diff --git a/crates/apollo_dashboard/resources/dev_grafana.json b/crates/apollo_dashboard/resources/dev_grafana.json index 00379ea91d6..4e1e992686c 100644 --- a/crates/apollo_dashboard/resources/dev_grafana.json +++ b/crates/apollo_dashboard/resources/dev_grafana.json @@ -1324,6 +1324,15 @@ ], "extra_params": {} }, + { + "title": "Committer Lag (blocks)", + "description": "Blocks the committer trails consensus (consensus_block_number - committer_offset), per namespace", + "type": "timeseries", + "exprs": [ + "max by (namespace) (consensus_block_number{cluster=~\"$cluster\", namespace=~\"$namespace\"}) - max by (namespace) (committer_offset{cluster=~\"$cluster\", namespace=~\"$namespace\"})" + ], + "extra_params": {} + }, { "title": "Total Block Duration", "description": "Average total block duration over a 1m window", @@ -1336,6 +1345,19 @@ "log_query": "\"Block\" AND \"durations in ms\"" } }, + { + "title": "Block Commit Latency", + "description": "Distribution of single-block commit duration", + "type": "timeseries", + "exprs": [ + "histogram_quantile(0.50, sum by (le) (rate(committer_block_commit_latency_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\"}[5m])))", + "histogram_quantile(0.95, sum by (le) (rate(committer_block_commit_latency_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\"}[5m])))" + ], + "extra_params": { + "unit": "s", + "log_query": "\"Block\" AND \"durations in ms\"" + } + }, { "title": "Total Block Duration per Modification", "description": "Average total block duration per modification over a 1m window", diff --git a/crates/apollo_dashboard/resources/dev_grafana_alerts.json b/crates/apollo_dashboard/resources/dev_grafana_alerts.json index fa163733319..900892281c6 100644 --- a/crates/apollo_dashboard/resources/dev_grafana_alerts.json +++ b/crates/apollo_dashboard/resources/dev_grafana_alerts.json @@ -274,6 +274,58 @@ "for": "30s", "severity": "p4" }, + { + "name": "committer_block_commit_latency_too_high", + "title": "Committer block commit latency too high", + "ruleGroup": "evaluation_rate_default", + "expr": "((sum(rate(committer_block_commit_latency_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"}[2m])) or vector(0)) / clamp_min(sum(rate(committer_block_commit_latency_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[2m])) or vector(0), 0.0000001)) and on() (is_observer{cluster=~\"$cluster\", namespace=~\"$namespace\"} == 0)", + "conditions": [ + { + "evaluator": { + "params": [ + 9.0 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "for": "30s", + "severity": "p4" + }, + { + "name": "committer_block_number_lag", + "title": "Committer block number lag", + "ruleGroup": "evaluation_rate_default", + "expr": "(max by (namespace) (consensus_block_number{cluster=~\"$cluster\", namespace=~\"$namespace\"}) - max by (namespace) (committer_offset{cluster=~\"$cluster\", namespace=~\"$namespace\"})) and on (namespace) (is_observer{cluster=~\"$cluster\", namespace=~\"$namespace\"} == 0) and on (namespace) (increase(consensus_decisions_reached_by_sync{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m]) == 0)", + "conditions": [ + { + "evaluator": { + "params": [ + 9.0 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "for": "5m", + "severity": "p2" + }, { "name": "committer_remote_server_number_of_connections", "title": "committer - Remote server number of connections exceeds 80", diff --git a/crates/apollo_dashboard/src/alert_definitions.rs b/crates/apollo_dashboard/src/alert_definitions.rs index 4102e5eec2a..263a3d8c21f 100644 --- a/crates/apollo_dashboard/src/alert_definitions.rs +++ b/crates/apollo_dashboard/src/alert_definitions.rs @@ -50,6 +50,8 @@ use crate::alert_scenarios::block_production_delay::{ consensus_block_number_progress_is_slow, get_cende_write_blob_failure_alert, get_cende_write_blob_failure_once_alert, + get_committer_block_commit_latency_too_high, + get_committer_block_number_lag, get_consensus_p2p_peer_down, get_consensus_round_above_zero, get_consensus_round_above_zero_multiple_times, @@ -649,6 +651,8 @@ pub fn get_apollo_alerts() -> Alerts { alerts.append(&mut get_batched_transactions_stuck_vec()); alerts.push(consensus_block_number_progress_is_slow()); alerts.push(get_cende_write_blob_failure_alert()); + alerts.push(get_committer_block_commit_latency_too_high()); + alerts.push(get_committer_block_number_lag()); alerts.append(&mut get_consensus_block_number_stuck_vec()); alerts.append(&mut get_consensus_p2p_not_enough_peers_for_quorum_vec()); alerts.push(get_consensus_p2p_peer_down()); diff --git a/crates/apollo_dashboard/src/alert_scenarios/block_production_delay.rs b/crates/apollo_dashboard/src/alert_scenarios/block_production_delay.rs index 6e6324f38c7..6768bc899d6 100644 --- a/crates/apollo_dashboard/src/alert_scenarios/block_production_delay.rs +++ b/crates/apollo_dashboard/src/alert_scenarios/block_production_delay.rs @@ -1,4 +1,10 @@ -use apollo_consensus::metrics::{CONSENSUS_BLOCK_NUMBER, CONSENSUS_ROUND_ABOVE_ZERO}; +use apollo_committer::metrics::{COMMITTER_BLOCK_COMMIT_LATENCY, COMMITTER_OFFSET}; +use apollo_consensus::metrics::{ + CONSENSUS_BLOCK_NUMBER, + CONSENSUS_DECISIONS_REACHED_BY_SYNC, + CONSENSUS_ROUND_ABOVE_ZERO, + IS_OBSERVER, +}; use apollo_consensus_manager::metrics::CONSENSUS_NUM_CONNECTED_PEERS; use apollo_consensus_orchestrator::metrics::CENDE_WRITE_BLOB_FAILURE; use apollo_infra_utils::template::Template; @@ -104,6 +110,62 @@ pub(crate) fn get_cende_write_blob_failure_once_alert() -> Alert { ) } +/// 9 fires at the STORED_BLOCK_HASH_BUFFER gate (10): integer gauges + strict `>`. +const COMMITTER_LAG_ALERT_THRESHOLD_BLOCKS: f64 = 9.0; + +pub(crate) fn get_committer_block_number_lag() -> Alert { + Alert::new( + "committer_block_number_lag", + "Committer block number lag", + EvaluationRate::Default, + // Per-namespace lag (committer vs its own consensus); excludes observer and syncing nodes. + format!( + "(max by (namespace) ({consensus}) - max by (namespace) ({committer})) and on \ + (namespace) ({observer} == 0) and on (namespace) (increase({sync}[5m]) == 0)", + consensus = CONSENSUS_BLOCK_NUMBER.get_name_with_filter(), + committer = COMMITTER_OFFSET.get_name_with_filter(), + observer = IS_OBSERVER.get_name_with_filter(), + sync = CONSENSUS_DECISIONS_REACHED_BY_SYNC.get_name_with_filter(), + ), + vec![AlertCondition::new( + AlertComparisonOp::GreaterThan, + COMMITTER_LAG_ALERT_THRESHOLD_BLOCKS, + AlertLogicalOp::And, + )], + // Debounce brief spikes before paging. + "5m", + AlertSeverity::Regular, + // Observers excluded in the expr; keep Applicable to avoid a second is_observer gate. + ObserverApplicability::Applicable, + ) +} + +/// Roughly 3x the committer's baseline commit latency. +const COMMIT_LATENCY_ALERT_THRESHOLD_SECS: f64 = 9.0; + +pub(crate) fn get_committer_block_commit_latency_too_high() -> Alert { + Alert::new( + "committer_block_commit_latency_too_high", + "Committer block commit latency too high", + EvaluationRate::Default, + // 2m average commit latency, so a regression is caught before lag accumulates. + format!( + "(sum(rate({}[2m])) or vector(0)) / clamp_min(sum(rate({}[2m])) or vector(0), \ + 0.0000001)", + COMMITTER_BLOCK_COMMIT_LATENCY.get_name_sum_with_filter(), + COMMITTER_BLOCK_COMMIT_LATENCY.get_name_count_with_filter(), + ), + vec![AlertCondition::new( + AlertComparisonOp::GreaterThan, + COMMIT_LATENCY_ALERT_THRESHOLD_SECS, + AlertLogicalOp::And, + )], + PENDING_DURATION_DEFAULT, + AlertSeverity::WorkingHours, + ObserverApplicability::NotApplicable, + ) +} + pub(crate) fn consensus_block_number_progress_is_slow() -> Alert { const ALERT_NAME: &str = "consensus_block_number_progress_is_slow"; Alert::new( diff --git a/crates/apollo_dashboard/src/panels/committer.rs b/crates/apollo_dashboard/src/panels/committer.rs index d1b600f60a8..d12b85701ae 100644 --- a/crates/apollo_dashboard/src/panels/committer.rs +++ b/crates/apollo_dashboard/src/panels/committer.rs @@ -3,6 +3,7 @@ use apollo_committer::metrics::{ AVERAGE_READ_RATE, AVERAGE_WRITE_RATE, BLOCKS_COMMITTED, + COMMITTER_BLOCK_COMMIT_LATENCY, COMMITTER_OFFSET, COMPUTE_DURATION_PER_BLOCK, COUNT_CLASSES_TRIE_MODIFICATIONS_PER_BLOCK, @@ -14,6 +15,8 @@ use apollo_committer::metrics::{ TOTAL_BLOCK_DURATION_PER_MODIFICATION, WRITE_DURATION_PER_BLOCK, }; +use apollo_consensus::metrics::CONSENSUS_BLOCK_NUMBER; +use apollo_metrics::metric_definitions::POD_LABEL_FILTER; use apollo_metrics::metrics::MetricQueryName; use crate::dashboard::Row; @@ -33,6 +36,19 @@ fn get_offset_panel() -> Panel { ) } +fn get_committer_lag_panel() -> Panel { + // Drop pod filter: the two metrics are on different pods, so a pod-scoped diff empties. + let consensus = CONSENSUS_BLOCK_NUMBER.get_name_with_filter().replace(POD_LABEL_FILTER, ""); + let committer = COMMITTER_OFFSET.get_name_with_filter().replace(POD_LABEL_FILTER, ""); + Panel::new( + "Committer Lag (blocks)", + "Blocks the committer trails consensus (consensus_block_number - committer_offset), per \ + namespace", + format!("max by (namespace) ({consensus}) - max by (namespace) ({committer})"), + PanelType::TimeSeries, + ) +} + /// Returns a panel that shows the average of a counter per block over a 1m window. fn average_per_block_panel( name: impl ToString, @@ -70,6 +86,16 @@ fn get_total_block_duration_panel() -> Panel { ) } +fn get_block_commit_latency_panel() -> Panel { + Panel::from_hist( + &COMMITTER_BLOCK_COMMIT_LATENCY, + "Block Commit Latency", + "Distribution of single-block commit duration", + ) + .with_unit(Unit::Seconds) + .with_log_query(BLOCK_DURATIONS_LOG_QUERY) +} + fn get_total_block_duration_per_modification_panel() -> Panel { average_per_block_panel( "Total Block Duration per Modification", @@ -209,7 +235,9 @@ pub(crate) fn get_committer_row() -> Row { "Committer", vec![ get_offset_panel(), + get_committer_lag_panel(), get_total_block_duration_panel(), + get_block_commit_latency_panel(), get_total_block_duration_per_modification_panel(), get_read_duration_per_block_panel(), get_average_read_rate_panel(),