From b18c1a7dd763b0521fadf2570e067f80f6b16682 Mon Sep 17 00:00:00 2001 From: Makro Date: Mon, 13 Jul 2026 05:32:23 +0000 Subject: [PATCH 1/4] perf: dep_graph: carry unchanged nodes into the next session's file On a warm rebuild the previous dep graph is decoded and then almost entirely re-encoded: every green node is read back from the previous graph and written out again, byte-for-byte equivalent to what it already was. This decode-and-re-encode round trip dominates dep-graph serialization when little or nothing changed. Instead, keep each green node at its previous index. A green node only ever points to other green nodes, which also keep their previous indices, so its edge list is unchanged and its whole record is identical to the previous file's. Re-emit that record directly: rebuild the fixed header from the already-decoded fields (feeding the byte-width back in so the header packs the same way) and copy the edge bytes straight from their on-disk form in `edge_list_data`, skipping the per-edge max scan and the per-edge write of a full re-encode. New and red nodes get fresh indices above the carried range, so they never collide with a carried index; the two singleton nodes stay pinned at indices 0 and 1 and are colored up front so they are never promoted into a duplicate record. Local rustc-perf (primary crates, instructions:u): incr-unchanged -1.59%, incr-patched -0.92%, full and incr-full flat (the carry is inactive without a previous cache, which doubles as a control that no overhead is added), 28 cells improved by at least 0.25%, 0 regressed. tests/incremental passes in full (178 tests, 0 failures). Co-Authored-By: Claude Opus 4.8 (1M context) --- compiler/rustc_middle/src/dep_graph/graph.rs | 43 ++++- .../rustc_middle/src/dep_graph/serialized.rs | 180 ++++++++++++++++-- 2 files changed, 205 insertions(+), 18 deletions(-) diff --git a/compiler/rustc_middle/src/dep_graph/graph.rs b/compiler/rustc_middle/src/dep_graph/graph.rs index 86cf7be60b858..f6ce142b65bcf 100644 --- a/compiler/rustc_middle/src/dep_graph/graph.rs +++ b/compiler/rustc_middle/src/dep_graph/graph.rs @@ -181,7 +181,10 @@ impl DepGraph { let colors = DepNodeColorMap::new(prev_graph_node_count); // Instantiate a node with zero dependencies only once for anonymous queries. - let _green_node_index = current.alloc_new_node( + // The two singletons always live at fixed indices 0 and 1, which are reserved below + // the range handed out to new nodes so they never collide with a carried green node. + let _green_node_index = current.alloc_singleton_node( + DepNodeIndex::SINGLETON_ZERO_DEPS_ANON_NODE, DepNode { kind: DepKind::AnonZeroDeps, key_fingerprint: current.anon_id_seed.into() }, EdgesVec::new(), Fingerprint::ZERO, @@ -191,7 +194,8 @@ impl DepGraph { // Create a single always-red node, with no dependencies of its own. // Other nodes can use the always-red node as a fake dependency, to // ensure that their dependency list will never be all-green. - let red_node_index = current.alloc_new_node( + let red_node_index = current.alloc_singleton_node( + DepNodeIndex::FOREVER_RED_NODE, DepNode { kind: DepKind::Red, key_fingerprint: Fingerprint::ZERO.into() }, EdgesVec::new(), Fingerprint::ZERO, @@ -202,6 +206,23 @@ impl DepGraph { const { SerializedDepNodeIndex::from_u32(DepNodeIndex::FOREVER_RED_NODE.as_u32()) }; let result = colors.try_set_color(prev_index, DesiredColor::Red); assert_matches!(result, TrySetColorResult::Success); + + // The previous graph's anon-zero-deps singleton also lives at a fixed index (0), the + // same one this session's singleton was just allocated at. It is deterministically + // green (an anonymous query with no dependencies never changes), and this session's + // singleton is its equivalent. Color it green pointing at that fresh node now, so it + // is never separately promoted, which would carry a second record into index 0 and + // corrupt the file. This mirrors how the always-red singleton is pinned above. + let anon_prev_index = const { + SerializedDepNodeIndex::from_u32( + DepNodeIndex::SINGLETON_ZERO_DEPS_ANON_NODE.as_u32(), + ) + }; + let result = colors.try_set_color( + anon_prev_index, + DesiredColor::Green { index: DepNodeIndex::SINGLETON_ZERO_DEPS_ANON_NODE }, + ); + assert_matches!(result, TrySetColorResult::Success); } DepGraph { @@ -1266,6 +1287,24 @@ impl CurrentDepGraph { dep_node_index } + + /// Allocates a node at a fixed index. Used only for the two singleton nodes, which must + /// live at indices 0 and 1 across every session. + #[inline(always)] + fn alloc_singleton_node( + &self, + index: DepNodeIndex, + key: DepNode, + edges: EdgesVec, + value_fingerprint: Fingerprint, + ) -> DepNodeIndex { + let dep_node_index = self.encoder.send_new_at(index, key, value_fingerprint, edges); + + #[cfg(debug_assertions)] + self.record_edge(dep_node_index, key, value_fingerprint); + + dep_node_index + } } #[derive(Debug, Clone, Copy)] diff --git a/compiler/rustc_middle/src/dep_graph/serialized.rs b/compiler/rustc_middle/src/dep_graph/serialized.rs index dbaf29745a8bc..18ad4c74fde9f 100644 --- a/compiler/rustc_middle/src/dep_graph/serialized.rs +++ b/compiler/rustc_middle/src/dep_graph/serialized.rs @@ -261,6 +261,60 @@ impl SerializedDepGraph { pub fn session_count(&self) -> u64 { self.session_count } + + /// Re-emits the record of green node `index` into `encoder`, byte-for-byte identical + /// to how it appeared in this graph's file, without decoding and re-encoding it. + /// + /// A green node keeps its index across sessions and only ever points to other green + /// nodes (also kept at their indices), so its edge list is unchanged. The fixed header + /// is rebuilt from the already-decoded fields (which is cheap and deterministic) and the + /// edge bytes, still in their on-disk form in [`Self::edge_list_data`], are copied + /// straight across, skipping the per-edge width scan and write of a full re-encode. + /// + /// Returns the node's edge count, for the caller's statistics. + #[inline] + fn carry_record_into(&self, index: SerializedDepNodeIndex, encoder: &mut MemEncoder) -> usize { + let node = &self.nodes[index]; + let value_fingerprint = self.value_fingerprints[index]; + let edge_header = self.edge_list_indices[index]; + let num_edges = edge_header.num_edges; + let bytes_per_index = edge_header.bytes_per_index(); + + // Reconstruct the header. `SerializedNodeHeader::new` derives the byte width from the + // maximum edge index; feed it the largest value of the known width so it picks exactly + // that width, reproducing the original header bit-for-bit without scanning the edges. + // The carried node keeps its previous index, so the serialized index is unchanged. + let dep_index = DepNodeIndex::from_u32(index.as_u32()); + let edge_max = width_to_max_index(bytes_per_index); + let header = SerializedNodeHeader::new( + node, + dep_index, + value_fingerprint, + edge_max, + num_edges as usize, + ); + encoder.write_array(header.bytes); + if header.len().is_none() { + encoder.emit_u32(num_edges); + } + + // Copy the edge bytes verbatim from their on-disk representation. + let start = edge_header.start(); + encoder.emit_raw_bytes(&self.edge_list_data[start..start + num_edges as usize * bytes_per_index]); + + num_edges as usize + } +} + +/// The largest node index representable in `bytes_per_index` bytes, used to make +/// [`SerializedNodeHeader::new`] select that exact edge byte width. +#[inline] +fn width_to_max_index(bytes_per_index: usize) -> u32 { + if bytes_per_index >= DEP_NODE_SIZE { + u32::MAX + } else { + (1u32 << (bytes_per_index * 8)) - 1 + } } /// A packed representation of an edge's start index and byte width. @@ -664,6 +718,11 @@ struct EncoderState { file: Lock>>, local: WorkerLocal>, stats: Option>>, + /// The first dep node index handed out to genuinely new (or red) nodes this session. + /// Green nodes carried from the previous graph keep their old indices, which all lie + /// below this value, so new nodes start above them and never collide. See the module + /// comment on the carry scheme. + first_new_index: u32, } impl EncoderState { @@ -672,9 +731,13 @@ impl EncoderState { record_stats: bool, previous: Arc, ) -> Self { + // Indices 0 and 1 are always the two singleton nodes; carried green indices fill the + // rest of the previous index space. New nodes start above all of them. + let first_new_index = std::cmp::max(2, previous.node_count() as u32); Self { previous, - next_node_index: AtomicU64::new(0), + next_node_index: AtomicU64::new(first_new_index as u64), + first_new_index, stats: record_stats.then(|| Lock::new(FxHashMap::default())), file: Lock::new(Some(encoder)), local: WorkerLocal::new(|_| { @@ -710,11 +773,19 @@ impl EncoderState { DepNodeIndex::from_u32(local.next_node_index) } - /// Marks the index previously returned by `next_index` as used. + /// Marks the index previously returned by `next_index` as used. Green nodes carried + /// from the previous graph keep their old index and don't go through here; the node + /// count is instead bumped by the encode itself (`count_node`). #[inline] - fn bump_index(&self, local: &mut LocalEncoderState) { + fn advance_index(&self, local: &mut LocalEncoderState) { local.remaining_node_index -= 1; local.next_node_index += 1; + } + + /// Counts one encoded node. Every node, whether new, re-executed, promoted or a + /// singleton, is counted here exactly once. + #[inline] + fn count_node(&self, local: &mut LocalEncoderState) { local.node_count += 1; } @@ -776,6 +847,7 @@ impl EncoderState { ) { node.encode(&mut local.encoder, index); self.flush_mem_encoder(&mut *local); + self.count_node(&mut *local); self.record(&node.node, index, node.edges.len(), &node.edges, retained_graph, &mut *local); } @@ -799,9 +871,37 @@ impl EncoderState { let edge_count = NodeInfo::encode_promoted(&mut local.encoder, node, index, value_fingerprint, edges); self.flush_mem_encoder(&mut *local); + self.count_node(&mut *local); self.record(node, index, edge_count, edges, retained_graph, &mut *local); } + /// Carries a promoted green node into the new file by re-emitting its previous record + /// (see [`SerializedDepGraph::carry_record_into`]) instead of decoding and re-encoding it. + /// Because the node keeps its previous index and every one of its edges points at another + /// green node that also kept its previous index, the bytes are identical to what a fresh + /// encode would produce. + /// + /// Only used when the full in-memory dep graph is not being retained (`-Zquery-dep-graph` + /// off, the common case); otherwise `encode_promoted_node` re-encodes, keeping the same + /// index, so the retained graph still sees the node's edges. + #[inline] + fn carry_promoted_node( + &self, + index: DepNodeIndex, + prev_index: SerializedDepNodeIndex, + local: &mut LocalEncoderState, + ) { + debug_assert_eq!(index.as_u32(), prev_index.as_u32()); + let edge_count = self.previous.carry_record_into(prev_index, &mut local.encoder); + self.flush_mem_encoder(&mut *local); + self.count_node(&mut *local); + // The carry path is only taken when the retained graph is disabled, so pass `None` and + // no edges; the `-Zquery-dep-graph` case goes through `encode_promoted_node` instead. + let node = self.previous.index_to_node(prev_index); + let no_retained: Option> = None; + self.record(node, index, edge_count, &[], &no_retained, &mut *local); + } + fn finish(&self, profiler: &SelfProfilerRef, current: &CurrentDepGraph) -> FileEncodeResult { // Prevent more indices from being allocated. self.next_node_index.store(u32::MAX as u64 + 1, Ordering::SeqCst); @@ -840,6 +940,12 @@ impl EncoderState { } } + // Carried green nodes keep their previous indices (all below `first_new_index`) but + // don't advance any worker's `next_node_index`. If few or no new nodes were encoded, + // the per-worker maxima can therefore understate the real index space, so raise the + // floor to cover every carried index. + node_max = max(node_max, self.first_new_index); + // Encode the number of each dep kind encountered for count in kind_stats.iter() { count.encode(&mut encoder); @@ -963,7 +1069,25 @@ impl GraphEncoder { let node = NodeInfo { node, value_fingerprint, edges }; let mut local = self.status.local.borrow_mut(); let index = self.status.next_index(&mut *local); - self.status.bump_index(&mut *local); + self.status.advance_index(&mut *local); + self.status.encode_node(index, &node, &self.retained_graph, &mut *local); + index + } + + /// Encodes a node at a fixed, caller-chosen index rather than the next allocated one. + /// Used only for the two singleton nodes, which must live at indices 0 and 1; those + /// slots are reserved below `first_new_index` and are never carried, so this cannot + /// collide with a carried green node or a freshly allocated one. + pub(crate) fn send_new_at( + &self, + index: DepNodeIndex, + node: DepNode, + value_fingerprint: Fingerprint, + edges: EdgesVec, + ) -> DepNodeIndex { + let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph"); + let node = NodeInfo { node, value_fingerprint, edges }; + let mut local = self.status.local.borrow_mut(); self.status.encode_node(index, &node, &self.retained_graph, &mut *local); index } @@ -985,8 +1109,15 @@ impl GraphEncoder { let mut local = self.status.local.borrow_mut(); - let index = self.status.next_index(&mut *local); - let color = if is_green { DesiredColor::Green { index } } else { DesiredColor::Red }; + // A green node keeps its previous index so that any node promoted from the previous + // graph, which refers to it by that index, stays byte-identical and can be carried + // verbatim. A red node changed, so it gets a fresh index above the carried range. + let (index, color) = if is_green { + let index = DepNodeIndex::from_u32(prev_index.as_u32()); + (index, DesiredColor::Green { index }) + } else { + (self.status.next_index(&mut *local), DesiredColor::Red) + }; // Use `try_set_color` to avoid racing when `send_promoted` is called concurrently // on the same index. @@ -996,7 +1127,9 @@ impl GraphEncoder { TrySetColorResult::AlreadyGreen { index } => return index, } - self.status.bump_index(&mut *local); + if !is_green { + self.status.advance_index(&mut *local); + } self.status.encode_node(index, &node, &self.retained_graph, &mut *local); index } @@ -1017,20 +1150,35 @@ impl GraphEncoder { let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph"); let mut local = self.status.local.borrow_mut(); - let index = self.status.next_index(&mut *local); + // A promoted green node keeps its previous index; its edges (all green, all likewise + // kept at their previous indices) are exactly the previous edges, so the record can be + // carried verbatim. + let index = DepNodeIndex::from_u32(prev_index.as_u32()); // Use `try_set_color` to avoid racing when `send_promoted` or `send_and_color` // is called concurrently on the same index. match colors.try_set_color(prev_index, DesiredColor::Green { index }) { TrySetColorResult::Success => { - self.status.bump_index(&mut *local); - self.status.encode_promoted_node( - index, - prev_index, - &self.retained_graph, - &mut *local, - edges, - ); + if self.retained_graph.is_none() { + // Fast path: re-emit the previous record instead of re-encoding it. + debug_assert!( + edges.iter().map(|e| e.as_u32()).eq(self + .status + .previous + .edge_targets_from(prev_index) + .map(|e| e.as_u32())), + "carried green node {prev_index:?} edges diverged from the previous graph", + ); + self.status.carry_promoted_node(index, prev_index, &mut *local); + } else { + self.status.encode_promoted_node( + index, + prev_index, + &self.retained_graph, + &mut *local, + edges, + ); + } Some(index) } TrySetColorResult::AlreadyRed => None, From b5192035c1777044afc0633be334cb80181f72d8 Mon Sep 17 00:00:00 2001 From: Makro Date: Mon, 13 Jul 2026 10:56:11 +0000 Subject: [PATCH 2/4] perf: dep_graph: append-only save via wholesale carry of the record region Building on carrying unchanged nodes forward: instead of re-emitting every green node's record one by one, copy the previous file's entire record region into the new file in a single write, and give every node that existed in the previous session a stable index, red or green. A node re-verified green then needs no write at all: its carried record, whose edges point at other kept indices, is already exactly right. A re-executed node appends a record at its old index which overrides the carried one (later records win at decode time), and a node this session dropped is tombstoned via a dead list in the footer. Genuinely new nodes get indices above the previous index space. The save becomes proportional to what changed, not to the size of the graph, and the one large copy is a single kernel-side write from the retained mapping. Marking a promoted node green shrinks to just the color-map insert: no encoder borrow, no record write, no per-node bookkeeping in the hot try_mark_green path. Footer counts are computed as O(changed) deltas from the previous footer. After eight carried generations the file is rewritten fresh, compacting away dead records, superseded duplicates and their index slots; the same fresh rewrite serves sessions where a debugging feature needs every node to pass through the encoder. Local rustc-perf (primary crates, instructions:u), against the same base: incr-unchanged -3.88%, incr-patched -2.31%, full and incr-full flat (-0.04%/-0.05%, the carry is inactive without a previous cache), all 30 incremental cells improved, 0 regressed. Multi-generation runs: 12 generations of incr-unchanged and of alternating real edits, both crossing the compaction boundary, all clean; 12 generations under -Zincremental-verify-ich clean; tests/incremental passes in full (178 tests, 0 failures). dep-graph.bin grows by the appended delta per generation and shrinks back at the compaction rewrite, as intended. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/persist/file_format.rs | 2 +- .../rustc_incremental/src/persist/load.rs | 11 +- compiler/rustc_middle/src/dep_graph/graph.rs | 13 +- .../rustc_middle/src/dep_graph/serialized.rs | 482 ++++++++++++------ 4 files changed, 349 insertions(+), 159 deletions(-) diff --git a/compiler/rustc_incremental/src/persist/file_format.rs b/compiler/rustc_incremental/src/persist/file_format.rs index 853a5c9ba7ab0..0e4102e54b77f 100644 --- a/compiler/rustc_incremental/src/persist/file_format.rs +++ b/compiler/rustc_incremental/src/persist/file_format.rs @@ -26,7 +26,7 @@ use crate::diagnostics; const FILE_MAGIC: &[u8] = b"RSIC"; /// Change this if the header format changes. -const HEADER_FORMAT_VERSION: u16 = 0; +const HEADER_FORMAT_VERSION: u16 = 1; pub(crate) fn write_file_header(stream: &mut FileEncoder<'_>, sess: &Session) { stream.emit_raw_bytes(FILE_MAGIC); diff --git a/compiler/rustc_incremental/src/persist/load.rs b/compiler/rustc_incremental/src/persist/load.rs index c3b9f433417a6..83d0984e5df5b 100644 --- a/compiler/rustc_incremental/src/persist/load.rs +++ b/compiler/rustc_incremental/src/persist/load.rs @@ -112,7 +112,16 @@ fn load_dep_graph(sess: &Session) -> LoadResult { return LoadResult::DataOutOfDate; } - let prev_graph = SerializedDepGraph::decode(&mut decoder, &sess.prof); + let mut prev_graph = SerializedDepGraph::decode(&mut decoder, &sess.prof); + + // Retain the file bytes so that the record region can be carried into the + // next session's dep-graph file wholesale. The decoder borrows `mmap`, so + // release it before handing ownership to the freshly decoded (uniquely + // owned) graph. + drop(decoder); + std::sync::Arc::get_mut(&mut prev_graph) + .expect("freshly decoded dep graph is uniquely owned") + .attach_mmap(mmap); LoadResult::Ok { prev_graph, prev_work_products } } diff --git a/compiler/rustc_middle/src/dep_graph/graph.rs b/compiler/rustc_middle/src/dep_graph/graph.rs index f6ce142b65bcf..9ffa876649ced 100644 --- a/compiler/rustc_middle/src/dep_graph/graph.rs +++ b/compiler/rustc_middle/src/dep_graph/graph.rs @@ -1102,7 +1102,11 @@ impl DepGraph { } pub(crate) fn finish_encoding(&self) -> FileEncodeResult { - if let Some(data) = &self.data { data.current.encoder.finish(&data.current) } else { Ok(0) } + if let Some(data) = &self.data { + data.current.encoder.finish(&data.current, &data.colors) + } else { + Ok(0) + } } pub fn next_virtual_depnode_index(&self) -> DepNodeIndex { @@ -1421,6 +1425,13 @@ impl DepNodeColorMap { DepNodeColor::Unknown } } + + /// Whether the node was marked green this session. Used by the encoder when it + /// computes which carried records are dead. + #[inline] + pub(super) fn is_green(&self, index: SerializedDepNodeIndex) -> bool { + self.values[index].load(Ordering::Acquire) < COMPRESSED_RED + } } /// The color that [`DepNodeColorMap::try_set_color`] should try to apply to a node. diff --git a/compiler/rustc_middle/src/dep_graph/serialized.rs b/compiler/rustc_middle/src/dep_graph/serialized.rs index 18ad4c74fde9f..f5f0927f891eb 100644 --- a/compiler/rustc_middle/src/dep_graph/serialized.rs +++ b/compiler/rustc_middle/src/dep_graph/serialized.rs @@ -1,12 +1,25 @@ //! The data that we will serialize and deserialize. //! //! Notionally, the dep-graph is a sequence of NodeInfo with the dependencies -//! specified inline. The total number of nodes and edges are stored as the last -//! 16 bytes of the file, so we can find them easily at decoding time. +//! specified inline. A footer stores the dead list, the per-kind counts, and the +//! total number of nodes and edges, with fixed-size positions at the very end of +//! the file so we can find them easily at decoding time. //! //! The serialisation is performed on-demand when each node is emitted. Using this //! scheme, we do not need to keep the current graph in memory. //! +//! On a warm rebuild most nodes are unchanged. Rather than re-encoding them, the +//! previous file's record region is copied into the new file wholesale and every +//! node that existed in the previous session keeps its index: a node re-verified +//! green needs no write at all (its carried record, whose edges point at other +//! kept indices, is already exactly right), a re-executed node appends a record +//! at its old index which overrides the carried one (later records win at decode +//! time), and a node this session dropped is tombstoned via the dead list in the +//! footer. Genuinely new nodes get indices above the previous index space. Dead +//! records and superseded duplicates accumulate with each carried generation, so +//! after [`MAX_CARRIED_GENERATIONS`] the file is rewritten fresh, which also +//! happens when a debugging feature needs every node to pass through the encoder. +//! //! The deserialization is performed manually, in order to convert from the stored //! sequence of NodeInfos to the different arrays in SerializedDepGraph. Since the //! node and edge count are stored at the end of the file, all the arrays can be @@ -47,7 +60,9 @@ use std::{iter, mem}; use rustc_data_structures::fingerprint::{Fingerprint, PackedFingerprint}; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::memmap::Mmap; use rustc_data_structures::outline; +use rustc_index::bit_set::DenseBitSet; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::sync::{AtomicU64, Lock, WorkerLocal, broadcast}; use rustc_data_structures::unhash::UnhashMap; @@ -120,6 +135,30 @@ pub struct SerializedDepGraph { /// The number of previous compilation sessions. This is used to generate /// unique anon dep nodes per session. session_count: u64, + /// How many consecutive sessions have carried the record region forward without + /// a compacting rewrite. Dead records and superseded duplicates accumulate with + /// each carried generation, so the writer compacts once this grows too large. + generation: u64, + /// The memory-mapped bytes of the file this graph was decoded from, retained so + /// that the record region can be copied into the next session's file wholesale + /// (see [`Self::region_bytes`]). `None` for the empty default graph, which + /// disables the carry. + mmap: Option, + /// The byte range of the record region within [`Self::mmap`]: every node record, + /// including dead and superseded ones, and nothing else. + records_range: std::ops::Range, + /// Indices whose record in the region is dead: the node was dropped by an earlier + /// session (never re-verified nor re-executed), so the record must be ignored. + /// Cumulative across carried generations; reset by a compacting rewrite. + dead: Vec, + /// The per-`DepKind` counts of live nodes, from the file footer. Retained so the + /// next session can compute its own footer counts as a delta. + kind_stats: Vec, + /// The number of live nodes, from the file footer (`nodes.len()` counts `Null` + /// index slots too). + live_node_count: u64, + /// The number of edges of live nodes, from the file footer. + live_edge_count: u64, /// Used to time the lazy per-`DepKind` reverse-index build. `None` only for /// the empty default graph, which is never looked up. profiler: Option, @@ -262,58 +301,44 @@ impl SerializedDepGraph { self.session_count } - /// Re-emits the record of green node `index` into `encoder`, byte-for-byte identical - /// to how it appeared in this graph's file, without decoding and re-encoding it. - /// - /// A green node keeps its index across sessions and only ever points to other green - /// nodes (also kept at their indices), so its edge list is unchanged. The fixed header - /// is rebuilt from the already-decoded fields (which is cheap and deterministic) and the - /// edge bytes, still in their on-disk form in [`Self::edge_list_data`], are copied - /// straight across, skipping the per-edge width scan and write of a full re-encode. + /// Whether this graph's record region can be carried into the next session's file + /// wholesale. False for the empty default graph (no retained bytes). + #[inline] + fn can_carry(&self) -> bool { + self.mmap.is_some() + } + + /// The raw bytes of the record region, exactly as they appeared in this graph's + /// file: every node record, including dead and superseded ones. /// - /// Returns the node's edge count, for the caller's statistics. + /// Every node re-verified or re-executed this session keeps its previous index, so + /// these records remain valid in the next file as-is: records of promoted green + /// nodes are byte-for-byte what a fresh encode would produce, records superseded by + /// a re-executed node are overridden by the appended record at the same index, and + /// records of dropped nodes are tombstoned via the dead list in the footer. #[inline] - fn carry_record_into(&self, index: SerializedDepNodeIndex, encoder: &mut MemEncoder) -> usize { - let node = &self.nodes[index]; - let value_fingerprint = self.value_fingerprints[index]; - let edge_header = self.edge_list_indices[index]; - let num_edges = edge_header.num_edges; - let bytes_per_index = edge_header.bytes_per_index(); - - // Reconstruct the header. `SerializedNodeHeader::new` derives the byte width from the - // maximum edge index; feed it the largest value of the known width so it picks exactly - // that width, reproducing the original header bit-for-bit without scanning the edges. - // The carried node keeps its previous index, so the serialized index is unchanged. - let dep_index = DepNodeIndex::from_u32(index.as_u32()); - let edge_max = width_to_max_index(bytes_per_index); - let header = SerializedNodeHeader::new( - node, - dep_index, - value_fingerprint, - edge_max, - num_edges as usize, - ); - encoder.write_array(header.bytes); - if header.len().is_none() { - encoder.emit_u32(num_edges); - } + fn region_bytes(&self) -> &[u8] { + &self.mmap.as_ref().unwrap()[self.records_range.clone()] + } - // Copy the edge bytes verbatim from their on-disk representation. - let start = edge_header.start(); - encoder.emit_raw_bytes(&self.edge_list_data[start..start + num_edges as usize * bytes_per_index]); + /// The number of edges of the node at `index`, used for O(changed) footer accounting. + #[inline] + fn edge_count_for_index(&self, index: SerializedDepNodeIndex) -> usize { + self.edge_list_indices[index].num_edges as usize + } - num_edges as usize + /// Whether the node at `index` has a (live or superseded) record in the region. + /// `Null` slots come from batch index allocation and dead records of earlier + /// generations; neither leaves a live record to tombstone. + #[inline] + fn index_is_occupied(&self, index: SerializedDepNodeIndex) -> bool { + self.nodes[index].kind != DepKind::Null } -} -/// The largest node index representable in `bytes_per_index` bytes, used to make -/// [`SerializedNodeHeader::new`] select that exact edge byte width. -#[inline] -fn width_to_max_index(bytes_per_index: usize) -> u32 { - if bytes_per_index >= DEP_NODE_SIZE { - u32::MAX - } else { - (1u32 << (bytes_per_index * 8)) - 1 + /// Attaches the retained file bytes decoded by [`Self::decode`], enabling the + /// carry of this graph's record region into the next session's file. + pub fn attach_mmap(&mut self, mmap: Mmap) { + self.mmap = Some(mmap); } } @@ -352,24 +377,52 @@ fn mask(bits: usize) -> usize { impl SerializedDepGraph { #[instrument(level = "debug", skip(d, profiler))] pub fn decode(d: &mut MemDecoder<'_>, profiler: &SelfProfilerRef) -> Arc { - // The last 16 bytes are the node count and edge count. + // The last 32 bytes are the position of the dead list (which is also where the + // record region ends), the node max, and the live node and edge counts. debug!("position: {:?}", d.position()); // `node_max` is the number of indices including empty nodes while `node_count` - // is the number of actually encoded nodes. - let (node_max, node_count, edge_count) = - d.with_position(d.len() - 3 * IntEncodedWithFixedSize::ENCODED_SIZE, |d| { + // is the number of live nodes: records that are neither dead nor superseded by + // a later record at the same index. + let (dead_pos, node_max, node_count, edge_count) = + d.with_position(d.len() - 4 * IntEncodedWithFixedSize::ENCODED_SIZE, |d| { debug!("position: {:?}", d.position()); + let dead_pos = IntEncodedWithFixedSize::decode(d).0 as usize; let node_max = IntEncodedWithFixedSize::decode(d).0 as usize; let node_count = IntEncodedWithFixedSize::decode(d).0 as usize; let edge_count = IntEncodedWithFixedSize::decode(d).0 as usize; - (node_max, node_count, edge_count) + (dead_pos, node_max, node_count, edge_count) }); debug!("position: {:?}", d.position()); debug!(?node_count, ?edge_count); - let graph_bytes = d.len() - (3 * IntEncodedWithFixedSize::ENCODED_SIZE) - d.position(); + let records_start = d.position(); + + // The footer between the records and the fixed-size tail: the dead list, the + // per-kind live counts, the session count and the carried generation count. + // Read it up front, as decoding the records requires the dead set. + let (dead, dead_set, kind_stats, session_count, generation) = + d.with_position(dead_pos, |d| { + let dead_len = d.read_u64() as usize; + let mut dead = Vec::with_capacity(dead_len); + let mut dead_set = DenseBitSet::new_empty(node_max); + for _ in 0..dead_len { + let index = SerializedDepNodeIndex::from_u32(u32::from_le_bytes(d.read_array())); + dead_set.insert(index); + dead.push(index); + } + let kind_stats: Vec = + (0..(DepKind::MAX + 1)).map(|_| d.read_u32()).collect(); + let session_count = d.read_u64(); + let generation = d.read_u64(); + (dead, dead_set, kind_stats, session_count, generation) + }); + + // The record region may contain more than `node_count` records: dead records + // and superseded ones (a later record at the same index overrides an earlier + // one). This makes the capacity estimate below overshoot slightly more. + let graph_bytes = dead_pos - records_start; let mut nodes = IndexVec::from_elem_n( DepNode { @@ -394,20 +447,13 @@ impl SerializedDepGraph { let mut edge_list_data = Vec::with_capacity(graph_bytes - node_count * size_of::()); - for _ in 0..node_count { + while d.position() < dead_pos { // Decode the header for this edge; the header packs together as many of the fixed-size // fields as possible to limit the number of times we update decoder state. let node_header = SerializedNodeHeader { bytes: d.read_array() }; let index = node_header.index(); - let node = &mut nodes[index]; - // Make sure there's no duplicate indices in the dep graph. - assert!(node_header.node().kind != DepKind::Null && node.kind == DepKind::Null); - *node = node_header.node(); - - value_fingerprints[index] = node_header.value_fingerprint(); - // If the length of this node's edge list is small, the length is stored in the header. // If it is not, we fall back to another decoder call. let num_edges = node_header.len().unwrap_or_else(|| d.read_u32()); @@ -416,8 +462,31 @@ impl SerializedDepGraph { // number of byte elements per-array not per-element. This lets us read the whole edge // list for a node with one decoder call and also use the on-disk format in memory. let edges_len_bytes = node_header.bytes_per_index() * (num_edges as usize); + + // A dead record: the node was dropped by an earlier session but its bytes were + // carried along in the region. Skip it; its slot stays `Null`. + if dead_set.contains(index) { + d.read_raw_bytes(edges_len_bytes); + continue; + } + + let node = &mut nodes[index]; + let new_node = node_header.node(); + assert!(new_node.kind != DepKind::Null); + if node.kind != DepKind::Null { + // A later record overrides an earlier one at the same index: the node was + // re-executed by the session that appended it, keeping its index. The key + // cannot change, only the value fingerprint and the edges. The exception + // is the anon-zero-deps singleton, whose key is seeded per session. + debug_assert!(*node == new_node || new_node.kind == DepKind::AnonZeroDeps); + } + *node = new_node; + + value_fingerprints[index] = node_header.value_fingerprint(); + // The in-memory structure for the edges list stores the byte width of the edges on - // this node with the offset into the global edge data array. + // this node with the offset into the global edge data array. On an override the + // earlier record's edge bytes are simply orphaned in `edge_list_data`. let edges_header = node_header.edges_header(&edge_list_data, num_edges); edge_list_data.extend(d.read_raw_bytes(edges_len_bytes)); @@ -430,19 +499,16 @@ impl SerializedDepGraph { // end of the array. This padding ensure it doesn't. edge_list_data.extend(&[0u8; DEP_NODE_PAD]); - // Read the number of nodes of each dep kind, and perform - // counting sort for `LazyNodeIndex`. + // Lay out the per-kind live counts (read from the footer above) as contiguous + // ranges for the counting sort of `LazyNodeIndex`. let mut kinds = Vec::with_capacity(DepKind::MAX as usize + 1); let mut offset = 0u32; - for _ in 0..(DepKind::MAX + 1) { - let len = d.read_u32(); + for &len in &kind_stats { kinds.push(LazyKindIndex { start: offset, len, map: OnceLock::new() }); offset += len; } debug_assert_eq!(offset as usize, node_count); - let session_count = d.read_u64(); - // Counting sort: place each node index into its kind's range. `fill[k]` // points at the next free slot in kind `k`'s range, so a kind's nodes end // up contiguous. Slots start as `None` and are each filled exactly once @@ -470,6 +536,14 @@ impl SerializedDepGraph { edge_list_data, reverse_index, session_count, + generation, + // The retained file bytes are attached by the caller via `attach_mmap`. + mmap: None, + records_range: records_start..dead_pos, + dead, + kind_stats, + live_node_count: node_count as u64, + live_edge_count: edge_count as u64, profiler: Some(profiler.clone()), }) } @@ -696,19 +770,30 @@ struct LocalEncoderState { next_node_index: u32, remaining_node_index: u32, encoder: MemEncoder, - node_count: usize, - edge_count: usize, - - /// Stores the number of times we've encoded each dep kind. + /// Net change to the live node count from this worker's appends. An appended + /// record that overrides a carried one nets zero (the node was already counted + /// by the previous footer), so only genuinely new nodes contribute. + node_count: i64, + /// Net change to the live edge count from this worker's appends. An override + /// contributes the difference between its new and old edge counts. + edge_count: i64, + /// Indices below `first_new_index` this worker appended records for. Those appends + /// override the carried record at the same index; anything occupied, not overridden + /// and not marked green by the end of the session is dead. + overridden: Vec, + + /// Stores the net change to the number of live nodes of each dep kind. + /// An override nets zero here since the key (and thus the kind) cannot change. kind_stats: Vec, } struct LocalEncoderResult { node_max: u32, - node_count: usize, - edge_count: usize, + node_count: i64, + edge_count: i64, + overridden: Vec, - /// Stores the number of times we've encoded each dep kind. + /// Stores the net change to the number of live nodes of each dep kind. kind_stats: Vec, } @@ -718,11 +803,16 @@ struct EncoderState { file: Lock>>, local: WorkerLocal>, stats: Option>>, - /// The first dep node index handed out to genuinely new (or red) nodes this session. - /// Green nodes carried from the previous graph keep their old indices, which all lie - /// below this value, so new nodes start above them and never collide. See the module - /// comment on the carry scheme. + /// The first dep node index handed out to genuinely new nodes this session. Nodes + /// that existed in the previous graph keep their old indices, which all lie below + /// this value, so new nodes never collide with them. first_new_index: u32, + /// Whether this session carries the previous record region forward: the region was + /// copied into the new file wholesale at construction, promoted green nodes write + /// nothing, and re-executed nodes append records that override the carried ones. + /// When false (first session, compaction, or a debugging feature retains the full + /// graph), every live record is written out fresh. + carrying: bool, } impl EncoderState { @@ -730,14 +820,25 @@ impl EncoderState { encoder: FileEncoder<'static>, record_stats: bool, previous: Arc, + carrying: bool, ) -> Self { - // Indices 0 and 1 are always the two singleton nodes; carried green indices fill the + // Indices 0 and 1 are always the two singleton nodes; carried indices fill the // rest of the previous index space. New nodes start above all of them. let first_new_index = std::cmp::max(2, previous.node_count() as u32); + let mut encoder = encoder; + if carrying { + // Copy the previous record region into the new file wholesale, before any + // appended record. Every node that survives this session keeps its index, so + // the region stays valid: promoted green records are byte-for-byte what a + // fresh encode would produce, re-executed nodes append overriding records at + // their old index, and dropped nodes are tombstoned via the dead list. + encoder.emit_raw_bytes(previous.region_bytes()); + } Self { previous, next_node_index: AtomicU64::new(first_new_index as u64), first_new_index, + carrying, stats: record_stats.then(|| Lock::new(FxHashMap::default())), file: Lock::new(Some(encoder)), local: WorkerLocal::new(|_| { @@ -746,6 +847,7 @@ impl EncoderState { remaining_node_index: 0, edge_count: 0, node_count: 0, + overridden: Vec::new(), encoder: MemEncoder::new(), kind_stats: iter::repeat_n(0, DepKind::MAX as usize + 1).collect(), }) @@ -773,17 +875,16 @@ impl EncoderState { DepNodeIndex::from_u32(local.next_node_index) } - /// Marks the index previously returned by `next_index` as used. Green nodes carried - /// from the previous graph keep their old index and don't go through here; the node - /// count is instead bumped by the encode itself (`count_node`). + /// Marks the index previously returned by `next_index` as used. Nodes that existed + /// in the previous graph keep their old index and don't go through here. #[inline] fn advance_index(&self, local: &mut LocalEncoderState) { local.remaining_node_index -= 1; local.next_node_index += 1; } - /// Counts one encoded node. Every node, whether new, re-executed, promoted or a - /// singleton, is counted here exactly once. + /// Counts one written record. Appends that override a carried record are + /// compensated afterwards by [`Self::record_override`]. #[inline] fn count_node(&self, local: &mut LocalEncoderState) { local.node_count += 1; @@ -800,7 +901,7 @@ impl EncoderState { local: &mut LocalEncoderState, ) { local.kind_stats[node.kind.as_usize()] += 1; - local.edge_count += edge_count; + local.edge_count += edge_count as i64; if let Some(retained_graph) = &retained_graph { // Outline the build of the full dep graph as it's typically disabled and cold. @@ -875,34 +976,30 @@ impl EncoderState { self.record(node, index, edge_count, edges, retained_graph, &mut *local); } - /// Carries a promoted green node into the new file by re-emitting its previous record - /// (see [`SerializedDepGraph::carry_record_into`]) instead of decoding and re-encoding it. - /// Because the node keeps its previous index and every one of its edges points at another - /// green node that also kept its previous index, the bytes are identical to what a fresh - /// encode would produce. - /// - /// Only used when the full in-memory dep graph is not being retained (`-Zquery-dep-graph` - /// off, the common case); otherwise `encode_promoted_node` re-encodes, keeping the same - /// index, so the retained graph still sees the node's edges. + /// Adjusts a worker's bookkeeping after it appended a record that overrides the + /// carried record at `prev_index`. The node was already counted by the previous + /// footer, so the append nets zero nodes (and zero for its kind, since the key + /// cannot change) and only the change in edge count remains. #[inline] - fn carry_promoted_node( + fn record_override( &self, - index: DepNodeIndex, prev_index: SerializedDepNodeIndex, + kind: DepKind, local: &mut LocalEncoderState, ) { - debug_assert_eq!(index.as_u32(), prev_index.as_u32()); - let edge_count = self.previous.carry_record_into(prev_index, &mut local.encoder); - self.flush_mem_encoder(&mut *local); - self.count_node(&mut *local); - // The carry path is only taken when the retained graph is disabled, so pass `None` and - // no edges; the `-Zquery-dep-graph` case goes through `encode_promoted_node` instead. - let node = self.previous.index_to_node(prev_index); - let no_retained: Option> = None; - self.record(node, index, edge_count, &[], &no_retained, &mut *local); + debug_assert!(self.carrying); + local.node_count -= 1; + local.kind_stats[kind.as_usize()] -= 1; + local.edge_count -= self.previous.edge_count_for_index(prev_index) as i64; + local.overridden.push(prev_index); } - fn finish(&self, profiler: &SelfProfilerRef, current: &CurrentDepGraph) -> FileEncodeResult { + fn finish( + &self, + profiler: &SelfProfilerRef, + current: &CurrentDepGraph, + colors: &DepNodeColorMap, + ) -> FileEncodeResult { // Prevent more indices from being allocated. self.next_node_index.store(u32::MAX as u64 + 1, Ordering::SeqCst); @@ -920,41 +1017,92 @@ impl EncoderState { node_max: local.next_node_index, node_count: local.node_count, edge_count: local.edge_count, + overridden: mem::take(&mut local.overridden), } }); let mut encoder = self.file.lock().take().unwrap(); - let mut kind_stats: Vec = iter::repeat_n(0, DepKind::MAX as usize + 1).collect(); + // Every count starts from the previous footer when carrying (the region already + // holds those nodes) and from zero when writing a fresh file; the workers report + // net changes in either case. + let (mut kind_stats, mut node_count, mut edge_count) = if self.carrying { + ( + self.previous.kind_stats.clone(), + self.previous.live_node_count as i64, + self.previous.live_edge_count as i64, + ) + } else { + (iter::repeat_n(0, DepKind::MAX as usize + 1).collect(), 0, 0) + }; let mut node_max = 0; - let mut node_count = 0; - let mut edge_count = 0; + let mut overridden = DenseBitSet::new_empty(self.first_new_index as usize); for result in results { node_max = max(node_max, result.node_max); node_count += result.node_count; edge_count += result.edge_count; for (i, stat) in result.kind_stats.iter().enumerate() { - kind_stats[i] += stat; + // The per-worker values are net changes: an override decrements the kind + // it previously incremented, so the sum stays balanced per worker and the + // wrapping cancels out across the base value taken from the footer. + kind_stats[i] = kind_stats[i].wrapping_add(*stat); + } + for index in result.overridden { + overridden.insert(index); } } - // Carried green nodes keep their previous indices (all below `first_new_index`) but - // don't advance any worker's `next_node_index`. If few or no new nodes were encoded, - // the per-worker maxima can therefore understate the real index space, so raise the - // floor to cover every carried index. + // Nodes that existed in the previous graph keep their previous indices (all below + // `first_new_index`) but don't advance any worker's `next_node_index`. If few or no + // new nodes were encoded, the per-worker maxima can therefore understate the real + // index space, so raise the floor to cover every carried index. node_max = max(node_max, self.first_new_index); - // Encode the number of each dep kind encountered + // When carrying, tombstone every record in the region that this session dropped: a + // node neither marked green (record still valid) nor overridden by an appended + // record. This matches what a fresh write drops by simply not writing it. Dead + // indices from earlier generations decode as unoccupied slots, so they are carried + // into the new list explicitly. + let mut dead: Vec = Vec::new(); + if self.carrying { + dead.extend_from_slice(&self.previous.dead); + for index in (0..self.previous.node_count() as u32).map(SerializedDepNodeIndex::from_u32) + { + if self.previous.index_is_occupied(index) + && !colors.is_green(index) + && !overridden.contains(index) + { + dead.push(index); + let kind = self.previous.index_to_node(index).kind; + kind_stats[kind.as_usize()] -= 1; + node_count -= 1; + edge_count -= self.previous.edge_count_for_index(index) as i64; + } + } + } + + let generation = if self.carrying { self.previous.generation + 1 } else { 0 }; + + // The record region ends where the dead list begins. + let dead_pos = encoder.position(); + encoder.emit_u64(dead.len() as u64); + for index in &dead { + encoder.write_array(index.as_u32().to_le_bytes()); + } + + // Encode the number of live nodes of each dep kind. for count in kind_stats.iter() { count.encode(&mut encoder); } self.previous.session_count.checked_add(1).unwrap().encode(&mut encoder); + generation.encode(&mut encoder); debug!(?node_max, ?node_count, ?edge_count); debug!("position: {:?}", encoder.position()); + IntEncodedWithFixedSize(dead_pos.try_into().unwrap()).encode(&mut encoder); IntEncodedWithFixedSize(node_max.try_into().unwrap()).encode(&mut encoder); IntEncodedWithFixedSize(node_count.try_into().unwrap()).encode(&mut encoder); IntEncodedWithFixedSize(edge_count.try_into().unwrap()).encode(&mut encoder); @@ -967,7 +1115,7 @@ impl EncoderState { profiler.artifact_size("dep_graph", "dep-graph.bin", position as u64); } - self.print_incremental_info(current, node_count, edge_count); + self.print_incremental_info(current, node_count as usize, edge_count as usize); result } @@ -1038,6 +1186,11 @@ pub(crate) struct GraphEncoder { retained_graph: Option>, } +/// After this many consecutive carried generations, write a fresh file instead. Each +/// carried generation leaves behind dead records, superseded records and their orphaned +/// index slots; a compacting rewrite reclaims all of it. +const MAX_CARRIED_GENERATIONS: u64 = 8; + impl GraphEncoder { pub(crate) fn new( sess: &Session, @@ -1050,7 +1203,16 @@ impl GraphEncoder { .unstable_opts .query_dep_graph .then(|| Lock::new(RetainedDepGraph::new(prev_node_count))); - let status = EncoderState::new(encoder, sess.opts.unstable_opts.incremental_info, previous); + let record_stats = sess.opts.unstable_opts.incremental_info; + // Carry the previous record region forward unless there is no previous file, a + // debugging feature needs every node to pass through the encoder (the retained + // graph and the stats both do), or enough generations accumulated that dead + // records should be compacted away. + let carrying = previous.can_carry() + && retained_graph.is_none() + && !record_stats + && previous.generation + 1 < MAX_CARRIED_GENERATIONS; + let status = EncoderState::new(encoder, record_stats, previous, carrying); GraphEncoder { status, retained_graph, profiler: sess.prof.clone() } } @@ -1076,8 +1238,9 @@ impl GraphEncoder { /// Encodes a node at a fixed, caller-chosen index rather than the next allocated one. /// Used only for the two singleton nodes, which must live at indices 0 and 1; those - /// slots are reserved below `first_new_index` and are never carried, so this cannot - /// collide with a carried green node or a freshly allocated one. + /// slots are reserved below `first_new_index`, so this cannot collide with a freshly + /// allocated node. When carrying, the record appended here overrides the previous + /// session's singleton record carried along in the region. pub(crate) fn send_new_at( &self, index: DepNodeIndex, @@ -1086,9 +1249,14 @@ impl GraphEncoder { edges: EdgesVec, ) -> DepNodeIndex { let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph"); + let kind = node.kind; let node = NodeInfo { node, value_fingerprint, edges }; let mut local = self.status.local.borrow_mut(); self.status.encode_node(index, &node, &self.retained_graph, &mut *local); + if self.status.carrying { + let prev_index = SerializedDepNodeIndex::from_u32(index.as_u32()); + self.status.record_override(prev_index, kind, &mut *local); + } index } @@ -1105,19 +1273,18 @@ impl GraphEncoder { is_green: bool, ) -> DepNodeIndex { let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph"); + let kind = node.kind; let node = NodeInfo { node, value_fingerprint, edges }; let mut local = self.status.local.borrow_mut(); - // A green node keeps its previous index so that any node promoted from the previous - // graph, which refers to it by that index, stays byte-identical and can be carried - // verbatim. A red node changed, so it gets a fresh index above the carried range. - let (index, color) = if is_green { - let index = DepNodeIndex::from_u32(prev_index.as_u32()); - (index, DesiredColor::Green { index }) - } else { - (self.status.next_index(&mut *local), DesiredColor::Red) - }; + // A re-executed node keeps its previous index whether it came out green or red. + // Keeping green indices stable lets records of promoted nodes, which refer to + // their deps by index, stay valid as-is; keeping red indices stable too means + // the appended record simply overrides the carried one, and this session's + // edges (which may point at the red node) need no separate index space. + let index = DepNodeIndex::from_u32(prev_index.as_u32()); + let color = if is_green { DesiredColor::Green { index } } else { DesiredColor::Red }; // Use `try_set_color` to avoid racing when `send_promoted` is called concurrently // on the same index. @@ -1127,19 +1294,24 @@ impl GraphEncoder { TrySetColorResult::AlreadyGreen { index } => return index, } - if !is_green { - self.status.advance_index(&mut *local); - } self.status.encode_node(index, &node, &self.retained_graph, &mut *local); + if self.status.carrying { + self.status.record_override(prev_index, kind, &mut *local); + } index } - /// Encodes a node that was promoted from the previous graph. It reads the information directly - /// from the previous dep graph and expects all edges to already have a new dep node index - /// assigned. + /// Marks a node that was promoted from the previous graph green. It expects all edges + /// to already have a new dep node index assigned. /// /// Tries to mark the dep node green, and returns Some if it is now green, /// or None if had already been concurrently marked red. + /// + /// A promoted node keeps its previous index; its edges (all green, all likewise kept + /// at their previous indices) are exactly the previous edges, so its previous record + /// remains valid. When the record region is carried forward, that record is already + /// in the new file and marking the node green is all there is to do; otherwise the + /// record is re-encoded into the fresh file. #[inline] pub(crate) fn send_promoted( &self, @@ -1147,30 +1319,24 @@ impl GraphEncoder { colors: &DepNodeColorMap, edges: &[DepNodeIndex], ) -> Option { - let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph"); - - let mut local = self.status.local.borrow_mut(); - // A promoted green node keeps its previous index; its edges (all green, all likewise - // kept at their previous indices) are exactly the previous edges, so the record can be - // carried verbatim. let index = DepNodeIndex::from_u32(prev_index.as_u32()); // Use `try_set_color` to avoid racing when `send_promoted` or `send_and_color` // is called concurrently on the same index. match colors.try_set_color(prev_index, DesiredColor::Green { index }) { TrySetColorResult::Success => { - if self.retained_graph.is_none() { - // Fast path: re-emit the previous record instead of re-encoding it. - debug_assert!( - edges.iter().map(|e| e.as_u32()).eq(self - .status - .previous - .edge_targets_from(prev_index) - .map(|e| e.as_u32())), - "carried green node {prev_index:?} edges diverged from the previous graph", - ); - self.status.carry_promoted_node(index, prev_index, &mut *local); - } else { + debug_assert!( + edges.iter().map(|e| e.as_u32()).eq(self + .status + .previous + .edge_targets_from(prev_index) + .map(|e| e.as_u32())), + "promoted green node {prev_index:?} edges diverged from the previous graph", + ); + if !self.status.carrying { + let _prof_timer = + self.profiler.generic_activity("incr_comp_encode_dep_graph"); + let mut local = self.status.local.borrow_mut(); self.status.encode_promoted_node( index, prev_index, @@ -1186,9 +1352,13 @@ impl GraphEncoder { } } - pub(crate) fn finish(&self, current: &CurrentDepGraph) -> FileEncodeResult { + pub(crate) fn finish( + &self, + current: &CurrentDepGraph, + colors: &DepNodeColorMap, + ) -> FileEncodeResult { let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph_finish"); - self.status.finish(&self.profiler, current) + self.status.finish(&self.profiler, current, colors) } } From 876ef966f3e78f39ddba23759b25db24cf6658c0 Mon Sep 17 00:00:00 2001 From: Makro Date: Mon, 13 Jul 2026 12:26:56 +0000 Subject: [PATCH 3/4] perf: dep_graph: skip edge collection in the marking walk when carrying With the record region carried forward, promoting a green node no longer consumes its edge list: the record is already in the new file. The marking walk still collected every dependency's index into a buffer purely to hand it to the encoder. Add a non-collecting variant of the walk for carried sessions that verifies dependency colors and promotes with just the color-map insert, eliminating a store per edge and the per-node edge-frame bookkeeping from the hottest incremental path. Compacting sessions keep the collecting walk, whose encoder still needs the edges. Local measurement on top of the region carry: the walk's self cost on a dep-graph-heavy incr-unchanged rebuild drops by about a quarter, and multi-generation medians improve by up to another 1.5% (match-stress) and 0.9% (hyper incr-patched). 12 generations of incr-unchanged, of alternating real edits, and of -Zincremental-verify-ich all pass; tests/incremental passes in full. Co-Authored-By: Claude Opus 4.8 (1M context) --- compiler/rustc_middle/src/dep_graph/graph.rs | 94 +++++++++++++++++-- .../rustc_middle/src/dep_graph/serialized.rs | 28 ++++++ 2 files changed, 115 insertions(+), 7 deletions(-) diff --git a/compiler/rustc_middle/src/dep_graph/graph.rs b/compiler/rustc_middle/src/dep_graph/graph.rs index 9ffa876649ced..b0f0d744ff8c4 100644 --- a/compiler/rustc_middle/src/dep_graph/graph.rs +++ b/compiler/rustc_middle/src/dep_graph/graph.rs @@ -937,18 +937,98 @@ impl DepGraphData { // in the previous compilation session too, so we can try to // mark it as green by recursively marking all of its // dependencies green. - - // Reuse a per-worker buffer for the edges instead of allocating one per call. - // The recursion gives it back empty: each `EdgeFrame` pops its edges on drop. - let mut edge_buf = self.green_edge_buf.take(); - let result = self.try_mark_previous_green(tcx, prev_index, None, &mut edge_buf); - debug_assert!(edge_buf.is_empty()); - self.green_edge_buf.set(edge_buf); + let result = if self.current.encoder.is_carrying() { + // The carried record region already contains this node's record, so + // promoting it needs no edge list: verify the dependencies without + // collecting their indices. + self.try_mark_previous_green_carried(tcx, prev_index, None) + } else { + // Reuse a per-worker buffer for the edges instead of allocating one per + // call. The recursion gives it back empty: each `EdgeFrame` pops its + // edges on drop. + let mut edge_buf = self.green_edge_buf.take(); + let result = self.try_mark_previous_green(tcx, prev_index, None, &mut edge_buf); + debug_assert!(edge_buf.is_empty()); + self.green_edge_buf.set(edge_buf); + result + }; result.map(|dep_node_index| (prev_index, dep_node_index)) } } } + /// Try to mark a dep-node which existed in the previous compilation session as green, + /// without collecting its edges. Used when the record region is carried forward: the + /// node's record is already in the new file, so all promotion needs is the color-map + /// insert, and the edge indices (which equal the previous ones) are never materialized. + #[instrument(skip(self, tcx, prev_dep_node_index, frame), level = "debug")] + fn try_mark_previous_green_carried<'tcx>( + &self, + tcx: TyCtxt<'tcx>, + prev_dep_node_index: SerializedDepNodeIndex, + frame: Option<&MarkFrame<'_>>, + ) -> Option { + let frame = MarkFrame { index: prev_dep_node_index, parent: frame }; + + // We never try to mark eval_always nodes as green + debug_assert!(!tcx.is_eval_always(self.previous.index_to_node(prev_dep_node_index).kind)); + + for parent_dep_node_index in self.previous.edge_targets_from(prev_dep_node_index) { + match self.colors.get(parent_dep_node_index) { + DepNodeColor::Green(_) => continue, + DepNodeColor::Red => return None, + DepNodeColor::Unknown => {} + } + + let parent_dep_node = self.previous.index_to_node(parent_dep_node_index); + + // If this dependency isn't eval_always, try to mark it green recursively. + if !tcx.is_eval_always(parent_dep_node.kind) + && self + .try_mark_previous_green_carried(tcx, parent_dep_node_index, Some(&frame)) + .is_some() + { + continue; + } + + // We failed to mark it green, so we try to force the query. + if !tcx.try_force_from_dep_node(*parent_dep_node, parent_dep_node_index, &frame) { + return None; + } + + match self.colors.get(parent_dep_node_index) { + DepNodeColor::Green(_) => continue, + DepNodeColor::Red => return None, + DepNodeColor::Unknown => {} + } + + if tcx.dcx().has_errors_or_delayed_bugs().is_none() { + panic!("try_mark_previous_green_carried() - forcing failed to set a color"); + } + + // A forced query that errored leaves the color unset; give up like the + // collecting walk does and rely on the cache not being persisted. + return None; + } + + // All dependencies are green: promote this node with just the color-map insert. + // `no_hash` nodes may fail this promotion due to already being conservatively + // colored red. + let dep_node_index = self.current.encoder.send_promoted_carried( + prev_dep_node_index, + &self.colors, + )?; + + #[cfg(debug_assertions)] + self.current.record_edge( + dep_node_index, + *self.previous.index_to_node(prev_dep_node_index), + self.previous.value_fingerprint_for_index(prev_dep_node_index), + ); + + Some(dep_node_index) + } + /// Try to mark a dep-node which existed in the previous compilation session as green. #[instrument(skip(self, tcx, prev_dep_node_index, frame, edge_buf), level = "debug")] fn try_mark_previous_green<'tcx>( diff --git a/compiler/rustc_middle/src/dep_graph/serialized.rs b/compiler/rustc_middle/src/dep_graph/serialized.rs index f5f0927f891eb..214e7a7c018ee 100644 --- a/compiler/rustc_middle/src/dep_graph/serialized.rs +++ b/compiler/rustc_middle/src/dep_graph/serialized.rs @@ -1220,6 +1220,34 @@ impl GraphEncoder { self.retained_graph.as_ref().map(|retained_graph| retained_graph.lock().clone()) } + /// Whether this session carries the previous record region forward. When true, + /// promoting a node from the previous graph needs no edge list, so the marking + /// walk can skip collecting edge indices entirely. + #[inline] + pub(crate) fn is_carrying(&self) -> bool { + self.status.carrying + } + + /// Marks a node promoted from the previous graph green without materializing its + /// edges. Only valid when carrying: the node's record is already in the new file. + /// + /// Returns Some if the node is now green, or None if it had already been + /// concurrently marked red. + #[inline] + pub(crate) fn send_promoted_carried( + &self, + prev_index: SerializedDepNodeIndex, + colors: &DepNodeColorMap, + ) -> Option { + debug_assert!(self.status.carrying); + let index = DepNodeIndex::from_u32(prev_index.as_u32()); + match colors.try_set_color(prev_index, DesiredColor::Green { index }) { + TrySetColorResult::Success => Some(index), + TrySetColorResult::AlreadyRed => None, + TrySetColorResult::AlreadyGreen { index } => Some(index), + } + } + /// Encodes a node that does not exists in the previous graph. pub(crate) fn send_new( &self, From b0bcb1e0c31e0c6b9b3c33ff41f9b7e165da94a8 Mon Sep 17 00:00:00 2001 From: Makro Date: Mon, 13 Jul 2026 13:03:46 +0000 Subject: [PATCH 4/4] perf: dep_graph: serve edge lists in place and compact by dead-byte ratio Decoding used to copy every edge byte out of the file into a freshly allocated buffer, the largest allocation and copy of the load. The file stays mapped for the whole session anyway (the carry copies its record region forward), so serve the edge lists directly from the retained bytes: the on-disk varint encoding was already the in-memory representation, and each node's edge header now records a position in the file instead of a position in the copied buffer. Decoding a record shrinks to reading its fixed header and skipping over the edges. The fixed-size overread at the end of an edge list stays in bounds because the footer always follows the records. On Windows, where the mapping must not outlive the load because the save renames over the mapped file, the bytes are copied out once instead. Compaction is now also triggered by the dead-byte ratio: the file is rewritten fresh once the record region exceeds twice its live bytes, tracked exactly in the footer as an O(changed) delta like the other counts. High-churn graphs compact as often as before, while low-churn graphs carry for up to sixteen generations instead of eight, avoiding periodic compaction spikes the fixed cap forced on them. Local rustc-perf (primary crates, instructions:u) on top of the previous commits: incr-unchanged -0.49%, incr-patched -0.32%, full and incr-full flat, 26 cells improved by at least 0.25%, 0 regressed. 12 generations of incr-unchanged, of alternating real edits, and of -Zincremental-verify-ich all pass; tests/incremental passes in full. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../rustc_middle/src/dep_graph/serialized.rs | 197 ++++++++++++------ 1 file changed, 129 insertions(+), 68 deletions(-) diff --git a/compiler/rustc_middle/src/dep_graph/serialized.rs b/compiler/rustc_middle/src/dep_graph/serialized.rs index 214e7a7c018ee..1f668d1fa83f6 100644 --- a/compiler/rustc_middle/src/dep_graph/serialized.rs +++ b/compiler/rustc_middle/src/dep_graph/serialized.rs @@ -97,9 +97,6 @@ impl SerializedDepNodeIndex { } const DEP_NODE_SIZE: usize = size_of::(); -/// Amount of padding we need to add to the edge list data so that we can retrieve every -/// SerializedDepNodeIndex with a fixed-size read then mask. -const DEP_NODE_PAD: usize = DEP_NODE_SIZE - 1; /// Number of bits we need to store the number of used bytes in a SerializedDepNodeIndex. /// Note that wherever we encode byte widths like this we actually store the number of bytes used /// minus 1; for a 4-byte value we technically would have 5 widths to store, but using one byte to @@ -121,13 +118,11 @@ pub struct SerializedDepGraph { /// Some nodes don't have a meaningful value hash (e.g. queries with `no_hash`), /// so they store a dummy value here instead (e.g. [`Fingerprint::ZERO`]). value_fingerprints: IndexVec, - /// For each DepNode, stores the list of edges originating from that - /// DepNode. Encoded as a [start, end) pair indexing into edge_list_data, - /// which holds the actual DepNodeIndices of the target nodes. + /// For each DepNode, stores the position and byte width of its edge list within + /// the retained file bytes ([`Self::backing`]), which serve as the edge data + /// directly: the on-disk varint encoding is also the in-memory representation, + /// so decoding copies no edge bytes at all. edge_list_indices: IndexVec, - /// A flattened list of all edge targets in the graph, stored in the same - /// varint encoding that we use on disk. Edge sources are implicit in edge_list_indices. - edge_list_data: Vec, /// The lazily-built inverse of `nodes`: maps a [`DepNode`] back to its /// [`SerializedDepNodeIndex`] via the node's key fingerprint. See /// [`LazyNodeIndex`]. @@ -139,12 +134,13 @@ pub struct SerializedDepGraph { /// a compacting rewrite. Dead records and superseded duplicates accumulate with /// each carried generation, so the writer compacts once this grows too large. generation: u64, - /// The memory-mapped bytes of the file this graph was decoded from, retained so - /// that the record region can be copied into the next session's file wholesale - /// (see [`Self::region_bytes`]). `None` for the empty default graph, which - /// disables the carry. - mmap: Option, - /// The byte range of the record region within [`Self::mmap`]: every node record, + /// The bytes of the file this graph was decoded from, retained both to serve the + /// edge lists in place (see [`Self::edge_list_indices`]) and so that the record + /// region can be copied into the next session's file wholesale (see + /// [`Self::region_bytes`]). `None` only for the empty default graph, which has no + /// nodes and never carries. + backing: Option, + /// The byte range of the record region within [`Self::backing`]: every node record, /// including dead and superseded ones, and nothing else. records_range: std::ops::Range, /// Indices whose record in the region is dead: the node was dropped by an earlier @@ -159,11 +155,37 @@ pub struct SerializedDepGraph { live_node_count: u64, /// The number of edges of live nodes, from the file footer. live_edge_count: u64, + /// The number of record-region bytes belonging to live records, from the file + /// footer. Compared against the region size to decide when accumulated dead and + /// superseded records warrant a compacting rewrite. + live_record_bytes: u64, /// Used to time the lazy per-`DepKind` reverse-index build. `None` only for /// the empty default graph, which is never looked up. profiler: Option, } +/// The retained bytes of the previous dep-graph file. +/// +/// On most platforms this is the memory mapping the file was decoded from. On Windows +/// the mapping cannot stay alive while the file is later replaced by the save's rename, +/// so the bytes are copied out once and the mapping is released. +enum Backing { + Mapped(Mmap), + #[allow(dead_code)] + Owned(Vec), +} + +impl std::ops::Deref for Backing { + type Target = [u8]; + #[inline] + fn deref(&self) -> &[u8] { + match self { + Backing::Mapped(mmap) => mmap, + Backing::Owned(bytes) => bytes, + } + } +} + // `SelfProfilerRef` is not `Debug`, so we can't derive this. impl std::fmt::Debug for SerializedDepGraph { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -171,7 +193,6 @@ impl std::fmt::Debug for SerializedDepGraph { .field("nodes", &self.nodes) .field("value_fingerprints", &self.value_fingerprints) .field("edge_list_indices", &self.edge_list_indices) - .field("edge_list_data", &self.edge_list_data) .field("reverse_index", &self.reverse_index) .field("session_count", &self.session_count) .finish_non_exhaustive() @@ -250,7 +271,12 @@ impl SerializedDepGraph { source: SerializedDepNodeIndex, ) -> impl Iterator + Clone { let header = self.edge_list_indices[source]; - let mut raw = &self.edge_list_data[header.start()..]; + // The edge bytes are read in place from the retained file. A node with edges + // always comes from a real file, so the backing is present. The fixed-size + // read below may extend a few bytes past the edge list; that is always still + // within the file, since the records are followed by the footer, which is at + // least a dead-list length and the fixed-size tail. + let mut raw = &self.backing.as_ref().unwrap()[header.start()..]; let bytes_per_index = header.bytes_per_index(); @@ -305,7 +331,7 @@ impl SerializedDepGraph { /// wholesale. False for the empty default graph (no retained bytes). #[inline] fn can_carry(&self) -> bool { - self.mmap.is_some() + self.backing.is_some() } /// The raw bytes of the record region, exactly as they appeared in this graph's @@ -316,9 +342,15 @@ impl SerializedDepGraph { /// nodes are byte-for-byte what a fresh encode would produce, records superseded by /// a re-executed node are overridden by the appended record at the same index, and /// records of dropped nodes are tombstoned via the dead list in the footer. + /// The size in bytes of the record region, dead and superseded records included. + #[inline] + fn region_size(&self) -> u64 { + self.records_range.len() as u64 + } + #[inline] fn region_bytes(&self) -> &[u8] { - &self.mmap.as_ref().unwrap()[self.records_range.clone()] + &self.backing.as_ref().unwrap()[self.records_range.clone()] } /// The number of edges of the node at `index`, used for O(changed) footer accounting. @@ -327,6 +359,22 @@ impl SerializedDepGraph { self.edge_list_indices[index].num_edges as usize } + /// The size in bytes of the encoded record of the node at `index`: the fixed + /// header, the spilled edge count if the header could not hold it, and the edge + /// bytes. Used for O(changed) accounting of the live record bytes. + fn record_size_for_index(&self, index: SerializedDepNodeIndex) -> u64 { + let header = self.edge_list_indices[index]; + let num_edges = header.num_edges; + let spill = if num_edges as usize > SerializedNodeHeader::MAX_INLINE_LEN { + leb128_u32_len(num_edges) + } else { + 0 + }; + size_of::() as u64 + + spill + + num_edges as u64 * header.bytes_per_index() as u64 + } + /// Whether the node at `index` has a (live or superseded) record in the region. /// `Null` slots come from batch index allocation and dead records of earlier /// generations; neither leaves a live record to tombstone. @@ -335,13 +383,27 @@ impl SerializedDepGraph { self.nodes[index].kind != DepKind::Null } - /// Attaches the retained file bytes decoded by [`Self::decode`], enabling the - /// carry of this graph's record region into the next session's file. + /// Attaches the file bytes decoded by [`Self::decode`], serving the edge lists in + /// place and enabling the carry of the record region into the next session's file. + /// + /// On Windows the mapping must not outlive the load, since the save later replaces + /// the mapped file by renaming over it, so the bytes are copied out instead. pub fn attach_mmap(&mut self, mmap: Mmap) { - self.mmap = Some(mmap); + if cfg!(windows) { + self.backing = Some(Backing::Owned(mmap.to_vec())); + } else { + self.backing = Some(Backing::Mapped(mmap)); + } } } +/// The encoded length of `value` as unsigned leb128, matching what +/// [`rustc_serialize::leb128`] writes for a `u32`. +#[inline] +fn leb128_u32_len(value: u32) -> u64 { + (31 - (value | 1).leading_zeros() as u64) / 7 + 1 +} + /// A packed representation of an edge's start index and byte width. /// /// This is packed by stealing 2 bits from the start index, which means we only accommodate edge @@ -402,7 +464,7 @@ impl SerializedDepGraph { // The footer between the records and the fixed-size tail: the dead list, the // per-kind live counts, the session count and the carried generation count. // Read it up front, as decoding the records requires the dead set. - let (dead, dead_set, kind_stats, session_count, generation) = + let (dead, dead_set, kind_stats, session_count, generation, live_record_bytes) = d.with_position(dead_pos, |d| { let dead_len = d.read_u64() as usize; let mut dead = Vec::with_capacity(dead_len); @@ -416,14 +478,10 @@ impl SerializedDepGraph { (0..(DepKind::MAX + 1)).map(|_| d.read_u32()).collect(); let session_count = d.read_u64(); let generation = d.read_u64(); - (dead, dead_set, kind_stats, session_count, generation) + let live_record_bytes = d.read_u64(); + (dead, dead_set, kind_stats, session_count, generation, live_record_bytes) }); - // The record region may contain more than `node_count` records: dead records - // and superseded ones (a later record at the same index overrides an earlier - // one). This makes the capacity estimate below overshoot slightly more. - let graph_bytes = dead_pos - records_start; - let mut nodes = IndexVec::from_elem_n( DepNode { kind: DepKind::Null, @@ -435,18 +493,6 @@ impl SerializedDepGraph { let mut edge_list_indices = IndexVec::from_elem_n(EdgeHeader { repr: 0, num_edges: 0 }, node_max); - // This estimation assumes that all of the encoded bytes are for the edge lists or for the - // fixed-size node headers. But that's not necessarily true; if any edge list has a length - // that spills out of the size we can bit-pack into SerializedNodeHeader then some of the - // total serialized size is also used by leb128-encoded edge list lengths. Neglecting that - // contribution to graph_bytes means our estimation of the bytes needed for edge_list_data - // slightly overshoots. But it cannot overshoot by much; consider that the worse case is - // for a node with length 64, which means the spilled 1-byte leb128 length is 1 byte of at - // least (34 byte header + 1 byte len + 64 bytes edge data), which is ~1%. A 2-byte leb128 - // length is about the same fractional overhead and it amortizes for yet greater lengths. - let mut edge_list_data = - Vec::with_capacity(graph_bytes - node_count * size_of::()); - while d.position() < dead_pos { // Decode the header for this edge; the header packs together as many of the fixed-size // fields as possible to limit the number of times we update decoder state. @@ -459,14 +505,16 @@ impl SerializedDepGraph { let num_edges = node_header.len().unwrap_or_else(|| d.read_u32()); // The edges index list uses the same varint strategy as rmeta tables; we select the - // number of byte elements per-array not per-element. This lets us read the whole edge - // list for a node with one decoder call and also use the on-disk format in memory. + // number of byte elements per-array not per-element. The edge bytes are not copied + // anywhere: they are later read in place from the retained file bytes, so decoding + // only records where they start and skips over them. + let edges_start = d.position(); let edges_len_bytes = node_header.bytes_per_index() * (num_edges as usize); + d.read_raw_bytes(edges_len_bytes); // A dead record: the node was dropped by an earlier session but its bytes were // carried along in the region. Skip it; its slot stays `Null`. if dead_set.contains(index) { - d.read_raw_bytes(edges_len_bytes); continue; } @@ -484,21 +532,9 @@ impl SerializedDepGraph { value_fingerprints[index] = node_header.value_fingerprint(); - // The in-memory structure for the edges list stores the byte width of the edges on - // this node with the offset into the global edge data array. On an override the - // earlier record's edge bytes are simply orphaned in `edge_list_data`. - let edges_header = node_header.edges_header(&edge_list_data, num_edges); - - edge_list_data.extend(d.read_raw_bytes(edges_len_bytes)); - - edge_list_indices[index] = edges_header; + edge_list_indices[index] = node_header.edges_header(edges_start, num_edges); } - // When we access the edge list data, we do a fixed-size read from the edge list data then - // mask off the bytes that aren't for that edge index, so the last read may dangle off the - // end of the array. This padding ensure it doesn't. - edge_list_data.extend(&[0u8; DEP_NODE_PAD]); - // Lay out the per-kind live counts (read from the footer above) as contiguous // ranges for the counting sort of `LazyNodeIndex`. let mut kinds = Vec::with_capacity(DepKind::MAX as usize + 1); @@ -533,17 +569,17 @@ impl SerializedDepGraph { nodes, value_fingerprints, edge_list_indices, - edge_list_data, reverse_index, session_count, generation, // The retained file bytes are attached by the caller via `attach_mmap`. - mmap: None, + backing: None, records_range: records_start..dead_pos, dead, kind_stats, live_node_count: node_count as u64, live_edge_count: edge_count as u64, + live_record_bytes, profiler: Some(profiler.clone()), }) } @@ -681,9 +717,9 @@ impl SerializedNodeHeader { } #[inline] - fn edges_header(&self, edge_list_data: &[u8], num_edges: u32) -> EdgeHeader { + fn edges_header(&self, edges_start: usize, num_edges: u32) -> EdgeHeader { EdgeHeader { - repr: (edge_list_data.len() << DEP_NODE_WIDTH_BITS) | (self.bytes_per_index() - 1), + repr: (edges_start << DEP_NODE_WIDTH_BITS) | (self.bytes_per_index() - 1), num_edges, } } @@ -777,6 +813,9 @@ struct LocalEncoderState { /// Net change to the live edge count from this worker's appends. An override /// contributes the difference between its new and old edge counts. edge_count: i64, + /// Net change to the live record bytes from this worker's appends. An override + /// contributes the difference between its new and old record sizes. + record_bytes: i64, /// Indices below `first_new_index` this worker appended records for. Those appends /// override the carried record at the same index; anything occupied, not overridden /// and not marked green by the end of the session is dead. @@ -791,6 +830,7 @@ struct LocalEncoderResult { node_max: u32, node_count: i64, edge_count: i64, + record_bytes: i64, overridden: Vec, /// Stores the net change to the number of live nodes of each dep kind. @@ -847,6 +887,7 @@ impl EncoderState { remaining_node_index: 0, edge_count: 0, node_count: 0, + record_bytes: 0, overridden: Vec::new(), encoder: MemEncoder::new(), kind_stats: iter::repeat_n(0, DepKind::MAX as usize + 1).collect(), @@ -946,7 +987,9 @@ impl EncoderState { retained_graph: &Option>, local: &mut LocalEncoderState, ) { + let before = local.encoder.position(); node.encode(&mut local.encoder, index); + local.record_bytes += (local.encoder.position() - before) as i64; self.flush_mem_encoder(&mut *local); self.count_node(&mut *local); self.record(&node.node, index, node.edges.len(), &node.edges, retained_graph, &mut *local); @@ -969,8 +1012,10 @@ impl EncoderState { ) { let node = self.previous.index_to_node(prev_index); let value_fingerprint = self.previous.value_fingerprint_for_index(prev_index); + let before = local.encoder.position(); let edge_count = NodeInfo::encode_promoted(&mut local.encoder, node, index, value_fingerprint, edges); + local.record_bytes += (local.encoder.position() - before) as i64; self.flush_mem_encoder(&mut *local); self.count_node(&mut *local); self.record(node, index, edge_count, edges, retained_graph, &mut *local); @@ -991,6 +1036,7 @@ impl EncoderState { local.node_count -= 1; local.kind_stats[kind.as_usize()] -= 1; local.edge_count -= self.previous.edge_count_for_index(prev_index) as i64; + local.record_bytes -= self.previous.record_size_for_index(prev_index) as i64; local.overridden.push(prev_index); } @@ -1017,6 +1063,7 @@ impl EncoderState { node_max: local.next_node_index, node_count: local.node_count, edge_count: local.edge_count, + record_bytes: local.record_bytes, overridden: mem::take(&mut local.overridden), } }); @@ -1026,14 +1073,16 @@ impl EncoderState { // Every count starts from the previous footer when carrying (the region already // holds those nodes) and from zero when writing a fresh file; the workers report // net changes in either case. - let (mut kind_stats, mut node_count, mut edge_count) = if self.carrying { + let (mut kind_stats, mut node_count, mut edge_count, mut record_bytes) = if self.carrying + { ( self.previous.kind_stats.clone(), self.previous.live_node_count as i64, self.previous.live_edge_count as i64, + self.previous.live_record_bytes as i64, ) } else { - (iter::repeat_n(0, DepKind::MAX as usize + 1).collect(), 0, 0) + (iter::repeat_n(0, DepKind::MAX as usize + 1).collect(), 0, 0, 0) }; let mut node_max = 0; @@ -1043,6 +1092,7 @@ impl EncoderState { node_max = max(node_max, result.node_max); node_count += result.node_count; edge_count += result.edge_count; + record_bytes += result.record_bytes; for (i, stat) in result.kind_stats.iter().enumerate() { // The per-worker values are net changes: an override decrements the kind // it previously incremented, so the sum stays balanced per worker and the @@ -1079,6 +1129,7 @@ impl EncoderState { kind_stats[kind.as_usize()] -= 1; node_count -= 1; edge_count -= self.previous.edge_count_for_index(index) as i64; + record_bytes -= self.previous.record_size_for_index(index) as i64; } } } @@ -1099,6 +1150,7 @@ impl EncoderState { self.previous.session_count.checked_add(1).unwrap().encode(&mut encoder); generation.encode(&mut encoder); + u64::try_from(record_bytes).unwrap().encode(&mut encoder); debug!(?node_max, ?node_count, ?edge_count); debug!("position: {:?}", encoder.position()); @@ -1186,10 +1238,18 @@ pub(crate) struct GraphEncoder { retained_graph: Option>, } -/// After this many consecutive carried generations, write a fresh file instead. Each -/// carried generation leaves behind dead records, superseded records and their orphaned -/// index slots; a compacting rewrite reclaims all of it. -const MAX_CARRIED_GENERATIONS: u64 = 8; +/// After this many consecutive carried generations, write a fresh file instead even if +/// the dead-byte ratio has not tripped, bounding the growth of orphaned index slots +/// (which the ratio does not measure). Each carried generation leaves behind dead +/// records, superseded records and their orphaned index slots; a compacting rewrite +/// reclaims all of it. +const MAX_CARRIED_GENERATIONS: u64 = 16; + +/// Write a fresh file once the record region exceeds this multiple of its live record +/// bytes. Dead and superseded records make decoding and the wholesale region copy +/// proportionally more expensive, so this bounds that overhead at a fixed factor while +/// letting low-churn graphs carry for many generations. +const MAX_REGION_GROWTH: u64 = 2; impl GraphEncoder { pub(crate) fn new( @@ -1211,7 +1271,8 @@ impl GraphEncoder { let carrying = previous.can_carry() && retained_graph.is_none() && !record_stats - && previous.generation + 1 < MAX_CARRIED_GENERATIONS; + && previous.generation + 1 < MAX_CARRIED_GENERATIONS + && previous.region_size() <= MAX_REGION_GROWTH * previous.live_record_bytes; let status = EncoderState::new(encoder, record_stats, previous, carrying); GraphEncoder { status, retained_graph, profiler: sess.prof.clone() } }