diff --git a/compiler/rustc_incremental/src/persist/file_format.rs b/compiler/rustc_incremental/src/persist/file_format.rs index 853a5c9ba7ab0..0e4102e54b77f 100644 --- a/compiler/rustc_incremental/src/persist/file_format.rs +++ b/compiler/rustc_incremental/src/persist/file_format.rs @@ -26,7 +26,7 @@ use crate::diagnostics; const FILE_MAGIC: &[u8] = b"RSIC"; /// Change this if the header format changes. -const HEADER_FORMAT_VERSION: u16 = 0; +const HEADER_FORMAT_VERSION: u16 = 1; pub(crate) fn write_file_header(stream: &mut FileEncoder<'_>, sess: &Session) { stream.emit_raw_bytes(FILE_MAGIC); diff --git a/compiler/rustc_incremental/src/persist/load.rs b/compiler/rustc_incremental/src/persist/load.rs index c3b9f433417a6..83d0984e5df5b 100644 --- a/compiler/rustc_incremental/src/persist/load.rs +++ b/compiler/rustc_incremental/src/persist/load.rs @@ -112,7 +112,16 @@ fn load_dep_graph(sess: &Session) -> LoadResult { return LoadResult::DataOutOfDate; } - let prev_graph = SerializedDepGraph::decode(&mut decoder, &sess.prof); + let mut prev_graph = SerializedDepGraph::decode(&mut decoder, &sess.prof); + + // Retain the file bytes so that the record region can be carried into the + // next session's dep-graph file wholesale. The decoder borrows `mmap`, so + // release it before handing ownership to the freshly decoded (uniquely + // owned) graph. + drop(decoder); + std::sync::Arc::get_mut(&mut prev_graph) + .expect("freshly decoded dep graph is uniquely owned") + .attach_mmap(mmap); LoadResult::Ok { prev_graph, prev_work_products } } diff --git a/compiler/rustc_middle/src/dep_graph/graph.rs b/compiler/rustc_middle/src/dep_graph/graph.rs index 86cf7be60b858..b0f0d744ff8c4 100644 --- a/compiler/rustc_middle/src/dep_graph/graph.rs +++ b/compiler/rustc_middle/src/dep_graph/graph.rs @@ -181,7 +181,10 @@ impl DepGraph { let colors = DepNodeColorMap::new(prev_graph_node_count); // Instantiate a node with zero dependencies only once for anonymous queries. - let _green_node_index = current.alloc_new_node( + // The two singletons always live at fixed indices 0 and 1, which are reserved below + // the range handed out to new nodes so they never collide with a carried green node. + let _green_node_index = current.alloc_singleton_node( + DepNodeIndex::SINGLETON_ZERO_DEPS_ANON_NODE, DepNode { kind: DepKind::AnonZeroDeps, key_fingerprint: current.anon_id_seed.into() }, EdgesVec::new(), Fingerprint::ZERO, @@ -191,7 +194,8 @@ impl DepGraph { // Create a single always-red node, with no dependencies of its own. // Other nodes can use the always-red node as a fake dependency, to // ensure that their dependency list will never be all-green. - let red_node_index = current.alloc_new_node( + let red_node_index = current.alloc_singleton_node( + DepNodeIndex::FOREVER_RED_NODE, DepNode { kind: DepKind::Red, key_fingerprint: Fingerprint::ZERO.into() }, EdgesVec::new(), Fingerprint::ZERO, @@ -202,6 +206,23 @@ impl DepGraph { const { SerializedDepNodeIndex::from_u32(DepNodeIndex::FOREVER_RED_NODE.as_u32()) }; let result = colors.try_set_color(prev_index, DesiredColor::Red); assert_matches!(result, TrySetColorResult::Success); + + // The previous graph's anon-zero-deps singleton also lives at a fixed index (0), the + // same one this session's singleton was just allocated at. It is deterministically + // green (an anonymous query with no dependencies never changes), and this session's + // singleton is its equivalent. Color it green pointing at that fresh node now, so it + // is never separately promoted, which would carry a second record into index 0 and + // corrupt the file. This mirrors how the always-red singleton is pinned above. + let anon_prev_index = const { + SerializedDepNodeIndex::from_u32( + DepNodeIndex::SINGLETON_ZERO_DEPS_ANON_NODE.as_u32(), + ) + }; + let result = colors.try_set_color( + anon_prev_index, + DesiredColor::Green { index: DepNodeIndex::SINGLETON_ZERO_DEPS_ANON_NODE }, + ); + assert_matches!(result, TrySetColorResult::Success); } DepGraph { @@ -916,18 +937,98 @@ impl DepGraphData { // in the previous compilation session too, so we can try to // mark it as green by recursively marking all of its // dependencies green. - - // Reuse a per-worker buffer for the edges instead of allocating one per call. - // The recursion gives it back empty: each `EdgeFrame` pops its edges on drop. - let mut edge_buf = self.green_edge_buf.take(); - let result = self.try_mark_previous_green(tcx, prev_index, None, &mut edge_buf); - debug_assert!(edge_buf.is_empty()); - self.green_edge_buf.set(edge_buf); + let result = if self.current.encoder.is_carrying() { + // The carried record region already contains this node's record, so + // promoting it needs no edge list: verify the dependencies without + // collecting their indices. + self.try_mark_previous_green_carried(tcx, prev_index, None) + } else { + // Reuse a per-worker buffer for the edges instead of allocating one per + // call. The recursion gives it back empty: each `EdgeFrame` pops its + // edges on drop. + let mut edge_buf = self.green_edge_buf.take(); + let result = self.try_mark_previous_green(tcx, prev_index, None, &mut edge_buf); + debug_assert!(edge_buf.is_empty()); + self.green_edge_buf.set(edge_buf); + result + }; result.map(|dep_node_index| (prev_index, dep_node_index)) } } } + /// Try to mark a dep-node which existed in the previous compilation session as green, + /// without collecting its edges. Used when the record region is carried forward: the + /// node's record is already in the new file, so all promotion needs is the color-map + /// insert, and the edge indices (which equal the previous ones) are never materialized. + #[instrument(skip(self, tcx, prev_dep_node_index, frame), level = "debug")] + fn try_mark_previous_green_carried<'tcx>( + &self, + tcx: TyCtxt<'tcx>, + prev_dep_node_index: SerializedDepNodeIndex, + frame: Option<&MarkFrame<'_>>, + ) -> Option { + let frame = MarkFrame { index: prev_dep_node_index, parent: frame }; + + // We never try to mark eval_always nodes as green + debug_assert!(!tcx.is_eval_always(self.previous.index_to_node(prev_dep_node_index).kind)); + + for parent_dep_node_index in self.previous.edge_targets_from(prev_dep_node_index) { + match self.colors.get(parent_dep_node_index) { + DepNodeColor::Green(_) => continue, + DepNodeColor::Red => return None, + DepNodeColor::Unknown => {} + } + + let parent_dep_node = self.previous.index_to_node(parent_dep_node_index); + + // If this dependency isn't eval_always, try to mark it green recursively. + if !tcx.is_eval_always(parent_dep_node.kind) + && self + .try_mark_previous_green_carried(tcx, parent_dep_node_index, Some(&frame)) + .is_some() + { + continue; + } + + // We failed to mark it green, so we try to force the query. + if !tcx.try_force_from_dep_node(*parent_dep_node, parent_dep_node_index, &frame) { + return None; + } + + match self.colors.get(parent_dep_node_index) { + DepNodeColor::Green(_) => continue, + DepNodeColor::Red => return None, + DepNodeColor::Unknown => {} + } + + if tcx.dcx().has_errors_or_delayed_bugs().is_none() { + panic!("try_mark_previous_green_carried() - forcing failed to set a color"); + } + + // A forced query that errored leaves the color unset; give up like the + // collecting walk does and rely on the cache not being persisted. + return None; + } + + // All dependencies are green: promote this node with just the color-map insert. + // `no_hash` nodes may fail this promotion due to already being conservatively + // colored red. + let dep_node_index = self.current.encoder.send_promoted_carried( + prev_dep_node_index, + &self.colors, + )?; + + #[cfg(debug_assertions)] + self.current.record_edge( + dep_node_index, + *self.previous.index_to_node(prev_dep_node_index), + self.previous.value_fingerprint_for_index(prev_dep_node_index), + ); + + Some(dep_node_index) + } + /// Try to mark a dep-node which existed in the previous compilation session as green. #[instrument(skip(self, tcx, prev_dep_node_index, frame, edge_buf), level = "debug")] fn try_mark_previous_green<'tcx>( @@ -1081,7 +1182,11 @@ impl DepGraph { } pub(crate) fn finish_encoding(&self) -> FileEncodeResult { - if let Some(data) = &self.data { data.current.encoder.finish(&data.current) } else { Ok(0) } + if let Some(data) = &self.data { + data.current.encoder.finish(&data.current, &data.colors) + } else { + Ok(0) + } } pub fn next_virtual_depnode_index(&self) -> DepNodeIndex { @@ -1266,6 +1371,24 @@ impl CurrentDepGraph { dep_node_index } + + /// Allocates a node at a fixed index. Used only for the two singleton nodes, which must + /// live at indices 0 and 1 across every session. + #[inline(always)] + fn alloc_singleton_node( + &self, + index: DepNodeIndex, + key: DepNode, + edges: EdgesVec, + value_fingerprint: Fingerprint, + ) -> DepNodeIndex { + let dep_node_index = self.encoder.send_new_at(index, key, value_fingerprint, edges); + + #[cfg(debug_assertions)] + self.record_edge(dep_node_index, key, value_fingerprint); + + dep_node_index + } } #[derive(Debug, Clone, Copy)] @@ -1382,6 +1505,13 @@ impl DepNodeColorMap { DepNodeColor::Unknown } } + + /// Whether the node was marked green this session. Used by the encoder when it + /// computes which carried records are dead. + #[inline] + pub(super) fn is_green(&self, index: SerializedDepNodeIndex) -> bool { + self.values[index].load(Ordering::Acquire) < COMPRESSED_RED + } } /// The color that [`DepNodeColorMap::try_set_color`] should try to apply to a node. diff --git a/compiler/rustc_middle/src/dep_graph/serialized.rs b/compiler/rustc_middle/src/dep_graph/serialized.rs index dbaf29745a8bc..1f668d1fa83f6 100644 --- a/compiler/rustc_middle/src/dep_graph/serialized.rs +++ b/compiler/rustc_middle/src/dep_graph/serialized.rs @@ -1,12 +1,25 @@ //! The data that we will serialize and deserialize. //! //! Notionally, the dep-graph is a sequence of NodeInfo with the dependencies -//! specified inline. The total number of nodes and edges are stored as the last -//! 16 bytes of the file, so we can find them easily at decoding time. +//! specified inline. A footer stores the dead list, the per-kind counts, and the +//! total number of nodes and edges, with fixed-size positions at the very end of +//! the file so we can find them easily at decoding time. //! //! The serialisation is performed on-demand when each node is emitted. Using this //! scheme, we do not need to keep the current graph in memory. //! +//! On a warm rebuild most nodes are unchanged. Rather than re-encoding them, the +//! previous file's record region is copied into the new file wholesale and every +//! node that existed in the previous session keeps its index: a node re-verified +//! green needs no write at all (its carried record, whose edges point at other +//! kept indices, is already exactly right), a re-executed node appends a record +//! at its old index which overrides the carried one (later records win at decode +//! time), and a node this session dropped is tombstoned via the dead list in the +//! footer. Genuinely new nodes get indices above the previous index space. Dead +//! records and superseded duplicates accumulate with each carried generation, so +//! after [`MAX_CARRIED_GENERATIONS`] the file is rewritten fresh, which also +//! happens when a debugging feature needs every node to pass through the encoder. +//! //! The deserialization is performed manually, in order to convert from the stored //! sequence of NodeInfos to the different arrays in SerializedDepGraph. Since the //! node and edge count are stored at the end of the file, all the arrays can be @@ -47,7 +60,9 @@ use std::{iter, mem}; use rustc_data_structures::fingerprint::{Fingerprint, PackedFingerprint}; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::memmap::Mmap; use rustc_data_structures::outline; +use rustc_index::bit_set::DenseBitSet; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::sync::{AtomicU64, Lock, WorkerLocal, broadcast}; use rustc_data_structures::unhash::UnhashMap; @@ -82,9 +97,6 @@ impl SerializedDepNodeIndex { } const DEP_NODE_SIZE: usize = size_of::(); -/// Amount of padding we need to add to the edge list data so that we can retrieve every -/// SerializedDepNodeIndex with a fixed-size read then mask. -const DEP_NODE_PAD: usize = DEP_NODE_SIZE - 1; /// Number of bits we need to store the number of used bytes in a SerializedDepNodeIndex. /// Note that wherever we encode byte widths like this we actually store the number of bytes used /// minus 1; for a 4-byte value we technically would have 5 widths to store, but using one byte to @@ -106,13 +118,11 @@ pub struct SerializedDepGraph { /// Some nodes don't have a meaningful value hash (e.g. queries with `no_hash`), /// so they store a dummy value here instead (e.g. [`Fingerprint::ZERO`]). value_fingerprints: IndexVec, - /// For each DepNode, stores the list of edges originating from that - /// DepNode. Encoded as a [start, end) pair indexing into edge_list_data, - /// which holds the actual DepNodeIndices of the target nodes. + /// For each DepNode, stores the position and byte width of its edge list within + /// the retained file bytes ([`Self::backing`]), which serve as the edge data + /// directly: the on-disk varint encoding is also the in-memory representation, + /// so decoding copies no edge bytes at all. edge_list_indices: IndexVec, - /// A flattened list of all edge targets in the graph, stored in the same - /// varint encoding that we use on disk. Edge sources are implicit in edge_list_indices. - edge_list_data: Vec, /// The lazily-built inverse of `nodes`: maps a [`DepNode`] back to its /// [`SerializedDepNodeIndex`] via the node's key fingerprint. See /// [`LazyNodeIndex`]. @@ -120,11 +130,62 @@ pub struct SerializedDepGraph { /// The number of previous compilation sessions. This is used to generate /// unique anon dep nodes per session. session_count: u64, + /// How many consecutive sessions have carried the record region forward without + /// a compacting rewrite. Dead records and superseded duplicates accumulate with + /// each carried generation, so the writer compacts once this grows too large. + generation: u64, + /// The bytes of the file this graph was decoded from, retained both to serve the + /// edge lists in place (see [`Self::edge_list_indices`]) and so that the record + /// region can be copied into the next session's file wholesale (see + /// [`Self::region_bytes`]). `None` only for the empty default graph, which has no + /// nodes and never carries. + backing: Option, + /// The byte range of the record region within [`Self::backing`]: every node record, + /// including dead and superseded ones, and nothing else. + records_range: std::ops::Range, + /// Indices whose record in the region is dead: the node was dropped by an earlier + /// session (never re-verified nor re-executed), so the record must be ignored. + /// Cumulative across carried generations; reset by a compacting rewrite. + dead: Vec, + /// The per-`DepKind` counts of live nodes, from the file footer. Retained so the + /// next session can compute its own footer counts as a delta. + kind_stats: Vec, + /// The number of live nodes, from the file footer (`nodes.len()` counts `Null` + /// index slots too). + live_node_count: u64, + /// The number of edges of live nodes, from the file footer. + live_edge_count: u64, + /// The number of record-region bytes belonging to live records, from the file + /// footer. Compared against the region size to decide when accumulated dead and + /// superseded records warrant a compacting rewrite. + live_record_bytes: u64, /// Used to time the lazy per-`DepKind` reverse-index build. `None` only for /// the empty default graph, which is never looked up. profiler: Option, } +/// The retained bytes of the previous dep-graph file. +/// +/// On most platforms this is the memory mapping the file was decoded from. On Windows +/// the mapping cannot stay alive while the file is later replaced by the save's rename, +/// so the bytes are copied out once and the mapping is released. +enum Backing { + Mapped(Mmap), + #[allow(dead_code)] + Owned(Vec), +} + +impl std::ops::Deref for Backing { + type Target = [u8]; + #[inline] + fn deref(&self) -> &[u8] { + match self { + Backing::Mapped(mmap) => mmap, + Backing::Owned(bytes) => bytes, + } + } +} + // `SelfProfilerRef` is not `Debug`, so we can't derive this. impl std::fmt::Debug for SerializedDepGraph { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -132,7 +193,6 @@ impl std::fmt::Debug for SerializedDepGraph { .field("nodes", &self.nodes) .field("value_fingerprints", &self.value_fingerprints) .field("edge_list_indices", &self.edge_list_indices) - .field("edge_list_data", &self.edge_list_data) .field("reverse_index", &self.reverse_index) .field("session_count", &self.session_count) .finish_non_exhaustive() @@ -211,7 +271,12 @@ impl SerializedDepGraph { source: SerializedDepNodeIndex, ) -> impl Iterator + Clone { let header = self.edge_list_indices[source]; - let mut raw = &self.edge_list_data[header.start()..]; + // The edge bytes are read in place from the retained file. A node with edges + // always comes from a real file, so the backing is present. The fixed-size + // read below may extend a few bytes past the edge list; that is always still + // within the file, since the records are followed by the footer, which is at + // least a dead-list length and the fixed-size tail. + let mut raw = &self.backing.as_ref().unwrap()[header.start()..]; let bytes_per_index = header.bytes_per_index(); @@ -261,6 +326,82 @@ impl SerializedDepGraph { pub fn session_count(&self) -> u64 { self.session_count } + + /// Whether this graph's record region can be carried into the next session's file + /// wholesale. False for the empty default graph (no retained bytes). + #[inline] + fn can_carry(&self) -> bool { + self.backing.is_some() + } + + /// The raw bytes of the record region, exactly as they appeared in this graph's + /// file: every node record, including dead and superseded ones. + /// + /// Every node re-verified or re-executed this session keeps its previous index, so + /// these records remain valid in the next file as-is: records of promoted green + /// nodes are byte-for-byte what a fresh encode would produce, records superseded by + /// a re-executed node are overridden by the appended record at the same index, and + /// records of dropped nodes are tombstoned via the dead list in the footer. + /// The size in bytes of the record region, dead and superseded records included. + #[inline] + fn region_size(&self) -> u64 { + self.records_range.len() as u64 + } + + #[inline] + fn region_bytes(&self) -> &[u8] { + &self.backing.as_ref().unwrap()[self.records_range.clone()] + } + + /// The number of edges of the node at `index`, used for O(changed) footer accounting. + #[inline] + fn edge_count_for_index(&self, index: SerializedDepNodeIndex) -> usize { + self.edge_list_indices[index].num_edges as usize + } + + /// The size in bytes of the encoded record of the node at `index`: the fixed + /// header, the spilled edge count if the header could not hold it, and the edge + /// bytes. Used for O(changed) accounting of the live record bytes. + fn record_size_for_index(&self, index: SerializedDepNodeIndex) -> u64 { + let header = self.edge_list_indices[index]; + let num_edges = header.num_edges; + let spill = if num_edges as usize > SerializedNodeHeader::MAX_INLINE_LEN { + leb128_u32_len(num_edges) + } else { + 0 + }; + size_of::() as u64 + + spill + + num_edges as u64 * header.bytes_per_index() as u64 + } + + /// Whether the node at `index` has a (live or superseded) record in the region. + /// `Null` slots come from batch index allocation and dead records of earlier + /// generations; neither leaves a live record to tombstone. + #[inline] + fn index_is_occupied(&self, index: SerializedDepNodeIndex) -> bool { + self.nodes[index].kind != DepKind::Null + } + + /// Attaches the file bytes decoded by [`Self::decode`], serving the edge lists in + /// place and enabling the carry of the record region into the next session's file. + /// + /// On Windows the mapping must not outlive the load, since the save later replaces + /// the mapped file by renaming over it, so the bytes are copied out instead. + pub fn attach_mmap(&mut self, mmap: Mmap) { + if cfg!(windows) { + self.backing = Some(Backing::Owned(mmap.to_vec())); + } else { + self.backing = Some(Backing::Mapped(mmap)); + } + } +} + +/// The encoded length of `value` as unsigned leb128, matching what +/// [`rustc_serialize::leb128`] writes for a `u32`. +#[inline] +fn leb128_u32_len(value: u32) -> u64 { + (31 - (value | 1).leading_zeros() as u64) / 7 + 1 } /// A packed representation of an edge's start index and byte width. @@ -298,24 +439,48 @@ fn mask(bits: usize) -> usize { impl SerializedDepGraph { #[instrument(level = "debug", skip(d, profiler))] pub fn decode(d: &mut MemDecoder<'_>, profiler: &SelfProfilerRef) -> Arc { - // The last 16 bytes are the node count and edge count. + // The last 32 bytes are the position of the dead list (which is also where the + // record region ends), the node max, and the live node and edge counts. debug!("position: {:?}", d.position()); // `node_max` is the number of indices including empty nodes while `node_count` - // is the number of actually encoded nodes. - let (node_max, node_count, edge_count) = - d.with_position(d.len() - 3 * IntEncodedWithFixedSize::ENCODED_SIZE, |d| { + // is the number of live nodes: records that are neither dead nor superseded by + // a later record at the same index. + let (dead_pos, node_max, node_count, edge_count) = + d.with_position(d.len() - 4 * IntEncodedWithFixedSize::ENCODED_SIZE, |d| { debug!("position: {:?}", d.position()); + let dead_pos = IntEncodedWithFixedSize::decode(d).0 as usize; let node_max = IntEncodedWithFixedSize::decode(d).0 as usize; let node_count = IntEncodedWithFixedSize::decode(d).0 as usize; let edge_count = IntEncodedWithFixedSize::decode(d).0 as usize; - (node_max, node_count, edge_count) + (dead_pos, node_max, node_count, edge_count) }); debug!("position: {:?}", d.position()); debug!(?node_count, ?edge_count); - let graph_bytes = d.len() - (3 * IntEncodedWithFixedSize::ENCODED_SIZE) - d.position(); + let records_start = d.position(); + + // The footer between the records and the fixed-size tail: the dead list, the + // per-kind live counts, the session count and the carried generation count. + // Read it up front, as decoding the records requires the dead set. + let (dead, dead_set, kind_stats, session_count, generation, live_record_bytes) = + d.with_position(dead_pos, |d| { + let dead_len = d.read_u64() as usize; + let mut dead = Vec::with_capacity(dead_len); + let mut dead_set = DenseBitSet::new_empty(node_max); + for _ in 0..dead_len { + let index = SerializedDepNodeIndex::from_u32(u32::from_le_bytes(d.read_array())); + dead_set.insert(index); + dead.push(index); + } + let kind_stats: Vec = + (0..(DepKind::MAX + 1)).map(|_| d.read_u32()).collect(); + let session_count = d.read_u64(); + let generation = d.read_u64(); + let live_record_bytes = d.read_u64(); + (dead, dead_set, kind_stats, session_count, generation, live_record_bytes) + }); let mut nodes = IndexVec::from_elem_n( DepNode { @@ -328,67 +493,58 @@ impl SerializedDepGraph { let mut edge_list_indices = IndexVec::from_elem_n(EdgeHeader { repr: 0, num_edges: 0 }, node_max); - // This estimation assumes that all of the encoded bytes are for the edge lists or for the - // fixed-size node headers. But that's not necessarily true; if any edge list has a length - // that spills out of the size we can bit-pack into SerializedNodeHeader then some of the - // total serialized size is also used by leb128-encoded edge list lengths. Neglecting that - // contribution to graph_bytes means our estimation of the bytes needed for edge_list_data - // slightly overshoots. But it cannot overshoot by much; consider that the worse case is - // for a node with length 64, which means the spilled 1-byte leb128 length is 1 byte of at - // least (34 byte header + 1 byte len + 64 bytes edge data), which is ~1%. A 2-byte leb128 - // length is about the same fractional overhead and it amortizes for yet greater lengths. - let mut edge_list_data = - Vec::with_capacity(graph_bytes - node_count * size_of::()); - - for _ in 0..node_count { + while d.position() < dead_pos { // Decode the header for this edge; the header packs together as many of the fixed-size // fields as possible to limit the number of times we update decoder state. let node_header = SerializedNodeHeader { bytes: d.read_array() }; let index = node_header.index(); - let node = &mut nodes[index]; - // Make sure there's no duplicate indices in the dep graph. - assert!(node_header.node().kind != DepKind::Null && node.kind == DepKind::Null); - *node = node_header.node(); - - value_fingerprints[index] = node_header.value_fingerprint(); - // If the length of this node's edge list is small, the length is stored in the header. // If it is not, we fall back to another decoder call. let num_edges = node_header.len().unwrap_or_else(|| d.read_u32()); // The edges index list uses the same varint strategy as rmeta tables; we select the - // number of byte elements per-array not per-element. This lets us read the whole edge - // list for a node with one decoder call and also use the on-disk format in memory. + // number of byte elements per-array not per-element. The edge bytes are not copied + // anywhere: they are later read in place from the retained file bytes, so decoding + // only records where they start and skips over them. + let edges_start = d.position(); let edges_len_bytes = node_header.bytes_per_index() * (num_edges as usize); - // The in-memory structure for the edges list stores the byte width of the edges on - // this node with the offset into the global edge data array. - let edges_header = node_header.edges_header(&edge_list_data, num_edges); + d.read_raw_bytes(edges_len_bytes); - edge_list_data.extend(d.read_raw_bytes(edges_len_bytes)); + // A dead record: the node was dropped by an earlier session but its bytes were + // carried along in the region. Skip it; its slot stays `Null`. + if dead_set.contains(index) { + continue; + } - edge_list_indices[index] = edges_header; - } + let node = &mut nodes[index]; + let new_node = node_header.node(); + assert!(new_node.kind != DepKind::Null); + if node.kind != DepKind::Null { + // A later record overrides an earlier one at the same index: the node was + // re-executed by the session that appended it, keeping its index. The key + // cannot change, only the value fingerprint and the edges. The exception + // is the anon-zero-deps singleton, whose key is seeded per session. + debug_assert!(*node == new_node || new_node.kind == DepKind::AnonZeroDeps); + } + *node = new_node; + + value_fingerprints[index] = node_header.value_fingerprint(); - // When we access the edge list data, we do a fixed-size read from the edge list data then - // mask off the bytes that aren't for that edge index, so the last read may dangle off the - // end of the array. This padding ensure it doesn't. - edge_list_data.extend(&[0u8; DEP_NODE_PAD]); + edge_list_indices[index] = node_header.edges_header(edges_start, num_edges); + } - // Read the number of nodes of each dep kind, and perform - // counting sort for `LazyNodeIndex`. + // Lay out the per-kind live counts (read from the footer above) as contiguous + // ranges for the counting sort of `LazyNodeIndex`. let mut kinds = Vec::with_capacity(DepKind::MAX as usize + 1); let mut offset = 0u32; - for _ in 0..(DepKind::MAX + 1) { - let len = d.read_u32(); + for &len in &kind_stats { kinds.push(LazyKindIndex { start: offset, len, map: OnceLock::new() }); offset += len; } debug_assert_eq!(offset as usize, node_count); - let session_count = d.read_u64(); - // Counting sort: place each node index into its kind's range. `fill[k]` // points at the next free slot in kind `k`'s range, so a kind's nodes end // up contiguous. Slots start as `None` and are each filled exactly once @@ -413,9 +569,17 @@ impl SerializedDepGraph { nodes, value_fingerprints, edge_list_indices, - edge_list_data, reverse_index, session_count, + generation, + // The retained file bytes are attached by the caller via `attach_mmap`. + backing: None, + records_range: records_start..dead_pos, + dead, + kind_stats, + live_node_count: node_count as u64, + live_edge_count: edge_count as u64, + live_record_bytes, profiler: Some(profiler.clone()), }) } @@ -553,9 +717,9 @@ impl SerializedNodeHeader { } #[inline] - fn edges_header(&self, edge_list_data: &[u8], num_edges: u32) -> EdgeHeader { + fn edges_header(&self, edges_start: usize, num_edges: u32) -> EdgeHeader { EdgeHeader { - repr: (edge_list_data.len() << DEP_NODE_WIDTH_BITS) | (self.bytes_per_index() - 1), + repr: (edges_start << DEP_NODE_WIDTH_BITS) | (self.bytes_per_index() - 1), num_edges, } } @@ -642,19 +806,34 @@ struct LocalEncoderState { next_node_index: u32, remaining_node_index: u32, encoder: MemEncoder, - node_count: usize, - edge_count: usize, - - /// Stores the number of times we've encoded each dep kind. + /// Net change to the live node count from this worker's appends. An appended + /// record that overrides a carried one nets zero (the node was already counted + /// by the previous footer), so only genuinely new nodes contribute. + node_count: i64, + /// Net change to the live edge count from this worker's appends. An override + /// contributes the difference between its new and old edge counts. + edge_count: i64, + /// Net change to the live record bytes from this worker's appends. An override + /// contributes the difference between its new and old record sizes. + record_bytes: i64, + /// Indices below `first_new_index` this worker appended records for. Those appends + /// override the carried record at the same index; anything occupied, not overridden + /// and not marked green by the end of the session is dead. + overridden: Vec, + + /// Stores the net change to the number of live nodes of each dep kind. + /// An override nets zero here since the key (and thus the kind) cannot change. kind_stats: Vec, } struct LocalEncoderResult { node_max: u32, - node_count: usize, - edge_count: usize, + node_count: i64, + edge_count: i64, + record_bytes: i64, + overridden: Vec, - /// Stores the number of times we've encoded each dep kind. + /// Stores the net change to the number of live nodes of each dep kind. kind_stats: Vec, } @@ -664,6 +843,16 @@ struct EncoderState { file: Lock>>, local: WorkerLocal>, stats: Option>>, + /// The first dep node index handed out to genuinely new nodes this session. Nodes + /// that existed in the previous graph keep their old indices, which all lie below + /// this value, so new nodes never collide with them. + first_new_index: u32, + /// Whether this session carries the previous record region forward: the region was + /// copied into the new file wholesale at construction, promoted green nodes write + /// nothing, and re-executed nodes append records that override the carried ones. + /// When false (first session, compaction, or a debugging feature retains the full + /// graph), every live record is written out fresh. + carrying: bool, } impl EncoderState { @@ -671,10 +860,25 @@ impl EncoderState { encoder: FileEncoder<'static>, record_stats: bool, previous: Arc, + carrying: bool, ) -> Self { + // Indices 0 and 1 are always the two singleton nodes; carried indices fill the + // rest of the previous index space. New nodes start above all of them. + let first_new_index = std::cmp::max(2, previous.node_count() as u32); + let mut encoder = encoder; + if carrying { + // Copy the previous record region into the new file wholesale, before any + // appended record. Every node that survives this session keeps its index, so + // the region stays valid: promoted green records are byte-for-byte what a + // fresh encode would produce, re-executed nodes append overriding records at + // their old index, and dropped nodes are tombstoned via the dead list. + encoder.emit_raw_bytes(previous.region_bytes()); + } Self { previous, - next_node_index: AtomicU64::new(0), + next_node_index: AtomicU64::new(first_new_index as u64), + first_new_index, + carrying, stats: record_stats.then(|| Lock::new(FxHashMap::default())), file: Lock::new(Some(encoder)), local: WorkerLocal::new(|_| { @@ -683,6 +887,8 @@ impl EncoderState { remaining_node_index: 0, edge_count: 0, node_count: 0, + record_bytes: 0, + overridden: Vec::new(), encoder: MemEncoder::new(), kind_stats: iter::repeat_n(0, DepKind::MAX as usize + 1).collect(), }) @@ -710,11 +916,18 @@ impl EncoderState { DepNodeIndex::from_u32(local.next_node_index) } - /// Marks the index previously returned by `next_index` as used. + /// Marks the index previously returned by `next_index` as used. Nodes that existed + /// in the previous graph keep their old index and don't go through here. #[inline] - fn bump_index(&self, local: &mut LocalEncoderState) { + fn advance_index(&self, local: &mut LocalEncoderState) { local.remaining_node_index -= 1; local.next_node_index += 1; + } + + /// Counts one written record. Appends that override a carried record are + /// compensated afterwards by [`Self::record_override`]. + #[inline] + fn count_node(&self, local: &mut LocalEncoderState) { local.node_count += 1; } @@ -729,7 +942,7 @@ impl EncoderState { local: &mut LocalEncoderState, ) { local.kind_stats[node.kind.as_usize()] += 1; - local.edge_count += edge_count; + local.edge_count += edge_count as i64; if let Some(retained_graph) = &retained_graph { // Outline the build of the full dep graph as it's typically disabled and cold. @@ -774,8 +987,11 @@ impl EncoderState { retained_graph: &Option>, local: &mut LocalEncoderState, ) { + let before = local.encoder.position(); node.encode(&mut local.encoder, index); + local.record_bytes += (local.encoder.position() - before) as i64; self.flush_mem_encoder(&mut *local); + self.count_node(&mut *local); self.record(&node.node, index, node.edges.len(), &node.edges, retained_graph, &mut *local); } @@ -796,13 +1012,40 @@ impl EncoderState { ) { let node = self.previous.index_to_node(prev_index); let value_fingerprint = self.previous.value_fingerprint_for_index(prev_index); + let before = local.encoder.position(); let edge_count = NodeInfo::encode_promoted(&mut local.encoder, node, index, value_fingerprint, edges); + local.record_bytes += (local.encoder.position() - before) as i64; self.flush_mem_encoder(&mut *local); + self.count_node(&mut *local); self.record(node, index, edge_count, edges, retained_graph, &mut *local); } - fn finish(&self, profiler: &SelfProfilerRef, current: &CurrentDepGraph) -> FileEncodeResult { + /// Adjusts a worker's bookkeeping after it appended a record that overrides the + /// carried record at `prev_index`. The node was already counted by the previous + /// footer, so the append nets zero nodes (and zero for its kind, since the key + /// cannot change) and only the change in edge count remains. + #[inline] + fn record_override( + &self, + prev_index: SerializedDepNodeIndex, + kind: DepKind, + local: &mut LocalEncoderState, + ) { + debug_assert!(self.carrying); + local.node_count -= 1; + local.kind_stats[kind.as_usize()] -= 1; + local.edge_count -= self.previous.edge_count_for_index(prev_index) as i64; + local.record_bytes -= self.previous.record_size_for_index(prev_index) as i64; + local.overridden.push(prev_index); + } + + fn finish( + &self, + profiler: &SelfProfilerRef, + current: &CurrentDepGraph, + colors: &DepNodeColorMap, + ) -> FileEncodeResult { // Prevent more indices from being allocated. self.next_node_index.store(u32::MAX as u64 + 1, Ordering::SeqCst); @@ -820,35 +1063,98 @@ impl EncoderState { node_max: local.next_node_index, node_count: local.node_count, edge_count: local.edge_count, + record_bytes: local.record_bytes, + overridden: mem::take(&mut local.overridden), } }); let mut encoder = self.file.lock().take().unwrap(); - let mut kind_stats: Vec = iter::repeat_n(0, DepKind::MAX as usize + 1).collect(); + // Every count starts from the previous footer when carrying (the region already + // holds those nodes) and from zero when writing a fresh file; the workers report + // net changes in either case. + let (mut kind_stats, mut node_count, mut edge_count, mut record_bytes) = if self.carrying + { + ( + self.previous.kind_stats.clone(), + self.previous.live_node_count as i64, + self.previous.live_edge_count as i64, + self.previous.live_record_bytes as i64, + ) + } else { + (iter::repeat_n(0, DepKind::MAX as usize + 1).collect(), 0, 0, 0) + }; let mut node_max = 0; - let mut node_count = 0; - let mut edge_count = 0; + let mut overridden = DenseBitSet::new_empty(self.first_new_index as usize); for result in results { node_max = max(node_max, result.node_max); node_count += result.node_count; edge_count += result.edge_count; + record_bytes += result.record_bytes; for (i, stat) in result.kind_stats.iter().enumerate() { - kind_stats[i] += stat; + // The per-worker values are net changes: an override decrements the kind + // it previously incremented, so the sum stays balanced per worker and the + // wrapping cancels out across the base value taken from the footer. + kind_stats[i] = kind_stats[i].wrapping_add(*stat); } + for index in result.overridden { + overridden.insert(index); + } + } + + // Nodes that existed in the previous graph keep their previous indices (all below + // `first_new_index`) but don't advance any worker's `next_node_index`. If few or no + // new nodes were encoded, the per-worker maxima can therefore understate the real + // index space, so raise the floor to cover every carried index. + node_max = max(node_max, self.first_new_index); + + // When carrying, tombstone every record in the region that this session dropped: a + // node neither marked green (record still valid) nor overridden by an appended + // record. This matches what a fresh write drops by simply not writing it. Dead + // indices from earlier generations decode as unoccupied slots, so they are carried + // into the new list explicitly. + let mut dead: Vec = Vec::new(); + if self.carrying { + dead.extend_from_slice(&self.previous.dead); + for index in (0..self.previous.node_count() as u32).map(SerializedDepNodeIndex::from_u32) + { + if self.previous.index_is_occupied(index) + && !colors.is_green(index) + && !overridden.contains(index) + { + dead.push(index); + let kind = self.previous.index_to_node(index).kind; + kind_stats[kind.as_usize()] -= 1; + node_count -= 1; + edge_count -= self.previous.edge_count_for_index(index) as i64; + record_bytes -= self.previous.record_size_for_index(index) as i64; + } + } + } + + let generation = if self.carrying { self.previous.generation + 1 } else { 0 }; + + // The record region ends where the dead list begins. + let dead_pos = encoder.position(); + encoder.emit_u64(dead.len() as u64); + for index in &dead { + encoder.write_array(index.as_u32().to_le_bytes()); } - // Encode the number of each dep kind encountered + // Encode the number of live nodes of each dep kind. for count in kind_stats.iter() { count.encode(&mut encoder); } self.previous.session_count.checked_add(1).unwrap().encode(&mut encoder); + generation.encode(&mut encoder); + u64::try_from(record_bytes).unwrap().encode(&mut encoder); debug!(?node_max, ?node_count, ?edge_count); debug!("position: {:?}", encoder.position()); + IntEncodedWithFixedSize(dead_pos.try_into().unwrap()).encode(&mut encoder); IntEncodedWithFixedSize(node_max.try_into().unwrap()).encode(&mut encoder); IntEncodedWithFixedSize(node_count.try_into().unwrap()).encode(&mut encoder); IntEncodedWithFixedSize(edge_count.try_into().unwrap()).encode(&mut encoder); @@ -861,7 +1167,7 @@ impl EncoderState { profiler.artifact_size("dep_graph", "dep-graph.bin", position as u64); } - self.print_incremental_info(current, node_count, edge_count); + self.print_incremental_info(current, node_count as usize, edge_count as usize); result } @@ -932,6 +1238,19 @@ pub(crate) struct GraphEncoder { retained_graph: Option>, } +/// After this many consecutive carried generations, write a fresh file instead even if +/// the dead-byte ratio has not tripped, bounding the growth of orphaned index slots +/// (which the ratio does not measure). Each carried generation leaves behind dead +/// records, superseded records and their orphaned index slots; a compacting rewrite +/// reclaims all of it. +const MAX_CARRIED_GENERATIONS: u64 = 16; + +/// Write a fresh file once the record region exceeds this multiple of its live record +/// bytes. Dead and superseded records make decoding and the wholesale region copy +/// proportionally more expensive, so this bounds that overhead at a fixed factor while +/// letting low-churn graphs carry for many generations. +const MAX_REGION_GROWTH: u64 = 2; + impl GraphEncoder { pub(crate) fn new( sess: &Session, @@ -944,7 +1263,17 @@ impl GraphEncoder { .unstable_opts .query_dep_graph .then(|| Lock::new(RetainedDepGraph::new(prev_node_count))); - let status = EncoderState::new(encoder, sess.opts.unstable_opts.incremental_info, previous); + let record_stats = sess.opts.unstable_opts.incremental_info; + // Carry the previous record region forward unless there is no previous file, a + // debugging feature needs every node to pass through the encoder (the retained + // graph and the stats both do), or enough generations accumulated that dead + // records should be compacted away. + let carrying = previous.can_carry() + && retained_graph.is_none() + && !record_stats + && previous.generation + 1 < MAX_CARRIED_GENERATIONS + && previous.region_size() <= MAX_REGION_GROWTH * previous.live_record_bytes; + let status = EncoderState::new(encoder, record_stats, previous, carrying); GraphEncoder { status, retained_graph, profiler: sess.prof.clone() } } @@ -952,6 +1281,34 @@ impl GraphEncoder { self.retained_graph.as_ref().map(|retained_graph| retained_graph.lock().clone()) } + /// Whether this session carries the previous record region forward. When true, + /// promoting a node from the previous graph needs no edge list, so the marking + /// walk can skip collecting edge indices entirely. + #[inline] + pub(crate) fn is_carrying(&self) -> bool { + self.status.carrying + } + + /// Marks a node promoted from the previous graph green without materializing its + /// edges. Only valid when carrying: the node's record is already in the new file. + /// + /// Returns Some if the node is now green, or None if it had already been + /// concurrently marked red. + #[inline] + pub(crate) fn send_promoted_carried( + &self, + prev_index: SerializedDepNodeIndex, + colors: &DepNodeColorMap, + ) -> Option { + debug_assert!(self.status.carrying); + let index = DepNodeIndex::from_u32(prev_index.as_u32()); + match colors.try_set_color(prev_index, DesiredColor::Green { index }) { + TrySetColorResult::Success => Some(index), + TrySetColorResult::AlreadyRed => None, + TrySetColorResult::AlreadyGreen { index } => Some(index), + } + } + /// Encodes a node that does not exists in the previous graph. pub(crate) fn send_new( &self, @@ -963,11 +1320,35 @@ impl GraphEncoder { let node = NodeInfo { node, value_fingerprint, edges }; let mut local = self.status.local.borrow_mut(); let index = self.status.next_index(&mut *local); - self.status.bump_index(&mut *local); + self.status.advance_index(&mut *local); self.status.encode_node(index, &node, &self.retained_graph, &mut *local); index } + /// Encodes a node at a fixed, caller-chosen index rather than the next allocated one. + /// Used only for the two singleton nodes, which must live at indices 0 and 1; those + /// slots are reserved below `first_new_index`, so this cannot collide with a freshly + /// allocated node. When carrying, the record appended here overrides the previous + /// session's singleton record carried along in the region. + pub(crate) fn send_new_at( + &self, + index: DepNodeIndex, + node: DepNode, + value_fingerprint: Fingerprint, + edges: EdgesVec, + ) -> DepNodeIndex { + let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph"); + let kind = node.kind; + let node = NodeInfo { node, value_fingerprint, edges }; + let mut local = self.status.local.borrow_mut(); + self.status.encode_node(index, &node, &self.retained_graph, &mut *local); + if self.status.carrying { + let prev_index = SerializedDepNodeIndex::from_u32(index.as_u32()); + self.status.record_override(prev_index, kind, &mut *local); + } + index + } + /// Encodes a node that exists in the previous graph, but was re-executed. /// /// This will also ensure the dep node is colored either red or green. @@ -981,11 +1362,17 @@ impl GraphEncoder { is_green: bool, ) -> DepNodeIndex { let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph"); + let kind = node.kind; let node = NodeInfo { node, value_fingerprint, edges }; let mut local = self.status.local.borrow_mut(); - let index = self.status.next_index(&mut *local); + // A re-executed node keeps its previous index whether it came out green or red. + // Keeping green indices stable lets records of promoted nodes, which refer to + // their deps by index, stay valid as-is; keeping red indices stable too means + // the appended record simply overrides the carried one, and this session's + // edges (which may point at the red node) need no separate index space. + let index = DepNodeIndex::from_u32(prev_index.as_u32()); let color = if is_green { DesiredColor::Green { index } } else { DesiredColor::Red }; // Use `try_set_color` to avoid racing when `send_promoted` is called concurrently @@ -996,17 +1383,24 @@ impl GraphEncoder { TrySetColorResult::AlreadyGreen { index } => return index, } - self.status.bump_index(&mut *local); self.status.encode_node(index, &node, &self.retained_graph, &mut *local); + if self.status.carrying { + self.status.record_override(prev_index, kind, &mut *local); + } index } - /// Encodes a node that was promoted from the previous graph. It reads the information directly - /// from the previous dep graph and expects all edges to already have a new dep node index - /// assigned. + /// Marks a node that was promoted from the previous graph green. It expects all edges + /// to already have a new dep node index assigned. /// /// Tries to mark the dep node green, and returns Some if it is now green, /// or None if had already been concurrently marked red. + /// + /// A promoted node keeps its previous index; its edges (all green, all likewise kept + /// at their previous indices) are exactly the previous edges, so its previous record + /// remains valid. When the record region is carried forward, that record is already + /// in the new file and marking the node green is all there is to do; otherwise the + /// record is re-encoded into the fresh file. #[inline] pub(crate) fn send_promoted( &self, @@ -1014,23 +1408,32 @@ impl GraphEncoder { colors: &DepNodeColorMap, edges: &[DepNodeIndex], ) -> Option { - let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph"); - - let mut local = self.status.local.borrow_mut(); - let index = self.status.next_index(&mut *local); + let index = DepNodeIndex::from_u32(prev_index.as_u32()); // Use `try_set_color` to avoid racing when `send_promoted` or `send_and_color` // is called concurrently on the same index. match colors.try_set_color(prev_index, DesiredColor::Green { index }) { TrySetColorResult::Success => { - self.status.bump_index(&mut *local); - self.status.encode_promoted_node( - index, - prev_index, - &self.retained_graph, - &mut *local, - edges, + debug_assert!( + edges.iter().map(|e| e.as_u32()).eq(self + .status + .previous + .edge_targets_from(prev_index) + .map(|e| e.as_u32())), + "promoted green node {prev_index:?} edges diverged from the previous graph", ); + if !self.status.carrying { + let _prof_timer = + self.profiler.generic_activity("incr_comp_encode_dep_graph"); + let mut local = self.status.local.borrow_mut(); + self.status.encode_promoted_node( + index, + prev_index, + &self.retained_graph, + &mut *local, + edges, + ); + } Some(index) } TrySetColorResult::AlreadyRed => None, @@ -1038,9 +1441,13 @@ impl GraphEncoder { } } - pub(crate) fn finish(&self, current: &CurrentDepGraph) -> FileEncodeResult { + pub(crate) fn finish( + &self, + current: &CurrentDepGraph, + colors: &DepNodeColorMap, + ) -> FileEncodeResult { let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph_finish"); - self.status.finish(&self.profiler, current) + self.status.finish(&self.profiler, current, colors) } }