diff --git a/compiler/rustc_incremental/src/diagnostics.rs b/compiler/rustc_incremental/src/diagnostics.rs index 6e291b7ea3abb..0351c4c66ada3 100644 --- a/compiler/rustc_incremental/src/diagnostics.rs +++ b/compiler/rustc_incremental/src/diagnostics.rs @@ -251,6 +251,14 @@ pub(crate) struct MoveDepGraph<'a> { pub err: std::io::Error, } +#[derive(Diagnostic)] +#[diag("failed to move query cache from `{$from}` to `{$to}`: {$err}")] +pub(crate) struct MoveQueryCache<'a> { + pub from: &'a Path, + pub to: &'a Path, + pub err: std::io::Error, +} + #[derive(Diagnostic)] #[diag("failed to create dependency graph at `{$path}`: {$err}")] pub(crate) struct CreateDepGraph<'a> { diff --git a/compiler/rustc_incremental/src/persist/file_format.rs b/compiler/rustc_incremental/src/persist/file_format.rs index 853a5c9ba7ab0..56dd2fd960b70 100644 --- a/compiler/rustc_incremental/src/persist/file_format.rs +++ b/compiler/rustc_incremental/src/persist/file_format.rs @@ -26,7 +26,7 @@ use crate::diagnostics; const FILE_MAGIC: &[u8] = b"RSIC"; /// Change this if the header format changes. -const HEADER_FORMAT_VERSION: u16 = 0; +const HEADER_FORMAT_VERSION: u16 = 2; pub(crate) fn write_file_header(stream: &mut FileEncoder<'_>, sess: &Session) { stream.emit_raw_bytes(FILE_MAGIC); @@ -39,6 +39,12 @@ pub(crate) fn write_file_header(stream: &mut FileEncoder<'_>, sess: &Session) { stream.emit_raw_bytes(rustc_version.as_bytes()); } +/// The number of bytes [`write_file_header`] emits, which is the position at +/// which a file's data region starts. +pub(crate) fn header_size(sess: &Session) -> usize { + FILE_MAGIC.len() + size_of::() + size_of::() + rustc_version(sess).len() +} + pub(crate) fn save_in(sess: &Session, path_buf: PathBuf, name: &str, encode: F) where F: FnOnce(FileEncoder<'static>) -> FileEncodeResult, diff --git a/compiler/rustc_incremental/src/persist/fs.rs b/compiler/rustc_incremental/src/persist/fs.rs index c40aa49c29d11..7e4f9dd621702 100644 --- a/compiler/rustc_incremental/src/persist/fs.rs +++ b/compiler/rustc_incremental/src/persist/fs.rs @@ -130,6 +130,7 @@ const DEP_GRAPH_FILENAME: &str = "dep-graph.bin"; const STAGING_DEP_GRAPH_FILENAME: &str = "dep-graph.part.bin"; const WORK_PRODUCTS_FILENAME: &str = "work-products.bin"; const QUERY_CACHE_FILENAME: &str = "query-cache.bin"; +const STAGING_QUERY_CACHE_FILENAME: &str = "query-cache.part.bin"; // We encode integers using the following base, so they are shorter than decimal // or hexadecimal numbers (we want short file and directory names). Since these @@ -159,6 +160,14 @@ pub(crate) fn query_cache_path(sess: &Session) -> PathBuf { in_incr_comp_dir_sess(sess, QUERY_CACHE_FILENAME) } +/// Returns the path the query cache is written to before it replaces the +/// previous session's file. The new file is written while the previous one +/// is still memory-mapped (its data region is carried forward into the new +/// file), so it cannot be written to the final path directly. +pub(crate) fn staging_query_cache_path(sess: &Session) -> PathBuf { + in_incr_comp_dir_sess(sess, STAGING_QUERY_CACHE_FILENAME) +} + /// Locks a given session directory. fn lock_file_path(session_dir: &Path) -> PathBuf { let crate_dir = session_dir.parent().unwrap(); diff --git a/compiler/rustc_incremental/src/persist/save.rs b/compiler/rustc_incremental/src/persist/save.rs index 544ab66766f39..e1935b0a6eade 100644 --- a/compiler/rustc_incremental/src/persist/save.rs +++ b/compiler/rustc_incremental/src/persist/save.rs @@ -1,7 +1,7 @@ use std::fs; use rustc_data_structures::sync::par_join; -use rustc_middle::dep_graph::{DepGraph, WorkProductMap}; +use rustc_middle::dep_graph::{CachePromotionMode, DepGraph, WorkProductMap}; use rustc_middle::query::on_disk_cache; use rustc_middle::ty::TyCtxt; use rustc_serialize::Encodable as RustcEncodable; @@ -35,6 +35,7 @@ pub(crate) fn save_dep_graph(tcx: TyCtxt<'_>) { } let query_cache_path = query_cache_path(sess); + let staging_query_cache_path = staging_query_cache_path(sess); let dep_graph_path = dep_graph_path(sess); let staging_dep_graph_path = staging_dep_graph_path(sess); @@ -61,27 +62,61 @@ pub(crate) fn save_dep_graph(tcx: TyCtxt<'_>) { // even if there was no previous session. let on_disk_cache = tcx.query_system.on_disk_cache.as_ref().unwrap(); - // For every green dep node that has a disk-cached value from the - // previous session, make sure the value is loaded into the memory - // cache, so that it will be serialized as part of this session. - // - // This reads data from the previous session, so it needs to happen - // before dropping the mmap. - // - // FIXME(Zalathar): This step is intended to be cheap, but still does - // quite a lot of work, especially in builds with few or no changes. - // Can we be smarter about how we identify values that need promotion? - // Can we promote values without decoding them into the memory cache? - tcx.dep_graph.exec_cache_promotions(tcx); - - // Drop the memory map so that we can remove the file and write to it. - on_disk_cache.close_serialized_data_mmap(); - - file_format::save_in(sess, query_cache_path, "query cache", |encoder| { - tcx.sess.time("incr_comp_serialize_result_cache", || { - on_disk_cache::OnDiskCache::serialize(tcx, encoder) - }) - }); + // When possible, the values of green nodes are carried forward + // from the previous cache file byte for byte, so its contents + // must stay readable while the new file is written: the mapping + // is kept alive until serialization is done. Carrying is + // periodically skipped to compact the cache (see + // `can_carry_data`). + let carry = on_disk_cache.can_carry_data(file_format::header_size(sess)); + + let carried_data = if carry { + // Carried values are not decoded, so the verification that + // loading performs would not run for them. Preserve its + // coverage: verify the same subset of not-loaded values + // that promotion (below) would have, without re-encoding. + tcx.dep_graph.exec_cache_promotions(tcx, CachePromotionMode::VerifyOnly); + + on_disk_cache.take_serialized_data_mmap() + } else { + // For every green dep node that has a disk-cached value from + // the previous session, make sure the value is loaded into + // the memory cache, so that it will be serialized as part of + // this session. + // + // This reads data from the previous session, so it needs to + // happen before dropping the mmap. + tcx.dep_graph.exec_cache_promotions(tcx, CachePromotionMode::Promote); + + // The mapping is not needed anymore. + on_disk_cache.close_serialized_data_mmap(); + None + }; + + // The new file is written to a staging path and swapped in + // when complete: the old file stays mapped while its data + // region is copied into the new one, and on Windows a + // mapped file cannot be removed or replaced. + file_format::save_in( + sess, + staging_query_cache_path.clone(), + "query cache", + |encoder| { + tcx.sess.time("incr_comp_serialize_result_cache", || { + on_disk_cache::OnDiskCache::serialize(tcx, encoder, carried_data) + }) + }, + ); + + // `serialize` consumed and dropped the mapping, so the old + // file can be replaced now. + if let Err(err) = fs::rename(&staging_query_cache_path, &query_cache_path) { + sess.dcx().emit_err(diagnostics::MoveQueryCache { + from: &staging_query_cache_path, + to: &query_cache_path, + err, + }); + } }); }, ); diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs index b6fda22775c2a..c149e49c2438a 100644 --- a/compiler/rustc_middle/src/dep_graph/dep_node.rs +++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs @@ -208,19 +208,41 @@ pub struct DepKindVTable<'tcx> { fn(tcx: TyCtxt<'tcx>, dep_node: DepNode, prev_index: SerializedDepNodeIndex) -> bool, >, - /// Load the on-disk cached value of a query into memory. The node is known - /// to be green, with `prev_index` its index in the previous session's dep - /// graph and `dep_node_index` its index in the current session's dep graph. + /// Load the on-disk cached value of a query into memory (or, depending on + /// [`CachePromotionMode`], only verify it). The node is known to be + /// green, with `prev_index` its index in the previous session's dep graph + /// and `dep_node_index` its index in the current session's dep graph. + /// + /// Used when saving the query cache (see `OnDiskCache::serialize`): + /// without carrying, values that are not in memory would be lost to the + /// next session; with carrying, values keep their bytes but the + /// verification that decoding performs still needs to run. pub promote_from_disk_fn: Option< fn( tcx: TyCtxt<'tcx>, dep_node: DepNode, prev_index: SerializedDepNodeIndex, dep_node_index: DepNodeIndex, + mode: CachePromotionMode, ), >, } +/// What [`DepKindVTable::promote_from_disk_fn`] should do with a disk-cached +/// value that is not in the memory cache. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub enum CachePromotionMode { + /// Load the value into the memory cache, so that it is serialized as part + /// of this session. Used when the previous cache file's data is not + /// carried forward into the new file. + Promote, + /// Only decode the value and verify its fingerprint, without keeping it. + /// Used when the previous file's data is carried forward: the value's + /// bytes are copied into the new file directly, but the verification + /// that loading performs should still happen. + VerifyOnly, +} + /// A "work product" corresponds to a `.o` (or other) file that we /// save in between runs. These IDs do not have a `DefId` but rather /// some independent path or string that persists between runs without diff --git a/compiler/rustc_middle/src/dep_graph/graph.rs b/compiler/rustc_middle/src/dep_graph/graph.rs index 66ccde118a6f7..c91973392c7f8 100644 --- a/compiler/rustc_middle/src/dep_graph/graph.rs +++ b/compiler/rustc_middle/src/dep_graph/graph.rs @@ -26,7 +26,7 @@ use {super::debug::EdgeFilter, std::env}; use super::edges::{ReadsRecorder, SMALL_READS_MAX, TaskReads}; use super::retained::RetainedDepGraph; use super::serialized::{GraphEncoder, SerializedDepGraph, SerializedDepNodeIndex}; -use super::{DepKind, DepNode, WorkProductId, read_deps, with_deps}; +use super::{CachePromotionMode, DepKind, DepNode, WorkProductId, read_deps, with_deps}; use crate::ich::StableHashState; use crate::ty::TyCtxt; use crate::verify_ich::incremental_verify_ich; @@ -75,8 +75,12 @@ rustc_index::newtype_index! { rustc_data_structures::static_assert_size!(Option, 4); impl DepNodeIndex { - const SINGLETON_ZERO_DEPS_ANON_NODE: DepNodeIndex = DepNodeIndex::ZERO; + pub(super) const SINGLETON_ZERO_DEPS_ANON_NODE: DepNodeIndex = DepNodeIndex::ZERO; pub const FOREVER_RED_NODE: DepNodeIndex = DepNodeIndex::from_u32(1); + + /// Indices below this belong to the singleton nodes, which sit at the same index in + /// every session. + pub(super) const FIRST_ALLOCATED: u32 = 2; } impl From for QueryInvocationId { @@ -186,27 +190,41 @@ impl DepGraph { let colors = DepNodeColorMap::new(prev_index_space_len); // Instantiate a node with zero dependencies only once for anonymous queries. - let _green_node_index = current.alloc_new_node( + current.alloc_singleton_node( + DepNodeIndex::SINGLETON_ZERO_DEPS_ANON_NODE, DepNode { kind: DepKind::AnonZeroDeps, key_fingerprint: current.anon_id_seed.into() }, &[], Fingerprint::ZERO, ); - assert_eq!(_green_node_index, DepNodeIndex::SINGLETON_ZERO_DEPS_ANON_NODE); // Create a single always-red node, with no dependencies of its own. // Other nodes can use the always-red node as a fake dependency, to // ensure that their dependency list will never be all-green. - let red_node_index = current.alloc_new_node( + current.alloc_singleton_node( + DepNodeIndex::FOREVER_RED_NODE, DepNode { kind: DepKind::Red, key_fingerprint: Fingerprint::ZERO.into() }, &[], Fingerprint::ZERO, ); - assert_eq!(red_node_index, DepNodeIndex::FOREVER_RED_NODE); if prev_index_space_len > 0 { let prev_index = const { SerializedDepNodeIndex::from_u32(DepNodeIndex::FOREVER_RED_NODE.as_u32()) }; let result = colors.try_set_color(prev_index, DesiredColor::Red); assert_matches!(result, TrySetColorResult::Success); + + // The record just written covers the previous anon singleton too: an anonymous + // node with no dependencies never changes, and nothing looks an anon node up by + // key, so the fresh session seed in this one's key does not matter. Color the + // previous one green up front, or promoting it would write a second record to + // that index. + let prev_index = const { + SerializedDepNodeIndex::from_u32( + DepNodeIndex::SINGLETON_ZERO_DEPS_ANON_NODE.as_u32(), + ) + }; + let color = DesiredColor::Green { index: DepNodeIndex::SINGLETON_ZERO_DEPS_ANON_NODE }; + let result = colors.try_set_color(prev_index, color); + assert_matches!(result, TrySetColorResult::Success); } DepGraph { @@ -700,7 +718,6 @@ impl DepGraphData { matches!(self.colors.get(prev_index), DepNodeColor::Green(_)) } - #[inline] pub fn prev_value_fingerprint_of(&self, prev_index: SerializedDepNodeIndex) -> Fingerprint { self.previous.value_fingerprint_for_index(prev_index) } @@ -1050,26 +1067,60 @@ impl DepGraph { } } - /// This method loads all on-disk cacheable query results into memory, so - /// they can be written out to the new cache file again. Most query results - /// will already be in memory but in the case where we marked something as - /// green but then did not need the value, that value will never have been - /// loaded from disk. + /// Invokes `f` for every node of the previous session that was marked + /// green during this session, together with its current-session index. + /// Used when saving the query cache, to reference the still-valid values + /// of green nodes at their positions in the previous cache file. + pub fn for_each_green_prev_index(&self, f: &mut dyn FnMut(SerializedDepNodeIndex)) { + let data = self.data.as_ref().unwrap(); + for prev_index in data.colors.values.indices() { + if let DepNodeColor::Green(dep_node_index) = data.colors.get(prev_index) { + // A green node keeps its index across sessions. + debug_assert_eq!(prev_index.as_u32(), dep_node_index.as_u32()); + f(prev_index); + } + } + } + + /// With [`CachePromotionMode::Promote`], loads all on-disk cacheable + /// query results into memory, so they can be written out to the new cache + /// file again. Most query results will already be in memory but in the + /// case where we marked something as green but then did not need the + /// value, that value will never have been loaded from disk. + /// + /// With [`CachePromotionMode::VerifyOnly`], used when the on-disk values + /// are carried forward instead of re-encoded, only decodes and verifies + /// the values that `Promote` would have verified, and drops them again. /// /// This method will only load queries that will end up in the disk cache. /// Other queries will not be executed. - pub fn exec_cache_promotions<'tcx>(&self, tcx: TyCtxt<'tcx>) { - let _prof_timer = tcx.prof.generic_activity("incr_comp_query_cache_promotion"); + pub fn exec_cache_promotions<'tcx>(&self, tcx: TyCtxt<'tcx>, mode: CachePromotionMode) { + let _prof_timer = tcx.prof.generic_activity(match mode { + CachePromotionMode::Promote => "incr_comp_query_cache_promotion", + CachePromotionMode::VerifyOnly => "incr_comp_query_cache_verification", + }); let data = self.data.as_ref().unwrap(); for prev_index in data.colors.values.indices() { match data.colors.get(prev_index) { DepNodeColor::Green(dep_node_index) => { + // When only verifying, filter by the same predicate the + // promotion path applies after decoding, so that all the + // per-node work below is skipped for values that would + // not be verified anyway. + if mode == CachePromotionMode::VerifyOnly + && !crate::verify_ich::should_verify_loaded_value( + tcx, + data.previous.value_fingerprint_for_index(prev_index), + ) + { + continue; + } let dep_node = data.previous.index_to_node(prev_index); if let Some(promote_fn) = tcx.dep_kind_vtable(dep_node.kind).promote_from_disk_fn { - promote_fn(tcx, *dep_node, prev_index, dep_node_index) + promote_fn(tcx, *dep_node, prev_index, dep_node_index, mode) }; } DepNodeColor::Unknown | DepNodeColor::Red => { @@ -1267,6 +1318,20 @@ impl CurrentDepGraph { dep_node_index } + + #[inline(always)] + fn alloc_singleton_node( + &self, + index: DepNodeIndex, + key: DepNode, + edges: &[DepNodeIndex], + value_fingerprint: Fingerprint, + ) { + self.encoder.send_new_at(index, key, value_fingerprint, edges); + + #[cfg(debug_assertions)] + self.record_edge(index, key, value_fingerprint); + } } #[derive(Debug, Clone, Copy)] diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs index 4f9cb03ff663e..3140436d35ee7 100644 --- a/compiler/rustc_middle/src/dep_graph/mod.rs +++ b/compiler/rustc_middle/src/dep_graph/mod.rs @@ -3,7 +3,8 @@ use std::panic; use tracing::instrument; pub use self::dep_node::{ - DepKind, DepKindVTable, DepNode, WorkProductId, dep_kind_from_label, label_strs, + CachePromotionMode, DepKind, DepKindVTable, DepNode, WorkProductId, dep_kind_from_label, + label_strs, }; pub use self::dep_node_key::DepNodeKey; pub use self::graph::{ diff --git a/compiler/rustc_middle/src/dep_graph/serialized.rs b/compiler/rustc_middle/src/dep_graph/serialized.rs index daebc887055ac..09b31b2155b15 100644 --- a/compiler/rustc_middle/src/dep_graph/serialized.rs +++ b/compiler/rustc_middle/src/dep_graph/serialized.rs @@ -38,12 +38,21 @@ //! Dep-graph indices are bulk allocated to threads inside `LocalEncoderState`. Having threads //! own these indices helps avoid races when they are conditionally used when marking nodes green. //! It also reduces congestion on the shared index count. +//! +//! The encoder also *carries* nodes: a node that exists in the previous graph keeps its +//! previous index, so a node that was marked green can have its previous record +//! re-emitted without rebuilding it, since its edge targets also kept their indices. Indices of +//! deleted nodes are left unoccupied, and the next session hands them to new nodes before +//! extending the index space. That is sound because nothing in the file references an +//! unoccupied index: a carried record's edges point only at nodes that were live when the file +//! was written. The index space therefore tracks the most nodes the graph has ever held at +//! once, rather than growing for as long as the incremental directory lives. use std::cell::RefCell; use std::cmp::max; +use std::iter; use std::sync::atomic::Ordering; use std::sync::{Arc, OnceLock}; -use std::{iter, mem}; use rustc_data_structures::fingerprint::{Fingerprint, PackedFingerprint}; use rustc_data_structures::fx::FxHashMap; @@ -58,7 +67,9 @@ use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; use rustc_session::Session; use tracing::{debug, instrument}; -use super::graph::{CurrentDepGraph, DepNodeColorMap, DesiredColor, TrySetColorResult}; +use super::graph::{ + CurrentDepGraph, DepNodeColor, DepNodeColorMap, DesiredColor, TrySetColorResult, +}; use super::retained::RetainedDepGraph; use super::{DepKind, DepNode, DepNodeIndex}; @@ -119,6 +130,11 @@ pub struct SerializedDepGraph { /// The number of nodes actually encoded, which is below [`Self::index_space_len`] /// whenever a thread left part of its batch of indices unused. live_node_count: usize, + /// Indices that hold no node. Nothing on disk references them: a carried record's + /// edges point only at nodes that were live when the file was written, and the query + /// caches only store data for nodes that were green or executed. So a new node can + /// take one over. + unoccupied_indices: Vec, /// The number of previous compilation sessions. This is used to generate /// unique anon dep nodes per session. session_count: u64, @@ -266,6 +282,35 @@ impl SerializedDepGraph { self.live_node_count } + /// Writes node `index`'s record into `encoder` exactly as it appears in this graph, and + /// returns its edge count. + /// + /// Decoding does not keep the header bytes, so the header is packed anew; the edges are + /// copied over still encoded. + #[inline] + fn re_emit_record(&self, index: SerializedDepNodeIndex, encoder: &mut MemEncoder) -> usize { + let edge_header = self.edge_list_indices[index]; + let num_edges = edge_header.num_edges; + let bytes_per_index = edge_header.bytes_per_index(); + let header = SerializedNodeHeader::with_bytes_per_index( + &self.nodes[index], + DepNodeIndex::from_u32(index.as_u32()), + self.value_fingerprints[index], + bytes_per_index, + num_edges as usize, + ); + encoder.write_array(header.bytes); + if header.len().is_none() { + encoder.emit_u32(num_edges); + } + + let start = edge_header.start(); + encoder + .emit_raw_bytes(&self.edge_list_data[start..][..num_edges as usize * bytes_per_index]); + + num_edges as usize + } + #[inline] pub fn session_count(&self) -> u64 { self.session_count @@ -404,10 +449,15 @@ impl SerializedDepGraph { // (the counts sum to the number of non-`Null` nodes). let mut nodes_by_kind = vec![None; node_count]; let mut fill: Vec = kinds.iter().map(|k| k.start).collect(); + let mut unoccupied_indices = Vec::with_capacity(node_max - node_count); for (idx, node) in nodes.iter_enumerated() { // Unused indices from batch allocation stay `Null`; they carry no - // encoded node and are never looked up by fingerprint, so skip them. + // encoded node and are never looked up by fingerprint. Collect them + // for this session to hand to new nodes. if node.kind == DepKind::Null { + if idx.as_u32() >= DepNodeIndex::FIRST_ALLOCATED { + unoccupied_indices.push(idx); + } continue; } let k = node.kind.as_usize(); @@ -425,6 +475,7 @@ impl SerializedDepGraph { edge_list_data, reverse_index, live_node_count: node_count, + unoccupied_indices, session_count, profiler: Some(profiler.clone()), }) @@ -479,14 +530,27 @@ impl SerializedNodeHeader { value_fingerprint: Fingerprint, edge_max_index: u32, edge_count: usize, + ) -> Self { + let free_bytes = edge_max_index.leading_zeros() as usize / 8; + let bytes_per_index = max(1, DEP_NODE_SIZE - free_bytes); + Self::with_bytes_per_index(node, index, value_fingerprint, bytes_per_index, edge_count) + } + + /// Like [`Self::new`], for a caller that already knows how wide the edge indices are and + /// so does not need the largest of them worked out first. + #[inline] + fn with_bytes_per_index( + node: &DepNode, + index: DepNodeIndex, + value_fingerprint: Fingerprint, + bytes_per_index: usize, + edge_count: usize, ) -> Self { debug_assert_eq!(Self::TOTAL_BITS, Self::LEN_BITS + Self::WIDTH_BITS + Self::KIND_BITS); + debug_assert!((1..=DEP_NODE_SIZE).contains(&bytes_per_index)); let mut head = node.kind.as_u16(); - - let free_bytes = edge_max_index.leading_zeros() as usize / 8; - let bytes_per_index = (DEP_NODE_SIZE - free_bytes).saturating_sub(1); - head |= (bytes_per_index as u16) << Self::KIND_BITS; + head |= ((bytes_per_index - 1) as u16) << Self::KIND_BITS; // Encode number of edges + 1 so that we can reserve 0 to indicate that the len doesn't fit // in this bitfield. @@ -611,12 +675,26 @@ struct Stat { struct LocalEncoderState { next_node_index: u32, remaining_node_index: u32, - encoder: MemEncoder, + /// Taken by [`EncoderState::finish`] when the buffer is written out. A node encoded + /// after that has nowhere to go and panics in [`Self::encoder`]; carried nodes allocate + /// no index, so the poisoned index counter alone cannot catch them. + encoder: Option, node_count: usize, edge_count: usize, /// Stores the number of times we've encoded each dep kind. kind_stats: Vec, + + /// This thread's share of [`EncoderState::free_indices`], served before the + /// contiguous batch. + free_indices: Vec, +} + +impl LocalEncoderState { + #[inline] + fn encoder(&mut self) -> &mut MemEncoder { + self.encoder.as_mut().expect("dep node encoded after the graph was written out") + } } struct LocalEncoderResult { @@ -628,12 +706,21 @@ struct LocalEncoderResult { kind_stats: Vec, } +/// How many indices a thread takes from [`EncoderState::free_indices`] at a time. +const FREE_INDEX_BATCH: usize = 64; + struct EncoderState { next_node_index: AtomicU64, previous: Arc, file: Lock>>, local: WorkerLocal>, stats: Option>>, + /// The first index handed out by [`Self::next_index`] once [`Self::free_indices`] runs + /// out. Carried indices all lie below it. + first_new_index: u32, + /// The previous session's unoccupied indices, handed to new nodes before the index + /// space is extended past [`Self::first_new_index`]. + free_indices: Lock>, } impl EncoderState { @@ -642,18 +729,23 @@ impl EncoderState { record_stats: bool, previous: Arc, ) -> Self { + let first_new_index = max(DepNodeIndex::FIRST_ALLOCATED, previous.index_space_len() as u32); + let free_indices = previous.unoccupied_indices.clone(); Self { previous, - next_node_index: AtomicU64::new(0), + first_new_index, + free_indices: Lock::new(free_indices), + next_node_index: AtomicU64::new(first_new_index as u64), stats: record_stats.then(|| Lock::new(FxHashMap::default())), file: Lock::new(Some(encoder)), local: WorkerLocal::new(|_| { RefCell::new(LocalEncoderState { next_node_index: 0, remaining_node_index: 0, + free_indices: Vec::new(), edge_count: 0, node_count: 0, - encoder: MemEncoder::new(), + encoder: Some(MemEncoder::new()), kind_stats: iter::repeat_n(0, DepKind::MAX as usize + 1).collect(), }) }), @@ -662,7 +754,25 @@ impl EncoderState { #[inline] fn next_index(&self, local: &mut LocalEncoderState) -> DepNodeIndex { + if let Some(&index) = local.free_indices.last() { + return DepNodeIndex::from_u32(index.as_u32()); + } + if local.remaining_node_index == 0 { + // Serve the previous session's unoccupied indices before extending the index + // space. Whatever a thread does not use of either kind of batch is left + // unoccupied in the file and comes back through here next session. + { + let mut free_indices = self.free_indices.lock(); + let len = free_indices.len(); + if len > 0 { + local + .free_indices + .extend(free_indices.drain(len - len.min(FREE_INDEX_BATCH)..)); + return DepNodeIndex::from_u32(local.free_indices.last().unwrap().as_u32()); + } + } + const COUNT: u32 = 256; // We assume that there won't be enough active threads to overflow `u64` from `u32::MAX` here. @@ -683,8 +793,16 @@ impl EncoderState { /// Marks the index previously returned by `next_index` as used. #[inline] fn bump_index(&self, local: &mut LocalEncoderState) { - local.remaining_node_index -= 1; - local.next_node_index += 1; + if local.free_indices.pop().is_none() { + local.remaining_node_index -= 1; + local.next_node_index += 1; + } + } + + /// Counts one encoded node. Separate from [`Self::bump_index`] because not every encoded + /// node is allocated an index: singletons and carried nodes already have one. + #[inline] + fn count_node(&self, local: &mut LocalEncoderState) { local.node_count += 1; } @@ -729,7 +847,7 @@ impl EncoderState { #[inline] fn flush_mem_encoder(&self, local: &mut LocalEncoderState) { - let data = &mut local.encoder.data; + let data = &mut local.encoder().data; if data.len() > 64 * 1024 { self.file.lock().as_mut().unwrap().emit_raw_bytes(&data[..]); data.clear(); @@ -744,42 +862,47 @@ impl EncoderState { retained_graph: &Option>, local: &mut LocalEncoderState, ) { - node.encode(&mut local.encoder, index); + node.encode(local.encoder(), index); self.flush_mem_encoder(&mut *local); + self.count_node(&mut *local); self.record(&node.node, index, node.edges.len(), node.edges, retained_graph, &mut *local); } - /// Encodes a node that was promoted from the previous graph, reading the node and its - /// fingerprint directly from the previous dep graph. It expects all edges to already - /// have a new dep node index assigned. + /// Re-emits a node's record from the previous graph instead of encoding it again. + /// + /// The node and its edge targets keep their previous indices, so the record on disk is + /// still the right one. `edges` holds the same targets as that record, gathered by the + /// marking walk; only the retained graph reads it. #[inline] - fn encode_promoted_node( + fn carry_node( &self, - index: DepNodeIndex, prev_index: SerializedDepNodeIndex, retained_graph: &Option>, local: &mut LocalEncoderState, edges: &[DepNodeIndex], ) { - let node = NodeInfo { - node: *self.previous.index_to_node(prev_index), - value_fingerprint: self.previous.value_fingerprint_for_index(prev_index), - edges, - }; - self.encode_node(index, &node, retained_graph, local); + let edge_count = self.previous.re_emit_record(prev_index, local.encoder()); + debug_assert_eq!(edge_count, edges.len()); + self.flush_mem_encoder(&mut *local); + self.count_node(&mut *local); + let node = self.previous.index_to_node(prev_index); + let index = DepNodeIndex::from_u32(prev_index.as_u32()); + self.record(node, index, edge_count, edges, retained_graph, &mut *local); } fn finish(&self, profiler: &SelfProfilerRef, current: &CurrentDepGraph) -> FileEncodeResult { // Prevent more indices from being allocated. self.next_node_index.store(u32::MAX as u64 + 1, Ordering::SeqCst); + self.free_indices.lock().clear(); let results = broadcast(|_| { let mut local = self.local.borrow_mut(); // Prevent more indices from being allocated on this thread. local.remaining_node_index = 0; + local.free_indices.clear(); - let data = mem::take(&mut local.encoder.data); + let data = local.encoder.take().expect("dep graph written out twice").data; self.file.lock().as_mut().unwrap().emit_raw_bytes(&data); LocalEncoderResult { @@ -794,7 +917,9 @@ impl EncoderState { let mut kind_stats: Vec = iter::repeat_n(0, DepKind::MAX as usize + 1).collect(); - let mut node_max = 0; + // Nothing allocates the singleton indices or the ones carried over, so the per-thread + // maxima below do not account for them. + let mut node_max = self.first_new_index; let mut node_count = 0; let mut edge_count = 0; @@ -828,7 +953,7 @@ impl EncoderState { profiler.artifact_size("dep_graph", "dep-graph.bin", position as u64); } - self.print_incremental_info(current, node_count, edge_count); + self.print_incremental_info(current, node_count, edge_count, node_max as usize); result } @@ -838,6 +963,7 @@ impl EncoderState { current: &CurrentDepGraph, total_node_count: usize, total_edge_count: usize, + node_max: usize, ) { if let Some(record_stats) = &self.stats { let record_stats = record_stats.lock(); @@ -856,6 +982,7 @@ impl EncoderState { eprintln!("[incremental]"); eprintln!("[incremental] Total Node Count: {}", total_node_count); eprintln!("[incremental] Total Edge Count: {}", total_edge_count); + eprintln!("[incremental] Index Space: {}", node_max); if cfg!(debug_assertions) { let total_read_count = current.total_read_count.load(Ordering::Relaxed); @@ -935,6 +1062,22 @@ impl GraphEncoder { index } + /// Encodes a node at one of the indices reserved below [`DepNodeIndex::FIRST_ALLOCATED`], + /// where only the singleton nodes live. + pub(crate) fn send_new_at( + &self, + index: DepNodeIndex, + node: DepNode, + value_fingerprint: Fingerprint, + edges: &[DepNodeIndex], + ) { + debug_assert!(index.as_u32() < DepNodeIndex::FIRST_ALLOCATED); + let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph"); + let node = NodeInfo { node, value_fingerprint, edges }; + let mut local = self.status.local.borrow_mut(); + self.status.encode_node(index, &node, &self.retained_graph, &mut *local); + } + /// Encodes a node that exists in the previous graph, but was re-executed. /// /// This will also ensure the dep node is colored either red or green. @@ -952,7 +1095,7 @@ impl GraphEncoder { let mut local = self.status.local.borrow_mut(); - let index = self.status.next_index(&mut *local); + let index = DepNodeIndex::from_u32(prev_index.as_u32()); let color = if is_green { DesiredColor::Green { index } } else { DesiredColor::Red }; // Use `try_set_color` to avoid racing when `send_promoted` is called concurrently @@ -963,7 +1106,6 @@ impl GraphEncoder { TrySetColorResult::AlreadyGreen { index } => return index, } - self.status.bump_index(&mut *local); self.status.encode_node(index, &node, &self.retained_graph, &mut *local); index } @@ -984,20 +1126,20 @@ impl GraphEncoder { let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph"); let mut local = self.status.local.borrow_mut(); - let index = self.status.next_index(&mut *local); + let index = DepNodeIndex::from_u32(prev_index.as_u32()); // Use `try_set_color` to avoid racing when `send_promoted` or `send_and_color` // is called concurrently on the same index. match colors.try_set_color(prev_index, DesiredColor::Green { index }) { TrySetColorResult::Success => { - self.status.bump_index(&mut *local); - self.status.encode_promoted_node( - index, - prev_index, - &self.retained_graph, - &mut *local, - edges, + debug_assert!( + self.status + .previous + .edge_targets_from(prev_index) + .all(|target| matches!(colors.get(target), DepNodeColor::Green(_))), + "carried node {prev_index:?} names a target that is not green", ); + self.status.carry_node(prev_index, &self.retained_graph, &mut *local, edges); Some(index) } TrySetColorResult::AlreadyRed => None, diff --git a/compiler/rustc_middle/src/query/on_disk_cache.rs b/compiler/rustc_middle/src/query/on_disk_cache.rs index 1dd510da06886..2f1581ee51f5e 100644 --- a/compiler/rustc_middle/src/query/on_disk_cache.rs +++ b/compiler/rustc_middle/src/query/on_disk_cache.rs @@ -29,6 +29,9 @@ use crate::mono::MonoItem; use crate::ty::codec::{RefDecodable, TyDecoder, TyEncoder}; use crate::ty::{self, Ty, TyCtxt}; +#[cfg(test)] +mod tests; + const TAG_FILE_FOOTER: u128 = 0xC0FFEE_C0FFEE_C0FFEE_C0FFEE_C0FFEE; // A normal span encoded with both location information and a `SyntaxContext` @@ -53,6 +56,17 @@ pub struct OnDiskCache { // The complete cache data in serialized form. serialized_data: RwLock>, + // The byte range of the previous session's data region (everything before + // the footer). When possible, this region is carried forward verbatim into + // the next cache file, so that the values of green nodes never need to be + // decoded and re-encoded. A value keeps working because its node keeps its + // index, which is also the tag embedded in its bytes. Values of deleted + // nodes ride along unreferenced until a rewrite, and since indices get + // reused, such a value's tag can match a later node's index; the tag check + // alone therefore cannot catch an entry mispointed into the region. + start_pos: usize, + footer_pos: usize, + file_index_to_stable_id: FxHashMap, // Caches that are populated lazily during decoding. @@ -68,12 +82,13 @@ pub struct OnDiskCache { alloc_decoding_state: AllocDecodingState, - // A map from syntax context ids to the position of their associated - // `SyntaxContextData`. We use a `u32` instead of a `SyntaxContext` - // to represent the fact that we are storing *encoded* ids. When we decode - // a `SyntaxContext`, a new id will be allocated from the global `HygieneData`, - // which will almost certainly be different than the serialized id. - syntax_contexts: FxHashMap, + /// The previous session's raw allocation index, kept for seeding the next + /// session's index when the data region is carried forward. + prev_interpret_alloc_index: Vec, + + // One table per data region of the cache file, ordered oldest first. + // See `SyntaxContextTable`. + syntax_context_tables: Vec, // A map from the `DefPathHash` of an `ExpnId` to the position // of their associated `ExpnData`. Ideally, we would store a `DefId`, // but we need to decode this before we've constructed a `TyCtxt` (which @@ -84,8 +99,6 @@ pub struct OnDiskCache { // we could look up the `ExpnData` from the metadata of foreign crates, // but it seemed easier to have `OnDiskCache` be independent of the `CStore`. expn_data: UnhashMap, - // Additional information used when decoding hygiene data. - hygiene_context: HygieneDecodeContext, // Maps `ExpnHash`es to their raw value from the *previous* // compilation session. This is used as an initial 'guess' when // we try to map an `ExpnHash` to its value in the current @@ -103,8 +116,8 @@ struct Footer { // Most uses only need values up to u32::MAX, but benchmarking indicates that we can use a u64 // without measurable overhead. This permits larger const allocations without ICEing. interpret_alloc_index: Vec, - // See `OnDiskCache.syntax_contexts` - syntax_contexts: FxHashMap, + // See `SyntaxContextTable`, one table per data region, ordered oldest first. + syntax_context_tables: Vec, // See `OnDiskCache.expn_data` expn_data: UnhashMap, foreign_expn_data: UnhashMap, @@ -113,7 +126,7 @@ struct Footer { #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Encodable, Decodable)] struct SourceFileIndex(u32); -#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq, Encodable, Decodable)] +#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq, PartialOrd, Ord, Encodable, Decodable)] pub struct AbsoluteBytePos(u64); impl AbsoluteBytePos { @@ -128,7 +141,62 @@ impl AbsoluteBytePos { } } -#[derive(Encodable, Decodable, Clone, Debug)] +/// Maps the syntax context ids encoded in one region of the cache file to the +/// positions of their associated `SyntaxContextData`. The ids are `u32`s rather +/// than `SyntaxContext`s to represent the fact that they are *encoded* ids: +/// decoding a `SyntaxContext` allocates a new id from the global `HygieneData`, +/// which will almost certainly differ from the serialized one. +/// +/// Encoded ids are only meaningful together with the data region that was +/// encoded in the same session, so each region has its own table, selected by +/// the position an id is decoded from (see [`syntax_context_table_for`]). The +/// stored positions are absolute: a table stays valid only for as long as its +/// region's bytes keep their absolute positions in the file. Regions carried +/// forward from earlier sessions keep their tables; this session's region ends +/// at the footer. +struct SyntaxContextTable { + /// Position one past the end of the data region this table describes. + region_end: AbsoluteBytePos, + /// The position of the `SyntaxContextData` for each id encoded in the region. + positions: FxHashMap, + /// Runtime cache of the ids already decoded this session. Decoded ids are + /// id-space specific, hence one cache per table. Not serialized. + decode_context: HygieneDecodeContext, +} + +impl SyntaxContextTable { + fn new(region_end: AbsoluteBytePos, positions: FxHashMap) -> Self { + SyntaxContextTable { region_end, positions, decode_context: Default::default() } + } +} + +impl Encodable for SyntaxContextTable { + fn encode(&self, e: &mut E) { + self.region_end.encode(e); + self.positions.encode(e); + } +} + +impl Decodable for SyntaxContextTable { + fn decode(d: &mut D) -> Self { + let region_end = Decodable::decode(d); + let positions = Decodable::decode(d); + SyntaxContextTable::new(region_end, positions) + } +} + +/// Selects the table for the data region containing `position`: the first +/// table whose region ends past it. Requires `tables` to be ordered by +/// ascending `region_end`. Returns `None` if the position lies past the last +/// region. +fn syntax_context_table_for( + tables: &[SyntaxContextTable], + position: AbsoluteBytePos, +) -> Option<&SyntaxContextTable> { + tables.get(tables.partition_point(|table| table.region_end <= position)) +} + +#[derive(Encodable, Decodable, Clone, Debug, PartialEq, Eq, Hash)] struct EncodedSourceFileId { stable_source_file_id: StableSourceFileId, stable_crate_id: StableCrateId, @@ -164,32 +232,38 @@ impl OnDiskCache { let footer: Footer = decoder.with_position(footer_pos, |decoder| decode_tagged(decoder, TAG_FILE_FOOTER)); + // `syntax_context_table_for` selects tables by binary search. + debug_assert!(footer.syntax_context_tables.is_sorted_by_key(|table| table.region_end)); Ok(Self { serialized_data: RwLock::new(Some(data)), + start_pos, + footer_pos, file_index_to_stable_id: footer.file_index_to_stable_id, file_index_to_file: Default::default(), query_values_index: footer.query_values_index.into_iter().collect(), side_effects_index: footer.side_effects_index.into_iter().collect(), + prev_interpret_alloc_index: footer.interpret_alloc_index.clone(), alloc_decoding_state: AllocDecodingState::new(footer.interpret_alloc_index), - syntax_contexts: footer.syntax_contexts, + syntax_context_tables: footer.syntax_context_tables, expn_data: footer.expn_data, foreign_expn_data: footer.foreign_expn_data, - hygiene_context: Default::default(), }) } pub fn new_empty() -> Self { Self { serialized_data: RwLock::new(None), + start_pos: 0, + footer_pos: 0, file_index_to_stable_id: Default::default(), file_index_to_file: Default::default(), query_values_index: Default::default(), side_effects_index: Default::default(), + prev_interpret_alloc_index: Vec::new(), alloc_decoding_state: AllocDecodingState::new(Vec::new()), - syntax_contexts: FxHashMap::default(), + syntax_context_tables: Vec::new(), expn_data: UnhashMap::default(), foreign_expn_data: UnhashMap::default(), - hygiene_context: Default::default(), } } @@ -199,38 +273,139 @@ impl OnDiskCache { *self.serialized_data.write() = None; } + /// Take ownership of the serialized backing `Mmap`, so its data region can + /// be carried forward into the next cache file while the old file itself + /// is unlinked and replaced. + pub fn take_serialized_data_mmap(&self) -> Option { + self.serialized_data.write().take() + } + + /// Bound the number of carried generations: every generation keeps its + /// own syntax context table alive and dead data from red nodes + /// accumulates, so occasionally fall back to a full re-encode, which + /// compacts the cache again. + const MAX_CARRIED_GENERATIONS: usize = 8; + + /// Whether [`OnDiskCache::serialize`] can carry the previous cache file's + /// data region forward into a new file whose data region starts at + /// `expected_start_pos`. When it cannot, values that are not in memory + /// must be promoted into the memory cache before the previous file's + /// data is dropped, or they will be lost to the next session. + pub fn can_carry_data(&self, expected_start_pos: usize) -> bool { + self.footer_pos > self.start_pos + // The copied region keeps its offsets only if the new file's + // header has the same length as the old one. + && self.start_pos == expected_start_pos + && self.syntax_context_tables.len() < Self::MAX_CARRIED_GENERATIONS + && self.serialized_data.read().as_ref().is_some_and(|data| data.len() >= self.footer_pos) + } + /// Serialize the current-session data that will be loaded by [`OnDiskCache`] /// in a subsequent incremental compilation session. - pub fn serialize(tcx: TyCtxt<'_>, encoder: FileEncoder<'static>) -> FileEncodeResult { + /// + /// When `carried_data` holds the previous session's cache contents, its + /// data region is copied into the new file verbatim, and the values of + /// green dep nodes are referenced at their old positions instead of being + /// decoded into memory and re-encoded. All position-dependent references + /// inside the region (type and symbol shorthands, allocation data) stay + /// valid because the region keeps its exact offsets. + /// + /// The caller must only pass `carried_data` when [`OnDiskCache::can_carry_data`] + /// held; otherwise it must promote on-disk values into the memory cache + /// before dropping them (see `save_dep_graph`). + pub fn serialize( + tcx: TyCtxt<'_>, + mut encoder: FileEncoder<'static>, + carried_data: Option, + ) -> FileEncodeResult { // Serializing the `DepGraph` should not modify it. tcx.dep_graph.with_ignore(|| { + let on_disk_cache = tcx.query_system.on_disk_cache.as_ref().unwrap(); + + let carried: Option<&[u8]> = carried_data.as_deref().map(|data| { + // `can_carry_data` verified this against the header size the + // caller expected; the copied region keeps its offsets only + // if the new file's header has the same length as the old + // one, so a mismatch here would corrupt the cache. + assert_eq!(on_disk_cache.start_pos, encoder.position()); + &data[on_disk_cache.start_pos..on_disk_cache.footer_pos] + }); + + // Copy the previous data region before anything else is encoded. + if let Some(bytes) = carried { + encoder.emit_raw_bytes(bytes); + } + // Allocate `SourceFileIndex`es. let (file_to_file_index, file_index_to_stable_id) = { let files = tcx.sess.source_map().files(); let mut file_to_file_index = FxHashMap::with_capacity_and_hasher(files.len(), Default::default()); - let mut file_index_to_stable_id = - FxHashMap::with_capacity_and_hasher(files.len(), Default::default()); - for (index, file) in files.iter().enumerate() { - let index = SourceFileIndex(index as u32); - let file_ptr: *const SourceFile = &raw const **file; - file_to_file_index.insert(file_ptr, index); - let source_file_id = EncodedSourceFileId::new(tcx, file); - file_index_to_stable_id.insert(index, source_file_id); - } + if carried.is_some() { + // Spans in the carried region reference the previous + // sessions' file indices: preserve every old assignment, + // including ones whose file is gone (their indices must + // not be reused), and append new files after them. + let mut file_index_to_stable_id = on_disk_cache.file_index_to_stable_id.clone(); + // The maps are only used for lookups and max computation here, + // so the iteration order does not affect the output. + #[allow(rustc::potential_query_instability)] + let mut stable_id_to_index: FxHashMap< + EncodedSourceFileId, + SourceFileIndex, + > = file_index_to_stable_id.iter().map(|(&i, id)| (id.clone(), i)).collect(); + #[allow(rustc::potential_query_instability)] + let next_index_init = + file_index_to_stable_id.keys().map(|i| i.0).max().map_or(0, |m| m + 1); + let mut next_index = next_index_init; + + for file in files.iter() { + let source_file_id = EncodedSourceFileId::new(tcx, file); + let index = *stable_id_to_index + .entry(source_file_id.clone()) + .or_insert_with(|| { + let index = SourceFileIndex(next_index); + next_index += 1; + index + }); + let file_ptr: *const SourceFile = &raw const **file; + file_to_file_index.insert(file_ptr, index); + file_index_to_stable_id.insert(index, source_file_id); + } + (file_to_file_index, file_index_to_stable_id) + } else { + let mut file_index_to_stable_id = + FxHashMap::with_capacity_and_hasher(files.len(), Default::default()); + + for (index, file) in files.iter().enumerate() { + let index = SourceFileIndex(index as u32); + let file_ptr: *const SourceFile = &raw const **file; + file_to_file_index.insert(file_ptr, index); + let source_file_id = EncodedSourceFileId::new(tcx, file); + file_index_to_stable_id.insert(index, source_file_id); + } - (file_to_file_index, file_index_to_stable_id) + (file_to_file_index, file_index_to_stable_id) + } }; let hygiene_encode_context = HygieneEncodeContext::default(); + // Allocation indices embedded in the carried region reference the + // previous sessions' allocation table: keep its entries (their + // data lives in the copied region at unchanged positions) and + // make this session's encoder assign indices after them. + let alloc_index_offset = + if carried.is_some() { on_disk_cache.prev_interpret_alloc_index.len() } else { 0 }; + let mut encoder = CacheEncoder { tcx, encoder, type_shorthands: Default::default(), predicate_shorthands: Default::default(), interpret_allocs: Default::default(), + alloc_index_offset: alloc_index_offset.try_into().unwrap(), caching_source_map_view: CachingSourceMapView::new(tcx.sess.source_map()), file_to_file_index, hygiene_context: &hygiene_encode_context, @@ -239,6 +414,19 @@ impl OnDiskCache { side_effects_index: Default::default(), }; + // Reference the values of green nodes at their positions in the + // carried region. Values that are in memory anyway (for example + // because the query re-executed) are also encoded freshly below; + // fresh entries are appended after these carried entries, and the + // load path lets later entries win. + if carried.is_some() { + tcx.dep_graph.for_each_green_prev_index(&mut |prev_index| { + if let Some(&pos) = on_disk_cache.query_values_index.get(&prev_index) { + encoder.query_values_index.push((prev_index, pos)); + } + }); + } + // Encode query return values. tcx.sess.time("encode_query_values", || { tcx.encode_query_values(&mut encoder); @@ -250,7 +438,13 @@ impl OnDiskCache { } let interpret_alloc_index = { - let mut interpret_alloc_index = Vec::new(); + // Carried values reference the previous sessions' allocation + // entries by index: keep them, and append this session's. + let mut interpret_alloc_index = if carried.is_some() { + on_disk_cache.prev_interpret_alloc_index.clone() + } else { + Vec::new() + }; let mut n = 0; loop { let new_n = encoder.interpret_allocs.len(); @@ -272,8 +466,19 @@ impl OnDiskCache { }; let mut syntax_contexts = FxHashMap::default(); - let mut expn_data = UnhashMap::default(); - let mut foreign_expn_data = UnhashMap::default(); + // Expansions are keyed by their session-independent hash, so + // carried entries (whose data lives in the copied region) share + // one table with this session's; fresh entries overwrite. + let mut expn_data = if carried.is_some() { + on_disk_cache.expn_data.clone() + } else { + UnhashMap::default() + }; + let mut foreign_expn_data = if carried.is_some() { + on_disk_cache.foreign_expn_data.clone() + } else { + UnhashMap::default() + }; // Encode all hygiene data (`SyntaxContextData` and `ExpnData`) from the current // session. @@ -300,6 +505,22 @@ impl OnDiskCache { let footer_pos = encoder.position() as u64; let query_values_index = mem::take(&mut encoder.query_values_index); let side_effects_index = mem::take(&mut encoder.side_effects_index); + + // The carried regions keep their syntax context tables (their + // encoded ids live in their own id spaces); this session's table + // covers the region up to the footer. + let mut syntax_context_tables: Vec = if carried.is_some() { + on_disk_cache + .syntax_context_tables + .iter() + .map(|table| SyntaxContextTable::new(table.region_end, table.positions.clone())) + .collect() + } else { + Vec::new() + }; + syntax_context_tables + .push(SyntaxContextTable::new(AbsoluteBytePos(footer_pos), syntax_contexts)); + encoder.encode_tagged( TAG_FILE_FOOTER, &Footer { @@ -307,7 +528,7 @@ impl OnDiskCache { query_values_index, side_effects_index, interpret_alloc_index, - syntax_contexts, + syntax_context_tables, expn_data, foreign_expn_data, }, @@ -385,10 +606,9 @@ impl OnDiskCache { file_index_to_file: &self.file_index_to_file, file_index_to_stable_id: &self.file_index_to_stable_id, alloc_decoding_session: self.alloc_decoding_state.new_decoding_session(), - syntax_contexts: &self.syntax_contexts, + syntax_context_tables: &self.syntax_context_tables, expn_data: &self.expn_data, foreign_expn_data: &self.foreign_expn_data, - hygiene_context: &self.hygiene_context, }; f(&mut decoder) } @@ -405,10 +625,9 @@ pub struct CacheDecoder<'a, 'tcx> { file_index_to_file: &'a Lock>>, file_index_to_stable_id: &'a FxHashMap, alloc_decoding_session: AllocDecodingSession<'a>, - syntax_contexts: &'a FxHashMap, + syntax_context_tables: &'a [SyntaxContextTable], expn_data: &'a UnhashMap, foreign_expn_data: &'a UnhashMap, - hygiene_context: &'a HygieneDecodeContext, } impl<'a, 'tcx> CacheDecoder<'a, 'tcx> { @@ -548,11 +767,16 @@ impl<'a, 'tcx> Decodable> for Vec { impl<'a, 'tcx> SpanDecoder for CacheDecoder<'a, 'tcx> { fn decode_syntax_context(&mut self) -> SyntaxContext { - let syntax_contexts = self.syntax_contexts; - rustc_span::hygiene::decode_syntax_context(self, self.hygiene_context, |this, id| { + // Select the table belonging to the region this id is being decoded from. + let position = AbsoluteBytePos::new(self.opaque.position()); + let table = + syntax_context_table_for(self.syntax_context_tables, position).unwrap_or_else(|| { + bug!("syntax context decoded from {position:?}, past the last data region") + }); + rustc_span::hygiene::decode_syntax_context(self, &table.decode_context, |this, id| { // This closure is invoked if we haven't already decoded the data for the `SyntaxContext` we are deserializing. // We look up the position of the associated `SyntaxData` and decode it. - let pos = syntax_contexts.get(&id).unwrap(); + let pos = table.positions.get(&id).unwrap(); this.with_position(pos.to_usize(), |decoder| { let data: SyntaxContextKey = decode_tagged(decoder, TAG_SYNTAX_CONTEXT); data @@ -787,6 +1011,9 @@ pub struct CacheEncoder<'a, 'tcx> { type_shorthands: FxHashMap, usize>, predicate_shorthands: FxHashMap, usize>, interpret_allocs: FxIndexSet, + /// Number of allocation entries carried over from previous sessions; + /// indices assigned by this encoder start after them. + alloc_index_offset: u32, caching_source_map_view: CachingSourceMapView<'tcx>, file_to_file_index: FxHashMap<*const SourceFile, SourceFileIndex>, hygiene_context: &'a HygieneEncodeContext, @@ -970,7 +1197,7 @@ impl<'a, 'tcx> TyEncoder<'tcx> for CacheEncoder<'a, 'tcx> { fn encode_alloc_id(&mut self, alloc_id: &interpret::AllocId) { let (index, _) = self.interpret_allocs.insert_full(*alloc_id); - index.encode(self); + (self.alloc_index_offset as usize + index).encode(self); } } diff --git a/compiler/rustc_middle/src/query/on_disk_cache/tests.rs b/compiler/rustc_middle/src/query/on_disk_cache/tests.rs new file mode 100644 index 0000000000000..cce8f736f58da --- /dev/null +++ b/compiler/rustc_middle/src/query/on_disk_cache/tests.rs @@ -0,0 +1,22 @@ +use super::*; + +fn table(region_end: u64) -> SyntaxContextTable { + SyntaxContextTable::new(AbsoluteBytePos(region_end), FxHashMap::default()) +} + +fn selected_region(tables: &[SyntaxContextTable], position: u64) -> Option { + syntax_context_table_for(tables, AbsoluteBytePos(position)).map(|table| table.region_end.0) +} + +#[test] +fn syntax_context_table_selection() { + assert_eq!(selected_region(&[], 0), None); + + // A table covers the positions up to, but excluding, its region end. + let tables = [table(100), table(250)]; + assert_eq!(selected_region(&tables, 0), Some(100)); + assert_eq!(selected_region(&tables, 99), Some(100)); + assert_eq!(selected_region(&tables, 100), Some(250)); + assert_eq!(selected_region(&tables, 249), Some(250)); + assert_eq!(selected_region(&tables, 250), None); +} diff --git a/compiler/rustc_middle/src/verify_ich.rs b/compiler/rustc_middle/src/verify_ich.rs index b0bc65bcc8a5c..831aa9ad4974c 100644 --- a/compiler/rustc_middle/src/verify_ich.rs +++ b/compiler/rustc_middle/src/verify_ich.rs @@ -9,6 +9,20 @@ use crate::dep_graph::{DepGraphData, SerializedDepNodeIndex}; use crate::ich::StableHashState; use crate::ty::TyCtxt; +/// Whether a value loaded from the on-disk cache should have its fingerprint +/// verified with `incremental_verify_ich`. If `-Zincremental-verify-ich` is +/// specified, re-hash results from the cache and make sure that they have the +/// expected fingerprint. +/// +/// If not, we still seek to verify a subset of fingerprints loaded from disk. +/// Re-hashing results is fairly expensive, so we can't currently afford to +/// verify every hash. This subset should still give us some coverage of +/// potential bugs. +pub fn should_verify_loaded_value(tcx: TyCtxt<'_>, prev_fingerprint: Fingerprint) -> bool { + prev_fingerprint.split().1.as_u64().is_multiple_of(32) + || tcx.sess.opts.unstable_opts.incremental_verify_ich +} + #[inline] #[instrument(skip(tcx, dep_graph_data, result, hash_result, format_value), level = "debug")] pub fn incremental_verify_ich<'tcx, V>( diff --git a/compiler/rustc_query_impl/src/dep_kind_vtables.rs b/compiler/rustc_query_impl/src/dep_kind_vtables.rs index 5adcf6c7bb576..9881bfeae0d3d 100644 --- a/compiler/rustc_query_impl/src/dep_kind_vtables.rs +++ b/compiler/rustc_query_impl/src/dep_kind_vtables.rs @@ -118,9 +118,9 @@ where }, ), promote_from_disk_fn: (can_recover && is_cache_on_disk).then_some( - |tcx, dep_node, prev_index, dep_node_index| { + |tcx, dep_node, prev_index, dep_node_index, mode| { let query = Q::query_vtable(tcx); - promote_from_disk_inner(tcx, query, dep_node, prev_index, dep_node_index) + promote_from_disk_inner(tcx, query, dep_node, prev_index, dep_node_index, mode) }, ), } diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs index a9192d0417712..c05f18bf6f436 100644 --- a/compiler/rustc_query_impl/src/execution.rs +++ b/compiler/rustc_query_impl/src/execution.rs @@ -1,7 +1,6 @@ use std::hash::Hash; use std::mem::ManuallyDrop; -use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::hash_table::{Entry, HashTable}; use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_data_structures::sync::{DynSend, DynSync}; @@ -13,7 +12,7 @@ use rustc_middle::query::{ QueryState, QueryVTable, }; use rustc_middle::ty::TyCtxt; -use rustc_middle::verify_ich::incremental_verify_ich; +use rustc_middle::verify_ich::{incremental_verify_ich, should_verify_loaded_value}; use rustc_span::{DUMMY_SP, Span}; use tracing::debug; @@ -485,20 +484,6 @@ fn execute_job_incr<'tcx, C: QueryCache>( (result, dep_node_index) } -/// Whether a value loaded from the on-disk cache should have its fingerprint -/// verified with `incremental_verify_ich`. If `-Zincremental-verify-ich` is -/// specified, re-hash results from the cache and make sure that they have the -/// expected fingerprint. -/// -/// If not, we still seek to verify a subset of fingerprints loaded from disk. -/// Re-hashing results is fairly expensive, so we can't currently afford to -/// verify every hash. This subset should still give us some coverage of -/// potential bugs. -pub(crate) fn should_verify_loaded_value(tcx: TyCtxt<'_>, prev_fingerprint: Fingerprint) -> bool { - prev_fingerprint.split().1.as_u64().is_multiple_of(32) - || tcx.sess.opts.unstable_opts.incremental_verify_ich -} - /// Given that the dep node for this query+key is green, obtain a value for it /// by loading one from disk if possible, or by invoking its query provider if /// necessary. diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index c53293447040b..0b3953efe068e 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -6,18 +6,20 @@ use rustc_data_structures::unord::UnordMap; use rustc_middle::bug; #[expect(unused_imports, reason = "used by doc comments")] use rustc_middle::dep_graph::DepKindVTable; -use rustc_middle::dep_graph::{DepNode, DepNodeIndex, DepNodeKey, SerializedDepNodeIndex}; +use rustc_middle::dep_graph::{ + CachePromotionMode, DepNode, DepNodeIndex, DepNodeKey, SerializedDepNodeIndex, +}; use rustc_middle::query::erase::{Erasable, Erased}; use rustc_middle::query::on_disk_cache::{CacheDecoder, CacheEncoder}; use rustc_middle::query::{QueryCache, QueryJobId, QueryVTable, erase}; use rustc_middle::ty::TyCtxt; use rustc_middle::ty::tls::{self, ImplicitCtxt}; -use rustc_middle::verify_ich::incremental_verify_ich; +use rustc_middle::verify_ich::{incremental_verify_ich, should_verify_loaded_value}; use rustc_serialize::{Decodable, Encodable}; use rustc_span::def_id::LOCAL_CRATE; use crate::error::{QueryOverflow, QueryOverflowNote}; -use crate::execution::{all_inactive, should_verify_loaded_value}; +use crate::execution::all_inactive; use crate::job::find_dep_kind_root; use crate::query_impl::for_each_query_vtable; use crate::{CollectActiveJobsKind, collect_active_query_jobs}; @@ -143,6 +145,7 @@ pub(crate) fn promote_from_disk_inner<'tcx, C: QueryCache>( dep_node: DepNode, prev_index: SerializedDepNodeIndex, dep_node_index: DepNodeIndex, + mode: CachePromotionMode, ) { debug_assert!(tcx.dep_graph.is_green(&dep_node)); @@ -159,7 +162,9 @@ pub(crate) fn promote_from_disk_inner<'tcx, C: QueryCache>( return; } - // If the value is already in memory, then promotion isn't needed. + // If the value is already in memory, then it was verified when it was + // loaded (or computed afresh) and will be re-encoded, so neither + // promotion nor verification is needed. if query.cache.lookup(&key).is_some() { return; } @@ -177,21 +182,38 @@ pub(crate) fn promote_from_disk_inner<'tcx, C: QueryCache>( bug!("failed to load disk-cached value for green node {dep_node:?}"); }; - // Verify the fingerprints of the same subset of loaded values as - // `load_from_disk_or_invoke_provider_green` does. - let prev_fingerprint = dep_graph_data.prev_value_fingerprint_of(prev_index); - if should_verify_loaded_value(tcx, prev_fingerprint) { - incremental_verify_ich( - tcx, - dep_graph_data, - &value, - prev_index, - query.hash_value_fn, - query.format_value, - ); + match mode { + CachePromotionMode::Promote => { + // Verify the fingerprints of the same subset of loaded values as + // `load_from_disk_or_invoke_provider_green` does. + let prev_fingerprint = dep_graph_data.prev_value_fingerprint_of(prev_index); + if should_verify_loaded_value(tcx, prev_fingerprint) { + incremental_verify_ich( + tcx, + dep_graph_data, + &value, + prev_index, + query.hash_value_fn, + query.format_value, + ); + } + + query.cache.complete(key, value, dep_node_index); + } + CachePromotionMode::VerifyOnly => { + // The caller already selected this node for verification, and its + // on-disk bytes are carried forward as they are: verify the + // decoded value and drop it. + incremental_verify_ich( + tcx, + dep_graph_data, + &value, + prev_index, + query.hash_value_fn, + query.format_value, + ); + } } - - query.cache.complete(key, value, dep_node_index); } pub(crate) fn try_load_from_disk<'tcx, V>(