Carry green query values forward from the previous cache file [-10.6% incr-unchanged]

compiler/rustc_incremental/src/diagnostics.rs

@@ -251,6 +251,14 @@ pub(crate) struct MoveDepGraph<'a> {
     pub err: std::io::Error,
 }
 
+#[derive(Diagnostic)]
+#[diag("failed to move query cache from `{$from}` to `{$to}`: {$err}")]
+pub(crate) struct MoveQueryCache<'a> {
+    pub from: &'a Path,
+    pub to: &'a Path,
+    pub err: std::io::Error,
+}
+
 #[derive(Diagnostic)]
 #[diag("failed to create dependency graph at `{$path}`: {$err}")]
 pub(crate) struct CreateDepGraph<'a> {

compiler/rustc_incremental/src/persist/file_format.rs

@@ -26,7 +26,7 @@ use crate::diagnostics;
 const FILE_MAGIC: &[u8] = b"RSIC";
 
 /// Change this if the header format changes.
-const HEADER_FORMAT_VERSION: u16 = 0;
+const HEADER_FORMAT_VERSION: u16 = 2;
 
 pub(crate) fn write_file_header(stream: &mut FileEncoder<'_>, sess: &Session) {
     stream.emit_raw_bytes(FILE_MAGIC);
@@ -39,6 +39,12 @@ pub(crate) fn write_file_header(stream: &mut FileEncoder<'_>, sess: &Session) {
     stream.emit_raw_bytes(rustc_version.as_bytes());
 }
 
+/// The number of bytes [`write_file_header`] emits, which is the position at
+/// which a file's data region starts.
+pub(crate) fn header_size(sess: &Session) -> usize {
+    FILE_MAGIC.len() + size_of::<u16>() + size_of::<u8>() + rustc_version(sess).len()
+}
+
 pub(crate) fn save_in<F>(sess: &Session, path_buf: PathBuf, name: &str, encode: F)
 where
     F: FnOnce(FileEncoder<'static>) -> FileEncodeResult,

compiler/rustc_incremental/src/persist/fs.rs

@@ -130,6 +130,7 @@ const DEP_GRAPH_FILENAME: &str = "dep-graph.bin";
 const STAGING_DEP_GRAPH_FILENAME: &str = "dep-graph.part.bin";
 const WORK_PRODUCTS_FILENAME: &str = "work-products.bin";
 const QUERY_CACHE_FILENAME: &str = "query-cache.bin";
+const STAGING_QUERY_CACHE_FILENAME: &str = "query-cache.part.bin";
 
 // We encode integers using the following base, so they are shorter than decimal
 // or hexadecimal numbers (we want short file and directory names). Since these
@@ -159,6 +160,14 @@ pub(crate) fn query_cache_path(sess: &Session) -> PathBuf {
     in_incr_comp_dir_sess(sess, QUERY_CACHE_FILENAME)
 }
 
+/// Returns the path the query cache is written to before it replaces the
+/// previous session's file. The new file is written while the previous one
+/// is still memory-mapped (its data region is carried forward into the new
+/// file), so it cannot be written to the final path directly.
+pub(crate) fn staging_query_cache_path(sess: &Session) -> PathBuf {
+    in_incr_comp_dir_sess(sess, STAGING_QUERY_CACHE_FILENAME)
+}
+
 /// Locks a given session directory.
 fn lock_file_path(session_dir: &Path) -> PathBuf {
     let crate_dir = session_dir.parent().unwrap();

compiler/rustc_incremental/src/persist/save.rs

@@ -1,7 +1,7 @@
 use std::fs;
 
 use rustc_data_structures::sync::par_join;
-use rustc_middle::dep_graph::{DepGraph, WorkProductMap};
+use rustc_middle::dep_graph::{CachePromotionMode, DepGraph, WorkProductMap};
 use rustc_middle::query::on_disk_cache;
 use rustc_middle::ty::TyCtxt;
 use rustc_serialize::Encodable as RustcEncodable;
@@ -35,6 +35,7 @@ pub(crate) fn save_dep_graph(tcx: TyCtxt<'_>) {
         }
 
         let query_cache_path = query_cache_path(sess);
+        let staging_query_cache_path = staging_query_cache_path(sess);
         let dep_graph_path = dep_graph_path(sess);
         let staging_dep_graph_path = staging_dep_graph_path(sess);
 
@@ -61,27 +62,61 @@ pub(crate) fn save_dep_graph(tcx: TyCtxt<'_>) {
                     // even if there was no previous session.
                     let on_disk_cache = tcx.query_system.on_disk_cache.as_ref().unwrap();
 
-                    // For every green dep node that has a disk-cached value from the
-                    // previous session, make sure the value is loaded into the memory
-                    // cache, so that it will be serialized as part of this session.
-                    //
-                    // This reads data from the previous session, so it needs to happen
-                    // before dropping the mmap.
-                    //
-                    // FIXME(Zalathar): This step is intended to be cheap, but still does
-                    // quite a lot of work, especially in builds with few or no changes.
-                    // Can we be smarter about how we identify values that need promotion?
-                    // Can we promote values without decoding them into the memory cache?
-                    tcx.dep_graph.exec_cache_promotions(tcx);
-
-                    // Drop the memory map so that we can remove the file and write to it.
-                    on_disk_cache.close_serialized_data_mmap();
-
-                    file_format::save_in(sess, query_cache_path, "query cache", |encoder| {
-                        tcx.sess.time("incr_comp_serialize_result_cache", || {
-                            on_disk_cache::OnDiskCache::serialize(tcx, encoder)
-                        })
-                    });
+                    // When possible, the values of green nodes are carried forward
+                    // from the previous cache file byte for byte, so its contents
+                    // must stay readable while the new file is written: the mapping
+                    // is kept alive until serialization is done. Carrying is
+                    // periodically skipped to compact the cache (see
+                    // `can_carry_data`).
+                    let carry = on_disk_cache.can_carry_data(file_format::header_size(sess));
+
+                    let carried_data = if carry {
+                        // Carried values are not decoded, so the verification that
+                        // loading performs would not run for them. Preserve its
+                        // coverage: verify the same subset of not-loaded values
+                        // that promotion (below) would have, without re-encoding.
+                        tcx.dep_graph.exec_cache_promotions(tcx, CachePromotionMode::VerifyOnly);
+
+                        on_disk_cache.take_serialized_data_mmap()
+                    } else {
+                        // For every green dep node that has a disk-cached value from
+                        // the previous session, make sure the value is loaded into
+                        // the memory cache, so that it will be serialized as part of
+                        // this session.
+                        //
+                        // This reads data from the previous session, so it needs to
+                        // happen before dropping the mmap.
+                        tcx.dep_graph.exec_cache_promotions(tcx, CachePromotionMode::Promote);
+
+                        // The mapping is not needed anymore.
+                        on_disk_cache.close_serialized_data_mmap();
+                        None
+                    };
+
+                    // The new file is written to a staging path and swapped in
+                    // when complete: the old file stays mapped while its data
+                    // region is copied into the new one, and on Windows a
+                    // mapped file cannot be removed or replaced.
+                    file_format::save_in(
+                        sess,
+                        staging_query_cache_path.clone(),
+                        "query cache",
+                        |encoder| {
+                            tcx.sess.time("incr_comp_serialize_result_cache", || {
+                                on_disk_cache::OnDiskCache::serialize(tcx, encoder, carried_data)
+                            })
+                        },
+                    );
+
+                    // `serialize` consumed and dropped the mapping, so the old
+                    // file can be replaced now.
+                    if let Err(err) = fs::rename(&staging_query_cache_path, &query_cache_path) {
+                        sess.dcx().emit_err(diagnostics::MoveQueryCache {
+                            from: &staging_query_cache_path,
+                            to: &query_cache_path,
+                            err,
+                        });
+                    }
                 });
             },
         );

compiler/rustc_middle/src/dep_graph/dep_node.rs

@@ -208,19 +208,41 @@ pub struct DepKindVTable<'tcx> {
         fn(tcx: TyCtxt<'tcx>, dep_node: DepNode, prev_index: SerializedDepNodeIndex) -> bool,
     >,
 
-    /// Load the on-disk cached value of a query into memory. The node is known
-    /// to be green, with `prev_index` its index in the previous session's dep
-    /// graph and `dep_node_index` its index in the current session's dep graph.
+    /// Load the on-disk cached value of a query into memory (or, depending on
+    /// [`CachePromotionMode`], only verify it). The node is known to be
+    /// green, with `prev_index` its index in the previous session's dep graph
+    /// and `dep_node_index` its index in the current session's dep graph.
+    ///
+    /// Used when saving the query cache (see `OnDiskCache::serialize`):
+    /// without carrying, values that are not in memory would be lost to the
+    /// next session; with carrying, values keep their bytes but the
+    /// verification that decoding performs still needs to run.
     pub promote_from_disk_fn: Option<
         fn(
             tcx: TyCtxt<'tcx>,
             dep_node: DepNode,
             prev_index: SerializedDepNodeIndex,
             dep_node_index: DepNodeIndex,
+            mode: CachePromotionMode,
         ),
     >,
 }
 
+/// What [`DepKindVTable::promote_from_disk_fn`] should do with a disk-cached
+/// value that is not in the memory cache.
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub enum CachePromotionMode {
+    /// Load the value into the memory cache, so that it is serialized as part
+    /// of this session. Used when the previous cache file's data is not
+    /// carried forward into the new file.
+    Promote,
+    /// Only decode the value and verify its fingerprint, without keeping it.
+    /// Used when the previous file's data is carried forward: the value's
+    /// bytes are copied into the new file directly, but the verification
+    /// that loading performs should still happen.
+    VerifyOnly,
+}
+
 /// A "work product" corresponds to a `.o` (or other) file that we
 /// save in between runs. These IDs do not have a `DefId` but rather
 /// some independent path or string that persists between runs without

compiler/rustc_middle/src/dep_graph/graph.rs

@@ -26,7 +26,7 @@ use {super::debug::EdgeFilter, std::env};
 use super::edges::{ReadsRecorder, SMALL_READS_MAX, TaskReads};
 use super::retained::RetainedDepGraph;
 use super::serialized::{GraphEncoder, SerializedDepGraph, SerializedDepNodeIndex};
-use super::{DepKind, DepNode, WorkProductId, read_deps, with_deps};
+use super::{CachePromotionMode, DepKind, DepNode, WorkProductId, read_deps, with_deps};
 use crate::ich::StableHashState;
 use crate::ty::TyCtxt;
 use crate::verify_ich::incremental_verify_ich;
@@ -718,7 +718,6 @@ impl DepGraphData {
         matches!(self.colors.get(prev_index), DepNodeColor::Green(_))
     }
 
-    #[inline]
     pub fn prev_value_fingerprint_of(&self, prev_index: SerializedDepNodeIndex) -> Fingerprint {
         self.previous.value_fingerprint_for_index(prev_index)
     }
@@ -1068,26 +1067,60 @@ impl DepGraph {
         }
     }
 
-    /// This method loads all on-disk cacheable query results into memory, so
-    /// they can be written out to the new cache file again. Most query results
-    /// will already be in memory but in the case where we marked something as
-    /// green but then did not need the value, that value will never have been
-    /// loaded from disk.
+    /// Invokes `f` for every node of the previous session that was marked
+    /// green during this session, together with its current-session index.
+    /// Used when saving the query cache, to reference the still-valid values
+    /// of green nodes at their positions in the previous cache file.
+    pub fn for_each_green_prev_index(&self, f: &mut dyn FnMut(SerializedDepNodeIndex)) {
+        let data = self.data.as_ref().unwrap();
+        for prev_index in data.colors.values.indices() {
+            if let DepNodeColor::Green(dep_node_index) = data.colors.get(prev_index) {
+                // A green node keeps its index across sessions.
+                debug_assert_eq!(prev_index.as_u32(), dep_node_index.as_u32());
+                f(prev_index);
+            }
+        }
+    }
+
+    /// With [`CachePromotionMode::Promote`], loads all on-disk cacheable
+    /// query results into memory, so they can be written out to the new cache
+    /// file again. Most query results will already be in memory but in the
+    /// case where we marked something as green but then did not need the
+    /// value, that value will never have been loaded from disk.
+    ///
+    /// With [`CachePromotionMode::VerifyOnly`], used when the on-disk values
+    /// are carried forward instead of re-encoded, only decodes and verifies
+    /// the values that `Promote` would have verified, and drops them again.
     ///
     /// This method will only load queries that will end up in the disk cache.
     /// Other queries will not be executed.
-    pub fn exec_cache_promotions<'tcx>(&self, tcx: TyCtxt<'tcx>) {
-        let _prof_timer = tcx.prof.generic_activity("incr_comp_query_cache_promotion");
+    pub fn exec_cache_promotions<'tcx>(&self, tcx: TyCtxt<'tcx>, mode: CachePromotionMode) {
+        let _prof_timer = tcx.prof.generic_activity(match mode {
+            CachePromotionMode::Promote => "incr_comp_query_cache_promotion",
+            CachePromotionMode::VerifyOnly => "incr_comp_query_cache_verification",
+        });
 
         let data = self.data.as_ref().unwrap();
         for prev_index in data.colors.values.indices() {
             match data.colors.get(prev_index) {
                 DepNodeColor::Green(dep_node_index) => {
+                    // When only verifying, filter by the same predicate the
+                    // promotion path applies after decoding, so that all the
+                    // per-node work below is skipped for values that would
+                    // not be verified anyway.
+                    if mode == CachePromotionMode::VerifyOnly
+                        && !crate::verify_ich::should_verify_loaded_value(
+                            tcx,
+                            data.previous.value_fingerprint_for_index(prev_index),
+                        )
+                    {
+                        continue;
+                    }
                     let dep_node = data.previous.index_to_node(prev_index);
                     if let Some(promote_fn) =
                         tcx.dep_kind_vtable(dep_node.kind).promote_from_disk_fn
                     {
-                        promote_fn(tcx, *dep_node, prev_index, dep_node_index)
+                        promote_fn(tcx, *dep_node, prev_index, dep_node_index, mode)
                     };
                 }
                 DepNodeColor::Unknown | DepNodeColor::Red => {

compiler/rustc_middle/src/dep_graph/mod.rs

@@ -3,7 +3,8 @@ use std::panic;
 use tracing::instrument;
 
 pub use self::dep_node::{
-    DepKind, DepKindVTable, DepNode, WorkProductId, dep_kind_from_label, label_strs,
+    CachePromotionMode, DepKind, DepKindVTable, DepNode, WorkProductId, dep_kind_from_label,
+    label_strs,
 };
 pub use self::dep_node_key::DepNodeKey;
 pub use self::graph::{