Avoid excessive memcpys with the new solver

Cargo.lock

@@ -4526,6 +4526,7 @@ dependencies = [
  "rustc_macros",
  "rustc_type_ir",
  "rustc_type_ir_macros",
+ "thin-vec",
  "tracing",
 ]
 

compiler/rustc_middle/src/traits/solve.rs

@@ -96,3 +96,14 @@ impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for ExternalConstraints<'tcx> {
         normalization_nested_goals.visit_with(visitor)
     }
 }
+
+// Some types are used a lot. Make sure they don't unintentionally get bigger.
+#[cfg(target_pointer_width = "64")]
+mod size_asserts {
+    use rustc_data_structures::static_assert_size;
+
+    use super::*;
+    // tidy-alphabetical-start
+    static_assert_size!(GoalStalledOn<'_>, 56);
+    // tidy-alphabetical-end
+}

compiler/rustc_next_trait_solver/Cargo.toml

@@ -11,6 +11,7 @@ rustc_index = { path = "../rustc_index", default-features = false }
 rustc_macros = { path = "../rustc_macros", optional = true }
 rustc_type_ir = { path = "../rustc_type_ir", default-features = false }
 rustc_type_ir_macros = { path = "../rustc_type_ir_macros" }
+thin-vec = "0.2.19"
 tracing = "0.1"
 # tidy-alphabetical-end
 

compiler/rustc_next_trait_solver/src/canonical/canonicalizer.rs

@@ -6,6 +6,7 @@ use rustc_type_ir::{
     Interner, PlaceholderConst, PlaceholderType, Region, TypeFlags, TypeFoldable, TypeFolder,
     TypeSuperFoldable, TypeVisitableExt,
 };
+use thin_vec::ThinVec;
 
 use crate::delegate::SolverDelegate;
 
@@ -65,7 +66,7 @@ pub(super) struct Canonicalizer<'a, D: SolverDelegate<Interner = I>, I: Interner
     canonicalize_mode: CanonicalizeMode,
 
     // Mutable fields.
-    variables: Vec<I::GenericArg>,
+    variables: ThinVec<I::GenericArg>,
     var_kinds: Vec<CanonicalVarKind<I>>,
     variable_lookup_table: HashMap<I::GenericArg, usize>,
     /// Maps each `sub_unification_table_root_var` to the index of the first
@@ -91,10 +92,10 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
             delegate,
             canonicalize_mode: CanonicalizeMode::Response { max_input_universe },
 
-            variables: Vec::new(),
+            variables: Default::default(),
             variable_lookup_table: Default::default(),
             sub_root_lookup_table: Default::default(),
-            var_kinds: Vec::new(),
+            var_kinds: Default::default(),
 
             cache: Default::default(),
         };
@@ -113,10 +114,14 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
     fn canonicalize_param_env(
         delegate: &'a D,
         param_env: I::ParamEnv,
-    ) -> (I::ParamEnv, Vec<I::GenericArg>, Vec<CanonicalVarKind<I>>, HashMap<I::GenericArg, usize>)
-    {
+    ) -> (
+        I::ParamEnv,
+        ThinVec<I::GenericArg>,
+        Vec<CanonicalVarKind<I>>,
+        HashMap<I::GenericArg, usize>,
+    ) {
         if !param_env.has_type_flags(NEEDS_CANONICAL) {
-            return (param_env, Vec::new(), Vec::new(), Default::default());
+            return (param_env, ThinVec::new(), Vec::new(), Default::default());
         }
 
         // Check whether we can use the global cache for this param_env. As we only use
@@ -134,10 +139,10 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
                         delegate,
                         canonicalize_mode: CanonicalizeMode::Input(CanonicalizeInputKind::ParamEnv),
 
-                        variables: Vec::new(),
+                        variables: Default::default(),
                         variable_lookup_table: Default::default(),
                         sub_root_lookup_table: Default::default(),
-                        var_kinds: Vec::new(),
+                        var_kinds: Default::default(),
 
                         cache: Default::default(),
                     };
@@ -159,7 +164,7 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
                     // FIXME(nnethercote): for reasons I don't understand, this `new`+`extend`
                     // combination is faster than `variables.clone()`, because it somehow avoids
                     // some allocations.
-                    let mut variables = Vec::new();
+                    let mut variables = ThinVec::new();
                     variables.extend(cache_variables.iter().copied());
                     (param_env, variables, var_kinds.clone(), variable_lookup_table.clone())
                 },
@@ -169,10 +174,10 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
                 delegate,
                 canonicalize_mode: CanonicalizeMode::Input(CanonicalizeInputKind::ParamEnv),
 
-                variables: Vec::new(),
+                variables: Default::default(),
                 variable_lookup_table: Default::default(),
                 sub_root_lookup_table: Default::default(),
-                var_kinds: Vec::new(),
+                var_kinds: Default::default(),
 
                 cache: Default::default(),
             };
@@ -198,7 +203,7 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
     pub(super) fn canonicalize_input<P: TypeFoldable<I>>(
         delegate: &'a D,
         input: QueryInput<I, P>,
-    ) -> (Vec<I::GenericArg>, ty::Canonical<I, QueryInput<I, P>>) {
+    ) -> (ThinVec<I::GenericArg>, ty::Canonical<I, QueryInput<I, P>>) {
         // First canonicalize the `param_env` while keeping `'static`
         let (param_env, variables, var_kinds, variable_lookup_table) =
             Canonicalizer::canonicalize_param_env(delegate, input.goal.param_env);
@@ -280,7 +285,7 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
         ty::BoundVar::from(idx)
     }
 
-    fn finalize(self) -> (ty::UniverseIndex, Vec<I::GenericArg>, I::CanonicalVarKinds) {
+    fn finalize(self) -> (ty::UniverseIndex, ThinVec<I::GenericArg>, I::CanonicalVarKinds) {
         let mut var_kinds = self.var_kinds;
         // See the rustc-dev-guide section about how we deal with universes
         // during canonicalization in the new solver.

compiler/rustc_next_trait_solver/src/canonical/mod.rs

@@ -21,6 +21,7 @@ use rustc_type_ir::{
     self as ty, Canonical, CanonicalVarKind, CanonicalVarValues, InferCtxtLike, Interner, Region,
     TypeFoldable, TypingMode, TypingModeEqWrapper, eager_resolve_vars,
 };
+use thin_vec::ThinVec;
 use tracing::instrument;
 
 use crate::delegate::SolverDelegate;
@@ -57,7 +58,7 @@ pub(super) fn canonicalize_goal<D, I>(
     goal: Goal<I, I::Predicate>,
     opaque_types: &[(ty::OpaqueTypeKey<I>, I::Ty)],
     typing_mode: TypingMode<I>,
-) -> (Vec<I::GenericArg>, CanonicalInput<I, I::Predicate>)
+) -> (ThinVec<I::GenericArg>, CanonicalInput<I, I::Predicate>)
 where
     D: SolverDelegate<Interner = I>,
     I: Interner,
@@ -541,7 +542,7 @@ pub fn instantiate_canonical_state<D, I, T>(
     delegate: &D,
     span: I::Span,
     param_env: I::ParamEnv,
-    orig_values: &mut Vec<I::GenericArg>,
+    orig_values: &mut ThinVec<I::GenericArg>,
     state: inspect::CanonicalState<I, T>,
 ) -> T
 where

compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs

@@ -19,6 +19,7 @@ use rustc_type_ir::{
     OpaqueTypeKey, PredicateKind, Region, TypeFoldable, TypeSuperVisitable, TypeVisitable,
     TypeVisitableExt, TypeVisitor, TypingMode, eager_resolve_vars,
 };
+use thin_vec::ThinVec;
 use tracing::{Level, debug, instrument, trace, warn};
 
 use super::has_only_region_constraints;
@@ -847,11 +848,11 @@ where
         &self,
         canonical_goal: CanonicalInput<I>,
         certainty: Certainty,
-        mut stalled_vars: Vec<I::GenericArg>,
+        mut stalled_vars: ThinVec<I::GenericArg>,
         previously_succeeded_in_erased: SucceededInErased<I>,
     ) -> GoalStalledOn<I> {
         // Remove the canonicalized universal vars, since we only care about stalled existentials.
-        let mut sub_roots = Vec::new();
+        let mut sub_roots = ThinVec::new();
         stalled_vars.retain(|arg| match arg.kind() {
             // Lifetimes can never stall goals.
             ty::GenericArgKind::Lifetime(_) => false,

compiler/rustc_trait_selection/src/solve/delegate.rs

@@ -21,6 +21,7 @@ use rustc_middle::ty::{
 };
 use rustc_next_trait_solver::solve::{GoalStalledOn, GoalStalledOnOpaques};
 use rustc_span::{DUMMY_SP, Span};
+use thin_vec::{ThinVec, thin_vec};
 
 use crate::traits::{EvaluateConstErr, ObligationCause, sizedness_fast_path, specialization_graph};
 
@@ -54,12 +55,12 @@ impl<'tcx> SolverDelegate<'tcx> {
 /// Create a [`ComputeGoalFastPathOutcome`] signalling the goal is stalled
 /// on a list of [`ty::GenericArg`]
 fn goal_stalled_on_args<'tcx>(
-    stalled_vars: Vec<ty::GenericArg<'tcx>>,
+    stalled_vars: ThinVec<ty::GenericArg<'tcx>>,
 ) -> ComputeGoalFastPathOutcome<'tcx> {
     ComputeGoalFastPathOutcome::TriviallyStalled {
         stalled_on: GoalStalledOn {
             stalled_vars,
-            sub_roots: Vec::new(),
+            sub_roots: ThinVec::new(),
             stalled_certainty: Certainty::AMBIGUOUS,
             opaques: GoalStalledOnOpaques::No,
         },
@@ -70,12 +71,12 @@ fn goal_stalled_on_args<'tcx>(
 /// on a list of [`ty::GenericArg`] *or* the opaque type storage being nonempty.
 ///
 fn goal_stalled_on_args_or_nonempty_opaques<'tcx>(
-    stalled_vars: Vec<ty::GenericArg<'tcx>>,
+    stalled_vars: ThinVec<ty::GenericArg<'tcx>>,
 ) -> ComputeGoalFastPathOutcome<'tcx> {
     ComputeGoalFastPathOutcome::TriviallyStalled {
         stalled_on: GoalStalledOn {
             stalled_vars,
-            sub_roots: Vec::new(),
+            sub_roots: ThinVec::new(),
             stalled_certainty: Certainty::AMBIGUOUS,
             opaques: GoalStalledOnOpaques::Yes {
                 num_opaques_in_storage: 0,
@@ -89,7 +90,7 @@ fn goal_stalled_on_args_or_nonempty_opaques<'tcx>(
 }
 
 struct CollectNonRegionInfer<'tcx> {
-    infers: Vec<ty::GenericArg<'tcx>>,
+    infers: ThinVec<ty::GenericArg<'tcx>>,
     visited: FxHashSet<Ty<'tcx>>,
 }
 
@@ -162,7 +163,7 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate<
                 // FIXME: Properly consider opaques here.
                 && self.known_no_opaque_types_in_storage()
                 {
-                    goal_stalled_on_args_or_nonempty_opaques(vec![self_ty.into()])
+                    goal_stalled_on_args_or_nonempty_opaques(thin_vec![self_ty.into()])
                 } else if trait_pred.polarity() == ty::PredicatePolarity::Positive {
                     match self.0.tcx.as_lang_item(trait_pred.def_id()) {
                         Some(LangItem::Sized) | Some(LangItem::MetaSized) => {
@@ -249,7 +250,7 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate<
                 match (self.shallow_resolve(a).kind(), self.shallow_resolve(b).kind()) {
                     (&ty::Infer(ty::TyVar(a_vid)), &ty::Infer(ty::TyVar(b_vid))) => {
                         self.sub_unify_ty_vids_raw(a_vid, b_vid);
-                        goal_stalled_on_args(vec![a.into(), b.into()])
+                        goal_stalled_on_args(thin_vec![a.into(), b.into()])
                     }
                     _ => Outcome::NoFastPath,
                 }
@@ -261,7 +262,7 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate<
 
                 let arg = self.shallow_resolve_const(ct);
                 if arg.is_ct_infer() {
-                    goal_stalled_on_args(vec![arg.into()])
+                    goal_stalled_on_args(thin_vec![arg.into()])
                 } else {
                     Outcome::NoFastPath
                 }
@@ -275,7 +276,7 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate<
                 if arg.is_trivially_wf(self.tcx) {
                     Outcome::TriviallyHolds
                 } else if arg.is_infer() {
-                    goal_stalled_on_args(vec![arg.into_arg()])
+                    goal_stalled_on_args(thin_vec![arg.into_arg()])
                 } else {
                     Outcome::NoFastPath
                 }

compiler/rustc_trait_selection/src/solve/fulfill.rs

@@ -202,7 +202,7 @@ where
         let mut errors = Vec::new();
         loop {
             let mut any_changed = false;
-            for (mut obligation, stalled_on) in self.obligations.drain_pending(|_, _| true) {
+            for (mut obligation, stalled_on) in mem::take(&mut self.obligations.pending) {
                 let goal = obligation.as_goal();
                 let delegate = <&SolverDelegate<'tcx>>::from(infcx);
 
@@ -398,3 +398,16 @@ impl<'tcx> FromSolverError<'tcx, NextSolverError<'tcx>> for ScrubbedTraitError<'
         }
     }
 }
+
+// Some types are used a lot. Make sure they don't unintentionally get bigger.
+#[cfg(target_pointer_width = "64")]
+mod size_asserts {
+    use rustc_data_structures::static_assert_size;
+
+    use super::*;
+    // tidy-alphabetical-start
+    // Before #160005 this pair was greater than 128 bytes, which triggered the use of (slow)
+    // `memcpy` for moving elements of `PendingObligations`.
+    static_assert_size!((PredicateObligation<'_>, Option<GoalStalledOn<TyCtxt<'_>>>), 104);
+    // tidy-alphabetical-end
+}

compiler/rustc_trait_selection/src/solve/inspect/analyse.rs

@@ -19,6 +19,7 @@ use rustc_middle::{bug, ty};
 use rustc_next_trait_solver::canonical::instantiate_canonical_state;
 use rustc_next_trait_solver::solve::{MaybeCause, MaybeInfo, SolverDelegateEvalExt as _, inspect};
 use rustc_span::Span;
+use thin_vec::ThinVec;
 use tracing::instrument;
 
 use crate::solve::delegate::SolverDelegate;
@@ -30,7 +31,7 @@ pub struct InspectConfig {
 pub struct InspectGoal<'a, 'tcx> {
     infcx: &'a SolverDelegate<'tcx>,
     depth: usize,
-    orig_values: Vec<ty::GenericArg<'tcx>>,
+    orig_values: ThinVec<ty::GenericArg<'tcx>>,
     goal: Goal<'tcx, ty::Predicate<'tcx>>,
     result: Result<Certainty, NoSolution>,
     final_revision: &'tcx inspect::Probe<TyCtxt<'tcx>>,

compiler/rustc_type_ir/Cargo.toml

@@ -41,5 +41,6 @@ nightly = [
     "rustc_type_ir_macros/nightly",
     "smallvec/may_dangle",
     "smallvec/union",
+    "thin-vec/unstable",
 ]
 # tidy-alphabetical-end

compiler/rustc_type_ir/src/canonical.rs

@@ -8,6 +8,7 @@ use rustc_macros::{Decodable_NoContext, Encodable_NoContext, StableHash_NoContex
 use rustc_type_ir_macros::{
     GenericTypeVisitable, Lift_Generic, TypeFoldable_Generic, TypeVisitable_Generic,
 };
+use thin_vec::ThinVec;
 
 use crate::data_structures::HashMap;
 use crate::inherent::*;
@@ -366,7 +367,7 @@ impl<I: Interner> Index<ty::BoundVar> for CanonicalVarValues<I> {
 #[derive_where(Clone, Debug; I: Interner)]
 pub struct CanonicalParamEnvCacheEntry<I: Interner> {
     pub param_env: I::ParamEnv,
-    pub variables: Vec<I::GenericArg>,
+    pub variables: ThinVec<I::GenericArg>,
     pub variable_lookup_table: HashMap<I::GenericArg, usize>,
     pub var_kinds: Vec<CanonicalVarKind<I>>,
 }

compiler/rustc_type_ir/src/solve/inspect.rs

@@ -19,6 +19,7 @@
 
 use derive_where::derive_where;
 use rustc_type_ir_macros::{GenericTypeVisitable, TypeFoldable_Generic, TypeVisitable_Generic};
+use thin_vec::ThinVec;
 
 use crate::solve::{CandidateSource, Certainty, Goal, GoalSource, QueryResult};
 use crate::{Canonical, CanonicalVarValues, Interner};
@@ -48,7 +49,7 @@ pub type CanonicalState<I, T> = Canonical<I, State<I, T>>;
 #[derive_where(PartialEq, Eq, Hash; I: Interner)]
 pub struct GoalEvaluation<I: Interner> {
     pub uncanonicalized_goal: Goal<I, I::Predicate>,
-    pub orig_values: Vec<I::GenericArg>,
+    pub orig_values: ThinVec<I::GenericArg>,
     pub final_revision: I::Probe,
     pub result: QueryResult<I>,
 }

compiler/rustc_type_ir/src/solve/mod.rs

@@ -10,6 +10,7 @@ use rustc_macros::{Decodable_NoContext, Encodable_NoContext, StableHash, StableH
 use rustc_type_ir_macros::{
     GenericTypeVisitable, Lift_Generic, TypeFoldable_Generic, TypeVisitable_Generic,
 };
+use thin_vec::ThinVec;
 use tracing::debug;
 
 use crate::lang_items::SolverTraitLangItem;
@@ -982,8 +983,10 @@ pub enum GoalStalledOnOpaques<I: Interner> {
 /// The conditions that must change for a goal to warrant
 #[derive_where(Clone, Debug; I: Interner)]
 pub struct GoalStalledOn<I: Interner> {
-    pub stalled_vars: Vec<I::GenericArg>,
-    pub sub_roots: Vec<TyVid>,
+    // `ThinVec` is important for performance. See #160005.
+    pub stalled_vars: ThinVec<I::GenericArg>,
+    // `ThinVec` is important for performance. See #160005.
+    pub sub_roots: ThinVec<TyVid>,
     /// The certainty that will be returned on subsequent evaluations if this
     /// goal remains stalled.
     pub stalled_certainty: Certainty,