[wip] terminator perf experiments

Cargo.lock

@@ -5574,9 +5574,9 @@ dependencies = [
 
 [[package]]
 name = "thin-vec"
-version = "0.2.18"
+version = "0.2.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0f7e269b48f0a7dd0146680fa24b50cc67fc0373f086a5b2f99bd084639b482"
+checksum = "79def32ffcd477db1ff26f76dab9e3a91f0bd42a85ca96577089b24623056f9d"
 
 [[package]]
 name = "thiserror"

compiler/rustc_ast/Cargo.toml

@@ -15,6 +15,6 @@ rustc_macros = { path = "../rustc_macros" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_span = { path = "../rustc_span" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-thin-vec = "0.2.18"
+thin-vec = { version = "0.2.19", features = ["unstable"] }
 tracing = "0.1"
 # tidy-alphabetical-end

compiler/rustc_ast_lowering/Cargo.toml

@@ -22,6 +22,6 @@ rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-thin-vec = "0.2.18"
+thin-vec = { version = "0.2.19", features = ["unstable"] }
 tracing = "0.1"
 # tidy-alphabetical-end

compiler/rustc_ast_pretty/Cargo.toml

@@ -13,5 +13,5 @@ rustc_span = { path = "../rustc_span" }
 
 [dev-dependencies]
 # tidy-alphabetical-start
-thin-vec = "0.2.18"
+thin-vec = { version = "0.2.19", features = ["unstable"] }
 # tidy-alphabetical-end

compiler/rustc_attr_parsing/Cargo.toml

@@ -20,5 +20,5 @@ rustc_parse_format = { path = "../rustc_parse_format" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
-thin-vec = "0.2.18"
+thin-vec = { version = "0.2.19", features = ["unstable"] }
 # tidy-alphabetical-end

compiler/rustc_builtin_macros/Cargo.toml

@@ -29,7 +29,7 @@ rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-thin-vec = "0.2.18"
+thin-vec = { version = "0.2.19", features = ["unstable"] }
 tracing = "0.1"
 # tidy-alphabetical-end
 

compiler/rustc_data_structures/Cargo.toml

@@ -30,7 +30,7 @@ smallvec = { version = "1.8.1", features = [
 ] }
 stacker = "0.1.17"
 tempfile = "3.2"
-thin-vec = "0.2.18"
+thin-vec = { version = "0.2.19", features = ["unstable"] }
 tracing = "0.1"
 # tidy-alphabetical-end
 

compiler/rustc_expand/Cargo.toml

@@ -30,6 +30,6 @@ rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 scoped-tls = "1.0"
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-thin-vec = "0.2.18"
+thin-vec = { version = "0.2.19", features = ["unstable"] }
 tracing = "0.1"
 # tidy-alphabetical-end

compiler/rustc_hir/Cargo.toml

@@ -23,6 +23,6 @@ rustc_serialize = { path = "../rustc_serialize" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-thin-vec = "0.2.18"
+thin-vec = { version = "0.2.19", features = ["unstable"] }
 tracing = "0.1"
 # tidy-alphabetical-end

compiler/rustc_infer/Cargo.toml

@@ -17,6 +17,6 @@ rustc_middle = { path = "../rustc_middle" }
 rustc_span = { path = "../rustc_span" }
 rustc_type_ir = { path = "../rustc_type_ir" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-thin-vec = "0.2.18"
+thin-vec = { version = "0.2.19", features = ["unstable"] }
 tracing = "0.1"
 # tidy-alphabetical-end

compiler/rustc_middle/Cargo.toml

@@ -33,7 +33,7 @@ rustc_target = { path = "../rustc_target" }
 rustc_thread_pool = { path = "../rustc_thread_pool" }
 rustc_type_ir = { path = "../rustc_type_ir" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-thin-vec = "0.2.18"
+thin-vec = { version = "0.2.19", features = ["unstable"] }
 tracing = "0.1"
 # tidy-alphabetical-end
 

compiler/rustc_middle/src/mir/mod.rs

@@ -1732,11 +1732,11 @@ mod size_asserts {
 
     use super::*;
     // tidy-alphabetical-start
-    static_assert_size!(BasicBlockData<'_>, 144);
+    static_assert_size!(BasicBlockData<'_>, 136);
     static_assert_size!(LocalDecl<'_>, 40);
     static_assert_size!(SourceScopeData<'_>, 64);
     static_assert_size!(Statement<'_>, 40);
-    static_assert_size!(Terminator<'_>, 104);
+    static_assert_size!(Terminator<'_>, 96);
     static_assert_size!(VarDebugInfo<'_>, 88);
     // tidy-alphabetical-end
 }

compiler/rustc_middle/src/mir/pretty.rs

@@ -1083,7 +1083,7 @@ impl<'tcx> TerminatorKind<'tcx> {
             | CoroutineDrop => vec![],
             Goto { .. } => vec!["".into()],
             SwitchInt { ref targets, .. } => targets
-                .values
+                .all_values()
                 .iter()
                 .map(|&u| Cow::Owned(u.to_string()))
                 .chain(iter::once("otherwise".into()))

compiler/rustc_middle/src/mir/syntax.rs

@@ -13,7 +13,7 @@ use rustc_macros::{StableHash, TyDecodable, TyEncodable, TypeFoldable, TypeVisit
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{Span, Spanned, Symbol};
 use rustc_target::asm::InlineAsmRegOrRegClass;
-use smallvec::SmallVec;
+use thin_vec::ThinVec;
 
 use super::{BasicBlock, Const, Local, UserTypeProjection};
 use crate::mir::coverage::CoverageKind;
@@ -793,7 +793,7 @@ pub enum TerminatorKind<'tcx> {
         /// reused across function calls without duplicating the contents.
         /// The span for each arg is also included
         /// (e.g. `a` and `b` in `x.foo(a, b)`).
-        args: Box<[Spanned<Operand<'tcx>>]>,
+        args: ThinVec<Spanned<Operand<'tcx>>>,
         /// Where the returned value will be written
         destination: Place<'tcx>,
         /// Where to go after this call returns. If none, the call necessarily diverges.
@@ -830,7 +830,7 @@ pub enum TerminatorKind<'tcx> {
         /// These are owned by the callee, which is free to modify them.
         /// This allows the memory occupied by "by-value" arguments to be
         /// reused across function calls without duplicating the contents.
-        args: Box<[Spanned<Operand<'tcx>>]>,
+        args: ThinVec<Spanned<Operand<'tcx>>>,
         // FIXME(explicit_tail_calls): should we have the span for `become`? is this span accurate? do we need it?
         /// This `Span` is the span of the function, without the dot and receiver
         /// (e.g. `foo(a, b)` in `x.foo(a, b)`
@@ -959,7 +959,7 @@ pub enum TerminatorKind<'tcx> {
         /// Valid targets for the inline assembly.
         /// The first element is the fallthrough destination, unless
         /// asm_macro == InlineAsmMacro::NakedAsm or InlineAsmOptions::NORETURN is set.
-        targets: Box<[BasicBlock]>,
+        targets: ThinVec<BasicBlock>,
 
         /// Action to be taken if the inline assembly unwinds. This is present
         /// if and only if InlineAsmOptions::MAY_UNWIND is set.
@@ -974,32 +974,39 @@ pub enum BackwardIncompatibleDropReason {
 }
 
 #[derive(Debug, Clone, TyEncodable, TyDecodable, StableHash, PartialEq)]
-pub struct SwitchTargets {
-    /// Possible values. For each value, the location to branch to is found in
-    /// the corresponding element in the `targets` vector.
-    pub(super) values: SmallVec<[Pu128; 1]>,
-
-    /// Possible branch targets. The last element of this vector is used for
-    /// the "otherwise" branch, so `targets.len() == values.len() + 1` always
-    /// holds.
-    //
-    // Note: This invariant is non-obvious and easy to violate. This would be a
-    // more rigorous representation:
-    //
-    //   normal: SmallVec<[(Pu128, BasicBlock); 1]>,
-    //   otherwise: BasicBlock,
-    //
-    // But it's important to have the targets in a sliceable type, because
-    // target slices show up elsewhere. E.g. `TerminatorKind::InlineAsm` has a
-    // boxed slice, and `TerminatorKind::FalseEdge` has a single target that
-    // can be converted to a slice with `slice::from_ref`.
-    //
-    // Why does this matter? In functions like `TerminatorKind::successors` we
-    // return `impl Iterator` and a non-slice-of-targets representation here
-    // causes problems because multiple different concrete iterator types would
-    // be involved and we would need a boxed trait object, which requires an
-    // allocation, which is expensive if done frequently.
-    pub(super) targets: SmallVec<[BasicBlock; 2]>,
+pub enum SwitchTargets {
+    Otherwise(BasicBlock),
+    Single {
+        values: [Pu128; 1],
+        targets: [BasicBlock; 2],
+    },
+    Many {
+        /// Possible values. For each value, the location to branch to is found in
+        /// the corresponding element in the `targets` vector.
+        values: ThinVec<Pu128>,
+
+        /// Possible branch targets. The last element of this vector is used for
+        /// the "otherwise" branch, so `targets.len() == values.len() + 1` always
+        /// holds.
+        //
+        // Note: This invariant is non-obvious and easy to violate. This would be a
+        // more rigorous representation:
+        //
+        //   normal: SmallVec<[(Pu128, BasicBlock); 1]>,
+        //   otherwise: BasicBlock,
+        //
+        // But it's important to have the targets in a sliceable type, because
+        // target slices show up elsewhere. E.g. `TerminatorKind::InlineAsm` has a
+        // boxed slice, and `TerminatorKind::FalseEdge` has a single target that
+        // can be converted to a slice with `slice::from_ref`.
+        //
+        // Why does this matter? In functions like `TerminatorKind::successors` we
+        // return `impl Iterator` and a non-slice-of-targets representation here
+        // causes problems because multiple different concrete iterator types would
+        // be involved and we would need a boxed trait object, which requires an
+        // allocation, which is expensive if done frequently.
+        targets: ThinVec<BasicBlock>,
+    },
 }
 
 /// Action to be taken when a stack unwind happens.
@@ -1736,6 +1743,7 @@ mod size_asserts {
     static_assert_size!(PlaceElem<'_>, 24);
     static_assert_size!(Rvalue<'_>, 40);
     static_assert_size!(StatementKind<'_>, 16);
-    static_assert_size!(TerminatorKind<'_>, 80);
+    static_assert_size!(SwitchTargets, 24);
+    static_assert_size!(TerminatorKind<'_>, 72);
     // tidy-alphabetical-end
 }

compiler/rustc_middle/src/mir/terminator.rs

@@ -7,7 +7,7 @@ use rustc_data_structures::packed::Pu128;
 use rustc_hir::LangItem;
 use rustc_hir::attrs::AttributeKind;
 use rustc_macros::{StableHash, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable};
-use smallvec::{SmallVec, smallvec};
+use smallvec::SmallVec;
 use thin_vec::ThinVec;
 
 use super::*;
@@ -18,23 +18,31 @@ impl SwitchTargets {
     /// The iterator may be empty, in which case the `SwitchInt` instruction is equivalent to
     /// `goto otherwise;`.
     pub fn new(targets: impl Iterator<Item = (u128, BasicBlock)>, otherwise: BasicBlock) -> Self {
-        let (values, mut targets): (SmallVec<_>, SmallVec<_>) =
-            targets.map(|(v, t)| (Pu128(v), t)).unzip();
+        let mut targets = targets.map(|(v, t)| (Pu128(v), t));
+
+        let Some(first) = targets.next() else { return Self::Otherwise(otherwise) };
+
+        let Some(second) = targets.next() else {
+            return Self::Single { values: [first.0], targets: [first.1, otherwise] };
+        };
+
+        let (values, mut targets): (ThinVec<_>, ThinVec<_>) =
+            [first, second].into_iter().chain(targets).unzip();
         targets.push(otherwise);
-        Self { values, targets }
+        Self::Many { values, targets }
     }
 
     /// Builds a switch targets definition that jumps to `then` if the tested value equals `value`,
     /// and to `else_` if not.
     pub fn static_if(value: u128, then: BasicBlock, else_: BasicBlock) -> Self {
-        Self { values: smallvec![Pu128(value)], targets: smallvec![then, else_] }
+        Self::Single { values: [Pu128(value)], targets: [then, else_] }
     }
 
     /// Inverse of `SwitchTargets::static_if`.
     #[inline]
     pub fn as_static_if(&self) -> Option<(u128, BasicBlock, BasicBlock)> {
-        if let &[value] = &self.values[..]
-            && let &[then, else_] = &self.targets[..]
+        if let &[value] = &self.all_values()[..]
+            && let &[then, else_] = &self.all_targets()[..]
         {
             Some((value.get(), then, else_))
         } else {
@@ -45,7 +53,7 @@ impl SwitchTargets {
     /// Returns the fallback target that is jumped to when none of the values match the operand.
     #[inline]
     pub fn otherwise(&self) -> BasicBlock {
-        *self.targets.last().unwrap()
+        *self.all_targets().last().unwrap()
     }
 
     /// Returns an iterator over the switch targets.
@@ -56,29 +64,45 @@ impl SwitchTargets {
     /// Note that this may yield 0 elements. Only the `otherwise` branch is mandatory.
     #[inline]
     pub fn iter(&self) -> SwitchTargetsIter<'_> {
-        SwitchTargetsIter { inner: iter::zip(&self.values, &self.targets) }
+        SwitchTargetsIter { inner: iter::zip(self.all_values(), self.all_targets()) }
     }
 
     /// Returns a slice with all possible jump targets (including the fallback target).
     #[inline]
     pub fn all_targets(&self) -> &[BasicBlock] {
-        &self.targets
+        match self {
+            SwitchTargets::Otherwise(target) => slice::from_ref(target),
+            SwitchTargets::Single { targets, values: _ } => &targets[..],
+            SwitchTargets::Many { targets, values: _ } => &targets[..],
+        }
     }
 
     #[inline]
     pub fn all_targets_mut(&mut self) -> &mut [BasicBlock] {
-        &mut self.targets
+        match self {
+            SwitchTargets::Otherwise(target) => slice::from_mut(target),
+            SwitchTargets::Single { targets, values: _ } => &mut targets[..],
+            SwitchTargets::Many { targets, values: _ } => &mut targets[..],
+        }
     }
 
     /// Returns a slice with all considered values (not including the fallback).
     #[inline]
     pub fn all_values(&self) -> &[Pu128] {
-        &self.values
+        match self {
+            SwitchTargets::Otherwise(_) => &[],
+            SwitchTargets::Single { targets: _, values } => &values[..],
+            SwitchTargets::Many { targets: _, values } => &values[..],
+        }
     }
 
     #[inline]
     pub fn all_values_mut(&mut self) -> &mut [Pu128] {
-        &mut self.values
+        match self {
+            SwitchTargets::Otherwise(_) => &mut [],
+            SwitchTargets::Single { targets: _, values } => &mut values[..],
+            SwitchTargets::Many { targets: _, values } => &mut values[..],
+        }
     }
 
     /// Finds the `BasicBlock` to which this `SwitchInt` will branch given the
@@ -93,17 +117,30 @@ impl SwitchTargets {
     #[inline]
     pub fn add_target(&mut self, value: u128, bb: BasicBlock) {
         let value = Pu128(value);
-        if self.values.contains(&value) {
+        if self.all_values().contains(&value) {
             bug!("target value {:?} already present", value);
         }
-        self.values.push(value);
-        self.targets.insert(self.targets.len() - 1, bb);
+        match self {
+            SwitchTargets::Otherwise(prev_target) => {
+                *self = SwitchTargets::Single { values: [value], targets: [*prev_target, bb] }
+            }
+            SwitchTargets::Single { values, targets } => {
+                *self = SwitchTargets::Many {
+                    values: ThinVec::from_iter(values.iter().copied().chain(iter::once(value))),
+                    targets: ThinVec::from_iter(targets.iter().copied().chain(iter::once(bb))),
+                }
+            }
+            SwitchTargets::Many { values, targets } => {
+                values.push(value);
+                targets.insert(targets.len() - 1, bb);
+            }
+        }
     }
 
     /// Returns true if all targets (including the fallback target) are distinct.
     #[inline]
     pub fn is_distinct(&self) -> bool {
-        self.targets.iter().collect::<FxHashSet<_>>().len() == self.targets.len()
+        self.all_targets().iter().collect::<FxHashSet<_>>().len() == self.all_targets().len()
     }
 }
 
@@ -548,7 +585,7 @@ mod helper {
                     mk_successors(targets, Some(u), None)
                 }
                 InlineAsm { ref targets, unwind: _, .. } => mk_successors(targets, None, None),
-                SwitchInt { ref targets, .. } => mk_successors(&targets.targets, None, None),
+                SwitchInt { ref targets, .. } => mk_successors(targets.all_targets(), None, None),
                 // FalseEdge
                 FalseEdge { ref real_target, imaginary_target } => {
                     mk_successors(slice::from_ref(real_target), Some(imaginary_target), None)
@@ -607,7 +644,7 @@ mod helper {
                     }
                 }
                 SwitchInt { targets, .. } => {
-                    for target in &mut targets.targets {
+                    for target in targets.all_targets_mut() {
                         f(target);
                     }
                 }