[WIP] MIR move elimination pass

compiler/rustc_index/src/bit_set.rs

@@ -324,6 +324,15 @@ impl<T: Idx> DenseBitSet<T> {
         // out-of-domain bits, so we need to clear them.
         self.clear_excess_bits();
     }
+
+    /// Sets `self = self & (a | b)` without allocating a temporary for `a | b`.
+    ///
+    /// Returns `true` if `self` changed.
+    pub fn intersect_with_union(&mut self, a: &DenseBitSet<T>, b: &DenseBitSet<T>) -> bool {
+        assert_eq!(self.domain_size, a.domain_size);
+        assert_eq!(self.domain_size, b.domain_size);
+        bitwise3(&mut self.words, &a.words, &b.words, |s, a, b| s & (a | b))
+    }
 }
 
 // dense REL dense
@@ -1084,6 +1093,23 @@ where
     changed != 0
 }
 
+#[inline]
+fn bitwise3<Op>(out_vec: &mut [Word], in_vec1: &[Word], in_vec2: &[Word], op: Op) -> bool
+where
+    Op: Fn(Word, Word, Word) -> Word,
+{
+    assert_eq!(out_vec.len(), in_vec1.len());
+    assert_eq!(out_vec.len(), in_vec2.len());
+    let mut changed = 0;
+    for ((out_elem, in_elem1), in_elem2) in iter::zip(iter::zip(out_vec, in_vec1), in_vec2) {
+        let old_val = *out_elem;
+        let new_val = op(old_val, *in_elem1, *in_elem2);
+        *out_elem = new_val;
+        changed |= old_val ^ new_val;
+    }
+    changed != 0
+}
+
 /// Returns true if a call to [`update_words`] would modify `lhs`, i.e.
 /// `lhs[i] != op(lhs[i], rhs[i])` for some `i`.
 #[inline]

compiler/rustc_index/src/interval.rs

@@ -1,6 +1,7 @@
 use std::iter::Step;
 use std::marker::PhantomData;
-use std::ops::{Bound, Range, RangeBounds};
+use std::ops::{Bound, RangeBounds};
+use std::range::RangeInclusive;
 
 use smallvec::SmallVec;
 
@@ -59,11 +60,14 @@ impl<I: Idx> IntervalSet<I> {
     }
 
     /// Iterates through intervals stored in the set, in order.
-    pub fn iter_intervals(&self) -> impl Iterator<Item = std::ops::Range<I>>
+    pub fn iter_intervals(&self) -> impl Iterator<Item = RangeInclusive<I>>
     where
         I: Step,
     {
-        self.map.iter().map(|&(start, end)| I::new(start as usize)..I::new(end as usize + 1))
+        self.map.iter().map(|&(start, end)| RangeInclusive {
+            start: I::new(start as usize),
+            last: I::new(end as usize),
+        })
     }
 
     /// Returns true if we increased the number of elements present.
@@ -204,17 +208,38 @@ impl<I: Idx> IntervalSet<I> {
         needle <= *prev_end
     }
 
+    /// Returns whether any point in `range` is contained in the set.
+    pub fn intersects_range(&self, range: impl RangeBounds<I> + Clone) -> bool {
+        let start = inclusive_start(range.clone());
+        let Some(end) = inclusive_end(self.domain, range) else {
+            // empty range
+            return false;
+        };
+        if start > end {
+            return false;
+        }
+
+        // Find the last interval whose start is <= end.
+        let Some(last) = self.map.partition_point(|r| r.0 <= end).checked_sub(1) else {
+            // All ranges in the map start after the new range's end
+            return false;
+        };
+        let (_, prev_end) = &self.map[last];
+        start <= *prev_end
+    }
+
     pub fn superset(&self, other: &IntervalSet<I>) -> bool
     where
         I: Step,
     {
         let mut sup_iter = self.iter_intervals();
         let mut current = None;
-        let contains = |sup: Range<I>, sub: Range<I>, current: &mut Option<Range<I>>| {
-            if sup.end < sub.start {
-                // if `sup.end == sub.start`, the next sup doesn't contain `sub.start`
+        let contains = |sup: RangeInclusive<I>,
+                        sub: RangeInclusive<I>,
+                        current: &mut Option<RangeInclusive<I>>| {
+            if sup.last < sub.start {
                 None // continue to the next sup
-            } else if sup.end >= sub.end && sup.start <= sub.start {
+            } else if sup.last >= sub.last && sup.start <= sub.start {
                 *current = Some(sup); // save the current sup
                 Some(true)
             } else {
@@ -224,8 +249,8 @@ impl<I: Idx> IntervalSet<I> {
         other.iter_intervals().all(|sub| {
             current
                 .take()
-                .and_then(|sup| contains(sup, sub.clone(), &mut current))
-                .or_else(|| sup_iter.find_map(|sup| contains(sup, sub.clone(), &mut current)))
+                .and_then(|sup| contains(sup, sub, &mut current))
+                .or_else(|| sup_iter.find_map(|sup| contains(sup, sub, &mut current)))
                 .unwrap_or(false)
         })
     }
@@ -242,11 +267,11 @@ impl<I: Idx> IntervalSet<I> {
             let mut other_current = other_iter.next()?;
 
             loop {
-                if self_current.end <= other_current.start {
+                if self_current.last < other_current.start {
                     self_current = self_iter.next()?;
                     continue;
                 }
-                if other_current.end <= self_current.start {
+                if other_current.last < self_current.start {
                     other_current = other_iter.next()?;
                     continue;
                 }
@@ -370,6 +395,12 @@ impl<R: Idx, C: Step + Idx> SparseIntervalMatrix<R, C> {
         self.rows.get(row)
     }
 
+    pub fn clear_row(&mut self, row: R) {
+        if let Some(row) = self.rows.get_mut(row) {
+            row.clear();
+        }
+    }
+
     fn ensure_row(&mut self, row: R) -> &mut IntervalSet<C> {
         self.rows.ensure_contains_elem(row, || IntervalSet::new(self.column_size))
     }
@@ -393,6 +424,16 @@ impl<R: Idx, C: Step + Idx> SparseIntervalMatrix<R, C> {
         write_row.union(read_row)
     }
 
+    pub fn disjoint_rows(&self, a: R, b: R) -> bool
+    where
+        C: Step,
+    {
+        let (Some(a), Some(b)) = (self.rows.get(a), self.rows.get(b)) else {
+            return true;
+        };
+        a.disjoint(b)
+    }
+
     pub fn insert_all_into_row(&mut self, row: R) {
         self.ensure_row(row).insert_all();
     }

compiler/rustc_index/src/interval/tests.rs

@@ -5,7 +5,7 @@ fn insert_collapses() {
     let mut set = IntervalSet::<u32>::new(10000);
     set.insert_range(9831..=9837);
     set.insert_range(43..=9830);
-    assert_eq!(set.iter_intervals().collect::<Vec<_>>(), [43..9838]);
+    assert_eq!(set.iter_intervals().collect::<Vec<_>>(), [(43..=9837).into()]);
 }
 
 #[test]

compiler/rustc_middle/src/mir/syntax.rs

@@ -124,7 +124,6 @@ pub enum RuntimePhase {
     /// disallowed:
     /// * [`TerminatorKind::Yield`]
     /// * [`TerminatorKind::CoroutineDrop`]
-    /// * [`Rvalue::Aggregate`] for any `AggregateKind` except `Array`
     /// * [`Rvalue::CopyForDeref`]
     /// * [`PlaceElem::OpaqueCast`]
     /// * [`LocalInfo::DerefTemp`](super::LocalInfo::DerefTemp)
@@ -1442,9 +1441,6 @@ pub enum Rvalue<'tcx> {
     /// This is needed because dataflow analysis needs to distinguish
     /// `dest = Foo { x: ..., y: ... }` from `dest.x = ...; dest.y = ...;` in the case that `Foo`
     /// has a destructor.
-    ///
-    /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Coroutine`. After
-    /// coroutine lowering, `Coroutine` aggregate kinds are disallowed too.
     Aggregate(Box<AggregateKind<'tcx>>, IndexVec<FieldIdx, Operand<'tcx>>),
 
     /// A CopyForDeref is equivalent to a read from a place at the

compiler/rustc_mir_dataflow/src/framework/direction.rs

@@ -212,7 +212,7 @@ impl Direction for Backward {
     ) where
         A: Analysis<'tcx>,
     {
-        vis.visit_block_end(state);
+        vis.visit_block_end(state, block);
 
         let loc = Location { block, statement_index: block_data.statements.len() };
         let term = block_data.terminator();
@@ -229,7 +229,7 @@ impl Direction for Backward {
             vis.visit_after_primary_statement_effect(analysis, state, stmt, loc);
         }
 
-        vis.visit_block_start(state);
+        vis.visit_block_start(state, block);
     }
 }
 
@@ -391,7 +391,7 @@ impl Direction for Forward {
     ) where
         A: Analysis<'tcx>,
     {
-        vis.visit_block_start(state);
+        vis.visit_block_start(state, block);
 
         for (statement_index, stmt) in block_data.statements.iter().enumerate() {
             let loc = Location { block, statement_index };
@@ -408,6 +408,6 @@ impl Direction for Forward {
         analysis.apply_primary_terminator_effect(state, term, loc);
         vis.visit_after_primary_terminator_effect(analysis, state, term, loc);
 
-        vis.visit_block_end(state);
+        vis.visit_block_end(state, block);
     }
 }

compiler/rustc_mir_dataflow/src/framework/graphviz.rs

@@ -660,13 +660,13 @@ where
     A: Analysis<'tcx>,
     A::Domain: DebugWithContext<A>,
 {
-    fn visit_block_start(&mut self, state: &A::Domain) {
+    fn visit_block_start(&mut self, state: &A::Domain, _block: BasicBlock) {
         if A::Direction::IS_FORWARD {
             self.prev_state.clone_from(state);
         }
     }
 
-    fn visit_block_end(&mut self, state: &A::Domain) {
+    fn visit_block_end(&mut self, state: &A::Domain, _block: BasicBlock) {
         if A::Direction::IS_BACKWARD {
             self.prev_state.clone_from(state);
         }

compiler/rustc_mir_dataflow/src/framework/visitor.rs

@@ -46,7 +46,7 @@ pub trait ResultsVisitor<'tcx, A>
 where
     A: Analysis<'tcx>,
 {
-    fn visit_block_start(&mut self, _state: &A::Domain) {}
+    fn visit_block_start(&mut self, _state: &A::Domain, _block: BasicBlock) {}
 
     /// Called after the "early" effect of the given statement is applied to `state`.
     fn visit_after_early_statement_effect(
@@ -90,5 +90,5 @@ where
     ) {
     }
 
-    fn visit_block_end(&mut self, _state: &A::Domain) {}
+    fn visit_block_end(&mut self, _state: &A::Domain, _block: BasicBlock) {}
 }

compiler/rustc_mir_dataflow/src/impls/mod.rs

@@ -1,6 +1,7 @@
 mod borrowed_locals;
 mod initialized;
 mod liveness;
+mod precise_liveness;
 mod storage_liveness;
 
 pub use self::borrowed_locals::{MaybeBorrowedLocals, borrowed_locals};
@@ -12,6 +13,9 @@ pub use self::liveness::{
     DefUse, MaybeLiveLocals, MaybeTransitiveLiveLocals,
     TransferFunction as LivenessTransferFunction,
 };
+pub use self::precise_liveness::{
+    SplitPointEffect, SplitPointIndex, dump_liveness_matrix, liveness_matrix,
+};
 pub use self::storage_liveness::{
     MaybeRequiresStorage, MaybeStorageDead, MaybeStorageLive, always_storage_live_locals,
 };