authorbors <bors@rust-lang.org> 2024-07-03 18:52:04 UTC
committerbors <bors@rust-lang.org> 2024-07-03 18:52:04 UTC
log2b90614e94cfb400820cfc10fe63b0db74f9e67a
treea51d4dc4c105955b3a8766e7b0b471d8f8ab8c41
parent1cfd47fe0b78f48a04ac8fce792a406b638da40b
parent76244d4dbc768e15e429c1f66ec021884f369f5f

Auto merge of #127036 - cjgillot:sparse-state, r=oli-obk

Make jump threading state sparse Continuation of https://github.com/rust-lang/rust/pull/127024 Both dataflow const-prop and jump threading involve cloning the state vector a lot. This PR replaces the data structure by a sparse vector, considering: - that jump threading state is typically very sparse (at most 1 or 2 set entries); - that dataflow const-prop is disabled by default; - that place/value map is very eager, and prone to creating an overly large state. The first commit is shared with the previous PR to avoid needless conflicts. r? `@oli-obk`

3 files changed, 141 insertions(+), 80 deletions(-)

compiler/rustc_mir_dataflow/src/framework/lattice.rs+14
......@@ -76,6 +76,8 @@ pub trait MeetSemiLattice: Eq {
7676/// A set that has a "bottom" element, which is less than or equal to any other element.
7777pub trait HasBottom {
7878 const BOTTOM: Self;
79
80 fn is_bottom(&self) -> bool;
7981}
8082
8183/// A set that has a "top" element, which is greater than or equal to any other element.
......@@ -114,6 +116,10 @@ impl MeetSemiLattice for bool {
114116
115117impl HasBottom for bool {
116118 const BOTTOM: Self = false;
119
120 fn is_bottom(&self) -> bool {
121 !self
122 }
117123}
118124
119125impl HasTop for bool {
......@@ -267,6 +273,10 @@ impl<T: Clone + Eq> MeetSemiLattice for FlatSet<T> {
267273
268274impl<T> HasBottom for FlatSet<T> {
269275 const BOTTOM: Self = Self::Bottom;
276
277 fn is_bottom(&self) -> bool {
278 matches!(self, Self::Bottom)
279 }
270280}
271281
272282impl<T> HasTop for FlatSet<T> {
......@@ -291,6 +301,10 @@ impl<T> MaybeReachable<T> {
291301
292302impl<T> HasBottom for MaybeReachable<T> {
293303 const BOTTOM: Self = MaybeReachable::Unreachable;
304
305 fn is_bottom(&self) -> bool {
306 matches!(self, Self::Unreachable)
307 }
294308}
295309
296310impl<T: HasTop> HasTop for MaybeReachable<T> {
compiler/rustc_mir_dataflow/src/value_analysis.rs+113-75
......@@ -36,10 +36,10 @@ use std::collections::VecDeque;
3636use std::fmt::{Debug, Formatter};
3737use std::ops::Range;
3838
39use rustc_data_structures::fx::FxHashMap;
39use rustc_data_structures::fx::{FxHashMap, StdEntry};
4040use rustc_data_structures::stack::ensure_sufficient_stack;
4141use rustc_index::bit_set::BitSet;
42use rustc_index::{IndexSlice, IndexVec};
42use rustc_index::IndexVec;
4343use rustc_middle::bug;
4444use rustc_middle::mir::visit::{MutatingUseContext, PlaceContext, Visitor};
4545use rustc_middle::mir::*;
......@@ -336,14 +336,13 @@ impl<'tcx, T: ValueAnalysis<'tcx>> AnalysisDomain<'tcx> for ValueAnalysisWrapper
336336 const NAME: &'static str = T::NAME;
337337
338338 fn bottom_value(&self, _body: &Body<'tcx>) -> Self::Domain {
339 State(StateData::Unreachable)
339 State::Unreachable
340340 }
341341
342342 fn initialize_start_block(&self, body: &Body<'tcx>, state: &mut Self::Domain) {
343343 // The initial state maps all tracked places of argument projections to ⊤ and the rest to ⊥.
344 assert!(matches!(state.0, StateData::Unreachable));
345 let values = IndexVec::from_elem_n(T::Value::BOTTOM, self.0.map().value_count);
346 *state = State(StateData::Reachable(values));
344 assert!(matches!(state, State::Unreachable));
345 *state = State::new_reachable();
347346 for arg in body.args_iter() {
348347 state.flood(PlaceRef { local: arg, projection: &[] }, self.0.map());
349348 }
......@@ -415,27 +414,54 @@ rustc_index::newtype_index!(
415414
416415/// See [`State`].
417416#[derive(PartialEq, Eq, Debug)]
418enum StateData<V> {
419 Reachable(IndexVec<ValueIndex, V>),
420 Unreachable,
417pub struct StateData<V> {
418 bottom: V,
419 /// This map only contains values that are not `⊥`.
420 map: FxHashMap<ValueIndex, V>,
421}
422
423impl<V: HasBottom> StateData<V> {
424 fn new() -> StateData<V> {
425 StateData { bottom: V::BOTTOM, map: FxHashMap::default() }
426 }
427
428 fn get(&self, idx: ValueIndex) -> &V {
429 self.map.get(&idx).unwrap_or(&self.bottom)
430 }
431
432 fn insert(&mut self, idx: ValueIndex, elem: V) {
433 if elem.is_bottom() {
434 self.map.remove(&idx);
435 } else {
436 self.map.insert(idx, elem);
437 }
438 }
421439}
422440
423441impl<V: Clone> Clone for StateData<V> {
424442 fn clone(&self) -> Self {
425 match self {
426 Self::Reachable(x) => Self::Reachable(x.clone()),
427 Self::Unreachable => Self::Unreachable,
428 }
443 StateData { bottom: self.bottom.clone(), map: self.map.clone() }
429444 }
430445
431446 fn clone_from(&mut self, source: &Self) {
432 match (&mut *self, source) {
433 (Self::Reachable(x), Self::Reachable(y)) => {
434 // We go through `raw` here, because `IndexVec` currently has a naive `clone_from`.
435 x.raw.clone_from(&y.raw);
447 self.map.clone_from(&source.map)
448 }
449}
450
451impl<V: JoinSemiLattice + Clone + HasBottom> JoinSemiLattice for StateData<V> {
452 fn join(&mut self, other: &Self) -> bool {
453 let mut changed = false;
454 #[allow(rustc::potential_query_instability)]
455 for (i, v) in other.map.iter() {
456 match self.map.entry(*i) {
457 StdEntry::Vacant(e) => {
458 e.insert(v.clone());
459 changed = true
460 }
461 StdEntry::Occupied(e) => changed |= e.into_mut().join(v),
436462 }
437 _ => *self = source.clone(),
438463 }
464 changed
439465 }
440466}
441467
......@@ -450,33 +476,47 @@ impl<V: Clone> Clone for StateData<V> {
450476///
451477/// Flooding means assigning a value (by default `⊤`) to all tracked projections of a given place.
452478#[derive(PartialEq, Eq, Debug)]
453pub struct State<V>(StateData<V>);
479pub enum State<V> {
480 Unreachable,
481 Reachable(StateData<V>),
482}
454483
455484impl<V: Clone> Clone for State<V> {
456485 fn clone(&self) -> Self {
457 Self(self.0.clone())
486 match self {
487 Self::Reachable(x) => Self::Reachable(x.clone()),
488 Self::Unreachable => Self::Unreachable,
489 }
458490 }
459491
460492 fn clone_from(&mut self, source: &Self) {
461 self.0.clone_from(&source.0);
493 match (&mut *self, source) {
494 (Self::Reachable(x), Self::Reachable(y)) => {
495 x.clone_from(&y);
496 }
497 _ => *self = source.clone(),
498 }
462499 }
463500}
464501
465impl<V: Clone> State<V> {
466 pub fn new(init: V, map: &Map) -> State<V> {
467 let values = IndexVec::from_elem_n(init, map.value_count);
468 State(StateData::Reachable(values))
502impl<V: Clone + HasBottom> State<V> {
503 pub fn new_reachable() -> State<V> {
504 State::Reachable(StateData::new())
469505 }
470506
471 pub fn all(&self, f: impl Fn(&V) -> bool) -> bool {
472 match self.0 {
473 StateData::Unreachable => true,
474 StateData::Reachable(ref values) => values.iter().all(f),
507 pub fn all_bottom(&self) -> bool {
508 match self {
509 State::Unreachable => false,
510 State::Reachable(ref values) =>
511 {
512 #[allow(rustc::potential_query_instability)]
513 values.map.values().all(V::is_bottom)
514 }
475515 }
476516 }
477517
478518 fn is_reachable(&self) -> bool {
479 matches!(&self.0, StateData::Reachable(_))
519 matches!(self, State::Reachable(_))
480520 }
481521
482522 /// Assign `value` to all places that are contained in `place` or may alias one.
......@@ -519,10 +559,8 @@ impl<V: Clone> State<V> {
519559 map: &Map,
520560 value: V,
521561 ) {
522 let StateData::Reachable(values) = &mut self.0 else { return };
523 map.for_each_aliasing_place(place, tail_elem, &mut |vi| {
524 values[vi] = value.clone();
525 });
562 let State::Reachable(values) = self else { return };
563 map.for_each_aliasing_place(place, tail_elem, &mut |vi| values.insert(vi, value.clone()));
526564 }
527565
528566 /// Low-level method that assigns to a place.
......@@ -541,9 +579,9 @@ impl<V: Clone> State<V> {
541579 ///
542580 /// The target place must have been flooded before calling this method.
543581 pub fn insert_value_idx(&mut self, target: PlaceIndex, value: V, map: &Map) {
544 let StateData::Reachable(values) = &mut self.0 else { return };
582 let State::Reachable(values) = self else { return };
545583 if let Some(value_index) = map.places[target].value_index {
546 values[value_index] = value;
584 values.insert(value_index, value)
547585 }
548586 }
549587
......@@ -555,14 +593,14 @@ impl<V: Clone> State<V> {
555593 ///
556594 /// The target place must have been flooded before calling this method.
557595 pub fn insert_place_idx(&mut self, target: PlaceIndex, source: PlaceIndex, map: &Map) {
558 let StateData::Reachable(values) = &mut self.0 else { return };
596 let State::Reachable(values) = self else { return };
559597
560598 // If both places are tracked, we copy the value to the target.
561599 // If the target is tracked, but the source is not, we do nothing, as invalidation has
562600 // already been performed.
563601 if let Some(target_value) = map.places[target].value_index {
564602 if let Some(source_value) = map.places[source].value_index {
565 values[target_value] = values[source_value].clone();
603 values.insert(target_value, values.get(source_value).clone());
566604 }
567605 }
568606 for target_child in map.children(target) {
......@@ -616,11 +654,11 @@ impl<V: Clone> State<V> {
616654
617655 /// Retrieve the value stored for a place index, or `None` if it is not tracked.
618656 pub fn try_get_idx(&self, place: PlaceIndex, map: &Map) -> Option<V> {
619 match &self.0 {
620 StateData::Reachable(values) => {
621 map.places[place].value_index.map(|v| values[v].clone())
657 match self {
658 State::Reachable(values) => {
659 map.places[place].value_index.map(|v| values.get(v).clone())
622660 }
623 StateData::Unreachable => None,
661 State::Unreachable => None,
624662 }
625663 }
626664
......@@ -631,10 +669,10 @@ impl<V: Clone> State<V> {
631669 where
632670 V: HasBottom + HasTop,
633671 {
634 match &self.0 {
635 StateData::Reachable(_) => self.try_get(place, map).unwrap_or(V::TOP),
672 match self {
673 State::Reachable(_) => self.try_get(place, map).unwrap_or(V::TOP),
636674 // Because this is unreachable, we can return any value we want.
637 StateData::Unreachable => V::BOTTOM,
675 State::Unreachable => V::BOTTOM,
638676 }
639677 }
640678
......@@ -645,10 +683,10 @@ impl<V: Clone> State<V> {
645683 where
646684 V: HasBottom + HasTop,
647685 {
648 match &self.0 {
649 StateData::Reachable(_) => self.try_get_discr(place, map).unwrap_or(V::TOP),
686 match self {
687 State::Reachable(_) => self.try_get_discr(place, map).unwrap_or(V::TOP),
650688 // Because this is unreachable, we can return any value we want.
651 StateData::Unreachable => V::BOTTOM,
689 State::Unreachable => V::BOTTOM,
652690 }
653691 }
654692
......@@ -659,10 +697,10 @@ impl<V: Clone> State<V> {
659697 where
660698 V: HasBottom + HasTop,
661699 {
662 match &self.0 {
663 StateData::Reachable(_) => self.try_get_len(place, map).unwrap_or(V::TOP),
700 match self {
701 State::Reachable(_) => self.try_get_len(place, map).unwrap_or(V::TOP),
664702 // Because this is unreachable, we can return any value we want.
665 StateData::Unreachable => V::BOTTOM,
703 State::Unreachable => V::BOTTOM,
666704 }
667705 }
668706
......@@ -673,11 +711,11 @@ impl<V: Clone> State<V> {
673711 where
674712 V: HasBottom + HasTop,
675713 {
676 match &self.0 {
677 StateData::Reachable(values) => {
678 map.places[place].value_index.map(|v| values[v].clone()).unwrap_or(V::TOP)
714 match self {
715 State::Reachable(values) => {
716 map.places[place].value_index.map(|v| values.get(v).clone()).unwrap_or(V::TOP)
679717 }
680 StateData::Unreachable => {
718 State::Unreachable => {
681719 // Because this is unreachable, we can return any value we want.
682720 V::BOTTOM
683721 }
......@@ -685,15 +723,15 @@ impl<V: Clone> State<V> {
685723 }
686724}
687725
688impl<V: JoinSemiLattice + Clone> JoinSemiLattice for State<V> {
726impl<V: JoinSemiLattice + Clone + HasBottom> JoinSemiLattice for State<V> {
689727 fn join(&mut self, other: &Self) -> bool {
690 match (&mut self.0, &other.0) {
691 (_, StateData::Unreachable) => false,
692 (StateData::Unreachable, _) => {
728 match (&mut *self, other) {
729 (_, State::Unreachable) => false,
730 (State::Unreachable, _) => {
693731 *self = other.clone();
694732 true
695733 }
696 (StateData::Reachable(this), StateData::Reachable(other)) => this.join(other),
734 (State::Reachable(this), State::Reachable(ref other)) => this.join(other),
697735 }
698736 }
699737}
......@@ -1194,9 +1232,9 @@ where
11941232 T::Value: Debug,
11951233{
11961234 fn fmt_with(&self, ctxt: &ValueAnalysisWrapper<T>, f: &mut Formatter<'_>) -> std::fmt::Result {
1197 match &self.0 {
1198 StateData::Reachable(values) => debug_with_context(values, None, ctxt.0.map(), f),
1199 StateData::Unreachable => write!(f, "unreachable"),
1235 match self {
1236 State::Reachable(values) => debug_with_context(values, None, ctxt.0.map(), f),
1237 State::Unreachable => write!(f, "unreachable"),
12001238 }
12011239 }
12021240
......@@ -1206,8 +1244,8 @@ where
12061244 ctxt: &ValueAnalysisWrapper<T>,
12071245 f: &mut Formatter<'_>,
12081246 ) -> std::fmt::Result {
1209 match (&self.0, &old.0) {
1210 (StateData::Reachable(this), StateData::Reachable(old)) => {
1247 match (self, old) {
1248 (State::Reachable(this), State::Reachable(old)) => {
12111249 debug_with_context(this, Some(old), ctxt.0.map(), f)
12121250 }
12131251 _ => Ok(()), // Consider printing something here.
......@@ -1215,21 +1253,21 @@ where
12151253 }
12161254}
12171255
1218fn debug_with_context_rec<V: Debug + Eq>(
1256fn debug_with_context_rec<V: Debug + Eq + HasBottom>(
12191257 place: PlaceIndex,
12201258 place_str: &str,
1221 new: &IndexSlice<ValueIndex, V>,
1222 old: Option<&IndexSlice<ValueIndex, V>>,
1259 new: &StateData<V>,
1260 old: Option<&StateData<V>>,
12231261 map: &Map,
12241262 f: &mut Formatter<'_>,
12251263) -> std::fmt::Result {
12261264 if let Some(value) = map.places[place].value_index {
12271265 match old {
1228 None => writeln!(f, "{}: {:?}", place_str, new[value])?,
1266 None => writeln!(f, "{}: {:?}", place_str, new.get(value))?,
12291267 Some(old) => {
1230 if new[value] != old[value] {
1231 writeln!(f, "\u{001f}-{}: {:?}", place_str, old[value])?;
1232 writeln!(f, "\u{001f}+{}: {:?}", place_str, new[value])?;
1268 if new.get(value) != old.get(value) {
1269 writeln!(f, "\u{001f}-{}: {:?}", place_str, old.get(value))?;
1270 writeln!(f, "\u{001f}+{}: {:?}", place_str, new.get(value))?;
12331271 }
12341272 }
12351273 }
......@@ -1261,9 +1299,9 @@ fn debug_with_context_rec<V: Debug + Eq>(
12611299 Ok(())
12621300}
12631301
1264fn debug_with_context<V: Debug + Eq>(
1265 new: &IndexSlice<ValueIndex, V>,
1266 old: Option<&IndexSlice<ValueIndex, V>>,
1302fn debug_with_context<V: Debug + Eq + HasBottom>(
1303 new: &StateData<V>,
1304 old: Option<&StateData<V>>,
12671305 map: &Map,
12681306 f: &mut Formatter<'_>,
12691307) -> std::fmt::Result {
compiler/rustc_mir_transform/src/jump_threading.rs+14-5
......@@ -47,6 +47,7 @@ use rustc_middle::mir::visit::Visitor;
4747use rustc_middle::mir::*;
4848use rustc_middle::ty::layout::LayoutOf;
4949use rustc_middle::ty::{self, ScalarInt, TyCtxt};
50use rustc_mir_dataflow::lattice::HasBottom;
5051use rustc_mir_dataflow::value_analysis::{Map, PlaceIndex, State, TrackElem};
5152use rustc_span::DUMMY_SP;
5253use rustc_target::abi::{TagEncoding, Variants};
......@@ -158,9 +159,17 @@ impl Condition {
158159 }
159160}
160161
161#[derive(Copy, Clone, Debug, Default)]
162#[derive(Copy, Clone, Debug)]
162163struct ConditionSet<'a>(&'a [Condition]);
163164
165impl HasBottom for ConditionSet<'_> {
166 const BOTTOM: Self = ConditionSet(&[]);
167
168 fn is_bottom(&self) -> bool {
169 self.0.is_empty()
170 }
171}
172
164173impl<'a> ConditionSet<'a> {
165174 fn iter(self) -> impl Iterator<Item = Condition> + 'a {
166175 self.0.iter().copied()
......@@ -177,7 +186,7 @@ impl<'a> ConditionSet<'a> {
177186
178187impl<'tcx, 'a> TOFinder<'tcx, 'a> {
179188 fn is_empty(&self, state: &State<ConditionSet<'a>>) -> bool {
180 state.all(|cs| cs.0.is_empty())
189 state.all_bottom()
181190 }
182191
183192 /// Recursion entry point to find threading opportunities.
......@@ -198,7 +207,7 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> {
198207 debug!(?discr);
199208
200209 let cost = CostChecker::new(self.tcx, self.param_env, None, self.body);
201 let mut state = State::new(ConditionSet::default(), self.map);
210 let mut state = State::new_reachable();
202211
203212 let conds = if let Some((value, then, else_)) = targets.as_static_if() {
204213 let value = ScalarInt::try_from_uint(value, discr_layout.size)?;
......@@ -255,7 +264,7 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> {
255264 // _1 = 5 // Whatever happens here, it won't change the result of a `SwitchInt`.
256265 // _1 = 6
257266 if let Some((lhs, tail)) = self.mutated_statement(stmt) {
258 state.flood_with_tail_elem(lhs.as_ref(), tail, self.map, ConditionSet::default());
267 state.flood_with_tail_elem(lhs.as_ref(), tail, self.map, ConditionSet::BOTTOM);
259268 }
260269 }
261270
......@@ -609,7 +618,7 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> {
609618 // We can recurse through this terminator.
610619 let mut state = state();
611620 if let Some(place_to_flood) = place_to_flood {
612 state.flood_with(place_to_flood.as_ref(), self.map, ConditionSet::default());
621 state.flood_with(place_to_flood.as_ref(), self.map, ConditionSet::BOTTOM);
613622 }
614623 self.find_opportunity(bb, state, cost.clone(), depth + 1);
615624 }