[DO NOT MERGE] bootstrap with -Znext-solver=globally

compiler/rustc_ast_passes/src/feature_gate.rs

@@ -1,6 +1,7 @@
 use rustc_ast as ast;
 use rustc_ast::visit::{self, AssocCtxt, FnCtxt, FnKind, Visitor};
 use rustc_ast::{NodeId, PatKind, attr, token};
+use rustc_errors::E0001;
 use rustc_feature::{AttributeGate, BUILTIN_ATTRIBUTE_MAP, BuiltinAttribute, Features};
 use rustc_session::Session;
 use rustc_session::parse::{feature_err, feature_warn};
@@ -648,10 +649,13 @@ fn check_new_solver_banned_features(sess: &Session, features: &Features) {
         .map(|feat| feat.attr_sp)
     {
         #[allow(rustc::symbol_intern_string_literal)]
-        sess.dcx().emit_err(errors::IncompatibleFeatures {
-            spans: vec![gce_span],
-            f1: Symbol::intern("-Znext-solver=globally"),
-            f2: sym::generic_const_exprs,
-        });
+        sess.dcx()
+            .create_fatal(errors::IncompatibleFeatures {
+                spans: vec![gce_span],
+                f1: Symbol::intern("-Znext-solver=globally"),
+                f2: sym::generic_const_exprs,
+            })
+            .with_code(E0001)
+            .emit();
     }
 }

compiler/rustc_borrowck/messages.ftl

@@ -156,12 +156,6 @@ borrowck_moved_due_to_usage_in_operator =
         *[false] operator
     }
 
-borrowck_opaque_type_lifetime_mismatch =
-    opaque type used twice with different lifetimes
-    .label = lifetime `{$arg}` used here
-    .prev_lifetime_label = lifetime `{$prev}` previously used here
-    .note = if all non-lifetime generic parameters are the same, but the lifetime parameters differ, it is not possible to differentiate the opaque types
-
 borrowck_partial_var_move_by_use_in_closure =
     variable {$is_partial ->
         [true] partially moved

compiler/rustc_borrowck/src/consumers.rs

@@ -15,7 +15,7 @@ pub use super::polonius::legacy::{
     RichLocation, RustcFacts,
 };
 pub use super::region_infer::RegionInferenceContext;
-use crate::{BorrowCheckRootCtxt, do_mir_borrowck};
+use crate::BorrowCheckRootCtxt;
 
 /// Options determining the output behavior of [`get_body_with_borrowck_facts`].
 ///
@@ -101,6 +101,6 @@ pub fn get_body_with_borrowck_facts(
     def_id: LocalDefId,
     options: ConsumerOptions,
 ) -> BodyWithBorrowckFacts<'_> {
-    let mut root_cx = BorrowCheckRootCtxt::new(tcx, def_id);
-    *do_mir_borrowck(&mut root_cx, def_id, Some(options)).1.unwrap()
+    let root_cx = BorrowCheckRootCtxt::new(tcx, def_id);
+    *root_cx.borrowck_root(Some(options)).1.unwrap()
 }

compiler/rustc_borrowck/src/dataflow.rs

@@ -1,7 +1,6 @@
 use std::fmt;
 
 use rustc_data_structures::fx::FxIndexMap;
-use rustc_data_structures::graph;
 use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::mir::{
     self, BasicBlock, Body, CallReturnPlaces, Location, Place, TerminatorEdges,
@@ -317,9 +316,8 @@ impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> {
             loans_out_of_scope_at_location: FxIndexMap::default(),
         };
         for (loan_idx, loan_data) in borrow_set.iter_enumerated() {
-            let issuing_region = loan_data.region;
             let loan_issued_at = loan_data.reserve_location;
-            prec.precompute_loans_out_of_scope(loan_idx, issuing_region, loan_issued_at);
+            prec.precompute_loans_out_of_scope(loan_idx, loan_issued_at);
         }
 
         prec.loans_out_of_scope_at_location
@@ -328,45 +326,7 @@ impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> {
     /// Loans are in scope while they are live: whether they are contained within any live region.
     /// In the location-insensitive analysis, a loan will be contained in a region if the issuing
     /// region can reach it in the subset graph. So this is a reachability problem.
-    fn precompute_loans_out_of_scope(
-        &mut self,
-        loan_idx: BorrowIndex,
-        issuing_region: RegionVid,
-        loan_issued_at: Location,
-    ) {
-        let sccs = self.regioncx.constraint_sccs();
-        let universal_regions = self.regioncx.universal_regions();
-
-        // The loop below was useful for the location-insensitive analysis but shouldn't be
-        // impactful in the location-sensitive case. It seems that it does, however, as without it a
-        // handful of tests fail. That likely means some liveness or outlives data related to choice
-        // regions is missing
-        // FIXME: investigate the impact of loans traversing applied member constraints and why some
-        // tests fail otherwise.
-        //
-        // We first handle the cases where the loan doesn't go out of scope, depending on the
-        // issuing region's successors.
-        for successor in graph::depth_first_search(&self.regioncx.region_graph(), issuing_region) {
-            // Via applied member constraints
-            //
-            // The issuing region can flow into the choice regions, and they are either:
-            // - placeholders or free regions themselves,
-            // - or also transitively outlive a free region.
-            //
-            // That is to say, if there are applied member constraints here, the loan escapes the
-            // function and cannot go out of scope. We could early return here.
-            //
-            // For additional insurance via fuzzing and crater, we verify that the constraint's min
-            // choice indeed escapes the function. In the future, we could e.g. turn this check into
-            // a debug assert and early return as an optimization.
-            let scc = sccs.scc(successor);
-            for constraint in self.regioncx.applied_member_constraints(scc) {
-                if universal_regions.is_universal_region(constraint.min_choice) {
-                    return;
-                }
-            }
-        }
-
+    fn precompute_loans_out_of_scope(&mut self, loan_idx: BorrowIndex, loan_issued_at: Location) {
         let first_block = loan_issued_at.block;
         let first_bb_data = &self.body.basic_blocks[first_block];
 

compiler/rustc_borrowck/src/diagnostics/region_errors.rs

@@ -23,7 +23,6 @@ use rustc_trait_selection::error_reporting::infer::nice_region_error::{
     self, HirTraitObjectVisitor, NiceRegionError, TraitObjectVisitor, find_anon_type,
     find_param_with_region, suggest_adding_lifetime_params,
 };
-use rustc_trait_selection::error_reporting::infer::region::unexpected_hidden_region_diagnostic;
 use rustc_trait_selection::infer::InferCtxtExt;
 use rustc_trait_selection::traits::{Obligation, ObligationCtxt};
 use tracing::{debug, instrument, trace};
@@ -84,9 +83,6 @@ impl<'tcx> RegionErrors<'tcx> {
         let guar = self.1.sess.dcx().delayed_bug(format!("{val:?}"));
         self.0.push((val, guar));
     }
-    pub(crate) fn is_empty(&self) -> bool {
-        self.0.is_empty()
-    }
     pub(crate) fn into_iter(
         self,
     ) -> impl Iterator<Item = (RegionErrorKind<'tcx>, ErrorGuaranteed)> {
@@ -108,18 +104,6 @@ pub(crate) enum RegionErrorKind<'tcx> {
     /// A generic bound failure for a type test (`T: 'a`).
     TypeTestError { type_test: TypeTest<'tcx> },
 
-    /// An unexpected hidden region for an opaque type.
-    UnexpectedHiddenRegion {
-        /// The span for the member constraint.
-        span: Span,
-        /// The hidden type.
-        hidden_ty: Ty<'tcx>,
-        /// The opaque type.
-        key: ty::OpaqueTypeKey<'tcx>,
-        /// The unexpected region.
-        member_region: ty::Region<'tcx>,
-    },
-
     /// Higher-ranked subtyping error.
     BoundUniversalRegionError {
         /// The placeholder free region.
@@ -312,9 +296,6 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
         // buffered in the `MirBorrowckCtxt`.
 
         let mut outlives_suggestion = OutlivesSuggestionBuilder::default();
-        let mut last_unexpected_hidden_region: Option<(Span, Ty<'_>, ty::OpaqueTypeKey<'tcx>)> =
-            None;
-
         for (nll_error, _) in nll_errors.into_iter() {
             match nll_error {
                 RegionErrorKind::TypeTestError { type_test } => {
@@ -364,30 +345,6 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                     }
                 }
 
-                RegionErrorKind::UnexpectedHiddenRegion { span, hidden_ty, key, member_region } => {
-                    let named_ty =
-                        self.regioncx.name_regions_for_member_constraint(self.infcx.tcx, hidden_ty);
-                    let named_key =
-                        self.regioncx.name_regions_for_member_constraint(self.infcx.tcx, key);
-                    let named_region = self
-                        .regioncx
-                        .name_regions_for_member_constraint(self.infcx.tcx, member_region);
-                    let diag = unexpected_hidden_region_diagnostic(
-                        self.infcx,
-                        self.mir_def_id(),
-                        span,
-                        named_ty,
-                        named_region,
-                        named_key,
-                    );
-                    if last_unexpected_hidden_region != Some((span, named_ty, named_key)) {
-                        self.buffer_error(diag);
-                        last_unexpected_hidden_region = Some((span, named_ty, named_key));
-                    } else {
-                        diag.delay_as_bug();
-                    }
-                }
-
                 RegionErrorKind::BoundUniversalRegionError {
                     longer_fr,
                     placeholder,

compiler/rustc_borrowck/src/lib.rs

@@ -19,10 +19,14 @@ use std::borrow::Cow;
 use std::cell::RefCell;
 use std::marker::PhantomData;
 use std::ops::{ControlFlow, Deref};
+use std::rc::Rc;
 
 use borrow_set::LocalsStateAtExit;
+use polonius_engine::AllFacts;
+use region_infer::opaque_types::DeferredOpaqueTypeError;
 use root_cx::BorrowCheckRootCtxt;
 use rustc_abi::FieldIdx;
+use rustc_data_structures::frozen::Frozen;
 use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_data_structures::graph::dominators::Dominators;
 use rustc_errors::LintDiagnostic;
@@ -31,6 +35,7 @@ use rustc_hir::CRATE_HIR_ID;
 use rustc_hir::def_id::LocalDefId;
 use rustc_index::bit_set::{DenseBitSet, MixedBitSet};
 use rustc_index::{IndexSlice, IndexVec};
+use rustc_infer::infer::outlives::env::RegionBoundPairs;
 use rustc_infer::infer::{
     InferCtxt, NllRegionVariableOrigin, RegionVariableOrigin, TyCtxtInferExt,
 };
@@ -46,23 +51,28 @@ use rustc_mir_dataflow::impls::{
 use rustc_mir_dataflow::move_paths::{
     InitIndex, InitLocation, LookupResult, MoveData, MovePathIndex,
 };
+use rustc_mir_dataflow::points::DenseLocationMap;
 use rustc_mir_dataflow::{Analysis, Results, ResultsVisitor, visit_results};
 use rustc_session::lint::builtin::{TAIL_EXPR_DROP_ORDER, UNUSED_MUT};
 use rustc_span::{ErrorGuaranteed, Span, Symbol};
 use smallvec::SmallVec;
 use tracing::{debug, instrument};
+use type_check::free_region_relations::UniversalRegionRelations;
+use type_check::{Locations, MirTypeckRegionConstraints, MirTypeckResults};
 
 use crate::borrow_set::{BorrowData, BorrowSet};
-use crate::consumers::{BodyWithBorrowckFacts, ConsumerOptions};
+use crate::consumers::{BodyWithBorrowckFacts, ConsumerOptions, RustcFacts};
 use crate::dataflow::{BorrowIndex, Borrowck, BorrowckDomain, Borrows};
 use crate::diagnostics::{
     AccessKind, BorrowckDiagnosticsBuffer, IllegalMoveOriginKind, MoveError, RegionName,
 };
 use crate::path_utils::*;
 use crate::place_ext::PlaceExt;
 use crate::places_conflict::{PlaceConflictBias, places_conflict};
-use crate::polonius::PoloniusDiagnosticsContext;
-use crate::polonius::legacy::{PoloniusLocationTable, PoloniusOutput};
+use crate::polonius::legacy::{
+    PoloniusFacts, PoloniusFactsExt, PoloniusLocationTable, PoloniusOutput,
+};
+use crate::polonius::{PoloniusContext, PoloniusDiagnosticsContext};
 use crate::prefixes::PrefixSet;
 use crate::region_infer::RegionInferenceContext;
 use crate::renumber::RegionCtxt;
@@ -74,7 +84,6 @@ mod constraints;
 mod dataflow;
 mod def_use;
 mod diagnostics;
-mod member_constraints;
 mod nll;
 mod path_utils;
 mod place_ext;
@@ -125,20 +134,8 @@ fn mir_borrowck(
         let opaque_types = ConcreteOpaqueTypes(Default::default());
         Ok(tcx.arena.alloc(opaque_types))
     } else {
-        let mut root_cx = BorrowCheckRootCtxt::new(tcx, def);
-        // We need to manually borrowck all nested bodies from the HIR as
-        // we do not generate MIR for dead code. Not doing so causes us to
-        // never check closures in dead code.
-        let nested_bodies = tcx.nested_bodies_within(def);
-        for def_id in nested_bodies {
-            root_cx.get_or_insert_nested(def_id);
-        }
-
-        let PropagatedBorrowCheckResults { closure_requirements, used_mut_upvars } =
-            do_mir_borrowck(&mut root_cx, def, None).0;
-        debug_assert!(closure_requirements.is_none());
-        debug_assert!(used_mut_upvars.is_empty());
-        root_cx.finalize()
+        let root_cx = BorrowCheckRootCtxt::new(tcx, def);
+        root_cx.borrowck_root(None).0
     }
 }
 
@@ -150,6 +147,8 @@ struct PropagatedBorrowCheckResults<'tcx> {
     used_mut_upvars: SmallVec<[FieldIdx; 8]>,
 }
 
+type DeferredClosureRequirements<'tcx> = Vec<(LocalDefId, ty::GenericArgsRef<'tcx>, Locations)>;
+
 /// After we borrow check a closure, we are left with various
 /// requirements that we have inferred between the free regions that
 /// appear in the closure's signature or on its field types. These
@@ -288,6 +287,24 @@ impl<'tcx> ClosureOutlivesSubjectTy<'tcx> {
     }
 }
 
+struct BorrowckState<'tcx> {
+    infcx: BorrowckInferCtxt<'tcx>,
+    body_owned: Body<'tcx>,
+    promoted: IndexVec<Promoted, Body<'tcx>>,
+    move_data: MoveData<'tcx>,
+    borrow_set: BorrowSet<'tcx>,
+    location_table: PoloniusLocationTable,
+    location_map: Rc<DenseLocationMap>,
+    universal_region_relations: Frozen<UniversalRegionRelations<'tcx>>,
+    region_bound_pairs: Frozen<RegionBoundPairs<'tcx>>,
+    known_type_outlives_obligations: Frozen<Vec<ty::PolyTypeOutlivesPredicate<'tcx>>>,
+    constraints: MirTypeckRegionConstraints<'tcx>,
+    deferred_closure_requirements: DeferredClosureRequirements<'tcx>,
+    deferred_opaque_type_errors: Vec<DeferredOpaqueTypeError<'tcx>>,
+    polonius_facts: Option<AllFacts<RustcFacts>>,
+    polonius_context: Option<PoloniusContext>,
+}
+
 /// Perform the actual borrow checking.
 ///
 /// Use `consumer_options: None` for the default behavior of returning
@@ -296,11 +313,11 @@ impl<'tcx> ClosureOutlivesSubjectTy<'tcx> {
 ///
 /// For nested bodies this should only be called through `root_cx.get_or_insert_nested`.
 #[instrument(skip(root_cx), level = "debug")]
-fn do_mir_borrowck<'tcx>(
+fn start_do_mir_borrowck<'tcx>(
     root_cx: &mut BorrowCheckRootCtxt<'tcx>,
     def: LocalDefId,
     consumer_options: Option<ConsumerOptions>,
-) -> (PropagatedBorrowCheckResults<'tcx>, Option<Box<BodyWithBorrowckFacts<'tcx>>>) {
+) -> BorrowckState<'tcx> {
     let tcx = root_cx.tcx;
     let infcx = BorrowckInferCtxt::new(tcx, def);
     let (input_body, promoted) = tcx.mir_promoted(def);
@@ -321,6 +338,7 @@ fn do_mir_borrowck<'tcx>(
     let body = &body_owned; // no further changes
 
     let location_table = PoloniusLocationTable::new(body);
+    let location_map = Rc::new(DenseLocationMap::new(body));
 
     let move_data = MoveData::gather_moves(body, tcx, |_| true);
 
@@ -331,6 +349,80 @@ fn do_mir_borrowck<'tcx>(
     let locals_are_invalidated_at_exit = tcx.hir_body_owner_kind(def).is_fn_or_closure();
     let borrow_set = BorrowSet::build(tcx, body, locals_are_invalidated_at_exit, &move_data);
 
+    let is_polonius_legacy_enabled = infcx.tcx.sess.opts.unstable_opts.polonius.is_legacy_enabled();
+    let polonius_input = consumer_options.map(|c| c.polonius_input()).unwrap_or_default()
+        || is_polonius_legacy_enabled;
+    let mut polonius_facts =
+        (polonius_input || PoloniusFacts::enabled(infcx.tcx)).then_some(PoloniusFacts::default());
+
+    // Run the MIR type-checker.
+    let MirTypeckResults {
+        constraints,
+        universal_region_relations,
+        region_bound_pairs,
+        known_type_outlives_obligations,
+        deferred_closure_requirements,
+        polonius_context,
+    } = type_check::type_check(
+        root_cx,
+        &infcx,
+        &body,
+        &promoted,
+        universal_regions,
+        &location_table,
+        &borrow_set,
+        &mut polonius_facts,
+        flow_inits,
+        &move_data,
+        Rc::clone(&location_map),
+    );
+
+    BorrowckState {
+        infcx,
+        body_owned,
+        promoted,
+        move_data,
+        borrow_set,
+        location_table,
+        location_map,
+        universal_region_relations,
+        region_bound_pairs,
+        known_type_outlives_obligations,
+        constraints,
+        deferred_closure_requirements,
+        deferred_opaque_type_errors: Default::default(),
+        polonius_facts,
+        polonius_context,
+    }
+}
+
+fn resume_do_mir_borrowck<'tcx>(
+    root_cx: &mut BorrowCheckRootCtxt<'tcx>,
+    consumer_options: Option<ConsumerOptions>,
+    BorrowckState {
+        infcx,
+        body_owned,
+        promoted,
+        move_data,
+        borrow_set,
+        location_table,
+        location_map,
+        universal_region_relations,
+        region_bound_pairs: _,
+        known_type_outlives_obligations: _,
+        constraints,
+        deferred_closure_requirements,
+        deferred_opaque_type_errors,
+        polonius_facts,
+        polonius_context,
+    }: BorrowckState<'tcx>,
+) -> (PropagatedBorrowCheckResults<'tcx>, Option<Box<BodyWithBorrowckFacts<'tcx>>>) {
+    assert!(!infcx.has_opaque_types_in_storage());
+    assert!(deferred_closure_requirements.is_empty());
+    let tcx = root_cx.tcx;
+    let body = &body_owned;
+    let def = body.source.def_id().expect_local();
+
     // Compute non-lexical lifetimes.
     let nll::NllOutput {
         regioncx,
@@ -340,18 +432,23 @@ fn do_mir_borrowck<'tcx>(
         nll_errors,
         polonius_diagnostics,
     } = nll::compute_regions(
-        root_cx,
         &infcx,
-        universal_regions,
         body,
-        &promoted,
         &location_table,
-        flow_inits,
         &move_data,
         &borrow_set,
+        location_map,
         consumer_options,
+        universal_region_relations,
+        constraints,
+        polonius_facts,
+        polonius_context,
     );
 
+    if nll_errors.has_errors().is_none() && !deferred_opaque_type_errors.is_empty() {
+        regioncx.emit_deferred_opaque_type_errors(root_cx, &infcx, deferred_opaque_type_errors);
+    }
+
     // Dump MIR results into a file, if that is enabled. This lets us
     // write unit-tests, as well as helping with debugging.
     nll::dump_nll_mir(&infcx, body, &regioncx, &opt_closure_req, &borrow_set);
@@ -577,12 +674,7 @@ pub(crate) struct BorrowckInferCtxt<'tcx> {
 
 impl<'tcx> BorrowckInferCtxt<'tcx> {
     pub(crate) fn new(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> Self {
-        let typing_mode = if tcx.use_typing_mode_borrowck() {
-            TypingMode::borrowck(tcx, def_id)
-        } else {
-            TypingMode::analysis_in_body(tcx, def_id)
-        };
-        let infcx = tcx.infer_ctxt().build(typing_mode);
+        let infcx = tcx.infer_ctxt().build(TypingMode::borrowck(tcx, def_id));
         let param_env = tcx.param_env(def_id);
         BorrowckInferCtxt { infcx, reg_var_to_origin: RefCell::new(Default::default()), param_env }
     }

compiler/rustc_borrowck/src/nll.rs

@@ -5,33 +5,33 @@ use std::path::PathBuf;
 use std::rc::Rc;
 use std::str::FromStr;
 
-use polonius_engine::{Algorithm, Output};
+use polonius_engine::{Algorithm, AllFacts, Output};
+use rustc_data_structures::frozen::Frozen;
 use rustc_index::IndexSlice;
 use rustc_middle::mir::pretty::{PrettyPrintMirOptions, dump_mir_with_options};
 use rustc_middle::mir::{Body, PassWhere, Promoted, create_dump_file, dump_enabled, dump_mir};
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{self, TyCtxt};
-use rustc_mir_dataflow::ResultsCursor;
-use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
 use rustc_mir_dataflow::move_paths::MoveData;
 use rustc_mir_dataflow::points::DenseLocationMap;
 use rustc_session::config::MirIncludeSpans;
 use rustc_span::sym;
 use tracing::{debug, instrument};
 
 use crate::borrow_set::BorrowSet;
-use crate::consumers::ConsumerOptions;
+use crate::consumers::{ConsumerOptions, RustcFacts};
 use crate::diagnostics::RegionErrors;
-use crate::polonius::PoloniusDiagnosticsContext;
 use crate::polonius::legacy::{
     PoloniusFacts, PoloniusFactsExt, PoloniusLocationTable, PoloniusOutput,
 };
+use crate::polonius::{PoloniusContext, PoloniusDiagnosticsContext};
 use crate::region_infer::RegionInferenceContext;
-use crate::type_check::{self, MirTypeckResults};
+use crate::type_check::MirTypeckRegionConstraints;
+use crate::type_check::free_region_relations::UniversalRegionRelations;
 use crate::universal_regions::UniversalRegions;
 use crate::{
-    BorrowCheckRootCtxt, BorrowckInferCtxt, ClosureOutlivesSubject, ClosureRegionRequirements,
-    polonius, renumber,
+    BorrowckInferCtxt, BorrowckState, ClosureOutlivesSubject, ClosureRegionRequirements, polonius,
+    renumber,
 };
 
 /// The output of `nll::compute_regions`. This includes the computed `RegionInferenceContext`, any
@@ -72,51 +72,47 @@ pub(crate) fn replace_regions_in_mir<'tcx>(
     universal_regions
 }
 
-/// Computes the (non-lexical) regions from the input MIR.
+pub(crate) fn compute_closure_requirements_modulo_opaques<'tcx>(
+    partial_result: &BorrowckState<'tcx>,
+) -> Option<ClosureRegionRequirements<'tcx>> {
+    let BorrowckState {
+        infcx,
+        body_owned,
+        location_map,
+        universal_region_relations,
+        constraints,
+        ..
+    } = partial_result;
+
+    let mut regioncx = RegionInferenceContext::new(
+        &infcx,
+        constraints.clone(),
+        universal_region_relations.clone(),
+        location_map.clone(),
+    );
+    let (closure_region_requirements, _nll_errors) = regioncx.solve(infcx, &body_owned, None);
+    closure_region_requirements
+}
+
+/// Computes and checks the region graph for the given constraints.
 ///
 /// This may result in errors being reported.
-pub(crate) fn compute_regions<'a, 'tcx>(
-    root_cx: &mut BorrowCheckRootCtxt<'tcx>,
+pub(crate) fn compute_regions<'tcx>(
     infcx: &BorrowckInferCtxt<'tcx>,
-    universal_regions: UniversalRegions<'tcx>,
     body: &Body<'tcx>,
-    promoted: &IndexSlice<Promoted, Body<'tcx>>,
     location_table: &PoloniusLocationTable,
-    flow_inits: ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>,
     move_data: &MoveData<'tcx>,
     borrow_set: &BorrowSet<'tcx>,
+    location_map: Rc<DenseLocationMap>,
     consumer_options: Option<ConsumerOptions>,
+    universal_region_relations: Frozen<UniversalRegionRelations<'tcx>>,
+    constraints: MirTypeckRegionConstraints<'tcx>,
+    mut polonius_facts: Option<AllFacts<RustcFacts>>,
+    polonius_context: Option<PoloniusContext>,
 ) -> NllOutput<'tcx> {
     let is_polonius_legacy_enabled = infcx.tcx.sess.opts.unstable_opts.polonius.is_legacy_enabled();
-    let polonius_input = consumer_options.map(|c| c.polonius_input()).unwrap_or_default()
-        || is_polonius_legacy_enabled;
     let polonius_output = consumer_options.map(|c| c.polonius_output()).unwrap_or_default()
         || is_polonius_legacy_enabled;
-    let mut polonius_facts =
-        (polonius_input || PoloniusFacts::enabled(infcx.tcx)).then_some(PoloniusFacts::default());
-
-    let location_map = Rc::new(DenseLocationMap::new(body));
-
-    // Run the MIR type-checker.
-    let MirTypeckResults {
-        constraints,
-        universal_region_relations,
-        opaque_type_values,
-        polonius_context,
-    } = type_check::type_check(
-        root_cx,
-        infcx,
-        body,
-        promoted,
-        universal_regions,
-        location_table,
-        borrow_set,
-        &mut polonius_facts,
-        flow_inits,
-        move_data,
-        Rc::clone(&location_map),
-    );
-
     // If requested, emit legacy polonius facts.
     polonius::legacy::emit_facts(
         &mut polonius_facts,
@@ -164,12 +160,9 @@ pub(crate) fn compute_regions<'a, 'tcx>(
         regioncx.solve(infcx, body, polonius_output.clone());
 
     if let Some(guar) = nll_errors.has_errors() {
-        // Suppress unhelpful extra errors in `infer_opaque_types`.
         infcx.set_tainted_by_errors(guar);
     }
 
-    regioncx.infer_opaque_types(root_cx, infcx, opaque_type_values);
-
     NllOutput {
         regioncx,
         polonius_input: polonius_facts.map(Box::new),

compiler/rustc_borrowck/src/region_infer/reverse_sccs.rs

@@ -5,7 +5,6 @@ use rustc_data_structures::graph;
 use rustc_data_structures::graph::vec_graph::VecGraph;
 use rustc_middle::ty::RegionVid;
 
-use crate::RegionInferenceContext;
 use crate::constraints::ConstraintSccIndex;
 use crate::region_infer::ConstraintSccs;
 use crate::universal_regions::UniversalRegions;
@@ -57,15 +56,3 @@ impl ReverseSccGraph {
             .filter(move |r| duplicates.insert(*r))
     }
 }
-
-impl RegionInferenceContext<'_> {
-    /// Compute the reverse SCC-based constraint graph (lazily).
-    pub(super) fn compute_reverse_scc_graph(&mut self) {
-        if self.rev_scc_graph.is_some() {
-            return;
-        }
-
-        self.rev_scc_graph =
-            Some(ReverseSccGraph::compute(&self.constraint_sccs, self.universal_regions()));
-    }
-}

compiler/rustc_borrowck/src/region_infer/values.rs

@@ -37,6 +37,7 @@ pub(crate) enum RegionElement {
 
 /// Records the CFG locations where each region is live. When we initially compute liveness, we use
 /// an interval matrix storing liveness ranges for each region-vid.
+#[derive(Clone)]
 pub(crate) struct LivenessValues {
     /// The map from locations to points.
     location_map: Rc<DenseLocationMap>,
@@ -193,7 +194,7 @@ impl LivenessValues {
 /// Maps from `ty::PlaceholderRegion` values that are used in the rest of
 /// rustc to the internal `PlaceholderIndex` values that are used in
 /// NLL.
-#[derive(Debug, Default)]
+#[derive(Clone, Debug, Default)]
 pub(crate) struct PlaceholderIndices {
     indices: FxIndexSet<ty::PlaceholderRegion>,
 }

compiler/rustc_borrowck/src/root_cx.rs

@@ -2,19 +2,28 @@ use rustc_abi::FieldIdx;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::def_id::LocalDefId;
 use rustc_middle::bug;
-use rustc_middle::ty::{OpaqueHiddenType, Ty, TyCtxt, TypeVisitableExt};
+use rustc_middle::ty::{EarlyBinder, OpaqueHiddenType, Ty, TyCtxt, TypeVisitableExt};
 use rustc_span::ErrorGuaranteed;
 use smallvec::SmallVec;
 
-use crate::{ClosureRegionRequirements, ConcreteOpaqueTypes, PropagatedBorrowCheckResults};
+use crate::consumers::{BodyWithBorrowckFacts, ConsumerOptions};
+use crate::nll::compute_closure_requirements_modulo_opaques;
+use crate::type_check::apply_closure_requirements_considering_opaques;
+use crate::{
+    BorrowckState, ClosureRegionRequirements, ConcreteOpaqueTypes, PropagatedBorrowCheckResults,
+    resume_do_mir_borrowck, start_do_mir_borrowck,
+};
 
 /// The shared context used by both the root as well as all its nested
 /// items.
 pub(super) struct BorrowCheckRootCtxt<'tcx> {
     pub tcx: TyCtxt<'tcx>,
     root_def_id: LocalDefId,
     concrete_opaque_types: ConcreteOpaqueTypes<'tcx>,
-    nested_bodies: FxHashMap<LocalDefId, PropagatedBorrowCheckResults<'tcx>>,
+    partial_results: FxHashMap<LocalDefId, Option<BorrowckState<'tcx>>>,
+    closure_requirements_modulo_opaques:
+        FxHashMap<LocalDefId, Option<ClosureRegionRequirements<'tcx>>>,
+    final_results: FxHashMap<LocalDefId, PropagatedBorrowCheckResults<'tcx>>,
     tainted_by_errors: Option<ErrorGuaranteed>,
 }
 
@@ -24,11 +33,17 @@ impl<'tcx> BorrowCheckRootCtxt<'tcx> {
             tcx,
             root_def_id,
             concrete_opaque_types: Default::default(),
-            nested_bodies: Default::default(),
+            partial_results: Default::default(),
+            closure_requirements_modulo_opaques: Default::default(),
+            final_results: Default::default(),
             tainted_by_errors: None,
         }
     }
 
+    pub(super) fn root_def_id(&self) -> LocalDefId {
+        self.root_def_id
+    }
+
     /// Collect all defining uses of opaque types inside of this typeck root. This
     /// expects the hidden type to be mapped to the definition parameters of the opaque
     /// and errors if we end up with distinct hidden types.
@@ -58,47 +73,144 @@ impl<'tcx> BorrowCheckRootCtxt<'tcx> {
         }
     }
 
+    pub(super) fn get_concrete_opaque_type(
+        &mut self,
+        def_id: LocalDefId,
+    ) -> Option<EarlyBinder<'tcx, OpaqueHiddenType<'tcx>>> {
+        self.concrete_opaque_types.0.get(&def_id).map(|ty| EarlyBinder::bind(*ty))
+    }
+
     pub(super) fn set_tainted_by_errors(&mut self, guar: ErrorGuaranteed) {
         self.tainted_by_errors = Some(guar);
     }
 
-    pub(super) fn get_or_insert_nested(
-        &mut self,
-        def_id: LocalDefId,
-    ) -> &PropagatedBorrowCheckResults<'tcx> {
+    fn compute_partial_results(&mut self, def_id: LocalDefId) {
+        debug_assert_eq!(
+            self.tcx.typeck_root_def_id(def_id.to_def_id()),
+            self.root_def_id.to_def_id()
+        );
+        if !self.partial_results.contains_key(&def_id) {
+            let result = start_do_mir_borrowck(self, def_id, None);
+            // We only need to store the partial result if it depends on opaque types
+            // or depends on a nested item which does.
+            let relies_on_opaques = result.infcx.has_opaque_types_in_storage()
+                || !result.deferred_closure_requirements.is_empty();
+
+            let to_insert = if !relies_on_opaques {
+                let final_result = resume_do_mir_borrowck(self, None, result).0;
+                if self.final_results.insert(def_id, final_result).is_some() {
+                    bug!("unexpected previous final result for {def_id:?}");
+                }
+                None
+            } else {
+                Some(result)
+            };
+
+            if self.partial_results.insert(def_id, to_insert).is_some() {
+                bug!("unexpected previous partial result for: {def_id:?}")
+            }
+        }
+    }
+
+    fn get_or_insert_final(&mut self, def_id: LocalDefId) -> &PropagatedBorrowCheckResults<'tcx> {
         debug_assert_eq!(
             self.tcx.typeck_root_def_id(def_id.to_def_id()),
             self.root_def_id.to_def_id()
         );
-        if !self.nested_bodies.contains_key(&def_id) {
-            let result = super::do_mir_borrowck(self, def_id, None).0;
-            if let Some(prev) = self.nested_bodies.insert(def_id, result) {
+        if !self.final_results.contains_key(&def_id) {
+            let mut yield_do_mir_borrowck = self.partial_results.remove(&def_id).unwrap().unwrap();
+            self.handle_opaque_type_uses(&mut yield_do_mir_borrowck);
+            apply_closure_requirements_considering_opaques(self, &mut yield_do_mir_borrowck);
+            let result = resume_do_mir_borrowck(self, None, yield_do_mir_borrowck).0;
+            if let Some(prev) = self.final_results.insert(def_id, result) {
                 bug!("unexpected previous nested body: {prev:?}");
             }
         }
 
-        self.nested_bodies.get(&def_id).unwrap()
+        self.final_results.get(&def_id).unwrap()
     }
 
-    pub(super) fn closure_requirements(
+    pub(crate) fn get_closure_requirements_considering_regions(
         &mut self,
-        nested_body_def_id: LocalDefId,
+        def_id: LocalDefId,
     ) -> &Option<ClosureRegionRequirements<'tcx>> {
-        &self.get_or_insert_nested(nested_body_def_id).closure_requirements
+        &self.get_or_insert_final(def_id).closure_requirements
+    }
+
+    /// Get the closure requirements of the nested body `def_id`. In case
+    /// this nested body relies on opaques, checking that the hidden type
+    /// matches the final definition may introduce new region constraints
+    /// we aren't considering yet.
+    ///
+    /// In these cases, we need to later add these requirements again, including
+    /// the constraints from opaque types.
+    pub(crate) fn get_closure_requirements_modulo_opaques(
+        &mut self,
+        def_id: LocalDefId,
+    ) -> (&Option<ClosureRegionRequirements<'tcx>>, bool) {
+        self.compute_partial_results(def_id);
+        // In case the nested item does not use any opaque types or depend on nested items
+        // which do, we eagerly compute its final result to avoid duplicate work.
+        if let Some(final_result) = self.final_results.get(&def_id) {
+            (&final_result.closure_requirements, false)
+        } else if self.closure_requirements_modulo_opaques.contains_key(&def_id) {
+            (self.closure_requirements_modulo_opaques.get(&def_id).unwrap(), true)
+        } else {
+            let partial_result = self.partial_results.get(&def_id).unwrap().as_ref().unwrap();
+            let modulo_opaques = compute_closure_requirements_modulo_opaques(partial_result);
+            self.closure_requirements_modulo_opaques.insert(def_id, modulo_opaques);
+            (self.closure_requirements_modulo_opaques.get(&def_id).unwrap(), true)
+        }
     }
 
     pub(super) fn used_mut_upvars(
         &mut self,
         nested_body_def_id: LocalDefId,
     ) -> &SmallVec<[FieldIdx; 8]> {
-        &self.get_or_insert_nested(nested_body_def_id).used_mut_upvars
+        &self.get_or_insert_final(nested_body_def_id).used_mut_upvars
     }
 
-    pub(super) fn finalize(self) -> Result<&'tcx ConcreteOpaqueTypes<'tcx>, ErrorGuaranteed> {
-        if let Some(guar) = self.tainted_by_errors {
+    /// The actual borrowck routine. This should only be called for the typeck root,
+    /// not for any nested bodies.
+    pub(super) fn borrowck_root(
+        mut self,
+        consumer_options: Option<ConsumerOptions>,
+    ) -> (
+        Result<&'tcx ConcreteOpaqueTypes<'tcx>, ErrorGuaranteed>,
+        Option<Box<BodyWithBorrowckFacts<'tcx>>>,
+    ) {
+        let root_def_id = self.root_def_id;
+        let mut yield_do_mir_borrowck =
+            start_do_mir_borrowck(&mut self, root_def_id, consumer_options);
+
+        self.handle_opaque_type_uses(&mut yield_do_mir_borrowck);
+        apply_closure_requirements_considering_opaques(&mut self, &mut yield_do_mir_borrowck);
+        let (PropagatedBorrowCheckResults { closure_requirements, used_mut_upvars }, consumer_data) =
+            resume_do_mir_borrowck(&mut self, consumer_options, yield_do_mir_borrowck);
+
+        // We need to manually borrowck all nested bodies from the HIR as
+        // we do not generate MIR for dead code. Not doing so causes us to
+        // never check closures in dead code.
+        let nested_bodies = self.tcx.nested_bodies_within(root_def_id);
+        for def_id in nested_bodies {
+            if !self.final_results.contains_key(&def_id) {
+                self.compute_partial_results(def_id);
+                let _ = self.get_or_insert_final(def_id);
+            }
+        }
+
+        #[allow(rustc::potential_query_instability)]
+        if cfg!(debug_assertions) {
+            assert!(closure_requirements.is_none());
+            assert!(used_mut_upvars.is_empty());
+            assert!(self.partial_results.values().all(|entry| entry.is_none()));
+        }
+
+        let result = if let Some(guar) = self.tainted_by_errors {
             Err(guar)
         } else {
-            Ok(self.tcx.arena.alloc(self.concrete_opaque_types))
-        }
+            Ok(&*self.tcx.arena.alloc(self.concrete_opaque_types))
+        };
+        (result, consumer_data)
     }
 }

compiler/rustc_borrowck/src/session_diagnostics.rs

@@ -1,7 +1,7 @@
 use rustc_errors::MultiSpan;
 use rustc_errors::codes::*;
 use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
-use rustc_middle::ty::{GenericArg, Ty};
+use rustc_middle::ty::Ty;
 use rustc_span::Span;
 
 use crate::diagnostics::RegionName;
@@ -294,19 +294,6 @@ pub(crate) struct MoveBorrow<'a> {
     pub borrow_span: Span,
 }
 
-#[derive(Diagnostic)]
-#[diag(borrowck_opaque_type_lifetime_mismatch)]
-pub(crate) struct LifetimeMismatchOpaqueParam<'tcx> {
-    pub arg: GenericArg<'tcx>,
-    pub prev: GenericArg<'tcx>,
-    #[primary_span]
-    #[label]
-    #[note]
-    pub span: Span,
-    #[label(borrowck_prev_lifetime_label)]
-    pub prev_span: Span,
-}
-
 #[derive(Subdiagnostic)]
 pub(crate) enum CaptureReasonLabel<'a> {
     #[label(borrowck_moved_due_to_call)]

compiler/rustc_borrowck/src/type_check/canonical.rs

@@ -2,8 +2,9 @@ use std::fmt;
 
 use rustc_errors::ErrorGuaranteed;
 use rustc_infer::infer::canonical::Canonical;
+use rustc_infer::infer::outlives::env::RegionBoundPairs;
 use rustc_middle::bug;
-use rustc_middle::mir::ConstraintCategory;
+use rustc_middle::mir::{Body, ConstraintCategory};
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable, Upcast};
 use rustc_span::Span;
 use rustc_span::def_id::DefId;
@@ -13,8 +14,70 @@ use rustc_trait_selection::traits::query::type_op::custom::CustomTypeOp;
 use rustc_trait_selection::traits::query::type_op::{self, TypeOpOutput};
 use tracing::{debug, instrument};
 
-use super::{Locations, NormalizeLocation, TypeChecker};
+use super::{Locations, MirTypeckRegionConstraints, NormalizeLocation, TypeChecker};
+use crate::BorrowckInferCtxt;
 use crate::diagnostics::ToUniverseInfo;
+use crate::type_check::constraint_conversion;
+use crate::universal_regions::UniversalRegions;
+
+#[instrument(skip(infcx, constraints, op), level = "trace")]
+pub(crate) fn fully_perform_op_raw<'tcx, R: fmt::Debug, Op>(
+    infcx: &BorrowckInferCtxt<'tcx>,
+    body: &Body<'tcx>,
+    universal_regions: &UniversalRegions<'tcx>,
+    region_bound_pairs: &RegionBoundPairs<'tcx>,
+    known_type_outlives_obligations: &[ty::PolyTypeOutlivesPredicate<'tcx>],
+    constraints: &mut MirTypeckRegionConstraints<'tcx>,
+    locations: Locations,
+    category: ConstraintCategory<'tcx>,
+    op: Op,
+) -> Result<R, ErrorGuaranteed>
+where
+    Op: type_op::TypeOp<'tcx, Output = R>,
+    Op::ErrorInfo: ToUniverseInfo<'tcx>,
+{
+    let old_universe = infcx.universe();
+
+    let TypeOpOutput { output, constraints: query_constraints, error_info } =
+        op.fully_perform(infcx, locations.span(body))?;
+    if cfg!(debug_assertions) {
+        let data = infcx.take_and_reset_region_constraints();
+        if !data.is_empty() {
+            panic!("leftover region constraints: {data:#?}");
+        }
+    }
+
+    debug!(?output, ?query_constraints);
+
+    if let Some(data) = query_constraints {
+        constraint_conversion::ConstraintConversion::new(
+            infcx,
+            universal_regions,
+            region_bound_pairs,
+            infcx.param_env,
+            known_type_outlives_obligations,
+            locations,
+            locations.span(body),
+            category,
+            constraints,
+        )
+        .convert_all(data);
+    }
+
+    // If the query has created new universes and errors are going to be emitted, register the
+    // cause of these new universes for improved diagnostics.
+    let universe = infcx.universe();
+    if old_universe != universe
+        && let Some(error_info) = error_info
+    {
+        let universe_info = error_info.to_universe_info(old_universe);
+        for u in (old_universe + 1)..=universe {
+            constraints.universe_causes.insert(u, universe_info.clone());
+        }
+    }
+
+    Ok(output)
+}
 
 impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
     /// Given some operation `op` that manipulates types, proves
@@ -27,7 +90,6 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
     /// **Any `rustc_infer::infer` operations that might generate region
     /// constraints should occur within this method so that those
     /// constraints can be properly localized!**
-    #[instrument(skip(self, op), level = "trace")]
     pub(super) fn fully_perform_op<R: fmt::Debug, Op>(
         &mut self,
         locations: Locations,
@@ -38,36 +100,17 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         Op: type_op::TypeOp<'tcx, Output = R>,
         Op::ErrorInfo: ToUniverseInfo<'tcx>,
     {
-        let old_universe = self.infcx.universe();
-
-        let TypeOpOutput { output, constraints, error_info } =
-            op.fully_perform(self.infcx, locations.span(self.body))?;
-        if cfg!(debug_assertions) {
-            let data = self.infcx.take_and_reset_region_constraints();
-            if !data.is_empty() {
-                panic!("leftover region constraints: {data:#?}");
-            }
-        }
-
-        debug!(?output, ?constraints);
-
-        if let Some(data) = constraints {
-            self.push_region_constraints(locations, category, data);
-        }
-
-        // If the query has created new universes and errors are going to be emitted, register the
-        // cause of these new universes for improved diagnostics.
-        let universe = self.infcx.universe();
-        if old_universe != universe
-            && let Some(error_info) = error_info
-        {
-            let universe_info = error_info.to_universe_info(old_universe);
-            for u in (old_universe + 1)..=universe {
-                self.constraints.universe_causes.insert(u, universe_info.clone());
-            }
-        }
-
-        Ok(output)
+        fully_perform_op_raw(
+            self.infcx,
+            self.body,
+            self.universal_regions,
+            self.region_bound_pairs,
+            self.known_type_outlives_obligations,
+            self.constraints,
+            locations,
+            category,
+            op,
+        )
     }
 
     pub(super) fn instantiate_canonical<T>(

compiler/rustc_borrowck/src/type_check/free_region_relations.rs

@@ -17,7 +17,7 @@ use type_op::TypeOpOutput;
 use crate::type_check::{Locations, MirTypeckRegionConstraints, constraint_conversion};
 use crate::universal_regions::UniversalRegions;
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub(crate) struct UniversalRegionRelations<'tcx> {
     pub(crate) universal_regions: UniversalRegions<'tcx>,
 
@@ -156,13 +156,6 @@ impl UniversalRegionRelations<'_> {
         self.outlives.contains(fr1, fr2)
     }
 
-    /// Returns `true` if fr1 is known to equal fr2.
-    ///
-    /// This will only ever be true for universally quantified regions.
-    pub(crate) fn equal(&self, fr1: RegionVid, fr2: RegionVid) -> bool {
-        self.outlives.contains(fr1, fr2) && self.outlives.contains(fr2, fr1)
-    }
-
     /// Returns a vector of free regions `x` such that `fr1: x` is
     /// known to hold.
     pub(crate) fn regions_outlived_by(&self, fr1: RegionVid) -> Vec<RegionVid> {

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -26,8 +26,8 @@ use rustc_middle::ty::adjustment::PointerCoercion;
 use rustc_middle::ty::cast::CastTy;
 use rustc_middle::ty::{
     self, Binder, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, CoroutineArgsExt,
-    Dynamic, GenericArgsRef, OpaqueHiddenType, OpaqueTypeKey, RegionVid, Ty, TyCtxt,
-    TypeVisitableExt, UserArgs, UserTypeAnnotationIndex, fold_regions,
+    Dynamic, GenericArgsRef, Ty, TyCtxt, TypeVisitableExt, UserArgs, UserTypeAnnotationIndex,
+    fold_regions,
 };
 use rustc_middle::{bug, span_bug};
 use rustc_mir_dataflow::ResultsCursor;
@@ -44,15 +44,16 @@ use tracing::{debug, instrument, trace};
 use crate::borrow_set::BorrowSet;
 use crate::constraints::{OutlivesConstraint, OutlivesConstraintSet};
 use crate::diagnostics::UniverseInfo;
-use crate::member_constraints::MemberConstraintSet;
 use crate::polonius::legacy::{PoloniusFacts, PoloniusLocationTable};
 use crate::polonius::{PoloniusContext, PoloniusLivenessContext};
 use crate::region_infer::TypeTest;
 use crate::region_infer::values::{LivenessValues, PlaceholderIndex, PlaceholderIndices};
 use crate::session_diagnostics::{MoveUnsized, SimdIntrinsicArgConst};
 use crate::type_check::free_region_relations::{CreateResult, UniversalRegionRelations};
 use crate::universal_regions::{DefiningTy, UniversalRegions};
-use crate::{BorrowCheckRootCtxt, BorrowckInferCtxt, path_utils};
+use crate::{
+    BorrowCheckRootCtxt, BorrowckInferCtxt, BorrowckState, DeferredClosureRequirements, path_utils,
+};
 
 macro_rules! span_mirbug {
     ($context:expr, $elem:expr, $($message:tt)*) => ({
@@ -69,12 +70,11 @@ macro_rules! span_mirbug {
     })
 }
 
-mod canonical;
+pub(crate) mod canonical;
 mod constraint_conversion;
 pub(crate) mod free_region_relations;
 mod input_output;
 pub(crate) mod liveness;
-mod opaque_types;
 mod relate_tys;
 
 /// Type checks the given `mir` in the context of the inference
@@ -118,7 +118,6 @@ pub(crate) fn type_check<'a, 'tcx>(
         placeholder_index_to_region: IndexVec::default(),
         liveness_constraints: LivenessValues::with_specific_points(Rc::clone(&location_map)),
         outlives_constraints: OutlivesConstraintSet::default(),
-        member_constraints: MemberConstraintSet::default(),
         type_tests: Vec::default(),
         universe_causes: FxIndexMap::default(),
     };
@@ -159,6 +158,7 @@ pub(crate) fn type_check<'a, 'tcx>(
         polonius_facts,
         borrow_set,
         constraints: &mut constraints,
+        deferred_closure_requirements: Default::default(),
         polonius_liveness,
     };
 
@@ -169,10 +169,10 @@ pub(crate) fn type_check<'a, 'tcx>(
 
     liveness::generate(&mut typeck, &location_map, flow_inits, move_data);
 
-    let opaque_type_values =
-        opaque_types::take_opaques_and_register_member_constraints(&mut typeck);
-
     // We're done with typeck, we can finalize the polonius liveness context for region inference.
+    //
+    // FIXME: Handling opaque type uses may introduce new regions. This likely has to be moved to
+    // a later point.
     let polonius_context = typeck.polonius_liveness.take().map(|liveness_context| {
         PoloniusContext::create_from_liveness(
             liveness_context,
@@ -181,10 +181,13 @@ pub(crate) fn type_check<'a, 'tcx>(
         )
     });
 
+    let deferred_closure_requirements = typeck.deferred_closure_requirements;
     MirTypeckResults {
         constraints,
         universal_region_relations,
-        opaque_type_values,
+        region_bound_pairs,
+        known_type_outlives_obligations,
+        deferred_closure_requirements,
         polonius_context,
     }
 }
@@ -224,6 +227,7 @@ struct TypeChecker<'a, 'tcx> {
     polonius_facts: &'a mut Option<PoloniusFacts>,
     borrow_set: &'a BorrowSet<'tcx>,
     constraints: &'a mut MirTypeckRegionConstraints<'tcx>,
+    deferred_closure_requirements: DeferredClosureRequirements<'tcx>,
     /// When using `-Zpolonius=next`, the liveness helper data used to create polonius constraints.
     polonius_liveness: Option<PoloniusLivenessContext>,
 }
@@ -233,12 +237,15 @@ struct TypeChecker<'a, 'tcx> {
 pub(crate) struct MirTypeckResults<'tcx> {
     pub(crate) constraints: MirTypeckRegionConstraints<'tcx>,
     pub(crate) universal_region_relations: Frozen<UniversalRegionRelations<'tcx>>,
-    pub(crate) opaque_type_values: FxIndexMap<OpaqueTypeKey<'tcx>, OpaqueHiddenType<'tcx>>,
+    pub(crate) region_bound_pairs: Frozen<RegionBoundPairs<'tcx>>,
+    pub(crate) known_type_outlives_obligations: Frozen<Vec<ty::PolyTypeOutlivesPredicate<'tcx>>>,
+    pub(crate) deferred_closure_requirements: DeferredClosureRequirements<'tcx>,
     pub(crate) polonius_context: Option<PoloniusContext>,
 }
 
 /// A collection of region constraints that must be satisfied for the
 /// program to be considered well-typed.
+#[derive(Clone)]
 pub(crate) struct MirTypeckRegionConstraints<'tcx> {
     /// Maps from a `ty::Placeholder` to the corresponding
     /// `PlaceholderIndex` bit that we will use for it.
@@ -265,8 +272,6 @@ pub(crate) struct MirTypeckRegionConstraints<'tcx> {
 
     pub(crate) outlives_constraints: OutlivesConstraintSet<'tcx>,
 
-    pub(crate) member_constraints: MemberConstraintSet<'tcx, RegionVid>,
-
     pub(crate) universe_causes: FxIndexMap<ty::UniverseIndex, UniverseInfo<'tcx>>,
 
     pub(crate) type_tests: Vec<TypeTest<'tcx>>,
@@ -275,7 +280,7 @@ pub(crate) struct MirTypeckRegionConstraints<'tcx> {
 impl<'tcx> MirTypeckRegionConstraints<'tcx> {
     /// Creates a `Region` for a given `PlaceholderRegion`, or returns the
     /// region that corresponds to a previously created one.
-    fn placeholder_region(
+    pub(crate) fn placeholder_region(
         &mut self,
         infcx: &InferCtxt<'tcx>,
         placeholder: ty::PlaceholderRegion,
@@ -368,14 +373,6 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         self.body
     }
 
-    fn to_region_vid(&mut self, r: ty::Region<'tcx>) -> RegionVid {
-        if let ty::RePlaceholder(placeholder) = r.kind() {
-            self.constraints.placeholder_region(self.infcx, placeholder).as_var()
-        } else {
-            self.universal_regions.to_region_vid(r)
-        }
-    }
-
     fn unsized_feature_enabled(&self) -> bool {
         let features = self.tcx().features();
         features.unsized_locals() || features.unsized_fn_params()
@@ -2502,7 +2499,13 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         args: GenericArgsRef<'tcx>,
         locations: Locations,
     ) -> ty::InstantiatedPredicates<'tcx> {
-        if let Some(closure_requirements) = &self.root_cx.closure_requirements(def_id) {
+        let (closure_requirements, needs_defer) =
+            self.root_cx.get_closure_requirements_modulo_opaques(def_id);
+        if needs_defer {
+            self.deferred_closure_requirements.push((def_id, args, locations));
+        }
+
+        if let Some(closure_requirements) = closure_requirements {
             constraint_conversion::ConstraintConversion::new(
                 self.infcx,
                 self.universal_regions,
@@ -2556,6 +2559,41 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
     }
 }
 
+pub(crate) fn apply_closure_requirements_considering_opaques<'tcx>(
+    root_cx: &mut BorrowCheckRootCtxt<'tcx>,
+    borrowck_state: &mut BorrowckState<'tcx>,
+) {
+    let BorrowckState {
+        infcx,
+        body_owned,
+        universal_region_relations,
+        region_bound_pairs,
+        known_type_outlives_obligations,
+        constraints,
+        deferred_closure_requirements,
+        ..
+    } = borrowck_state;
+
+    for (def_id, args, locations) in mem::take(deferred_closure_requirements).into_iter() {
+        if let Some(closure_requirements) =
+            root_cx.get_closure_requirements_considering_regions(def_id)
+        {
+            constraint_conversion::ConstraintConversion::new(
+                infcx,
+                &universal_region_relations.universal_regions,
+                region_bound_pairs,
+                infcx.param_env,
+                known_type_outlives_obligations,
+                locations,
+                body_owned.span,            // irrelevant; will be overridden.
+                ConstraintCategory::Boring, // same as above.
+                constraints,
+            )
+            .apply_closure_requirements(closure_requirements, def_id, args);
+        }
+    }
+}
+
 trait NormalizeLocation: fmt::Debug + Copy {
     fn to_locations(self) -> Locations;
 }

compiler/rustc_borrowck/src/universal_regions.rs

@@ -39,7 +39,7 @@ use tracing::{debug, instrument};
 use crate::BorrowckInferCtxt;
 use crate::renumber::RegionCtxt;
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub(crate) struct UniversalRegions<'tcx> {
     indices: UniversalRegionIndices<'tcx>,
 
@@ -199,7 +199,7 @@ impl<'tcx> DefiningTy<'tcx> {
     }
 }
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 struct UniversalRegionIndices<'tcx> {
     /// For those regions that may appear in the parameter environment
     /// ('static and early-bound regions), we maintain a map from the

compiler/rustc_data_structures/src/frozen.rs

@@ -46,7 +46,7 @@
 //!    Frozen::freeze(new_bar)`).
 
 /// An owned immutable value.
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub struct Frozen<T>(T);
 
 impl<T> Frozen<T> {

compiler/rustc_hir_typeck/src/callee.rs

@@ -14,7 +14,7 @@ use rustc_middle::ty::adjustment::{
 use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt, TypeVisitableExt};
 use rustc_middle::{bug, span_bug};
 use rustc_span::def_id::LocalDefId;
-use rustc_span::{Span, sym};
+use rustc_span::{Span, Symbol, sym};
 use rustc_trait_selection::error_reporting::traits::DefIdOrName;
 use rustc_trait_selection::infer::InferCtxtExt as _;
 use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt as _;
@@ -66,7 +66,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         arg_exprs: &'tcx [hir::Expr<'tcx>],
         expected: Expectation<'tcx>,
     ) -> Ty<'tcx> {
-        let original_callee_ty = match &callee_expr.kind {
+        let expr_ty = match &callee_expr.kind {
             hir::ExprKind::Path(hir::QPath::Resolved(..) | hir::QPath::TypeRelative(..)) => self
                 .check_expr_with_expectation_and_args(
                     callee_expr,
@@ -76,8 +76,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             _ => self.check_expr(callee_expr),
         };
 
-        let expr_ty = self.structurally_resolve_type(call_expr.span, original_callee_ty);
-
         let mut autoderef = self.autoderef(callee_expr.span, expr_ty);
         let mut result = None;
         while result.is_none() && autoderef.next().is_some() {
@@ -144,7 +142,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         autoderef: &Autoderef<'a, 'tcx>,
     ) -> Option<CallStep<'tcx>> {
         let adjusted_ty =
-            self.structurally_resolve_type(autoderef.span(), autoderef.final_ty(false));
+            self.try_structurally_resolve_type(autoderef.span(), autoderef.final_ty(false));
 
         // If the callee is a bare function or a closure, then we're all set.
         match *adjusted_ty.kind() {
@@ -241,6 +239,25 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 return None;
             }
 
+            // We only want to confirm a call step here if the infer var
+            // originated from the defining use of an opaque.
+            ty::Infer(ty::TyVar(vid))
+                if let Some(alias_ty) = self.find_sup_as_registered_opaque(vid) =>
+            {
+                return self
+                    .try_overloaded_call_traits_for_alias(call_expr, alias_ty, arg_exprs)
+                    .map(|(autoref, method)| {
+                        let mut adjustments = self.adjust_steps(autoderef);
+                        adjustments.extend(autoref);
+                        self.apply_adjustments(callee_expr, adjustments);
+                        CallStep::Overloaded(method)
+                    });
+            }
+
+            ty::Infer(_) => {
+                return None;
+            }
+
             ty::Error(_) => {
                 return None;
             }
@@ -305,40 +322,111 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         // Try the options that are least restrictive on the caller first.
         for (opt_trait_def_id, method_name, borrow) in call_trait_choices {
-            let Some(trait_def_id) = opt_trait_def_id else { continue };
+            if let Some(confirmed) = self.try_overloaded_call_trait(
+                call_expr,
+                adjusted_ty,
+                opt_arg_exprs,
+                opt_trait_def_id,
+                method_name,
+                borrow,
+            ) {
+                return Some(confirmed);
+            }
+        }
+
+        None
+    }
+
+    fn try_overloaded_call_trait(
+        &self,
+        call_expr: &hir::Expr<'_>,
+        call_ty: Ty<'tcx>,
+        opt_arg_exprs: Option<&'tcx [hir::Expr<'tcx>]>,
+        opt_trait_def_id: Option<DefId>,
+        method_name: Symbol,
+        borrow: bool,
+    ) -> Option<(Option<Adjustment<'tcx>>, MethodCallee<'tcx>)> {
+        let Some(trait_def_id) = opt_trait_def_id else {
+            return None;
+        };
+
+        let opt_input_type = opt_arg_exprs.map(|arg_exprs| {
+            Ty::new_tup_from_iter(self.tcx, arg_exprs.iter().map(|e| self.next_ty_var(e.span)))
+        });
+
+        let Some(ok) = self.lookup_method_for_operator(
+            self.misc(call_expr.span),
+            method_name,
+            trait_def_id,
+            call_ty,
+            opt_input_type,
+        ) else {
+            return None;
+        };
+        let method = self.register_infer_ok_obligations(ok);
+        let mut autoref = None;
+        if borrow {
+            // Check for &self vs &mut self in the method signature. Since this is either
+            // the Fn or FnMut trait, it should be one of those.
+            let ty::Ref(_, _, mutbl) = *method.sig.inputs()[0].kind() else {
+                bug!("Expected `FnMut`/`Fn` to take receiver by-ref/by-mut")
+            };
 
-            let opt_input_type = opt_arg_exprs.map(|arg_exprs| {
-                Ty::new_tup_from_iter(self.tcx, arg_exprs.iter().map(|e| self.next_ty_var(e.span)))
+            // For initial two-phase borrow
+            // deployment, conservatively omit
+            // overloaded function call ops.
+            let mutbl = AutoBorrowMutability::new(mutbl, AllowTwoPhase::No);
+
+            autoref = Some(Adjustment {
+                kind: Adjust::Borrow(AutoBorrow::Ref(mutbl)),
+                target: method.sig.inputs()[0],
             });
+        }
 
-            if let Some(ok) = self.lookup_method_for_operator(
-                self.misc(call_expr.span),
-                method_name,
-                trait_def_id,
-                adjusted_ty,
-                opt_input_type,
-            ) {
-                let method = self.register_infer_ok_obligations(ok);
-                let mut autoref = None;
-                if borrow {
-                    // Check for &self vs &mut self in the method signature. Since this is either
-                    // the Fn or FnMut trait, it should be one of those.
-                    let ty::Ref(_, _, mutbl) = method.sig.inputs()[0].kind() else {
-                        bug!("Expected `FnMut`/`Fn` to take receiver by-ref/by-mut")
-                    };
-
-                    // For initial two-phase borrow
-                    // deployment, conservatively omit
-                    // overloaded function call ops.
-                    let mutbl = AutoBorrowMutability::new(*mutbl, AllowTwoPhase::No);
-
-                    autoref = Some(Adjustment {
-                        kind: Adjust::Borrow(AutoBorrow::Ref(mutbl)),
-                        target: method.sig.inputs()[0],
-                    });
-                }
+        Some((autoref, method))
+    }
 
-                return Some((autoref, method));
+    fn try_overloaded_call_traits_for_alias(
+        &self,
+        call_expr: &'tcx hir::Expr<'tcx>,
+        alias_ty: ty::AliasTy<'tcx>,
+        arg_exprs: &'tcx [rustc_hir::Expr<'tcx>],
+    ) -> Option<(Option<Adjustment<'tcx>>, MethodCallee<'tcx>)> {
+        let call_ty = alias_ty.to_ty(self.tcx);
+
+        let call_traits = [
+            (self.tcx.lang_items().fn_trait(), sym::call, true),
+            (self.tcx.lang_items().fn_mut_trait(), sym::call_mut, true),
+            (self.tcx.lang_items().fn_once_trait(), sym::call_once, false),
+            (self.tcx.lang_items().async_fn_trait(), sym::async_call, true),
+            (self.tcx.lang_items().async_fn_mut_trait(), sym::async_call_mut, true),
+            (self.tcx.lang_items().async_fn_once_trait(), sym::async_call_once, false),
+        ];
+        // We only want to try a call trait if it shows up in the bounds
+        // of the opaque. We confirm the first one that shows up in the
+        // bounds list, which can lead to inference weirdness but doesn't
+        // matter today.
+        for clause in
+            self.tcx.item_self_bounds(alias_ty.def_id).iter_instantiated(self.tcx, alias_ty.args)
+        {
+            let Some(poly_trait_ref) = clause.as_trait_clause() else {
+                continue;
+            };
+
+            if let Some(&(opt_trait_def_id, method_name, borrow)) =
+                call_traits.iter().find(|(trait_def_id, _, _)| {
+                    trait_def_id.is_some_and(|trait_def_id| trait_def_id == poly_trait_ref.def_id())
+                })
+                && let Some(confirmed) = self.try_overloaded_call_trait(
+                    call_expr,
+                    call_ty,
+                    Some(arg_exprs),
+                    opt_trait_def_id,
+                    method_name,
+                    borrow,
+                )
+            {
+                return Some(confirmed);
             }
         }
 

compiler/rustc_hir_typeck/src/expr.rs

@@ -1591,7 +1591,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ) -> Ty<'tcx> {
         let rcvr_t = self.check_expr(rcvr);
         // no need to check for bot/err -- callee does that
-        let rcvr_t = self.structurally_resolve_type(rcvr.span, rcvr_t);
+        let rcvr_t = self.try_structurally_resolve_type(rcvr.span, rcvr_t);
 
         match self.lookup_method(rcvr_t, segment, segment.ident.span, expr, rcvr, args) {
             Ok(method) => {

compiler/rustc_hir_typeck/src/fn_ctxt/inspect_obligations.rs

@@ -1,7 +1,6 @@
 //! A utility module to inspect currently ambiguous obligations in the current context.
 
 use rustc_infer::traits::{self, ObligationCause, PredicateObligations};
-use rustc_middle::traits::solve::GoalSource;
 use rustc_middle::ty::{self, Ty, TypeVisitableExt};
 use rustc_span::Span;
 use rustc_trait_selection::solve::inspect::{
@@ -119,21 +118,7 @@ impl<'a, 'tcx> ProofTreeVisitor<'tcx> for NestedObligationsForSelfTy<'a, 'tcx> {
     fn visit_goal(&mut self, inspect_goal: &InspectGoal<'_, 'tcx>) {
         let tcx = self.fcx.tcx;
         let goal = inspect_goal.goal();
-        if self.fcx.predicate_has_self_ty(goal.predicate, self.self_ty)
-            // We do not push the instantiated forms of goals as it would cause any
-            // aliases referencing bound vars to go from having escaping bound vars to
-            // being able to be normalized to an inference variable.
-            //
-            // This is mostly just a hack as arbitrary nested goals could still contain
-            // such aliases while having a different `GoalSource`. Closure signature inference
-            // however can't really handle *every* higher ranked `Fn` goal also being present
-            // in the form of `?c: Fn<(<?x as Trait<'!a>>::Assoc)`.
-            //
-            // This also just better matches the behaviour of the old solver where we do not
-            // encounter instantiated forms of goals, only nested goals that referred to bound
-            // vars from instantiated goals.
-            && !matches!(inspect_goal.source(), GoalSource::InstantiateHigherRanked)
-        {
+        if self.fcx.predicate_has_self_ty(goal.predicate, self.self_ty) {
             self.obligations_for_self_ty.push(traits::Obligation::new(
                 tcx,
                 self.root_cause.clone(),

compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs

@@ -19,7 +19,6 @@ use rustc_middle::ty::{self, Const, Ty, TyCtxt, TypeVisitableExt};
 use rustc_session::Session;
 use rustc_span::{self, DUMMY_SP, ErrorGuaranteed, Ident, Span, sym};
 use rustc_trait_selection::error_reporting::TypeErrCtxt;
-use rustc_trait_selection::error_reporting::infer::sub_relations::SubRelations;
 use rustc_trait_selection::traits::{ObligationCause, ObligationCauseCode, ObligationCtxt};
 
 use crate::coercion::DynamicCoerceMany;
@@ -177,14 +176,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ///
     /// [`InferCtxtErrorExt::err_ctxt`]: rustc_trait_selection::error_reporting::InferCtxtErrorExt::err_ctxt
     pub(crate) fn err_ctxt(&'a self) -> TypeErrCtxt<'a, 'tcx> {
-        let mut sub_relations = SubRelations::default();
-        sub_relations.add_constraints(
-            self,
-            self.fulfillment_cx.borrow_mut().pending_obligations().iter().map(|o| o.predicate),
-        );
         TypeErrCtxt {
             infcx: &self.infcx,
-            sub_relations: RefCell::new(sub_relations),
             typeck_results: Some(self.typeck_results.borrow()),
             fallback_has_occurred: self.fallback_has_occurred.get(),
             normalize_fn_sig: Box::new(|fn_sig| {

compiler/rustc_hir_typeck/src/lib.rs

@@ -226,6 +226,10 @@ fn typeck_with_inspect<'tcx>(
 
     fcx.select_obligations_where_possible(|_| {});
 
+    if fcx.next_trait_solver() {
+        fcx.handle_opaque_type_uses_next();
+    }
+
     debug!(pending_obligations = ?fcx.fulfillment_cx.borrow().pending_obligations());
 
     // This must be the last thing before `report_ambiguity_errors`.

compiler/rustc_hir_typeck/src/method/confirm.rs

@@ -190,7 +190,7 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
         assert_eq!(n, pick.autoderefs);
 
         let mut adjustments = self.adjust_steps(&autoderef);
-        let mut target = self.structurally_resolve_type(autoderef.span(), ty);
+        let mut target = self.try_structurally_resolve_type(autoderef.span(), ty);
 
         match pick.autoref_or_ptr_adjustment {
             Some(probe::AutorefOrPtrAdjustment::Autoref { mutbl, unsize }) => {

compiler/rustc_hir_typeck/src/method/probe.rs

@@ -11,7 +11,7 @@ use rustc_hir::HirId;
 use rustc_hir::def::DefKind;
 use rustc_hir_analysis::autoderef::{self, Autoderef};
 use rustc_infer::infer::canonical::{Canonical, OriginalQueryValues, QueryResponse};
-use rustc_infer::infer::{self, DefineOpaqueTypes, InferOk, TyCtxtInferExt};
+use rustc_infer::infer::{self, DefineOpaqueTypes, InferCtxt, InferOk, TyCtxtInferExt};
 use rustc_infer::traits::ObligationCauseCode;
 use rustc_middle::middle::stability;
 use rustc_middle::query::Providers;
@@ -443,7 +443,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // If we encountered an `_` type or an error type during autoderef, this is
         // ambiguous.
         if let Some(bad_ty) = &steps.opt_bad_ty {
-            if is_suggestion.0 {
+            // Ended up encountering a type variable when doing autoderef,
+            // but it may not be a type variable after processing obligations
+            // in our local `FnCtxt`, so don't call `structurally_resolve_type`.
+            let ty = &bad_ty.ty;
+            let ty = self
+                .probe_instantiate_query_response(span, &orig_values, ty)
+                .unwrap_or_else(|_| span_bug!(span, "instantiating {:?} failed?", ty));
+            if bad_ty.is_opaque_type || final_ty_is_opaque(&self.infcx, ty.value) {
+                // FIXME(-Znext-solver): This isn't really what we want :<
+                assert!(self.tcx.next_trait_solver_globally());
+            } else if is_suggestion.0 {
                 // Ambiguity was encountered during a suggestion. There's really
                 // not much use in suggesting methods in this case.
                 return Err(MethodError::NoMatch(NoMatchData {
@@ -469,13 +479,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     },
                 );
             } else {
-                // Ended up encountering a type variable when doing autoderef,
-                // but it may not be a type variable after processing obligations
-                // in our local `FnCtxt`, so don't call `structurally_resolve_type`.
-                let ty = &bad_ty.ty;
-                let ty = self
-                    .probe_instantiate_query_response(span, &orig_values, ty)
-                    .unwrap_or_else(|_| span_bug!(span, "instantiating {:?} failed?", ty));
                 let ty = self.resolve_vars_if_possible(ty.value);
                 let guar = match *ty.kind() {
                     ty::Infer(ty::TyVar(_)) => {
@@ -583,60 +586,75 @@ fn method_autoderef_steps<'tcx>(
     let mut reached_raw_pointer = false;
     let arbitrary_self_types_enabled =
         tcx.features().arbitrary_self_types() || tcx.features().arbitrary_self_types_pointers();
-    let (mut steps, reached_recursion_limit): (Vec<_>, bool) = if arbitrary_self_types_enabled {
-        let reachable_via_deref =
-            autoderef_via_deref.by_ref().map(|_| true).chain(std::iter::repeat(false));
-
-        let mut autoderef_via_receiver =
-            Autoderef::new(infcx, param_env, hir::def_id::CRATE_DEF_ID, DUMMY_SP, self_ty)
-                .include_raw_pointers()
-                .use_receiver_trait()
-                .silence_errors();
-        let steps = autoderef_via_receiver
-            .by_ref()
-            .zip(reachable_via_deref)
-            .map(|((ty, d), reachable_via_deref)| {
-                let step = CandidateStep {
-                    self_ty: infcx
-                        .make_query_response_ignoring_pending_obligations(inference_vars, ty),
-                    autoderefs: d,
-                    from_unsafe_deref: reached_raw_pointer,
-                    unsize: false,
-                    reachable_via_deref,
-                };
-                if ty.is_raw_ptr() {
-                    // all the subsequent steps will be from_unsafe_deref
-                    reached_raw_pointer = true;
-                }
-                step
-            })
-            .collect();
-        (steps, autoderef_via_receiver.reached_recursion_limit())
-    } else {
-        let steps = autoderef_via_deref
-            .by_ref()
-            .map(|(ty, d)| {
-                let step = CandidateStep {
-                    self_ty: infcx
-                        .make_query_response_ignoring_pending_obligations(inference_vars, ty),
-                    autoderefs: d,
-                    from_unsafe_deref: reached_raw_pointer,
-                    unsize: false,
-                    reachable_via_deref: true,
-                };
-                if ty.is_raw_ptr() {
-                    // all the subsequent steps will be from_unsafe_deref
-                    reached_raw_pointer = true;
-                }
-                step
-            })
-            .collect();
-        (steps, autoderef_via_deref.reached_recursion_limit())
-    };
-    let final_ty = autoderef_via_deref.final_ty(true);
+    let (mut steps, final_ty, reached_recursion_limit): (Vec<_>, _, _) =
+        if arbitrary_self_types_enabled {
+            let reachable_via_deref =
+                autoderef_via_deref.by_ref().map(|_| true).chain(std::iter::repeat(false));
+
+            let mut autoderef_via_receiver =
+                Autoderef::new(infcx, param_env, hir::def_id::CRATE_DEF_ID, DUMMY_SP, self_ty)
+                    .include_raw_pointers()
+                    .use_receiver_trait()
+                    .silence_errors();
+            let steps = autoderef_via_receiver
+                .by_ref()
+                .zip(reachable_via_deref)
+                .map(|((ty, d), reachable_via_deref)| {
+                    let step = CandidateStep {
+                        self_ty: infcx
+                            .make_query_response_ignoring_pending_obligations(inference_vars, ty),
+                        autoderefs: d,
+                        from_unsafe_deref: reached_raw_pointer,
+                        unsize: false,
+                        reachable_via_deref,
+                    };
+                    if ty.is_raw_ptr() {
+                        // all the subsequent steps will be from_unsafe_deref
+                        reached_raw_pointer = true;
+                    }
+                    step
+                })
+                .collect();
+            (
+                steps,
+                // FIXME(arbitrary_self_types): This is sus.
+                autoderef_via_deref.final_ty(true),
+                autoderef_via_receiver.reached_recursion_limit(),
+            )
+        } else {
+            let steps = autoderef_via_deref
+                .by_ref()
+                .map(|(ty, d)| {
+                    let step = CandidateStep {
+                        self_ty: infcx
+                            .make_query_response_ignoring_pending_obligations(inference_vars, ty),
+                        autoderefs: d,
+                        from_unsafe_deref: reached_raw_pointer,
+                        unsize: false,
+                        reachable_via_deref: true,
+                    };
+                    if ty.is_raw_ptr() {
+                        // all the subsequent steps will be from_unsafe_deref
+                        reached_raw_pointer = true;
+                    }
+                    step
+                })
+                .collect();
+            (
+                steps,
+                autoderef_via_deref.final_ty(true),
+                autoderef_via_deref.reached_recursion_limit(),
+            )
+        };
     let opt_bad_ty = match final_ty.kind() {
-        ty::Infer(ty::TyVar(_)) | ty::Error(_) => Some(MethodAutoderefBadTy {
+        ty::Infer(ty::TyVar(_)) => Some(MethodAutoderefBadTy {
+            reached_raw_pointer,
+            is_opaque_type: final_ty_is_opaque(infcx, final_ty),
+            ty: infcx.make_query_response_ignoring_pending_obligations(inference_vars, final_ty),
+        }),
+        ty::Error(_) => Some(MethodAutoderefBadTy {
             reached_raw_pointer,
+            is_opaque_type: false,
             ty: infcx.make_query_response_ignoring_pending_obligations(inference_vars, final_ty),
         }),
         ty::Array(elem_ty, _) => {
@@ -669,6 +687,20 @@ fn method_autoderef_steps<'tcx>(
     }
 }
 
+/// Returns `true` in case the final type is the hidden type of an opaque.
+#[instrument(level = "debug", skip(infcx), ret)]
+fn final_ty_is_opaque<'tcx>(infcx: &InferCtxt<'tcx>, final_ty: Ty<'tcx>) -> bool {
+    // nyaaaa~
+    if infcx.next_trait_solver() {
+        let &ty::Infer(ty::TyVar(vid)) = final_ty.kind() else {
+            return false;
+        };
+        infcx.find_sup_as_registered_opaque(vid).is_some()
+    } else {
+        false
+    }
+}
+
 impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
     fn new(
         fcx: &'a FnCtxt<'a, 'tcx>,
@@ -1882,31 +1914,39 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                     (xform_self_ty, xform_ret_ty) =
                         self.xform_self_ty(probe.item, trait_ref.self_ty(), trait_ref.args);
                     xform_self_ty = ocx.normalize(cause, self.param_env, xform_self_ty);
-                    match self_ty.kind() {
-                        // HACK: opaque types will match anything for which their bounds hold.
-                        // Thus we need to prevent them from trying to match the `&_` autoref
-                        // candidates that get created for `&self` trait methods.
-                        ty::Alias(ty::Opaque, alias_ty)
-                            if !self.next_trait_solver()
-                                && self.infcx.can_define_opaque_ty(alias_ty.def_id)
-                                && !xform_self_ty.is_ty_var() =>
-                        {
-                            return ProbeResult::NoMatch;
-                        }
-                        _ => match ocx.relate(
-                            cause,
-                            self.param_env,
-                            self.variance(),
-                            self_ty,
-                            xform_self_ty,
-                        ) {
-                            Ok(()) => {}
-                            Err(err) => {
-                                debug!("--> cannot relate self-types {:?}", err);
+
+                    // HACK: opaque types will match anything for which their bounds hold.
+                    // Thus we need to prevent them from trying to match the `&_` autoref
+                    // candidates that get created for `&self` trait methods.
+                    if self.mode == Mode::MethodCall {
+                        match self_ty.kind() {
+                            ty::Infer(ty::TyVar(_)) => {
+                                assert!(self.infcx.next_trait_solver());
+                                if !xform_self_ty.is_ty_var() {
+                                    return ProbeResult::NoMatch;
+                                }
+                            }
+                            ty::Alias(ty::Opaque, alias_ty)
+                                if !self.infcx.next_trait_solver()
+                                    && self.infcx.can_define_opaque_ty(alias_ty.def_id)
+                                    && !xform_self_ty.is_ty_var() =>
+                            {
+                                assert!(!self.infcx.next_trait_solver());
                                 return ProbeResult::NoMatch;
                             }
-                        },
+                            _ => {}
+                        }
+                    }
+
+                    match ocx.relate(cause, self.param_env, self.variance(), self_ty, xform_self_ty)
+                    {
+                        Ok(()) => {}
+                        Err(err) => {
+                            debug!("--> cannot relate self-types {:?}", err);
+                            return ProbeResult::NoMatch;
+                        }
                     }
+
                     let obligation = traits::Obligation::new(
                         self.tcx,
                         cause.clone(),

compiler/rustc_hir_typeck/src/opaque_types.rs

@@ -1,5 +1,123 @@
-use super::FnCtxt;
-impl<'tcx> FnCtxt<'_, 'tcx> {
+use rustc_middle::ty::{
+    DefiningScopeKind, EarlyBinder, OpaqueHiddenType, OpaqueTypeKey, Ty, TypeVisitableExt,
+};
+use rustc_trait_selection::opaque_types::{
+    InvalidOpaqueTypeArgs, check_opaque_type_parameter_valid,
+};
+use tracing::{debug, instrument};
+
+use crate::FnCtxt;
+
+impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
+    #[instrument(level = "debug", skip(self))]
+    pub(super) fn handle_opaque_type_uses_next(&mut self) {
+        // We clone the opaques instead of stealing them here as they are still used for
+        // normalization in the next generation trait solver.
+        //
+        // FIXME(-Znext-solver): Opaque types defined after this would simply get dropped
+        // at the end of typeck. Ideally we can feed some query here to no longer define
+        // new opaque uses but instead always reveal by using the definitions inferred here.
+        let mut opaque_types: Vec<_> = self.infcx.clone_opaque_types();
+        let num_entries = self.inner.borrow_mut().opaque_types().num_entries();
+        let prev = self.checked_opaque_types_storage_entries.replace(Some(num_entries));
+        debug_assert_eq!(prev, None);
+        for entry in &mut opaque_types {
+            *entry = self.resolve_vars_if_possible(*entry);
+        }
+        debug!(?opaque_types);
+
+        self.collect_defining_uses(&opaque_types);
+        self.apply_defining_uses(&opaque_types);
+    }
+
+    fn collect_defining_uses(
+        &mut self,
+        opaque_types: &[(OpaqueTypeKey<'tcx>, OpaqueHiddenType<'tcx>)],
+    ) {
+        let tcx = self.tcx;
+        let typeck_results = &mut *self.typeck_results.borrow_mut();
+        for &(opaque_type_key, hidden_type) in opaque_types {
+            match check_opaque_type_parameter_valid(
+                &self,
+                opaque_type_key,
+                hidden_type.span,
+                DefiningScopeKind::HirTypeck,
+            ) {
+                Ok(()) => {}
+                Err(InvalidOpaqueTypeArgs::AlreadyReported(guar)) => {
+                    typeck_results
+                        .concrete_opaque_types
+                        .insert(opaque_type_key.def_id, OpaqueHiddenType::new_error(tcx, guar));
+                }
+                // Not a defining use, ignore and treat as revealing use instead.
+                Err(
+                    InvalidOpaqueTypeArgs::NotAParam { .. }
+                    | InvalidOpaqueTypeArgs::DuplicateParam { .. },
+                ) => continue,
+            }
+
+            // We ignore uses of the opaque if they have any inference variables
+            // as this can frequently happen with recursive calls.
+            //
+            // See `tests/ui/traits/next-solver/opaques/universal-args-non-defining.rs`.
+            if hidden_type.ty.has_non_region_infer() {
+                continue;
+            }
+
+            let hidden_type = hidden_type.remap_generic_params_to_declaration_params(
+                opaque_type_key,
+                tcx,
+                DefiningScopeKind::HirTypeck,
+            );
+
+            if let Some(prev) =
+                typeck_results.concrete_opaque_types.insert(opaque_type_key.def_id, hidden_type)
+            {
+                let entry =
+                    typeck_results.concrete_opaque_types.get_mut(&opaque_type_key.def_id).unwrap();
+                if prev.ty != hidden_type.ty {
+                    if let Some(guar) = typeck_results.tainted_by_errors {
+                        entry.ty = Ty::new_error(tcx, guar);
+                    } else {
+                        let (Ok(guar) | Err(guar)) =
+                            prev.build_mismatch_error(&hidden_type, tcx).map(|d| d.emit());
+                        entry.ty = Ty::new_error(tcx, guar);
+                    }
+                }
+
+                // Pick a better span if there is one.
+                // FIXME(oli-obk): collect multiple spans for better diagnostics down the road.
+                entry.span = prev.span.substitute_dummy(hidden_type.span);
+            }
+        }
+
+        // FIXME(-Znext-solver): Check that all opaques have been defined hre.
+    }
+
+    fn apply_defining_uses(
+        &mut self,
+        opaque_types: &[(OpaqueTypeKey<'tcx>, OpaqueHiddenType<'tcx>)],
+    ) {
+        let tcx = self.tcx;
+        for &(key, hidden_type) in opaque_types {
+            let Some(&expected) =
+                self.typeck_results.borrow_mut().concrete_opaque_types.get(&key.def_id)
+            else {
+                let guar =
+                    tcx.dcx().span_err(hidden_type.span, "non-defining use in the defining scope");
+                self.typeck_results
+                    .borrow_mut()
+                    .concrete_opaque_types
+                    .insert(key.def_id, OpaqueHiddenType::new_error(tcx, guar));
+                self.set_tainted_by_errors(guar);
+                continue;
+            };
+
+            let expected = EarlyBinder::bind(expected.ty).instantiate(tcx, key.args);
+            self.demand_eqtype(hidden_type.span, expected, hidden_type.ty);
+        }
+    }
+
     /// We may in theory add further uses of an opaque after cloning the opaque
     /// types storage during writeback when computing the defining uses.
     ///

compiler/rustc_hir_typeck/src/writeback.rs

@@ -534,26 +534,34 @@ impl<'cx, 'tcx> WritebackCx<'cx, 'tcx> {
         }
     }
 
+    fn visit_opaque_types_next(&mut self) {
+        let fcx_typeck_results = self.fcx.typeck_results.borrow();
+        assert_eq!(fcx_typeck_results.hir_owner, self.typeck_results.hir_owner);
+        for (&def_id, &hidden_type) in &fcx_typeck_results.concrete_opaque_types {
+            assert!(!hidden_type.has_infer());
+            self.typeck_results.concrete_opaque_types.insert(def_id, hidden_type);
+        }
+    }
+
     #[instrument(skip(self), level = "debug")]
     fn visit_opaque_types(&mut self) {
+        if self.fcx.next_trait_solver() {
+            return self.visit_opaque_types_next();
+        }
+
         let tcx = self.tcx();
-        // We clone the opaques instead of stealing them here as they are still used for
-        // normalization in the next generation trait solver.
-        let opaque_types = self.fcx.infcx.clone_opaque_types();
+        let opaque_types = self.fcx.infcx.take_opaque_types();
         let num_entries = self.fcx.inner.borrow_mut().opaque_types().num_entries();
         let prev = self.fcx.checked_opaque_types_storage_entries.replace(Some(num_entries));
         debug_assert_eq!(prev, None);
         for (opaque_type_key, hidden_type) in opaque_types {
             let hidden_type = self.resolve(hidden_type, &hidden_type.span);
             let opaque_type_key = self.resolve(opaque_type_key, &hidden_type.span);
-
-            if !self.fcx.next_trait_solver() {
-                if let ty::Alias(ty::Opaque, alias_ty) = hidden_type.ty.kind()
-                    && alias_ty.def_id == opaque_type_key.def_id.to_def_id()
-                    && alias_ty.args == opaque_type_key.args
-                {
-                    continue;
-                }
+            if let ty::Alias(ty::Opaque, alias_ty) = hidden_type.ty.kind()
+                && alias_ty.def_id == opaque_type_key.def_id.to_def_id()
+                && alias_ty.args == opaque_type_key.args
+            {
+                continue;
             }
 
             if let Err(err) = check_opaque_type_parameter_valid(
@@ -950,8 +958,10 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Resolver<'cx, 'tcx> {
     }
 
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
-        debug_assert!(!r.is_bound(), "Should not be resolving bound region.");
-        self.fcx.tcx.lifetimes.re_erased
+        match r.kind() {
+            ty::ReBound(..) => r,
+            _ => self.fcx.tcx.lifetimes.re_erased,
+        }
     }
 
     fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {

compiler/rustc_infer/src/infer/canonical/canonicalizer.rs

@@ -17,8 +17,7 @@ use tracing::debug;
 
 use crate::infer::InferCtxt;
 use crate::infer::canonical::{
-    Canonical, CanonicalQueryInput, CanonicalTyVarKind, CanonicalVarInfo, CanonicalVarKind,
-    OriginalQueryValues,
+    Canonical, CanonicalQueryInput, CanonicalVarInfo, CanonicalVarKind, OriginalQueryValues,
 };
 
 impl<'tcx> InferCtxt<'tcx> {
@@ -299,6 +298,7 @@ struct Canonicalizer<'cx, 'tcx> {
     // Note that indices is only used once `var_values` is big enough to be
     // heap-allocated.
     indices: FxHashMap<GenericArg<'tcx>, BoundVar>,
+    sub_root_lookup_table: FxHashMap<ty::TyVid, usize>,
     canonicalize_mode: &'cx dyn CanonicalizeMode,
     needs_canonical_flags: TypeFlags,
 
@@ -367,9 +367,10 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
                             // FIXME: perf problem described in #55921.
                             ui = ty::UniverseIndex::ROOT;
                         }
+                        let sub_root = self.get_or_insert_sub_root(vid);
                         self.canonicalize_ty_var(
                             CanonicalVarInfo {
-                                kind: CanonicalVarKind::Ty(CanonicalTyVarKind::General(ui)),
+                                kind: CanonicalVarKind::Ty { universe: ui, sub_root },
                             },
                             t,
                         )
@@ -382,21 +383,15 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
                 if nt != t {
                     return self.fold_ty(nt);
                 } else {
-                    self.canonicalize_ty_var(
-                        CanonicalVarInfo { kind: CanonicalVarKind::Ty(CanonicalTyVarKind::Int) },
-                        t,
-                    )
+                    self.canonicalize_ty_var(CanonicalVarInfo { kind: CanonicalVarKind::Int }, t)
                 }
             }
             ty::Infer(ty::FloatVar(vid)) => {
                 let nt = self.infcx.unwrap().opportunistic_resolve_float_var(vid);
                 if nt != t {
                     return self.fold_ty(nt);
                 } else {
-                    self.canonicalize_ty_var(
-                        CanonicalVarInfo { kind: CanonicalVarKind::Ty(CanonicalTyVarKind::Float) },
-                        t,
-                    )
+                    self.canonicalize_ty_var(CanonicalVarInfo { kind: CanonicalVarKind::Float }, t)
                 }
             }
 
@@ -576,6 +571,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
             variables: SmallVec::from_slice(base.variables),
             query_state,
             indices: FxHashMap::default(),
+            sub_root_lookup_table: Default::default(),
             binder_index: ty::INNERMOST,
         };
         if canonicalizer.query_state.var_values.spilled() {
@@ -670,6 +666,13 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
         }
     }
 
+    fn get_or_insert_sub_root(&mut self, vid: ty::TyVid) -> ty::BoundVar {
+        let root_vid = self.infcx.unwrap().sub_root_var(vid);
+        let idx =
+            *self.sub_root_lookup_table.entry(root_vid).or_insert_with(|| self.variables.len());
+        ty::BoundVar::from(idx)
+    }
+
     /// Replaces the universe indexes used in `var_values` with their index in
     /// `query_state.universe_map`. This minimizes the maximum universe used in
     /// the canonicalized value.
@@ -690,11 +693,11 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
             .iter()
             .map(|v| CanonicalVarInfo {
                 kind: match v.kind {
-                    CanonicalVarKind::Ty(CanonicalTyVarKind::Int | CanonicalTyVarKind::Float) => {
+                    CanonicalVarKind::Int | CanonicalVarKind::Float => {
                         return *v;
                     }
-                    CanonicalVarKind::Ty(CanonicalTyVarKind::General(u)) => {
-                        CanonicalVarKind::Ty(CanonicalTyVarKind::General(reverse_universe_map[&u]))
+                    CanonicalVarKind::Ty { universe, sub_root } => {
+                        CanonicalVarKind::Ty { universe: reverse_universe_map[&universe], sub_root }
                     }
                     CanonicalVarKind::Region(u) => {
                         CanonicalVarKind::Region(reverse_universe_map[&u])