Rollup of 6 pull requests

compiler/rustc_middle/src/queries.rs

@@ -131,10 +131,10 @@ use crate::{mir, thir};
 // `Providers` that the driver creates (using several `rustc_*` crates).
 //
 // The result type of each query must implement `Clone`. Additionally
-// `ty::query::from_cycle_error::FromCycleError` can be implemented which produces an appropriate
+// `QueryVTable::handle_cycle_error_fn` can be used to produce an appropriate
 // placeholder (error) value if the query resulted in a query cycle.
-// Queries without a `FromCycleError` implementation will raise a fatal error on query
-// cycles instead.
+// Queries without a custom `handle_cycle_error_fn` implementation will raise a
+// fatal error on query cycles instead.
 rustc_queries! {
     /// Caches the expansion of a derive proc macro, e.g. `#[derive(Serialize)]`.
     /// The key is:
@@ -577,7 +577,7 @@ rustc_queries! {
 
     /// Checks whether a type is representable or infinitely sized
     //
-    // Infinitely sized types will cause a cycle. The `value_from_cycle_error` impl will print
+    // Infinitely sized types will cause a cycle. The query's `handle_cycle_error_fn` will print
     // a custom error about the infinite size and then abort compilation. (In the past we
     // recovered and continued, but in practice that leads to confusing subsequent error
     // messages about cycles that then abort.)

compiler/rustc_middle/src/query/job.rs

@@ -6,7 +6,7 @@ use std::sync::Arc;
 use parking_lot::{Condvar, Mutex};
 use rustc_span::Span;
 
-use crate::query::CycleError;
+use crate::query::Cycle;
 use crate::ty::TyCtxt;
 
 /// A value uniquely identifying an active query job.
@@ -59,7 +59,7 @@ pub struct QueryWaiter<'tcx> {
     pub parent: Option<QueryJobId>,
     pub condvar: Condvar,
     pub span: Span,
-    pub cycle: Mutex<Option<CycleError<'tcx>>>,
+    pub cycle: Mutex<Option<Cycle<'tcx>>>,
 }
 
 #[derive(Clone, Debug)]
@@ -79,7 +79,7 @@ impl<'tcx> QueryLatch<'tcx> {
         tcx: TyCtxt<'tcx>,
         query: Option<QueryJobId>,
         span: Span,
-    ) -> Result<(), CycleError<'tcx>> {
+    ) -> Result<(), Cycle<'tcx>> {
         let mut waiters_guard = self.waiters.lock();
         let Some(waiters) = &mut *waiters_guard else {
             return Ok(()); // already complete

compiler/rustc_middle/src/query/mod.rs

@@ -5,8 +5,8 @@ pub use self::into_query_key::IntoQueryKey;
 pub use self::job::{QueryJob, QueryJobId, QueryLatch, QueryWaiter};
 pub use self::keys::{AsLocalQueryKey, LocalCrate, QueryKey};
 pub use self::plumbing::{
-    ActiveKeyStatus, CycleError, EnsureMode, QueryMode, QueryState, QuerySystem, QueryVTable,
-    TyCtxtAt, TyCtxtEnsureDone, TyCtxtEnsureOk, TyCtxtEnsureResult,
+    ActiveKeyStatus, Cycle, EnsureMode, QueryMode, QueryState, QuerySystem, QueryVTable, TyCtxtAt,
+    TyCtxtEnsureDone, TyCtxtEnsureOk, TyCtxtEnsureResult,
 };
 pub use self::stack::QueryStackFrame;
 pub use crate::queries::Providers;

compiler/rustc_middle/src/query/plumbing.rs

@@ -47,12 +47,12 @@ pub enum ActiveKeyStatus<'tcx> {
 }
 
 #[derive(Debug)]
-pub struct CycleError<'tcx> {
+pub struct Cycle<'tcx> {
     /// The query and related span that uses the cycle.
     pub usage: Option<QueryStackFrame<'tcx>>,
 
     /// The span here corresponds to the reason for which this query was required.
-    pub cycle: Vec<QueryStackFrame<'tcx>>,
+    pub frames: Vec<QueryStackFrame<'tcx>>,
 }
 
 #[derive(Debug)]
@@ -111,13 +111,10 @@ pub struct QueryVTable<'tcx, C: QueryCache> {
 
     /// Function pointer that handles a cycle error. `error` must be consumed, e.g. with `emit` (if
     /// it should be emitted) or `delay_as_bug` (if it need not be emitted because an alternative
-    /// error is created and emitted).
-    pub value_from_cycle_error: fn(
-        tcx: TyCtxt<'tcx>,
-        key: C::Key,
-        cycle_error: CycleError<'tcx>,
-        error: Diag<'_>,
-    ) -> C::Value,
+    /// error is created and emitted). A value may be returned, or (more commonly) the function may
+    /// just abort after emitting the error.
+    pub handle_cycle_error_fn:
+        fn(tcx: TyCtxt<'tcx>, key: C::Key, cycle: Cycle<'tcx>, error: Diag<'_>) -> C::Value,
 
     pub format_value: fn(&C::Value) -> String,
 

compiler/rustc_query_impl/src/dep_kind_vtables.rs

@@ -4,7 +4,7 @@ use rustc_middle::dep_graph::{DepKindVTable, DepNodeKey, KeyFingerprintStyle};
 use rustc_middle::query::QueryCache;
 
 use crate::GetQueryVTable;
-use crate::plumbing::{force_from_dep_node_inner, promote_from_disk_inner};
+use crate::plumbing::promote_from_disk_inner;
 
 /// [`DepKindVTable`] constructors for special dep kinds that aren't queries.
 #[expect(non_snake_case, reason = "use non-snake case to avoid collision with query names")]
@@ -111,10 +111,16 @@ where
     DepKindVTable {
         is_eval_always,
         key_fingerprint_style,
-        force_from_dep_node_fn: (can_recover && !is_no_force)
-            .then_some(force_from_dep_node_inner::<Q>),
-        promote_from_disk_fn: (can_recover && is_cache_on_disk)
-            .then_some(promote_from_disk_inner::<Q>),
+        force_from_dep_node_fn: (can_recover && !is_no_force).then_some(
+            |tcx, dep_node, _prev_index| {
+                let query = Q::query_vtable(tcx);
+                crate::execution::force_query_dep_node(tcx, query, dep_node)
+            },
+        ),
+        promote_from_disk_fn: (can_recover && is_cache_on_disk).then_some(|tcx, dep_node| {
+            let query = Q::query_vtable(tcx);
+            promote_from_disk_inner(tcx, query, dep_node)
+        }),
     }
 }
 

compiler/rustc_query_impl/src/execution.rs

@@ -8,16 +8,16 @@ use rustc_data_structures::{outline, sharded, sync};
 use rustc_errors::FatalError;
 use rustc_middle::dep_graph::{DepGraphData, DepNodeKey, SerializedDepNodeIndex};
 use rustc_middle::query::{
-    ActiveKeyStatus, CycleError, EnsureMode, QueryCache, QueryJob, QueryJobId, QueryKey,
-    QueryLatch, QueryMode, QueryState, QueryVTable,
+    ActiveKeyStatus, Cycle, EnsureMode, QueryCache, QueryJob, QueryJobId, QueryKey, QueryLatch,
+    QueryMode, QueryState, QueryVTable,
 };
 use rustc_middle::ty::TyCtxt;
 use rustc_middle::verify_ich::incremental_verify_ich;
 use rustc_span::{DUMMY_SP, Span};
 use tracing::warn;
 
 use crate::dep_graph::{DepNode, DepNodeIndex};
-use crate::job::{QueryJobInfo, QueryJobMap, find_cycle_in_stack, report_cycle};
+use crate::job::{QueryJobInfo, QueryJobMap, create_cycle_error, find_cycle_in_stack};
 use crate::plumbing::{current_query_job, loadable_from_disk, next_job_id, start_query};
 use crate::query_impl::for_each_query_vtable;
 
@@ -108,14 +108,14 @@ fn collect_active_query_jobs_inner<'tcx, C>(
 
 #[cold]
 #[inline(never)]
-fn mk_cycle<'tcx, C: QueryCache>(
+fn handle_cycle<'tcx, C: QueryCache>(
     query: &'tcx QueryVTable<'tcx, C>,
     tcx: TyCtxt<'tcx>,
     key: C::Key,
-    cycle_error: CycleError<'tcx>,
+    cycle: Cycle<'tcx>,
 ) -> C::Value {
-    let error = report_cycle(tcx, &cycle_error);
-    (query.value_from_cycle_error)(tcx, key, cycle_error, error)
+    let error = create_cycle_error(tcx, &cycle);
+    (query.handle_cycle_error_fn)(tcx, key, cycle, error)
 }
 
 /// Guard object representing the responsibility to execute a query job and
@@ -194,7 +194,7 @@ where
 
 #[cold]
 #[inline(never)]
-fn cycle_error<'tcx, C: QueryCache>(
+fn find_and_handle_cycle<'tcx, C: QueryCache>(
     query: &'tcx QueryVTable<'tcx, C>,
     tcx: TyCtxt<'tcx>,
     key: C::Key,
@@ -205,8 +205,8 @@ fn cycle_error<'tcx, C: QueryCache>(
     // We need the complete map to ensure we find a cycle to break.
     let job_map = collect_active_query_jobs(tcx, CollectActiveJobsKind::FullNoContention);
 
-    let error = find_cycle_in_stack(try_execute, job_map, &current_query_job(), span);
-    (mk_cycle(query, tcx, key, error), None)
+    let cycle = find_cycle_in_stack(try_execute, job_map, &current_query_job(), span);
+    (handle_cycle(query, tcx, key, cycle), None)
 }
 
 #[inline(always)]
@@ -250,7 +250,7 @@ fn wait_for_query<'tcx, C: QueryCache>(
 
             (v, Some(index))
         }
-        Err(cycle) => (mk_cycle(query, tcx, key, cycle), None),
+        Err(cycle) => (handle_cycle(query, tcx, key, cycle), None),
     }
 }
 
@@ -334,7 +334,7 @@ fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>(
 
                         // If we are single-threaded we know that we have cycle error,
                         // so we just return the error.
-                        cycle_error(query, tcx, key, id, span)
+                        find_and_handle_cycle(query, tcx, key, id, span)
                     }
                 }
                 ActiveKeyStatus::Poisoned => FatalError.raise(),
@@ -659,20 +659,26 @@ pub(super) fn execute_query_incr_inner<'tcx, C: QueryCache>(
     Some(result)
 }
 
-pub(crate) fn force_query<'tcx, C: QueryCache>(
-    query: &'tcx QueryVTable<'tcx, C>,
+/// Inner implementation of [`DepKindVTable::force_from_dep_node_fn`][force_fn]
+/// for query nodes.
+///
+/// [force_fn]: rustc_middle::dep_graph::DepKindVTable::force_from_dep_node_fn
+pub(crate) fn force_query_dep_node<'tcx, C: QueryCache>(
     tcx: TyCtxt<'tcx>,
-    key: C::Key,
+    query: &'tcx QueryVTable<'tcx, C>,
     dep_node: DepNode,
-) {
-    // We may be concurrently trying both execute and force a query.
-    // Ensure that only one of them runs the query.
-    if let Some((_, index)) = query.cache.lookup(&key) {
-        tcx.prof.query_cache_hit(index.into());
-        return;
-    }
+) -> bool {
+    let Some(key) = C::Key::try_recover_key(tcx, &dep_node) else {
+        // We couldn't recover a key from the node's key fingerprint.
+        // Tell the caller that we couldn't force the node.
+        return false;
+    };
 
     ensure_sufficient_stack(|| {
         try_execute_query::<C, true>(query, tcx, DUMMY_SP, key, Some(dep_node))
     });
+
+    // We did manage to recover a key and force the node, though it's up to
+    // the caller to check whether the node ended up marked red or green.
+    true
 }

compiler/rustc_query_impl/src/from_cycle_error.rs → compiler/rustc_query_impl/src/handle_cycle_error.rs

@@ -10,33 +10,33 @@ use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_middle::bug;
 use rustc_middle::queries::{QueryVTables, TaggedQueryKey};
-use rustc_middle::query::CycleError;
+use rustc_middle::query::Cycle;
 use rustc_middle::query::erase::erase_val;
 use rustc_middle::ty::layout::LayoutError;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_span::def_id::{DefId, LocalDefId};
 use rustc_span::{ErrorGuaranteed, Span};
 
-use crate::job::report_cycle;
+use crate::job::create_cycle_error;
 
 pub(crate) fn specialize_query_vtables<'tcx>(vtables: &mut QueryVTables<'tcx>) {
-    vtables.fn_sig.value_from_cycle_error = |tcx, key, _, err| {
+    vtables.fn_sig.handle_cycle_error_fn = |tcx, key, _, err| {
         let guar = err.delay_as_bug();
         erase_val(fn_sig(tcx, key, guar))
     };
 
-    vtables.check_representability.value_from_cycle_error =
+    vtables.check_representability.handle_cycle_error_fn =
         |tcx, _, cycle, _err| check_representability(tcx, cycle);
 
-    vtables.check_representability_adt_ty.value_from_cycle_error =
+    vtables.check_representability_adt_ty.handle_cycle_error_fn =
         |tcx, _, cycle, _err| check_representability(tcx, cycle);
 
-    vtables.variances_of.value_from_cycle_error = |tcx, key, _, err| {
+    vtables.variances_of.handle_cycle_error_fn = |tcx, key, _, err| {
         let _guar = err.delay_as_bug();
         erase_val(variances_of(tcx, key))
     };
 
-    vtables.layout_of.value_from_cycle_error = |tcx, _, cycle, err| {
+    vtables.layout_of.handle_cycle_error_fn = |tcx, _, cycle, err| {
         let _guar = err.delay_as_bug();
         erase_val(Err(layout_of(tcx, cycle)))
     }
@@ -72,10 +72,10 @@ fn fn_sig<'tcx>(
     )))
 }
 
-fn check_representability<'tcx>(tcx: TyCtxt<'tcx>, cycle_error: CycleError<'tcx>) -> ! {
+fn check_representability<'tcx>(tcx: TyCtxt<'tcx>, cycle: Cycle<'tcx>) -> ! {
     let mut item_and_field_ids = Vec::new();
     let mut representable_ids = FxHashSet::default();
-    for frame in &cycle_error.cycle {
+    for frame in &cycle.frames {
         if let TaggedQueryKey::check_representability(def_id) = frame.tagged_key
             && tcx.def_kind(def_id) == DefKind::Field
         {
@@ -88,7 +88,7 @@ fn check_representability<'tcx>(tcx: TyCtxt<'tcx>, cycle_error: CycleError<'tcx>
             item_and_field_ids.push((item_id.expect_local(), field_id));
         }
     }
-    for frame in &cycle_error.cycle {
+    for frame in &cycle.frames {
         if let TaggedQueryKey::check_representability_adt_ty(key) = frame.tagged_key
             && let Some(adt) = key.ty_adt_def()
             && let Some(def_id) = adt.did().as_local()
@@ -127,14 +127,11 @@ fn search_for_cycle_permutation<Q, T>(
     otherwise()
 }
 
-fn layout_of<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    cycle_error: CycleError<'tcx>,
-) -> &'tcx ty::layout::LayoutError<'tcx> {
+fn layout_of<'tcx>(tcx: TyCtxt<'tcx>, cycle: Cycle<'tcx>) -> &'tcx ty::layout::LayoutError<'tcx> {
     let diag = search_for_cycle_permutation(
-        &cycle_error.cycle,
-        |cycle| {
-            if let TaggedQueryKey::layout_of(key) = cycle[0].tagged_key
+        &cycle.frames,
+        |frames| {
+            if let TaggedQueryKey::layout_of(key) = frames[0].tagged_key
                 && let ty::Coroutine(def_id, _) = key.value.kind()
                 && let Some(def_id) = def_id.as_local()
                 && let def_kind = tcx.def_kind(def_id)
@@ -158,7 +155,7 @@ fn layout_of<'tcx>(
                     tcx.def_kind_descr_article(def_kind, def_id.to_def_id()),
                     tcx.def_kind_descr(def_kind, def_id.to_def_id()),
                 );
-                for (i, frame) in cycle.iter().enumerate() {
+                for (i, frame) in frames.iter().enumerate() {
                     let TaggedQueryKey::layout_of(frame_key) = frame.tagged_key else {
                         continue;
                     };
@@ -169,7 +166,7 @@ fn layout_of<'tcx>(
                         continue;
                     };
                     let frame_span =
-                        frame.tagged_key.default_span(tcx, cycle[(i + 1) % cycle.len()].span);
+                        frame.tagged_key.default_span(tcx, frames[(i + 1) % frames.len()].span);
                     if frame_span.is_dummy() {
                         continue;
                     }
@@ -203,7 +200,7 @@ fn layout_of<'tcx>(
                 ControlFlow::Continue(())
             }
         },
-        || report_cycle(tcx, &cycle_error),
+        || create_cycle_error(tcx, &cycle),
     );
 
     let guar = diag.emit();