diff --git a/compiler/rustc_middle/src/query/inner.rs b/compiler/rustc_middle/src/query/inner.rs index 4e62fbbec77d5..775669231228a 100644 --- a/compiler/rustc_middle/src/query/inner.rs +++ b/compiler/rustc_middle/src/query/inner.rs @@ -33,15 +33,15 @@ where /// Shared implementation of `tcx.$query(..)` and `tcx.at(span).$query(..)` /// for all queries. #[inline(always)] -pub(crate) fn query_get_at<'tcx, Cache>( +pub(crate) fn query_get_at<'tcx, C>( tcx: TyCtxt<'tcx>, - execute_query: fn(TyCtxt<'tcx>, Span, Cache::Key, QueryMode) -> Option, - query_cache: &Cache, + execute_query: fn(TyCtxt<'tcx>, Span, C::Key, QueryMode) -> Option, + query_cache: &C, span: Span, - key: Cache::Key, -) -> Cache::Value + key: C::Key, +) -> C::Value where - Cache: QueryCache, + C: QueryCache, { match try_get_cached(tcx, query_cache, &key) { Some(value) => value, @@ -52,14 +52,14 @@ where /// Shared implementation of `tcx.ensure_ok().$query(..)` for most queries, /// and `tcx.ensure_done().$query(..)` for all queries. #[inline] -pub(crate) fn query_ensure<'tcx, Cache>( +pub(crate) fn query_ensure<'tcx, C>( tcx: TyCtxt<'tcx>, - execute_query: fn(TyCtxt<'tcx>, Span, Cache::Key, QueryMode) -> Option, - query_cache: &Cache, - key: Cache::Key, + execute_query: fn(TyCtxt<'tcx>, Span, C::Key, QueryMode) -> Option, + query_cache: &C, + key: C::Key, ensure_mode: EnsureMode, ) where - Cache: QueryCache, + C: QueryCache, { if try_get_cached(tcx, query_cache, &key).is_none() { execute_query(tcx, DUMMY_SP, key, QueryMode::Ensure { ensure_mode }); @@ -69,17 +69,17 @@ pub(crate) fn query_ensure<'tcx, Cache>( /// Shared implementation of `tcx.ensure_ok().$query(..)` for queries that /// have the `return_result_from_ensure_ok` modifier. #[inline] -pub(crate) fn query_ensure_error_guaranteed<'tcx, Cache, T>( +pub(crate) fn query_ensure_error_guaranteed<'tcx, C, T>( tcx: TyCtxt<'tcx>, - execute_query: fn(TyCtxt<'tcx>, Span, Cache::Key, QueryMode) -> Option, - query_cache: &Cache, - key: Cache::Key, + execute_query: fn(TyCtxt<'tcx>, Span, C::Key, QueryMode) -> Option, + query_cache: &C, + key: C::Key, // This arg is needed to match the signature of `query_ensure`, // but should always be `EnsureMode::Ok`. ensure_mode: EnsureMode, ) -> Result<(), ErrorGuaranteed> where - Cache: QueryCache>>, + C: QueryCache>>, Result: Erasable, { assert_matches!(ensure_mode, EnsureMode::Ok); @@ -101,21 +101,20 @@ where } /// Common implementation of query feeding, used by `define_feedable!`. -pub(crate) fn query_feed<'tcx, Cache>( +pub(crate) fn query_feed<'tcx, C>( tcx: TyCtxt<'tcx>, dep_kind: DepKind, - query_vtable: &QueryVTable<'tcx, Cache>, - cache: &Cache, - key: Cache::Key, - value: Cache::Value, + query_vtable: &QueryVTable<'tcx, C>, + key: C::Key, + value: C::Value, ) where - Cache: QueryCache, - Cache::Key: DepNodeKey<'tcx>, + C: QueryCache, + C::Key: DepNodeKey<'tcx>, { let format_value = query_vtable.format_value; // Check whether the in-memory cache already has a value for this key. - match try_get_cached(tcx, cache, &key) { + match try_get_cached(tcx, &query_vtable.cache, &key) { Some(old) => { // The query already has a cached value for this key. // That's OK if both values are the same, i.e. they have the same hash, @@ -158,7 +157,7 @@ pub(crate) fn query_feed<'tcx, Cache>( query_vtable.hash_result, query_vtable.format_value, ); - cache.complete(key, value, dep_node_index); + query_vtable.cache.complete(key, value, dep_node_index); } } } diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index 2fee3713c4e01..9141f49bd45de 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -14,9 +14,7 @@ pub use sealed::IntoQueryParam; use crate::dep_graph; use crate::dep_graph::{DepKind, DepNode, DepNodeIndex, SerializedDepNodeIndex}; use crate::ich::StableHashingContext; -use crate::queries::{ - ExternProviders, PerQueryVTables, Providers, QueryArenas, QueryCaches, QueryEngine, QueryStates, -}; +use crate::queries::{ExternProviders, Providers, QueryArenas, QueryVTables}; use crate::query::on_disk_cache::{CacheEncoder, EncodedDepNodeIndex, OnDiskCache}; use crate::query::stack::{QueryStackDeferred, QueryStackFrame, QueryStackFrameExtra}; use crate::query::{QueryCache, QueryInfo, QueryJob}; @@ -113,7 +111,7 @@ pub enum EnsureMode { Done, } -/// Stores function pointers and other metadata for a particular query. +/// Stores data and metadata (e.g. function pointers) for a particular query. pub struct QueryVTable<'tcx, C: QueryCache> { pub name: &'static str, @@ -129,10 +127,8 @@ pub struct QueryVTable<'tcx, C: QueryCache> { pub dep_kind: DepKind, /// How this query deals with query cycle errors. pub cycle_error_handling: CycleErrorHandling, - // Offset of this query's state field in the QueryStates struct - pub query_state: usize, - // Offset of this query's cache field in the QueryCaches struct - pub query_cache: usize, + pub state: QueryState<'tcx, C::Key>, + pub cache: C, pub will_cache_on_disk_for_key_fn: Option>, /// Function pointer that calls `tcx.$query(key)` for this query and @@ -161,6 +157,8 @@ pub struct QueryVTable<'tcx, C: QueryCache> { /// /// Used when reporting query cycle errors and similar problems. pub description_fn: fn(TyCtxt<'tcx>, C::Key) -> String, + + pub execute_query_fn: fn(TyCtxt<'tcx>, Span, C::Key, QueryMode) -> Option, } impl<'tcx, C: QueryCache> fmt::Debug for QueryVTable<'tcx, C> { @@ -181,30 +179,6 @@ impl<'tcx, C: QueryCache> QueryVTable<'tcx, C> { self.will_cache_on_disk_for_key_fn.map_or(false, |f| f(tcx, key)) } - // Don't use this method to access query results, instead use the methods on TyCtxt. - #[inline(always)] - pub fn query_state(&self, tcx: TyCtxt<'tcx>) -> &'tcx QueryState<'tcx, C::Key> { - // Safety: - // This is just manually doing the subfield referencing through pointer math. - unsafe { - &*(&tcx.query_system.states as *const QueryStates<'tcx>) - .byte_add(self.query_state) - .cast::>() - } - } - - // Don't use this method to access query results, instead use the methods on TyCtxt. - #[inline(always)] - pub fn query_cache(&self, tcx: TyCtxt<'tcx>) -> &'tcx C { - // Safety: - // This is just manually doing the subfield referencing through pointer math. - unsafe { - &*(&tcx.query_system.caches as *const QueryCaches<'tcx>) - .byte_add(self.query_cache) - .cast::() - } - } - #[inline(always)] pub fn try_load_from_disk( &self, @@ -243,7 +217,6 @@ impl<'tcx, C: QueryCache> QueryVTable<'tcx, C> { } pub struct QuerySystemFns { - pub engine: QueryEngine, pub local_providers: Providers, pub extern_providers: ExternProviders, pub encode_query_results: for<'tcx> fn( @@ -255,10 +228,8 @@ pub struct QuerySystemFns { } pub struct QuerySystem<'tcx> { - pub states: QueryStates<'tcx>, pub arenas: WorkerLocal>, - pub caches: QueryCaches<'tcx>, - pub query_vtables: PerQueryVTables<'tcx>, + pub query_vtables: QueryVTables<'tcx>, /// This provides access to the incremental compilation on-disk cache for query results. /// Do not access this directly. It is only meant to be used by @@ -521,13 +492,6 @@ macro_rules! define_callbacks { )* } - #[derive(Default)] - pub struct QueryCaches<'tcx> { - $( - pub $name: $name::Storage<'tcx>, - )* - } - impl<'tcx> $crate::query::TyCtxtEnsureOk<'tcx> { $( $(#[$attr])* @@ -546,8 +510,8 @@ macro_rules! define_callbacks { (crate::query::inner::query_ensure) )( self.tcx, - self.tcx.query_system.fns.engine.$name, - &self.tcx.query_system.caches.$name, + self.tcx.query_system.query_vtables.$name.execute_query_fn, + &self.tcx.query_system.query_vtables.$name.cache, $crate::query::IntoQueryParam::into_query_param(key), $crate::query::EnsureMode::Ok, ) @@ -562,8 +526,8 @@ macro_rules! define_callbacks { pub fn $name(self, key: query_helper_param_ty!($($K)*)) { crate::query::inner::query_ensure( self.tcx, - self.tcx.query_system.fns.engine.$name, - &self.tcx.query_system.caches.$name, + self.tcx.query_system.query_vtables.$name.execute_query_fn, + &self.tcx.query_system.query_vtables.$name.cache, $crate::query::IntoQueryParam::into_query_param(key), $crate::query::EnsureMode::Done, ); @@ -591,8 +555,8 @@ macro_rules! define_callbacks { erase::restore_val::<$V>(inner::query_get_at( self.tcx, - self.tcx.query_system.fns.engine.$name, - &self.tcx.query_system.caches.$name, + self.tcx.query_system.query_vtables.$name.execute_query_fn, + &self.tcx.query_system.query_vtables.$name.cache, self.span, $crate::query::IntoQueryParam::into_query_param(key), )) @@ -615,7 +579,6 @@ macro_rules! define_callbacks { self.tcx, dep_graph::DepKind::$name, &self.tcx.query_system.query_vtables.$name, - &self.tcx.query_system.caches.$name, key, erased_value, ); @@ -625,21 +588,12 @@ macro_rules! define_callbacks { )* /// Holds a `QueryVTable` for each query. - /// - /// ("Per" just makes this pluralized name more visually distinct.) - pub struct PerQueryVTables<'tcx> { + pub struct QueryVTables<'tcx> { $( pub $name: ::rustc_middle::query::plumbing::QueryVTable<'tcx, $name::Storage<'tcx>>, )* } - #[derive(Default)] - pub struct QueryStates<'tcx> { - $( - pub $name: $crate::query::QueryState<'tcx, $name::Key<'tcx>>, - )* - } - pub struct Providers { $( /// This is the provider for the query. Use `Find references` on this to @@ -699,17 +653,6 @@ macro_rules! define_callbacks { impl Clone for ExternProviders { fn clone(&self) -> Self { *self } } - - pub struct QueryEngine { - $( - pub $name: for<'tcx> fn( - TyCtxt<'tcx>, - Span, - $name::Key<'tcx>, - $crate::query::QueryMode, - ) -> Option<$crate::query::erase::Erased<$V>>, - )* - } }; } diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs index 7cc20fef6c3db..655dbe52df3e6 100644 --- a/compiler/rustc_query_impl/src/execution.rs +++ b/compiler/rustc_query_impl/src/execution.rs @@ -244,12 +244,12 @@ fn wait_for_query<'tcx, C: QueryCache>( match result { Ok(()) => { - let Some((v, index)) = query.query_cache(tcx).lookup(&key) else { + let Some((v, index)) = query.cache.lookup(&key) else { outline(|| { // We didn't find the query result in the query cache. Check if it was // poisoned due to a panic instead. let key_hash = sharded::make_hash(&key); - let shard = query.query_state(tcx).active.lock_shard_by_hash(key_hash); + let shard = query.state.active.lock_shard_by_hash(key_hash); match shard.find(key_hash, equivalent_key(&key)) { // The query we waited on panicked. Continue unwinding here. Some((_, ActiveKeyStatus::Poisoned)) => FatalError.raise(), @@ -280,9 +280,8 @@ fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>( // query+key, which we should reuse instead of creating a new one. dep_node: Option, ) -> (C::Value, Option) { - let state = query.query_state(tcx); let key_hash = sharded::make_hash(&key); - let mut state_lock = state.active.lock_shard_by_hash(key_hash); + let mut state_lock = query.state.active.lock_shard_by_hash(key_hash); // For the parallel compiler we need to check both the query cache and query state structures // while holding the state lock to ensure that 1) the query has not yet completed and 2) the @@ -291,7 +290,7 @@ fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>( // executing, but another thread may have already completed the query and stores it result // in the query cache. if tcx.sess.threads() > 1 { - if let Some((value, index)) = query.query_cache(tcx).lookup(&key) { + if let Some((value, index)) = query.cache.lookup(&key) { tcx.prof.query_cache_hit(index.into()); return (value, Some(index)); } @@ -310,7 +309,7 @@ fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>( // Drop the lock before we start executing the query drop(state_lock); - execute_job::(query, tcx, state, key, key_hash, id, dep_node) + execute_job::(query, tcx, key, key_hash, id, dep_node) } Entry::Occupied(mut entry) => { match &mut entry.get_mut().1 { @@ -342,7 +341,6 @@ fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>( fn execute_job<'tcx, C: QueryCache, const INCR: bool>( query: &'tcx QueryVTable<'tcx, C>, tcx: TyCtxt<'tcx>, - state: &'tcx QueryState<'tcx, C::Key>, key: C::Key, key_hash: u64, id: QueryJobId, @@ -350,7 +348,7 @@ fn execute_job<'tcx, C: QueryCache, const INCR: bool>( ) -> (C::Value, Option) { // Set up a guard object that will automatically poison the query if a // panic occurs while executing the query (or any intermediate plumbing). - let job_guard = ActiveJobGuard { state, key, key_hash }; + let job_guard = ActiveJobGuard { state: &query.state, key, key_hash }; debug_assert_eq!(tcx.dep_graph.is_fully_enabled(), INCR); @@ -361,7 +359,7 @@ fn execute_job<'tcx, C: QueryCache, const INCR: bool>( execute_job_non_incr(query, tcx, key, id) }; - let cache = query.query_cache(tcx); + let cache = &query.cache; if query.feedable { // We should not compute queries that also got a value via feeding. // This can't happen, as query feeding adds the very dependencies to the fed query @@ -706,7 +704,7 @@ pub(crate) fn force_query<'tcx, C: QueryCache>( ) { // We may be concurrently trying both execute and force a query. // Ensure that only one of them runs the query. - if let Some((_, index)) = query.query_cache(tcx).lookup(&key) { + if let Some((_, index)) = query.cache.lookup(&key) { tcx.prof.query_cache_hit(index.into()); return; } diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 63cba4bb6172b..890c0b2ef92c7 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -10,9 +10,7 @@ use rustc_data_structures::sync::AtomicU64; use rustc_middle::dep_graph; -use rustc_middle::queries::{ - self, ExternProviders, Providers, QueryCaches, QueryEngine, QueryStates, -}; +use rustc_middle::queries::{self, ExternProviders, Providers}; use rustc_middle::query::on_disk_cache::{CacheEncoder, EncodedDepNodeIndex, OnDiskCache}; use rustc_middle::query::plumbing::{QuerySystem, QuerySystemFns, QueryVTable}; use rustc_middle::query::{AsLocalKey, QueryCache, QueryMode}; @@ -58,13 +56,10 @@ pub fn query_system<'tcx>( incremental: bool, ) -> QuerySystem<'tcx> { QuerySystem { - states: Default::default(), arenas: Default::default(), - caches: Default::default(), - query_vtables: make_query_vtables(), + query_vtables: make_query_vtables(incremental), on_disk_cache, fns: QuerySystemFns { - engine: engine(incremental), local_providers, extern_providers, encode_query_results: encode_all_query_results, diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index a25be9bb0156d..7b5fd76c31958 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -306,14 +306,15 @@ where QueryStackFrameExtra::new(description, span, def_kind) } -pub(crate) fn create_deferred_query_stack_frame<'tcx, Cache>( +pub(crate) fn create_deferred_query_stack_frame<'tcx, C>( tcx: TyCtxt<'tcx>, - vtable: &'tcx QueryVTable<'tcx, Cache>, - key: Cache::Key, + vtable: &'tcx QueryVTable<'tcx, C>, + key: C::Key, ) -> QueryStackFrame> where - Cache: QueryCache, - Cache::Key: Key + DynSend + DynSync, + C: QueryCache, + C::Key: Key + DynSend + DynSync, + QueryVTable<'tcx, C>: DynSync, { let kind = vtable.dep_kind; @@ -335,9 +336,8 @@ pub(crate) fn encode_query_results_inner<'a, 'tcx, C, V>( { let _timer = tcx.prof.generic_activity_with_arg("encode_query_results_for", query.name); - assert!(all_inactive(query.query_state(tcx))); - let cache = query.query_cache(tcx); - cache.iter(&mut |key, value, dep_node| { + assert!(all_inactive(&query.state)); + query.cache.iter(&mut |key, value, dep_node| { if query.will_cache_on_disk_for_key(tcx, key) { let dep_node = SerializedDepNodeIndex::new(dep_node.index()); @@ -357,7 +357,7 @@ pub(crate) fn query_key_hash_verify<'tcx, C: QueryCache>( ) { let _timer = tcx.prof.generic_activity_with_arg("query_key_hash_verify_for", query.name); - let cache = query.query_cache(tcx); + let cache = &query.cache; let mut map = UnordMap::with_capacity(cache.len()); cache.iter(&mut |key, _, _| { let node = DepNode::construct(tcx, query.dep_kind, key); @@ -548,7 +548,7 @@ macro_rules! define_queries { } } - pub(crate) fn make_query_vtable<'tcx>() + pub(crate) fn make_query_vtable<'tcx>(incremental: bool) -> QueryVTable<'tcx, queries::$name::Storage<'tcx>> { QueryVTable { @@ -559,8 +559,8 @@ macro_rules! define_queries { feedable: is_feedable!([$($modifiers)*]), dep_kind: dep_graph::DepKind::$name, cycle_error_handling: cycle_error_handling!([$($modifiers)*]), - query_state: std::mem::offset_of!(QueryStates<'tcx>, $name), - query_cache: std::mem::offset_of!(QueryCaches<'tcx>, $name), + state: Default::default(), + cache: Default::default(), will_cache_on_disk_for_key_fn: if_cache_on_disk!([$($modifiers)*] { Some(::rustc_middle::queries::_cache_on_disk_if_fns::$name) } { @@ -603,6 +603,11 @@ macro_rules! define_queries { hash_result: hash_result!([$($modifiers)*][queries::$name::Value<'tcx>]), format_value: |value| format!("{:?}", erase::restore_val::>(*value)), description_fn: $crate::queries::_description_fns::$name, + execute_query_fn: if incremental { + query_impl::$name::get_query_incr::__rust_end_short_backtrace + } else { + query_impl::$name::get_query_non_incr::__rust_end_short_backtrace + }, } } @@ -632,7 +637,8 @@ macro_rules! define_queries { }; // Call `gather_active_jobs_inner` to do the actual work. - let res = crate::execution::gather_active_jobs_inner(&tcx.query_system.states.$name, + let res = crate::execution::gather_active_jobs_inner( + &tcx.query_system.query_vtables.$name.state, tcx, make_frame, require_complete, @@ -658,7 +664,7 @@ macro_rules! define_queries { $crate::profiling_support::alloc_self_profile_query_strings_for_query_cache( tcx, stringify!($name), - &tcx.query_system.caches.$name, + &tcx.query_system.query_vtables.$name.cache, string_cache, ) } @@ -686,22 +692,10 @@ macro_rules! define_queries { } })*} - pub(crate) fn engine(incremental: bool) -> QueryEngine { - if incremental { - QueryEngine { - $($name: query_impl::$name::get_query_incr::__rust_end_short_backtrace,)* - } - } else { - QueryEngine { - $($name: query_impl::$name::get_query_non_incr::__rust_end_short_backtrace,)* - } - } - } - - pub fn make_query_vtables<'tcx>() -> queries::PerQueryVTables<'tcx> { - queries::PerQueryVTables { + pub fn make_query_vtables<'tcx>(incremental: bool) -> queries::QueryVTables<'tcx> { + queries::QueryVTables { $( - $name: query_impl::$name::make_query_vtable(), + $name: query_impl::$name::make_query_vtable(incremental), )* } }