Skip to content

Commit 16b9941

Browse files
SparrowLiiSparrowLii
committed
make sharded query caches only for multiple threads
1 parent 9bd60a6 commit 16b9941

File tree

8 files changed

+275
-49
lines changed

8 files changed

+275
-49
lines changed

‎compiler/rustc_interface/src/passes.rs

+1
Original file line numberDiff line numberDiff line change
@@ -731,6 +731,7 @@ pub fn create_global_ctxt<'tcx>(
731731
extern_providers,
732732
query_result_on_disk_cache,
733733
incremental,
734+
!rustc_data_structures::sync::is_dyn_thread_safe(),
734735
),
735736
)
736737
})

‎compiler/rustc_middle/src/query/plumbing.rs

+64-14
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ use crate::dep_graph::DepKind;
33
use crate::query::on_disk_cache::CacheEncoder;
44
use crate::query::on_disk_cache::EncodedDepNodeIndex;
55
use crate::query::on_disk_cache::OnDiskCache;
6+
#[cfg(parallel_compiler)]
7+
use crate::query::MtQueryCaches;
68
use crate::query::{
79
DynamicQueries, ExternProviders, Providers, QueryArenas, QueryCaches, QueryEngine, QueryStates,
810
};
@@ -20,6 +22,8 @@ pub(crate) use rustc_query_system::query::QueryJobId;
2022
use rustc_query_system::query::*;
2123
use rustc_query_system::HandleCycleError;
2224
use rustc_span::{Span, DUMMY_SP};
25+
#[cfg(not(parallel_compiler))]
26+
use std::marker::PhantomData;
2327
use std::ops::Deref;
2428

2529
pub struct QueryKeyStringCache {
@@ -32,13 +36,17 @@ impl QueryKeyStringCache {
3236
}
3337
}
3438

35-
pub struct DynamicQuery<'tcx, C: QueryCache> {
39+
pub struct DynamicQuery<'tcx, C: QueryCache, C2: QueryCache<Key = C::Key, Value = C::Value>> {
3640
pub name: &'static str,
3741
pub eval_always: bool,
3842
pub dep_kind: DepKind,
3943
pub handle_cycle_error: HandleCycleError,
4044
pub query_state: FieldOffset<QueryStates<'tcx>, QueryState<C::Key, DepKind>>,
4145
pub query_cache: FieldOffset<QueryCaches<'tcx>, C>,
46+
#[cfg(not(parallel_compiler))]
47+
pub mt_query_cache: PhantomData<C2>,
48+
#[cfg(parallel_compiler)]
49+
pub mt_query_cache: FieldOffset<MtQueryCaches<'tcx>, C2>,
4250
pub cache_on_disk: fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool,
4351
pub execute_query: fn(tcx: TyCtxt<'tcx>, k: C::Key) -> C::Value,
4452
pub compute: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value,
@@ -72,6 +80,10 @@ pub struct QuerySystem<'tcx> {
7280
pub states: QueryStates<'tcx>,
7381
pub arenas: QueryArenas<'tcx>,
7482
pub caches: QueryCaches<'tcx>,
83+
#[cfg(parallel_compiler)]
84+
pub single_thread: bool,
85+
#[cfg(parallel_compiler)]
86+
pub mt_caches: MtQueryCaches<'tcx>,
7587
pub dynamic_queries: DynamicQueries<'tcx>,
7688

7789
/// This provides access to the incremental compilation on-disk cache for query results.
@@ -138,6 +150,36 @@ impl<'tcx> TyCtxt<'tcx> {
138150
}
139151
}
140152

153+
#[macro_export]
154+
#[cfg(not(parallel_compiler))]
155+
macro_rules! with_query_caches {
156+
($func: ident($($params: expr,)* :$tcx: expr, $name:ident, $($rest: expr,)*)) => {
157+
$func($($params,)* &$tcx.query_system.caches.$name, $($rest,)*)
158+
};
159+
($tcx: expr, $name:ident, $func: ident($($params: expr,)*)) => {
160+
$tcx.query_system.caches.$name.$func($($params,)*)
161+
}
162+
}
163+
164+
#[macro_export]
165+
#[cfg(parallel_compiler)]
166+
macro_rules! with_query_caches {
167+
($func: ident($($params: expr,)* :$tcx: expr, $name:ident, $($rest: expr,)*)) => {
168+
if $tcx.query_system.single_thread {
169+
$func($($params,)* &$tcx.query_system.caches.$name, $($rest,)*)
170+
} else {
171+
$func($($params,)* &$tcx.query_system.mt_caches.$name, $($rest,)*)
172+
}
173+
};
174+
($tcx: expr, $name:ident, $func: ident($($params: expr,)*)) => {
175+
if $tcx.query_system.single_thread {
176+
$tcx.query_system.caches.$name.$func($($params,)*)
177+
} else {
178+
$tcx.query_system.mt_caches.$name.$func($($params,)*)
179+
}
180+
}
181+
}
182+
141183
#[inline]
142184
pub fn query_get_at<'tcx, Cache>(
143185
tcx: TyCtxt<'tcx>,
@@ -286,6 +328,10 @@ macro_rules! define_callbacks {
286328
<$($K)* as keys::Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>>
287329
>::Cache;
288330

331+
pub type MtStorage<'tcx> = <
332+
<$($K)* as keys::Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>>
333+
>::MtCache;
334+
289335
// Ensure that keys grow no larger than 64 bytes
290336
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
291337
const _: () = {
@@ -339,31 +385,36 @@ macro_rules! define_callbacks {
339385
$($(#[$attr])* pub $name: queries::$name::Storage<'tcx>,)*
340386
}
341387

388+
#[derive(Default)]
389+
pub struct MtQueryCaches<'tcx> {
390+
$($(#[$attr])* pub $name: queries::$name::MtStorage<'tcx>,)*
391+
}
392+
342393
impl<'tcx> TyCtxtEnsure<'tcx> {
343394
$($(#[$attr])*
344395
#[inline(always)]
345396
pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
346-
query_ensure(
397+
with_query_caches!(query_ensure(
347398
self.tcx,
348399
self.tcx.query_system.fns.engine.$name,
349-
&self.tcx.query_system.caches.$name,
400+
:self.tcx, $name,
350401
key.into_query_param(),
351402
false,
352-
);
403+
));
353404
})*
354405
}
355406

356407
impl<'tcx> TyCtxtEnsureWithValue<'tcx> {
357408
$($(#[$attr])*
358409
#[inline(always)]
359410
pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
360-
query_ensure(
411+
with_query_caches!(query_ensure(
361412
self.tcx,
362413
self.tcx.query_system.fns.engine.$name,
363-
&self.tcx.query_system.caches.$name,
414+
:self.tcx, $name,
364415
key.into_query_param(),
365416
true,
366-
);
417+
));
367418
})*
368419
}
369420

@@ -382,19 +433,19 @@ macro_rules! define_callbacks {
382433
#[inline(always)]
383434
pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V
384435
{
385-
restore::<$V>(query_get_at(
436+
restore::<$V>(with_query_caches!(query_get_at(
386437
self.tcx,
387438
self.tcx.query_system.fns.engine.$name,
388-
&self.tcx.query_system.caches.$name,
439+
:self.tcx, $name,
389440
self.span,
390441
key.into_query_param(),
391-
))
442+
)))
392443
})*
393444
}
394445

395446
pub struct DynamicQueries<'tcx> {
396447
$(
397-
pub $name: DynamicQuery<'tcx, queries::$name::Storage<'tcx>>,
448+
pub $name: DynamicQuery<'tcx, queries::$name::Storage<'tcx>, queries::$name::MtStorage<'tcx>>,
398449
)*
399450
}
400451

@@ -484,10 +535,9 @@ macro_rules! define_feedable {
484535
let tcx = self.tcx;
485536
let erased = queries::$name::provided_to_erased(tcx, value);
486537
let value = restore::<$V>(erased);
487-
let cache = &tcx.query_system.caches.$name;
488538

489539
let hasher: Option<fn(&mut StableHashingContext<'_>, &_) -> _> = hash_result!([$($modifiers)*]);
490-
match try_get_cached(tcx, cache, &key) {
540+
match with_query_caches!(try_get_cached(tcx, :tcx, $name, &key,)) {
491541
Some(old) => {
492542
let old = restore::<$V>(old);
493543
if let Some(hasher) = hasher {
@@ -523,7 +573,7 @@ macro_rules! define_feedable {
523573
&value,
524574
hash_result!([$($modifiers)*]),
525575
);
526-
cache.complete(key, erased, dep_node_index);
576+
with_query_caches!(tcx, $name, complete(key, erased, dep_node_index,));
527577
}
528578
}
529579
}

‎compiler/rustc_query_impl/src/lib.rs

+30-7
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ use rustc_middle::query::plumbing::{
2929
DynamicQuery, QueryKeyStringCache, QuerySystem, QuerySystemFns,
3030
};
3131
use rustc_middle::query::AsLocalKey;
32+
#[cfg(parallel_compiler)]
33+
use rustc_middle::query::MtQueryCaches;
3234
use rustc_middle::query::{
3335
queries, DynamicQueries, ExternProviders, Providers, QueryCaches, QueryEngine, QueryStates,
3436
};
@@ -53,33 +55,40 @@ pub use self::profiling_support::alloc_self_profile_query_strings;
5355
struct DynamicConfig<
5456
'tcx,
5557
C: QueryCache,
58+
C2: QueryCache<Key = C::Key, Value = C::Value>,
5659
const ANON: bool,
5760
const DEPTH_LIMIT: bool,
5861
const FEEDABLE: bool,
5962
> {
60-
dynamic: &'tcx DynamicQuery<'tcx, C>,
63+
dynamic: &'tcx DynamicQuery<'tcx, C, C2>,
6164
}
6265

63-
impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Copy
64-
for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE>
66+
impl<'tcx, C: QueryCache, C2, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Copy
67+
for DynamicConfig<'tcx, C, C2, ANON, DEPTH_LIMIT, FEEDABLE>
68+
where
69+
C2: QueryCache<Key = C::Key, Value = C::Value>,
6570
{
6671
}
67-
impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Clone
68-
for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE>
72+
impl<'tcx, C: QueryCache, C2, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Clone
73+
for DynamicConfig<'tcx, C, C2, ANON, DEPTH_LIMIT, FEEDABLE>
74+
where
75+
C2: QueryCache<Key = C::Key, Value = C::Value>,
6976
{
7077
fn clone(&self) -> Self {
7178
DynamicConfig { dynamic: self.dynamic }
7279
}
7380
}
7481

75-
impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool>
76-
QueryConfig<QueryCtxt<'tcx>> for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE>
82+
impl<'tcx, C: QueryCache, C2, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool>
83+
QueryConfig<QueryCtxt<'tcx>> for DynamicConfig<'tcx, C, C2, ANON, DEPTH_LIMIT, FEEDABLE>
7784
where
85+
C2: QueryCache<Key = C::Key, Value = C::Value>,
7886
for<'a> C::Key: HashStable<StableHashingContext<'a>>,
7987
{
8088
type Key = C::Key;
8189
type Value = C::Value;
8290
type Cache = C;
91+
type MtCache = C2;
8392

8493
#[inline(always)]
8594
fn name(self) -> &'static str {
@@ -107,6 +116,15 @@ where
107116
self.dynamic.query_cache.apply(&qcx.tcx.query_system.caches)
108117
}
109118

119+
#[inline(always)]
120+
#[cfg(parallel_compiler)]
121+
fn mt_query_cache<'a>(self, qcx: QueryCtxt<'tcx>) -> &'a Self::MtCache
122+
where
123+
'tcx: 'a,
124+
{
125+
self.dynamic.mt_query_cache.apply(&qcx.tcx.query_system.mt_caches)
126+
}
127+
110128
#[inline(always)]
111129
fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
112130
(self.dynamic.execute_query)(tcx, key)
@@ -207,11 +225,16 @@ pub fn query_system<'tcx>(
207225
extern_providers: ExternProviders,
208226
on_disk_cache: Option<OnDiskCache<'tcx>>,
209227
incremental: bool,
228+
_single_thread: bool,
210229
) -> QuerySystem<'tcx> {
211230
QuerySystem {
212231
states: Default::default(),
213232
arenas: Default::default(),
214233
caches: Default::default(),
234+
#[cfg(parallel_compiler)]
235+
single_thread: _single_thread,
236+
#[cfg(parallel_compiler)]
237+
mt_caches: Default::default(),
215238
dynamic_queries: dynamic_queries(),
216239
on_disk_cache,
217240
fns: QuerySystemFns {

‎compiler/rustc_query_impl/src/plumbing.rs

+33-18
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ use rustc_query_system::query::{
2323
force_query, QueryCache, QueryConfig, QueryContext, QueryJobId, QueryMap, QuerySideEffects,
2424
QueryStackFrame,
2525
};
26-
use rustc_query_system::{LayoutOfDepth, QueryOverflow};
26+
use rustc_query_system::{with_qcx_query_caches, LayoutOfDepth, QueryOverflow};
2727
use rustc_serialize::Decodable;
2828
use rustc_serialize::Encodable;
2929
use rustc_session::Limit;
@@ -73,6 +73,12 @@ impl QueryContext for QueryCtxt<'_> {
7373
)
7474
}
7575

76+
#[inline]
77+
#[cfg(parallel_compiler)]
78+
fn single_thread(self) -> bool {
79+
self.query_system.single_thread
80+
}
81+
7682
#[inline]
7783
fn current_query_job(self) -> Option<QueryJobId> {
7884
tls::with_related_context(self.tcx, |icx| icx.query)
@@ -352,19 +358,22 @@ pub(crate) fn encode_query_results<'a, 'tcx, Q>(
352358
qcx.profiler().verbose_generic_activity_with_arg("encode_query_results_for", query.name());
353359

354360
assert!(query.query_state(qcx).all_inactive());
355-
let cache = query.query_cache(qcx);
356-
cache.iter(&mut |key, value, dep_node| {
357-
if query.cache_on_disk(qcx.tcx, &key) {
358-
let dep_node = SerializedDepNodeIndex::new(dep_node.index());
359-
360-
// Record position of the cache entry.
361-
query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position())));
362-
363-
// Encode the type check tables with the `SerializedDepNodeIndex`
364-
// as tag.
365-
encoder.encode_tagged(dep_node, &Q::restore(*value));
366-
}
367-
});
361+
with_qcx_query_caches!(
362+
query,
363+
qcx,
364+
iter(&mut |key, value, dep_node| {
365+
if query.cache_on_disk(qcx.tcx, &key) {
366+
let dep_node = SerializedDepNodeIndex::new(dep_node.index());
367+
368+
// Record position of the cache entry.
369+
query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position())));
370+
371+
// Encode the type check tables with the `SerializedDepNodeIndex`
372+
// as tag.
373+
encoder.encode_tagged(dep_node, &Q::restore(*value));
374+
}
375+
})
376+
);
368377
}
369378

370379
fn try_load_from_on_disk_cache<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode)
@@ -562,14 +571,18 @@ macro_rules! define_queries {
562571
}
563572
}
564573

565-
pub fn dynamic_query<'tcx>() -> DynamicQuery<'tcx, queries::$name::Storage<'tcx>> {
574+
pub fn dynamic_query<'tcx>() -> DynamicQuery<'tcx, queries::$name::Storage<'tcx>, queries::$name::MtStorage<'tcx>> {
566575
DynamicQuery {
567576
name: stringify!($name),
568577
eval_always: is_eval_always!([$($modifiers)*]),
569578
dep_kind: dep_graph::DepKind::$name,
570579
handle_cycle_error: handle_cycle_error!([$($modifiers)*]),
571580
query_state: offset_of!(QueryStates<'tcx> => $name),
572581
query_cache: offset_of!(QueryCaches<'tcx> => $name),
582+
#[cfg(not(parallel_compiler))]
583+
mt_query_cache: PhantomData,
584+
#[cfg(parallel_compiler)]
585+
mt_query_cache: offset_of!(MtQueryCaches<'tcx> => $name),
573586
cache_on_disk: |tcx, key| ::rustc_middle::query::cached::$name(tcx, key),
574587
execute_query: |tcx, key| erase(tcx.$name(key)),
575588
compute: |tcx, key| {
@@ -632,6 +645,7 @@ macro_rules! define_queries {
632645
type Config = DynamicConfig<
633646
'tcx,
634647
queries::$name::Storage<'tcx>,
648+
queries::$name::MtStorage<'tcx>,
635649
{ is_anon!([$($modifiers)*]) },
636650
{ depth_limit!([$($modifiers)*]) },
637651
{ feedable!([$($modifiers)*]) },
@@ -664,12 +678,13 @@ macro_rules! define_queries {
664678
}
665679

666680
pub fn alloc_self_profile_query_strings<'tcx>(tcx: TyCtxt<'tcx>, string_cache: &mut QueryKeyStringCache) {
667-
$crate::profiling_support::alloc_self_profile_query_strings_for_query_cache(
681+
use $crate::profiling_support::alloc_self_profile_query_strings_for_query_cache;
682+
with_query_caches!(alloc_self_profile_query_strings_for_query_cache(
668683
tcx,
669684
stringify!($name),
670-
&tcx.query_system.caches.$name,
685+
:tcx, $name,
671686
string_cache,
672-
)
687+
))
673688
}
674689

675690
item_if_cached! { [$($modifiers)*] {

0 commit comments

Comments
 (0)