Skip to content

Commit ba2f204

Browse files
borsbors
committed
Auto merge of #128506 - compiler-errors:by-move-body, r=cjgillot
Stop storing a special inner body for the coroutine by-move body for async closures ...and instead, just synthesize an item which is treated mostly normally by the MIR pipeline. This PR does a few things: * We synthesize a new `DefId` for the by-move body of a closure, which has its `mir_built` fed with the output of the `ByMoveBody` MIR transformation, and some other relevant queries. * This has the `DefKind::ByMoveBody`, which we use to distinguish it from "real" bodies (that come from HIR) which need to be borrowck'd. Introduce `TyCtxt::is_synthetic_mir` to skip over `mir_borrowck` which is called by `mir_promoted`; borrowck isn't really possible to make work ATM since it heavily relies being called on a body generated from HIR, and is redundant by the construction of the by-move-body. * Remove the special `PassManager` hacks for handling the inner `by_move_body` stored within the coroutine's mir body. Instead, this body is fed like a regular MIR body, so it's goes through all of the `tcx.*_mir` stages normally (build -> promoted -> ...etc... -> optimized) ✨. * Remove the `InstanceKind::ByMoveBody` shim, since now we have a "regular" def id, we can just use `InstanceKind::Item`. This also allows us to remove the corresponding hacks from codegen, such as in `fn_sig_for_fn_abi` ✨. Notable remarks: * ~~I know it's kind of weird to be using `DefKind::Closure` here, since it's not a distinct closure but just a new MIR body. I don't believe it really matters, but I could also use a different `DefKind`... maybe one that we could use for synthetic MIR bodies in general?~~ edit: We're doing this now.
2 parents 600edc9 + 93295ff commit ba2f204

File tree

40 files changed

+276
-303
lines changed

40 files changed

+276
-303
lines changed

Diff for: ‎compiler/rustc_const_eval/src/interpret/call.rs

-1
Original file line numberDiff line numberDiff line change
@@ -576,7 +576,6 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
576576
| ty::InstanceKind::ReifyShim(..)
577577
| ty::InstanceKind::ClosureOnceShim { .. }
578578
| ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
579-
| ty::InstanceKind::CoroutineKindShim { .. }
580579
| ty::InstanceKind::FnPtrShim(..)
581580
| ty::InstanceKind::DropGlue(..)
582581
| ty::InstanceKind::CloneShim(..)

Diff for: ‎compiler/rustc_hir/src/def.rs

+9-2
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,9 @@ pub enum DefKind {
133133
/// we treat them all the same, and code which needs to distinguish them can match
134134
/// or `hir::ClosureKind` or `type_of`.
135135
Closure,
136+
/// The definition of a synthetic coroutine body created by the lowering of a
137+
/// coroutine-closure, such as an async closure.
138+
SyntheticCoroutineBody,
136139
}
137140

138141
impl DefKind {
@@ -177,6 +180,7 @@ impl DefKind {
177180
DefKind::Closure => "closure",
178181
DefKind::ExternCrate => "extern crate",
179182
DefKind::GlobalAsm => "global assembly block",
183+
DefKind::SyntheticCoroutineBody => "synthetic mir body",
180184
}
181185
}
182186

@@ -236,7 +240,8 @@ impl DefKind {
236240
| DefKind::ForeignMod
237241
| DefKind::GlobalAsm
238242
| DefKind::Impl { .. }
239-
| DefKind::OpaqueTy => None,
243+
| DefKind::OpaqueTy
244+
| DefKind::SyntheticCoroutineBody => None,
240245
}
241246
}
242247

@@ -276,6 +281,7 @@ impl DefKind {
276281
DefKind::GlobalAsm => DefPathData::GlobalAsm,
277282
DefKind::Impl { .. } => DefPathData::Impl,
278283
DefKind::Closure => DefPathData::Closure,
284+
DefKind::SyntheticCoroutineBody => DefPathData::Closure,
279285
}
280286
}
281287

@@ -291,7 +297,8 @@ impl DefKind {
291297
| DefKind::AssocFn
292298
| DefKind::Ctor(..)
293299
| DefKind::Closure
294-
| DefKind::Static { .. } => true,
300+
| DefKind::Static { .. }
301+
| DefKind::SyntheticCoroutineBody => true,
295302
DefKind::Mod
296303
| DefKind::Struct
297304
| DefKind::Union

Diff for: ‎compiler/rustc_hir_analysis/src/check/wfcheck.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -2174,7 +2174,8 @@ fn lint_redundant_lifetimes<'tcx>(
21742174
| DefKind::Field
21752175
| DefKind::LifetimeParam
21762176
| DefKind::GlobalAsm
2177-
| DefKind::Closure => return,
2177+
| DefKind::Closure
2178+
| DefKind::SyntheticCoroutineBody => return,
21782179
}
21792180

21802181
// The ordering of this lifetime map is a bit subtle.

Diff for: ‎compiler/rustc_hir_analysis/src/lib.rs

-4
Original file line numberDiff line numberDiff line change
@@ -200,10 +200,6 @@ pub fn check_crate(tcx: TyCtxt<'_>) {
200200
}
201201
});
202202

203-
// Freeze definitions as we don't add new ones at this point. This improves performance by
204-
// allowing lock-free access to them.
205-
tcx.untracked().definitions.freeze();
206-
207203
// FIXME: Remove this when we implement creating `DefId`s
208204
// for anon constants during their parents' typeck.
209205
// Typeck all body owners in parallel will produce queries

Diff for: ‎compiler/rustc_interface/src/passes.rs

+14
Original file line numberDiff line numberDiff line change
@@ -784,7 +784,20 @@ fn run_required_analyses(tcx: TyCtxt<'_>) {
784784
}
785785
);
786786
});
787+
787788
rustc_hir_analysis::check_crate(tcx);
789+
sess.time("MIR_coroutine_by_move_body", || {
790+
tcx.hir().par_body_owners(|def_id| {
791+
if tcx.needs_coroutine_by_move_body_def_id(def_id) {
792+
tcx.ensure_with_value().coroutine_by_move_body_def_id(def_id);
793+
}
794+
});
795+
});
796+
// Freeze definitions as we don't add new ones at this point.
797+
// We need to wait until now since we synthesize a by-move body
798+
// This improves performance by allowing lock-free access to them.
799+
tcx.untracked().definitions.freeze();
800+
788801
sess.time("MIR_borrow_checking", || {
789802
tcx.hir().par_body_owners(|def_id| {
790803
// Run unsafety check because it's responsible for stealing and
@@ -816,6 +829,7 @@ fn run_required_analyses(tcx: TyCtxt<'_>) {
816829
);
817830
}
818831
});
832+
819833
sess.time("layout_testing", || layout_test::test_layout(tcx));
820834
sess.time("abi_testing", || abi_test::test_abi(tcx));
821835

Diff for: ‎compiler/rustc_metadata/src/rmeta/encoder.rs

+23-11
Original file line numberDiff line numberDiff line change
@@ -872,7 +872,8 @@ fn should_encode_span(def_kind: DefKind) -> bool {
872872
| DefKind::OpaqueTy
873873
| DefKind::Field
874874
| DefKind::Impl { .. }
875-
| DefKind::Closure => true,
875+
| DefKind::Closure
876+
| DefKind::SyntheticCoroutineBody => true,
876877
DefKind::ForeignMod | DefKind::GlobalAsm => false,
877878
}
878879
}
@@ -902,6 +903,7 @@ fn should_encode_attrs(def_kind: DefKind) -> bool {
902903
// https://github.com./model-checking/kani and is not a performance
903904
// or maintenance issue for us.
904905
DefKind::Closure => true,
906+
DefKind::SyntheticCoroutineBody => false,
905907
DefKind::TyParam
906908
| DefKind::ConstParam
907909
| DefKind::Ctor(..)
@@ -948,7 +950,8 @@ fn should_encode_expn_that_defined(def_kind: DefKind) -> bool {
948950
| DefKind::Field
949951
| DefKind::LifetimeParam
950952
| DefKind::GlobalAsm
951-
| DefKind::Closure => false,
953+
| DefKind::Closure
954+
| DefKind::SyntheticCoroutineBody => false,
952955
}
953956
}
954957

@@ -984,7 +987,8 @@ fn should_encode_visibility(def_kind: DefKind) -> bool {
984987
| DefKind::GlobalAsm
985988
| DefKind::Impl { .. }
986989
| DefKind::Closure
987-
| DefKind::ExternCrate => false,
990+
| DefKind::ExternCrate
991+
| DefKind::SyntheticCoroutineBody => false,
988992
}
989993
}
990994

@@ -1019,7 +1023,8 @@ fn should_encode_stability(def_kind: DefKind) -> bool {
10191023
| DefKind::InlineConst
10201024
| DefKind::GlobalAsm
10211025
| DefKind::Closure
1022-
| DefKind::ExternCrate => false,
1026+
| DefKind::ExternCrate
1027+
| DefKind::SyntheticCoroutineBody => false,
10231028
}
10241029
}
10251030

@@ -1061,6 +1066,7 @@ fn should_encode_mir(
10611066
}
10621067
// Coroutines require optimized MIR to compute layout.
10631068
DefKind::Closure if tcx.is_coroutine(def_id.to_def_id()) => (false, true),
1069+
DefKind::SyntheticCoroutineBody => (false, true),
10641070
// Full-fledged functions + closures
10651071
DefKind::AssocFn | DefKind::Fn | DefKind::Closure => {
10661072
let generics = tcx.generics_of(def_id);
@@ -1109,7 +1115,8 @@ fn should_encode_variances<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, def_kind: Def
11091115
| DefKind::InlineConst
11101116
| DefKind::GlobalAsm
11111117
| DefKind::Closure
1112-
| DefKind::ExternCrate => false,
1118+
| DefKind::ExternCrate
1119+
| DefKind::SyntheticCoroutineBody => false,
11131120
DefKind::TyAlias => tcx.type_alias_is_lazy(def_id),
11141121
}
11151122
}
@@ -1137,7 +1144,8 @@ fn should_encode_generics(def_kind: DefKind) -> bool {
11371144
| DefKind::Impl { .. }
11381145
| DefKind::Field
11391146
| DefKind::TyParam
1140-
| DefKind::Closure => true,
1147+
| DefKind::Closure
1148+
| DefKind::SyntheticCoroutineBody => true,
11411149
DefKind::Mod
11421150
| DefKind::ForeignMod
11431151
| DefKind::ConstParam
@@ -1168,7 +1176,8 @@ fn should_encode_type(tcx: TyCtxt<'_>, def_id: LocalDefId, def_kind: DefKind) ->
11681176
| DefKind::Closure
11691177
| DefKind::ConstParam
11701178
| DefKind::AnonConst
1171-
| DefKind::InlineConst => true,
1179+
| DefKind::InlineConst
1180+
| DefKind::SyntheticCoroutineBody => true,
11721181

11731182
DefKind::OpaqueTy => {
11741183
let origin = tcx.opaque_type_origin(def_id);
@@ -1240,7 +1249,8 @@ fn should_encode_fn_sig(def_kind: DefKind) -> bool {
12401249
| DefKind::Use
12411250
| DefKind::LifetimeParam
12421251
| DefKind::GlobalAsm
1243-
| DefKind::ExternCrate => false,
1252+
| DefKind::ExternCrate
1253+
| DefKind::SyntheticCoroutineBody => false,
12441254
}
12451255
}
12461256

@@ -1277,7 +1287,8 @@ fn should_encode_constness(def_kind: DefKind) -> bool {
12771287
| DefKind::Use
12781288
| DefKind::LifetimeParam
12791289
| DefKind::GlobalAsm
1280-
| DefKind::ExternCrate => false,
1290+
| DefKind::ExternCrate
1291+
| DefKind::SyntheticCoroutineBody => false,
12811292
}
12821293
}
12831294

@@ -1310,7 +1321,8 @@ fn should_encode_const(def_kind: DefKind) -> bool {
13101321
| DefKind::Use
13111322
| DefKind::LifetimeParam
13121323
| DefKind::GlobalAsm
1313-
| DefKind::ExternCrate => false,
1324+
| DefKind::ExternCrate
1325+
| DefKind::SyntheticCoroutineBody => false,
13141326
}
13151327
}
13161328

@@ -1458,7 +1470,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
14581470
record!(self.tables.associated_type_for_effects[def_id] <- assoc_def_id);
14591471
}
14601472
}
1461-
if def_kind == DefKind::Closure
1473+
if let DefKind::Closure | DefKind::SyntheticCoroutineBody = def_kind
14621474
&& let Some(coroutine_kind) = self.tcx.coroutine_kind(def_id)
14631475
{
14641476
self.tables.coroutine_kind.set(def_id.index, Some(coroutine_kind))

Diff for: ‎compiler/rustc_metadata/src/rmeta/table.rs

+1
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@ fixed_size_enum! {
171171
( Macro(MacroKind::Bang) )
172172
( Macro(MacroKind::Attr) )
173173
( Macro(MacroKind::Derive) )
174+
( SyntheticCoroutineBody )
174175
}
175176
}
176177

Diff for: ‎compiler/rustc_middle/src/hir/map/mod.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,7 @@ impl<'hir> Map<'hir> {
307307
}
308308
DefKind::InlineConst => BodyOwnerKind::Const { inline: true },
309309
DefKind::Ctor(..) | DefKind::Fn | DefKind::AssocFn => BodyOwnerKind::Fn,
310-
DefKind::Closure => BodyOwnerKind::Closure,
310+
DefKind::Closure | DefKind::SyntheticCoroutineBody => BodyOwnerKind::Closure,
311311
DefKind::Static { safety: _, mutability, nested: false } => {
312312
BodyOwnerKind::Static(mutability)
313313
}

Diff for: ‎compiler/rustc_middle/src/mir/mod.rs

-17
Original file line numberDiff line numberDiff line change
@@ -267,18 +267,6 @@ pub struct CoroutineInfo<'tcx> {
267267
/// Coroutine drop glue. This field is populated after the state transform pass.
268268
pub coroutine_drop: Option<Body<'tcx>>,
269269

270-
/// The body of the coroutine, modified to take its upvars by move rather than by ref.
271-
///
272-
/// This is used by coroutine-closures, which must return a different flavor of coroutine
273-
/// when called using `AsyncFnOnce::call_once`. It is produced by the `ByMoveBody` pass which
274-
/// is run right after building the initial MIR, and will only be populated for coroutines
275-
/// which come out of the async closure desugaring.
276-
///
277-
/// This body should be processed in lockstep with the containing body -- any optimization
278-
/// passes, etc, should be applied to this body as well. This is done automatically if
279-
/// using `run_passes`.
280-
pub by_move_body: Option<Body<'tcx>>,
281-
282270
/// The layout of a coroutine. This field is populated after the state transform pass.
283271
pub coroutine_layout: Option<CoroutineLayout<'tcx>>,
284272

@@ -298,7 +286,6 @@ impl<'tcx> CoroutineInfo<'tcx> {
298286
coroutine_kind,
299287
yield_ty: Some(yield_ty),
300288
resume_ty: Some(resume_ty),
301-
by_move_body: None,
302289
coroutine_drop: None,
303290
coroutine_layout: None,
304291
}
@@ -665,10 +652,6 @@ impl<'tcx> Body<'tcx> {
665652
self.coroutine.as_ref().and_then(|coroutine| coroutine.coroutine_drop.as_ref())
666653
}
667654

668-
pub fn coroutine_by_move_body(&self) -> Option<&Body<'tcx>> {
669-
self.coroutine.as_ref()?.by_move_body.as_ref()
670-
}
671-
672655
#[inline]
673656
pub fn coroutine_kind(&self) -> Option<CoroutineKind> {
674657
self.coroutine.as_ref().map(|coroutine| coroutine.coroutine_kind)

Diff for: ‎compiler/rustc_middle/src/mir/mono.rs

-1
Original file line numberDiff line numberDiff line change
@@ -415,7 +415,6 @@ impl<'tcx> CodegenUnit<'tcx> {
415415
| InstanceKind::Virtual(..)
416416
| InstanceKind::ClosureOnceShim { .. }
417417
| InstanceKind::ConstructCoroutineInClosureShim { .. }
418-
| InstanceKind::CoroutineKindShim { .. }
419418
| InstanceKind::DropGlue(..)
420419
| InstanceKind::CloneShim(..)
421420
| InstanceKind::ThreadLocalShim(..)

Diff for: ‎compiler/rustc_middle/src/mir/visit.rs

-1
Original file line numberDiff line numberDiff line change
@@ -349,7 +349,6 @@ macro_rules! make_mir_visitor {
349349
coroutine_closure_def_id: _def_id,
350350
receiver_by_ref: _,
351351
} |
352-
ty::InstanceKind::CoroutineKindShim { coroutine_def_id: _def_id } |
353352
ty::InstanceKind::AsyncDropGlueCtorShim(_def_id, None) |
354353
ty::InstanceKind::DropGlue(_def_id, None) => {}
355354

Diff for: ‎compiler/rustc_middle/src/query/mod.rs

+15
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,7 @@ rustc_queries! {
326326
query predicates_of(key: DefId) -> ty::GenericPredicates<'tcx> {
327327
desc { |tcx| "computing predicates of `{}`", tcx.def_path_str(key) }
328328
cache_on_disk_if { key.is_local() }
329+
feedable
329330
}
330331

331332
query opaque_types_defined_by(
@@ -498,6 +499,7 @@ rustc_queries! {
498499
/// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/mir/construction.html
499500
query mir_built(key: LocalDefId) -> &'tcx Steal<mir::Body<'tcx>> {
500501
desc { |tcx| "building MIR for `{}`", tcx.def_path_str(key) }
502+
feedable
501503
}
502504

503505
/// Try to build an abstract representation of the given constant.
@@ -742,6 +744,7 @@ rustc_queries! {
742744
query constness(key: DefId) -> hir::Constness {
743745
desc { |tcx| "checking if item is const: `{}`", tcx.def_path_str(key) }
744746
separate_provide_extern
747+
feedable
745748
}
746749

747750
query asyncness(key: DefId) -> ty::Asyncness {
@@ -760,10 +763,22 @@ rustc_queries! {
760763
desc { |tcx| "checking if item is promotable: `{}`", tcx.def_path_str(key) }
761764
}
762765

766+
/// The body of the coroutine, modified to take its upvars by move rather than by ref.
767+
///
768+
/// This is used by coroutine-closures, which must return a different flavor of coroutine
769+
/// when called using `AsyncFnOnce::call_once`. It is produced by the `ByMoveBody` pass which
770+
/// is run right after building the initial MIR, and will only be populated for coroutines
771+
/// which come out of the async closure desugaring.
772+
query coroutine_by_move_body_def_id(def_id: DefId) -> DefId {
773+
desc { |tcx| "looking up the coroutine by-move body for `{}`", tcx.def_path_str(def_id) }
774+
separate_provide_extern
775+
}
776+
763777
/// Returns `Some(coroutine_kind)` if the node pointed to by `def_id` is a coroutine.
764778
query coroutine_kind(def_id: DefId) -> Option<hir::CoroutineKind> {
765779
desc { |tcx| "looking up coroutine kind of `{}`", tcx.def_path_str(def_id) }
766780
separate_provide_extern
781+
feedable
767782
}
768783

769784
query coroutine_for_closure(def_id: DefId) -> DefId {

Diff for: ‎compiler/rustc_middle/src/ty/context.rs

+18
Original file line numberDiff line numberDiff line change
@@ -1579,6 +1579,12 @@ impl<'tcx> TyCtxt<'tcx> {
15791579
)
15801580
}
15811581

1582+
// Whether the body owner is synthetic, which in this case means it does not correspond to
1583+
// meaningful HIR. This is currently used to skip over MIR borrowck.
1584+
pub fn is_synthetic_mir(self, def_id: impl Into<DefId>) -> bool {
1585+
matches!(self.def_kind(def_id.into()), DefKind::SyntheticCoroutineBody)
1586+
}
1587+
15821588
/// Returns `true` if the node pointed to by `def_id` is a general coroutine that implements `Coroutine`.
15831589
/// This means it is neither an `async` or `gen` construct.
15841590
pub fn is_general_coroutine(self, def_id: DefId) -> bool {
@@ -3168,6 +3174,18 @@ impl<'tcx> TyCtxt<'tcx> {
31683174
self.impl_trait_header(def_id).map_or(ty::ImplPolarity::Positive, |h| h.polarity)
31693175
}
31703176

3177+
pub fn needs_coroutine_by_move_body_def_id(self, def_id: LocalDefId) -> bool {
3178+
if let Some(hir::CoroutineKind::Desugared(_, hir::CoroutineSource::Closure)) =
3179+
self.coroutine_kind(def_id)
3180+
&& let ty::Coroutine(_, args) = self.type_of(def_id).instantiate_identity().kind()
3181+
&& args.as_coroutine().kind_ty().to_opt_closure_kind() != Some(ty::ClosureKind::FnOnce)
3182+
{
3183+
true
3184+
} else {
3185+
false
3186+
}
3187+
}
3188+
31713189
/// Whether this is a trait implementation that has `#[diagnostic::do_not_recommend]`
31723190
pub fn do_not_recommend_impl(self, def_id: DefId) -> bool {
31733191
matches!(self.def_kind(def_id), DefKind::Impl { of_trait: true })

0 commit comments

Comments
 (0)