From 92f7d895625e22c6e9703b24a6dcc315b41252ca Mon Sep 17 00:00:00 2001 From: Ding Xiang Fei Date: Tue, 22 Apr 2025 19:28:51 +0000 Subject: [PATCH 1/4] relocate upvars with saved locals for analysis ... and treat coroutine upvar captures as saved locals as well. This allows the liveness analysis to determine which captures are truly saved across a yield point and which are initially used but discarded at first yield points. In the event that upvar captures are promoted, most certainly because a coroutine suspends at least once, the slots in the promotion prefix shall be reused. This means that the copies emitted in the upvar relocation MIR pass will eventually elided and eliminated in the codegen phase, hence no additional runtime cost is realised. Additional MIR dumps are inserted so that it is easier to inspect the bodies of async closures, including those that captures the state by-value. Debug information is updated to point at the correct location for upvars in borrow checking errors and final debuginfo. A language change that this patch enables is now actually reverted, so that lifetimes on relocated upvars are invariant with the upvars outside of the coroutine body. We are deferring the language change to a later discussion. Co-authored-by: Dario Nieuwenhuis --- compiler/rustc_abi/src/layout.rs | 9 +- compiler/rustc_abi/src/layout/coroutine.rs | 132 +++-- compiler/rustc_abi/src/lib.rs | 4 +- .../src/diagnostics/mutability_errors.rs | 105 ++-- compiler/rustc_borrowck/src/lib.rs | 35 +- compiler/rustc_borrowck/src/path_utils.rs | 28 +- compiler/rustc_borrowck/src/type_check/mod.rs | 47 +- compiler/rustc_codegen_cranelift/src/base.rs | 3 + .../src/debuginfo/metadata.rs | 6 +- .../src/debuginfo/metadata/enums/cpp_like.rs | 2 - .../src/debuginfo/metadata/enums/mod.rs | 34 +- .../src/debuginfo/metadata/enums/native.rs | 4 - compiler/rustc_codegen_ssa/src/mir/operand.rs | 9 +- compiler/rustc_codegen_ssa/src/mir/rvalue.rs | 3 + .../rustc_const_eval/src/interpret/step.rs | 3 + compiler/rustc_index/src/vec.rs | 35 ++ compiler/rustc_middle/src/mir/mod.rs | 5 + compiler/rustc_middle/src/mir/query.rs | 17 +- compiler/rustc_middle/src/mir/statement.rs | 14 +- compiler/rustc_middle/src/ty/instance.rs | 7 +- compiler/rustc_middle/src/ty/layout.rs | 5 +- compiler/rustc_middle/src/ty/mod.rs | 84 +++- compiler/rustc_middle/src/ty/sty.rs | 7 - .../rustc_mir_build/src/builder/custom/mod.rs | 1 + compiler/rustc_mir_transform/src/coroutine.rs | 262 ++++++++-- .../src/coroutine/by_move_body.rs | 62 ++- .../rustc_mir_transform/src/coroutine/drop.rs | 18 +- .../src/coroutine/relocate_upvars.rs | 451 ++++++++++++++++++ .../src/deref_separator.rs | 1 + compiler/rustc_mir_transform/src/lib.rs | 12 +- .../rustc_mir_transform/src/pass_manager.rs | 13 +- compiler/rustc_mir_transform/src/patch.rs | 4 + compiler/rustc_mir_transform/src/shim.rs | 6 +- .../src/shim/async_destructor_ctor.rs | 56 ++- compiler/rustc_mir_transform/src/simplify.rs | 15 +- compiler/rustc_mir_transform/src/validate.rs | 13 +- compiler/rustc_session/src/config.rs | 31 +- compiler/rustc_session/src/options.rs | 15 + .../src/error_reporting/traits/suggestions.rs | 24 +- .../src/traits/query/dropck_outlives.rs | 11 +- compiler/rustc_ty_utils/src/layout.rs | 32 +- .../clippy/tests/ui/future_not_send.stderr | 4 +- .../clippy/tests/ui/large_futures.stderr | 10 +- .../clippy/tests/ui/needless_lifetimes.fixed | 1 + .../clippy/tests/ui/needless_lifetimes.rs | 1 + .../clippy/tests/ui/needless_lifetimes.stderr | 84 ++-- src/tools/miri/tests/pass/async-drop.rs | 8 +- .../pass/async-drop.stackrelocate.stdout | 23 + .../tests/pass/async-drop.treerelocate.stdout | 23 + tests/codegen-llvm/async-fn-debug.rs | 10 +- tests/debuginfo/coroutine-objects.rs | 12 +- .../debuginfo/coroutine-objects_relocated.rs | 89 ++++ ...#0}.coroutine_drop_async.0.panic-abort.mir | 4 +- ...0}.coroutine_drop_async.0.panic-unwind.mir | 4 +- ...await.a-{closure#0}.coroutine_resume.0.mir | 1 + ...await.b-{closure#0}.coroutine_resume.0.mir | 49 +- ...ny.main-{closure#0}.coroutine_resume.0.mir | 7 +- ...y.run2-{closure#0}.Inline.panic-abort.diff | 2 +- ....run2-{closure#0}.Inline.panic-unwind.diff | 2 +- ... => async-drop-initial.classic.run.stdout} | 1 + .../async-drop-initial.relocate.run.stdout | 23 + .../async-drop/async-drop-initial.rs | 65 ++- .../async-awaiting-fut.classic.stdout | 76 +++ ...out => async-awaiting-fut.relocate.stdout} | 54 ++- .../future-sizes/async-awaiting-fut.rs | 3 + .../async-await/future-sizes/future-as-arg.rs | 9 +- .../future-sizes/large-arg.classic.stdout | 64 +++ .../future-sizes/large-arg.relocate.stdout | 61 +++ .../ui/async-await/future-sizes/large-arg.rs | 3 + .../async-await/future-sizes/large-arg.stdout | 60 --- ...-ranked-auto-trait-8.no_assumptions.stderr | 11 +- tests/ui/async-await/issue-70818.stderr | 12 +- .../issue-74072-lifetime-name-annotations.rs | 6 +- ...sue-74072-lifetime-name-annotations.stderr | 4 +- tests/ui/async-await/issue-86507.rs | 7 +- tests/ui/async-await/issue-86507.stderr | 14 +- tests/ui/coroutine/auto-trait-regions.rs | 1 + tests/ui/coroutine/auto-trait-regions.stderr | 18 +- ...-impl.stderr => clone-impl.classic.stderr} | 76 +-- tests/ui/coroutine/clone-impl.relocate.stderr | 127 +++++ tests/ui/coroutine/clone-impl.rs | 5 +- .../ref-escapes-but-not-over-yield.rs | 7 +- .../ref-escapes-but-not-over-yield.stderr | 2 +- tests/ui/coroutine/ref-upvar-not-send.rs | 30 +- tests/ui/coroutine/ref-upvar-not-send.stderr | 40 +- .../coroutine/unsized-capture-across-yield.rs | 3 +- .../large_future.attribute.stderr | 8 +- .../large_future.option-relocate.stderr | 23 + .../large_future.option.stderr | 8 +- .../ui/lint/large_assignments/large_future.rs | 6 +- tests/ui/mir/lint/storage-live.stderr | 2 +- .../{async.stdout => async.classic.stdout} | 9 +- .../ui/print_type_sizes/async.relocate.stdout | 32 ++ tests/ui/print_type_sizes/async.rs | 3 + .../print_type_sizes/coroutine.classic.stdout | 15 + .../coroutine.relocate.stdout | 15 + tests/ui/print_type_sizes/coroutine.rs | 3 + tests/ui/print_type_sizes/coroutine.stdout | 10 - ...re.stdout => async-closure.classic.stdout} | 46 +- .../async-closure.relocate.stdout | 127 +++++ .../ui/rustc_public-ir-print/async-closure.rs | 3 + 101 files changed, 2370 insertions(+), 681 deletions(-) create mode 100644 compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs create mode 100644 src/tools/miri/tests/pass/async-drop.stackrelocate.stdout create mode 100644 src/tools/miri/tests/pass/async-drop.treerelocate.stdout create mode 100644 tests/debuginfo/coroutine-objects_relocated.rs rename tests/ui/async-await/async-drop/{async-drop-initial.run.stdout => async-drop-initial.classic.run.stdout} (94%) create mode 100644 tests/ui/async-await/async-drop/async-drop-initial.relocate.run.stdout create mode 100644 tests/ui/async-await/future-sizes/async-awaiting-fut.classic.stdout rename tests/ui/async-await/future-sizes/{async-awaiting-fut.stdout => async-awaiting-fut.relocate.stdout} (66%) create mode 100644 tests/ui/async-await/future-sizes/large-arg.classic.stdout create mode 100644 tests/ui/async-await/future-sizes/large-arg.relocate.stdout delete mode 100644 tests/ui/async-await/future-sizes/large-arg.stdout rename tests/ui/coroutine/{clone-impl.stderr => clone-impl.classic.stderr} (58%) create mode 100644 tests/ui/coroutine/clone-impl.relocate.stderr create mode 100644 tests/ui/lint/large_assignments/large_future.option-relocate.stderr rename tests/ui/print_type_sizes/{async.stdout => async.classic.stdout} (85%) create mode 100644 tests/ui/print_type_sizes/async.relocate.stdout create mode 100644 tests/ui/print_type_sizes/coroutine.classic.stdout create mode 100644 tests/ui/print_type_sizes/coroutine.relocate.stdout delete mode 100644 tests/ui/print_type_sizes/coroutine.stdout rename tests/ui/rustc_public-ir-print/{async-closure.stdout => async-closure.classic.stdout} (65%) create mode 100644 tests/ui/rustc_public-ir-print/async-closure.relocate.stdout diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index c2405553756b9..b92801d093ce8 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -3,6 +3,7 @@ use std::fmt::{self, Write}; use std::ops::{Bound, Deref}; use std::{cmp, iter}; +pub use coroutine::PackCoroutineLayout; use rustc_hashes::Hash64; use rustc_index::Idx; use rustc_index::bit_set::BitMatrix; @@ -210,17 +211,21 @@ impl LayoutCalculator { >( &self, local_layouts: &IndexSlice, - prefix_layouts: IndexVec, + relocated_upvars: &IndexSlice>, + upvar_layouts: IndexVec, variant_fields: &IndexSlice>, storage_conflicts: &BitMatrix, + pack: PackCoroutineLayout, tag_to_layout: impl Fn(Scalar) -> F, ) -> LayoutCalculatorResult { coroutine::layout( self, local_layouts, - prefix_layouts, + relocated_upvars, + upvar_layouts, variant_fields, storage_conflicts, + pack, tag_to_layout, ) } diff --git a/compiler/rustc_abi/src/layout/coroutine.rs b/compiler/rustc_abi/src/layout/coroutine.rs index 2b22276d4aed7..bbe2a2e6a61db 100644 --- a/compiler/rustc_abi/src/layout/coroutine.rs +++ b/compiler/rustc_abi/src/layout/coroutine.rs @@ -30,6 +30,17 @@ use crate::{ StructKind, TagEncoding, Variants, WrappingRange, }; +/// This option controls how coroutine saved locals are packed +/// into the coroutine state data +#[derive(Debug, Clone, Copy)] +pub enum PackCoroutineLayout { + /// The classic layout where captures are always promoted to coroutine state prefix + Classic, + /// Captures are first saved into the `UNRESUME` state and promoted + /// when they are used across more than one suspension + CapturesOnly, +} + /// Overlap eligibility and variant assignment for each CoroutineSavedLocal. #[derive(Clone, Debug, PartialEq)] enum SavedLocalEligibility { @@ -145,9 +156,11 @@ pub(super) fn layout< >( calc: &super::LayoutCalculator, local_layouts: &IndexSlice, - mut prefix_layouts: IndexVec, + relocated_upvars: &IndexSlice>, + upvar_layouts: IndexVec, variant_fields: &IndexSlice>, storage_conflicts: &BitMatrix, + pack: PackCoroutineLayout, tag_to_layout: impl Fn(Scalar) -> F, ) -> super::LayoutCalculatorResult { use SavedLocalEligibility::*; @@ -155,10 +168,11 @@ pub(super) fn layout< let (ineligible_locals, assignments) = coroutine_saved_local_eligibility(local_layouts.len(), variant_fields, storage_conflicts); - // Build a prefix layout, including "promoting" all ineligible - // locals as part of the prefix. We compute the layout of all of - // these fields at once to get optimal packing. - let tag_index = prefix_layouts.next_index(); + // Build a prefix layout, consisting of only the state tag and, as per request, upvars + let tag_index = match pack { + PackCoroutineLayout::CapturesOnly => FieldIdx::new(0), + PackCoroutineLayout::Classic => upvar_layouts.next_index(), + }; // `variant_fields` already accounts for the reserved variants, so no need to add them. let max_discr = (variant_fields.len() - 1) as u128; @@ -169,17 +183,28 @@ pub(super) fn layout< }; let promoted_layouts = ineligible_locals.iter().map(|local| local_layouts[local]); - prefix_layouts.push(tag_to_layout(tag)); - prefix_layouts.extend(promoted_layouts); + // FIXME: when we introduce more pack scheme, we need to change the prefix layout here + let prefix_layouts: IndexVec<_, _> = match pack { + PackCoroutineLayout::Classic => { + // Classic scheme packs the states as follows + // [ .. , , ] ++ + // In addition, UNRESUME overlaps with the part + upvar_layouts.into_iter().chain([tag_to_layout(tag)]).chain(promoted_layouts).collect() + } + PackCoroutineLayout::CapturesOnly => { + [tag_to_layout(tag)].into_iter().chain(promoted_layouts).collect() + } + }; + debug!(?prefix_layouts, ?pack); let prefix = calc.univariant(&prefix_layouts, &ReprOptions::default(), StructKind::AlwaysSized)?; let (prefix_size, prefix_align) = (prefix.size, prefix.align); - // Split the prefix layout into the "outer" fields (upvars and - // discriminant) and the "promoted" fields. Promoted fields will - // get included in each variant that requested them in - // CoroutineLayout. + // Split the prefix layout into the discriminant and + // the "promoted" fields. + // Promoted fields will get included in each variant + // that requested them in CoroutineLayout. debug!("prefix = {:#?}", prefix); let (outer_fields, promoted_offsets, promoted_memory_index) = match prefix.fields { FieldsShape::Arbitrary { mut offsets, memory_index } => { @@ -218,19 +243,67 @@ pub(super) fn layout< let variants = variant_fields .iter_enumerated() .map(|(index, variant_fields)| { + let is_unresumed = index == VariantIdx::new(0); + if is_unresumed && matches!(pack, PackCoroutineLayout::Classic) { + let fields = FieldsShape::Arbitrary { + offsets: (0..tag_index.index()).map(|i| outer_fields.offset(i)).collect(), + memory_index: (0..tag_index.index()) + .map(|i| { + (outer_fields.memory_index(i) + promoted_memory_index.len()) as u32 + }) + .collect(), + }; + let align = prefix.align; + let size = prefix.size; + return Ok(LayoutData { + fields, + variants: Variants::Single { index }, + backend_repr: BackendRepr::Memory { sized: true }, + largest_niche: None, + uninhabited: false, + align, + size, + max_repr_align: None, + unadjusted_abi_align: align.abi, + randomization_seed: Default::default(), + }); + } + let mut is_ineligible = IndexVec::from_elem_n(None, variant_fields.len()); + for (field, &local) in variant_fields.iter_enumerated() { + if is_unresumed { + if let Some(inner_local) = relocated_upvars[local] + && let Ineligible(Some(promoted_field)) = assignments[inner_local] + { + is_ineligible.insert(field, promoted_field); + continue; + } + } + match assignments[local] { + Assigned(v) if v == index => {} + Ineligible(Some(promoted_field)) => { + is_ineligible.insert(field, promoted_field); + } + Ineligible(None) => { + panic!("an ineligible local should have been promoted into the prefix") + } + Assigned(_) => { + panic!("an eligible local should have been assigned to exactly one variant") + } + Unassigned => { + panic!("each saved local should have been inspected at least once") + } + } + } // Only include overlap-eligible fields when we compute our variant layout. - let variant_only_tys = variant_fields - .iter() - .filter(|local| match assignments[**local] { - Unassigned => unreachable!(), - Assigned(v) if v == index => true, - Assigned(_) => unreachable!("assignment does not match variant"), - Ineligible(_) => false, + let fields: IndexVec<_, _> = variant_fields + .iter_enumerated() + .filter_map(|(field, &local)| { + if is_ineligible.contains(field) { None } else { Some(local_layouts[local]) } }) - .map(|local| local_layouts[*local]); + .collect(); let mut variant = calc.univariant( - &variant_only_tys.collect::>(), + &fields, &ReprOptions::default(), StructKind::Prefixed(prefix_size, prefix_align.abi), )?; @@ -254,19 +327,14 @@ pub(super) fn layout< IndexVec::from_elem_n(FieldIdx::new(invalid_field_idx), invalid_field_idx); let mut offsets_and_memory_index = iter::zip(offsets, memory_index); - let combined_offsets = variant_fields + let combined_offsets = is_ineligible .iter_enumerated() - .map(|(i, local)| { - let (offset, memory_index) = match assignments[*local] { - Unassigned => unreachable!(), - Assigned(_) => { - let (offset, memory_index) = offsets_and_memory_index.next().unwrap(); - (offset, promoted_memory_index.len() as u32 + memory_index) - } - Ineligible(field_idx) => { - let field_idx = field_idx.unwrap(); - (promoted_offsets[field_idx], promoted_memory_index[field_idx]) - } + .map(|(i, &is_ineligible)| { + let (offset, memory_index) = if let Some(field_idx) = is_ineligible { + (promoted_offsets[field_idx], promoted_memory_index[field_idx]) + } else { + let (offset, memory_index) = offsets_and_memory_index.next().unwrap(); + (offset, promoted_memory_index.len() as u32 + memory_index) }; combined_inverse_memory_index[memory_index] = i; offset diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index 14e256b8045df..60f9d35e60bd8 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -65,7 +65,9 @@ pub use callconv::{Heterogeneous, HomogeneousAggregate, Reg, RegKind}; pub use canon_abi::{ArmCall, CanonAbi, InterruptKind, X86Call}; pub use extern_abi::{ExternAbi, all_names}; #[cfg(feature = "nightly")] -pub use layout::{FIRST_VARIANT, FieldIdx, Layout, TyAbiInterface, TyAndLayout, VariantIdx}; +pub use layout::{ + FIRST_VARIANT, FieldIdx, Layout, PackCoroutineLayout, TyAbiInterface, TyAndLayout, VariantIdx, +}; pub use layout::{LayoutCalculator, LayoutCalculatorError}; /// Requirements for a `StableHashingContext` to be used in this crate. diff --git a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs index ea264c8064ad1..edc7c73968325 100644 --- a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs +++ b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs @@ -422,49 +422,18 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { Place::ty_from(local, proj_base, self.body, self.infcx.tcx).ty )); - let captured_place = self.upvars[upvar_index.index()]; - - err.span_label(span, format!("cannot {act}")); - - let upvar_hir_id = captured_place.get_root_variable(); - - if let Node::Pat(pat) = self.infcx.tcx.hir_node(upvar_hir_id) - && let hir::PatKind::Binding(hir::BindingMode::NONE, _, upvar_ident, _) = - pat.kind - { - if upvar_ident.name == kw::SelfLower { - for (_, node) in self.infcx.tcx.hir_parent_iter(upvar_hir_id) { - if let Some(fn_decl) = node.fn_decl() { - if !matches!( - fn_decl.implicit_self, - hir::ImplicitSelfKind::RefImm | hir::ImplicitSelfKind::RefMut - ) { - err.span_suggestion_verbose( - upvar_ident.span.shrink_to_lo(), - "consider changing this to be mutable", - "mut ", - Applicability::MachineApplicable, - ); - break; - } - } - } - } else { - err.span_suggestion_verbose( - upvar_ident.span.shrink_to_lo(), - "consider changing this to be mutable", - "mut ", - Applicability::MachineApplicable, - ); - } - } + self.suggest_mutable_upvar(*upvar_index, the_place_err, &mut err, span, act); + } - let tcx = self.infcx.tcx; - if let ty::Ref(_, ty, Mutability::Mut) = the_place_err.ty(self.body, tcx).ty.kind() - && let ty::Closure(id, _) = *ty.kind() - { - self.show_mutating_upvar(tcx, id.expect_local(), the_place_err, &mut err); - } + PlaceRef { local, projection: [] } + if let Some(upvar_index) = self + .body + .local_upvar_map + .iter_enumerated() + .filter_map(|(field, &local)| local.map(|local| (field, local))) + .find_map(|(field, relocated)| (relocated == local).then_some(field)) => + { + self.suggest_mutable_upvar(upvar_index, the_place_err, &mut err, span, act); } // Complete hack to approximate old AST-borrowck diagnostic: if the span starts @@ -570,6 +539,58 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { } } + fn suggest_mutable_upvar( + &self, + upvar_index: FieldIdx, + the_place_err: PlaceRef<'tcx>, + err: &mut Diag<'infcx>, + span: Span, + act: &str, + ) { + let captured_place = self.upvars[upvar_index.index()]; + + err.span_label(span, format!("cannot {act}")); + + let upvar_hir_id = captured_place.get_root_variable(); + + if let Node::Pat(pat) = self.infcx.tcx.hir_node(upvar_hir_id) + && let hir::PatKind::Binding(hir::BindingMode::NONE, _, upvar_ident, _) = pat.kind + { + if upvar_ident.name == kw::SelfLower { + for (_, node) in self.infcx.tcx.hir_parent_iter(upvar_hir_id) { + if let Some(fn_decl) = node.fn_decl() { + if !matches!( + fn_decl.implicit_self, + hir::ImplicitSelfKind::RefImm | hir::ImplicitSelfKind::RefMut + ) { + err.span_suggestion_verbose( + upvar_ident.span.shrink_to_lo(), + "consider changing this to be mutable", + "mut ", + Applicability::MachineApplicable, + ); + break; + } + } + } + } else { + err.span_suggestion_verbose( + upvar_ident.span.shrink_to_lo(), + "consider changing this to be mutable", + "mut ", + Applicability::MachineApplicable, + ); + } + } + + let tcx = self.infcx.tcx; + if let ty::Ref(_, ty, Mutability::Mut) = the_place_err.ty(self.body, tcx).ty.kind() + && let ty::Closure(id, _) = *ty.kind() + { + self.show_mutating_upvar(tcx, id.expect_local(), the_place_err, err); + } + } + /// Suggest `map[k] = v` => `map.insert(k, v)` and the like. fn suggest_map_index_mut_alternatives(&self, ty: Ty<'tcx>, err: &mut Diag<'infcx>, span: Span) { let Some(adt) = ty.ty_adt_def() else { return }; diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs index ce78ae203a4d8..ea16cacc06c9b 100644 --- a/compiler/rustc_borrowck/src/lib.rs +++ b/compiler/rustc_borrowck/src/lib.rs @@ -26,6 +26,7 @@ use root_cx::BorrowCheckRootCtxt; use rustc_abi::FieldIdx; use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; use rustc_data_structures::graph::dominators::Dominators; +use rustc_data_structures::unord::UnordMap; use rustc_errors::LintDiagnostic; use rustc_hir as hir; use rustc_hir::CRATE_HIR_ID; @@ -429,6 +430,7 @@ fn do_mir_borrowck<'tcx>( regioncx: ®ioncx, used_mut: Default::default(), used_mut_upvars: SmallVec::new(), + local_from_upvars: UnordMap::default(), borrow_set: &borrow_set, upvars: &[], local_names: OnceCell::from(IndexVec::from_elem(None, &promoted_body.local_decls)), @@ -454,6 +456,12 @@ fn do_mir_borrowck<'tcx>( promoted_mbcx.report_move_errors(); } + let mut local_from_upvars = UnordMap::default(); + for (field, &local) in body.local_upvar_map.iter_enumerated() { + let Some(local) = local else { continue }; + local_from_upvars.insert(local, field); + } + let mut mbcx = MirBorrowckCtxt { root_cx, infcx: &infcx, @@ -468,6 +476,7 @@ fn do_mir_borrowck<'tcx>( regioncx: ®ioncx, used_mut: Default::default(), used_mut_upvars: SmallVec::new(), + local_from_upvars, borrow_set: &borrow_set, upvars: tcx.closure_captures(def), local_names: OnceCell::new(), @@ -702,6 +711,9 @@ struct MirBorrowckCtxt<'a, 'infcx, 'tcx> { /// If the function we're checking is a closure, then we'll need to report back the list of /// mutable upvars that have been used. This field keeps track of them. used_mut_upvars: SmallVec<[FieldIdx; 8]>, + /// Since upvars may be moved to real locals, we need to map mutations to the locals back to + /// the upvars, so that used_mut_upvars is up-to-date. + local_from_upvars: UnordMap, /// Region inference context. This contains the results from region inference and lets us e.g. /// find out which CFG points are contained in each borrow region. regioncx: &'a RegionInferenceContext<'tcx>, @@ -2420,7 +2432,9 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { // at this point, we have set up the error reporting state. if let Some(init_index) = previously_initialized { - if let (AccessKind::Mutate, Some(_)) = (error_access, place.as_local()) { + if let (AccessKind::Mutate, Some(local)) = (error_access, place.as_local()) + && !self.local_from_upvars.contains_key(&local) + { // If this is a mutate access to an immutable local variable with no projections // report the error as an illegal reassignment let init = &self.move_data.inits[init_index]; @@ -2448,10 +2462,12 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { // If the local may have been initialized, and it is now currently being // mutated, then it is justified to be annotated with the `mut` // keyword, since the mutation may be a possible reassignment. - if is_local_mutation_allowed != LocalMutationIsAllowed::Yes - && self.is_local_ever_initialized(local, state).is_some() - { - self.used_mut.insert(local); + if !matches!(is_local_mutation_allowed, LocalMutationIsAllowed::Yes) { + if self.is_local_ever_initialized(local, state).is_some() { + self.used_mut.insert(local); + } else if let Some(&field) = self.local_from_upvars.get(&local) { + self.used_mut_upvars.push(field); + } } } RootPlace { @@ -2469,6 +2485,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { projection: place_projection, }) { self.used_mut_upvars.push(field); + } else if let Some(&field) = self.local_from_upvars.get(&place_local) { + self.used_mut_upvars.push(field); } } } @@ -2622,6 +2640,13 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { /// of a closure type. fn is_upvar_field_projection(&self, place_ref: PlaceRef<'tcx>) -> Option { path_utils::is_upvar_field_projection(self.infcx.tcx, &self.upvars, place_ref, self.body()) + .or_else(|| { + path_utils::is_relocated_upvar_field_projection( + &self.upvars, + &self.local_from_upvars, + place_ref, + ) + }) } fn dominators(&self) -> &Dominators { diff --git a/compiler/rustc_borrowck/src/path_utils.rs b/compiler/rustc_borrowck/src/path_utils.rs index 2c94a32d369ce..14ca03859b40e 100644 --- a/compiler/rustc_borrowck/src/path_utils.rs +++ b/compiler/rustc_borrowck/src/path_utils.rs @@ -2,8 +2,9 @@ use std::ops::ControlFlow; use rustc_abi::FieldIdx; use rustc_data_structures::graph::dominators::Dominators; -use rustc_middle::mir::{BasicBlock, Body, Location, Place, PlaceRef, ProjectionElem}; -use rustc_middle::ty::TyCtxt; +use rustc_data_structures::unord::UnordMap; +use rustc_middle::mir::{BasicBlock, Body, Local, Location, Place, PlaceRef, ProjectionElem}; +use rustc_middle::ty::{CapturedPlace, TyCtxt}; use tracing::debug; use crate::borrow_set::{BorrowData, BorrowSet, TwoPhaseActivation}; @@ -133,7 +134,7 @@ pub(super) fn borrow_of_local_data(place: Place<'_>) -> bool { /// of a closure type. pub(crate) fn is_upvar_field_projection<'tcx>( tcx: TyCtxt<'tcx>, - upvars: &[&rustc_middle::ty::CapturedPlace<'tcx>], + upvars: &[&CapturedPlace<'tcx>], place_ref: PlaceRef<'tcx>, body: &Body<'tcx>, ) -> Option { @@ -159,3 +160,24 @@ pub(crate) fn is_upvar_field_projection<'tcx>( _ => None, } } + +pub(crate) fn is_relocated_upvar_field_projection<'tcx>( + upvars: &[&CapturedPlace<'tcx>], + local_from_upvars: &UnordMap, + mut place_ref: PlaceRef<'tcx>, +) -> Option { + let mut by_ref = false; + + if let Some((place_base, ProjectionElem::Deref)) = place_ref.last_projection() { + place_ref = place_base; + by_ref = true; + } + if let Some(local) = place_ref.as_local() + && let Some(&field) = local_from_upvars.get(&local) + && (!by_ref || upvars[field.index()].is_by_ref()) + { + Some(field) + } else { + None + } +} diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index 0d363935f149e..b0c6337a007aa 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -25,8 +25,8 @@ use rustc_middle::traits::query::NoSolution; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::cast::CastTy; use rustc_middle::ty::{ - self, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, CoroutineArgsExt, - GenericArgsRef, Ty, TyCtxt, TypeVisitableExt, UserArgs, UserTypeAnnotationIndex, fold_regions, + self, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, GenericArgsRef, Ty, TyCtxt, + TypeVisitableExt, UserArgs, UserTypeAnnotationIndex, fold_regions, }; use rustc_middle::{bug, span_bug}; use rustc_mir_dataflow::move_paths::MoveData; @@ -160,6 +160,7 @@ pub(crate) fn type_check<'tcx>( polonius_liveness, }; + typeck.mark_upvar_locals_invariant(&normalized_inputs_and_output); typeck.check_user_type_annotations(); typeck.visit_body(body); typeck.equate_inputs_and_outputs(&normalized_inputs_and_output); @@ -396,6 +397,31 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { } } + /// In case that upvars are relocated into inner locals, + /// we should assert that the respective region variables are invariant + /// to each other. + /// + /// NOTE(@dingxiangfei2009): to implementers, invariant relations are built in order to + /// maintain parity with how the language has handled captures in view of lifetimes. + /// This may change depending on how #140132 would be resolved. + #[instrument(level = "debug", skip(self))] + fn mark_upvar_locals_invariant(&mut self, inputs_outputs: &[Ty<'tcx>]) { + let upvar_tys = self.universal_regions.defining_ty.upvar_tys(); + let span = self.body.span; + for (local, upvar_ty) in self.body.local_upvar_map.iter().zip(upvar_tys) { + let &Some(local) = local else { continue }; + let local_ty = self.body.local_decls[local].ty; + if let Err(_no_solution) = + self.eq_types(local_ty, upvar_ty, Locations::All(span), ConstraintCategory::Boring) + { + span_bug!( + span, + "upvars are relocated to locals equipped with unequatable types, upvar has type {upvar_ty:?} but local has {local_ty:?}" + ); + } + } + } + #[instrument(skip(self, data), level = "debug")] fn push_region_constraints( &mut self, @@ -2158,14 +2184,15 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { } } AggregateKind::Coroutine(_, args) => { - // It doesn't make sense to look at a field beyond the prefix; - // these require a variant index, and are not initialized in - // aggregate rvalues. - match args.as_coroutine().prefix_tys().get(field_index.as_usize()) { - Some(ty) => Ok(*ty), - None => Err(FieldAccessError::OutOfRange { - field_count: args.as_coroutine().prefix_tys().len(), - }), + // It doesn't make sense to look at a field beyond the captured + // upvars. + // Otherwise it require a variant index, and are not initialized + // in aggregate rvalues. + let upvar_tys = &args.as_coroutine().upvar_tys(); + if let Some(ty) = upvar_tys.get(field_index.as_usize()) { + Ok(*ty) + } else { + Err(FieldAccessError::OutOfRange { field_count: upvar_tys.len() }) } } AggregateKind::CoroutineClosure(_, args) => { diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index bc0a0f034b236..b4eea3456a773 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -912,6 +912,9 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: let variant_dest = lval.downcast_variant(fx, variant_index); (variant_index, variant_dest, active_field_index) } + mir::AggregateKind::Coroutine(_def_id, _args) => { + (FIRST_VARIANT, lval.downcast_variant(fx, FIRST_VARIANT), None) + } _ => (FIRST_VARIANT, lval, None), }; if active_field_index.is_some() { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 0e9dbfba658d2..40114c18ca081 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -15,9 +15,7 @@ use rustc_middle::bug; use rustc_middle::ty::layout::{ HasTypingEnv, LayoutOf, TyAndLayout, WIDE_PTR_ADDR, WIDE_PTR_EXTRA, }; -use rustc_middle::ty::{ - self, AdtKind, CoroutineArgsExt, ExistentialTraitRef, Instance, Ty, TyCtxt, Visibility, -}; +use rustc_middle::ty::{self, AdtKind, ExistentialTraitRef, Instance, Ty, TyCtxt, Visibility}; use rustc_session::config::{self, DebugInfo, Lto}; use rustc_span::{DUMMY_SP, FileName, FileNameDisplayPreference, SourceFile, Symbol, hygiene}; use rustc_symbol_mangling::typeid_for_trait_ref; @@ -1131,7 +1129,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( closure_or_coroutine_di_node: &'ll DIType, ) -> SmallVec<&'ll DIType> { let (&def_id, up_var_tys) = match closure_or_coroutine_ty.kind() { - ty::Coroutine(def_id, args) => (def_id, args.as_coroutine().prefix_tys()), + ty::Coroutine(def_id, args) => (def_id, args.as_coroutine().upvar_tys()), ty::Closure(def_id, args) => (def_id, args.as_closure().upvar_tys()), ty::CoroutineClosure(def_id, args) => (def_id, args.as_coroutine_closure().upvar_tys()), _ => { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index a5c808957410e..ee38573574f20 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -723,7 +723,6 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>( let coroutine_layout = cx.tcx.coroutine_layout(coroutine_def_id, coroutine_args.args).unwrap(); - let common_upvar_names = cx.tcx.closure_saved_names_of_captured_variables(coroutine_def_id); let variant_range = coroutine_args.variant_range(coroutine_def_id, cx.tcx); let variant_count = (variant_range.start.as_u32()..variant_range.end.as_u32()).len(); @@ -763,7 +762,6 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>( coroutine_type_and_layout, coroutine_type_di_node, coroutine_layout, - common_upvar_names, ); let span = coroutine_layout.variant_source_info[variant_index].span; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index 7c701926d2c5e..07bd9aeb63c85 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -5,12 +5,10 @@ use rustc_codegen_ssa::debuginfo::type_names::{compute_debuginfo_type_name, cpp_ use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; use rustc_codegen_ssa::traits::MiscCodegenMethods; use rustc_hir::def::CtorKind; -use rustc_index::IndexSlice; use rustc_middle::bug; use rustc_middle::mir::CoroutineLayout; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty, VariantDef}; -use rustc_span::Symbol; use super::type_map::{DINodeCreationResult, UniqueTypeId}; use super::{SmallVec, size_and_align_of}; @@ -286,7 +284,6 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( coroutine_type_and_layout: TyAndLayout<'tcx>, coroutine_type_di_node: &'ll DIType, coroutine_layout: &CoroutineLayout<'tcx>, - common_upvar_names: &IndexSlice, ) -> &'ll DIType { let variant_name = CoroutineArgs::variant_name(variant_index); let unique_type_id = UniqueTypeId::for_enum_variant_struct_type( @@ -297,11 +294,6 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( let variant_layout = coroutine_type_and_layout.for_variant(cx, variant_index); - let coroutine_args = match coroutine_type_and_layout.ty.kind() { - ty::Coroutine(_, args) => args.as_coroutine(), - _ => unreachable!(), - }; - type_map::build_type_with_children( cx, type_map::stub( @@ -316,7 +308,7 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( ), |cx, variant_struct_type_di_node| { // Fields that just belong to this variant/state - let state_specific_fields: SmallVec<_> = (0..variant_layout.fields.count()) + (0..variant_layout.fields.count()) .map(|field_index| { let coroutine_saved_local = coroutine_layout.variant_fields[variant_index] [FieldIdx::from_usize(field_index)]; @@ -339,29 +331,7 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( None, ) }) - .collect(); - - // Fields that are common to all states - let common_fields: SmallVec<_> = coroutine_args - .prefix_tys() - .iter() - .zip(common_upvar_names) - .enumerate() - .map(|(index, (upvar_ty, upvar_name))| { - build_field_di_node( - cx, - variant_struct_type_di_node, - upvar_name.as_str(), - cx.layout_of(upvar_ty), - coroutine_type_and_layout.fields.offset(index), - DIFlags::FlagZero, - type_di_node(cx, upvar_ty), - None, - ) - }) - .collect(); - - state_specific_fields.into_iter().chain(common_fields).collect() + .collect() }, |cx| build_generic_type_param_di_nodes(cx, coroutine_type_and_layout.ty), ) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index 62d38d463aba7..54b3cbc39d129 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -186,9 +186,6 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( ) }; - let common_upvar_names = - cx.tcx.closure_saved_names_of_captured_variables(coroutine_def_id); - // Build variant struct types let variant_struct_type_di_nodes: SmallVec<_> = variants .indices() @@ -216,7 +213,6 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( coroutine_type_and_layout, coroutine_type_di_node, coroutine_layout, - common_upvar_names, ), source_info, } diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index d851c3329802c..33e1e3cbeb0fc 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -927,13 +927,12 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue { } impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + #[instrument(level = "debug", skip(self, bx), ret)] fn maybe_codegen_consume_direct( &mut self, bx: &mut Bx, place_ref: mir::PlaceRef<'tcx>, ) -> Option> { - debug!("maybe_codegen_consume_direct(place_ref={:?})", place_ref); - match self.locals[place_ref.local] { LocalRef::Operand(mut o) => { // We only need to handle the projections that @@ -978,13 +977,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } + #[instrument(level = "debug", skip(self, bx), ret)] pub fn codegen_consume( &mut self, bx: &mut Bx, place_ref: mir::PlaceRef<'tcx>, ) -> OperandRef<'tcx, Bx::Value> { - debug!("codegen_consume(place_ref={:?})", place_ref); - let ty = self.monomorphized_place_ty(place_ref); let layout = bx.cx().layout_of(ty); @@ -1003,13 +1001,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.load_operand(place) } + #[instrument(level = "debug", skip(self, bx), ret)] pub fn codegen_operand( &mut self, bx: &mut Bx, operand: &mir::Operand<'tcx>, ) -> OperandRef<'tcx, Bx::Value> { - debug!("codegen_operand(operand={:?})", operand); - match *operand { mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => { self.codegen_consume(bx, place.as_ref()) diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 8a67b8d6e5f13..807c837e29a74 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -157,6 +157,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let variant_dest = dest.project_downcast(bx, variant_index); (variant_index, variant_dest, active_field_index) } + mir::AggregateKind::Coroutine(_, _) => { + (FIRST_VARIANT, dest.project_downcast(bx, FIRST_VARIANT), None) + } _ => (FIRST_VARIANT, dest, None), }; if active_field_index.is_some() { diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index 23d362de308f3..3c39f22cdf22e 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -316,6 +316,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let variant_dest = self.project_downcast(dest, variant_index)?; (variant_index, variant_dest, active_field_index) } + mir::AggregateKind::Coroutine(_def_id, _args) => { + (FIRST_VARIANT, self.project_downcast(dest, FIRST_VARIANT)?, None) + } mir::AggregateKind::RawPtr(..) => { // Pointers don't have "fields" in the normal sense, so the // projection-based code below would either fail in projection diff --git a/compiler/rustc_index/src/vec.rs b/compiler/rustc_index/src/vec.rs index 13f0dda180be9..90eb212e69bf5 100644 --- a/compiler/rustc_index/src/vec.rs +++ b/compiler/rustc_index/src/vec.rs @@ -197,6 +197,11 @@ impl IndexVec { pub fn append(&mut self, other: &mut Self) { self.raw.append(&mut other.raw); } + + #[inline] + pub fn debug_map_view(&self) -> IndexSliceMapView<'_, I, T> { + IndexSliceMapView(self.as_slice()) + } } /// `IndexVec` is often used as a map, so it provides some map-like APIs. @@ -220,14 +225,44 @@ impl IndexVec> { pub fn contains(&self, index: I) -> bool { self.get(index).and_then(Option::as_ref).is_some() } + + #[inline] + pub fn debug_map_view_compact(&self) -> IndexSliceMapViewCompact<'_, I, T> { + IndexSliceMapViewCompact(self.as_slice()) + } } +pub struct IndexSliceMapView<'a, I: Idx, T>(&'a IndexSlice); +pub struct IndexSliceMapViewCompact<'a, I: Idx, T>(&'a IndexSlice>); + impl fmt::Debug for IndexVec { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&self.raw, fmt) } } +impl<'a, I: Idx, T: fmt::Debug> fmt::Debug for IndexSliceMapView<'a, I, T> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut entries = fmt.debug_map(); + for (idx, val) in self.0.iter_enumerated() { + entries.entry(&idx, val); + } + entries.finish() + } +} + +impl<'a, I: Idx, T: fmt::Debug> fmt::Debug for IndexSliceMapViewCompact<'a, I, T> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut entries = fmt.debug_map(); + for (idx, val) in self.0.iter_enumerated() { + if let Some(val) = val { + entries.entry(&idx, val); + } + } + entries.finish() + } +} + impl Deref for IndexVec { type Target = IndexSlice; diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index c977e5329c292..647c43c03fa80 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -333,6 +333,9 @@ pub struct Body<'tcx> { #[type_foldable(identity)] #[type_visitable(ignore)] pub function_coverage_info: Option>, + + /// Coroutine local-upvar map + pub local_upvar_map: IndexVec>, } impl<'tcx> Body<'tcx> { @@ -376,6 +379,7 @@ impl<'tcx> Body<'tcx> { tainted_by_errors, coverage_info_hi: None, function_coverage_info: None, + local_upvar_map: IndexVec::new(), }; body.is_polymorphic = body.has_non_region_param(); body @@ -407,6 +411,7 @@ impl<'tcx> Body<'tcx> { tainted_by_errors: None, coverage_info_hi: None, function_coverage_info: None, + local_upvar_map: IndexVec::new(), }; body.is_polymorphic = body.has_non_region_param(); body diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs index a8a95c699d880..83b3262d7ebf3 100644 --- a/compiler/rustc_middle/src/mir/query.rs +++ b/compiler/rustc_middle/src/mir/query.rs @@ -9,7 +9,9 @@ use rustc_hir::def_id::LocalDefId; use rustc_index::IndexVec; use rustc_index::bit_set::BitMatrix; use rustc_macros::{HashStable, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable}; +use rustc_session::config::PackCoroutineLayout; use rustc_span::{Span, Symbol}; +use rustc_type_ir::data_structures::IndexMap; use super::{ConstValue, SourceInfo}; use crate::ty::{self, CoroutineArgsExt, OpaqueHiddenType, Ty}; @@ -17,7 +19,7 @@ use crate::ty::{self, CoroutineArgsExt, OpaqueHiddenType, Ty}; rustc_index::newtype_index! { #[derive(HashStable)] #[encodable] - #[debug_format = "_{}"] + #[debug_format = "corsl_{}"] pub struct CoroutineSavedLocal {} } @@ -55,6 +57,18 @@ pub struct CoroutineLayout<'tcx> { #[type_foldable(identity)] #[type_visitable(ignore)] pub storage_conflicts: BitMatrix, + + /// This map `A -> B` allows later MIR passes, error reporters + /// and layout calculator to relate saved locals `A` sourced from upvars + /// and locals `B` that upvars are moved into. + #[type_foldable(identity)] + #[type_visitable(ignore)] + pub relocated_upvars: IndexMap, + + /// Coroutine layout packing + #[type_foldable(identity)] + #[type_visitable(ignore)] + pub pack: PackCoroutineLayout, } impl Debug for CoroutineLayout<'_> { @@ -80,6 +94,7 @@ impl Debug for CoroutineLayout<'_> { map.finish() }) .field("storage_conflicts", &self.storage_conflicts) + .field("relocated_upvars", &self.relocated_upvars) .finish() } } diff --git a/compiler/rustc_middle/src/mir/statement.rs b/compiler/rustc_middle/src/mir/statement.rs index 6e52bc775ef73..84930a44871bb 100644 --- a/compiler/rustc_middle/src/mir/statement.rs +++ b/compiler/rustc_middle/src/mir/statement.rs @@ -133,13 +133,13 @@ impl<'tcx> PlaceTy<'tcx> { .get(f.index()) .copied() .unwrap_or_else(|| bug!("field {f:?} out of range: {self_ty:?}")), - // Only prefix fields (upvars and current state) are - // accessible without a variant index. - ty::Coroutine(_, args) => { - args.as_coroutine().prefix_tys().get(f.index()).copied().unwrap_or_else(|| { - bug!("field {f:?} out of range of prefixes for {self_ty}") - }) - } + // Only upvars are accessible without a variant index. + ty::Coroutine(_, args) => args + .as_coroutine() + .upvar_tys() + .get(f.index()) + .copied() + .unwrap_or_else(|| bug!("field {f:?} out of range: {self_ty:?}")), ty::Tuple(tys) => tys .get(f.index()) .copied() diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs index 3a51f79f12169..e0f01c708d085 100644 --- a/compiler/rustc_middle/src/ty/instance.rs +++ b/compiler/rustc_middle/src/ty/instance.rs @@ -409,15 +409,14 @@ impl<'tcx> fmt::Display for Instance<'tcx> { // async_drop_in_place::coroutine's poll function (through FutureDropPollShim proxy) fn resolve_async_drop_poll<'tcx>(mut cor_ty: Ty<'tcx>) -> Instance<'tcx> { let first_cor = cor_ty; - let ty::Coroutine(poll_def_id, proxy_args) = first_cor.kind() else { + let &ty::Coroutine(poll_def_id, proxy_args) = first_cor.kind() else { bug!(); }; - let poll_def_id = *poll_def_id; let mut child_ty = cor_ty; loop { - if let ty::Coroutine(child_def, child_args) = child_ty.kind() { + if let &ty::Coroutine(child_def, child_args) = child_ty.kind() { cor_ty = child_ty; - if *child_def == poll_def_id { + if child_def == poll_def_id { child_ty = child_args.first().unwrap().expect_ty(); continue; } else { diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index aed94f9aa04d8..6dace191f958f 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -921,9 +921,10 @@ where ), Variants::Multiple { tag, tag_field, .. } => { if FieldIdx::from_usize(i) == tag_field { - return TyMaybeWithLayout::TyAndLayout(tag_layout(tag)); + TyMaybeWithLayout::TyAndLayout(tag_layout(tag)) + } else { + TyMaybeWithLayout::Ty(args.as_coroutine().upvar_tys()[i]) } - TyMaybeWithLayout::Ty(args.as_coroutine().prefix_tys()[i]) } }, diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index a7298af502ef1..714757c67a580 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -39,13 +39,14 @@ use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LocalDefIdMap}; use rustc_hir::definitions::DisambiguatorState; use rustc_hir::{LangItem, attrs as attr, find_attr}; use rustc_index::IndexVec; -use rustc_index::bit_set::BitMatrix; +use rustc_index::bit_set::{BitMatrix, DenseBitSet}; use rustc_macros::{ Decodable, Encodable, HashStable, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable, extension, }; use rustc_query_system::ich::StableHashingContext; use rustc_serialize::{Decodable, Encodable}; +use rustc_session::config::PackCoroutineLayout; pub use rustc_session::lint::RegisteredTools; use rustc_span::hygiene::MacroKind; use rustc_span::{DUMMY_SP, ExpnId, ExpnKind, Ident, Span, Symbol, sym}; @@ -62,6 +63,7 @@ pub use rustc_type_ir::solve::SizedTraitKind; pub use rustc_type_ir::*; #[allow(hidden_glob_reexports, unused_imports)] use rustc_type_ir::{InferCtxtLike, Interner}; +use smallvec::SmallVec; use tracing::{debug, instrument}; pub use vtable::*; use {rustc_ast as ast, rustc_hir as hir}; @@ -113,7 +115,7 @@ pub use self::visit::*; use crate::error::{OpaqueHiddenTypeMismatch, TypeMismatchReason}; use crate::metadata::ModChild; use crate::middle::privacy::EffectiveVisibilities; -use crate::mir::{Body, CoroutineLayout, CoroutineSavedLocal, SourceInfo}; +use crate::mir::{Body, CoroutineLayout, CoroutineSavedLocal, CoroutineSavedTy, SourceInfo}; use crate::query::{IntoQueryParam, Providers}; use crate::ty; use crate::ty::codec::{TyDecoder, TyEncoder}; @@ -1881,39 +1883,77 @@ impl<'tcx> TyCtxt<'tcx> { .ok_or_else(|| self.layout_error(LayoutError::Unknown(ty()))) } + /// Construct the trivial coroutine layout for async_drop of a coroutine + #[instrument(level = "debug", ret, skip(self))] + fn build_async_drop_trivial_coroutine_layout( + self, + def_id: DefId, + upvar_ty: Ty<'tcx>, + ) -> CoroutineLayout<'tcx> { + let upvar_tys = [upvar_ty]; + let nr_upvars = 1; + let span = self.def_span(def_id); + let source_info = SourceInfo::outermost(span); + let mut field_tys = IndexVec::new(); + let field_names = [None].into(); + let upvar_saved_locals: SmallVec<[_; 4]> = upvar_tys + .into_iter() + .map(|ty| field_tys.push(CoroutineSavedTy { ty, source_info, ignore_for_traits: true })) + .collect(); + // Even minimal, the trivial coroutine has 3 states (RESERVED_VARIANTS), + // so variant_fields and variant_source_info should have 3 elements. + let mut variant_fields: IndexVec> = + iter::repeat(IndexVec::new()).take(CoroutineArgs::RESERVED_VARIANTS).collect(); + variant_fields[VariantIdx::ZERO].extend(upvar_saved_locals.clone()); + let variant_source_info: IndexVec = + iter::repeat(source_info).take(CoroutineArgs::RESERVED_VARIANTS).collect(); + let mut storage_conflicts = BitMatrix::new(nr_upvars, nr_upvars); + if let Some(&first) = upvar_saved_locals.first() + && let Some(&last) = upvar_saved_locals.last() + { + let mut upvars = DenseBitSet::new_empty(nr_upvars); + upvars.insert_range(first..=last); + for &local in &upvar_saved_locals { + storage_conflicts.union_row_with(&upvars, local); + } + } + let relocated_upvars = Default::default(); + CoroutineLayout { + field_tys, + field_names, + variant_fields, + variant_source_info, + storage_conflicts, + relocated_upvars, + pack: PackCoroutineLayout::CapturesOnly, + } + } + /// Returns layout of a coroutine. Layout might be unavailable if the /// coroutine is tainted by errors. + #[instrument(level = "debug", skip(self), ret)] pub fn coroutine_layout( self, def_id: DefId, args: GenericArgsRef<'tcx>, ) -> Result<&'tcx CoroutineLayout<'tcx>, &'tcx LayoutError<'tcx>> { if self.is_async_drop_in_place_coroutine(def_id) { - // layout of `async_drop_in_place::{closure}` in case, - // when T is a coroutine, contains this internal coroutine's ptr in upvars - // and doesn't require any locals. Here is an `empty coroutine's layout` + // Layout of `async_drop_in_place::{closure}` when T is a coroutine + // It contains exactly one upvar which is a raw pointer to this inner coroutine + // and it does not suspend. let arg_cor_ty = args.first().unwrap().expect_ty(); - if arg_cor_ty.is_coroutine() { - let span = self.def_span(def_id); - let source_info = SourceInfo::outermost(span); - // Even minimal, empty coroutine has 3 states (RESERVED_VARIANTS), - // so variant_fields and variant_source_info should have 3 elements. - let variant_fields: IndexVec> = - iter::repeat(IndexVec::new()).take(CoroutineArgs::RESERVED_VARIANTS).collect(); - let variant_source_info: IndexVec = - iter::repeat(source_info).take(CoroutineArgs::RESERVED_VARIANTS).collect(); - let proxy_layout = CoroutineLayout { - field_tys: [].into(), - field_names: [].into(), - variant_fields, - variant_source_info, - storage_conflicts: BitMatrix::new(0, 0), - }; - return Ok(self.arena.alloc(proxy_layout)); + if let &ty::Coroutine(_did, _args) = arg_cor_ty.kind() { + debug!("it is a trivial coroutine"); + let upvar_ty = Ty::new_mut_ptr(self, arg_cor_ty); + Ok(self + .arena + .alloc(self.build_async_drop_trivial_coroutine_layout(def_id, upvar_ty))) } else { + debug!("it is an async drop coroutine"); self.async_drop_coroutine_layout(def_id, args) } } else { + debug!("it is an ordinary coroutine"); self.ordinary_coroutine_layout(def_id, args) } } diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index 755fc68d86f34..6fea11516d841 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -149,13 +149,6 @@ impl<'tcx> ty::CoroutineArgs> { }) }) } - - /// This is the types of the fields of a coroutine which are not stored in a - /// variant. - #[inline] - fn prefix_tys(self) -> &'tcx List> { - self.upvar_tys() - } } #[derive(Debug, Copy, Clone, HashStable, TypeFoldable, TypeVisitable)] diff --git a/compiler/rustc_mir_build/src/builder/custom/mod.rs b/compiler/rustc_mir_build/src/builder/custom/mod.rs index 792ad6d782cf3..9f39bf236c312 100644 --- a/compiler/rustc_mir_build/src/builder/custom/mod.rs +++ b/compiler/rustc_mir_build/src/builder/custom/mod.rs @@ -62,6 +62,7 @@ pub(super) fn build_custom_mir<'tcx>( pass_count: 0, coverage_info_hi: None, function_coverage_info: None, + local_upvar_map: IndexVec::new(), }; body.local_decls.push(LocalDecl::new(return_ty, return_ty_span)); diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index 761d5461a996f..ad61ae78a9e86 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -52,7 +52,9 @@ mod by_move_body; mod drop; -use std::{iter, ops}; +mod relocate_upvars; + +use std::ops::Deref; pub(super) use by_move_body::coroutine_by_move_body_def_id; use drop::{ @@ -60,8 +62,10 @@ use drop::{ create_coroutine_drop_shim_proxy_async, elaborate_coroutine_drops, expand_async_drops, has_expandable_async_drops, insert_clean_drop, }; +pub(super) use relocate_upvars::RelocateUpvars; use rustc_abi::{FieldIdx, VariantIdx}; use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::unord::UnordMap; use rustc_errors::pluralize; use rustc_hir as hir; use rustc_hir::lang_items::LangItem; @@ -82,14 +86,16 @@ use rustc_mir_dataflow::impls::{ use rustc_mir_dataflow::{ Analysis, Results, ResultsCursor, ResultsVisitor, visit_reachable_results, }; +use rustc_session::config::PackCoroutineLayout; use rustc_span::def_id::{DefId, LocalDefId}; use rustc_span::source_map::dummy_spanned; use rustc_span::symbol::sym; -use rustc_span::{DUMMY_SP, Span}; +use rustc_span::{DUMMY_SP, Span, Symbol, kw}; use rustc_target::spec::PanicStrategy; use rustc_trait_selection::error_reporting::InferCtxtErrorExt; use rustc_trait_selection::infer::TyCtxtInferExt as _; use rustc_trait_selection::traits::{ObligationCause, ObligationCauseCode, ObligationCtxt}; +use smallvec::{SmallVec, smallvec}; use tracing::{debug, instrument, trace}; use crate::deref_separator::deref_finder; @@ -103,6 +109,8 @@ struct RenameLocalVisitor<'tcx> { tcx: TyCtxt<'tcx>, } +const VARIANT_UNRESUMED: VariantIdx = VariantIdx::from_usize(CoroutineArgs::UNRESUMED); + impl<'tcx> MutVisitor<'tcx> for RenameLocalVisitor<'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx @@ -199,8 +207,11 @@ struct TransformVisitor<'tcx> { // A map from a suspension point in a block to the locals which have live storage at that point storage_liveness: IndexVec>>, + // A rev-lookup from basic blocks with yielding terminator to the suspension point index, + suspension_point_at_block: UnordMap, + // A list of suspension points, generated during the transform - suspension_points: Vec>, + suspension_points: IndexVec>>, // The set of locals that have no `StorageLive`/`StorageDead` annotations. always_live_locals: DenseBitSet, @@ -403,6 +414,16 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { // Replace an Local in the remap with a coroutine struct access if let Some(&Some((ty, variant_index, idx))) = self.remap.get(place.local) { replace_base(place, self.make_field(variant_index, idx, ty), self.tcx); + } else if let Place { local: ty::CAPTURE_STRUCT_LOCAL, projection } = *place + && let [first @ ProjectionElem::Field(..), rest @ ..] = &**projection + { + let projections: Vec<_> = [ProjectionElem::Downcast(None, VARIANT_UNRESUMED), *first] + .into_iter() + .chain(rest.iter().copied()) + .collect(); + let new_place = + Place::from(ty::CAPTURE_STRUCT_LOCAL).project_deeper(&projections, self.tcx); + *place = new_place; } } @@ -431,8 +452,9 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { // We must assign the value first in case it gets declared dead below self.make_state(v, source_info, is_return, &mut data.statements); let state = if let Some((resume, mut resume_arg)) = resume { - // Yield - let state = CoroutineArgs::RESERVED_VARIANTS + self.suspension_points.len(); + // This is a `yield` + let suspension_point_idx = *self.suspension_point_at_block.get(&block).unwrap(); + let state = CoroutineArgs::RESERVED_VARIANTS + suspension_point_idx.as_usize(); // The resume arg target location might itself be remapped if its base local is // live across a yield. @@ -454,12 +476,8 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { } } - self.suspension_points.push(SuspensionPoint { - state, - resume, - resume_arg, - drop, - storage_liveness, + self.suspension_points.get_or_insert_with(suspension_point_idx, || { + SuspensionPoint { state, resume, resume_arg, drop, storage_liveness } }); VariantIdx::new(state) @@ -485,19 +503,19 @@ fn make_aggregate_adt<'tcx>( } fn make_coroutine_state_argument_indirect<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let coroutine_ty = body.local_decls.raw[1].ty; + let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; let ref_coroutine_ty = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, coroutine_ty); // Replace the by value coroutine argument - body.local_decls.raw[1].ty = ref_coroutine_ty; + body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty = ref_coroutine_ty; // Add a deref to accesses of the coroutine state SelfArgVisitor::new(tcx, ProjectionElem::Deref).visit_body(body); } fn make_coroutine_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let ref_coroutine_ty = body.local_decls.raw[1].ty; + let ref_coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; let pin_did = tcx.require_lang_item(LangItem::Pin, body.span); let pin_adt_ref = tcx.adt_def(pin_did); @@ -505,7 +523,7 @@ fn make_coroutine_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body let pin_ref_coroutine_ty = Ty::new_adt(tcx, pin_adt_ref, args); // Replace the by ref coroutine argument - body.local_decls.raw[1].ty = pin_ref_coroutine_ty; + body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty = pin_ref_coroutine_ty; // Add the Pin field access to accesses of the coroutine state SelfArgVisitor::new(tcx, ProjectionElem::Field(FieldIdx::ZERO, ref_coroutine_ty)) @@ -643,15 +661,20 @@ fn transform_gen_context<'tcx>(body: &mut Body<'tcx>) { body.arg_count = 1; } +#[derive(Debug)] struct LivenessInfo { /// Which locals are live across any suspension point. saved_locals: CoroutineSavedLocals, + /// Always live locals + always_live_locals: DenseBitSet, + /// The set of saved locals live at each suspension point. - live_locals_at_suspension_points: Vec>, + live_locals_at_suspension_points: + IndexVec>, /// Parallel vec to the above with SourceInfo for each yield terminator. - source_info_at_suspension_points: Vec, + source_info_at_suspension_points: IndexVec, /// For every saved local, the set of other saved locals that are /// storage-live at the same time as this local. We cannot overlap locals in @@ -661,6 +684,14 @@ struct LivenessInfo { /// For every suspending block, the locals which are storage-live across /// that suspension point. storage_liveness: IndexVec>>, + + /// A rev-lookup of basic blocks to the suspension point index + suspension_point_at_block: UnordMap, +} + +rustc_index::newtype_index! { + #[debug_format = "suspend_{}"] + struct SuspensionPointIdx {} } /// Computes which locals have to be stored in the state-machine for the @@ -674,12 +705,12 @@ struct LivenessInfo { fn locals_live_across_suspend_points<'tcx>( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, - always_live_locals: &DenseBitSet, + always_live_locals: DenseBitSet, movable: bool, ) -> LivenessInfo { // Calculate when MIR locals have live storage. This gives us an upper bound of their // lifetimes. - let mut storage_live = MaybeStorageLive::new(std::borrow::Cow::Borrowed(always_live_locals)) + let mut storage_live = MaybeStorageLive::new(std::borrow::Cow::Borrowed(&always_live_locals)) .iterate_to_fixpoint(tcx, body, None) .into_results_cursor(body); @@ -712,11 +743,13 @@ fn locals_live_across_suspend_points<'tcx>( MaybeLiveLocals.iterate_to_fixpoint(tcx, body, Some("coroutine")).into_results_cursor(body); let mut storage_liveness_map = IndexVec::from_elem(None, &body.basic_blocks); - let mut live_locals_at_suspension_points = Vec::new(); - let mut source_info_at_suspension_points = Vec::new(); + let mut live_locals_at_suspension_points = IndexVec::::default(); + let mut source_info_at_suspension_points = IndexVec::default(); let mut live_locals_at_any_suspension_point = DenseBitSet::new_empty(body.local_decls.len()); + let mut suspension_point_at_block = UnordMap::default(); - for (block, data) in body.basic_blocks.iter_enumerated() { + for &block in body.basic_blocks.reverse_postorder() { + let data = &body.basic_blocks[block]; if let TerminatorKind::Yield { .. } = data.terminator().kind { let loc = Location { block, statement_index: data.statements.len() }; @@ -750,7 +783,7 @@ fn locals_live_across_suspend_points<'tcx>( live_locals.intersect(requires_storage_cursor.get()); // The coroutine argument is ignored. - live_locals.remove(SELF_ARG); + live_locals.remove(ty::CAPTURE_STRUCT_LOCAL); debug!("loc = {:?}, live_locals = {:?}", loc, live_locals); @@ -758,8 +791,9 @@ fn locals_live_across_suspend_points<'tcx>( // any suspension points live_locals_at_any_suspension_point.union(&live_locals); - live_locals_at_suspension_points.push(live_locals); + let idx = live_locals_at_suspension_points.push(live_locals); source_info_at_suspension_points.push(data.terminator().source_info); + suspension_point_at_block.insert(block, idx); } } @@ -783,10 +817,12 @@ fn locals_live_across_suspend_points<'tcx>( LivenessInfo { saved_locals, + always_live_locals, live_locals_at_suspension_points, source_info_at_suspension_points, storage_conflicts, storage_liveness: storage_liveness_map, + suspension_point_at_block, } } @@ -795,6 +831,7 @@ fn locals_live_across_suspend_points<'tcx>( /// `CoroutineSavedLocal` is indexed in terms of the elements in this set; /// i.e. `CoroutineSavedLocal::new(1)` corresponds to the second local /// included in this set. +#[derive(Debug)] struct CoroutineSavedLocals(DenseBitSet); impl CoroutineSavedLocals { @@ -827,7 +864,7 @@ impl CoroutineSavedLocals { } } -impl ops::Deref for CoroutineSavedLocals { +impl Deref for CoroutineSavedLocals { type Target = DenseBitSet; fn deref(&self) -> &Self::Target { @@ -946,25 +983,46 @@ impl StorageConflictVisitor<'_, '_> { } } +#[derive(Debug, Copy, Clone)] +struct UpvarInfo { + name: Symbol, + span: Span, +} + +#[instrument[level = "debug", skip(body), fields(body = ?body.source)]] fn compute_layout<'tcx>( liveness: LivenessInfo, body: &Body<'tcx>, + upvar_tys: &[Ty<'tcx>], + upvar_infos: &[UpvarInfo], + pack: PackCoroutineLayout, ) -> ( IndexVec, VariantIdx, FieldIdx)>>, CoroutineLayout<'tcx>, IndexVec>>, + UnordMap, ) { let LivenessInfo { saved_locals, + always_live_locals, live_locals_at_suspension_points, source_info_at_suspension_points, storage_conflicts, storage_liveness, + suspension_point_at_block, } = liveness; + // We need to later establish the map between upvars in UNRESUMED and locals in other states. + let local_upvar_map: UnordMap<_, _> = body + .local_upvar_map + .iter_enumerated() + .filter_map(|(field, local)| local.map(|local| (local, field))) + .collect(); + // Gather live local types and their indices. let mut locals = IndexVec::::new(); let mut tys = IndexVec::::new(); + let mut saved_local_upvar_map = UnordMap::default(); for (saved_local, local) in saved_locals.iter_enumerated() { debug!("coroutine saved local {:?} => {:?}", saved_local, local); @@ -992,7 +1050,52 @@ fn compute_layout<'tcx>( debug!(?decl); tys.push(decl); + + if let Some(&field) = local_upvar_map.get(&local) { + saved_local_upvar_map.insert(field, saved_local); + } } + // These are the "saved locals" sourced from the UNRESUMED state. + let upvar_saved_locals: IndexVec = upvar_tys + .iter() + .zip(upvar_infos) + .map(|(&ty, info)| { + tys.push(CoroutineSavedTy { + ty, + source_info: SourceInfo::outermost(info.span), + ignore_for_traits: false, + }) + }) + .collect(); + debug!(?upvar_saved_locals); + let storage_conflicts = if let Some(&first) = upvar_saved_locals.raw.first() + && let Some(&last) = upvar_saved_locals.raw.last() + { + let mut enlarged_storage_conflicts = BitMatrix::new(tys.len(), tys.len()); + let mut upvars = DenseBitSet::new_empty(tys.len()); + let mut ineligibles = upvars.clone(); + upvars.insert_range(first..=last); + for (saved_local, local) in saved_locals.iter_enumerated() { + if always_live_locals.contains(local) { + ineligibles.insert(saved_local); + } + } + upvars.union(&ineligibles); + for row in storage_conflicts.rows() { + for column in storage_conflicts.iter(row) { + enlarged_storage_conflicts.insert(row, column); + } + } + for &upvar in &upvar_saved_locals { + enlarged_storage_conflicts.union_row_with(&upvars, upvar); + } + for ineligible in ineligibles.iter() { + enlarged_storage_conflicts.union_row_with(&upvars, ineligible); + } + enlarged_storage_conflicts + } else { + storage_conflicts + }; // Leave empty variants for the UNRESUMED, RETURNED, and POISONED states. // In debuginfo, these will correspond to the beginning (UNRESUMED) or end @@ -1009,12 +1112,14 @@ fn compute_layout<'tcx>( // Build the coroutine variant field list. // Create a map from local indices to coroutine struct indices. + let variant_fields: [_; CoroutineArgs::RESERVED_VARIANTS] = + [upvar_saved_locals.clone(), IndexVec::new(), IndexVec::new()]; let mut variant_fields: IndexVec> = - iter::repeat(IndexVec::new()).take(CoroutineArgs::RESERVED_VARIANTS).collect(); + variant_fields.into_iter().collect(); let mut remap = IndexVec::from_elem_n(None, saved_locals.domain_size()); - for (suspension_point_idx, live_locals) in live_locals_at_suspension_points.iter().enumerate() { + for (suspension_point_idx, live_locals) in live_locals_at_suspension_points.iter_enumerated() { let variant_index = - VariantIdx::from(CoroutineArgs::RESERVED_VARIANTS + suspension_point_idx); + VariantIdx::from(CoroutineArgs::RESERVED_VARIANTS + suspension_point_idx.as_usize()); let mut fields = IndexVec::new(); for (idx, saved_local) in live_locals.iter().enumerate() { fields.push(saved_local); @@ -1030,29 +1135,43 @@ fn compute_layout<'tcx>( } debug!("coroutine variant_fields = {:?}", variant_fields); debug!("coroutine storage_conflicts = {:#?}", storage_conflicts); + debug!(remap = ?remap.debug_map_view_compact()); + debug!(locals = ?locals.debug_map_view()); let mut field_names = IndexVec::from_elem(None, &tys); for var in &body.var_debug_info { - let VarDebugInfoContents::Place(place) = &var.value else { continue }; - let Some(local) = place.as_local() else { continue }; - let Some(&Some((_, variant, field))) = remap.get(local) else { - continue; - }; - - let saved_local = variant_fields[variant][field]; - field_names.get_or_insert_with(saved_local, || var.name); + debug!(?var); + if let VarDebugInfoContents::Place(place) = &var.value + && let Some(local) = place.local_or_deref_local() + && let Some(&Some((_, variant, field))) = remap.get(local) + { + let saved_local = variant_fields[variant][field]; + field_names.get_or_insert_with(saved_local, || var.name); + } + } + for (capture, &saved_local) in upvar_infos.iter().zip(&upvar_saved_locals) { + field_names.get_or_insert_with(saved_local, || capture.name); } + debug!(field_names = ?field_names.debug_map_view()); + let relocated_upvars = upvar_saved_locals + .iter_enumerated() + .filter_map(|(field, &source)| { + saved_local_upvar_map.get(&field).map(|&dest| (source, dest)) + }) + .collect(); let layout = CoroutineLayout { field_tys: tys, field_names, variant_fields, variant_source_info, storage_conflicts, + relocated_upvars, + pack, }; debug!(?layout); - (remap, layout, storage_liveness) + (remap, layout, storage_liveness, suspension_point_at_block) } /// Replaces the entry point of `body` with a block that switches on the coroutine discriminant and @@ -1324,6 +1443,7 @@ fn create_cases<'tcx>( .suspension_points .iter() .filter_map(|point| { + let Some(point) = point else { bug!("all suspension points must be resolved now") }; // Find the target for this suspension point, if applicable operation.target_block(point).map(|target| { let mut statements = Vec::new(); @@ -1375,8 +1495,15 @@ pub(crate) fn mir_coroutine_witnesses<'tcx>( // The first argument is the coroutine type passed by value let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; - let movable = match *coroutine_ty.kind() { - ty::Coroutine(def_id, _) => tcx.coroutine_movability(def_id) == hir::Movability::Movable, + let (movable, upvar_tys, upvar_infos) = match *coroutine_ty.kind() { + ty::Coroutine(def_id, args) => ( + matches!(tcx.coroutine_movability(def_id), hir::Movability::Movable), + args.as_coroutine().upvar_tys(), + tcx.closure_captures(def_id.expect_local()) + .iter() + .map(|info| UpvarInfo { name: info.var_ident.name, span: info.var_ident.span }) + .collect::>(), + ), ty::Error(_) => return None, _ => span_bug!(body.span, "unexpected coroutine type {}", coroutine_ty), }; @@ -1384,12 +1511,19 @@ pub(crate) fn mir_coroutine_witnesses<'tcx>( // The witness simply contains all locals live across suspend points. let always_live_locals = always_storage_live_locals(body); - let liveness_info = locals_live_across_suspend_points(tcx, body, &always_live_locals, movable); + debug!(?always_live_locals); + let liveness_info = locals_live_across_suspend_points(tcx, body, always_live_locals, movable); // Extract locals which are live across suspension point into `layout` // `remap` gives a mapping from local indices onto coroutine struct indices // `storage_liveness` tells us which locals have live storage at suspension points - let (_, coroutine_layout, _) = compute_layout(liveness_info, body); + let (_, coroutine_layout, _, _) = compute_layout( + liveness_info, + body, + upvar_tys, + &upvar_infos, + tcx.sess.opts.unstable_opts.pack_coroutine_layout, + ); check_suspend_tys(tcx, &coroutine_layout, body); check_field_tys_sized(tcx, &coroutine_layout, def_id); @@ -1449,14 +1583,39 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { dump_mir(tcx, false, "coroutine_before", &0, body, |_, _| Ok(())); // The first argument is the coroutine type passed by value - let coroutine_ty = body.local_decls.raw[1].ty; + let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; let coroutine_kind = body.coroutine_kind().unwrap(); // Get the discriminant type and args which typeck computed - let ty::Coroutine(_, args) = coroutine_ty.kind() else { + let ty::Coroutine(def_id, args) = coroutine_ty.kind() else { tcx.dcx().span_bug(body.span, format!("unexpected coroutine type {coroutine_ty}")); }; - let discr_ty = args.as_coroutine().discr_ty(tcx); + let upvar_infos = match body.source.instance { + ty::InstanceKind::AsyncDropGlue(..) => { + smallvec![UpvarInfo { name: kw::SelfLower, span: DUMMY_SP }] + } + ty::InstanceKind::Item(_) => tcx + .closure_captures(def_id.expect_local()) + .iter() + .map(|info| UpvarInfo { name: info.var_ident.name, span: info.var_ident.span }) + .collect::>(), + ty::InstanceKind::Intrinsic(..) + | ty::InstanceKind::VTableShim(..) + | ty::InstanceKind::ReifyShim(..) + | ty::InstanceKind::FnPtrShim(..) + | ty::InstanceKind::Virtual(..) + | ty::InstanceKind::ClosureOnceShim { .. } + | ty::InstanceKind::ConstructCoroutineInClosureShim { .. } + | ty::InstanceKind::ThreadLocalShim(..) + | ty::InstanceKind::FutureDropPollShim(..) + | ty::InstanceKind::DropGlue(..) + | ty::InstanceKind::CloneShim(..) + | ty::InstanceKind::AsyncDropGlueCtorShim(..) + | ty::InstanceKind::FnPtrAddrShim(..) => unreachable!(), + }; + let coroutine_args = args.as_coroutine(); + let discr_ty = coroutine_args.discr_ty(tcx); + let upvar_tys = coroutine_args.upvar_tys(); let new_ret_ty = match coroutine_kind { CoroutineKind::Desugared(CoroutineDesugaring::Async, _) => { @@ -1536,9 +1695,9 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { let always_live_locals = always_storage_live_locals(body); - let movable = coroutine_kind.movability() == hir::Movability::Movable; + let movable = matches!(coroutine_kind.movability(), hir::Movability::Movable); let liveness_info = - locals_live_across_suspend_points(tcx, body, &always_live_locals, movable); + locals_live_across_suspend_points(tcx, body, always_live_locals.clone(), movable); if tcx.sess.opts.unstable_opts.validate_mir { let mut vis = EnsureCoroutineFieldAssignmentsNeverAlias { @@ -1553,7 +1712,13 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { // Extract locals which are live across suspension point into `layout` // `remap` gives a mapping from local indices onto coroutine struct indices // `storage_liveness` tells us which locals have live storage at suspension points - let (remap, layout, storage_liveness) = compute_layout(liveness_info, body); + let (remap, layout, storage_liveness, suspension_point_at_block) = compute_layout( + liveness_info, + body, + upvar_tys, + &upvar_infos, + tcx.sess.opts.unstable_opts.pack_coroutine_layout, + ); let can_return = can_return(tcx, body, body.typing_env(tcx)); @@ -1568,11 +1733,12 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { remap, storage_liveness, always_live_locals, - suspension_points: Vec::new(), + suspension_points: IndexVec::default(), old_ret_local, discr_ty, old_ret_ty, old_yield_ty, + suspension_point_at_block, }; transform.visit_body(body); diff --git a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs index 81d7b7ba02c2c..56c7e4524d894 100644 --- a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs +++ b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs @@ -69,7 +69,6 @@ use rustc_abi::{FieldIdx, VariantIdx}; use rustc_data_structures::steal::Steal; -use rustc_data_structures::unord::UnordMap; use rustc_hir as hir; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LocalDefId}; @@ -78,8 +77,18 @@ use rustc_middle::bug; use rustc_middle::hir::place::{Projection, ProjectionKind}; use rustc_middle::mir::visit::MutVisitor; use rustc_middle::mir::{self, dump_mir}; +use rustc_middle::ty::data_structures::IndexMap; use rustc_middle::ty::{self, InstanceKind, Ty, TyCtxt, TypeVisitableExt}; +struct CaptureInfo<'tcx> { + /// Field index of the capture in the parent coroutine structure + remapped_idx: FieldIdx, + /// Type of the capture in the parent coroutine structure + remapped_ty: Ty<'tcx>, + peel_deref: bool, + bridging_projections: Vec>, +} + pub(crate) fn coroutine_by_move_body_def_id<'tcx>( tcx: TyCtxt<'tcx>, coroutine_def_id: LocalDefId, @@ -126,23 +135,27 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( .tuple_fields() .len(); - let field_remapping: UnordMap<_, _> = ty::analyze_coroutine_closure_captures( + let field_remapping: IndexMap<_, _> = ty::analyze_coroutine_closure_captures( tcx.closure_captures(parent_def_id).iter().copied(), tcx.closure_captures(coroutine_def_id).iter().skip(num_args).copied(), |(parent_field_idx, parent_capture), (child_field_idx, child_capture)| { // Store this set of additional projections (fields and derefs). // We need to re-apply them later. - let mut child_precise_captures = - child_capture.place.projections[parent_capture.place.projections.len()..].to_vec(); + let child_precise_captures = child_capture.place.projections + [parent_capture.place.projections.len()..] + .iter() + .copied(); // If the parent capture is by-ref, then we need to apply an additional // deref before applying any further projections to this place. - if parent_capture.is_by_ref() { - child_precise_captures.insert( - 0, - Projection { ty: parent_capture.place.ty(), kind: ProjectionKind::Deref }, - ); - } + let bridging_projections = if parent_capture.is_by_ref() { + [Projection { ty: parent_capture.place.ty(), kind: ProjectionKind::Deref }] + .into_iter() + .chain(child_precise_captures) + .collect() + } else { + child_precise_captures.collect() + }; // If the child capture is by-ref, then we need to apply a "ref" // projection (i.e. `&`) at the end. But wait! We don't have that // as a projection kind. So instead, we can apply its dual and @@ -168,8 +181,8 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( // Finally, store the type of the parent's captured place. We need // this when building the field projection in the MIR body later on. - let mut parent_capture_ty = parent_capture.place.ty(); - parent_capture_ty = match parent_capture.info.capture_kind { + let parent_capture_ty = parent_capture.place.ty(); + let remapped_ty = match parent_capture.info.capture_kind { ty::UpvarCapture::ByValue | ty::UpvarCapture::ByUse => parent_capture_ty, ty::UpvarCapture::ByRef(kind) => Ty::new_ref( tcx, @@ -181,19 +194,19 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( Some(( FieldIdx::from_usize(child_field_idx + num_args), - ( - FieldIdx::from_usize(parent_field_idx + num_args), - parent_capture_ty, + CaptureInfo { + remapped_idx: FieldIdx::from_usize(parent_field_idx + num_args), + remapped_ty, peel_deref, - child_precise_captures, - ), + bridging_projections, + }, )) }, ) .flatten() .collect(); - if coroutine_kind == ty::ClosureKind::FnOnce { + if matches!(coroutine_kind, ty::ClosureKind::FnOnce) { assert_eq!(field_remapping.len(), tcx.closure_captures(parent_def_id).len()); // The by-move body is just the body :) return coroutine_def_id.to_def_id(); @@ -212,6 +225,7 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( ); let mut by_move_body = body.clone(); + dump_mir(tcx, false, "built", &"before", &by_move_body, |_, _| Ok(())); MakeByMoveBody { tcx, field_remapping, by_move_coroutine_ty }.visit_body(&mut by_move_body); // This path is unique since we're in a query so we'll only be called once with `parent_def_id` @@ -251,7 +265,7 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( struct MakeByMoveBody<'tcx> { tcx: TyCtxt<'tcx>, - field_remapping: UnordMap, bool, Vec>)>, + field_remapping: IndexMap>, by_move_coroutine_ty: Ty<'tcx>, } @@ -272,8 +286,12 @@ impl<'tcx> MutVisitor<'tcx> for MakeByMoveBody<'tcx> { if place.local == ty::CAPTURE_STRUCT_LOCAL && let Some((&mir::ProjectionElem::Field(idx, _), projection)) = place.projection.split_first() - && let Some(&(remapped_idx, remapped_ty, peel_deref, ref bridging_projections)) = - self.field_remapping.get(&idx) + && let Some(&CaptureInfo { + remapped_idx, + remapped_ty, + peel_deref, + ref bridging_projections, + }) = self.field_remapping.get(&idx) { // As noted before, if the parent closure captures a field by value, and // the child captures a field by ref, then for the by-move body we're @@ -350,7 +368,7 @@ impl<'tcx> MutVisitor<'tcx> for MakeByMoveBody<'tcx> { local: ty::CAPTURE_STRUCT_LOCAL, projection: [mir::ProjectionElem::Field(idx, _)], } = place.as_ref() - && let Some(&(_, _, true, _)) = self.field_remapping.get(&idx) + && let Some(CaptureInfo { peel_deref: true, .. }) = self.field_remapping.get(idx) { statement.kind = mir::StatementKind::Nop; } diff --git a/compiler/rustc_mir_transform/src/coroutine/drop.rs b/compiler/rustc_mir_transform/src/coroutine/drop.rs index 1a314e029f4a7..d68c377597443 100644 --- a/compiler/rustc_mir_transform/src/coroutine/drop.rs +++ b/compiler/rustc_mir_transform/src/coroutine/drop.rs @@ -231,8 +231,14 @@ pub(super) fn has_expandable_async_drops<'tcx>( if body[bb].is_cleanup { continue; } - let TerminatorKind::Drop { place, target: _, unwind: _, replace: _, drop: _, async_fut } = - body[bb].terminator().kind + let TerminatorKind::Drop { + place, + target: _, + unwind: _, + replace: _, + drop: _, + async_fut: Some(_), + } = body[bb].terminator().kind else { continue; }; @@ -240,12 +246,9 @@ pub(super) fn has_expandable_async_drops<'tcx>( if place_ty == coroutine_ty { continue; } - if async_fut.is_none() { - continue; - } return true; } - return false; + false } /// Expand Drop terminator for async drops into mainline poll-switch and dropline poll-switch @@ -508,7 +511,8 @@ pub(super) fn elaborate_coroutine_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body elaborate_drop( &mut elaborator, *source_info, - Place::from(SELF_ARG), + Place::from(SELF_ARG) + .project_deeper(&[ProjectionElem::Downcast(None, VARIANT_UNRESUMED)], tcx), (), *target, unwind, diff --git a/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs b/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs new file mode 100644 index 0000000000000..a9b0e8f382858 --- /dev/null +++ b/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs @@ -0,0 +1,451 @@ +//! MIR rewrite pass to promote upvars into native locals in the coroutine body +//! +//! # Summary +//! This pass performs the following transformations. +//! 1. It generates a fresh batch of locals for each captured upvars. +//! +//! For each upvar, whether used or not, a fresh local is created with the same type. +//! +//! 2. It replaces the places pointing into those upvars with places pointing into those locals instead +//! +//! Each place that starts with access into the coroutine structure `_1` is replaced with the fresh local as +//! the base. For instance, `(_1.4 as Some).0` is rewritten into `(_34 as Some).0` when `_34` is the fresh local +//! corresponding to the captured upvar stored in `_1.4`. +//! +//! 3. It assembles an prologue to replace the current entry block. +//! +//! This prologue block transfers every captured upvar into its corresponding fresh local, *via scratch locals*. +//! The upvars are first completely moved into the scratch locals in batch, and then moved into the destination +//! locals in batch. +//! The reason is that it is possible that coroutine layout may change and the source memory location of +//! an upvar may not necessarily be mapped exactly to the same place as in the `Unresumed` state. +//! While coroutine layout ensures that the same saved local has stable offsets throughout its lifetime, +//! technically the upvar in `Unresumed` state and their fresh locals are different saved locals. +//! This scratch locals re-estabilish safety so that the correct data permutation can take place. + +use std::borrow::Cow; + +use rustc_abi::FieldIdx; +use rustc_index::bit_set::DenseBitSet; +use rustc_index::{IndexSlice, IndexVec}; +use rustc_middle::bug; +use rustc_middle::mir::visit::MutVisitor; +use rustc_middle::mir::{ + self, BasicBlock, BasicBlockData, Body, Local, Place, ProjectionElem, START_BLOCK, SourceInfo, + Statement, StatementKind, Terminator, TerminatorKind, UnwindAction, +}; +use rustc_middle::ty::{self, TyCtxt}; +use rustc_mir_dataflow::Analysis; +use rustc_mir_dataflow::impls::{MaybeStorageLive, always_storage_live_locals}; +use rustc_span::{DUMMY_SP, Ident, Span, Symbol}; +use smallvec::{SmallVec, smallvec}; +use tracing::{debug, instrument}; + +use crate::pass_manager::MirPass; +use crate::patch::MirPatch; + +pub(crate) struct RelocateUpvars(bool); + +impl RelocateUpvars { + pub(crate) fn new(do_relocate: bool) -> Self { + Self(do_relocate) + } +} + +struct UpvarSubstitution<'tcx> { + /// Newly minted local into which the upvar is moved + local: Local, + /// The temporary local that the prologue will permute the upvars with + reloc: Local, + /// Place into the capture structure where this upvar is found + upvar_place: Place<'tcx>, + /// The span of the captured upvar from the parent body + span: Span, + /// Name of the upvar + name: Symbol, +} + +struct SubstituteUpvarVisitor<'tcx, 'a> { + tcx: TyCtxt<'tcx>, + mappings: &'a IndexSlice>, +} + +impl<'tcx, 'a> MutVisitor<'tcx> for SubstituteUpvarVisitor<'tcx, 'a> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_place( + &mut self, + place: &mut Place<'tcx>, + _context: mir::visit::PlaceContext, + location: mir::Location, + ) { + if let Place { local: ty::CAPTURE_STRUCT_LOCAL, projection } = place + && let [ProjectionElem::Field(field_idx, _ty), rest @ ..] = &***projection + { + let Some(&UpvarSubstitution { local, .. }) = self.mappings.get(*field_idx) else { + bug!( + "SubstituteUpvar: found {field_idx:?} @ {location:?} but there is no upvar for it" + ) + }; + let new_place = Place::from(local); + let new_place = new_place.project_deeper(rest, self.tcx); + *place = new_place; + } + } + + fn visit_terminator( + &mut self, + terminator: &mut mir::Terminator<'tcx>, + location: mir::Location, + ) { + if let TerminatorKind::Drop { place, .. } = &terminator.kind + && let Some(ty::CAPTURE_STRUCT_LOCAL) = place.as_local() + { + // This is a drop on the whole coroutine state, which we will processed later + return; + } + self.super_terminator(terminator, location) + } +} + +#[derive(Debug, Clone, Copy)] +struct RelocationInfo { + ident: Option, + immutable: bool, + by_ref: bool, +} + +impl<'tcx> MirPass<'tcx> for RelocateUpvars { + #[instrument(level = "debug", skip_all, fields(def_id = ?body.source))] + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + if !self.0 { + debug!("relocate upvar is set to no-op"); + return; + } + if body.yield_ty().is_none() { + // It fails the litmus test as a coroutine + debug!("not passing litmus test"); + return; + } + + // The first argument is the coroutine type passed by value + let coroutine_ty = if let Some(decl) = body.local_decls.get(ty::CAPTURE_STRUCT_LOCAL) { + decl.ty + } else { + debug!("not coroutine ty, skipping"); + return; + }; + + // We only care when there is at least one upvar + let (def_id, upvar_tys) = if let ty::Coroutine(def_id, args) = *coroutine_ty.kind() { + let args = args.as_coroutine(); + (def_id, args.upvar_tys()) + } else { + debug!("not coroutine ty again, skipping"); + return; + }; + if upvar_tys.is_empty() { + debug!("no upvar, skipping"); + return; + } + + let upvar_infos = match body.source.instance { + ty::InstanceKind::AsyncDropGlue(..) => { + smallvec![RelocationInfo { ident: None, immutable: true, by_ref: false }] + } + ty::InstanceKind::Item(_) => tcx + .closure_captures(def_id.expect_local()) + .iter() + .map(|info| RelocationInfo { + ident: Some(Ident::new(info.to_symbol(), info.var_ident.span)), + immutable: matches!(info.mutability, ty::Mutability::Not), + by_ref: matches!(info.info.capture_kind, ty::UpvarCapture::ByRef(..)), + }) + .collect::>(), + ty::InstanceKind::Intrinsic(..) + | ty::InstanceKind::VTableShim(..) + | ty::InstanceKind::ReifyShim(..) + | ty::InstanceKind::FnPtrShim(..) + | ty::InstanceKind::Virtual(..) + | ty::InstanceKind::ClosureOnceShim { .. } + | ty::InstanceKind::ConstructCoroutineInClosureShim { .. } + | ty::InstanceKind::ThreadLocalShim(..) + | ty::InstanceKind::FutureDropPollShim(..) + | ty::InstanceKind::DropGlue(..) + | ty::InstanceKind::CloneShim(..) + | ty::InstanceKind::AsyncDropGlueCtorShim(..) + | ty::InstanceKind::FnPtrAddrShim(..) => unreachable!(), + }; + + let mut substitution_mapping = IndexVec::new(); + let mut patch = MirPatch::new(body); + for (field_idx, (upvar_ty, &captured)) in upvar_tys.iter().zip(&upvar_infos).enumerate() { + let span = captured.ident.map_or(DUMMY_SP, |ident| ident.span); + let name = if let Some(ident) = captured.ident { + ident.name + } else { + Symbol::intern(&format!("_{}", field_idx)) + }; + + let immutable = + if captured.immutable { mir::Mutability::Not } else { mir::Mutability::Mut }; + let local = patch.new_local_with_info( + upvar_ty, + span, + mir::LocalInfo::User(mir::BindingForm::Var(mir::VarBindingForm { + binding_mode: rustc_ast::BindingMode( + if captured.by_ref { + rustc_ast::ByRef::Yes(immutable) + } else { + rustc_ast::ByRef::No + }, + immutable, + ), + opt_ty_info: None, + opt_match_place: None, + pat_span: span, + })), + captured.immutable, + ); + let reloc = patch.new_local_with_info(upvar_ty, span, mir::LocalInfo::Boring, true); + + let field_idx = FieldIdx::from_usize(field_idx); + let upvar_place = Place::from(ty::CAPTURE_STRUCT_LOCAL) + .project_deeper(&[ProjectionElem::Field(field_idx, upvar_ty)], tcx); + + substitution_mapping.push(UpvarSubstitution { local, reloc, upvar_place, span, name }); + } + patch.apply(body); + body.local_upvar_map = substitution_mapping.iter().map(|sub| Some(sub.local)).collect(); + SubstituteUpvarVisitor { tcx, mappings: &substitution_mapping }.visit_body(body); + + rewrite_drop_coroutine_struct(body, &substitution_mapping); + insert_substitution_prologue(body, &substitution_mapping); + patch_missing_storage_deads(tcx, body, &substitution_mapping); + hydrate_var_debug_info(body, &substitution_mapping); + } + + fn is_required(&self) -> bool { + true + } +} + +fn rewrite_one_drop_coroutine_struct<'tcx>( + patch: &mut MirPatch<'tcx>, + body: &Body<'tcx>, + block: BasicBlock, + substitution_mapping: &IndexSlice>, +) { + let data = &body.basic_blocks[block]; + let source_info = data.terminator().source_info; + let TerminatorKind::Drop { + place: _, + mut target, + mut unwind, + replace, + drop: dropline, + async_fut: None, + } = data.terminator().kind + else { + unreachable!("unexpected terminator {:?}", data.terminator().kind) + }; + let mut cleanup = match unwind { + UnwindAction::Cleanup(tgt) => tgt, + UnwindAction::Continue => patch.resume_block(), + UnwindAction::Unreachable => patch.unreachable_cleanup_block(), + UnwindAction::Terminate(reason) => patch.terminate_block(reason), + }; + for &UpvarSubstitution { local, .. } in substitution_mapping { + let place = local.into(); + let mut unwind_one = patch.new_block(BasicBlockData::new_stmts( + vec![Statement::new(source_info, StatementKind::StorageDead(local))], + Some(Terminator { + source_info, + kind: TerminatorKind::Goto { + target: if data.is_cleanup { target } else { cleanup }, + }, + }), + true, + )); + unwind_one = patch.new_block(BasicBlockData::new_stmts( + vec![], + Some(Terminator { + source_info, + kind: TerminatorKind::Drop { + place, + target: unwind_one, + unwind: UnwindAction::Terminate(mir::UnwindTerminateReason::InCleanup), + replace, + drop: None, + async_fut: None, + }, + }), + true, + )); + if data.is_cleanup { + unwind = UnwindAction::Cleanup(unwind_one); + cleanup = unwind_one; + target = unwind_one; + } else { + let mut drop_one = patch.new_block(BasicBlockData::new_stmts( + vec![Statement::new(source_info, StatementKind::StorageDead(local))], + Some(Terminator { source_info, kind: TerminatorKind::Goto { target } }), + false, + )); + drop_one = patch.new_block(BasicBlockData::new_stmts( + vec![], + Some(Terminator { + source_info, + kind: TerminatorKind::Drop { + place, + target: drop_one, + unwind, + replace, + drop: dropline, + async_fut: None, + }, + }), + false, + )); + target = drop_one; + unwind = UnwindAction::Cleanup(unwind_one); + cleanup = unwind_one; + } + } + patch.patch_terminator(block, TerminatorKind::Goto { target }); +} + +fn rewrite_drop_coroutine_struct<'tcx>( + body: &mut Body<'tcx>, + substitution_mapping: &IndexSlice>, +) { + let mut blocks = DenseBitSet::new_empty(body.basic_blocks.len()); + for (block, block_data) in body.basic_blocks.iter_enumerated() { + let Terminator { source_info: _, kind: TerminatorKind::Drop { place, .. } } = + block_data.terminator() + else { + continue; + }; + let Some(local) = place.as_local() else { continue }; + if local == ty::CAPTURE_STRUCT_LOCAL { + blocks.insert(block); + } + } + let mut patch = MirPatch::new(body); + for block in blocks.iter() { + rewrite_one_drop_coroutine_struct(&mut patch, body, block, substitution_mapping); + } + patch.apply(body); +} + +fn insert_substitution_prologue<'tcx>( + body: &mut Body<'tcx>, + substitution_mapping: &IndexSlice>, +) { + let mut patch = MirPatch::new(body); + let mut stmts = Vec::with_capacity(2 * substitution_mapping.len()); + for &UpvarSubstitution { local, reloc, upvar_place, span, name: _ } in substitution_mapping { + // For each upvar-local _$i + let source_info = SourceInfo::outermost(span); + // StorageLive(_$i) + stmts.push(Statement::new(source_info, StatementKind::StorageLive(local))); + // Use a fresh local _$i' here, so as to avoid potential field permutation + // StorageLive(_$i') + stmts.push(Statement::new(source_info, StatementKind::StorageLive(reloc))); + // _$i' = move $ + stmts.push(Statement::new( + source_info, + StatementKind::Assign(Box::new(( + reloc.into(), + mir::Rvalue::Use(mir::Operand::Move(upvar_place)), + ))), + )); + } + for &UpvarSubstitution { local, reloc, upvar_place: _, span, name: _ } in substitution_mapping { + let source_info = SourceInfo::outermost(span); + // _$i = move $i' + stmts.push(Statement::new( + source_info, + StatementKind::Assign(Box::new(( + local.into(), + mir::Rvalue::Use(mir::Operand::Move(reloc.into())), + ))), + )); + stmts.push(Statement::new(source_info, StatementKind::StorageDead(reloc))); + } + let source_info = SourceInfo::outermost(body.span); + let prologue = patch.new_block(BasicBlockData::new_stmts( + stmts, + Some(Terminator { source_info, kind: TerminatorKind::Goto { target: START_BLOCK } }), + false, + )); + patch.apply(body); + + // Manually patch so that prologue is the new entry-point + let preds = body.basic_blocks.predecessors()[START_BLOCK].clone(); + let basic_blocks = body.basic_blocks.as_mut(); + for pred in preds { + basic_blocks[pred].terminator_mut().successors_mut(|target| { + if *target == START_BLOCK { + *target = prologue; + } + }); + } + basic_blocks.swap(START_BLOCK, prologue); +} + +/// Occasionally there are upvar locals left without `StorageDead` because +/// they do not have destructors. +/// We need to mark them daed for correctness, as previously the entire +/// capture structure was marked dead and now we need to mark them one at +/// a time. +fn patch_missing_storage_deads<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mut Body<'tcx>, + substitution_mapping: &IndexSlice>, +) { + let always_live_locals = &always_storage_live_locals(body); + + let mut maybe_storage_live = MaybeStorageLive::new(Cow::Borrowed(always_live_locals)) + .iterate_to_fixpoint(tcx, body, None) + .into_results_cursor(body); + + let mut patch = MirPatch::new(body); + let mut upvar_locals = DenseBitSet::new_empty(body.local_decls.len()); + for subst in substitution_mapping { + upvar_locals.insert(subst.local); + } + for (block, data) in body.basic_blocks.iter_enumerated() { + if !data.is_cleanup && matches!(data.terminator().kind, TerminatorKind::Return) { + let nr_stmts = data.statements.len(); + maybe_storage_live + .seek_after_primary_effect(mir::Location { block, statement_index: nr_stmts }); + let mut missing_locals = maybe_storage_live.get().clone(); + missing_locals.intersect(&upvar_locals); + for (count, local) in missing_locals.iter().enumerate() { + patch.add_statement( + mir::Location { block, statement_index: nr_stmts + count }, + mir::StatementKind::StorageDead(local), + ); + } + } + } + patch.apply(body); +} + +fn hydrate_var_debug_info<'tcx>( + body: &mut Body<'tcx>, + substitution_mapping: &IndexSlice>, +) { + for subst in substitution_mapping { + body.var_debug_info.push(mir::VarDebugInfo { + name: subst.name, + source_info: SourceInfo::outermost(subst.span), + composite: None, + value: mir::VarDebugInfoContents::Place(Place::from(subst.local)), + argument_index: None, + }); + } +} diff --git a/compiler/rustc_mir_transform/src/deref_separator.rs b/compiler/rustc_mir_transform/src/deref_separator.rs index bc914ea656415..00d81cc3b3ccd 100644 --- a/compiler/rustc_mir_transform/src/deref_separator.rs +++ b/compiler/rustc_mir_transform/src/deref_separator.rs @@ -40,6 +40,7 @@ impl<'a, 'tcx> MutVisitor<'tcx> for DerefChecker<'a, 'tcx> { ty, self.local_decls[p_ref.local].source_info.span, LocalInfo::DerefTemp, + false, ); // We are adding current p_ref's projections to our diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 08f25276cecc1..df184c652305c 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -125,7 +125,7 @@ declare_passes! { pub mod cleanup_post_borrowck : CleanupPostBorrowck; mod copy_prop : CopyProp; - mod coroutine : StateTransform; + mod coroutine : RelocateUpvars, StateTransform; mod coverage : InstrumentCoverage; mod ctfe_limit : CtfeLimit; mod dataflow_const_prop : DataflowConstProp; @@ -442,7 +442,15 @@ fn mir_promoted( pm::run_passes( tcx, &mut body, - &[&promote_pass, &simplify::SimplifyCfg::PromoteConsts, &coverage::InstrumentCoverage], + &[ + &coroutine::RelocateUpvars::new(!matches!( + tcx.sess.opts.unstable_opts.pack_coroutine_layout, + rustc_session::config::PackCoroutineLayout::No + )), + &promote_pass, + &simplify::SimplifyCfg::PromoteConsts, + &coverage::InstrumentCoverage, + ], Some(MirPhase::Analysis(AnalysisPhase::Initial)), pm::Optimizations::Allowed, ); diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs index 5b3ddcc777be5..7cd4874db44a6 100644 --- a/compiler/rustc_mir_transform/src/pass_manager.rs +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -305,11 +305,20 @@ fn run_passes_inner<'tcx>( if dump_enabled { dump_mir_for_pass(tcx, body, name, true); } + let (dialect, phase_num) = body.phase.index(); if validate { - validate_body(tcx, body, format!("after pass {name}")); + validate_body( + tcx, + body, + format!("after pass {name} {dialect}-{phase_num}-{:03}", body.pass_count), + ); } if lint { - lint_body(tcx, body, format!("after pass {name}")); + lint_body( + tcx, + body, + format!("after pass {name} {dialect}-{phase_num}-{:03}", body.pass_count), + ); } body.pass_count += 1; diff --git a/compiler/rustc_mir_transform/src/patch.rs b/compiler/rustc_mir_transform/src/patch.rs index c781d1a5324b7..888fda94419d9 100644 --- a/compiler/rustc_mir_transform/src/patch.rs +++ b/compiler/rustc_mir_transform/src/patch.rs @@ -167,10 +167,14 @@ impl<'tcx> MirPatch<'tcx> { ty: Ty<'tcx>, span: Span, local_info: LocalInfo<'tcx>, + immutable: bool, ) -> Local { let index = self.next_local; self.next_local += 1; let mut new_decl = LocalDecl::new(ty, span); + if immutable { + new_decl = new_decl.immutable(); + } **new_decl.local_info.as_mut().unwrap_crate_local() = local_info; self.new_locals.push(new_decl); Local::new(index) diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index c6760b3583f20..da6bb52906191 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -48,7 +48,7 @@ impl<'tcx> MutVisitor<'tcx> for FixProxyFutureDropVisitor<'tcx> { _context: PlaceContext, _location: Location, ) { - if place.local == Local::from_u32(1) { + if place.local == ty::CAPTURE_STRUCT_LOCAL { if place.projection.len() == 1 { assert!(matches!( place.projection.first(), @@ -65,9 +65,8 @@ impl<'tcx> MutVisitor<'tcx> for FixProxyFutureDropVisitor<'tcx> { } } +#[instrument(level = "debug", skip(tcx))] fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body<'tcx> { - debug!("make_shim({:?})", instance); - let mut result = match instance { ty::InstanceKind::Item(..) => bug!("item {:?} passed to make_shim", instance), ty::InstanceKind::VTableShim(def_id) => { @@ -194,6 +193,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body< tcx, &mut body, &[ + &crate::coroutine::RelocateUpvars::new(true), // We must always relocate to ensure valid MIR &mentioned_items::MentionedItems, &abort_unwinding_calls::AbortUnwindingCalls, &add_call_guards::CriticalCallEdges, diff --git a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs index 18d09473c191e..76a0f1c2d4fcf 100644 --- a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs +++ b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs @@ -11,6 +11,7 @@ use rustc_middle::ty::{self, EarlyBinder, Ty, TyCtxt, TypeVisitableExt}; use super::*; use crate::patch::MirPatch; +#[instrument(level = "debug", skip(tcx))] pub(super) fn build_async_destructor_ctor_shim<'tcx>( tcx: TyCtxt<'tcx>, def_id: DefId, @@ -39,12 +40,12 @@ pub(super) fn build_async_destructor_ctor_shim<'tcx>( } // build_drop_shim analog for async drop glue (for generated coroutine poll function) +#[instrument(level = "debug", skip(tcx))] pub(super) fn build_async_drop_shim<'tcx>( tcx: TyCtxt<'tcx>, def_id: DefId, ty: Ty<'tcx>, ) -> Body<'tcx> { - debug!("build_async_drop_shim(def_id={:?}, ty={:?})", def_id, ty); let ty::Coroutine(_, parent_args) = ty.kind() else { bug!(); }; @@ -193,6 +194,7 @@ pub(super) fn build_future_drop_poll_shim<'tcx>( // `async_drop_in_place::{closure}.poll()` is converted into `T.future_drop_poll()`. // Every coroutine has its `poll` (calculate yourself a little further) // and its `future_drop_poll` (drop yourself a little further). +#[instrument(level = "debug", skip_all, fields(span))] fn build_adrop_for_coroutine_shim<'tcx>( tcx: TyCtxt<'tcx>, proxy_ty: Ty<'tcx>, @@ -200,33 +202,41 @@ fn build_adrop_for_coroutine_shim<'tcx>( span: Span, instance: ty::InstanceKind<'tcx>, ) -> Body<'tcx> { - let ty::Coroutine(coroutine_def_id, impl_args) = impl_ty.kind() else { + debug!("proxy_ty={proxy_ty:#?}"); + debug!("impl_ty={impl_ty:#?}"); + debug!("instance={instance:#?}"); + let &ty::Coroutine(coroutine_def_id, impl_args) = impl_ty.kind() else { bug!("build_adrop_for_coroutine_shim not for coroutine impl type: ({:?})", instance); }; let proxy_ref = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, proxy_ty); - // taking _1.0 (impl from Pin) - let pin_proxy_layout_local = Local::new(1); - let source_info = SourceInfo::outermost(span); - // converting `(_1: Pin<&mut CorLayout>, _2: &mut Context<'_>) -> Poll<()>` - // into `(_1: Pin<&mut ProxyLayout>, _2: &mut Context<'_>) -> Poll<()>` - // let mut _x: &mut CorLayout = &*_1.0.0; - // Replace old _1.0 accesses into _x accesses; - let body = tcx.optimized_mir(*coroutine_def_id).future_drop_poll().unwrap(); + let body = tcx.optimized_mir(coroutine_def_id).future_drop_poll().unwrap(); let mut body: Body<'tcx> = EarlyBinder::bind(body.clone()).instantiate(tcx, impl_args); + dump_mir(tcx, true, "build_adrop_for_coroutine_shim_before", &0, &body, |_, _| Ok(())); + debug!(?body.source.instance, "before"); body.source.instance = instance; body.phase = MirPhase::Runtime(RuntimePhase::Initial); body.var_debug_info.clear(); + // Here we convert `(_1: Pin<&mut InnerCoroutine>, _2: &mut Context<'_>) -> Poll<()>` + // into `(_1: Pin<&mut ProxyCoroutine>, _2: &mut Context<'_>) -> Poll<()>`. + // **The Schematic** + // We make a new local `_x`: + // let mut _x: &mut InnerCoroutine = &* ((*_1.0) as variant#0).0; + // Then replace projections on `_1.0` into projections on `_x`. + let pin_proxy_layout_local = Local::new(1); + let source_info = SourceInfo::outermost(span); let pin_adt_ref = tcx.adt_def(tcx.require_lang_item(LangItem::Pin, span)); let args = tcx.mk_args(&[proxy_ref.into()]); let pin_proxy_ref = Ty::new_adt(tcx, pin_adt_ref, args); + // This is the type of the vanilla `InnerCoroutine` let cor_ref = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, impl_ty); let proxy_ref_local = body.local_decls.push(LocalDecl::new(proxy_ref, span)); let cor_ref_local = body.local_decls.push(LocalDecl::new(cor_ref, span)); + // Batch replacement of `_1.0` with `_x` FixProxyFutureDropVisitor { tcx, replace_to: cor_ref_local }.visit_body(&mut body); - // Now changing first arg from Pin<&mut ImplCoroutine> to Pin<&mut ProxyCoroutine> + // Now changing first arg from Pin<&mut InnerCoroutine> to Pin<&mut ProxyCoroutine> body.local_decls[pin_proxy_layout_local] = LocalDecl::new(pin_proxy_ref, span); { @@ -250,11 +260,15 @@ fn build_adrop_for_coroutine_shim<'tcx>( if ty != proxy_ty { let ty_ptr = Ty::new_mut_ptr(tcx, ty); let impl_ptr_place = Place::from(cor_ptr_local).project_deeper( - &[PlaceElem::Deref, PlaceElem::Field(FieldIdx::ZERO, ty_ptr)], + &[ + PlaceElem::Deref, + PlaceElem::Downcast(None, VariantIdx::ZERO), + PlaceElem::Field(FieldIdx::ZERO, ty_ptr), + ], tcx, ); cor_ptr_local = body.local_decls.push(LocalDecl::new(ty_ptr, span)); - // _cor_ptr = _proxy.0.0 (... .0) + // _cor_ptr = ((*_proxy.0) as variant#0).0 (... .0) body.basic_blocks_mut()[START_BLOCK].statements.insert( idx, Statement::new( @@ -283,6 +297,7 @@ fn build_adrop_for_coroutine_shim<'tcx>( ), ); } + dump_mir(tcx, true, "build_adrop_for_coroutine_shim", &0, &body, |_, _| Ok(())); body } @@ -342,10 +357,16 @@ fn build_adrop_for_adrop_shim<'tcx>( proxy_ty.find_async_drop_impl_coroutine(tcx, |ty| { if ty != proxy_ty { let ty_ptr = Ty::new_mut_ptr(tcx, ty); - let impl_ptr_place = Place::from(cor_ptr_local) - .project_deeper(&[PlaceElem::Deref, PlaceElem::Field(FieldIdx::ZERO, ty_ptr)], tcx); + let impl_ptr_place = Place::from(cor_ptr_local).project_deeper( + &[ + PlaceElem::Deref, + PlaceElem::Downcast(None, VariantIdx::ZERO), + PlaceElem::Field(FieldIdx::ZERO, ty_ptr), + ], + tcx, + ); cor_ptr_local = locals.push(LocalDecl::new(ty_ptr, span)); - // _cor_ptr = _proxy.0.0 (... .0) + // _cor_ptr = ((*_proxy.0) as variant#0).0 (... .0) statements.push(Statement::new( source_info, StatementKind::Assign(Box::new(( @@ -421,5 +442,6 @@ fn build_adrop_for_adrop_shim<'tcx>( let source = MirSource::from_instance(instance); let mut body = new_body(source, blocks, locals, sig.inputs().len(), span); body.phase = MirPhase::Runtime(RuntimePhase::Initial); - return body; + dump_mir(tcx, true, "build_adrop_for_adrop_shim", &0, &body, |_, _| Ok(())); + body } diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index 468ef742dfb73..97375ae754e8f 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -431,9 +431,16 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyLocals { // Only bother running the `LocalUpdater` if we actually found locals to remove. if map.iter().any(Option::is_none) { // Update references to all vars and tmps now - let mut updater = LocalUpdater { map, tcx }; + let mut updater = LocalUpdater { map: &map, tcx }; updater.visit_body_preserves_cfg(body); + // Update mapping for local to upvar + for local in &mut body.local_upvar_map { + if let Some(idx) = local { + *local = *map.get(*idx).unwrap_or(&None); + } + } + body.local_decls.shrink_to_fit(); } } @@ -615,12 +622,12 @@ fn remove_unused_definitions_helper(used_locals: &mut UsedLocals, body: &mut Bod } } -struct LocalUpdater<'tcx> { - map: IndexVec>, +struct LocalUpdater<'tcx, 'a> { + map: &'a IndexSlice>, tcx: TyCtxt<'tcx>, } -impl<'tcx> MutVisitor<'tcx> for LocalUpdater<'tcx> { +impl<'tcx> MutVisitor<'tcx> for LocalUpdater<'tcx, '_> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index 99e4782e4700c..a326670bede65 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -14,7 +14,7 @@ use rustc_middle::mir::*; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::{ - self, CoroutineArgsExt, InstanceKind, ScalarInt, Ty, TyCtxt, TypeVisitableExt, Upcast, Variance, + self, InstanceKind, ScalarInt, Ty, TyCtxt, TypeVisitableExt, Upcast, Variance, }; use rustc_middle::{bug, span_bug}; use rustc_trait_selection::traits::ObligationCtxt; @@ -797,14 +797,11 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { }; ty::EarlyBinder::bind(f_ty.ty).instantiate(self.tcx, args) - } else { - let Some(&f_ty) = args.as_coroutine().prefix_tys().get(f.index()) - else { - fail_out_of_bounds(self, location); - return; - }; - + } else if let Some(&f_ty) = args.as_coroutine().upvar_tys().get(f.index()) { f_ty + } else { + fail_out_of_bounds(self, location); + return; }; check_equal(self, location, f_ty); diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 9793d8091e2ab..27c081477b2cf 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -3215,9 +3215,9 @@ pub(crate) mod dep_tracking { CrateType, DebugInfo, DebugInfoCompression, ErrorOutputType, FmtDebug, FunctionReturn, InliningThreshold, InstrumentCoverage, InstrumentXRay, LinkerPluginLto, LocationDetail, LtoCli, MirStripDebugInfo, NextSolverConfig, Offload, OomStrategy, OptLevel, OutFileName, - OutputType, OutputTypes, PatchableFunctionEntry, Polonius, RemapPathScopeComponents, - ResolveDocLinks, SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath, - SymbolManglingVersion, WasiExecModel, + OutputType, OutputTypes, PackCoroutineLayout, PatchableFunctionEntry, Polonius, + RemapPathScopeComponents, ResolveDocLinks, SourceFileHashAlgorithm, SplitDwarfKind, + SwitchWithOptPath, SymbolManglingVersion, WasiExecModel, }; use crate::lint; use crate::utils::NativeLib; @@ -3320,6 +3320,7 @@ pub(crate) mod dep_tracking { Polonius, InliningThreshold, FunctionReturn, + PackCoroutineLayout, Align, ); @@ -3574,6 +3575,30 @@ pub enum FunctionReturn { ThunkExtern, } +/// Layout optimisation for Coroutines +#[derive( + Clone, + Copy, + PartialEq, + Eq, + Hash, + HashStable_Generic, + Debug, + Default, + Decodable, + Encodable +)] +pub enum PackCoroutineLayout { + /// Keep coroutine captured variables throughout all states + #[default] + No, + + /// Allow coroutine captured variables that are used only once + /// before the first suspension to be freed up for storage + /// in all other suspension states + CapturesOnly, +} + /// Whether extra span comments are included when dumping MIR, via the `-Z mir-include-spans` flag. /// By default, only enabled in the NLL MIR dumps, and disabled in all other passes. #[derive(Clone, Copy, Default, PartialEq, Debug)] diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 6d5be2d92cd2a..0e5e8ad74643f 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -738,6 +738,7 @@ mod desc { pub(crate) const parse_panic_strategy: &str = "either `unwind` or `abort`"; pub(crate) const parse_on_broken_pipe: &str = "either `kill`, `error`, or `inherit`"; pub(crate) const parse_patchable_function_entry: &str = "either two comma separated integers (total_nops,prefix_nops), with prefix_nops <= total_nops, or one integer (total_nops)"; + pub(crate) const parse_pack_coroutine_layout: &str = "either `no` or `captures-only`"; pub(crate) const parse_opt_panic_strategy: &str = parse_panic_strategy; pub(crate) const parse_oom_strategy: &str = "either `panic` or `abort`"; pub(crate) const parse_relro_level: &str = "one of: `full`, `partial`, or `off`"; @@ -1880,6 +1881,18 @@ pub mod parse { true } + pub(crate) fn parse_pack_coroutine_layout( + slot: &mut PackCoroutineLayout, + v: Option<&str>, + ) -> bool { + *slot = match v { + Some("no") => PackCoroutineLayout::No, + Some("captures-only") => PackCoroutineLayout::CapturesOnly, + _ => return false, + }; + true + } + pub(crate) fn parse_inlining_threshold(slot: &mut InliningThreshold, v: Option<&str>) -> bool { match v { Some("always" | "yes") => { @@ -2433,6 +2446,8 @@ options! { "panic strategy for out-of-memory handling"), osx_rpath_install_name: bool = (false, parse_bool, [TRACKED], "pass `-install_name @rpath/...` to the macOS linker (default: no)"), + pack_coroutine_layout: PackCoroutineLayout = (PackCoroutineLayout::default(), parse_pack_coroutine_layout, [TRACKED], + "set strategy to pack coroutine state layout (default: no)"), packed_bundled_libs: bool = (false, parse_bool, [TRACKED], "change rlib format to store native libraries as archives"), panic_abort_tests: bool = (false, parse_bool, [TRACKED], diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs index e31ff8b872981..e9821bfe147d0 100644 --- a/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs +++ b/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs @@ -6,7 +6,7 @@ use std::iter; use std::path::PathBuf; use itertools::{EitherOrBoth, Itertools}; -use rustc_abi::ExternAbi; +use rustc_abi::{ExternAbi, FIRST_VARIANT}; use rustc_data_structures::fx::FxHashSet; use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_errors::codes::*; @@ -2465,18 +2465,28 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { && let Some(coroutine_info) = self.tcx.mir_coroutine_witnesses(coroutine_did) { debug!(?coroutine_info); - 'find_source: for (variant, source_info) in - coroutine_info.variant_fields.iter().zip(&coroutine_info.variant_source_info) + // Variants are scanned "in reverse" because suspension points + // tend to contain more diagnostic information than the unresumed state. + 'find_source: for ((variant_idx, variant), source_info) in coroutine_info + .variant_fields + .iter_enumerated() + .zip(&coroutine_info.variant_source_info) + .rev() { debug!(?variant); for &local in variant { let decl = &coroutine_info.field_tys[local]; debug!(?decl); if ty_matches(ty::Binder::dummy(decl.ty)) && !decl.ignore_for_traits { - interior_or_upvar_span = Some(CoroutineInteriorOrUpvar::Interior( - decl.source_info.span, - Some((source_info.span, from_awaited_ty)), - )); + let span = decl.source_info.span; + if variant_idx == FIRST_VARIANT { + interior_or_upvar_span = Some(CoroutineInteriorOrUpvar::Upvar(span)); + } else { + interior_or_upvar_span = Some(CoroutineInteriorOrUpvar::Interior( + span, + Some((source_info.span, from_awaited_ty)), + )); + } break 'find_source; } } diff --git a/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs b/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs index de40453289979..52d9460a3f48c 100644 --- a/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs +++ b/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs @@ -1,6 +1,8 @@ +use rustc_abi::VariantIdx; use rustc_data_structures::fx::FxHashSet; use rustc_infer::traits::query::type_op::DropckOutlives; use rustc_middle::traits::query::{DropckConstraint, DropckOutlivesResult}; +use rustc_middle::ty::data_structures::IndexSet; use rustc_middle::ty::{self, EarlyBinder, ParamEnvAnd, Ty, TyCtxt}; use rustc_span::Span; use tracing::{debug, instrument}; @@ -352,7 +354,14 @@ pub fn dtorck_constraint_for_ty_inner<'tcx>( // in the interior, we'll already detect the need for a drop by checking the interior. let typing_env = tcx.erase_regions(typing_env); let needs_drop = tcx.mir_coroutine_witnesses(def_id).is_some_and(|witness| { - witness.field_tys.iter().any(|field| field.ty.needs_drop(tcx, typing_env)) + let upvar_locals: IndexSet<_> = + witness.variant_fields[VariantIdx::ZERO].iter().copied().collect(); + // As a reminder, upvars also appear in the UNRESUMED state-variant + witness + .field_tys + .iter_enumerated() + .filter_map(|(corsl, ty)| (!upvar_locals.contains(&corsl)).then_some(ty)) + .any(|field| field.ty.needs_drop(tcx, typing_env)) }); if needs_drop { // Pushing types directly to `constraints.outlives` is equivalent diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index 79f7e228e2adc..03d6ab0502065 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -9,6 +9,7 @@ use rustc_abi::{ use rustc_hashes::Hash64; use rustc_index::IndexVec; use rustc_middle::bug; +use rustc_middle::mir::CoroutineSavedLocal; use rustc_middle::query::Providers; use rustc_middle::ty::layout::{ FloatExt, HasTyCtxt, IntegerExt, LayoutCx, LayoutError, LayoutOf, TyAndLayout, @@ -17,6 +18,7 @@ use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::{ self, AdtDef, CoroutineArgsExt, EarlyBinder, PseudoCanonicalInput, Ty, TyCtxt, TypeVisitableExt, }; +use rustc_session::config::PackCoroutineLayout; use rustc_session::{DataTypeKind, FieldInfo, FieldKind, SizeKind, VariantInfo}; use rustc_span::{Symbol, sym}; use tracing::{debug, instrument}; @@ -172,6 +174,7 @@ fn extract_const_value<'tcx>( } } +#[instrument(level = "debug", skip(cx), ret)] fn layout_of_uncached<'tcx>( cx: &LayoutCx<'tcx>, ty: Ty<'tcx>, @@ -486,7 +489,7 @@ fn layout_of_uncached<'tcx>( let info = tcx.coroutine_layout(def_id, args)?; - let local_layouts = info + let local_layouts: IndexVec<_, _> = info .field_tys .iter() .map(|local| { @@ -494,22 +497,31 @@ fn layout_of_uncached<'tcx>( let uninit_ty = Ty::new_maybe_uninit(tcx, field_ty.instantiate(tcx, args)); cx.spanned_layout_of(uninit_ty, local.source_info.span) }) - .try_collect::>()?; + .try_collect()?; - let prefix_layouts = args + let relocated_upvars = IndexVec::from_fn_n( + |local: CoroutineSavedLocal| info.relocated_upvars.get(&local).copied(), + info.field_tys.len(), + ); + let pack = match info.pack { + PackCoroutineLayout::No => rustc_abi::PackCoroutineLayout::Classic, + PackCoroutineLayout::CapturesOnly => rustc_abi::PackCoroutineLayout::CapturesOnly, + }; + let upvar_layouts = args .as_coroutine() - .prefix_tys() + .upvar_tys() .iter() .map(|ty| cx.layout_of(ty)) - .try_collect::>()?; - + .collect::>()?; let layout = cx .calc .coroutine( &local_layouts, - prefix_layouts, + &relocated_upvars, + upvar_layouts, &info.variant_fields, &info.storage_conflicts, + pack, |tag| TyAndLayout { ty: tag.primitive().to_ty(tcx), layout: tcx.mk_layout(LayoutData::scalar(cx, tag)), @@ -892,7 +904,11 @@ fn variant_info_for_coroutine<'tcx>( .then(|| Symbol::intern(&field_layout.ty.to_string())), } }) - .chain(upvar_fields.iter().copied()) + .chain( + if variant_idx == FIRST_VARIANT { &upvar_fields[..] } else { &[] } + .iter() + .copied(), + ) .collect(); // If the variant has no state-specific fields, then it's the size of the upvars. diff --git a/src/tools/clippy/tests/ui/future_not_send.stderr b/src/tools/clippy/tests/ui/future_not_send.stderr index e366dc2d21958..8521eb13f62d7 100644 --- a/src/tools/clippy/tests/ui/future_not_send.stderr +++ b/src/tools/clippy/tests/ui/future_not_send.stderr @@ -93,10 +93,10 @@ LL | pub async fn public_future(&self) { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ future returned by `public_future` is not `Send` | note: captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - --> tests/ui/future_not_send.rs:49:32 + --> tests/ui/future_not_send.rs:49:33 | LL | pub async fn public_future(&self) { - | ^^^^^ has type `&Dummy` which is not `Send`, because `Dummy` is not `Sync` + | ^^^^ has type `&Dummy` which is not `Send`, because `Dummy` is not `Sync` = note: `std::rc::Rc<[u8]>` doesn't implement `std::marker::Sync` error: future cannot be sent between threads safely diff --git a/src/tools/clippy/tests/ui/large_futures.stderr b/src/tools/clippy/tests/ui/large_futures.stderr index 4280c9e2af284..06956ef9d4a88 100644 --- a/src/tools/clippy/tests/ui/large_futures.stderr +++ b/src/tools/clippy/tests/ui/large_futures.stderr @@ -1,4 +1,4 @@ -error: large future with a size of 16385 bytes +error: large future with a size of 32769 bytes --> tests/ui/large_futures.rs:13:9 | LL | big_fut([0u8; 1024 * 16]).await; @@ -7,19 +7,19 @@ LL | big_fut([0u8; 1024 * 16]).await; = note: `-D clippy::large-futures` implied by `-D warnings` = help: to override `-D warnings` add `#[allow(clippy::large_futures)]` -error: large future with a size of 16386 bytes +error: large future with a size of 32770 bytes --> tests/ui/large_futures.rs:16:5 | LL | f.await | ^ help: consider `Box::pin` on it: `Box::pin(f)` -error: large future with a size of 16387 bytes +error: large future with a size of 32771 bytes --> tests/ui/large_futures.rs:21:9 | LL | wait().await; | ^^^^^^ help: consider `Box::pin` on it: `Box::pin(wait())` -error: large future with a size of 16387 bytes +error: large future with a size of 32771 bytes --> tests/ui/large_futures.rs:27:13 | LL | wait().await; @@ -31,7 +31,7 @@ error: large future with a size of 65540 bytes LL | foo().await; | ^^^^^ help: consider `Box::pin` on it: `Box::pin(foo())` -error: large future with a size of 49159 bytes +error: large future with a size of 98311 bytes --> tests/ui/large_futures.rs:38:5 | LL | calls_fut(fut).await; diff --git a/src/tools/clippy/tests/ui/needless_lifetimes.fixed b/src/tools/clippy/tests/ui/needless_lifetimes.fixed index 15ca409c95bd1..ef8a4a91677d0 100644 --- a/src/tools/clippy/tests/ui/needless_lifetimes.fixed +++ b/src/tools/clippy/tests/ui/needless_lifetimes.fixed @@ -3,6 +3,7 @@ #![warn(clippy::needless_lifetimes, clippy::elidable_lifetime_names)] #![allow( unused, + clippy::await_holding_lock, clippy::boxed_local, clippy::extra_unused_type_parameters, clippy::needless_pass_by_value, diff --git a/src/tools/clippy/tests/ui/needless_lifetimes.rs b/src/tools/clippy/tests/ui/needless_lifetimes.rs index af9649d729872..86da853dc238f 100644 --- a/src/tools/clippy/tests/ui/needless_lifetimes.rs +++ b/src/tools/clippy/tests/ui/needless_lifetimes.rs @@ -3,6 +3,7 @@ #![warn(clippy::needless_lifetimes, clippy::elidable_lifetime_names)] #![allow( unused, + clippy::await_holding_lock, clippy::boxed_local, clippy::extra_unused_type_parameters, clippy::needless_pass_by_value, diff --git a/src/tools/clippy/tests/ui/needless_lifetimes.stderr b/src/tools/clippy/tests/ui/needless_lifetimes.stderr index 138d0498c43e4..c7669e4d175b4 100644 --- a/src/tools/clippy/tests/ui/needless_lifetimes.stderr +++ b/src/tools/clippy/tests/ui/needless_lifetimes.stderr @@ -1,5 +1,5 @@ error: the following explicit lifetimes could be elided: 'a, 'b - --> tests/ui/needless_lifetimes.rs:19:23 + --> tests/ui/needless_lifetimes.rs:20:23 | LL | fn distinct_lifetimes<'a, 'b>(_x: &'a u8, _y: &'b u8, _z: u8) {} | ^^ ^^ ^^ ^^ @@ -13,7 +13,7 @@ LL + fn distinct_lifetimes(_x: &u8, _y: &u8, _z: u8) {} | error: the following explicit lifetimes could be elided: 'a, 'b - --> tests/ui/needless_lifetimes.rs:22:24 + --> tests/ui/needless_lifetimes.rs:23:24 | LL | fn distinct_and_static<'a, 'b>(_x: &'a u8, _y: &'b u8, _z: &'static u8) {} | ^^ ^^ ^^ ^^ @@ -25,7 +25,7 @@ LL + fn distinct_and_static(_x: &u8, _y: &u8, _z: &'static u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:33:15 + --> tests/ui/needless_lifetimes.rs:34:15 | LL | fn in_and_out<'a>(x: &'a u8, _y: u8) -> &'a u8 { | ^^ ^^ ^^ @@ -37,7 +37,7 @@ LL + fn in_and_out(x: &u8, _y: u8) -> &u8 { | error: the following explicit lifetimes could be elided: 'b - --> tests/ui/needless_lifetimes.rs:46:31 + --> tests/ui/needless_lifetimes.rs:47:31 | LL | fn multiple_in_and_out_2a<'a, 'b>(x: &'a u8, _y: &'b u8) -> &'a u8 { | ^^ ^^ @@ -49,7 +49,7 @@ LL + fn multiple_in_and_out_2a<'a>(x: &'a u8, _y: &u8) -> &'a u8 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:54:27 + --> tests/ui/needless_lifetimes.rs:55:27 | LL | fn multiple_in_and_out_2b<'a, 'b>(_x: &'a u8, y: &'b u8) -> &'b u8 { | ^^ ^^ @@ -61,7 +61,7 @@ LL + fn multiple_in_and_out_2b<'b>(_x: &u8, y: &'b u8) -> &'b u8 { | error: the following explicit lifetimes could be elided: 'b - --> tests/ui/needless_lifetimes.rs:72:26 + --> tests/ui/needless_lifetimes.rs:73:26 | LL | fn deep_reference_1a<'a, 'b>(x: &'a u8, _y: &'b u8) -> Result<&'a u8, ()> { | ^^ ^^ @@ -73,7 +73,7 @@ LL + fn deep_reference_1a<'a>(x: &'a u8, _y: &u8) -> Result<&'a u8, ()> { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:80:22 + --> tests/ui/needless_lifetimes.rs:81:22 | LL | fn deep_reference_1b<'a, 'b>(_x: &'a u8, y: &'b u8) -> Result<&'b u8, ()> { | ^^ ^^ @@ -85,7 +85,7 @@ LL + fn deep_reference_1b<'b>(_x: &u8, y: &'b u8) -> Result<&'b u8, ()> { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:90:21 + --> tests/ui/needless_lifetimes.rs:91:21 | LL | fn deep_reference_3<'a>(x: &'a u8, _y: u8) -> Result<&'a u8, ()> { | ^^ ^^ ^^ @@ -97,7 +97,7 @@ LL + fn deep_reference_3(x: &u8, _y: u8) -> Result<&u8, ()> { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:96:28 + --> tests/ui/needless_lifetimes.rs:97:28 | LL | fn where_clause_without_lt<'a, T>(x: &'a u8, _y: u8) -> Result<&'a u8, ()> | ^^ ^^ ^^ @@ -109,7 +109,7 @@ LL + fn where_clause_without_lt(x: &u8, _y: u8) -> Result<&u8, ()> | error: the following explicit lifetimes could be elided: 's - --> tests/ui/needless_lifetimes.rs:127:21 + --> tests/ui/needless_lifetimes.rs:128:21 | LL | fn self_and_out<'s>(&'s self) -> &'s u8 { | ^^ ^^ ^^ @@ -121,7 +121,7 @@ LL + fn self_and_out(&self) -> &u8 { | error: the following explicit lifetimes could be elided: 't - --> tests/ui/needless_lifetimes.rs:135:30 + --> tests/ui/needless_lifetimes.rs:136:30 | LL | fn self_and_in_out_1<'s, 't>(&'s self, _x: &'t u8) -> &'s u8 { | ^^ ^^ @@ -133,7 +133,7 @@ LL + fn self_and_in_out_1<'s>(&'s self, _x: &u8) -> &'s u8 { | error: the following explicit lifetimes could be elided: 's - --> tests/ui/needless_lifetimes.rs:143:26 + --> tests/ui/needless_lifetimes.rs:144:26 | LL | fn self_and_in_out_2<'s, 't>(&'s self, x: &'t u8) -> &'t u8 { | ^^ ^^ @@ -145,7 +145,7 @@ LL + fn self_and_in_out_2<'t>(&self, x: &'t u8) -> &'t u8 { | error: the following explicit lifetimes could be elided: 's, 't - --> tests/ui/needless_lifetimes.rs:148:29 + --> tests/ui/needless_lifetimes.rs:149:29 | LL | fn distinct_self_and_in<'s, 't>(&'s self, _x: &'t u8) {} | ^^ ^^ ^^ ^^ @@ -157,7 +157,7 @@ LL + fn distinct_self_and_in(&self, _x: &u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:172:21 + --> tests/ui/needless_lifetimes.rs:173:21 | LL | fn struct_with_lt4b<'a, 'b>(_foo: &'a Foo<'b>) -> &'b str { | ^^ ^^ @@ -169,7 +169,7 @@ LL + fn struct_with_lt4b<'b>(_foo: &Foo<'b>) -> &'b str { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:188:22 + --> tests/ui/needless_lifetimes.rs:189:22 | LL | fn trait_obj_elided2<'a>(_arg: &'a dyn Drop) -> &'a str { | ^^ ^^ ^^ @@ -181,7 +181,7 @@ LL + fn trait_obj_elided2(_arg: &dyn Drop) -> &str { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:199:20 + --> tests/ui/needless_lifetimes.rs:200:20 | LL | fn alias_with_lt4b<'a, 'b>(_foo: &'a FooAlias<'b>) -> &'b str { | ^^ ^^ @@ -193,7 +193,7 @@ LL + fn alias_with_lt4b<'b>(_foo: &FooAlias<'b>) -> &'b str { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:204:30 + --> tests/ui/needless_lifetimes.rs:205:30 | LL | fn named_input_elided_output<'a>(_arg: &'a str) -> &str { | ^^ ^^ ^ @@ -205,7 +205,7 @@ LL + fn named_input_elided_output(_arg: &str) -> &str { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:213:19 + --> tests/ui/needless_lifetimes.rs:214:19 | LL | fn trait_bound_ok<'a, T: WithLifetime<'static>>(_: &'a u8, _: T) { | ^^ ^^ @@ -217,7 +217,7 @@ LL + fn trait_bound_ok>(_: &u8, _: T) { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:249:24 + --> tests/ui/needless_lifetimes.rs:250:24 | LL | fn needless_lt<'a>(x: &'a u8) {} | ^^ ^^ @@ -229,7 +229,7 @@ LL + fn needless_lt(x: &u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:254:24 + --> tests/ui/needless_lifetimes.rs:255:24 | LL | fn needless_lt<'a>(_x: &'a u8) {} | ^^ ^^ @@ -241,7 +241,7 @@ LL + fn needless_lt(_x: &u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:285:55 + --> tests/ui/needless_lifetimes.rs:286:55 | LL | fn impl_trait_elidable_nested_anonymous_lifetimes<'a>(i: &'a i32, f: impl Fn(&i32) -> &i32) -> &'a i32 { | ^^ ^^ ^^ @@ -253,7 +253,7 @@ LL + fn impl_trait_elidable_nested_anonymous_lifetimes(i: &i32, f: impl Fn(& | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:295:26 + --> tests/ui/needless_lifetimes.rs:296:26 | LL | fn generics_elidable<'a, T: Fn(&i32) -> &i32>(i: &'a i32, f: T) -> &'a i32 { | ^^ ^^ ^^ @@ -265,7 +265,7 @@ LL + fn generics_elidable &i32>(i: &i32, f: T) -> &i32 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:308:30 + --> tests/ui/needless_lifetimes.rs:309:30 | LL | fn where_clause_elidable<'a, T>(i: &'a i32, f: T) -> &'a i32 | ^^ ^^ ^^ @@ -277,7 +277,7 @@ LL + fn where_clause_elidable(i: &i32, f: T) -> &i32 | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:324:28 + --> tests/ui/needless_lifetimes.rs:325:28 | LL | fn pointer_fn_elidable<'a>(i: &'a i32, f: fn(&i32) -> &i32) -> &'a i32 { | ^^ ^^ ^^ @@ -289,7 +289,7 @@ LL + fn pointer_fn_elidable(i: &i32, f: fn(&i32) -> &i32) -> &i32 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:338:28 + --> tests/ui/needless_lifetimes.rs:339:28 | LL | fn nested_fn_pointer_3<'a>(_: &'a i32) -> fn(fn(&i32) -> &i32) -> i32 { | ^^ ^^ @@ -301,7 +301,7 @@ LL + fn nested_fn_pointer_3(_: &i32) -> fn(fn(&i32) -> &i32) -> i32 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:342:28 + --> tests/ui/needless_lifetimes.rs:343:28 | LL | fn nested_fn_pointer_4<'a>(_: &'a i32) -> impl Fn(fn(&i32)) { | ^^ ^^ @@ -313,7 +313,7 @@ LL + fn nested_fn_pointer_4(_: &i32) -> impl Fn(fn(&i32)) { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:365:21 + --> tests/ui/needless_lifetimes.rs:366:21 | LL | fn implicit<'a>(&'a self) -> &'a () { | ^^ ^^ ^^ @@ -325,7 +325,7 @@ LL + fn implicit(&self) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:369:25 + --> tests/ui/needless_lifetimes.rs:370:25 | LL | fn implicit_mut<'a>(&'a mut self) -> &'a () { | ^^ ^^ ^^ @@ -337,7 +337,7 @@ LL + fn implicit_mut(&mut self) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:374:21 + --> tests/ui/needless_lifetimes.rs:375:21 | LL | fn explicit<'a>(self: &'a Arc) -> &'a () { | ^^ ^^ ^^ @@ -349,7 +349,7 @@ LL + fn explicit(self: &Arc) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:379:25 + --> tests/ui/needless_lifetimes.rs:380:25 | LL | fn explicit_mut<'a>(self: &'a mut Rc) -> &'a () { | ^^ ^^ ^^ @@ -361,7 +361,7 @@ LL + fn explicit_mut(self: &mut Rc) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:392:31 + --> tests/ui/needless_lifetimes.rs:393:31 | LL | fn lifetime_elsewhere<'a>(self: Box, here: &'a ()) -> &'a () { | ^^ ^^ ^^ @@ -373,7 +373,7 @@ LL + fn lifetime_elsewhere(self: Box, here: &()) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:399:21 + --> tests/ui/needless_lifetimes.rs:400:21 | LL | fn implicit<'a>(&'a self) -> &'a (); | ^^ ^^ ^^ @@ -385,7 +385,7 @@ LL + fn implicit(&self) -> &(); | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:401:30 + --> tests/ui/needless_lifetimes.rs:402:30 | LL | fn implicit_provided<'a>(&'a self) -> &'a () { | ^^ ^^ ^^ @@ -397,7 +397,7 @@ LL + fn implicit_provided(&self) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:407:21 + --> tests/ui/needless_lifetimes.rs:408:21 | LL | fn explicit<'a>(self: &'a Arc) -> &'a (); | ^^ ^^ ^^ @@ -409,7 +409,7 @@ LL + fn explicit(self: &Arc) -> &(); | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:410:30 + --> tests/ui/needless_lifetimes.rs:411:30 | LL | fn explicit_provided<'a>(self: &'a Arc) -> &'a () { | ^^ ^^ ^^ @@ -421,7 +421,7 @@ LL + fn explicit_provided(self: &Arc) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:421:31 + --> tests/ui/needless_lifetimes.rs:422:31 | LL | fn lifetime_elsewhere<'a>(self: Box, here: &'a ()) -> &'a (); | ^^ ^^ ^^ @@ -433,7 +433,7 @@ LL + fn lifetime_elsewhere(self: Box, here: &()) -> &(); | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:423:40 + --> tests/ui/needless_lifetimes.rs:424:40 | LL | fn lifetime_elsewhere_provided<'a>(self: Box, here: &'a ()) -> &'a () { | ^^ ^^ ^^ @@ -445,7 +445,7 @@ LL + fn lifetime_elsewhere_provided(self: Box, here: &()) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:433:12 + --> tests/ui/needless_lifetimes.rs:434:12 | LL | fn foo<'a>(x: &'a u8, y: &'_ u8) {} | ^^ ^^ @@ -457,7 +457,7 @@ LL + fn foo(x: &u8, y: &'_ u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:436:12 + --> tests/ui/needless_lifetimes.rs:437:12 | LL | fn bar<'a>(x: &'a u8, y: &'_ u8, z: &'_ u8) {} | ^^ ^^ @@ -469,7 +469,7 @@ LL + fn bar(x: &u8, y: &'_ u8, z: &'_ u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:444:18 + --> tests/ui/needless_lifetimes.rs:445:18 | LL | fn one_input<'a>(x: &'a u8) -> &'a u8 { | ^^ ^^ ^^ @@ -481,7 +481,7 @@ LL + fn one_input(x: &u8) -> &u8 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:450:42 + --> tests/ui/needless_lifetimes.rs:451:42 | LL | fn multiple_inputs_output_not_elided<'a, 'b>(x: &'a u8, y: &'b u8, z: &'b u8) -> &'b u8 { | ^^ ^^ @@ -493,7 +493,7 @@ LL + fn multiple_inputs_output_not_elided<'b>(x: &u8, y: &'b u8, z: &'b u8) | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:467:22 + --> tests/ui/needless_lifetimes.rs:468:22 | LL | fn one_input<'a>(x: &'a u8) -> &'a u8 { | ^^ ^^ ^^ diff --git a/src/tools/miri/tests/pass/async-drop.rs b/src/tools/miri/tests/pass/async-drop.rs index 4fa84384d9bdd..6754d4ffbef10 100644 --- a/src/tools/miri/tests/pass/async-drop.rs +++ b/src/tools/miri/tests/pass/async-drop.rs @@ -1,6 +1,8 @@ -//@revisions: stack tree -//@compile-flags: -Zmiri-strict-provenance -//@[tree]compile-flags: -Zmiri-tree-borrows +//@ revisions: stack tree stackrelocate treerelocate +//@ compile-flags: -Zmiri-strict-provenance -Zpack-coroutine-layout=no +//@ [tree] compile-flags: -Zmiri-tree-borrows +//@ [stackrelocate] compile-flags: -Zpack-coroutine-layout=captures-only +//@ [treerelocate] compile-flags: -Zmiri-tree-borrows -Zpack-coroutine-layout=captures-only // WARNING: If you would ever want to modify this test, // please consider modifying rustc's async drop test at diff --git a/src/tools/miri/tests/pass/async-drop.stackrelocate.stdout b/src/tools/miri/tests/pass/async-drop.stackrelocate.stdout new file mode 100644 index 0000000000000..fc53df2f1b485 --- /dev/null +++ b/src/tools/miri/tests/pass/async-drop.stackrelocate.stdout @@ -0,0 +1,23 @@ +AsyncInt::async_drop: 0 +AsyncInt::async_drop: 1 +AsyncInt::async_drop: 2 +AsyncInt::async_drop: 3 +AsyncInt::async_drop: 4 +AsyncStruct::async_drop: 6 +AsyncInt::async_drop: 7 +AsyncInt::async_drop: 8 +AsyncReference::async_drop: 10 +AsyncInt::async_drop: 11 +AsyncEnum(A)::async_drop: 12 +SyncInt::drop: 12 +AsyncEnum(B)::async_drop: 13 +AsyncInt::async_drop: 13 +SyncInt::drop: 14 +SyncThenAsync::drop: 15 +AsyncInt::async_drop: 16 +SyncInt::drop: 17 +AsyncInt::async_drop: 18 +AsyncInt::async_drop: 19 +AsyncInt::async_drop: 20 +AsyncUnion::async_drop: 21, 21 +AsyncInt::async_drop: 10 diff --git a/src/tools/miri/tests/pass/async-drop.treerelocate.stdout b/src/tools/miri/tests/pass/async-drop.treerelocate.stdout new file mode 100644 index 0000000000000..fc53df2f1b485 --- /dev/null +++ b/src/tools/miri/tests/pass/async-drop.treerelocate.stdout @@ -0,0 +1,23 @@ +AsyncInt::async_drop: 0 +AsyncInt::async_drop: 1 +AsyncInt::async_drop: 2 +AsyncInt::async_drop: 3 +AsyncInt::async_drop: 4 +AsyncStruct::async_drop: 6 +AsyncInt::async_drop: 7 +AsyncInt::async_drop: 8 +AsyncReference::async_drop: 10 +AsyncInt::async_drop: 11 +AsyncEnum(A)::async_drop: 12 +SyncInt::drop: 12 +AsyncEnum(B)::async_drop: 13 +AsyncInt::async_drop: 13 +SyncInt::drop: 14 +SyncThenAsync::drop: 15 +AsyncInt::async_drop: 16 +SyncInt::drop: 17 +AsyncInt::async_drop: 18 +AsyncInt::async_drop: 19 +AsyncInt::async_drop: 20 +AsyncUnion::async_drop: 21, 21 +AsyncInt::async_drop: 10 diff --git a/tests/codegen-llvm/async-fn-debug.rs b/tests/codegen-llvm/async-fn-debug.rs index ed704c7cc8b92..c430a12700414 100644 --- a/tests/codegen-llvm/async-fn-debug.rs +++ b/tests/codegen-llvm/async-fn-debug.rs @@ -38,17 +38,17 @@ async fn async_fn_test() { // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) // CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "3", scope: [[VARIANT]], -// CHECK-SAME: file: [[FILE]], line: 13, +// CHECK-SAME: file: [[FILE]], line: 14, // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) -// CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "4", scope: [[VARIANT]], -// CHECK-SAME: file: [[FILE]], line: 15, +// CHECK: [[S0:!.*]] = !DICompositeType(tag: DW_TAG_structure_type, name: "Suspend0", scope: [[GEN]], // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) -// CHECK: [[S1:!.*]] = !DICompositeType(tag: DW_TAG_structure_type, name: "Suspend1", scope: [[GEN]], +// CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "s", scope: [[S0]] // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) -// CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "s", scope: [[S1]] +// CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "4", scope: [[VARIANT]], +// CHECK-SAME: file: [[FILE]], line: 12, // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) // CHECK: [[DISC]] = !DIDerivedType(tag: DW_TAG_member, name: "__state", scope: [[GEN]], diff --git a/tests/debuginfo/coroutine-objects.rs b/tests/debuginfo/coroutine-objects.rs index 7ead154cbdb1f..8b7a19b545dad 100644 --- a/tests/debuginfo/coroutine-objects.rs +++ b/tests/debuginfo/coroutine-objects.rs @@ -4,29 +4,29 @@ // with memory layout of discriminant for this particular enum // ensure that LLDB won't crash at least (like #57822). -//@ compile-flags:-g +//@ compile-flags: -g -Z pack-coroutine-layout=no //@ disable-gdb-pretty-printers // === GDB TESTS =================================================================================== // gdb-command:run // gdb-command:print b -// gdb-check:$1 = coroutine_objects::main::{coroutine_env#0}::Unresumed{_ref__a: 0x[...]} +// gdb-check:$1 = coroutine_objects::main::{coroutine_env#0}::Unresumed{a: 0x[...]} // gdb-command:continue // gdb-command:print b -// gdb-check:$2 = coroutine_objects::main::{coroutine_env#0}::Suspend0{c: 6, d: 7, _ref__a: 0x[...]} +// gdb-check:$2 = coroutine_objects::main::{coroutine_env#0}::Suspend0{c: 6, d: 7} // gdb-command:continue // gdb-command:print b -// gdb-check:$3 = coroutine_objects::main::{coroutine_env#0}::Suspend1{c: 7, d: 8, _ref__a: 0x[...]} +// gdb-check:$3 = coroutine_objects::main::{coroutine_env#0}::Suspend1{c: 7, d: 8} // gdb-command:continue // gdb-command:print b -// gdb-check:$4 = coroutine_objects::main::{coroutine_env#0}::Returned{_ref__a: 0x[...]} +// gdb-check:$4 = coroutine_objects::main::{coroutine_env#0}::Returned // === LLDB TESTS ================================================================================== // lldb-command:run // lldb-command:v b -// lldb-check:(coroutine_objects::main::{coroutine_env#0}) b = { value = { _ref__a = 0x[...] } $discr$ = [...] } +// lldb-check:(coroutine_objects::main::{coroutine_env#0}) b = { value = { a = 0x[...] } $discr$ = [...] } // === CDB TESTS =================================================================================== diff --git a/tests/debuginfo/coroutine-objects_relocated.rs b/tests/debuginfo/coroutine-objects_relocated.rs new file mode 100644 index 0000000000000..83113f89607ed --- /dev/null +++ b/tests/debuginfo/coroutine-objects_relocated.rs @@ -0,0 +1,89 @@ +//@ min-lldb-version: 1800 + +// LLDB (18.1+) now supports DW_TAG_variant_part, but there is some bug in either compiler or LLDB +// with memory layout of discriminant for this particular enum +// ensure that LLDB won't crash at least (like #57822). + +//@ compile-flags: -g -Z pack-coroutine-layout=captures-only +//@ disable-gdb-pretty-printers + +// === GDB TESTS =================================================================================== + +// gdb-command:run +// gdb-command:print b +// gdb-check:$1 = coroutine_objects_relocated::main::{coroutine_env#0}::Unresumed{a: 0x[...]} +// gdb-command:continue +// gdb-command:print b +// gdb-check:$2 = coroutine_objects_relocated::main::{coroutine_env#0}::Suspend0{c: 6, d: 7, a: 0x[...]} +// gdb-command:continue +// gdb-command:print b +// gdb-check:$3 = coroutine_objects_relocated::main::{coroutine_env#0}::Suspend1{c: 7, d: 8, a: 0x[...]} +// gdb-command:continue +// gdb-command:print b +// gdb-check:$4 = coroutine_objects_relocated::main::{coroutine_env#0}::Returned + +// === LLDB TESTS ================================================================================== + +// lldb-command:run +// lldb-command:v b +// lldb-check:(coroutine_objects_relocated::main::{coroutine_env#0}) b = { value = { a = 0x[...] } $discr$ = [...] } + +// === CDB TESTS =================================================================================== + +// cdb-command: g +// cdb-command: dx b +// cdb-check: b : Unresumed [Type: enum2$] +// cdb-check: [+0x[...]] _ref__a : 0x[...] : 5 [Type: int *] + +// cdb-command: g +// cdb-command: dx b +// cdb-check: b : Suspend0 [Type: enum2$] +// cdb-check: [+0x[...]] c : 6 [Type: int] +// cdb-check: [+0x[...]] d : 7 [Type: int] +// cdb-check: [+0x[...]] _ref__a : 0x[...] : 5 [Type: int *] + +// cdb-command: g +// cdb-command: dx b +// cdb-check: b : Suspend1 [Type: enum2$] +// cdb-check: [+0x[...]] c : 7 [Type: int] +// cdb-check: [+0x[...]] d : 8 [Type: int] +// cdb-check: [+0x[...]] _ref__a : 0x[...] : 6 [Type: int *] + +// cdb-command: g +// cdb-command: dx b +// cdb-check: b : Returned [Type: enum2$] +// cdb-check: [+0x[...]] _ref__a : 0x[...] : 6 [Type: int *] + +#![feature(coroutines, coroutine_trait, stmt_expr_attributes)] + +use std::ops::Coroutine; +use std::pin::Pin; + +fn main() { + let mut a = 5; + let mut b = #[coroutine] + || { + let mut c = 6; + let mut d = 7; + + yield; + a += 1; + c += 1; + d += 1; + + yield; + println!("{} {} {}", a, c, d); + }; + _zzz(); // #break + Pin::new(&mut b).resume(()); + _zzz(); // #break + Pin::new(&mut b).resume(()); + _zzz(); // #break + Pin::new(&mut b).resume(()); + _zzz(); // #break +} + +#[inline(never)] +fn _zzz() { + () +} diff --git a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir index 347e4119cd0e0..918330c12321a 100644 --- a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir +++ b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir @@ -2,7 +2,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) -> Poll<()> { debug _task_context => _19; - debug x => ((*(_1.0: &mut {async fn body of a()})).0: T); + debug x => (((*(_1.0: &mut {async fn body of a()})) as variant#0).0: T); let mut _0: std::task::Poll<()>; let _3: T; let mut _4: impl std::future::Future; @@ -82,7 +82,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) } bb11: { - drop(((*(_1.0: &mut {async fn body of a()})).0: T)) -> [return: bb10, unwind unreachable]; + drop((((*(_1.0: &mut {async fn body of a()})) as variant#0).0: T)) -> [return: bb10, unwind unreachable]; } bb12: { diff --git a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir index b1cf5373f9191..74db5063ffe28 100644 --- a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir +++ b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir @@ -2,7 +2,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) -> Poll<()> { debug _task_context => _19; - debug x => ((*(_1.0: &mut {async fn body of a()})).0: T); + debug x => (((*(_1.0: &mut {async fn body of a()})) as variant#0).0: T); let mut _0: std::task::Poll<()>; let _3: T; let mut _4: impl std::future::Future; @@ -96,7 +96,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) } bb14: { - drop(((*(_1.0: &mut {async fn body of a()})).0: T)) -> [return: bb13, unwind: bb4]; + drop((((*(_1.0: &mut {async fn body of a()})) as variant#0).0: T)) -> [return: bb13, unwind: bb4]; } bb15 (cleanup): { diff --git a/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir b/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir index 7480324b17791..1a7c791474903 100644 --- a/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir +++ b/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir @@ -7,6 +7,7 @@ Panicked (2): [], }, storage_conflicts: BitMatrix(0x0) {}, + relocated_upvars: {}, } */ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) -> Poll<()> { diff --git a/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir b/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir index 9bff257e06392..b91ecc345148a 100644 --- a/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir +++ b/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir @@ -1,7 +1,7 @@ // MIR for `b::{closure#0}` 0 coroutine_resume /* coroutine_layout = CoroutineLayout { field_tys: { - _0: CoroutineSavedTy { + corsl_0: CoroutineSavedTy { ty: Coroutine( DefId(0:5 ~ async_await[ccf8]::a::{closure#0}), [ @@ -18,7 +18,7 @@ }, ignore_for_traits: false, }, - _1: CoroutineSavedTy { + corsl_1: CoroutineSavedTy { ty: Coroutine( DefId(0:5 ~ async_await[ccf8]::a::{closure#0}), [ @@ -40,13 +40,14 @@ Unresumed(0): [], Returned (1): [], Panicked (2): [], - Suspend0 (3): [_0], - Suspend1 (4): [_1], + Suspend0 (3): [corsl_1], + Suspend1 (4): [corsl_0], }, storage_conflicts: BitMatrix(2x2) { - (_0, _0), - (_1, _1), + (corsl_0, corsl_0), + (corsl_1, corsl_1), }, + relocated_upvars: {}, } */ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> Poll<()> { @@ -88,14 +89,14 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> let mut _38: &mut std::task::Context<'_>; let mut _39: u32; scope 1 { - debug __awaitee => (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}); + debug __awaitee => (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}); let _17: (); scope 2 { debug result => _17; } } scope 3 { - debug __awaitee => (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}); + debug __awaitee => (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}); let _33: (); scope 4 { debug result => _33; @@ -123,7 +124,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_5); PlaceMention(_4); nop; - (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}) = move _4; + (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}) = move _4; goto -> bb4; } @@ -133,7 +134,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageLive(_10); StorageLive(_11); StorageLive(_12); - _12 = &mut (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}); + _12 = &mut (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}); _11 = &mut (*_12); _10 = Pin::<&mut {async fn body of a()}>::new_unchecked(move _11) -> [return: bb5, unwind unreachable]; } @@ -180,7 +181,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_4); StorageDead(_19); StorageDead(_20); - discriminant((*(_1.0: &mut {async fn body of b()}))) = 3; + discriminant((*(_1.0: &mut {async fn body of b()}))) = 4; return; } @@ -193,7 +194,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_12); StorageDead(_9); StorageDead(_8); - drop((((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()})) -> [return: bb12, unwind unreachable]; + drop((((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()})) -> [return: bb12, unwind unreachable]; } bb11: { @@ -225,7 +226,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_22); PlaceMention(_21); nop; - (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}) = move _21; + (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}) = move _21; goto -> bb16; } @@ -235,7 +236,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageLive(_26); StorageLive(_27); StorageLive(_28); - _28 = &mut (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}); + _28 = &mut (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}); _27 = &mut (*_28); _26 = Pin::<&mut {async fn body of a()}>::new_unchecked(move _27) -> [return: bb17, unwind unreachable]; } @@ -277,7 +278,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_21); StorageDead(_35); StorageDead(_36); - discriminant((*(_1.0: &mut {async fn body of b()}))) = 4; + discriminant((*(_1.0: &mut {async fn body of b()}))) = 3; return; } @@ -290,7 +291,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_28); StorageDead(_25); StorageDead(_24); - drop((((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()})) -> [return: bb23, unwind unreachable]; + drop((((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()})) -> [return: bb23, unwind unreachable]; } bb22: { @@ -322,6 +323,14 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> } bb27: { + StorageLive(_21); + StorageLive(_35); + StorageLive(_36); + _35 = move _2; + goto -> bb22; + } + + bb28: { StorageLive(_3); StorageLive(_4); StorageLive(_19); @@ -330,14 +339,6 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> goto -> bb11; } - bb28: { - StorageLive(_21); - StorageLive(_35); - StorageLive(_36); - _35 = move _2; - goto -> bb22; - } - bb29: { assert(const false, "`async fn` resumed after completion") -> [success: bb29, unwind unreachable]; } diff --git a/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir b/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir index f8b3f68d21e63..73ce0b567f1ff 100644 --- a/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir +++ b/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir @@ -1,7 +1,7 @@ // MIR for `main::{closure#0}` 0 coroutine_resume /* coroutine_layout = CoroutineLayout { field_tys: { - _0: CoroutineSavedTy { + corsl_0: CoroutineSavedTy { ty: HasDrop, source_info: SourceInfo { span: $DIR/coroutine_tiny.rs:22:13: 22:15 (#0), @@ -14,11 +14,12 @@ Unresumed(0): [], Returned (1): [], Panicked (2): [], - Suspend0 (3): [_0], + Suspend0 (3): [corsl_0], }, storage_conflicts: BitMatrix(1x1) { - (_0, _0), + (corsl_0, corsl_0), }, + relocated_upvars: {}, } */ fn main::{closure#0}(_1: Pin<&mut {coroutine@$DIR/coroutine_tiny.rs:21:5: 21:13}>, _2: u8) -> CoroutineState<(), ()> { diff --git a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff index 22e6ea722ddaf..6ea67e6a3c455 100644 --- a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff +++ b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff @@ -189,7 +189,7 @@ + _31 = move _9; + _34 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); + _35 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ (((*_34) as variant#3).0: ActionPermit<'_, T>) = move ((*_35).0: ActionPermit<'_, T>); ++ (((*_34) as variant#3).0: ActionPermit<'_, T>) = move (((*_35) as variant#0).0: ActionPermit<'_, T>); + StorageLive(_12); + StorageLive(_13); + StorageLive(_14); diff --git a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff index 8b027e988b8e8..8e3d3ad6f2284 100644 --- a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff @@ -206,7 +206,7 @@ + _31 = move _9; + _34 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); + _35 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ (((*_34) as variant#3).0: ActionPermit<'_, T>) = move ((*_35).0: ActionPermit<'_, T>); ++ (((*_34) as variant#3).0: ActionPermit<'_, T>) = move (((*_35) as variant#0).0: ActionPermit<'_, T>); + StorageLive(_12); + StorageLive(_13); + StorageLive(_14); diff --git a/tests/ui/async-await/async-drop/async-drop-initial.run.stdout b/tests/ui/async-await/async-drop/async-drop-initial.classic.run.stdout similarity index 94% rename from tests/ui/async-await/async-drop/async-drop-initial.run.stdout rename to tests/ui/async-await/async-drop/async-drop-initial.classic.run.stdout index 9cae4331caf92..792880d835935 100644 --- a/tests/ui/async-await/async-drop/async-drop-initial.run.stdout +++ b/tests/ui/async-await/async-drop/async-drop-initial.classic.run.stdout @@ -20,3 +20,4 @@ AsyncInt::Dropper::poll: 18 AsyncInt::Dropper::poll: 19 AsyncInt::Dropper::poll: 20 AsyncUnion::Dropper::poll: 21, 21 +AsyncUnion::Dropper::poll: 21, 21 diff --git a/tests/ui/async-await/async-drop/async-drop-initial.relocate.run.stdout b/tests/ui/async-await/async-drop/async-drop-initial.relocate.run.stdout new file mode 100644 index 0000000000000..792880d835935 --- /dev/null +++ b/tests/ui/async-await/async-drop/async-drop-initial.relocate.run.stdout @@ -0,0 +1,23 @@ +AsyncInt::Dropper::poll: 0 +AsyncInt::Dropper::poll: 1 +AsyncInt::Dropper::poll: 2 +AsyncInt::Dropper::poll: 3 +AsyncInt::Dropper::poll: 4 +AsyncStruct::Dropper::poll: 6 +AsyncInt::Dropper::poll: 7 +AsyncInt::Dropper::poll: 8 +AsyncReference::Dropper::poll: 10 +AsyncInt::Dropper::poll: 11 +AsyncEnum(A)::Dropper::poll: 12 +SyncInt::drop: 12 +AsyncEnum(B)::Dropper::poll: 13 +AsyncInt::Dropper::poll: 13 +SyncInt::drop: 14 +SyncThenAsync::drop: 15 +AsyncInt::Dropper::poll: 16 +SyncInt::drop: 17 +AsyncInt::Dropper::poll: 18 +AsyncInt::Dropper::poll: 19 +AsyncInt::Dropper::poll: 20 +AsyncUnion::Dropper::poll: 21, 21 +AsyncUnion::Dropper::poll: 21, 21 diff --git a/tests/ui/async-await/async-drop/async-drop-initial.rs b/tests/ui/async-await/async-drop/async-drop-initial.rs index cd33c143fba01..64f370babdbdc 100644 --- a/tests/ui/async-await/async-drop/async-drop-initial.rs +++ b/tests/ui/async-await/async-drop/async-drop-initial.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only //@ run-pass //@ check-run-results @@ -11,26 +14,29 @@ //@ edition: 2021 // FIXME(zetanumbers): consider AsyncDestruct::async_drop cleanup tests -use core::future::{async_drop_in_place, AsyncDrop, Future}; +use core::future::{AsyncDrop, Future, async_drop_in_place}; use core::hint::black_box; use core::mem::{self, ManuallyDrop}; -use core::pin::{pin, Pin}; +use core::pin::{Pin, pin}; use core::task::{Context, Poll, Waker}; +use core::sync::atomic::{AtomicBool, Ordering}; -async fn test_async_drop(x: T, _size: usize) { +static PASS: AtomicBool = AtomicBool::new(false); + +async fn test_async_drop(x: T, #[allow(unused)] expect: usize) { let mut x = mem::MaybeUninit::new(x); let dtor = pin!(unsafe { async_drop_in_place(x.as_mut_ptr()) }); // FIXME(zetanumbers): This check fully depends on the layout of // the coroutine state, since async destructor combinators are just // async functions. + #[allow(unused)] + let got = mem::size_of_val(&*dtor); #[cfg(target_pointer_width = "64")] - assert_eq!( - mem::size_of_val(&*dtor), - _size, - "sizes did not match for async destructor of type {}", - core::any::type_name::(), - ); + if expect != got { + println!("sizes did not match for async destructor of type {}, expect {expect}, got {got}", core::any::type_name::()); + PASS.store(false, Ordering::Relaxed); + } test_idempotency(dtor).await; } @@ -47,27 +53,32 @@ where } fn main() { + PASS.store(true, Ordering::Relaxed); let waker = Waker::noop(); let mut cx = Context::from_waker(&waker); let i = 13; let fut = pin!(async { - test_async_drop(Int(0), 16).await; - test_async_drop(AsyncInt(0), 32).await; - test_async_drop([AsyncInt(1), AsyncInt(2)], 104).await; - test_async_drop((AsyncInt(3), AsyncInt(4)), 120).await; - test_async_drop(5, 16).await; + test_async_drop(Int(0), [16, 24][cfg!(classic) as usize]).await; + test_async_drop(AsyncInt(0), [32, 48][cfg!(classic) as usize]).await; + test_async_drop([AsyncInt(1), AsyncInt(2)], [96, 136][cfg!(classic) as usize]).await; + test_async_drop((AsyncInt(3), AsyncInt(4)), [112, 168][cfg!(classic) as usize]).await; + test_async_drop(5, [16, 24][cfg!(classic) as usize]).await; let j = 42; - test_async_drop(&i, 16).await; - test_async_drop(&j, 16).await; + test_async_drop(&i, [16, 24][cfg!(classic) as usize]).await; + test_async_drop(&j, [16, 24][cfg!(classic) as usize]).await; test_async_drop( AsyncStruct { b: AsyncInt(8), a: AsyncInt(7), i: 6 }, - 136, + if cfg!(panic = "unwind") { + [128, 192][cfg!(classic) as usize] + } else { + 136 + }, ).await; - test_async_drop(ManuallyDrop::new(AsyncInt(9)), 16).await; + test_async_drop(ManuallyDrop::new(AsyncInt(9)), [16, 24][cfg!(classic) as usize]).await; let foo = AsyncInt(10); - test_async_drop(AsyncReference { foo: &foo }, 32).await; + test_async_drop(AsyncReference { foo: &foo }, [32, 48][cfg!(classic) as usize]).await; let _ = ManuallyDrop::new(foo); let foo = AsyncInt(11); @@ -77,17 +88,17 @@ fn main() { let foo = AsyncInt(10); foo }, - 48, + [48, 72][cfg!(classic) as usize], ) .await; - test_async_drop(AsyncEnum::A(AsyncInt(12)), 104).await; - test_async_drop(AsyncEnum::B(SyncInt(13)), 104).await; + test_async_drop(AsyncEnum::A(AsyncInt(12)), [96, 144][cfg!(classic) as usize]).await; + test_async_drop(AsyncEnum::B(SyncInt(13)), [96, 144][cfg!(classic) as usize]).await; - test_async_drop(SyncInt(14), 16).await; + test_async_drop(SyncInt(14), [16, 24][cfg!(classic) as usize]).await; test_async_drop( SyncThenAsync { i: 15, a: AsyncInt(16), b: SyncInt(17), c: AsyncInt(18) }, - 120, + [112, 168][cfg!(classic) as usize], ) .await; @@ -104,14 +115,16 @@ fn main() { black_box(core::future::ready(())).await; foo }, - 48, + [48, 72][cfg!(classic) as usize], ) .await; - test_async_drop(AsyncUnion { signed: 21 }, 32).await; + test_async_drop(AsyncUnion { signed: 21 }, [32, 48][cfg!(classic) as usize]).await; + test_async_drop(AsyncUnion { signed: 21 }, [32, 48][cfg!(classic) as usize]).await; }); let res = fut.poll(&mut cx); assert_eq!(res, Poll::Ready(())); + assert!(PASS.load(Ordering::Relaxed)); } struct AsyncInt(i32); diff --git a/tests/ui/async-await/future-sizes/async-awaiting-fut.classic.stdout b/tests/ui/async-await/future-sizes/async-awaiting-fut.classic.stdout new file mode 100644 index 0000000000000..6e5011992474e --- /dev/null +++ b/tests/ui/async-await/future-sizes/async-awaiting-fut.classic.stdout @@ -0,0 +1,76 @@ +print-type-size type: `{async fn body of test()}`: 6150 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 0 bytes +print-type-size variant `Suspend0`: 6149 bytes +print-type-size local `.__awaitee`: 6149 bytes, type: {async fn body of calls_fut<{async fn body of big_fut()}>()} +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 6149 bytes, alignment: 1 bytes +print-type-size field `.value`: 6149 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 6149 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 6149 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 6149 bytes +print-type-size type: `{async fn body of calls_fut<{async fn body of big_fut()}>()}`: 6149 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 2049 bytes +print-type-size local `.fut`: 2049 bytes +print-type-size upvar `.fut`: 2049 bytes, offset: 1 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 4100 bytes +print-type-size padding: 2049 bytes +print-type-size local `.fut`: 2049 bytes, alignment: 1 bytes +print-type-size local `..coroutine_field4`: 1 bytes, type: bool +print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} +print-type-size variant `Suspend1`: 6148 bytes +print-type-size padding: 4098 bytes +print-type-size local `..coroutine_field4`: 1 bytes, alignment: 1 bytes, type: bool +print-type-size local `.__awaitee`: 2049 bytes, type: {async fn body of big_fut()} +print-type-size variant `Suspend2`: 4100 bytes +print-type-size padding: 2049 bytes +print-type-size local `.fut`: 2049 bytes, alignment: 1 bytes +print-type-size local `..coroutine_field4`: 1 bytes, type: bool +print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} +print-type-size variant `Returned`: 2049 bytes +print-type-size variant `Panicked`: 2049 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of big_fut()}>`: 2049 bytes, alignment: 1 bytes +print-type-size field `.value`: 2049 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of big_fut()}>`: 2049 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 2049 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 2049 bytes +print-type-size type: `{async fn body of big_fut()}`: 2049 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size local `.arg`: 1024 bytes +print-type-size upvar `.arg`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes +print-type-size variant `Returned`: 1024 bytes +print-type-size variant `Panicked`: 1024 bytes +print-type-size end padding: 1024 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1024 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::ManuallyDrop`: 1 bytes, alignment: 1 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of wait()}>`: 1 bytes, alignment: 1 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::mem::MaybeUninit`: 1 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of wait()}>`: 1 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::task::Poll<()>`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Ready`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size variant `Pending`: 0 bytes +print-type-size type: `{async fn body of wait()}`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 0 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes diff --git a/tests/ui/async-await/future-sizes/async-awaiting-fut.stdout b/tests/ui/async-await/future-sizes/async-awaiting-fut.relocate.stdout similarity index 66% rename from tests/ui/async-await/future-sizes/async-awaiting-fut.stdout rename to tests/ui/async-await/future-sizes/async-awaiting-fut.relocate.stdout index 642e27b2a57d6..c7342b38cc503 100644 --- a/tests/ui/async-await/future-sizes/async-awaiting-fut.stdout +++ b/tests/ui/async-await/future-sizes/async-awaiting-fut.relocate.stdout @@ -1,39 +1,36 @@ -print-type-size type: `{async fn body of test()}`: 3078 bytes, alignment: 1 bytes +print-type-size type: `{async fn body of test()}`: 2053 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 0 bytes -print-type-size variant `Suspend0`: 3077 bytes -print-type-size local `.__awaitee`: 3077 bytes, type: {async fn body of calls_fut<{async fn body of big_fut()}>()} +print-type-size variant `Suspend0`: 2052 bytes +print-type-size local `.__awaitee`: 2052 bytes, type: {async fn body of calls_fut<{async fn body of big_fut()}>()} print-type-size variant `Returned`: 0 bytes print-type-size variant `Panicked`: 0 bytes -print-type-size type: `std::mem::ManuallyDrop<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 3077 bytes, alignment: 1 bytes -print-type-size field `.value`: 3077 bytes -print-type-size type: `std::mem::MaybeUninit<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 3077 bytes, alignment: 1 bytes -print-type-size variant `MaybeUninit`: 3077 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 2052 bytes, alignment: 1 bytes +print-type-size field `.value`: 2052 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 2052 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 2052 bytes print-type-size field `.uninit`: 0 bytes -print-type-size field `.value`: 3077 bytes -print-type-size type: `{async fn body of calls_fut<{async fn body of big_fut()}>()}`: 3077 bytes, alignment: 1 bytes +print-type-size field `.value`: 2052 bytes +print-type-size type: `{async fn body of calls_fut<{async fn body of big_fut()}>()}`: 2052 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes -print-type-size variant `Unresumed`: 1025 bytes -print-type-size upvar `.fut`: 1025 bytes -print-type-size variant `Suspend0`: 2052 bytes -print-type-size upvar `.fut`: 1025 bytes +print-type-size variant `Unresumed`: 2051 bytes +print-type-size upvar `.fut`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size padding: 1026 bytes +print-type-size local `.fut`: 1025 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 1027 bytes print-type-size local `.fut`: 1025 bytes print-type-size local `..coroutine_field4`: 1 bytes, type: bool print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} -print-type-size variant `Suspend1`: 3076 bytes -print-type-size upvar `.fut`: 1025 bytes +print-type-size variant `Suspend1`: 2051 bytes print-type-size padding: 1025 bytes print-type-size local `..coroutine_field4`: 1 bytes, alignment: 1 bytes, type: bool print-type-size local `.__awaitee`: 1025 bytes, type: {async fn body of big_fut()} -print-type-size variant `Suspend2`: 2052 bytes -print-type-size upvar `.fut`: 1025 bytes +print-type-size variant `Suspend2`: 1027 bytes print-type-size local `.fut`: 1025 bytes print-type-size local `..coroutine_field4`: 1 bytes, type: bool print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} -print-type-size variant `Returned`: 1025 bytes -print-type-size upvar `.fut`: 1025 bytes -print-type-size variant `Panicked`: 1025 bytes -print-type-size upvar `.fut`: 1025 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes print-type-size type: `std::mem::ManuallyDrop<{async fn body of big_fut()}>`: 1025 bytes, alignment: 1 bytes print-type-size field `.value`: 1025 bytes print-type-size type: `std::mem::MaybeUninit<{async fn body of big_fut()}>`: 1025 bytes, alignment: 1 bytes @@ -43,11 +40,16 @@ print-type-size field `.value`: 1025 bytes print-type-size type: `{async fn body of big_fut()}`: 1025 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.arg`: 1024 bytes -print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.arg`: 1024 bytes -print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.arg`: 1024 bytes +print-type-size upvar `.arg`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.arg`: 1024 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1024 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1024 bytes print-type-size type: `std::mem::ManuallyDrop`: 1 bytes, alignment: 1 bytes print-type-size field `.value`: 1 bytes print-type-size type: `std::mem::ManuallyDrop<{async fn body of wait()}>`: 1 bytes, alignment: 1 bytes diff --git a/tests/ui/async-await/future-sizes/async-awaiting-fut.rs b/tests/ui/async-await/future-sizes/async-awaiting-fut.rs index a3f0bdc851481..e689c799bb96a 100644 --- a/tests/ui/async-await/future-sizes/async-awaiting-fut.rs +++ b/tests/ui/async-await/future-sizes/async-awaiting-fut.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only //@ compile-flags: -Z print-type-sizes --crate-type lib //@ edition:2021 //@ build-pass diff --git a/tests/ui/async-await/future-sizes/future-as-arg.rs b/tests/ui/async-await/future-sizes/future-as-arg.rs index 317c17b9b3e59..3bbbadc2099df 100644 --- a/tests/ui/async-await/future-sizes/future-as-arg.rs +++ b/tests/ui/async-await/future-sizes/future-as-arg.rs @@ -8,9 +8,10 @@ async fn use_future(fut: impl std::future::Future) { } fn main() { - let actual = std::mem::size_of_val( - &use_future(use_future(use_future(use_future(use_future(test([0; 16]))))))); + let actual = std::mem::size_of_val(&use_future(use_future(use_future(use_future( + use_future(test([0; 16])), + ))))); // Not using an exact number in case it slightly changes over different commits - let expected = 550; - assert!(actual > expected, "expected: >{expected}, actual: {actual}"); + let expected = 21; + assert!(actual >= expected, "expected: >{expected}, actual: {actual}"); } diff --git a/tests/ui/async-await/future-sizes/large-arg.classic.stdout b/tests/ui/async-await/future-sizes/large-arg.classic.stdout new file mode 100644 index 0000000000000..4e2a28c5de727 --- /dev/null +++ b/tests/ui/async-await/future-sizes/large-arg.classic.stdout @@ -0,0 +1,64 @@ +print-type-size type: `{async fn body of test()}`: 4100 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 0 bytes +print-type-size variant `Suspend0`: 4099 bytes +print-type-size local `.__awaitee`: 4099 bytes, type: {async fn body of a<[u8; 1024]>()} +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of a<[u8; 1024]>()}>`: 4099 bytes, alignment: 1 bytes +print-type-size field `.value`: 4099 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of a<[u8; 1024]>()}>`: 4099 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 4099 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 4099 bytes +print-type-size type: `{async fn body of a<[u8; 1024]>()}`: 4099 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size local `.t`: 1024 bytes +print-type-size upvar `.t`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 4098 bytes +print-type-size padding: 1024 bytes +print-type-size local `.__awaitee`: 3074 bytes, alignment: 1 bytes, type: {async fn body of b<[u8; 1024]>()} +print-type-size variant `Returned`: 1024 bytes +print-type-size variant `Panicked`: 1024 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of b<[u8; 1024]>()}>`: 3074 bytes, alignment: 1 bytes +print-type-size field `.value`: 3074 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of b<[u8; 1024]>()}>`: 3074 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 3074 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 3074 bytes +print-type-size type: `{async fn body of b<[u8; 1024]>()}`: 3074 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size local `.t`: 1024 bytes +print-type-size upvar `.t`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 3073 bytes +print-type-size padding: 1024 bytes +print-type-size local `.__awaitee`: 2049 bytes, alignment: 1 bytes, type: {async fn body of c<[u8; 1024]>()} +print-type-size variant `Returned`: 1024 bytes +print-type-size variant `Panicked`: 1024 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of c<[u8; 1024]>()}>`: 2049 bytes, alignment: 1 bytes +print-type-size field `.value`: 2049 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of c<[u8; 1024]>()}>`: 2049 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 2049 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 2049 bytes +print-type-size type: `{async fn body of c<[u8; 1024]>()}`: 2049 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size local `.t`: 1024 bytes +print-type-size upvar `.t`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes +print-type-size variant `Returned`: 1024 bytes +print-type-size variant `Panicked`: 1024 bytes +print-type-size end padding: 1024 bytes +print-type-size type: `std::task::Poll<[u8; 1024]>`: 1025 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Ready`: 1024 bytes +print-type-size field `.0`: 1024 bytes +print-type-size variant `Pending`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1024 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1024 bytes diff --git a/tests/ui/async-await/future-sizes/large-arg.relocate.stdout b/tests/ui/async-await/future-sizes/large-arg.relocate.stdout new file mode 100644 index 0000000000000..8e62f7f1775bf --- /dev/null +++ b/tests/ui/async-await/future-sizes/large-arg.relocate.stdout @@ -0,0 +1,61 @@ +print-type-size type: `{async fn body of test()}`: 1028 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 0 bytes +print-type-size variant `Suspend0`: 1027 bytes +print-type-size local `.__awaitee`: 1027 bytes, type: {async fn body of a<[u8; 1024]>()} +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of a<[u8; 1024]>()}>`: 1027 bytes, alignment: 1 bytes +print-type-size field `.value`: 1027 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of a<[u8; 1024]>()}>`: 1027 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1027 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1027 bytes +print-type-size type: `{async fn body of a<[u8; 1024]>()}`: 1027 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size upvar `.t`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.t`: 1024 bytes +print-type-size variant `Suspend0`: 1026 bytes +print-type-size local `.__awaitee`: 1026 bytes, type: {async fn body of b<[u8; 1024]>()} +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of b<[u8; 1024]>()}>`: 1026 bytes, alignment: 1 bytes +print-type-size field `.value`: 1026 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of b<[u8; 1024]>()}>`: 1026 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1026 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1026 bytes +print-type-size type: `{async fn body of b<[u8; 1024]>()}`: 1026 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size upvar `.t`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.t`: 1024 bytes +print-type-size variant `Suspend0`: 1025 bytes +print-type-size local `.__awaitee`: 1025 bytes, type: {async fn body of c<[u8; 1024]>()} +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of c<[u8; 1024]>()}>`: 1025 bytes, alignment: 1 bytes +print-type-size field `.value`: 1025 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of c<[u8; 1024]>()}>`: 1025 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1025 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1025 bytes +print-type-size type: `std::task::Poll<[u8; 1024]>`: 1025 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Ready`: 1024 bytes +print-type-size field `.0`: 1024 bytes +print-type-size variant `Pending`: 0 bytes +print-type-size type: `{async fn body of c<[u8; 1024]>()}`: 1025 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size upvar `.t`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.t`: 1024 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1024 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1024 bytes diff --git a/tests/ui/async-await/future-sizes/large-arg.rs b/tests/ui/async-await/future-sizes/large-arg.rs index 5fbae22a7714d..547abf1325689 100644 --- a/tests/ui/async-await/future-sizes/large-arg.rs +++ b/tests/ui/async-await/future-sizes/large-arg.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only //@ compile-flags: -Z print-type-sizes --crate-type=lib //@ edition: 2021 //@ build-pass diff --git a/tests/ui/async-await/future-sizes/large-arg.stdout b/tests/ui/async-await/future-sizes/large-arg.stdout deleted file mode 100644 index 67168a3d6ef74..0000000000000 --- a/tests/ui/async-await/future-sizes/large-arg.stdout +++ /dev/null @@ -1,60 +0,0 @@ -print-type-size type: `{async fn body of test()}`: 3076 bytes, alignment: 1 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `Unresumed`: 0 bytes -print-type-size variant `Suspend0`: 3075 bytes -print-type-size local `.__awaitee`: 3075 bytes, type: {async fn body of a<[u8; 1024]>()} -print-type-size variant `Returned`: 0 bytes -print-type-size variant `Panicked`: 0 bytes -print-type-size type: `std::mem::ManuallyDrop<{async fn body of a<[u8; 1024]>()}>`: 3075 bytes, alignment: 1 bytes -print-type-size field `.value`: 3075 bytes -print-type-size type: `std::mem::MaybeUninit<{async fn body of a<[u8; 1024]>()}>`: 3075 bytes, alignment: 1 bytes -print-type-size variant `MaybeUninit`: 3075 bytes -print-type-size field `.uninit`: 0 bytes -print-type-size field `.value`: 3075 bytes -print-type-size type: `{async fn body of a<[u8; 1024]>()}`: 3075 bytes, alignment: 1 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size variant `Suspend0`: 3074 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size local `.__awaitee`: 2050 bytes, type: {async fn body of b<[u8; 1024]>()} -print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size type: `std::mem::ManuallyDrop<{async fn body of b<[u8; 1024]>()}>`: 2050 bytes, alignment: 1 bytes -print-type-size field `.value`: 2050 bytes -print-type-size type: `std::mem::MaybeUninit<{async fn body of b<[u8; 1024]>()}>`: 2050 bytes, alignment: 1 bytes -print-type-size variant `MaybeUninit`: 2050 bytes -print-type-size field `.uninit`: 0 bytes -print-type-size field `.value`: 2050 bytes -print-type-size type: `{async fn body of b<[u8; 1024]>()}`: 2050 bytes, alignment: 1 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size variant `Suspend0`: 2049 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size local `.__awaitee`: 1025 bytes, type: {async fn body of c<[u8; 1024]>()} -print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size type: `std::mem::ManuallyDrop<{async fn body of c<[u8; 1024]>()}>`: 1025 bytes, alignment: 1 bytes -print-type-size field `.value`: 1025 bytes -print-type-size type: `std::mem::MaybeUninit<{async fn body of c<[u8; 1024]>()}>`: 1025 bytes, alignment: 1 bytes -print-type-size variant `MaybeUninit`: 1025 bytes -print-type-size field `.uninit`: 0 bytes -print-type-size field `.value`: 1025 bytes -print-type-size type: `std::task::Poll<[u8; 1024]>`: 1025 bytes, alignment: 1 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `Ready`: 1024 bytes -print-type-size field `.0`: 1024 bytes -print-type-size variant `Pending`: 0 bytes -print-type-size type: `{async fn body of c<[u8; 1024]>()}`: 1025 bytes, alignment: 1 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes diff --git a/tests/ui/async-await/higher-ranked-auto-trait-8.no_assumptions.stderr b/tests/ui/async-await/higher-ranked-auto-trait-8.no_assumptions.stderr index 6208675117b74..39be5f0887739 100644 --- a/tests/ui/async-await/higher-ranked-auto-trait-8.no_assumptions.stderr +++ b/tests/ui/async-await/higher-ranked-auto-trait-8.no_assumptions.stderr @@ -6,5 +6,14 @@ LL | needs_send(use_my_struct(second_struct)); // ERROR | = note: could not prove `impl Future: Send` -error: aborting due to 1 previous error +error: higher-ranked lifetime error + --> $DIR/higher-ranked-auto-trait-8.rs:26:5 + | +LL | needs_send(use_my_struct(second_struct)); // ERROR + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: could not prove `impl Future: Send` + = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no` + +error: aborting due to 2 previous errors diff --git a/tests/ui/async-await/issue-70818.stderr b/tests/ui/async-await/issue-70818.stderr index 07fd20cdd77ea..efc412cef098b 100644 --- a/tests/ui/async-await/issue-70818.stderr +++ b/tests/ui/async-await/issue-70818.stderr @@ -5,10 +5,10 @@ LL | async { (ty, ty1) } | ^^^^^^^^^^^^^^^^^^^ future created by async block is not `Send` | note: captured value is not `Send` - --> $DIR/issue-70818.rs:6:18 + --> $DIR/issue-70818.rs:4:27 | -LL | async { (ty, ty1) } - | ^^^ has type `U` which is not `Send` +LL | fn foo(ty: T, ty1: U) -> impl Future + Send { + | ^^^ has type `U` which is not `Send` note: required by a bound in an opaque type --> $DIR/issue-70818.rs:4:69 | @@ -26,10 +26,10 @@ LL | fn foo(ty: T, ty1: U) -> impl Future + Send { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ future created by async block is not `Send` | note: captured value is not `Send` - --> $DIR/issue-70818.rs:6:18 + --> $DIR/issue-70818.rs:4:27 | -LL | async { (ty, ty1) } - | ^^^ has type `U` which is not `Send` +LL | fn foo(ty: T, ty1: U) -> impl Future + Send { + | ^^^ has type `U` which is not `Send` help: consider restricting type parameter `U` with trait `Send` | LL | fn foo(ty: T, ty1: U) -> impl Future + Send { diff --git a/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs b/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs index f6c9fdd6d6806..63adb5a725172 100644 --- a/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs +++ b/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs @@ -9,7 +9,7 @@ pub async fn async_fn(x: &mut i32) -> &i32 { y } -pub fn async_closure(x: &mut i32) -> impl Future { +pub fn async_closure(x: &mut i32) -> impl Future { (async move || { //~^ ERROR lifetime may not live long enough //~| ERROR temporary value dropped while borrowed @@ -19,7 +19,7 @@ pub fn async_closure(x: &mut i32) -> impl Future { })() } -pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { +pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { (async move || -> &i32 { //~^ ERROR lifetime may not live long enough //~| ERROR temporary value dropped while borrowed @@ -29,7 +29,7 @@ pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future impl Future { +pub fn async_block(x: &mut i32) -> impl Future { async move { let y = &*x; *x += 1; //~ ERROR cannot assign to `*x` because it is borrowed diff --git a/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr b/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr index e1f268116fc54..cb559898c2fc5 100644 --- a/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr +++ b/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr @@ -44,7 +44,7 @@ LL | | })() error[E0716]: temporary value dropped while borrowed --> $DIR/issue-74072-lifetime-name-annotations.rs:13:5 | -LL | pub fn async_closure(x: &mut i32) -> impl Future { +LL | pub fn async_closure(x: &mut i32) -> impl Future { | - let's call the lifetime of this reference `'1` LL | // (async move || { LL | || @@ -93,7 +93,7 @@ LL | | })() error[E0716]: temporary value dropped while borrowed --> $DIR/issue-74072-lifetime-name-annotations.rs:23:5 | -LL | pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { +LL | pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { | - let's call the lifetime of this reference `'1` LL | // (async move || -> &i32 { LL | || diff --git a/tests/ui/async-await/issue-86507.rs b/tests/ui/async-await/issue-86507.rs index 484122a1ddcfd..6dbebc9328677 100644 --- a/tests/ui/async-await/issue-86507.rs +++ b/tests/ui/async-await/issue-86507.rs @@ -12,9 +12,14 @@ trait Foo { impl Foo for () { fn bar<'me, 'async_trait, T: Send>(x: &'me T) + //~^ NOTE captured value is not `Send` + //~| NOTE has type `&T` which is not `Send` -> Pin + Send + 'async_trait>> where 'me:'async_trait { - Box::pin( //~ ERROR future cannot be sent between threads safely + Box::pin( + //~^ ERROR future cannot be sent between threads safely + //~| NOTE future created by async block is not `Send` + //~| NOTE required for the cast from async move { let x = x; } diff --git a/tests/ui/async-await/issue-86507.stderr b/tests/ui/async-await/issue-86507.stderr index c71801dcfc854..32b9f795ee79d 100644 --- a/tests/ui/async-await/issue-86507.stderr +++ b/tests/ui/async-await/issue-86507.stderr @@ -1,19 +1,17 @@ error: future cannot be sent between threads safely - --> $DIR/issue-86507.rs:17:13 + --> $DIR/issue-86507.rs:19:13 | LL | / Box::pin( -LL | | async move { -LL | | let x = x; -LL | | } +... | LL | | ) | |_____________^ future created by async block is not `Send` | note: captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - --> $DIR/issue-86507.rs:19:29 + --> $DIR/issue-86507.rs:14:40 | -LL | let x = x; - | ^ has type `&T` which is not `Send`, because `T` is not `Sync` - = note: required for the cast from `Pin>` to `Pin + Send>>` +LL | fn bar<'me, 'async_trait, T: Send>(x: &'me T) + | ^ has type `&T` which is not `Send`, because `T` is not `Sync` + = note: required for the cast from `Pin>` to `Pin + Send>>` help: consider further restricting type parameter `T` with trait `Sync` | LL | fn bar<'me, 'async_trait, T: Send + std::marker::Sync>(x: &'me T) diff --git a/tests/ui/coroutine/auto-trait-regions.rs b/tests/ui/coroutine/auto-trait-regions.rs index 736555b31bbd1..8b3d1e8eaac58 100644 --- a/tests/ui/coroutine/auto-trait-regions.rs +++ b/tests/ui/coroutine/auto-trait-regions.rs @@ -30,6 +30,7 @@ fn main() { }; assert_foo(generator); //~^ ERROR implementation of `Foo` is not general enough + //~| ERROR implementation of `Foo` is not general enough // Allow impls which matches any lifetime let x = &OnlyFooIfRef(No); diff --git a/tests/ui/coroutine/auto-trait-regions.stderr b/tests/ui/coroutine/auto-trait-regions.stderr index beb689d868d46..09319eb26366c 100644 --- a/tests/ui/coroutine/auto-trait-regions.stderr +++ b/tests/ui/coroutine/auto-trait-regions.stderr @@ -1,5 +1,5 @@ error[E0626]: borrow may still be in use when coroutine yields - --> $DIR/auto-trait-regions.rs:45:19 + --> $DIR/auto-trait-regions.rs:46:19 | LL | let generator = #[coroutine] move || { | ------- within this coroutine @@ -15,7 +15,7 @@ LL | let generator = #[coroutine] static move || { | ++++++ error[E0626]: borrow may still be in use when coroutine yields - --> $DIR/auto-trait-regions.rs:45:30 + --> $DIR/auto-trait-regions.rs:46:30 | LL | let generator = #[coroutine] move || { | ------- within this coroutine @@ -40,7 +40,17 @@ LL | assert_foo(generator); = note: ...but `Foo` is actually implemented for the type `&'static OnlyFooIfStaticRef` error: implementation of `Foo` is not general enough - --> $DIR/auto-trait-regions.rs:51:5 + --> $DIR/auto-trait-regions.rs:31:5 + | +LL | assert_foo(generator); + | ^^^^^^^^^^^^^^^^^^^^^ implementation of `Foo` is not general enough + | + = note: `&'0 OnlyFooIfStaticRef` must implement `Foo`, for any lifetime `'0`... + = note: ...but `Foo` is actually implemented for the type `&'static OnlyFooIfStaticRef` + = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no` + +error: implementation of `Foo` is not general enough + --> $DIR/auto-trait-regions.rs:52:5 | LL | assert_foo(generator); | ^^^^^^^^^^^^^^^^^^^^^ implementation of `Foo` is not general enough @@ -48,6 +58,6 @@ LL | assert_foo(generator); = note: `Foo` would have to be implemented for the type `A<'0, '1>`, for any two lifetimes `'0` and `'1`... = note: ...but `Foo` is actually implemented for the type `A<'_, '2>`, for some specific lifetime `'2` -error: aborting due to 4 previous errors +error: aborting due to 5 previous errors For more information about this error, try `rustc --explain E0626`. diff --git a/tests/ui/coroutine/clone-impl.stderr b/tests/ui/coroutine/clone-impl.classic.stderr similarity index 58% rename from tests/ui/coroutine/clone-impl.stderr rename to tests/ui/coroutine/clone-impl.classic.stderr index f316902a42d08..73513cf8df612 100644 --- a/tests/ui/coroutine/clone-impl.stderr +++ b/tests/ui/coroutine/clone-impl.classic.stderr @@ -1,81 +1,81 @@ -error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:44:5: 44:12}` - --> $DIR/clone-impl.rs:48:5 +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}` + --> $DIR/clone-impl.rs:51:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:44:5: 44:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}` ... LL | check_copy(&gen_clone_0); - | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:44:5: 44:12}`, the trait `Copy` is not implemented for `Vec` + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}`, the trait `Copy` is not implemented for `Vec` | note: captured value does not implement `Copy` - --> $DIR/clone-impl.rs:46:14 + --> $DIR/clone-impl.rs:45:9 | -LL | drop(clonable_0); - | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` +LL | let clonable_0: Vec = Vec::new(); + | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` note: required by a bound in `check_copy` - --> $DIR/clone-impl.rs:92:18 + --> $DIR/clone-impl.rs:95:18 | LL | fn check_copy(_x: &T) {} | ^^^^ required by this bound in `check_copy` -error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:55:5: 55:12}` - --> $DIR/clone-impl.rs:60:5 +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}` + --> $DIR/clone-impl.rs:63:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:55:5: 55:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}` ... LL | check_copy(&gen_clone_1); - | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:55:5: 55:12}`, the trait `Copy` is not implemented for `Vec` + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}`, the trait `Copy` is not implemented for `Vec` | note: coroutine does not implement `Copy` as this value is used across a yield - --> $DIR/clone-impl.rs:57:9 + --> $DIR/clone-impl.rs:60:9 | LL | let v = vec!['a']; | - has type `Vec` which does not implement `Copy` LL | yield; | ^^^^^ yield occurs here, with `v` maybe used later note: required by a bound in `check_copy` - --> $DIR/clone-impl.rs:92:18 + --> $DIR/clone-impl.rs:95:18 | LL | fn check_copy(_x: &T) {} | ^^^^ required by this bound in `check_copy` -error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:68:5: 68:12}` - --> $DIR/clone-impl.rs:74:5 +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}` + --> $DIR/clone-impl.rs:77:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:68:5: 68:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}` ... LL | check_copy(&gen_clone_1); - | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:68:5: 68:12}`, the trait `Copy` is not implemented for `Vec` + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}`, the trait `Copy` is not implemented for `Vec` | note: captured value does not implement `Copy` - --> $DIR/clone-impl.rs:72:14 + --> $DIR/clone-impl.rs:69:9 | -LL | drop(clonable_1); - | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` +LL | let clonable_1: Vec = Vec::new(); + | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` note: required by a bound in `check_copy` - --> $DIR/clone-impl.rs:92:18 + --> $DIR/clone-impl.rs:95:18 | LL | fn check_copy(_x: &T) {} | ^^^^ required by this bound in `check_copy` -error[E0277]: the trait bound `NonClone: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}` - --> $DIR/clone-impl.rs:86:5 +error[E0277]: the trait bound `NonClone: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` + --> $DIR/clone-impl.rs:89:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` ... LL | check_copy(&gen_non_clone); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}`, the trait `Copy` is not implemented for `NonClone` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}`, the trait `Copy` is not implemented for `NonClone` | note: captured value does not implement `Copy` - --> $DIR/clone-impl.rs:84:14 + --> $DIR/clone-impl.rs:83:9 | -LL | drop(non_clonable); - | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Copy` +LL | let non_clonable: NonClone = NonClone; + | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Copy` note: required by a bound in `check_copy` - --> $DIR/clone-impl.rs:92:18 + --> $DIR/clone-impl.rs:95:18 | LL | fn check_copy(_x: &T) {} | ^^^^ required by this bound in `check_copy` @@ -85,22 +85,22 @@ LL + #[derive(Copy)] LL | struct NonClone; | -error[E0277]: the trait bound `NonClone: Clone` is not satisfied in `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}` - --> $DIR/clone-impl.rs:88:5 +error[E0277]: the trait bound `NonClone: Clone` is not satisfied in `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` + --> $DIR/clone-impl.rs:91:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` ... LL | check_clone(&gen_non_clone); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}`, the trait `Clone` is not implemented for `NonClone` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}`, the trait `Clone` is not implemented for `NonClone` | note: captured value does not implement `Clone` - --> $DIR/clone-impl.rs:84:14 + --> $DIR/clone-impl.rs:83:9 | -LL | drop(non_clonable); - | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Clone` +LL | let non_clonable: NonClone = NonClone; + | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Clone` note: required by a bound in `check_clone` - --> $DIR/clone-impl.rs:93:19 + --> $DIR/clone-impl.rs:96:19 | LL | fn check_clone(_x: &T) {} | ^^^^^ required by this bound in `check_clone` diff --git a/tests/ui/coroutine/clone-impl.relocate.stderr b/tests/ui/coroutine/clone-impl.relocate.stderr new file mode 100644 index 0000000000000..05849ac33ff64 --- /dev/null +++ b/tests/ui/coroutine/clone-impl.relocate.stderr @@ -0,0 +1,127 @@ +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}` + --> $DIR/clone-impl.rs:51:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}` +... +LL | check_copy(&gen_clone_0); + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}`, the trait `Copy` is not implemented for `Vec` + | +note: coroutine does not implement `Copy` as this value is used across a yield + --> $DIR/clone-impl.rs:48:9 + | +LL | let clonable_0: Vec = Vec::new(); + | ---------- has type `Vec` which does not implement `Copy` +... +LL | yield; + | ^^^^^ yield occurs here, with `clonable_0` maybe used later +note: required by a bound in `check_copy` + --> $DIR/clone-impl.rs:95:18 + | +LL | fn check_copy(_x: &T) {} + | ^^^^ required by this bound in `check_copy` + +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}` + --> $DIR/clone-impl.rs:63:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}` +... +LL | check_copy(&gen_clone_1); + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}`, the trait `Copy` is not implemented for `Vec` + | +note: coroutine does not implement `Copy` as this value is used across a yield + --> $DIR/clone-impl.rs:60:9 + | +LL | let v = vec!['a']; + | - has type `Vec` which does not implement `Copy` +LL | yield; + | ^^^^^ yield occurs here, with `v` maybe used later +note: required by a bound in `check_copy` + --> $DIR/clone-impl.rs:95:18 + | +LL | fn check_copy(_x: &T) {} + | ^^^^ required by this bound in `check_copy` + +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}` + --> $DIR/clone-impl.rs:77:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}` +... +LL | check_copy(&gen_clone_1); + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}`, the trait `Copy` is not implemented for `Vec` + | +note: coroutine does not implement `Copy` as this value is used across a yield + --> $DIR/clone-impl.rs:72:9 + | +LL | let clonable_1: Vec = Vec::new(); + | ---------- has type `Vec` which does not implement `Copy` +... +LL | yield; + | ^^^^^ yield occurs here, with `clonable_1` maybe used later +note: required by a bound in `check_copy` + --> $DIR/clone-impl.rs:95:18 + | +LL | fn check_copy(_x: &T) {} + | ^^^^ required by this bound in `check_copy` + +error[E0277]: the trait bound `NonClone: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` + --> $DIR/clone-impl.rs:89:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` +... +LL | check_copy(&gen_non_clone); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}`, the trait `Copy` is not implemented for `NonClone` + | +note: coroutine does not implement `Copy` as this value is used across a yield + --> $DIR/clone-impl.rs:86:9 + | +LL | let non_clonable: NonClone = NonClone; + | ------------ has type `NonClone` which does not implement `Copy` +... +LL | yield; + | ^^^^^ yield occurs here, with `non_clonable` maybe used later +note: required by a bound in `check_copy` + --> $DIR/clone-impl.rs:95:18 + | +LL | fn check_copy(_x: &T) {} + | ^^^^ required by this bound in `check_copy` +help: consider annotating `NonClone` with `#[derive(Copy)]` + | +LL + #[derive(Copy)] +LL | struct NonClone; + | + +error[E0277]: the trait bound `NonClone: Clone` is not satisfied in `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` + --> $DIR/clone-impl.rs:91:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` +... +LL | check_clone(&gen_non_clone); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}`, the trait `Clone` is not implemented for `NonClone` + | +note: coroutine does not implement `Clone` as this value is used across a yield + --> $DIR/clone-impl.rs:86:9 + | +LL | let non_clonable: NonClone = NonClone; + | ------------ has type `NonClone` which does not implement `Clone` +... +LL | yield; + | ^^^^^ yield occurs here, with `non_clonable` maybe used later +note: required by a bound in `check_clone` + --> $DIR/clone-impl.rs:96:19 + | +LL | fn check_clone(_x: &T) {} + | ^^^^^ required by this bound in `check_clone` +help: consider annotating `NonClone` with `#[derive(Clone)]` + | +LL + #[derive(Clone)] +LL | struct NonClone; + | + +error: aborting due to 5 previous errors + +For more information about this error, try `rustc --explain E0277`. diff --git a/tests/ui/coroutine/clone-impl.rs b/tests/ui/coroutine/clone-impl.rs index 9e04e256fc11e..17723b56449b3 100644 --- a/tests/ui/coroutine/clone-impl.rs +++ b/tests/ui/coroutine/clone-impl.rs @@ -1,7 +1,10 @@ // gate-test-coroutine_clone // Verifies that non-static coroutines can be cloned/copied if all their upvars and locals held // across awaits can be cloned/copied. -//@compile-flags: --diagnostic-width=300 +//@ revisions: classic relocate +//@ compile-flags: --diagnostic-width=300 +//@ [classic] compile-flags: -Zpack-coroutine-layout=no +//@ [relocate] compile-flags: -Zpack-coroutine-layout=captures-only #![feature(coroutines, coroutine_clone, stmt_expr_attributes)] diff --git a/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs b/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs index 0f9c56786da06..c862ba658aa01 100644 --- a/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs +++ b/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs @@ -1,16 +1,13 @@ #![feature(coroutines, stmt_expr_attributes)] fn foo(x: &i32) { - // In this case, a reference to `b` escapes the coroutine, but not - // because of a yield. We see that there is no yield in the scope of - // `b` and give the more generic error message. let mut a = &3; - let mut b = #[coroutine] + let b = #[coroutine] move || { yield (); let b = 5; a = &b; - //~^ ERROR borrowed data escapes outside of coroutine + //~^ ERROR: borrowed data escapes outside of coroutine }; } diff --git a/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr b/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr index 6fa7082c0b8b7..41b118d9f8f3a 100644 --- a/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr +++ b/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr @@ -1,5 +1,5 @@ error[E0521]: borrowed data escapes outside of coroutine - --> $DIR/ref-escapes-but-not-over-yield.rs:12:9 + --> $DIR/ref-escapes-but-not-over-yield.rs:9:9 | LL | let mut a = &3; | ----- `a` declared here, outside of the coroutine body diff --git a/tests/ui/coroutine/ref-upvar-not-send.rs b/tests/ui/coroutine/ref-upvar-not-send.rs index 89bb5e5495f45..a843e9ba29a64 100644 --- a/tests/ui/coroutine/ref-upvar-not-send.rs +++ b/tests/ui/coroutine/ref-upvar-not-send.rs @@ -11,21 +11,27 @@ fn assert_send(_: T) {} fn main() { let x: &*mut () = &std::ptr::null_mut(); + //~^ NOTE has type `&*mut ()` which is not `Send` + //~| NOTE captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` let y: &mut *mut () = &mut std::ptr::null_mut(); - assert_send(#[coroutine] move || { + //~^ NOTE has type `&mut *mut ()` which is not `Send` + //~| NOTE captured value is not `Send` because `&mut` references cannot be sent unless their referent is `Send` + assert_send( //~^ ERROR coroutine cannot be sent between threads safely //~| NOTE coroutine is not `Send` - yield; - let _x = x; - }); - //~^^ NOTE captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - //~| NOTE has type `&*mut ()` which is not `Send`, because `*mut ()` is not `Sync` - assert_send(#[coroutine] move || { + #[coroutine] + move || { + yield; + let _x = x; + }, + ); + assert_send( //~^ ERROR coroutine cannot be sent between threads safely //~| NOTE coroutine is not `Send` - yield; - let _y = y; - }); - //~^^ NOTE captured value is not `Send` because `&mut` references cannot be sent unless their referent is `Send` - //~| NOTE has type `&mut *mut ()` which is not `Send`, because `*mut ()` is not `Send` + #[coroutine] + move || { + yield; + let _y = y; + }, + ); } diff --git a/tests/ui/coroutine/ref-upvar-not-send.stderr b/tests/ui/coroutine/ref-upvar-not-send.stderr index 3a5e8ec4dab0c..1197583c0cc08 100644 --- a/tests/ui/coroutine/ref-upvar-not-send.stderr +++ b/tests/ui/coroutine/ref-upvar-not-send.stderr @@ -1,20 +1,21 @@ error: coroutine cannot be sent between threads safely - --> $DIR/ref-upvar-not-send.rs:15:5 + --> $DIR/ref-upvar-not-send.rs:19:5 | -LL | / assert_send(#[coroutine] move || { +LL | / assert_send( LL | | LL | | -LL | | yield; -LL | | let _x = x; -LL | | }); - | |______^ coroutine is not `Send` +LL | | #[coroutine] +... | +LL | | }, +LL | | ); + | |_____^ coroutine is not `Send` | = help: the trait `Sync` is not implemented for `*mut ()` note: captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - --> $DIR/ref-upvar-not-send.rs:19:18 + --> $DIR/ref-upvar-not-send.rs:13:9 | -LL | let _x = x; - | ^ has type `&*mut ()` which is not `Send`, because `*mut ()` is not `Sync` +LL | let x: &*mut () = &std::ptr::null_mut(); + | ^ has type `&*mut ()` which is not `Send`, because `*mut ()` is not `Sync` note: required by a bound in `assert_send` --> $DIR/ref-upvar-not-send.rs:6:19 | @@ -22,22 +23,23 @@ LL | fn assert_send(_: T) {} | ^^^^ required by this bound in `assert_send` error: coroutine cannot be sent between threads safely - --> $DIR/ref-upvar-not-send.rs:23:5 + --> $DIR/ref-upvar-not-send.rs:28:5 | -LL | / assert_send(#[coroutine] move || { +LL | / assert_send( LL | | LL | | -LL | | yield; -LL | | let _y = y; -LL | | }); - | |______^ coroutine is not `Send` +LL | | #[coroutine] +... | +LL | | }, +LL | | ); + | |_____^ coroutine is not `Send` | - = help: within `{coroutine@$DIR/ref-upvar-not-send.rs:23:30: 23:37}`, the trait `Send` is not implemented for `*mut ()` + = help: within `{coroutine@$DIR/ref-upvar-not-send.rs:32:9: 32:16}`, the trait `Send` is not implemented for `*mut ()` note: captured value is not `Send` because `&mut` references cannot be sent unless their referent is `Send` - --> $DIR/ref-upvar-not-send.rs:27:18 + --> $DIR/ref-upvar-not-send.rs:16:9 | -LL | let _y = y; - | ^ has type `&mut *mut ()` which is not `Send`, because `*mut ()` is not `Send` +LL | let y: &mut *mut () = &mut std::ptr::null_mut(); + | ^ has type `&mut *mut ()` which is not `Send`, because `*mut ()` is not `Send` note: required by a bound in `assert_send` --> $DIR/ref-upvar-not-send.rs:6:19 | diff --git a/tests/ui/coroutine/unsized-capture-across-yield.rs b/tests/ui/coroutine/unsized-capture-across-yield.rs index ee27ea064ec23..60c8f69bdb8f0 100644 --- a/tests/ui/coroutine/unsized-capture-across-yield.rs +++ b/tests/ui/coroutine/unsized-capture-across-yield.rs @@ -4,7 +4,8 @@ use std::ops::Coroutine; fn capture() -> impl Coroutine { - let b: [u8] = *(Box::new([]) as Box<[u8]>); //~ERROR he size for values of type `[u8]` cannot be known at compilation time + let b: [u8] = *(Box::new([]) as Box<[u8]>); + //~^ ERROR the size for values of type `[u8]` cannot be known at compilation time #[coroutine] move || { println!("{:?}", &b); diff --git a/tests/ui/lint/large_assignments/large_future.attribute.stderr b/tests/ui/lint/large_assignments/large_future.attribute.stderr index 734b7ff7ba22f..51e27c85cd258 100644 --- a/tests/ui/lint/large_assignments/large_future.attribute.stderr +++ b/tests/ui/lint/large_assignments/large_future.attribute.stderr @@ -1,5 +1,5 @@ -error: moving 10024 bytes - --> $DIR/large_future.rs:19:14 +error: moving 10040 bytes + --> $DIR/large_future.rs:21:14 | LL | let z = (x, 42); | ^ value moved from here @@ -11,8 +11,8 @@ note: the lint level is defined here LL | #![deny(large_assignments)] | ^^^^^^^^^^^^^^^^^ -error: moving 10024 bytes - --> $DIR/large_future.rs:20:13 +error: moving 10040 bytes + --> $DIR/large_future.rs:22:13 | LL | let a = z.0; | ^^^ value moved from here diff --git a/tests/ui/lint/large_assignments/large_future.option-relocate.stderr b/tests/ui/lint/large_assignments/large_future.option-relocate.stderr new file mode 100644 index 0000000000000..3c3a1fb3e5fb8 --- /dev/null +++ b/tests/ui/lint/large_assignments/large_future.option-relocate.stderr @@ -0,0 +1,23 @@ +error: moving 10024 bytes + --> $DIR/large_future.rs:21:14 + | +LL | let z = (x, 42); + | ^ value moved from here + | + = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]` +note: the lint level is defined here + --> $DIR/large_future.rs:1:9 + | +LL | #![deny(large_assignments)] + | ^^^^^^^^^^^^^^^^^ + +error: moving 10024 bytes + --> $DIR/large_future.rs:22:13 + | +LL | let a = z.0; + | ^^^ value moved from here + | + = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]` + +error: aborting due to 2 previous errors + diff --git a/tests/ui/lint/large_assignments/large_future.option.stderr b/tests/ui/lint/large_assignments/large_future.option.stderr index 734b7ff7ba22f..51e27c85cd258 100644 --- a/tests/ui/lint/large_assignments/large_future.option.stderr +++ b/tests/ui/lint/large_assignments/large_future.option.stderr @@ -1,5 +1,5 @@ -error: moving 10024 bytes - --> $DIR/large_future.rs:19:14 +error: moving 10040 bytes + --> $DIR/large_future.rs:21:14 | LL | let z = (x, 42); | ^ value moved from here @@ -11,8 +11,8 @@ note: the lint level is defined here LL | #![deny(large_assignments)] | ^^^^^^^^^^^^^^^^^ -error: moving 10024 bytes - --> $DIR/large_future.rs:20:13 +error: moving 10040 bytes + --> $DIR/large_future.rs:22:13 | LL | let a = z.0; | ^^^ value moved from here diff --git a/tests/ui/lint/large_assignments/large_future.rs b/tests/ui/lint/large_assignments/large_future.rs index 28c358bdbf086..43ae254663be9 100644 --- a/tests/ui/lint/large_assignments/large_future.rs +++ b/tests/ui/lint/large_assignments/large_future.rs @@ -3,8 +3,10 @@ #![cfg_attr(attribute, move_size_limit = "1000")] //@ build-fail //@ only-64bit -//@ revisions: attribute option -//@ [option]compile-flags: -Zmove-size-limit=1000 +//@ revisions: attribute option option-relocate +//@ compile-flags: -Zpack-coroutine-layout=no +//@ [option] compile-flags: -Zmove-size-limit=1000 +//@ [option-relocate] compile-flags: -Zmove-size-limit=1000 -Zpack-coroutine-layout=captures-only //@ edition:2018 //@ compile-flags: -Zmir-opt-level=0 diff --git a/tests/ui/mir/lint/storage-live.stderr b/tests/ui/mir/lint/storage-live.stderr index 50df9ae061fcc..fabf7f33745f0 100644 --- a/tests/ui/mir/lint/storage-live.stderr +++ b/tests/ui/mir/lint/storage-live.stderr @@ -1,4 +1,4 @@ -error: internal compiler error: broken MIR in Item(DefId(0:8 ~ storage_live[HASH]::multiple_storage)) (after pass CheckForceInline) at bb0[1]: +error: internal compiler error: broken MIR in Item(DefId(0:8 ~ storage_live[HASH]::multiple_storage)) (after pass CheckForceInline 1-1-000) at bb0[1]: StorageLive(_1) which already has storage here --> $DIR/storage-live.rs:21:13 | diff --git a/tests/ui/print_type_sizes/async.stdout b/tests/ui/print_type_sizes/async.classic.stdout similarity index 85% rename from tests/ui/print_type_sizes/async.stdout rename to tests/ui/print_type_sizes/async.classic.stdout index 83a6962e4cd13..c85b9f0046e51 100644 --- a/tests/ui/print_type_sizes/async.stdout +++ b/tests/ui/print_type_sizes/async.classic.stdout @@ -1,15 +1,14 @@ print-type-size type: `{async fn body of test()}`: 16386 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 8192 bytes -print-type-size upvar `.arg`: 8192 bytes +print-type-size local `.arg`: 8192 bytes +print-type-size upvar `.arg`: 8192 bytes, offset: 1 bytes, alignment: 1 bytes print-type-size variant `Suspend0`: 16385 bytes -print-type-size upvar `.arg`: 8192 bytes +print-type-size padding: 8192 bytes +print-type-size local `.__awaitee`: 1 bytes, alignment: 1 bytes, type: {async fn body of wait()} print-type-size local `.arg`: 8192 bytes -print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} print-type-size variant `Returned`: 8192 bytes -print-type-size upvar `.arg`: 8192 bytes print-type-size variant `Panicked`: 8192 bytes -print-type-size upvar `.arg`: 8192 bytes print-type-size type: `std::mem::ManuallyDrop<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes print-type-size field `.value`: 8192 bytes print-type-size type: `std::mem::MaybeUninit<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes diff --git a/tests/ui/print_type_sizes/async.relocate.stdout b/tests/ui/print_type_sizes/async.relocate.stdout new file mode 100644 index 0000000000000..6bc30dc71154d --- /dev/null +++ b/tests/ui/print_type_sizes/async.relocate.stdout @@ -0,0 +1,32 @@ +print-type-size type: `{async fn body of test()}`: 8194 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 8192 bytes +print-type-size upvar `.arg`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.arg`: 8192 bytes +print-type-size variant `Suspend0`: 8193 bytes +print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} +print-type-size local `.arg`: 8192 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size field `.value`: 8192 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 8192 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 8192 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of wait()}>`: 1 bytes, alignment: 1 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of wait()}>`: 1 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::task::Poll<()>`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Ready`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size variant `Pending`: 0 bytes +print-type-size type: `{async fn body of wait()}`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 0 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes diff --git a/tests/ui/print_type_sizes/async.rs b/tests/ui/print_type_sizes/async.rs index 805bccbcf6381..8155ae823dca3 100644 --- a/tests/ui/print_type_sizes/async.rs +++ b/tests/ui/print_type_sizes/async.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only //@ compile-flags: -Z print-type-sizes --crate-type lib //@ edition:2021 //@ build-pass diff --git a/tests/ui/print_type_sizes/coroutine.classic.stdout b/tests/ui/print_type_sizes/coroutine.classic.stdout new file mode 100644 index 0000000000000..ee097da2fa240 --- /dev/null +++ b/tests/ui/print_type_sizes/coroutine.classic.stdout @@ -0,0 +1,15 @@ +print-type-size type: `{coroutine@$DIR/coroutine.rs:14:5: 14:14}`: 16385 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 8192 bytes +print-type-size local `.array`: 8192 bytes +print-type-size upvar `.array`: 8192 bytes, offset: 1 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 8192 bytes +print-type-size variant `Returned`: 8192 bytes +print-type-size variant `Panicked`: 8192 bytes +print-type-size end padding: 8192 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size field `.value`: 8192 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 8192 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 8192 bytes diff --git a/tests/ui/print_type_sizes/coroutine.relocate.stdout b/tests/ui/print_type_sizes/coroutine.relocate.stdout new file mode 100644 index 0000000000000..79f5d093dc3ad --- /dev/null +++ b/tests/ui/print_type_sizes/coroutine.relocate.stdout @@ -0,0 +1,15 @@ +print-type-size type: `{coroutine@$DIR/coroutine.rs:14:5: 14:14}`: 8193 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 8192 bytes +print-type-size upvar `.array`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.array`: 8192 bytes +print-type-size variant `Suspend0`: 8192 bytes +print-type-size local `.array`: 8192 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size field `.value`: 8192 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 8192 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 8192 bytes diff --git a/tests/ui/print_type_sizes/coroutine.rs b/tests/ui/print_type_sizes/coroutine.rs index 1533578878944..407d0873c015d 100644 --- a/tests/ui/print_type_sizes/coroutine.rs +++ b/tests/ui/print_type_sizes/coroutine.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only //@ compile-flags: -Z print-type-sizes --crate-type=lib //@ build-pass //@ ignore-pass diff --git a/tests/ui/print_type_sizes/coroutine.stdout b/tests/ui/print_type_sizes/coroutine.stdout deleted file mode 100644 index 339bbddfc2a93..0000000000000 --- a/tests/ui/print_type_sizes/coroutine.stdout +++ /dev/null @@ -1,10 +0,0 @@ -print-type-size type: `{coroutine@$DIR/coroutine.rs:11:5: 11:14}`: 8193 bytes, alignment: 1 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `Unresumed`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes -print-type-size variant `Suspend0`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes -print-type-size variant `Returned`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes -print-type-size variant `Panicked`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes diff --git a/tests/ui/rustc_public-ir-print/async-closure.stdout b/tests/ui/rustc_public-ir-print/async-closure.classic.stdout similarity index 65% rename from tests/ui/rustc_public-ir-print/async-closure.stdout rename to tests/ui/rustc_public-ir-print/async-closure.classic.stdout index 73e9b8fc097ab..9204905d51eda 100644 --- a/tests/ui/rustc_public-ir-print/async-closure.stdout +++ b/tests/ui/rustc_public-ir-print/async-closure.classic.stdout @@ -3,7 +3,7 @@ fn foo() -> () { let mut _0: (); let _1: i32; - let _2: {async closure@$DIR/async-closure.rs:9:13: 9:21}; + let _2: {async closure@$DIR/async-closure.rs:12:13: 12:21}; let mut _3: &i32; debug y => _1; debug x => _2; @@ -13,7 +13,7 @@ fn foo() -> () { StorageLive(_2); StorageLive(_3); _3 = &_1; - _2 = {coroutine-closure@$DIR/async-closure.rs:9:13: 9:21}(move _3); + _2 = {coroutine-closure@$DIR/async-closure.rs:12:13: 12:21}(move _3); StorageDead(_3); _0 = (); StorageDead(_2); @@ -21,8 +21,8 @@ fn foo() -> () { return; } } -fn foo::{closure#0}(_1: &{async closure@$DIR/async-closure.rs:9:13: 9:21}) -> {async closure body@$DIR/async-closure.rs:9:22: 11:6} { - let mut _0: {async closure body@$DIR/async-closure.rs:9:22: 11:6}; +fn foo::{closure#0}(_1: &{async closure@$DIR/async-closure.rs:12:13: 12:21}) -> {async closure body@$DIR/async-closure.rs:12:22: 14:6} { + let mut _0: {async closure body@$DIR/async-closure.rs:12:22: 14:6}; let mut _2: &i32; let mut _3: &i32; debug y => (*((*_1).0: &i32)); @@ -30,39 +30,39 @@ fn foo::{closure#0}(_1: &{async closure@$DIR/async-closure.rs:9:13: 9:21}) -> {a StorageLive(_2); _3 = CopyForDeref(((*_1).0: &i32)); _2 = &(*_3); - _0 = {coroutine@$DIR/async-closure.rs:9:22: 11:6}(move _2); + _0 = {coroutine@$DIR/async-closure.rs:12:22: 14:6}(move _2); StorageDead(_2); return; } } -fn foo::{closure#0}::{closure#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}>, _2: &mut Context<'_>) -> Poll<()> { +fn foo::{closure#0}::{closure#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}>, _2: &mut Context<'_>) -> Poll<()> { let mut _0: Poll<()>; let _3: i32; let mut _4: &i32; let mut _5: (); let mut _6: &mut Context<'_>; let mut _7: u32; - let mut _8: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}; - let mut _9: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}; - let mut _10: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}; + let mut _8: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _9: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _10: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; debug _task_context => _6; - debug y => (*((*(_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6})).0: &i32)); + debug y => (*(((*(_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})) as variant#0).0: &i32)); debug y => _3; bb0: { - _8 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6})); + _8 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); _7 = discriminant((*_8)); switchInt(move _7) -> [0: bb1, 1: bb2, otherwise: bb3]; } bb1: { _6 = move _2; StorageLive(_3); - _9 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6})); - _4 = CopyForDeref(((*_9).0: &i32)); + _9 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + _4 = CopyForDeref((((*_9) as variant#0).0: &i32)); _3 = (*_4); _5 = (); StorageDead(_3); _0 = std::task::Poll::Ready(move _5); - _10 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6})); + _10 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); discriminant((*_10)) = 1; return; } @@ -73,34 +73,34 @@ fn foo::{closure#0}::{closure#0}(_1: Pin<&mut {async closure body@$DIR/async-clo unreachable; } } -fn foo::{closure#0}::{synthetic#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}>, _2: &mut Context<'_>) -> Poll<()> { +fn foo::{closure#0}::{synthetic#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}>, _2: &mut Context<'_>) -> Poll<()> { let mut _0: Poll<()>; let _3: i32; let mut _4: &i32; let mut _5: (); let mut _6: &mut Context<'_>; let mut _7: u32; - let mut _8: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}; - let mut _9: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}; - let mut _10: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}; + let mut _8: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _9: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _10: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; debug _task_context => _6; - debug y => (*((*(_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6})).0: &i32)); + debug y => (*(((*(_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})) as variant#0).0: &i32)); debug y => _3; bb0: { - _8 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6})); + _8 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); _7 = discriminant((*_8)); switchInt(move _7) -> [0: bb1, 1: bb2, otherwise: bb3]; } bb1: { _6 = move _2; StorageLive(_3); - _9 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6})); - _4 = CopyForDeref(((*_9).0: &i32)); + _9 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + _4 = CopyForDeref((((*_9) as variant#0).0: &i32)); _3 = (*_4); _5 = (); StorageDead(_3); _0 = std::task::Poll::Ready(move _5); - _10 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6})); + _10 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); discriminant((*_10)) = 1; return; } diff --git a/tests/ui/rustc_public-ir-print/async-closure.relocate.stdout b/tests/ui/rustc_public-ir-print/async-closure.relocate.stdout new file mode 100644 index 0000000000000..d22762c494891 --- /dev/null +++ b/tests/ui/rustc_public-ir-print/async-closure.relocate.stdout @@ -0,0 +1,127 @@ +// WARNING: This is highly experimental output it's intended for rustc_public developers only. +// If you find a bug or want to improve the output open a issue at https://github.com/rust-lang/project-stable-mir. +fn foo() -> () { + let mut _0: (); + let _1: i32; + let _2: {async closure@$DIR/async-closure.rs:12:13: 12:21}; + let mut _3: &i32; + debug y => _1; + debug x => _2; + bb0: { + StorageLive(_1); + _1 = 0_i32; + StorageLive(_2); + StorageLive(_3); + _3 = &_1; + _2 = {coroutine-closure@$DIR/async-closure.rs:12:13: 12:21}(move _3); + StorageDead(_3); + _0 = (); + StorageDead(_2); + StorageDead(_1); + return; + } +} +fn foo::{closure#0}(_1: &{async closure@$DIR/async-closure.rs:12:13: 12:21}) -> {async closure body@$DIR/async-closure.rs:12:22: 14:6} { + let mut _0: {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _2: &i32; + let mut _3: &i32; + debug y => (*((*_1).0: &i32)); + bb0: { + StorageLive(_2); + _3 = CopyForDeref(((*_1).0: &i32)); + _2 = &(*_3); + _0 = {coroutine@$DIR/async-closure.rs:12:22: 14:6}(move _2); + StorageDead(_2); + return; + } +} +fn foo::{closure#0}::{closure#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}>, _2: &mut Context<'_>) -> Poll<()> { + let mut _0: Poll<()>; + let _3: i32; + let _4: &i32; + let _5: &i32; + let mut _6: (); + let mut _7: &mut Context<'_>; + let mut _8: u32; + let mut _9: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _10: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _11: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + debug _task_context => _7; + debug y => (*_4); + debug y => _3; + debug y => _4; + bb0: { + _9 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + _8 = discriminant((*_9)); + switchInt(move _8) -> [0: bb1, 1: bb2, otherwise: bb3]; + } + bb1: { + _7 = move _2; + StorageLive(_4); + StorageLive(_5); + _10 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + _5 = move (((*_10) as variant#0).0: &i32); + _4 = move _5; + StorageDead(_5); + StorageLive(_3); + _3 = (*_4); + _6 = (); + StorageDead(_3); + StorageDead(_4); + _0 = std::task::Poll::Ready(move _6); + _11 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + discriminant((*_11)) = 1; + return; + } + bb2: { + assert(false, `async fn` resumed after completion) -> [success: bb2, unwind unreachable]; + } + bb3: { + unreachable; + } +} +fn foo::{closure#0}::{synthetic#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}>, _2: &mut Context<'_>) -> Poll<()> { + let mut _0: Poll<()>; + let _3: i32; + let _4: &i32; + let _5: &i32; + let mut _6: (); + let mut _7: &mut Context<'_>; + let mut _8: u32; + let mut _9: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _10: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _11: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + debug _task_context => _7; + debug y => (*_4); + debug y => _3; + debug y => _4; + bb0: { + _9 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + _8 = discriminant((*_9)); + switchInt(move _8) -> [0: bb1, 1: bb2, otherwise: bb3]; + } + bb1: { + _7 = move _2; + StorageLive(_4); + StorageLive(_5); + _10 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + _5 = move (((*_10) as variant#0).0: &i32); + _4 = move _5; + StorageDead(_5); + StorageLive(_3); + _3 = (*_4); + _6 = (); + StorageDead(_3); + StorageDead(_4); + _0 = std::task::Poll::Ready(move _6); + _11 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + discriminant((*_11)) = 1; + return; + } + bb2: { + assert(false, `async fn` resumed after completion) -> [success: bb2, unwind unreachable]; + } + bb3: { + unreachable; + } +} diff --git a/tests/ui/rustc_public-ir-print/async-closure.rs b/tests/ui/rustc_public-ir-print/async-closure.rs index 80f96e09cfc78..c492ea04c6908 100644 --- a/tests/ui/rustc_public-ir-print/async-closure.rs +++ b/tests/ui/rustc_public-ir-print/async-closure.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only //@ compile-flags: -Z unpretty=stable-mir --crate-type lib -C panic=abort -Zmir-opt-level=0 //@ check-pass //@ only-x86_64 From c2f588233b44f9e06a3cd9d6ac767e5dadce4e29 Mon Sep 17 00:00:00 2001 From: Xiangfei Ding Date: Wed, 16 Jul 2025 15:59:05 +0000 Subject: [PATCH 2/4] mir-opt; docstr --- .../src/coroutine/relocate_upvars.rs | 42 +++++- tests/mir-opt/coroutine-relocate-upvars.rs | 16 ++ ...re#0}.RelocateUpvars.after.panic-abort.mir | 138 ++++++++++++++++++ ...e#0}.RelocateUpvars.after.panic-unwind.mir | 138 ++++++++++++++++++ ...e#0}.RelocateUpvars.before.panic-abort.mir | 77 ++++++++++ ...#0}.RelocateUpvars.before.panic-unwind.mir | 77 ++++++++++ 6 files changed, 482 insertions(+), 6 deletions(-) create mode 100644 tests/mir-opt/coroutine-relocate-upvars.rs create mode 100644 tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-abort.mir create mode 100644 tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-unwind.mir create mode 100644 tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-abort.mir create mode 100644 tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-unwind.mir diff --git a/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs b/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs index a9b0e8f382858..4aed2d933125a 100644 --- a/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs +++ b/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs @@ -1,27 +1,57 @@ -//! MIR rewrite pass to promote upvars into native locals in the coroutine body +//! MIR rewrite pass to relocate upvars into native locals in the coroutine body //! //! # Summary +//! The current contract of coroutine upvars is as follows. +//! Coroutines are constructed, initially in state UNRESUMED, by copying or moving +//! captures into the `struct`-fields, which are also called prefix fields, +//! taking necessary references as per capture specification. +//! +//! ```text +//! Low address High address +//! ┌─────────┬─────────┬─────┬─────────────────────┬───────────────────────────────────────────────────────┬──────────────┐ +//! │ │ │ │ │ │ │ +//! │ Upvar 1 │ Upvar 2 │ ... │ Coroutine State Tag │ Ineligibles, aka. saved locals alive across 2+ states │ Other states │ +//! │ │ │ │ │ │ │ +//! └─────────┴─────────┴─────┴─────────────────────┴───────────────────────────────────────────────────────┴──────────────┘ +//! ``` +//! +//! In case some upvars are large and short-lived, the classic layout scheme can be wasteful. +//! One way to reduce the memory footprint is to +//! //! This pass performs the following transformations. //! 1. It generates a fresh batch of locals for each captured upvars. //! //! For each upvar, whether used or not, a fresh local is created with the same type. +//! The types respect the nature of the captures, being by-ref, by-ref-mut or by-value. +//! This is reflected in the results in the upvar analysis conducted in the HIR type-checking phase. //! //! 2. It replaces the places pointing into those upvars with places pointing into those locals instead //! //! Each place that starts with access into the coroutine structure `_1` is replaced with the fresh local as -//! the base. For instance, `(_1.4 as Some).0` is rewritten into `(_34 as Some).0` when `_34` is the fresh local +//! the base. +//! For instance, `(_1.4 as Some).0` is rewritten into `(_34 as Some).0` when `_34` is the fresh local //! corresponding to the captured upvar stored in `_1.4`. //! +//! This phase assumes that the initial built MIR respects the nature of captures. +//! For instance, if the upvar `_1.4` is instead a by-ref-mut capture of a value of type `T`, +//! this phase assumes that all access correctly built as operating on the place `(*_1.4)`. +//! Based on the assumption, this phase replaces `_1.4` with a fresh local `_34: &mut T` and +//! the correctness is still upheld. +//! //! 3. It assembles an prologue to replace the current entry block. //! //! This prologue block transfers every captured upvar into its corresponding fresh local, *via scratch locals*. //! The upvars are first completely moved into the scratch locals in batch, and then moved into the destination //! locals in batch. //! The reason is that it is possible that coroutine layout may change and the source memory location of -//! an upvar may not necessarily be mapped exactly to the same place as in the `Unresumed` state. -//! While coroutine layout ensures that the same saved local has stable offsets throughout its lifetime, -//! technically the upvar in `Unresumed` state and their fresh locals are different saved locals. -//! This scratch locals re-estabilish safety so that the correct data permutation can take place. +//! an upvar may not necessarily be mapped exactly to the same place as in the `UNRESUMED` state. +//! This is very possible, because the coroutine layout scheme at this moment remains opaque, +//! other than the contract that a saved local has a stable internal offset throughout its liveness span. +//! +//! While the current coroutine layout ensures that the same saved local has stable offsets throughout its lifetime, +//! technically the upvar in `UNRESUMED` state and their fresh locals are different saved locals. +//! This scratch locals re-establish safety so that the correct data permutation can take place, +//! when a future coroutine layout calculator sees the permutation fit. use std::borrow::Cow; diff --git a/tests/mir-opt/coroutine-relocate-upvars.rs b/tests/mir-opt/coroutine-relocate-upvars.rs new file mode 100644 index 0000000000000..35a86f6326432 --- /dev/null +++ b/tests/mir-opt/coroutine-relocate-upvars.rs @@ -0,0 +1,16 @@ +// skip-filecheck +//@ compile-flags: -Zpack-coroutine-layout=captures-only +#![feature(coroutines, coroutine_trait, stmt_expr_attributes)] + +// EMIT_MIR_FOR_EACH_PANIC_STRATEGY + +// EMIT_MIR coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.mir +// EMIT_MIR coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.mir +fn main() { + let mut x = String::new(); + let gen_ = #[coroutine] + || { + x = String::new(); + yield; + }; +} \ No newline at end of file diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-abort.mir b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-abort.mir new file mode 100644 index 0000000000000..655f0c8f47891 --- /dev/null +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-abort.mir @@ -0,0 +1,138 @@ +// MIR for `main::{closure#0}` after RelocateUpvars + +fn main::{closure#0}(_1: {coroutine@$DIR/coroutine-relocate-upvars.rs:12:5: 12:7}, _2: ()) -> () +yields () + { + debug x => (*_6); + debug x => _6; + let mut _0: (); + let mut _3: std::string::String; + let _4: (); + let mut _5: (); + let mut _6: &mut std::string::String; + let _7: &mut std::string::String; + + bb0: { + StorageLive(_6); + StorageLive(_7); + _7 = move (_1.0: &mut std::string::String); + _6 = move _7; + StorageDead(_7); + goto -> bb24; + } + + bb1: { + drop((*_6)) -> [return: bb2, unwind: bb3]; + } + + bb2: { + (*_6) = move _3; + drop(_3) -> [return: bb4, unwind: bb11, drop: bb8]; + } + + bb3 (cleanup): { + (*_6) = move _3; + drop(_3) -> [return: bb11, unwind terminate(cleanup)]; + } + + bb4: { + StorageDead(_3); + StorageLive(_4); + StorageLive(_5); + _5 = (); + _4 = yield(move _5) -> [resume: bb5, drop: bb7]; + } + + bb5: { + StorageDead(_5); + StorageDead(_4); + _0 = const (); + goto -> bb16; + } + + bb6: { + return; + } + + bb7: { + StorageDead(_5); + StorageDead(_4); + goto -> bb9; + } + + bb8: { + StorageDead(_3); + goto -> bb9; + } + + bb9: { + goto -> bb20; + } + + bb10: { + coroutine_drop; + } + + bb11 (cleanup): { + StorageDead(_3); + goto -> bb23; + } + + bb12 (cleanup): { + resume; + } + + bb13 (cleanup): { + StorageDead(_6); + goto -> bb12; + } + + bb14 (cleanup): { + drop(_6) -> [return: bb13, unwind terminate(cleanup)]; + } + + bb15: { + StorageDead(_6); + goto -> bb6; + } + + bb16: { + drop(_6) -> [return: bb15, unwind: bb12, drop: bb10]; + } + + bb17 (cleanup): { + StorageDead(_6); + goto -> bb12; + } + + bb18 (cleanup): { + drop(_6) -> [return: bb17, unwind terminate(cleanup)]; + } + + bb19: { + StorageDead(_6); + goto -> bb10; + } + + bb20: { + drop(_6) -> [return: bb19, unwind: bb12]; + } + + bb21 (cleanup): { + terminate(cleanup); + } + + bb22 (cleanup): { + StorageDead(_6); + goto -> bb12; + } + + bb23 (cleanup): { + drop(_6) -> [return: bb22, unwind terminate(cleanup)]; + } + + bb24: { + StorageLive(_3); + _3 = String::new() -> [return: bb1, unwind: bb11]; + } +} diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-unwind.mir b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-unwind.mir new file mode 100644 index 0000000000000..655f0c8f47891 --- /dev/null +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-unwind.mir @@ -0,0 +1,138 @@ +// MIR for `main::{closure#0}` after RelocateUpvars + +fn main::{closure#0}(_1: {coroutine@$DIR/coroutine-relocate-upvars.rs:12:5: 12:7}, _2: ()) -> () +yields () + { + debug x => (*_6); + debug x => _6; + let mut _0: (); + let mut _3: std::string::String; + let _4: (); + let mut _5: (); + let mut _6: &mut std::string::String; + let _7: &mut std::string::String; + + bb0: { + StorageLive(_6); + StorageLive(_7); + _7 = move (_1.0: &mut std::string::String); + _6 = move _7; + StorageDead(_7); + goto -> bb24; + } + + bb1: { + drop((*_6)) -> [return: bb2, unwind: bb3]; + } + + bb2: { + (*_6) = move _3; + drop(_3) -> [return: bb4, unwind: bb11, drop: bb8]; + } + + bb3 (cleanup): { + (*_6) = move _3; + drop(_3) -> [return: bb11, unwind terminate(cleanup)]; + } + + bb4: { + StorageDead(_3); + StorageLive(_4); + StorageLive(_5); + _5 = (); + _4 = yield(move _5) -> [resume: bb5, drop: bb7]; + } + + bb5: { + StorageDead(_5); + StorageDead(_4); + _0 = const (); + goto -> bb16; + } + + bb6: { + return; + } + + bb7: { + StorageDead(_5); + StorageDead(_4); + goto -> bb9; + } + + bb8: { + StorageDead(_3); + goto -> bb9; + } + + bb9: { + goto -> bb20; + } + + bb10: { + coroutine_drop; + } + + bb11 (cleanup): { + StorageDead(_3); + goto -> bb23; + } + + bb12 (cleanup): { + resume; + } + + bb13 (cleanup): { + StorageDead(_6); + goto -> bb12; + } + + bb14 (cleanup): { + drop(_6) -> [return: bb13, unwind terminate(cleanup)]; + } + + bb15: { + StorageDead(_6); + goto -> bb6; + } + + bb16: { + drop(_6) -> [return: bb15, unwind: bb12, drop: bb10]; + } + + bb17 (cleanup): { + StorageDead(_6); + goto -> bb12; + } + + bb18 (cleanup): { + drop(_6) -> [return: bb17, unwind terminate(cleanup)]; + } + + bb19: { + StorageDead(_6); + goto -> bb10; + } + + bb20: { + drop(_6) -> [return: bb19, unwind: bb12]; + } + + bb21 (cleanup): { + terminate(cleanup); + } + + bb22 (cleanup): { + StorageDead(_6); + goto -> bb12; + } + + bb23 (cleanup): { + drop(_6) -> [return: bb22, unwind terminate(cleanup)]; + } + + bb24: { + StorageLive(_3); + _3 = String::new() -> [return: bb1, unwind: bb11]; + } +} diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-abort.mir b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-abort.mir new file mode 100644 index 0000000000000..54790152c6a49 --- /dev/null +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-abort.mir @@ -0,0 +1,77 @@ +// MIR for `main::{closure#0}` before RelocateUpvars + +fn main::{closure#0}(_1: {coroutine@$DIR/coroutine-relocate-upvars.rs:12:5: 12:7}, _2: ()) -> () +yields () + { + debug x => (*(_1.0: &mut std::string::String)); + let mut _0: (); + let mut _3: std::string::String; + let _4: (); + let mut _5: (); + + bb0: { + StorageLive(_3); + _3 = String::new() -> [return: bb1, unwind: bb11]; + } + + bb1: { + drop((*(_1.0: &mut std::string::String))) -> [return: bb2, unwind: bb3]; + } + + bb2: { + (*(_1.0: &mut std::string::String)) = move _3; + drop(_3) -> [return: bb4, unwind: bb11, drop: bb8]; + } + + bb3 (cleanup): { + (*(_1.0: &mut std::string::String)) = move _3; + drop(_3) -> [return: bb11, unwind terminate(cleanup)]; + } + + bb4: { + StorageDead(_3); + StorageLive(_4); + StorageLive(_5); + _5 = (); + _4 = yield(move _5) -> [resume: bb5, drop: bb7]; + } + + bb5: { + StorageDead(_5); + StorageDead(_4); + _0 = const (); + drop(_1) -> [return: bb6, unwind: bb12, drop: bb10]; + } + + bb6: { + return; + } + + bb7: { + StorageDead(_5); + StorageDead(_4); + goto -> bb9; + } + + bb8: { + StorageDead(_3); + goto -> bb9; + } + + bb9: { + drop(_1) -> [return: bb10, unwind: bb12]; + } + + bb10: { + coroutine_drop; + } + + bb11 (cleanup): { + StorageDead(_3); + drop(_1) -> [return: bb12, unwind terminate(cleanup)]; + } + + bb12 (cleanup): { + resume; + } +} diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-unwind.mir b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-unwind.mir new file mode 100644 index 0000000000000..54790152c6a49 --- /dev/null +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-unwind.mir @@ -0,0 +1,77 @@ +// MIR for `main::{closure#0}` before RelocateUpvars + +fn main::{closure#0}(_1: {coroutine@$DIR/coroutine-relocate-upvars.rs:12:5: 12:7}, _2: ()) -> () +yields () + { + debug x => (*(_1.0: &mut std::string::String)); + let mut _0: (); + let mut _3: std::string::String; + let _4: (); + let mut _5: (); + + bb0: { + StorageLive(_3); + _3 = String::new() -> [return: bb1, unwind: bb11]; + } + + bb1: { + drop((*(_1.0: &mut std::string::String))) -> [return: bb2, unwind: bb3]; + } + + bb2: { + (*(_1.0: &mut std::string::String)) = move _3; + drop(_3) -> [return: bb4, unwind: bb11, drop: bb8]; + } + + bb3 (cleanup): { + (*(_1.0: &mut std::string::String)) = move _3; + drop(_3) -> [return: bb11, unwind terminate(cleanup)]; + } + + bb4: { + StorageDead(_3); + StorageLive(_4); + StorageLive(_5); + _5 = (); + _4 = yield(move _5) -> [resume: bb5, drop: bb7]; + } + + bb5: { + StorageDead(_5); + StorageDead(_4); + _0 = const (); + drop(_1) -> [return: bb6, unwind: bb12, drop: bb10]; + } + + bb6: { + return; + } + + bb7: { + StorageDead(_5); + StorageDead(_4); + goto -> bb9; + } + + bb8: { + StorageDead(_3); + goto -> bb9; + } + + bb9: { + drop(_1) -> [return: bb10, unwind: bb12]; + } + + bb10: { + coroutine_drop; + } + + bb11 (cleanup): { + StorageDead(_3); + drop(_1) -> [return: bb12, unwind terminate(cleanup)]; + } + + bb12 (cleanup): { + resume; + } +} From b15c01138877e2cfd6845d68b6f0cb627806162e Mon Sep 17 00:00:00 2001 From: Xiangfei Ding Date: Wed, 30 Jul 2025 15:38:52 +0000 Subject: [PATCH 3/4] update documentation --- compiler/rustc_middle/src/mir/mod.rs | 9 +++- .../src/coroutine/relocate_upvars.rs | 53 +++++++++++++++++-- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index 647c43c03fa80..345c4929aabad 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -334,7 +334,14 @@ pub struct Body<'tcx> { #[type_visitable(ignore)] pub function_coverage_info: Option>, - /// Coroutine local-upvar map + /// Coroutine local-upvar map, which maps the coroutine captures as fields + /// in coroutine state to the internal MIR locals. + /// This is to help borrow-checker assert lifetime invariance between types of + /// the capture, as appear exterior to the coroutine, and that of locals, + /// intto which the captures are relocated. + /// It also assists diagnostic to re-construct the identity of the captures. + /// MIR interpretation and instrumentation can disregard this information. + /// It carries no operation-semantic significance. pub local_upvar_map: IndexVec>, } diff --git a/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs b/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs index 4aed2d933125a..8dac0d65b7a0f 100644 --- a/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs +++ b/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs @@ -29,10 +29,57 @@ //! //! Each place that starts with access into the coroutine structure `_1` is replaced with the fresh local as //! the base. -//! For instance, `(_1.4 as Some).0` is rewritten into `(_34 as Some).0` when `_34` is the fresh local -//! corresponding to the captured upvar stored in `_1.4`. +//! Suppose we are to lower this coroutine into the MIR. +//! ```ignore (illustrative) +//! let mut captured = None; +//! let _ = #[coroutine] || { +//! yield (); +//! if let Some(inner) = &mut captured { +//! *inner = 42i32; // (*) +//! } +//! }; +//! ``` +//! `captured` is the only capture, whose mutable borrow is formally allotted to the first field `_1.0: &mut i32`. +//! The highlighted line `(*)` should be lowered, roughly, into MIR `(*_1.0) = const 42i32;`. +//! Now, by application of this pass, we create a new local `_4: &mut i32` and we perform the following +//! code transformation. +//! +//! A new block is constructed to just perform the relocation of this mutable borrow. +//! This block is inserted to the very beginning of the coroutine body control flow, +//! so that this is executed before any proper coroutine code as it transits from `UNRESUME` state to +//! any other state. +//! This "prologue" will look like the following. +//! ```ignore (illustrative) +//! StorageLive(_5); +//! StorageLive(_4); +//! _5 = move (_1.0); +//! _4 = move (_5); +//! StorageDead(_5); +//! ``` +//! Note that we also create a trampoline local `_5` of the same type. +//! +//! ### Intricacy around the trampoline local +//! The reason that we need the trampolines is because state transformation and coroutine +//! layout calculation is not aware of potential storage conflict between captures as struct fields +//! and other saved locals. +//! The only guarantee that we can work with is one where any coroutine layout calculator respects +//! the storage conflict contracts between *MIR locals*. +//! It is known that calculators do not consider struct fields, where captures reside, as MIR locals. +//! This is the source of potential memory overlaps. +//! For instance, in a hypothetical situation, +//! - `_1.0` is relocated to `_4`; +//! - `_1.1` is relocated to `_6`; +//! - `_4` and `_6` remains live at one of the first suspension state; +//! - `_4` occupies the same offset of `_1.1` and `_6` occupies the same offset of `_1.0` +//! as decided by some layout calculator; +//! In this scenario, without trampolining, the relocations introduce undefined behaviour. +//! +//! As a proposal for a future design, it is best that coroutine captures receive their own +//! MIR locals, possibly in a form of "function arguments" like `_1` itself. +//! The trampolining transformation already attempts to restore the semantics of MIR locals to +//! these captures and promoting them to "arguments" would make MIR safer to handle. //! -//! This phase assumes that the initial built MIR respects the nature of captures. +//! One should note that this phase assumes that the initial built MIR respects the nature of captures. //! For instance, if the upvar `_1.4` is instead a by-ref-mut capture of a value of type `T`, //! this phase assumes that all access correctly built as operating on the place `(*_1.4)`. //! Based on the assumption, this phase replaces `_1.4` with a fresh local `_34: &mut T` and From 0ceb235c8fee7c673dae295fe03ed799d01cf308 Mon Sep 17 00:00:00 2001 From: Xiangfei Ding Date: Mon, 25 Aug 2025 17:31:20 +0200 Subject: [PATCH 4/4] bless some tests Signed-off-by: Xiangfei Ding --- tests/codegen-llvm/async-fn-debug.rs | 4 +-- ...re#0}.RelocateUpvars.after.panic-abort.mir | 2 +- ...e#0}.RelocateUpvars.after.panic-unwind.mir | 2 +- ...e#0}.RelocateUpvars.before.panic-abort.mir | 2 +- ...#0}.RelocateUpvars.before.panic-unwind.mir | 2 +- ...upvars.rs => coroutine_relocate_upvars.rs} | 2 +- .../async-drop/async-drop-initial.rs | 30 ++++++++----------- 7 files changed, 19 insertions(+), 25 deletions(-) rename tests/mir-opt/{coroutine-relocate-upvars.rs => coroutine_relocate_upvars.rs} (99%) diff --git a/tests/codegen-llvm/async-fn-debug.rs b/tests/codegen-llvm/async-fn-debug.rs index c430a12700414..a1fc4f14db0b1 100644 --- a/tests/codegen-llvm/async-fn-debug.rs +++ b/tests/codegen-llvm/async-fn-debug.rs @@ -38,7 +38,7 @@ async fn async_fn_test() { // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) // CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "3", scope: [[VARIANT]], -// CHECK-SAME: file: [[FILE]], line: 14, +// CHECK-SAME: file: [[FILE]], line: 15, // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) // CHECK: [[S0:!.*]] = !DICompositeType(tag: DW_TAG_structure_type, name: "Suspend0", scope: [[GEN]], @@ -48,7 +48,7 @@ async fn async_fn_test() { // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) // CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "4", scope: [[VARIANT]], -// CHECK-SAME: file: [[FILE]], line: 12, +// CHECK-SAME: file: [[FILE]], line: 13, // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) // CHECK: [[DISC]] = !DIDerivedType(tag: DW_TAG_member, name: "__state", scope: [[GEN]], diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-abort.mir b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-abort.mir index 655f0c8f47891..9994cb8119540 100644 --- a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-abort.mir +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-abort.mir @@ -1,6 +1,6 @@ // MIR for `main::{closure#0}` after RelocateUpvars -fn main::{closure#0}(_1: {coroutine@$DIR/coroutine-relocate-upvars.rs:12:5: 12:7}, _2: ()) -> () +fn main::{closure#0}(_1: {coroutine@$DIR/coroutine_relocate_upvars.rs:12:5: 12:7}, _2: ()) -> () yields () { debug x => (*_6); diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-unwind.mir b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-unwind.mir index 655f0c8f47891..9994cb8119540 100644 --- a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-unwind.mir +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-unwind.mir @@ -1,6 +1,6 @@ // MIR for `main::{closure#0}` after RelocateUpvars -fn main::{closure#0}(_1: {coroutine@$DIR/coroutine-relocate-upvars.rs:12:5: 12:7}, _2: ()) -> () +fn main::{closure#0}(_1: {coroutine@$DIR/coroutine_relocate_upvars.rs:12:5: 12:7}, _2: ()) -> () yields () { debug x => (*_6); diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-abort.mir b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-abort.mir index 54790152c6a49..a6e54a7af077f 100644 --- a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-abort.mir +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-abort.mir @@ -1,6 +1,6 @@ // MIR for `main::{closure#0}` before RelocateUpvars -fn main::{closure#0}(_1: {coroutine@$DIR/coroutine-relocate-upvars.rs:12:5: 12:7}, _2: ()) -> () +fn main::{closure#0}(_1: {coroutine@$DIR/coroutine_relocate_upvars.rs:12:5: 12:7}, _2: ()) -> () yields () { debug x => (*(_1.0: &mut std::string::String)); diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-unwind.mir b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-unwind.mir index 54790152c6a49..a6e54a7af077f 100644 --- a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-unwind.mir +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-unwind.mir @@ -1,6 +1,6 @@ // MIR for `main::{closure#0}` before RelocateUpvars -fn main::{closure#0}(_1: {coroutine@$DIR/coroutine-relocate-upvars.rs:12:5: 12:7}, _2: ()) -> () +fn main::{closure#0}(_1: {coroutine@$DIR/coroutine_relocate_upvars.rs:12:5: 12:7}, _2: ()) -> () yields () { debug x => (*(_1.0: &mut std::string::String)); diff --git a/tests/mir-opt/coroutine-relocate-upvars.rs b/tests/mir-opt/coroutine_relocate_upvars.rs similarity index 99% rename from tests/mir-opt/coroutine-relocate-upvars.rs rename to tests/mir-opt/coroutine_relocate_upvars.rs index 35a86f6326432..42c4ca896a3e8 100644 --- a/tests/mir-opt/coroutine-relocate-upvars.rs +++ b/tests/mir-opt/coroutine_relocate_upvars.rs @@ -13,4 +13,4 @@ fn main() { x = String::new(); yield; }; -} \ No newline at end of file +} diff --git a/tests/ui/async-await/async-drop/async-drop-initial.rs b/tests/ui/async-await/async-drop/async-drop-initial.rs index 64f370babdbdc..f44dc88000390 100644 --- a/tests/ui/async-await/async-drop/async-drop-initial.rs +++ b/tests/ui/async-await/async-drop/async-drop-initial.rs @@ -18,8 +18,8 @@ use core::future::{AsyncDrop, Future, async_drop_in_place}; use core::hint::black_box; use core::mem::{self, ManuallyDrop}; use core::pin::{Pin, pin}; -use core::task::{Context, Poll, Waker}; use core::sync::atomic::{AtomicBool, Ordering}; +use core::task::{Context, Poll, Waker}; static PASS: AtomicBool = AtomicBool::new(false); @@ -34,7 +34,10 @@ async fn test_async_drop(x: T, #[allow(unused)] expect: usize) { let got = mem::size_of_val(&*dtor); #[cfg(target_pointer_width = "64")] if expect != got { - println!("sizes did not match for async destructor of type {}, expect {expect}, got {got}", core::any::type_name::()); + println!( + "sizes did not match for async destructor of type {}, expect {expect}, got {got}", + core::any::type_name::() + ); PASS.store(false, Ordering::Relaxed); } @@ -69,12 +72,9 @@ fn main() { test_async_drop(&j, [16, 24][cfg!(classic) as usize]).await; test_async_drop( AsyncStruct { b: AsyncInt(8), a: AsyncInt(7), i: 6 }, - if cfg!(panic = "unwind") { - [128, 192][cfg!(classic) as usize] - } else { - 136 - }, - ).await; + if cfg!(panic = "unwind") { [128, 192][cfg!(classic) as usize] } else { 136 }, + ) + .await; test_async_drop(ManuallyDrop::new(AsyncInt(9)), [16, 24][cfg!(classic) as usize]).await; let foo = AsyncInt(10); @@ -241,19 +241,13 @@ union AsyncUnion { impl Drop for AsyncUnion { fn drop(&mut self) { - println!( - "AsyncUnion::drop: {}, {}", - unsafe { self.signed }, - unsafe { self.unsigned }, - ); + println!("AsyncUnion::drop: {}, {}", unsafe { self.signed }, unsafe { self.unsigned },); } } impl AsyncDrop for AsyncUnion { async fn drop(self: Pin<&mut Self>) { - println!( - "AsyncUnion::Dropper::poll: {}, {}", - unsafe { self.signed }, - unsafe { self.unsigned }, - ); + println!("AsyncUnion::Dropper::poll: {}, {}", unsafe { self.signed }, unsafe { + self.unsigned + },); } }