From 5129ea575b79874213b2cb46c0eda0c9ceed79ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tommy=20M=E1=B6=9CMichen?= Date: Fri, 1 Aug 2025 10:58:24 -0700 Subject: [PATCH 1/3] [llvm][sroa] Disabled support for `invariant.group` --- llvm/lib/Transforms/Scalar/SROA.cpp | 15 ++------- llvm/test/Transforms/SROA/invariant-group.ll | 34 ++++++++++++++++---- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp index 23256cf2acbd2..03d9f32aa3f03 100644 --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -1247,8 +1247,7 @@ class AllocaSlices::SliceBuilder : public PtrUseVisitor { "Map index doesn't point back to a slice with this user."); } - // Disable SRoA for any intrinsics except for lifetime invariants and - // invariant group. + // Disable SRoA for any intrinsics except for lifetime invariants. // FIXME: What about debug intrinsics? This matches old behavior, but // doesn't make sense. void visitIntrinsicInst(IntrinsicInst &II) { @@ -1268,12 +1267,6 @@ class AllocaSlices::SliceBuilder : public PtrUseVisitor { return; } - if (II.isLaunderOrStripInvariantGroup()) { - insertUse(II, Offset, AllocSize, true); - enqueueUsers(II); - return; - } - Base::visitIntrinsicInst(II); } @@ -3607,8 +3600,7 @@ class AllocaSliceRewriter : public InstVisitor { } bool visitIntrinsicInst(IntrinsicInst &II) { - assert((II.isLifetimeStartOrEnd() || II.isLaunderOrStripInvariantGroup() || - II.isDroppable()) && + assert((II.isLifetimeStartOrEnd() || II.isDroppable()) && "Unexpected intrinsic!"); LLVM_DEBUG(dbgs() << " original: " << II << "\n"); @@ -3622,9 +3614,6 @@ class AllocaSliceRewriter : public InstVisitor { return true; } - if (II.isLaunderOrStripInvariantGroup()) - return true; - assert(II.getArgOperand(1) == OldPtr); // Lifetime intrinsics are only promotable if they cover the whole alloca. // Therefore, we drop lifetime intrinsics which don't cover the whole diff --git a/llvm/test/Transforms/SROA/invariant-group.ll b/llvm/test/Transforms/SROA/invariant-group.ll index 1be6f6e2fc32b..147ea3704f6b9 100644 --- a/llvm/test/Transforms/SROA/invariant-group.ll +++ b/llvm/test/Transforms/SROA/invariant-group.ll @@ -11,10 +11,17 @@ declare i32 @somevalue() define void @f() { ; CHECK-LABEL: @f( +; CHECK-NEXT: [[A:%.*]] = alloca [[T:%.*]], align 8 +; CHECK-NEXT: [[A1_I8_INV:%.*]] = call ptr @llvm.launder.invariant.group.p0(ptr [[A]]) +; CHECK-NEXT: [[A2:%.*]] = getelementptr inbounds [[T]], ptr [[A]], i32 0, i32 1 ; CHECK-NEXT: [[SV1:%.*]] = call i32 @somevalue() ; CHECK-NEXT: [[SV2:%.*]] = call i32 @somevalue() -; CHECK-NEXT: call void @h(i32 [[SV1]]) -; CHECK-NEXT: call void @h(i32 [[SV2]]) +; CHECK-NEXT: store i32 [[SV1]], ptr [[A1_I8_INV]], align 4, !invariant.group [[META0:![0-9]+]] +; CHECK-NEXT: store i32 [[SV2]], ptr [[A2]], align 4 +; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[A1_I8_INV]], align 4, !invariant.group [[META0]] +; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[A2]], align 4 +; CHECK-NEXT: call void @h(i32 [[V1]]) +; CHECK-NEXT: call void @h(i32 [[V2]]) ; CHECK-NEXT: ret void ; %a = alloca %t @@ -44,7 +51,7 @@ define void @g() { ; CHECK-NEXT: [[A2:%.*]] = getelementptr inbounds [[T]], ptr [[A]], i32 0, i32 1 ; CHECK-NEXT: [[SV1:%.*]] = call i32 @somevalue() ; CHECK-NEXT: [[SV2:%.*]] = call i32 @somevalue() -; CHECK-NEXT: store i32 [[SV1]], ptr [[A1_I8_INV]], align 4, !invariant.group [[META0:![0-9]+]] +; CHECK-NEXT: store i32 [[SV1]], ptr [[A1_I8_INV]], align 4, !invariant.group [[META0]] ; CHECK-NEXT: store i32 [[SV2]], ptr [[A2]], align 4 ; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[A1_I8_INV]], align 4, !invariant.group [[META0]] ; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[A2]], align 4 @@ -81,6 +88,9 @@ define void @g() { define void @store_and_launder() { ; CHECK-LABEL: @store_and_launder( +; CHECK-NEXT: [[VALPTR:%.*]] = alloca i32, align 4 +; CHECK-NEXT: store i32 0, ptr [[VALPTR]], align 4 +; CHECK-NEXT: [[BARR:%.*]] = call ptr @llvm.launder.invariant.group.p0(ptr [[VALPTR]]) ; CHECK-NEXT: ret void ; %valptr = alloca i32, align 4 @@ -91,7 +101,10 @@ define void @store_and_launder() { define i32 @launder_and_load() { ; CHECK-LABEL: @launder_and_load( -; CHECK-NEXT: ret i32 undef +; CHECK-NEXT: [[VALPTR:%.*]] = alloca i32, align 4 +; CHECK-NEXT: [[BARR:%.*]] = call ptr @llvm.launder.invariant.group.p0(ptr [[VALPTR]]) +; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[VALPTR]], align 4 +; CHECK-NEXT: ret i32 [[V2]] ; %valptr = alloca i32, align 4 %barr = call ptr @llvm.launder.invariant.group.p0(ptr %valptr) @@ -101,6 +114,9 @@ define i32 @launder_and_load() { define void @launder_and_ptr_arith() { ; CHECK-LABEL: @launder_and_ptr_arith( +; CHECK-NEXT: [[VALPTR:%.*]] = alloca i32, align 4 +; CHECK-NEXT: [[BARR:%.*]] = call ptr @llvm.launder.invariant.group.p0(ptr [[VALPTR]]) +; CHECK-NEXT: [[A2:%.*]] = getelementptr inbounds i32, ptr [[VALPTR]], i32 0 ; CHECK-NEXT: ret void ; %valptr = alloca i32, align 4 @@ -140,9 +156,13 @@ end: define void @partial_promotion_of_alloca() { ; CHECK-LABEL: @partial_promotion_of_alloca( -; CHECK-NEXT: [[STRUCT_PTR_SROA_2:%.*]] = alloca i32, align 4 -; CHECK-NEXT: store volatile i32 0, ptr [[STRUCT_PTR_SROA_2]], align 4 -; CHECK-NEXT: [[STRUCT_PTR_SROA_2_0_STRUCT_PTR_SROA_2_4_LOAD_VAL:%.*]] = load volatile i32, ptr [[STRUCT_PTR_SROA_2]], align 4 +; CHECK-NEXT: [[STRUCT_PTR:%.*]] = alloca [[T:%.*]], align 4 +; CHECK-NEXT: [[FIELD_PTR:%.*]] = getelementptr inbounds [[T]], ptr [[STRUCT_PTR]], i32 0, i32 0 +; CHECK-NEXT: store i32 0, ptr [[FIELD_PTR]], align 4 +; CHECK-NEXT: [[VOLATILE_FIELD_PTR:%.*]] = getelementptr inbounds [[T]], ptr [[STRUCT_PTR]], i32 0, i32 1 +; CHECK-NEXT: store volatile i32 0, ptr [[VOLATILE_FIELD_PTR]], align 4, !invariant.group [[META0]] +; CHECK-NEXT: [[BARR:%.*]] = call ptr @llvm.launder.invariant.group.p0(ptr [[STRUCT_PTR]]) +; CHECK-NEXT: [[LOAD_VAL:%.*]] = load volatile i32, ptr [[VOLATILE_FIELD_PTR]], align 4, !invariant.group [[META0]] ; CHECK-NEXT: ret void ; %struct_ptr = alloca %t, align 4 From 02c381f9acd6f3022b28b9fcfa3756477a4490d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tommy=20M=E1=B6=9CMichen?= Date: Fri, 1 Aug 2025 13:12:38 -0700 Subject: [PATCH 2/3] [llvm][sroa] Added `invariant.group` tests from #107557 and #151574 --- llvm/test/Transforms/SROA/invariant-group.ll | 70 ++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/llvm/test/Transforms/SROA/invariant-group.ll b/llvm/test/Transforms/SROA/invariant-group.ll index 147ea3704f6b9..a4a203ddfa750 100644 --- a/llvm/test/Transforms/SROA/invariant-group.ll +++ b/llvm/test/Transforms/SROA/invariant-group.ll @@ -175,6 +175,76 @@ define void @partial_promotion_of_alloca() { ret void } +define void @memcpy_after_laundering_alloca(ptr %ptr) { +; CHECK-LABEL: @memcpy_after_laundering_alloca( +; CHECK-NEXT: [[ALLOCA:%.*]] = alloca { i64, i64 }, align 8 +; CHECK-NEXT: [[LAUNDER:%.*]] = call ptr @llvm.launder.invariant.group.p0(ptr [[ALLOCA]]) +; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[LAUNDER]], ptr [[PTR:%.*]], i64 16, i1 false) +; CHECK-NEXT: ret void +; + %alloca = alloca { i64, i64 }, align 8 + %launder = call ptr @llvm.launder.invariant.group.p0(ptr %alloca) + call void @llvm.memcpy.p0.p0.i64(ptr %launder, ptr %ptr, i64 16, i1 false) + ret void +} + +define void @memcpy_after_laundering_alloca_slices(ptr %ptr) { +; CHECK-LABEL: @memcpy_after_laundering_alloca_slices( +; CHECK-NEXT: [[ALLOCA:%.*]] = alloca { [16 x i8], i64, [16 x i8] }, align 8 +; CHECK-NEXT: [[LAUNDER:%.*]] = call ptr @llvm.launder.invariant.group.p0(ptr [[ALLOCA]]) +; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[LAUNDER]], i64 16 +; CHECK-NEXT: store i64 0, ptr [[GEP]], align 4 +; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[LAUNDER]], ptr [[PTR:%.*]], i64 40, i1 false) +; CHECK-NEXT: ret void +; + %alloca = alloca { [16 x i8], i64, [16 x i8] }, align 8 + %launder = call ptr @llvm.launder.invariant.group.p0(ptr %alloca) + %gep = getelementptr i8, ptr %launder, i64 16 + store i64 0, ptr %gep + call void @llvm.memcpy.p0.p0.i64(ptr %launder, ptr %ptr, i64 40, i1 false) + ret void +} + +define void @launder_in_loop() { +; CHECK-LABEL: @launder_in_loop( +; CHECK-NEXT: [[STRUCT_PTR:%.*]] = alloca [[T:%.*]], i64 1, align 4 +; CHECK-NEXT: br label [[HEADER:%.*]] +; CHECK: header: +; CHECK-NEXT: br i1 true, label [[BODY:%.*]], label [[EXIT:%.*]] +; CHECK: body: +; CHECK-NEXT: [[STRUCT_PTR_FRESH:%.*]] = call ptr @llvm.launder.invariant.group.p0(ptr [[STRUCT_PTR]]) +; CHECK-NEXT: [[STRUCT:%.*]] = call [[T]] @[[MAKE_T:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]]() +; CHECK-NEXT: store [[T]] [[STRUCT]], ptr [[STRUCT_PTR_FRESH]], align 4, !invariant.group [[META0]] +; CHECK-NEXT: [[FIRST_PTR:%.*]] = getelementptr [[T]], ptr [[STRUCT_PTR_FRESH]], i32 0, i32 0 +; CHECK-NEXT: [[FIRST:%.*]] = load i32, ptr [[FIRST_PTR]], align 4 +; CHECK-NEXT: [[SECOND_PTR:%.*]] = getelementptr [[T]], ptr [[STRUCT_PTR_FRESH]], i32 0, i32 1 +; CHECK-NEXT: [[SECOND:%.*]] = load i32, ptr [[SECOND_PTR]], align 4 +; CHECK-NEXT: br label [[HEADER]] +; CHECK: exit: +; CHECK-NEXT: ret void +; + %struct_ptr = alloca %t, i64 1, align 4 + br label %header + +header: + br i1 true, label %body, label %exit + +body: ; preds = %6 + %struct_ptr_fresh = call ptr @llvm.launder.invariant.group.p0(ptr %struct_ptr) + %struct = call %t @make_t() + store %t %struct, ptr %struct_ptr_fresh, align 4, !invariant.group !0 + %first_ptr = getelementptr %t, ptr %struct_ptr_fresh, i32 0, i32 0 + %first = load i32, ptr %first_ptr, align 4 + %second_ptr = getelementptr %t, ptr %struct_ptr_fresh, i32 0, i32 1 + %second = load i32, ptr %second_ptr, align 4 + br label %header + +exit: + ret void +} + +declare %t @make_t() + declare void @use(ptr) !0 = !{} From ee5d72d0a8801d304490d461ca0e50cf1fd61928 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tommy=20M=E1=B6=9CMichen?= Date: Mon, 4 Aug 2025 10:04:31 -0700 Subject: [PATCH 3/3] [llvm][sroa] Removed unecessary loop from aggregate store with invariant.group test --- llvm/test/Transforms/SROA/invariant-group.ll | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/llvm/test/Transforms/SROA/invariant-group.ll b/llvm/test/Transforms/SROA/invariant-group.ll index a4a203ddfa750..c9c9e031ca95f 100644 --- a/llvm/test/Transforms/SROA/invariant-group.ll +++ b/llvm/test/Transforms/SROA/invariant-group.ll @@ -205,13 +205,9 @@ define void @memcpy_after_laundering_alloca_slices(ptr %ptr) { ret void } -define void @launder_in_loop() { -; CHECK-LABEL: @launder_in_loop( +define void @test_agg_store() { +; CHECK-LABEL: @test_agg_store( ; CHECK-NEXT: [[STRUCT_PTR:%.*]] = alloca [[T:%.*]], i64 1, align 4 -; CHECK-NEXT: br label [[HEADER:%.*]] -; CHECK: header: -; CHECK-NEXT: br i1 true, label [[BODY:%.*]], label [[EXIT:%.*]] -; CHECK: body: ; CHECK-NEXT: [[STRUCT_PTR_FRESH:%.*]] = call ptr @llvm.launder.invariant.group.p0(ptr [[STRUCT_PTR]]) ; CHECK-NEXT: [[STRUCT:%.*]] = call [[T]] @[[MAKE_T:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]]() ; CHECK-NEXT: store [[T]] [[STRUCT]], ptr [[STRUCT_PTR_FRESH]], align 4, !invariant.group [[META0]] @@ -219,17 +215,9 @@ define void @launder_in_loop() { ; CHECK-NEXT: [[FIRST:%.*]] = load i32, ptr [[FIRST_PTR]], align 4 ; CHECK-NEXT: [[SECOND_PTR:%.*]] = getelementptr [[T]], ptr [[STRUCT_PTR_FRESH]], i32 0, i32 1 ; CHECK-NEXT: [[SECOND:%.*]] = load i32, ptr [[SECOND_PTR]], align 4 -; CHECK-NEXT: br label [[HEADER]] -; CHECK: exit: ; CHECK-NEXT: ret void ; %struct_ptr = alloca %t, i64 1, align 4 - br label %header - -header: - br i1 true, label %body, label %exit - -body: ; preds = %6 %struct_ptr_fresh = call ptr @llvm.launder.invariant.group.p0(ptr %struct_ptr) %struct = call %t @make_t() store %t %struct, ptr %struct_ptr_fresh, align 4, !invariant.group !0 @@ -237,9 +225,6 @@ body: ; preds = %6 %first = load i32, ptr %first_ptr, align 4 %second_ptr = getelementptr %t, ptr %struct_ptr_fresh, i32 0, i32 1 %second = load i32, ptr %second_ptr, align 4 - br label %header - -exit: ret void }