Skip to content

Commit c568927

Browse files
JonPsson1JonPsson1
authored
[SystemZ] Properly support 16 byte atomic int/fp types and ops. (#73134)
- Clang FE now has MaxAtomicPromoteWidth / MaxAtomicInlineWidth set to 128, and now produces IR instead of calls to __atomic instrinsics for 16 bytes as well. - Atomic __int128 (and long double) variables are now aligned to 16 bytes by default (like gcc 14). - AtomicExpand pass now expands 16 byte operations as well. - tests for __atomic builtins for all integer widths, and __atomic_is_lock_free with friends. - TODO: AtomicExpand pass handles with this patch expansion of i128 atomicrmw:s. As a next step smaller integer types should also be possible to handle this way instead of by the backend.
1 parent a7993fa commit c568927

11 files changed

+2011
-62
lines changed

‎clang/lib/Basic/Targets/SystemZ.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ class LLVM_LIBRARY_VISIBILITY SystemZTargetInfo : public TargetInfo {
6060
resetDataLayout("E-m:e-i1:8:16-i8:8:16-i64:64-f128:64"
6161
"-v128:64-a:8:16-n32:64");
6262
}
63-
MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 64;
63+
MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 128;
6464
HasStrictFP = true;
6565
}
6666

‎clang/test/CodeGen/SystemZ/atomic_is_lock_free.c

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
// RUN: %clang_cc1 -triple s390x-linux-gnu -O1 -emit-llvm %s -o - | FileCheck %s
2+
//
3+
// Test __atomic_is_lock_free() and friends.
4+
5+
#include <stdatomic.h>
6+
#include <stdint.h>
7+
8+
typedef __attribute__((aligned(16))) __int128 __int128_Al16;
9+
10+
_Atomic __int128 Int128_Atomic;
11+
__int128_Al16 Int128_Al16;
12+
__int128 Int128;
13+
struct { int I[3]; } _Atomic AtomicStruct;
14+
_Atomic long double Atomic_fp128; // Also check the alignment of this.
15+
16+
// Check alignments of the variables. @AtomicStruct gets padded and its size
17+
// and alignment becomes 16. Only a power-of-2 size is considered, so 16 (not
18+
// 12) needs to be specified with the intrinsics below.
19+
//
20+
// CHECK: %struct.anon = type { [3 x i32] }
21+
// CHECK: @Int128 = {{.*}} i128 0, align 8
22+
// CHECK: @Int128_Atomic = {{.*}} i128 0, align 16
23+
// CHECK: @Int128_Al16 = {{.*}} i128 0, align 16
24+
// CHECK: @AtomicStruct = {{.*}} { %struct.anon, [4 x i8] } zeroinitializer, align 16
25+
// CHECK: @Atomic_fp128 = {{.*}} fp128 0xL00000000000000000000000000000000, align 16
26+
27+
28+
// CHECK-LABEL: @fun0
29+
// CHECK: ret i1 true
30+
_Bool fun0() {
31+
return __atomic_is_lock_free(16, &Int128_Atomic);
32+
}
33+
34+
// CHECK-LABEL: @fun1
35+
// CHECK: ret i1 true
36+
_Bool fun1() {
37+
return __atomic_always_lock_free(16, &Int128_Atomic);
38+
}
39+
40+
// CHECK-LABEL: @fun2
41+
// CHECK: ret i1 true
42+
_Bool fun2() {
43+
return __atomic_is_lock_free(16, &Int128_Al16);
44+
}
45+
46+
// CHECK-LABEL: @fun3
47+
// CHECK: ret i1 true
48+
_Bool fun3() {
49+
return __atomic_always_lock_free(16, &Int128_Al16);
50+
}
51+
52+
// CHECK-LABEL: @fun4
53+
// CHECK: call zeroext i1 @__atomic_is_lock_free
54+
_Bool fun4() {
55+
return __atomic_is_lock_free(16, &Int128);
56+
}
57+
58+
// CHECK-LABEL: @fun5
59+
// CHECK: ret i1 false
60+
_Bool fun5() {
61+
return __atomic_always_lock_free(16, &Int128);
62+
}
63+
64+
// CHECK-LABEL: @fun6
65+
// CHECK: ret i1 true
66+
_Bool fun6() {
67+
return __atomic_is_lock_free(16, 0);
68+
}
69+
70+
// CHECK-LABEL: @fun7
71+
// CHECK: ret i1 true
72+
_Bool fun7() {
73+
return __atomic_always_lock_free(16, 0);
74+
}
75+
76+
// CHECK-LABEL: @fun8
77+
// CHECK: ret i1 true
78+
_Bool fun8() {
79+
return __atomic_is_lock_free(16, &AtomicStruct);
80+
}
81+
82+
// CHECK-LABEL: @fun9
83+
// CHECK: ret i1 true
84+
_Bool fun9() {
85+
return __atomic_always_lock_free(16, &AtomicStruct);
86+
}
87+
88+
// CHECK-LABEL: @fun10
89+
// CHECK: ret i1 true
90+
_Bool fun10() {
91+
return atomic_is_lock_free(&Int128_Atomic);
92+
}
93+
94+
// CHECK-LABEL: @fun11
95+
// CHECK: ret i1 true
96+
_Bool fun11() {
97+
return __c11_atomic_is_lock_free(16);
98+
}

‎clang/test/CodeGen/SystemZ/gnu-atomic-builtins-i128-16Al.c

+257
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,257 @@
1+
// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
2+
// RUN: %clang_cc1 -triple s390x-linux-gnu -O1 -emit-llvm %s -o - | FileCheck %s
3+
//
4+
// Test GNU atomic builtins for __int128 aligned to 16 bytes, which should be
5+
// expanded to LLVM I/R by the front end.
6+
7+
#include <stdatomic.h>
8+
#include <stdint.h>
9+
10+
__int128 Ptr __attribute__((aligned(16)));
11+
__int128 Ret __attribute__((aligned(16)));
12+
__int128 Val __attribute__((aligned(16)));
13+
__int128 Exp __attribute__((aligned(16)));
14+
__int128 Des __attribute__((aligned(16)));
15+
16+
// CHECK-LABEL: @f1(
17+
// CHECK-NEXT: entry:
18+
// CHECK-NEXT: [[TMP0:%.*]] = load atomic i128, ptr @Ptr seq_cst, align 16
19+
// CHECK-NEXT: store i128 [[TMP0]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2:![0-9]+]]
20+
// CHECK-NEXT: ret void
21+
//
22+
__int128 f1() {
23+
return __atomic_load_n(&Ptr, memory_order_seq_cst);
24+
}
25+
26+
// CHECK-LABEL: @f2(
27+
// CHECK-NEXT: entry:
28+
// CHECK-NEXT: [[TMP0:%.*]] = load atomic i128, ptr @Ptr seq_cst, align 16
29+
// CHECK-NEXT: store i128 [[TMP0]], ptr @Ret, align 16
30+
// CHECK-NEXT: store i128 [[TMP0]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
31+
// CHECK-NEXT: ret void
32+
//
33+
__int128 f2() {
34+
__atomic_load(&Ptr, &Ret, memory_order_seq_cst);
35+
return Ret;
36+
}
37+
38+
// CHECK-LABEL: @f3(
39+
// CHECK-NEXT: entry:
40+
// CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
41+
// CHECK-NEXT: store atomic i128 [[TMP0]], ptr @Ptr seq_cst, align 16
42+
// CHECK-NEXT: ret void
43+
//
44+
void f3() {
45+
__atomic_store_n(&Ptr, Val, memory_order_seq_cst);
46+
}
47+
48+
// CHECK-LABEL: @f4(
49+
// CHECK-NEXT: entry:
50+
// CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr @Val, align 16
51+
// CHECK-NEXT: store atomic i128 [[TMP0]], ptr @Ptr seq_cst, align 16
52+
// CHECK-NEXT: ret void
53+
//
54+
void f4() {
55+
__atomic_store(&Ptr, &Val, memory_order_seq_cst);
56+
}
57+
58+
// CHECK-LABEL: @f5(
59+
// CHECK-NEXT: entry:
60+
// CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
61+
// CHECK-NEXT: [[TMP1:%.*]] = atomicrmw xchg ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
62+
// CHECK-NEXT: store i128 [[TMP1]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
63+
// CHECK-NEXT: ret void
64+
//
65+
__int128 f5() {
66+
return __atomic_exchange_n(&Ptr, Val, memory_order_seq_cst);
67+
}
68+
69+
// CHECK-LABEL: @f6(
70+
// CHECK-NEXT: entry:
71+
// CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr @Val, align 16
72+
// CHECK-NEXT: [[TMP1:%.*]] = atomicrmw xchg ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
73+
// CHECK-NEXT: store i128 [[TMP1]], ptr @Ret, align 16
74+
// CHECK-NEXT: store i128 [[TMP1]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
75+
// CHECK-NEXT: ret void
76+
//
77+
__int128 f6() {
78+
__atomic_exchange(&Ptr, &Val, &Ret, memory_order_seq_cst);
79+
return Ret;
80+
}
81+
82+
// CHECK-LABEL: @f7(
83+
// CHECK-NEXT: entry:
84+
// CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr @Des, align 16, !tbaa [[TBAA2]]
85+
// CHECK-NEXT: [[TMP1:%.*]] = load i128, ptr @Exp, align 16
86+
// CHECK-NEXT: [[TMP2:%.*]] = cmpxchg ptr @Ptr, i128 [[TMP1]], i128 [[TMP0]] seq_cst seq_cst, align 16
87+
// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { i128, i1 } [[TMP2]], 1
88+
// CHECK-NEXT: br i1 [[TMP3]], label [[CMPXCHG_CONTINUE:%.*]], label [[CMPXCHG_STORE_EXPECTED:%.*]]
89+
// CHECK: cmpxchg.store_expected:
90+
// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { i128, i1 } [[TMP2]], 0
91+
// CHECK-NEXT: store i128 [[TMP4]], ptr @Exp, align 16
92+
// CHECK-NEXT: br label [[CMPXCHG_CONTINUE]]
93+
// CHECK: cmpxchg.continue:
94+
// CHECK-NEXT: ret i1 [[TMP3]]
95+
//
96+
_Bool f7() {
97+
return __atomic_compare_exchange_n(&Ptr, &Exp, Des, 0,
98+
memory_order_seq_cst, memory_order_seq_cst);
99+
}
100+
101+
// CHECK-LABEL: @f8(
102+
// CHECK-NEXT: entry:
103+
// CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr @Exp, align 16
104+
// CHECK-NEXT: [[TMP1:%.*]] = load i128, ptr @Des, align 16
105+
// CHECK-NEXT: [[TMP2:%.*]] = cmpxchg ptr @Ptr, i128 [[TMP0]], i128 [[TMP1]] seq_cst seq_cst, align 16
106+
// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { i128, i1 } [[TMP2]], 1
107+
// CHECK-NEXT: br i1 [[TMP3]], label [[CMPXCHG_CONTINUE:%.*]], label [[CMPXCHG_STORE_EXPECTED:%.*]]
108+
// CHECK: cmpxchg.store_expected:
109+
// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { i128, i1 } [[TMP2]], 0
110+
// CHECK-NEXT: store i128 [[TMP4]], ptr @Exp, align 16
111+
// CHECK-NEXT: br label [[CMPXCHG_CONTINUE]]
112+
// CHECK: cmpxchg.continue:
113+
// CHECK-NEXT: ret i1 [[TMP3]]
114+
//
115+
_Bool f8() {
116+
return __atomic_compare_exchange(&Ptr, &Exp, &Des, 0,
117+
memory_order_seq_cst, memory_order_seq_cst);
118+
}
119+
120+
// CHECK-LABEL: @f9(
121+
// CHECK-NEXT: entry:
122+
// CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
123+
// CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
124+
// CHECK-NEXT: [[TMP2:%.*]] = add i128 [[TMP1]], [[TMP0]]
125+
// CHECK-NEXT: store i128 [[TMP2]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
126+
// CHECK-NEXT: ret void
127+
//
128+
__int128 f9() {
129+
return __atomic_add_fetch(&Ptr, Val, memory_order_seq_cst);
130+
}
131+
132+
// CHECK-LABEL: @f10(
133+
// CHECK-NEXT: entry:
134+
// CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
135+
// CHECK-NEXT: [[TMP1:%.*]] = atomicrmw sub ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
136+
// CHECK-NEXT: [[TMP2:%.*]] = sub i128 [[TMP1]], [[TMP0]]
137+
// CHECK-NEXT: store i128 [[TMP2]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
138+
// CHECK-NEXT: ret void
139+
//
140+
__int128 f10() {
141+
return __atomic_sub_fetch(&Ptr, Val, memory_order_seq_cst);
142+
}
143+
144+
// CHECK-LABEL: @f11(
145+
// CHECK-NEXT: entry:
146+
// CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
147+
// CHECK-NEXT: [[TMP1:%.*]] = atomicrmw and ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
148+
// CHECK-NEXT: [[TMP2:%.*]] = and i128 [[TMP1]], [[TMP0]]
149+
// CHECK-NEXT: store i128 [[TMP2]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
150+
// CHECK-NEXT: ret void
151+
//
152+
__int128 f11() {
153+
return __atomic_and_fetch(&Ptr, Val, memory_order_seq_cst);
154+
}
155+
156+
// CHECK-LABEL: @f12(
157+
// CHECK-NEXT: entry:
158+
// CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
159+
// CHECK-NEXT: [[TMP1:%.*]] = atomicrmw xor ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
160+
// CHECK-NEXT: [[TMP2:%.*]] = xor i128 [[TMP1]], [[TMP0]]
161+
// CHECK-NEXT: store i128 [[TMP2]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
162+
// CHECK-NEXT: ret void
163+
//
164+
__int128 f12() {
165+
return __atomic_xor_fetch(&Ptr, Val, memory_order_seq_cst);
166+
}
167+
168+
// CHECK-LABEL: @f13(
169+
// CHECK-NEXT: entry:
170+
// CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
171+
// CHECK-NEXT: [[TMP1:%.*]] = atomicrmw or ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
172+
// CHECK-NEXT: [[TMP2:%.*]] = or i128 [[TMP1]], [[TMP0]]
173+
// CHECK-NEXT: store i128 [[TMP2]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
174+
// CHECK-NEXT: ret void
175+
//
176+
__int128 f13() {
177+
return __atomic_or_fetch(&Ptr, Val, memory_order_seq_cst);
178+
}
179+
180+
// CHECK-LABEL: @f14(
181+
// CHECK-NEXT: entry:
182+
// CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
183+
// CHECK-NEXT: [[TMP1:%.*]] = atomicrmw nand ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
184+
// CHECK-NEXT: [[TMP2:%.*]] = and i128 [[TMP1]], [[TMP0]]
185+
// CHECK-NEXT: [[TMP3:%.*]] = xor i128 [[TMP2]], -1
186+
// CHECK-NEXT: store i128 [[TMP3]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
187+
// CHECK-NEXT: ret void
188+
//
189+
__int128 f14() {
190+
return __atomic_nand_fetch(&Ptr, Val, memory_order_seq_cst);
191+
}
192+
193+
// CHECK-LABEL: @f15(
194+
// CHECK-NEXT: entry:
195+
// CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
196+
// CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
197+
// CHECK-NEXT: store i128 [[TMP1]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
198+
// CHECK-NEXT: ret void
199+
//
200+
__int128 f15() {
201+
return __atomic_fetch_add(&Ptr, Val, memory_order_seq_cst);
202+
}
203+
204+
// CHECK-LABEL: @f16(
205+
// CHECK-NEXT: entry:
206+
// CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
207+
// CHECK-NEXT: [[TMP1:%.*]] = atomicrmw sub ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
208+
// CHECK-NEXT: store i128 [[TMP1]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
209+
// CHECK-NEXT: ret void
210+
//
211+
__int128 f16() {
212+
return __atomic_fetch_sub(&Ptr, Val, memory_order_seq_cst);
213+
}
214+
215+
// CHECK-LABEL: @f17(
216+
// CHECK-NEXT: entry:
217+
// CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
218+
// CHECK-NEXT: [[TMP1:%.*]] = atomicrmw and ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
219+
// CHECK-NEXT: store i128 [[TMP1]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
220+
// CHECK-NEXT: ret void
221+
//
222+
__int128 f17() {
223+
return __atomic_fetch_and(&Ptr, Val, memory_order_seq_cst);
224+
}
225+
226+
// CHECK-LABEL: @f18(
227+
// CHECK-NEXT: entry:
228+
// CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
229+
// CHECK-NEXT: [[TMP1:%.*]] = atomicrmw xor ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
230+
// CHECK-NEXT: store i128 [[TMP1]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
231+
// CHECK-NEXT: ret void
232+
//
233+
__int128 f18() {
234+
return __atomic_fetch_xor(&Ptr, Val, memory_order_seq_cst);
235+
}
236+
237+
// CHECK-LABEL: @f19(
238+
// CHECK-NEXT: entry:
239+
// CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
240+
// CHECK-NEXT: [[TMP1:%.*]] = atomicrmw or ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
241+
// CHECK-NEXT: store i128 [[TMP1]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
242+
// CHECK-NEXT: ret void
243+
//
244+
__int128 f19() {
245+
return __atomic_fetch_or(&Ptr, Val, memory_order_seq_cst);
246+
}
247+
248+
// CHECK-LABEL: @f20(
249+
// CHECK-NEXT: entry:
250+
// CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
251+
// CHECK-NEXT: [[TMP1:%.*]] = atomicrmw nand ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
252+
// CHECK-NEXT: store i128 [[TMP1]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
253+
// CHECK-NEXT: ret void
254+
//
255+
__int128 f20() {
256+
return __atomic_fetch_nand(&Ptr, Val, memory_order_seq_cst);
257+
}

0 commit comments

Comments
 (0)