1// Copyright 2015 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "textflag.h"
6#include "funcdata.h"
7
8// func Cas(ptr *int32, old, new int32) bool
9// Atomically:
10// if *ptr == old {
11// *ptr = new
12// return true
13// } else {
14// return false
15// }
16TEXT ·Cas(SB), NOSPLIT, $0-13
17 MOVL ptr+0(FP), BX
18 MOVL old+4(FP), AX
19 MOVL new+8(FP), CX
20 LOCK
21 CMPXCHGL CX, 0(BX)
22 SETEQ ret+12(FP)
23 RET
24
25TEXT ·Casint32(SB), NOSPLIT, $0-13
26 JMP ·Cas(SB)
27
28TEXT ·Casint64(SB), NOSPLIT, $0-21
29 JMP ·Cas64(SB)
30
31TEXT ·Casuintptr(SB), NOSPLIT, $0-13
32 JMP ·Cas(SB)
33
34TEXT ·CasRel(SB), NOSPLIT, $0-13
35 JMP ·Cas(SB)
36
37TEXT ·Loaduintptr(SB), NOSPLIT, $0-8
38 JMP ·Load(SB)
39
40TEXT ·Loaduint(SB), NOSPLIT, $0-8
41 JMP ·Load(SB)
42
43TEXT ·Storeint32(SB), NOSPLIT, $0-8
44 JMP ·Store(SB)
45
46TEXT ·Storeint64(SB), NOSPLIT, $0-12
47 JMP ·Store64(SB)
48
49TEXT ·Storeuintptr(SB), NOSPLIT, $0-8
50 JMP ·Store(SB)
51
52TEXT ·Xadduintptr(SB), NOSPLIT, $0-12
53 JMP ·Xadd(SB)
54
55TEXT ·Loadint32(SB), NOSPLIT, $0-8
56 JMP ·Load(SB)
57
58TEXT ·Loadint64(SB), NOSPLIT, $0-12
59 JMP ·Load64(SB)
60
61TEXT ·Xaddint32(SB), NOSPLIT, $0-12
62 JMP ·Xadd(SB)
63
64TEXT ·Xaddint64(SB), NOSPLIT, $0-20
65 JMP ·Xadd64(SB)
66
67// func Cas64(ptr *uint64, old, new uint64) bool
68// Atomically:
69// if *ptr == old {
70// *ptr = new
71// return true
72// } else {
73// return false
74// }
75TEXT ·Cas64(SB), NOSPLIT, $0-21
76 NO_LOCAL_POINTERS
77 MOVL ptr+0(FP), BP
78 TESTL $7, BP
79 JZ 2(PC)
80 CALL ·panicUnaligned(SB)
81 MOVL old_lo+4(FP), AX
82 MOVL old_hi+8(FP), DX
83 MOVL new_lo+12(FP), BX
84 MOVL new_hi+16(FP), CX
85 LOCK
86 CMPXCHG8B 0(BP)
87 SETEQ ret+20(FP)
88 RET
89
90// func Casp1(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool
91// Atomically:
92// if *ptr == old {
93// *ptr = new
94// return true
95// } else {
96// return false
97// }
98TEXT ·Casp1(SB), NOSPLIT, $0-13
99 MOVL ptr+0(FP), BX
100 MOVL old+4(FP), AX
101 MOVL new+8(FP), CX
102 LOCK
103 CMPXCHGL CX, 0(BX)
104 SETEQ ret+12(FP)
105 RET
106
107// uint32 Xadd(uint32 volatile *val, int32 delta)
108// Atomically:
109// *val += delta;
110// return *val;
111TEXT ·Xadd(SB), NOSPLIT, $0-12
112 MOVL ptr+0(FP), BX
113 MOVL delta+4(FP), AX
114 MOVL AX, CX
115 LOCK
116 XADDL AX, 0(BX)
117 ADDL CX, AX
118 MOVL AX, ret+8(FP)
119 RET
120
121TEXT ·Xadd64(SB), NOSPLIT, $0-20
122 NO_LOCAL_POINTERS
123 // no XADDQ so use CMPXCHG8B loop
124 MOVL ptr+0(FP), BP
125 TESTL $7, BP
126 JZ 2(PC)
127 CALL ·panicUnaligned(SB)
128 // DI:SI = delta
129 MOVL delta_lo+4(FP), SI
130 MOVL delta_hi+8(FP), DI
131 // DX:AX = *addr
132 MOVL 0(BP), AX
133 MOVL 4(BP), DX
134addloop:
135 // CX:BX = DX:AX (*addr) + DI:SI (delta)
136 MOVL AX, BX
137 MOVL DX, CX
138 ADDL SI, BX
139 ADCL DI, CX
140
141 // if *addr == DX:AX {
142 // *addr = CX:BX
143 // } else {
144 // DX:AX = *addr
145 // }
146 // all in one instruction
147 LOCK
148 CMPXCHG8B 0(BP)
149
150 JNZ addloop
151
152 // success
153 // return CX:BX
154 MOVL BX, ret_lo+12(FP)
155 MOVL CX, ret_hi+16(FP)
156 RET
157
158// uint8 Xchg8(uint8 *ptr, uint8 new)
159TEXT ·Xchg8(SB), NOSPLIT, $0-9
160 MOVL ptr+0(FP), BX
161 MOVB new+4(FP), AX
162 XCHGB AX, 0(BX)
163 MOVB AX, ret+8(FP)
164 RET
165
166TEXT ·Xchg(SB), NOSPLIT, $0-12
167 MOVL ptr+0(FP), BX
168 MOVL new+4(FP), AX
169 XCHGL AX, 0(BX)
170 MOVL AX, ret+8(FP)
171 RET
172
173TEXT ·Xchgint32(SB), NOSPLIT, $0-12
174 JMP ·Xchg(SB)
175
176TEXT ·Xchgint64(SB), NOSPLIT, $0-20
177 JMP ·Xchg64(SB)
178
179TEXT ·Xchguintptr(SB), NOSPLIT, $0-12
180 JMP ·Xchg(SB)
181
182TEXT ·Xchg64(SB),NOSPLIT,$0-20
183 NO_LOCAL_POINTERS
184 // no XCHGQ so use CMPXCHG8B loop
185 MOVL ptr+0(FP), BP
186 TESTL $7, BP
187 JZ 2(PC)
188 CALL ·panicUnaligned(SB)
189 // CX:BX = new
190 MOVL new_lo+4(FP), BX
191 MOVL new_hi+8(FP), CX
192 // DX:AX = *addr
193 MOVL 0(BP), AX
194 MOVL 4(BP), DX
195swaploop:
196 // if *addr == DX:AX
197 // *addr = CX:BX
198 // else
199 // DX:AX = *addr
200 // all in one instruction
201 LOCK
202 CMPXCHG8B 0(BP)
203 JNZ swaploop
204
205 // success
206 // return DX:AX
207 MOVL AX, ret_lo+12(FP)
208 MOVL DX, ret_hi+16(FP)
209 RET
210
211TEXT ·StorepNoWB(SB), NOSPLIT, $0-8
212 MOVL ptr+0(FP), BX
213 MOVL val+4(FP), AX
214 XCHGL AX, 0(BX)
215 RET
216
217TEXT ·Store(SB), NOSPLIT, $0-8
218 MOVL ptr+0(FP), BX
219 MOVL val+4(FP), AX
220 XCHGL AX, 0(BX)
221 RET
222
223TEXT ·StoreRel(SB), NOSPLIT, $0-8
224 JMP ·Store(SB)
225
226TEXT ·StoreReluintptr(SB), NOSPLIT, $0-8
227 JMP ·Store(SB)
228
229// uint64 atomicload64(uint64 volatile* addr);
230TEXT ·Load64(SB), NOSPLIT, $0-12
231 NO_LOCAL_POINTERS
232 MOVL ptr+0(FP), AX
233 TESTL $7, AX
234 JZ 2(PC)
235 CALL ·panicUnaligned(SB)
236 MOVQ (AX), M0
237 MOVQ M0, ret+4(FP)
238 EMMS
239 RET
240
241// void ·Store64(uint64 volatile* addr, uint64 v);
242TEXT ·Store64(SB), NOSPLIT, $0-12
243 NO_LOCAL_POINTERS
244 MOVL ptr+0(FP), AX
245 TESTL $7, AX
246 JZ 2(PC)
247 CALL ·panicUnaligned(SB)
248 // MOVQ and EMMS were introduced on the Pentium MMX.
249 MOVQ val+4(FP), M0
250 MOVQ M0, (AX)
251 EMMS
252 // This is essentially a no-op, but it provides required memory fencing.
253 // It can be replaced with MFENCE, but MFENCE was introduced only on the Pentium4 (SSE2).
254 XORL AX, AX
255 LOCK
256 XADDL AX, (SP)
257 RET
258
259// void ·Or8(byte volatile*, byte);
260TEXT ·Or8(SB), NOSPLIT, $0-5
261 MOVL ptr+0(FP), AX
262 MOVB val+4(FP), BX
263 LOCK
264 ORB BX, (AX)
265 RET
266
267// void ·And8(byte volatile*, byte);
268TEXT ·And8(SB), NOSPLIT, $0-5
269 MOVL ptr+0(FP), AX
270 MOVB val+4(FP), BX
271 LOCK
272 ANDB BX, (AX)
273 RET
274
275TEXT ·Store8(SB), NOSPLIT, $0-5
276 MOVL ptr+0(FP), BX
277 MOVB val+4(FP), AX
278 XCHGB AX, 0(BX)
279 RET
280
281// func Or(addr *uint32, v uint32)
282TEXT ·Or(SB), NOSPLIT, $0-8
283 MOVL ptr+0(FP), AX
284 MOVL val+4(FP), BX
285 LOCK
286 ORL BX, (AX)
287 RET
288
289// func And(addr *uint32, v uint32)
290TEXT ·And(SB), NOSPLIT, $0-8
291 MOVL ptr+0(FP), AX
292 MOVL val+4(FP), BX
293 LOCK
294 ANDL BX, (AX)
295 RET
296
297// func And32(addr *uint32, v uint32) old uint32
298TEXT ·And32(SB), NOSPLIT, $0-12
299 MOVL ptr+0(FP), BX
300 MOVL val+4(FP), CX
301casloop:
302 MOVL CX, DX
303 MOVL (BX), AX
304 ANDL AX, DX
305 LOCK
306 CMPXCHGL DX, (BX)
307 JNZ casloop
308 MOVL AX, ret+8(FP)
309 RET
310
311// func Or32(addr *uint32, v uint32) old uint32
312TEXT ·Or32(SB), NOSPLIT, $0-12
313 MOVL ptr+0(FP), BX
314 MOVL val+4(FP), CX
315casloop:
316 MOVL CX, DX
317 MOVL (BX), AX
318 ORL AX, DX
319 LOCK
320 CMPXCHGL DX, (BX)
321 JNZ casloop
322 MOVL AX, ret+8(FP)
323 RET
324
325// func And64(addr *uint64, v uint64) old uint64
326TEXT ·And64(SB), NOSPLIT, $0-20
327 MOVL ptr+0(FP), BP
328 // DI:SI = v
329 MOVL val_lo+4(FP), SI
330 MOVL val_hi+8(FP), DI
331 // DX:AX = *addr
332 MOVL 0(BP), AX
333 MOVL 4(BP), DX
334casloop:
335 // CX:BX = DX:AX (*addr) & DI:SI (mask)
336 MOVL AX, BX
337 MOVL DX, CX
338 ANDL SI, BX
339 ANDL DI, CX
340 LOCK
341 CMPXCHG8B 0(BP)
342 JNZ casloop
343 MOVL AX, ret_lo+12(FP)
344 MOVL DX, ret_hi+16(FP)
345 RET
346
347
348// func Or64(addr *uint64, v uint64) old uint64
349TEXT ·Or64(SB), NOSPLIT, $0-20
350 MOVL ptr+0(FP), BP
351 // DI:SI = v
352 MOVL val_lo+4(FP), SI
353 MOVL val_hi+8(FP), DI
354 // DX:AX = *addr
355 MOVL 0(BP), AX
356 MOVL 4(BP), DX
357casloop:
358 // CX:BX = DX:AX (*addr) | DI:SI (mask)
359 MOVL AX, BX
360 MOVL DX, CX
361 ORL SI, BX
362 ORL DI, CX
363 LOCK
364 CMPXCHG8B 0(BP)
365 JNZ casloop
366 MOVL AX, ret_lo+12(FP)
367 MOVL DX, ret_hi+16(FP)
368 RET
369
370// func Anduintptr(addr *uintptr, v uintptr) old uintptr
371TEXT ·Anduintptr(SB), NOSPLIT, $0-12
372 JMP ·And32(SB)
373
374// func Oruintptr(addr *uintptr, v uintptr) old uintptr
375TEXT ·Oruintptr(SB), NOSPLIT, $0-12
376 JMP ·Or32(SB)
View as plain text