Text file
src/runtime/asm_s390x.s
Documentation: runtime
1// Copyright 2016 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "go_tls.h"
7#include "funcdata.h"
8#include "textflag.h"
9
10// _rt0_s390x_lib is common startup code for s390x systems when
11// using -buildmode=c-archive or -buildmode=c-shared. The linker will
12// arrange to invoke this function as a global constructor (for
13// c-archive) or when the shared library is loaded (for c-shared).
14// We expect argc and argv to be passed in the usual C ABI registers
15// R2 and R3.
16TEXT _rt0_s390x_lib(SB), NOSPLIT|NOFRAME, $0
17 STMG R6, R15, 48(R15)
18 MOVD R2, _rt0_s390x_lib_argc<>(SB)
19 MOVD R3, _rt0_s390x_lib_argv<>(SB)
20
21 // Save R6-R15 in the register save area of the calling function.
22 STMG R6, R15, 48(R15)
23
24 // Allocate 80 bytes on the stack.
25 MOVD $-80(R15), R15
26
27 // Save F8-F15 in our stack frame.
28 FMOVD F8, 16(R15)
29 FMOVD F9, 24(R15)
30 FMOVD F10, 32(R15)
31 FMOVD F11, 40(R15)
32 FMOVD F12, 48(R15)
33 FMOVD F13, 56(R15)
34 FMOVD F14, 64(R15)
35 FMOVD F15, 72(R15)
36
37 // Synchronous initialization.
38 MOVD $runtime·libpreinit(SB), R1
39 BL R1
40
41 // Create a new thread to finish Go runtime initialization.
42 MOVD _cgo_sys_thread_create(SB), R1
43 CMP R1, $0
44 BEQ nocgo
45 MOVD $_rt0_s390x_lib_go(SB), R2
46 MOVD $0, R3
47 BL R1
48 BR restore
49
50nocgo:
51 MOVD $0x800000, R1 // stacksize
52 MOVD R1, 0(R15)
53 MOVD $_rt0_s390x_lib_go(SB), R1
54 MOVD R1, 8(R15) // fn
55 MOVD $runtime·newosproc(SB), R1
56 BL R1
57
58restore:
59 // Restore F8-F15 from our stack frame.
60 FMOVD 16(R15), F8
61 FMOVD 24(R15), F9
62 FMOVD 32(R15), F10
63 FMOVD 40(R15), F11
64 FMOVD 48(R15), F12
65 FMOVD 56(R15), F13
66 FMOVD 64(R15), F14
67 FMOVD 72(R15), F15
68 MOVD $80(R15), R15
69
70 // Restore R6-R15.
71 LMG 48(R15), R6, R15
72 RET
73
74// _rt0_s390x_lib_go initializes the Go runtime.
75// This is started in a separate thread by _rt0_s390x_lib.
76TEXT _rt0_s390x_lib_go(SB), NOSPLIT|NOFRAME, $0
77 MOVD _rt0_s390x_lib_argc<>(SB), R2
78 MOVD _rt0_s390x_lib_argv<>(SB), R3
79 MOVD $runtime·rt0_go(SB), R1
80 BR R1
81
82DATA _rt0_s390x_lib_argc<>(SB)/8, $0
83GLOBL _rt0_s390x_lib_argc<>(SB), NOPTR, $8
84DATA _rt0_s90x_lib_argv<>(SB)/8, $0
85GLOBL _rt0_s390x_lib_argv<>(SB), NOPTR, $8
86
87TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
88 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer
89 // C TLS base pointer in AR0:AR1
90
91 // initialize essential registers
92 XOR R0, R0
93
94 SUB $24, R15
95 MOVW R2, 8(R15) // argc
96 MOVD R3, 16(R15) // argv
97
98 // create istack out of the given (operating system) stack.
99 // _cgo_init may update stackguard.
100 MOVD $runtime·g0(SB), g
101 MOVD R15, R11
102 SUB $(64*1024), R11
103 MOVD R11, g_stackguard0(g)
104 MOVD R11, g_stackguard1(g)
105 MOVD R11, (g_stack+stack_lo)(g)
106 MOVD R15, (g_stack+stack_hi)(g)
107
108 // if there is a _cgo_init, call it using the gcc ABI.
109 MOVD _cgo_init(SB), R11
110 CMPBEQ R11, $0, nocgo
111 MOVW AR0, R4 // (AR0 << 32 | AR1) is the TLS base pointer; MOVD is translated to EAR
112 SLD $32, R4, R4
113 MOVW AR1, R4 // arg 2: TLS base pointer
114 MOVD $setg_gcc<>(SB), R3 // arg 1: setg
115 MOVD g, R2 // arg 0: G
116 // C functions expect 160 bytes of space on caller stack frame
117 // and an 8-byte aligned stack pointer
118 MOVD R15, R9 // save current stack (R9 is preserved in the Linux ABI)
119 SUB $160, R15 // reserve 160 bytes
120 MOVD $~7, R6
121 AND R6, R15 // 8-byte align
122 BL R11 // this call clobbers volatile registers according to Linux ABI (R0-R5, R14)
123 MOVD R9, R15 // restore stack
124 XOR R0, R0 // zero R0
125
126nocgo:
127 // update stackguard after _cgo_init
128 MOVD (g_stack+stack_lo)(g), R2
129 ADD $const_stackGuard, R2
130 MOVD R2, g_stackguard0(g)
131 MOVD R2, g_stackguard1(g)
132
133 // set the per-goroutine and per-mach "registers"
134 MOVD $runtime·m0(SB), R2
135
136 // save m->g0 = g0
137 MOVD g, m_g0(R2)
138 // save m0 to g0->m
139 MOVD R2, g_m(g)
140
141 BL runtime·check(SB)
142
143 // argc/argv are already prepared on stack
144 BL runtime·args(SB)
145 BL runtime·checkS390xCPU(SB)
146 BL runtime·osinit(SB)
147 BL runtime·schedinit(SB)
148
149 // create a new goroutine to start program
150 MOVD $runtime·mainPC(SB), R2 // entry
151 SUB $16, R15
152 MOVD R2, 8(R15)
153 MOVD $0, 0(R15)
154 BL runtime·newproc(SB)
155 ADD $16, R15
156
157 // start this M
158 BL runtime·mstart(SB)
159
160 MOVD $0, 1(R0)
161 RET
162
163DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
164GLOBL runtime·mainPC(SB),RODATA,$8
165
166TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
167 BRRK
168 RET
169
170TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
171 RET
172
173TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
174 CALL runtime·mstart0(SB)
175 RET // not reached
176
177/*
178 * go-routine
179 */
180
181// void gogo(Gobuf*)
182// restore state from Gobuf; longjmp
183TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
184 MOVD buf+0(FP), R5
185 MOVD gobuf_g(R5), R6
186 MOVD 0(R6), R7 // make sure g != nil
187 BR gogo<>(SB)
188
189TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
190 MOVD R6, g
191 BL runtime·save_g(SB)
192
193 MOVD 0(g), R4
194 MOVD gobuf_sp(R5), R15
195 MOVD gobuf_lr(R5), LR
196 MOVD gobuf_ctxt(R5), R12
197 MOVD $0, gobuf_sp(R5)
198 MOVD $0, gobuf_lr(R5)
199 MOVD $0, gobuf_ctxt(R5)
200 CMP R0, R0 // set condition codes for == test, needed by stack split
201 MOVD gobuf_pc(R5), R6
202 BR (R6)
203
204// void mcall(fn func(*g))
205// Switch to m->g0's stack, call fn(g).
206// Fn must never return. It should gogo(&g->sched)
207// to keep running g.
208TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT, $-8-8
209#ifdef GOEXPERIMENT_regabiargs
210 MOVD R2, R12 // context
211#else
212 MOVD fn+0(FP), R12 // context
213#endif
214 // Save caller state in g->sched
215 MOVD R15, (g_sched+gobuf_sp)(g)
216 MOVD LR, (g_sched+gobuf_pc)(g)
217 MOVD $0, (g_sched+gobuf_lr)(g)
218
219 // Switch to m->g0 & its stack, call fn.
220 MOVD g, R2
221 MOVD g_m(g), R4
222 MOVD m_g0(R4), g
223 BL runtime·save_g(SB)
224 CMP g, R2
225 BNE 2(PC)
226 BR runtime·badmcall(SB)
227 MOVD 0(R12), R4 // code pointer
228 MOVD (g_sched+gobuf_sp)(g), R15 // sp = m->g0->sched.sp
229 SUB $16, R15
230 MOVD R2, 8(R15)
231 MOVD $0, 0(R15)
232 BL (R4)
233 BR runtime·badmcall2(SB)
234
235// systemstack_switch is a dummy routine that systemstack leaves at the bottom
236// of the G stack. We need to distinguish the routine that
237// lives at the bottom of the G stack from the one that lives
238// at the top of the system stack because the one at the top of
239// the system stack terminates the stack walk (see topofstack()).
240TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
241 UNDEF
242 BL (LR) // make sure this function is not leaf
243 RET
244
245// func systemstack(fn func())
246TEXT runtime·systemstack(SB), NOSPLIT, $0-8
247 MOVD fn+0(FP), R3 // R3 = fn
248 MOVD R3, R12 // context
249 MOVD g_m(g), R4 // R4 = m
250
251 MOVD m_gsignal(R4), R5 // R5 = gsignal
252 CMPBEQ g, R5, noswitch
253
254 MOVD m_g0(R4), R5 // R5 = g0
255 CMPBEQ g, R5, noswitch
256
257 MOVD m_curg(R4), R6
258 CMPBEQ g, R6, switch
259
260 // Bad: g is not gsignal, not g0, not curg. What is it?
261 // Hide call from linker nosplit analysis.
262 MOVD $runtime·badsystemstack(SB), R3
263 BL (R3)
264 BL runtime·abort(SB)
265
266switch:
267 // save our state in g->sched. Pretend to
268 // be systemstack_switch if the G stack is scanned.
269 BL gosave_systemstack_switch<>(SB)
270
271 // switch to g0
272 MOVD R5, g
273 BL runtime·save_g(SB)
274 MOVD (g_sched+gobuf_sp)(g), R15
275
276 // call target function
277 MOVD 0(R12), R3 // code pointer
278 BL (R3)
279
280 // switch back to g
281 MOVD g_m(g), R3
282 MOVD m_curg(R3), g
283 BL runtime·save_g(SB)
284 MOVD (g_sched+gobuf_sp)(g), R15
285 MOVD $0, (g_sched+gobuf_sp)(g)
286 RET
287
288noswitch:
289 // already on m stack, just call directly
290 // Using a tail call here cleans up tracebacks since we won't stop
291 // at an intermediate systemstack.
292 MOVD 0(R12), R3 // code pointer
293 MOVD 0(R15), LR // restore LR
294 ADD $8, R15
295 BR (R3)
296
297// func switchToCrashStack0(fn func())
298TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
299 MOVD R2, R12 // context
300 MOVD g_m(g), R2 // curm
301
302 // set g to gcrash
303 MOVD $runtime·gcrash(SB), g // g = &gcrash
304 BL runtime·save_g(SB)
305 MOVD R2, g_m(g) // g.m = curm
306 MOVD g, m_g0(R2) // curm.g0 = g
307
308 // switch to crashstack
309 MOVD (g_stack+stack_hi)(g), R2
310 ADD $(-4*8), R2, R15
311
312 // call target function
313 MOVD 0(R12), R3 // code pointer
314 BL (R3)
315
316 // should never return
317 BL runtime·abort(SB)
318 UNDEF
319
320/*
321 * support for morestack
322 */
323
324// Called during function prolog when more stack is needed.
325// Caller has already loaded:
326// R3: framesize, R4: argsize, R5: LR
327//
328// The traceback routines see morestack on a g0 as being
329// the top of a stack (for example, morestack calling newstack
330// calling the scheduler calling newm calling gc), so we must
331// record an argument size. For that purpose, it has no arguments.
332TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
333 // Called from f.
334 // Set g->sched to context in f.
335 MOVD R15, (g_sched+gobuf_sp)(g)
336 MOVD LR, R8
337 MOVD R8, (g_sched+gobuf_pc)(g)
338 MOVD R5, (g_sched+gobuf_lr)(g)
339 MOVD R12, (g_sched+gobuf_ctxt)(g)
340
341 // Cannot grow scheduler stack (m->g0).
342 MOVD g_m(g), R7
343 MOVD m_g0(R7), R8
344 CMPBNE g, R8, 3(PC)
345 BL runtime·badmorestackg0(SB)
346 BL runtime·abort(SB)
347
348 // Cannot grow signal stack (m->gsignal).
349 MOVD m_gsignal(R7), R8
350 CMP g, R8
351 BNE 3(PC)
352 BL runtime·badmorestackgsignal(SB)
353 BL runtime·abort(SB)
354
355 // Called from f.
356 // Set m->morebuf to f's caller.
357 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
358 MOVD R15, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
359 MOVD g, (m_morebuf+gobuf_g)(R7)
360
361 // Call newstack on m->g0's stack.
362 MOVD m_g0(R7), g
363 BL runtime·save_g(SB)
364 MOVD (g_sched+gobuf_sp)(g), R15
365 // Create a stack frame on g0 to call newstack.
366 MOVD $0, -8(R15) // Zero saved LR in frame
367 SUB $8, R15
368 BL runtime·newstack(SB)
369
370 // Not reached, but make sure the return PC from the call to newstack
371 // is still in this function, and not the beginning of the next.
372 UNDEF
373
374TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
375 // Force SPWRITE. This function doesn't actually write SP,
376 // but it is called with a special calling convention where
377 // the caller doesn't save LR on stack but passes it as a
378 // register (R5), and the unwinder currently doesn't understand.
379 // Make it SPWRITE to stop unwinding. (See issue 54332)
380 MOVD R15, R15
381
382 MOVD $0, R12
383 BR runtime·morestack(SB)
384
385// reflectcall: call a function with the given argument list
386// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
387// we don't have variable-sized frames, so we use a small number
388// of constant-sized-frame functions to encode a few bits of size in the pc.
389// Caution: ugly multiline assembly macros in your future!
390
391#define DISPATCH(NAME,MAXSIZE) \
392 MOVD $MAXSIZE, R4; \
393 CMP R3, R4; \
394 BGT 3(PC); \
395 MOVD $NAME(SB), R5; \
396 BR (R5)
397// Note: can't just "BR NAME(SB)" - bad inlining results.
398
399TEXT ·reflectcall(SB), NOSPLIT, $-8-48
400 MOVWZ frameSize+32(FP), R3
401 DISPATCH(runtime·call16, 16)
402 DISPATCH(runtime·call32, 32)
403 DISPATCH(runtime·call64, 64)
404 DISPATCH(runtime·call128, 128)
405 DISPATCH(runtime·call256, 256)
406 DISPATCH(runtime·call512, 512)
407 DISPATCH(runtime·call1024, 1024)
408 DISPATCH(runtime·call2048, 2048)
409 DISPATCH(runtime·call4096, 4096)
410 DISPATCH(runtime·call8192, 8192)
411 DISPATCH(runtime·call16384, 16384)
412 DISPATCH(runtime·call32768, 32768)
413 DISPATCH(runtime·call65536, 65536)
414 DISPATCH(runtime·call131072, 131072)
415 DISPATCH(runtime·call262144, 262144)
416 DISPATCH(runtime·call524288, 524288)
417 DISPATCH(runtime·call1048576, 1048576)
418 DISPATCH(runtime·call2097152, 2097152)
419 DISPATCH(runtime·call4194304, 4194304)
420 DISPATCH(runtime·call8388608, 8388608)
421 DISPATCH(runtime·call16777216, 16777216)
422 DISPATCH(runtime·call33554432, 33554432)
423 DISPATCH(runtime·call67108864, 67108864)
424 DISPATCH(runtime·call134217728, 134217728)
425 DISPATCH(runtime·call268435456, 268435456)
426 DISPATCH(runtime·call536870912, 536870912)
427 DISPATCH(runtime·call1073741824, 1073741824)
428 MOVD $runtime·badreflectcall(SB), R5
429 BR (R5)
430
431#define CALLFN(NAME,MAXSIZE) \
432TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
433 NO_LOCAL_POINTERS; \
434 /* copy arguments to stack */ \
435 MOVD stackArgs+16(FP), R4; \
436 MOVWZ stackArgsSize+24(FP), R5; \
437 MOVD $stack-MAXSIZE(SP), R6; \
438loopArgs: /* copy 256 bytes at a time */ \
439 CMP R5, $256; \
440 BLT tailArgs; \
441 SUB $256, R5; \
442 MVC $256, 0(R4), 0(R6); \
443 MOVD $256(R4), R4; \
444 MOVD $256(R6), R6; \
445 BR loopArgs; \
446tailArgs: /* copy remaining bytes */ \
447 CMP R5, $0; \
448 BEQ callFunction; \
449 SUB $1, R5; \
450 EXRL $callfnMVC<>(SB), R5; \
451callFunction: \
452 MOVD f+8(FP), R12; \
453 MOVD regArgs+40(FP), R10; \
454 BL ·unspillArgs(SB); \
455 MOVD (R12), R10; \
456 PCDATA $PCDATA_StackMapIndex, $0; \
457 BL (R10); \
458 /* copy return values back */ \
459 MOVD regArgs+40(FP), R10; \
460 BL ·spillArgs(SB); \
461 MOVD stackArgsType+0(FP), R7; \
462 MOVD stackArgs+16(FP), R6; \
463 MOVWZ stackArgsSize+24(FP), R5; \
464 MOVD $stack-MAXSIZE(SP), R4; \
465 MOVWZ stackRetOffset+28(FP), R1; \
466 ADD R1, R4; \
467 ADD R1, R6; \
468 SUB R1, R5; \
469 BL callRet<>(SB); \
470 RET
471
472// callRet copies return values back at the end of call*. This is a
473// separate function so it can allocate stack space for the arguments
474// to reflectcallmove. It does not follow the Go ABI; it expects its
475// arguments in registers.
476TEXT callRet<>(SB), NOSPLIT, $40-0
477 NO_LOCAL_POINTERS;
478 MOVD R7, 8(R15)
479 MOVD R6, 16(R15)
480 MOVD R4, 24(R15)
481 MOVD R5, 32(R15)
482 MOVD R10, 40(R15)
483 BL runtime·reflectcallmove(SB)
484 RET
485
486CALLFN(·call16, 16)
487CALLFN(·call32, 32)
488CALLFN(·call64, 64)
489CALLFN(·call128, 128)
490CALLFN(·call256, 256)
491CALLFN(·call512, 512)
492CALLFN(·call1024, 1024)
493CALLFN(·call2048, 2048)
494CALLFN(·call4096, 4096)
495CALLFN(·call8192, 8192)
496CALLFN(·call16384, 16384)
497CALLFN(·call32768, 32768)
498CALLFN(·call65536, 65536)
499CALLFN(·call131072, 131072)
500CALLFN(·call262144, 262144)
501CALLFN(·call524288, 524288)
502CALLFN(·call1048576, 1048576)
503CALLFN(·call2097152, 2097152)
504CALLFN(·call4194304, 4194304)
505CALLFN(·call8388608, 8388608)
506CALLFN(·call16777216, 16777216)
507CALLFN(·call33554432, 33554432)
508CALLFN(·call67108864, 67108864)
509CALLFN(·call134217728, 134217728)
510CALLFN(·call268435456, 268435456)
511CALLFN(·call536870912, 536870912)
512CALLFN(·call1073741824, 1073741824)
513
514// Not a function: target for EXRL (execute relative long) instruction.
515TEXT callfnMVC<>(SB),NOSPLIT|NOFRAME,$0-0
516 MVC $1, 0(R4), 0(R6)
517
518TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0
519 RET
520
521// Save state of caller into g->sched,
522// but using fake PC from systemstack_switch.
523// Must only be called from functions with no locals ($0)
524// or else unwinding from systemstack_switch is incorrect.
525// Smashes R1.
526TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
527 MOVD $runtime·systemstack_switch(SB), R1
528 ADD $16, R1 // get past prologue
529 MOVD R1, (g_sched+gobuf_pc)(g)
530 MOVD R15, (g_sched+gobuf_sp)(g)
531 MOVD $0, (g_sched+gobuf_lr)(g)
532 // Assert ctxt is zero. See func save.
533 MOVD (g_sched+gobuf_ctxt)(g), R1
534 CMPBEQ R1, $0, 2(PC)
535 BL runtime·abort(SB)
536 RET
537
538// func asmcgocall(fn, arg unsafe.Pointer) int32
539// Call fn(arg) on the scheduler stack,
540// aligned appropriately for the gcc ABI.
541// See cgocall.go for more details.
542TEXT ·asmcgocall(SB),NOSPLIT,$0-20
543 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer
544 // C TLS base pointer in AR0:AR1
545 MOVD fn+0(FP), R3
546 MOVD arg+8(FP), R4
547
548 MOVD R15, R2 // save original stack pointer
549 MOVD g, R5
550
551 // Figure out if we need to switch to m->g0 stack.
552 // We get called to create new OS threads too, and those
553 // come in on the m->g0 stack already. Or we might already
554 // be on the m->gsignal stack.
555 MOVD g_m(g), R6
556 MOVD m_gsignal(R6), R7
557 CMPBEQ R7, g, g0
558 MOVD m_g0(R6), R7
559 CMPBEQ R7, g, g0
560 BL gosave_systemstack_switch<>(SB)
561 MOVD R7, g
562 BL runtime·save_g(SB)
563 MOVD (g_sched+gobuf_sp)(g), R15
564
565 // Now on a scheduling stack (a pthread-created stack).
566g0:
567 // Save room for two of our pointers, plus 160 bytes of callee
568 // save area that lives on the caller stack.
569 SUB $176, R15
570 MOVD $~7, R6
571 AND R6, R15 // 8-byte alignment for gcc ABI
572 MOVD R5, 168(R15) // save old g on stack
573 MOVD (g_stack+stack_hi)(R5), R5
574 SUB R2, R5
575 MOVD R5, 160(R15) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
576 MOVD $0, 0(R15) // clear back chain pointer (TODO can we give it real back trace information?)
577 MOVD R4, R2 // arg in R2
578 BL R3 // can clobber: R0-R5, R14, F0-F3, F5, F7-F15
579
580 XOR R0, R0 // set R0 back to 0.
581 // Restore g, stack pointer.
582 MOVD 168(R15), g
583 BL runtime·save_g(SB)
584 MOVD (g_stack+stack_hi)(g), R5
585 MOVD 160(R15), R6
586 SUB R6, R5
587 MOVD R5, R15
588
589 MOVW R2, ret+16(FP)
590 RET
591
592// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
593// See cgocall.go for more details.
594TEXT ·cgocallback(SB),NOSPLIT,$24-24
595 NO_LOCAL_POINTERS
596
597 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
598 // It is used to dropm while thread is exiting.
599 MOVD fn+0(FP), R1
600 CMPBNE R1, $0, loadg
601 // Restore the g from frame.
602 MOVD frame+8(FP), g
603 BR dropm
604
605loadg:
606 // Load m and g from thread-local storage.
607 MOVB runtime·iscgo(SB), R3
608 CMPBEQ R3, $0, nocgo
609 BL runtime·load_g(SB)
610
611nocgo:
612 // If g is nil, Go did not create the current thread,
613 // or if this thread never called into Go on pthread platforms.
614 // Call needm to obtain one for temporary use.
615 // In this case, we're running on the thread stack, so there's
616 // lots of space, but the linker doesn't know. Hide the call from
617 // the linker analysis by using an indirect call.
618 CMPBEQ g, $0, needm
619
620 MOVD g_m(g), R8
621 MOVD R8, savedm-8(SP)
622 BR havem
623
624needm:
625 MOVD g, savedm-8(SP) // g is zero, so is m.
626 MOVD $runtime·needAndBindM(SB), R3
627 BL (R3)
628
629 // Set m->sched.sp = SP, so that if a panic happens
630 // during the function we are about to execute, it will
631 // have a valid SP to run on the g0 stack.
632 // The next few lines (after the havem label)
633 // will save this SP onto the stack and then write
634 // the same SP back to m->sched.sp. That seems redundant,
635 // but if an unrecovered panic happens, unwindm will
636 // restore the g->sched.sp from the stack location
637 // and then systemstack will try to use it. If we don't set it here,
638 // that restored SP will be uninitialized (typically 0) and
639 // will not be usable.
640 MOVD g_m(g), R8
641 MOVD m_g0(R8), R3
642 MOVD R15, (g_sched+gobuf_sp)(R3)
643
644havem:
645 // Now there's a valid m, and we're running on its m->g0.
646 // Save current m->g0->sched.sp on stack and then set it to SP.
647 // Save current sp in m->g0->sched.sp in preparation for
648 // switch back to m->curg stack.
649 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
650 MOVD m_g0(R8), R3
651 MOVD (g_sched+gobuf_sp)(R3), R4
652 MOVD R4, savedsp-24(SP) // must match frame size
653 MOVD R15, (g_sched+gobuf_sp)(R3)
654
655 // Switch to m->curg stack and call runtime.cgocallbackg.
656 // Because we are taking over the execution of m->curg
657 // but *not* resuming what had been running, we need to
658 // save that information (m->curg->sched) so we can restore it.
659 // We can restore m->curg->sched.sp easily, because calling
660 // runtime.cgocallbackg leaves SP unchanged upon return.
661 // To save m->curg->sched.pc, we push it onto the curg stack and
662 // open a frame the same size as cgocallback's g0 frame.
663 // Once we switch to the curg stack, the pushed PC will appear
664 // to be the return PC of cgocallback, so that the traceback
665 // will seamlessly trace back into the earlier calls.
666 MOVD m_curg(R8), g
667 BL runtime·save_g(SB)
668 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
669 MOVD (g_sched+gobuf_pc)(g), R5
670 MOVD R5, -(24+8)(R4) // "saved LR"; must match frame size
671 // Gather our arguments into registers.
672 MOVD fn+0(FP), R1
673 MOVD frame+8(FP), R2
674 MOVD ctxt+16(FP), R3
675 MOVD $-(24+8)(R4), R15 // switch stack; must match frame size
676 MOVD R1, 8(R15)
677 MOVD R2, 16(R15)
678 MOVD R3, 24(R15)
679 BL runtime·cgocallbackg(SB)
680
681 // Restore g->sched (== m->curg->sched) from saved values.
682 MOVD 0(R15), R5
683 MOVD R5, (g_sched+gobuf_pc)(g)
684 MOVD $(24+8)(R15), R4 // must match frame size
685 MOVD R4, (g_sched+gobuf_sp)(g)
686
687 // Switch back to m->g0's stack and restore m->g0->sched.sp.
688 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
689 // so we do not have to restore it.)
690 MOVD g_m(g), R8
691 MOVD m_g0(R8), g
692 BL runtime·save_g(SB)
693 MOVD (g_sched+gobuf_sp)(g), R15
694 MOVD savedsp-24(SP), R4 // must match frame size
695 MOVD R4, (g_sched+gobuf_sp)(g)
696
697 // If the m on entry was nil, we called needm above to borrow an m,
698 // 1. for the duration of the call on non-pthread platforms,
699 // 2. or the duration of the C thread alive on pthread platforms.
700 // If the m on entry wasn't nil,
701 // 1. the thread might be a Go thread,
702 // 2. or it wasn't the first call from a C thread on pthread platforms,
703 // since then we skip dropm to reuse the m in the first call.
704 MOVD savedm-8(SP), R6
705 CMPBNE R6, $0, droppedm
706
707 // Skip dropm to reuse it in the next call, when a pthread key has been created.
708 MOVD _cgo_pthread_key_created(SB), R6
709 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
710 CMPBEQ R6, $0, dropm
711 MOVD (R6), R6
712 CMPBNE R6, $0, droppedm
713
714dropm:
715 MOVD $runtime·dropm(SB), R3
716 BL (R3)
717droppedm:
718
719 // Done!
720 RET
721
722// void setg(G*); set g. for use by needm.
723TEXT runtime·setg(SB), NOSPLIT, $0-8
724 MOVD gg+0(FP), g
725 // This only happens if iscgo, so jump straight to save_g
726 BL runtime·save_g(SB)
727 RET
728
729// void setg_gcc(G*); set g in C TLS.
730// Must obey the gcc calling convention.
731TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
732 // The standard prologue clobbers LR (R14), which is callee-save in
733 // the C ABI, so we have to use NOFRAME and save LR ourselves.
734 MOVD LR, R1
735 // Also save g, R10, and R11 since they're callee-save in C ABI
736 MOVD R10, R3
737 MOVD g, R4
738 MOVD R11, R5
739
740 MOVD R2, g
741 BL runtime·save_g(SB)
742
743 MOVD R5, R11
744 MOVD R4, g
745 MOVD R3, R10
746 MOVD R1, LR
747 RET
748
749TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
750 MOVW (R0), R0
751 UNDEF
752
753// int64 runtime·cputicks(void)
754TEXT runtime·cputicks(SB),NOSPLIT,$0-8
755 // The TOD clock on s390 counts from the year 1900 in ~250ps intervals.
756 // This means that since about 1972 the msb has been set, making the
757 // result of a call to STORE CLOCK (stck) a negative number.
758 // We clear the msb to make it positive.
759 STCK ret+0(FP) // serialises before and after call
760 MOVD ret+0(FP), R3 // R3 will wrap to 0 in the year 2043
761 SLD $1, R3
762 SRD $1, R3
763 MOVD R3, ret+0(FP)
764 RET
765
766#ifdef GOEXPERIMENT_regabiargs
767// spillArgs stores return values from registers to a *internal/abi.RegArgs in R10.
768TEXT runtime·spillArgs(SB),NOSPLIT,$0-0
769 MOVD R2, 0(R10)
770 MOVD R3, 8(R10)
771 MOVD R4, 16(R10)
772 MOVD R5, 24(R10)
773 MOVD R6, 32(R10)
774 MOVD R7, 40(R10)
775 MOVD R8, 48(R10)
776 MOVD R9, 56(R10)
777 FMOVD F0, 64(R10)
778 FMOVD F1, 72(R10)
779 FMOVD F2, 80(R10)
780 FMOVD F3, 88(R10)
781 FMOVD F4, 96(R10)
782 FMOVD F5, 104(R10)
783 FMOVD F6, 112(R10)
784 FMOVD F7, 120(R10)
785 FMOVD F8, 128(R10)
786 FMOVD F9, 136(R10)
787 FMOVD F10, 144(R10)
788 FMOVD F11, 152(R10)
789 FMOVD F12, 160(R10)
790 FMOVD F13, 168(R10)
791 FMOVD F14, 176(R10)
792 FMOVD F15, 184(R10)
793 RET
794
795// unspillArgs loads args into registers from a *internal/abi.RegArgs in R10.
796TEXT runtime·unspillArgs(SB),NOSPLIT,$0-0
797 MOVD 0(R10), R2
798 MOVD 8(R10), R3
799 MOVD 16(R10), R4
800 MOVD 24(R10), R5
801 MOVD 32(R10), R6
802 MOVD 40(R10), R7
803 MOVD 48(R10), R8
804 MOVD 56(R10), R9
805 FMOVD 64(R10), F0
806 FMOVD 72(R10), F1
807 FMOVD 80(R10), F2
808 FMOVD 88(R10), F3
809 FMOVD 96(R10), F4
810 FMOVD 104(R10), F5
811 FMOVD 112(R10), F6
812 FMOVD 120(R10), F7
813 FMOVD 128(R10), F8
814 FMOVD 136(R10), F9
815 FMOVD 144(R10), F10
816 FMOVD 152(R10), F11
817 FMOVD 160(R10), F12
818 FMOVD 168(R10), F13
819 FMOVD 176(R10), F14
820 FMOVD 184(R10), F15
821 RET
822#else
823
824TEXT runtime·spillArgs(SB),NOSPLIT,$0-0
825 RET
826
827TEXT runtime·unspillArgs(SB),NOSPLIT,$0-0
828 RET
829#endif
830
831// AES hashing not implemented for s390x
832TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
833 JMP runtime·memhashFallback<ABIInternal>(SB)
834TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
835 JMP runtime·strhashFallback<ABIInternal>(SB)
836TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
837 JMP runtime·memhash32Fallback<ABIInternal>(SB)
838TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
839 JMP runtime·memhash64Fallback<ABIInternal>(SB)
840
841// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
842// Must obey the gcc calling convention.
843TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
844 // g (R13), R10, R11 and LR (R14) are callee-save in the C ABI, so save them
845 MOVD g, R1
846 MOVD R10, R3
847 MOVD LR, R4
848 MOVD R11, R5
849
850 BL runtime·load_g(SB) // clobbers g (R13), R10, R11
851 MOVD g_m(g), R2
852 MOVD m_curg(R2), R2
853 MOVD (g_stack+stack_hi)(R2), R2
854
855 MOVD R1, g
856 MOVD R3, R10
857 MOVD R4, LR
858 MOVD R5, R11
859 RET
860
861// The top-most function running on a goroutine
862// returns to goexit+PCQuantum.
863TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
864 BYTE $0x07; BYTE $0x00; // 2-byte nop
865 BL runtime·goexit1(SB) // does not return
866 // traceback from goexit1 must hit code range of goexit
867 BYTE $0x07; BYTE $0x00; // 2-byte nop
868
869TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
870 // Stores are already ordered on s390x, so this is just a
871 // compile barrier.
872 RET
873
874// This is called from .init_array and follows the platform, not Go, ABI.
875// We are overly conservative. We could only save the registers we use.
876// However, since this function is only called once per loaded module
877// performance is unimportant.
878TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
879 // Save R6-R15 in the register save area of the calling function.
880 // Don't bother saving F8-F15 as we aren't doing any calls.
881 STMG R6, R15, 48(R15)
882
883 // append the argument (passed in R2, as per the ELF ABI) to the
884 // moduledata linked list.
885 MOVD runtime·lastmoduledatap(SB), R1
886 MOVD R2, moduledata_next(R1)
887 MOVD R2, runtime·lastmoduledatap(SB)
888
889 // Restore R6-R15.
890 LMG 48(R15), R6, R15
891 RET
892
893TEXT ·checkASM(SB),NOSPLIT,$0-1
894 MOVB $1, ret+0(FP)
895 RET
896
897// gcWriteBarrier informs the GC about heap pointer writes.
898//
899// gcWriteBarrier does NOT follow the Go ABI. It accepts the
900// number of bytes of buffer needed in R9, and returns a pointer
901// to the buffer space in R9.
902// It clobbers R10 (the temp register) and R1 (used by PLT stub).
903// It does not clobber any other general-purpose registers,
904// but may clobber others (e.g., floating point registers).
905TEXT gcWriteBarrier<>(SB),NOSPLIT,$96
906 // Save the registers clobbered by the fast path.
907 MOVD R4, 96(R15)
908retry:
909 MOVD g_m(g), R1
910 MOVD m_p(R1), R1
911 // Increment wbBuf.next position.
912 MOVD R9, R4
913 ADD (p_wbBuf+wbBuf_next)(R1), R4
914 // Is the buffer full?
915 MOVD (p_wbBuf+wbBuf_end)(R1), R10
916 CMPUBGT R4, R10, flush
917 // Commit to the larger buffer.
918 MOVD R4, (p_wbBuf+wbBuf_next)(R1)
919 // Make return value (the original next position)
920 SUB R9, R4, R9
921 // Restore registers.
922 MOVD 96(R15), R4
923 RET
924
925flush:
926 // Save all general purpose registers since these could be
927 // clobbered by wbBufFlush and were not saved by the caller.
928 STMG R2, R3, 8(R15)
929 MOVD R0, 24(R15)
930 // R1 already saved.
931 // R4 already saved.
932 STMG R5, R12, 32(R15) // save R5 - R12
933 // R13 is g.
934 // R14 is LR.
935 // R15 is SP.
936
937 CALL runtime·wbBufFlush(SB)
938
939 LMG 8(R15), R2, R3 // restore R2 - R3
940 MOVD 24(R15), R0 // restore R0
941 LMG 32(R15), R5, R12 // restore R5 - R12
942 JMP retry
943
944TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
945 MOVD $8, R9
946 JMP gcWriteBarrier<>(SB)
947TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
948 MOVD $16, R9
949 JMP gcWriteBarrier<>(SB)
950TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
951 MOVD $24, R9
952 JMP gcWriteBarrier<>(SB)
953TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
954 MOVD $32, R9
955 JMP gcWriteBarrier<>(SB)
956TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
957 MOVD $40, R9
958 JMP gcWriteBarrier<>(SB)
959TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
960 MOVD $48, R9
961 JMP gcWriteBarrier<>(SB)
962TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
963 MOVD $56, R9
964 JMP gcWriteBarrier<>(SB)
965TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
966 MOVD $64, R9
967 JMP gcWriteBarrier<>(SB)
968
969TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$144-0
970 NO_LOCAL_POINTERS
971 // Save all 16 int registers that could have an index in them.
972 // They may be pointers, but if they are they are dead.
973 STMG R0, R12, 24(R15)
974 // Note that R10 @ 104 is not needed, it is an assembler temp
975 // skip R13 aka G @ 128
976 // skip R14 aka LR @ 136
977 // skip R15 aka SP @ 144
978
979 MOVD R14, R2 // PC immediately after call to panicBounds
980 ADD $24, R15, R3 // pointer to save area
981 CALL runtime·panicBounds64<ABIInternal>(SB)
982 RET
View as plain text