Text file
src/runtime/asm_ppc64x.s
Documentation: runtime
1// Copyright 2014 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5//go:build ppc64 || ppc64le
6
7#include "go_asm.h"
8#include "go_tls.h"
9#include "funcdata.h"
10#include "textflag.h"
11#include "asm_ppc64x.h"
12#include "cgo/abi_ppc64x.h"
13
14// This is called using the host ABI. argc and argv arguments
15// should be in R3 and R4 respectively.
16TEXT _rt0_ppc64x_lib(SB),NOSPLIT|NOFRAME,$0
17 // Start with standard C stack frame layout and linkage, allocate
18 // 16 bytes of argument space, save callee-save regs, and set R0 to $0.
19 // Allocate an extra 16 bytes to account for the larger fixed frame size
20 // of aix/elfv1 (48 vs 32) to ensure 16 bytes of parameter save space.
21 STACK_AND_SAVE_HOST_TO_GO_ABI(32)
22 // The above will not preserve R2 (TOC). Save it in case Go is
23 // compiled without a TOC pointer (e.g -buildmode=default).
24 MOVD R2, 24(R1)
25
26 MOVD R3, _rt0_ppc64x_lib_argc<>(SB)
27 MOVD R4, _rt0_ppc64x_lib_argv<>(SB)
28
29 // Synchronous initialization.
30 MOVD $runtime·reginit(SB), R12
31 MOVD R12, CTR
32 BL (CTR)
33
34#ifdef GOOS_aix
35 // See runtime/cgo/gcc_aix_ppc64.c
36 MOVBZ runtime·isarchive(SB), R3 // Check buildmode = c-archive
37 CMP $0, R3
38 BEQ done
39#endif
40
41 MOVD $runtime·libpreinit(SB), R12
42 MOVD R12, CTR
43 BL (CTR)
44
45 // Create a new thread to do the runtime initialization and return.
46 // _cgo_sys_thread_create is a C function.
47 MOVD _cgo_sys_thread_create(SB), R12
48 CMP $0, R12
49 BEQ nocgo
50 MOVD $_rt0_ppc64x_lib_go(SB), R3
51 MOVD $0, R4
52#ifdef GO_PPC64X_HAS_FUNCDESC
53 // Load the real entry address from the first slot of the function descriptor.
54 MOVD 8(R12), R2
55 MOVD (R12), R12
56#endif
57 MOVD R12, CTR
58 BL (CTR)
59 MOVD 24(R1), R2 // Restore the old frame, and R2.
60 BR done
61
62nocgo:
63 MOVD $0x800000, R12 // stacksize = 8192KB
64 MOVD R12, 8+FIXED_FRAME(R1)
65 MOVD $_rt0_ppc64x_lib_go(SB), R12
66 MOVD R12, 16+FIXED_FRAME(R1)
67 MOVD $runtime·newosproc0(SB),R12
68 MOVD R12, CTR
69 BL (CTR)
70
71done:
72 UNSTACK_AND_RESTORE_GO_TO_HOST_ABI(32)
73 RET
74
75#ifdef GO_PPC64X_HAS_FUNCDESC
76DEFINE_PPC64X_FUNCDESC(_rt0_ppc64x_lib_go, __rt0_ppc64x_lib_go)
77TEXT __rt0_ppc64x_lib_go(SB),NOSPLIT,$0
78#else
79TEXT _rt0_ppc64x_lib_go(SB),NOSPLIT,$0
80#endif
81 MOVD _rt0_ppc64x_lib_argc<>(SB), R3
82 MOVD _rt0_ppc64x_lib_argv<>(SB), R4
83 MOVD $runtime·rt0_go(SB), R12
84 MOVD R12, CTR
85 BR (CTR)
86
87DATA _rt0_ppc64x_lib_argc<>(SB)/8, $0
88GLOBL _rt0_ppc64x_lib_argc<>(SB),NOPTR, $8
89DATA _rt0_ppc64x_lib_argv<>(SB)/8, $0
90GLOBL _rt0_ppc64x_lib_argv<>(SB),NOPTR, $8
91
92
93#ifdef GOOS_aix
94#define cgoCalleeStackSize 48
95#else
96#define cgoCalleeStackSize 32
97#endif
98
99TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
100 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer
101
102 // initialize essential registers
103 BL runtime·reginit(SB)
104
105 SUB $(FIXED_FRAME+16), R1
106 MOVD R2, 24(R1) // stash the TOC pointer away again now we've created a new frame
107 MOVW R3, FIXED_FRAME+0(R1) // argc
108 MOVD R4, FIXED_FRAME+8(R1) // argv
109
110 // create istack out of the given (operating system) stack.
111 // _cgo_init may update stackguard.
112 MOVD $runtime·g0(SB), g
113 BL runtime·save_g(SB)
114 MOVD $(-64*1024), R31
115 ADD R31, R1, R3
116 MOVD R3, g_stackguard0(g)
117 MOVD R3, g_stackguard1(g)
118 MOVD R3, (g_stack+stack_lo)(g)
119 MOVD R1, (g_stack+stack_hi)(g)
120
121 // If there is a _cgo_init, call it using the gcc ABI.
122 MOVD _cgo_init(SB), R12
123 CMP R12, $0
124 BEQ nocgo
125
126#ifdef GO_PPC64X_HAS_FUNCDESC
127 // Load the real entry address from the first slot of the function descriptor.
128 MOVD 8(R12), R2
129 MOVD (R12), R12
130#endif
131 MOVD R12, CTR // r12 = "global function entry point"
132 MOVD R13, R5 // arg 2: TLS base pointer
133 MOVD $setg_gcc<>(SB), R4 // arg 1: setg
134 MOVD g, R3 // arg 0: G
135 // C functions expect 32 (48 for AIX) bytes of space on caller
136 // stack frame and a 16-byte aligned R1
137 MOVD R1, R14 // save current stack
138 SUB $cgoCalleeStackSize, R1 // reserve the callee area
139 RLDCR $0, R1, $~15, R1 // 16-byte align
140 BL (CTR) // may clobber R0, R3-R12
141 MOVD R14, R1 // restore stack
142#ifndef GOOS_aix
143 MOVD 24(R1), R2
144#endif
145 XOR R0, R0 // fix R0
146
147nocgo:
148 // update stackguard after _cgo_init
149 MOVD (g_stack+stack_lo)(g), R3
150 ADD $const_stackGuard, R3
151 MOVD R3, g_stackguard0(g)
152 MOVD R3, g_stackguard1(g)
153
154 // set the per-goroutine and per-mach "registers"
155 MOVD $runtime·m0(SB), R3
156
157 // save m->g0 = g0
158 MOVD g, m_g0(R3)
159 // save m0 to g0->m
160 MOVD R3, g_m(g)
161
162 BL runtime·check(SB)
163
164 // args are already prepared
165 BL runtime·args(SB)
166 BL runtime·osinit(SB)
167 BL runtime·schedinit(SB)
168
169 // create a new goroutine to start program
170 MOVD $runtime·mainPC(SB), R3 // entry
171 MOVDU R3, -8(R1)
172 MOVDU R0, -8(R1)
173 MOVDU R0, -8(R1)
174 MOVDU R0, -8(R1)
175 MOVDU R0, -8(R1)
176 BL runtime·newproc(SB)
177 ADD $(8+FIXED_FRAME), R1
178
179 // start this M
180 BL runtime·mstart(SB)
181 // Prevent dead-code elimination of debugCallV2 and debugPinnerV1, which are
182 // intended to be called by debuggers.
183#ifdef GOARCH_ppc64le
184 MOVD $runtime·debugPinnerV1<ABIInternal>(SB), R31
185 MOVD $runtime·debugCallV2<ABIInternal>(SB), R31
186#endif
187 MOVD R0, 0(R0)
188 RET
189
190DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
191GLOBL runtime·mainPC(SB),RODATA,$8
192
193TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
194 TW $31, R0, R0
195 RET
196
197TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
198 RET
199
200// Any changes must be reflected to runtime/cgo/gcc_aix_ppc64.S:.crosscall_ppc64
201TEXT _cgo_reginit(SB),NOSPLIT|NOFRAME,$0-0
202 // crosscall_ppc64 and crosscall2 need to reginit, but can't
203 // get at the 'runtime.reginit' symbol.
204 BR runtime·reginit(SB)
205
206TEXT runtime·reginit(SB),NOSPLIT|NOFRAME,$0-0
207 // set R0 to zero, it's expected by the toolchain
208 XOR R0, R0
209 RET
210
211TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
212 BL runtime·mstart0(SB)
213 RET // not reached
214
215/*
216 * go-routine
217 */
218
219// void gogo(Gobuf*)
220// restore state from Gobuf; longjmp
221TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
222 MOVD buf+0(FP), R5
223 MOVD gobuf_g(R5), R6
224 MOVD 0(R6), R4 // make sure g != nil
225 BR gogo<>(SB)
226
227TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
228 MOVD R6, g
229 BL runtime·save_g(SB)
230
231 MOVD gobuf_sp(R5), R1
232 MOVD gobuf_lr(R5), R31
233#ifndef GOOS_aix
234 MOVD 24(R1), R2 // restore R2
235#endif
236 MOVD R31, LR
237 MOVD gobuf_ctxt(R5), R11
238 MOVD R0, gobuf_sp(R5)
239 MOVD R0, gobuf_lr(R5)
240 MOVD R0, gobuf_ctxt(R5)
241 CMP R0, R0 // set condition codes for == test, needed by stack split
242 MOVD gobuf_pc(R5), R12
243 MOVD R12, CTR
244 BR (CTR)
245
246// void mcall(fn func(*g))
247// Switch to m->g0's stack, call fn(g).
248// Fn must never return. It should gogo(&g->sched)
249// to keep running g.
250TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
251 // Save caller state in g->sched
252 // R11 should be safe across save_g??
253 MOVD R3, R11
254 MOVD R1, (g_sched+gobuf_sp)(g)
255 MOVD LR, R31
256 MOVD R31, (g_sched+gobuf_pc)(g)
257 MOVD R0, (g_sched+gobuf_lr)(g)
258
259 // Switch to m->g0 & its stack, call fn.
260 MOVD g, R3
261 MOVD g_m(g), R8
262 MOVD m_g0(R8), g
263 BL runtime·save_g(SB)
264 CMP g, R3
265 BNE 2(PC)
266 BR runtime·badmcall(SB)
267 MOVD 0(R11), R12 // code pointer
268 MOVD R12, CTR
269 MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp
270 // Don't need to do anything special for regabiargs here
271 // R3 is g; stack is set anyway
272 MOVDU R3, -8(R1)
273 MOVDU R0, -8(R1)
274 MOVDU R0, -8(R1)
275 MOVDU R0, -8(R1)
276 MOVDU R0, -8(R1)
277 BL (CTR)
278 MOVD 24(R1), R2
279 BR runtime·badmcall2(SB)
280
281// systemstack_switch is a dummy routine that systemstack leaves at the bottom
282// of the G stack. We need to distinguish the routine that
283// lives at the bottom of the G stack from the one that lives
284// at the top of the system stack because the one at the top of
285// the system stack terminates the stack walk (see topofstack()).
286TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
287 // We have several undefs here so that 16 bytes past
288 // $runtime·systemstack_switch lies within them whether or not the
289 // instructions that derive r2 from r12 are there.
290 UNDEF
291 UNDEF
292 UNDEF
293 BL (LR) // make sure this function is not leaf
294 RET
295
296// func systemstack(fn func())
297TEXT runtime·systemstack(SB), NOSPLIT, $0-8
298 MOVD fn+0(FP), R3 // R3 = fn
299 MOVD R3, R11 // context
300 MOVD g_m(g), R4 // R4 = m
301
302 MOVD m_gsignal(R4), R5 // R5 = gsignal
303 CMP g, R5
304 BEQ noswitch
305
306 MOVD m_g0(R4), R5 // R5 = g0
307 CMP g, R5
308 BEQ noswitch
309
310 MOVD m_curg(R4), R6
311 CMP g, R6
312 BEQ switch
313
314 // Bad: g is not gsignal, not g0, not curg. What is it?
315 // Hide call from linker nosplit analysis.
316 MOVD $runtime·badsystemstack(SB), R12
317 MOVD R12, CTR
318 BL (CTR)
319 BL runtime·abort(SB)
320
321switch:
322 // save our state in g->sched. Pretend to
323 // be systemstack_switch if the G stack is scanned.
324 BL gosave_systemstack_switch<>(SB)
325
326 // switch to g0
327 MOVD R5, g
328 BL runtime·save_g(SB)
329 MOVD (g_sched+gobuf_sp)(g), R1
330
331 // call target function
332 MOVD 0(R11), R12 // code pointer
333 MOVD R12, CTR
334 BL (CTR)
335
336 // restore TOC pointer. It seems unlikely that we will use systemstack
337 // to call a function defined in another module, but the results of
338 // doing so would be so confusing that it's worth doing this.
339 MOVD g_m(g), R3
340 MOVD m_curg(R3), g
341 MOVD (g_sched+gobuf_sp)(g), R3
342#ifndef GOOS_aix
343 MOVD 24(R3), R2
344#endif
345 // switch back to g
346 MOVD g_m(g), R3
347 MOVD m_curg(R3), g
348 BL runtime·save_g(SB)
349 MOVD (g_sched+gobuf_sp)(g), R1
350 MOVD R0, (g_sched+gobuf_sp)(g)
351 RET
352
353noswitch:
354 // already on m stack, just call directly
355 // On other arches we do a tail call here, but it appears to be
356 // impossible to tail call a function pointer in shared mode on
357 // ppc64 because the caller is responsible for restoring the TOC.
358 MOVD 0(R11), R12 // code pointer
359 MOVD R12, CTR
360 BL (CTR)
361#ifndef GOOS_aix
362 MOVD 24(R1), R2
363#endif
364 RET
365
366// func switchToCrashStack0(fn func())
367TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
368 MOVD R3, R11 // context register
369 MOVD g_m(g), R3 // curm
370
371 // set g to gcrash
372 MOVD $runtime·gcrash(SB), g // g = &gcrash
373 CALL runtime·save_g(SB) // clobbers R31
374 MOVD R3, g_m(g) // g.m = curm
375 MOVD g, m_g0(R3) // curm.g0 = g
376
377 // switch to crashstack
378 MOVD (g_stack+stack_hi)(g), R3
379 SUB $(4*8), R3
380 MOVD R3, R1
381
382 // call target function
383 MOVD 0(R11), R12 // code pointer
384 MOVD R12, CTR
385 BL (CTR)
386
387 // should never return
388 CALL runtime·abort(SB)
389 UNDEF
390
391/*
392 * support for morestack
393 */
394
395// Called during function prolog when more stack is needed.
396// Caller has already loaded:
397// R3: framesize, R4: argsize, R5: LR
398//
399// The traceback routines see morestack on a g0 as being
400// the top of a stack (for example, morestack calling newstack
401// calling the scheduler calling newm calling gc), so we must
402// record an argument size. For that purpose, it has no arguments.
403TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
404 // Called from f.
405 // Set g->sched to context in f.
406 MOVD R1, (g_sched+gobuf_sp)(g)
407 MOVD LR, R8
408 MOVD R8, (g_sched+gobuf_pc)(g)
409 MOVD R5, (g_sched+gobuf_lr)(g)
410 MOVD R11, (g_sched+gobuf_ctxt)(g)
411
412 // Cannot grow scheduler stack (m->g0).
413 MOVD g_m(g), R7
414 MOVD m_g0(R7), R8
415 CMP g, R8
416 BNE 3(PC)
417 BL runtime·badmorestackg0(SB)
418 BL runtime·abort(SB)
419
420 // Cannot grow signal stack (m->gsignal).
421 MOVD m_gsignal(R7), R8
422 CMP g, R8
423 BNE 3(PC)
424 BL runtime·badmorestackgsignal(SB)
425 BL runtime·abort(SB)
426
427 // Called from f.
428 // Set m->morebuf to f's caller.
429 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
430 MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
431 MOVD g, (m_morebuf+gobuf_g)(R7)
432
433 // Call newstack on m->g0's stack.
434 MOVD m_g0(R7), g
435 BL runtime·save_g(SB)
436 MOVD (g_sched+gobuf_sp)(g), R1
437 MOVDU R0, -(FIXED_FRAME+0)(R1) // create a call frame on g0
438 BL runtime·newstack(SB)
439
440 // Not reached, but make sure the return PC from the call to newstack
441 // is still in this function, and not the beginning of the next.
442 UNDEF
443
444TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
445 // Force SPWRITE. This function doesn't actually write SP,
446 // but it is called with a special calling convention where
447 // the caller doesn't save LR on stack but passes it as a
448 // register (R5), and the unwinder currently doesn't understand.
449 // Make it SPWRITE to stop unwinding. (See issue 54332)
450 // Use OR R0, R1 instead of MOVD R1, R1 as the MOVD instruction
451 // has a special affect on Power8,9,10 by lowering the thread
452 // priority and causing a slowdown in execution time
453
454 OR R0, R1
455 MOVD R0, R11
456 BR runtime·morestack(SB)
457
458// reflectcall: call a function with the given argument list
459// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
460// we don't have variable-sized frames, so we use a small number
461// of constant-sized-frame functions to encode a few bits of size in the pc.
462// Caution: ugly multiline assembly macros in your future!
463
464#define DISPATCH(NAME,MAXSIZE) \
465 MOVD $MAXSIZE, R31; \
466 CMP R3, R31; \
467 BGT 4(PC); \
468 MOVD $NAME(SB), R12; \
469 MOVD R12, CTR; \
470 BR (CTR)
471// Note: can't just "BR NAME(SB)" - bad inlining results.
472
473TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
474 MOVWZ frameSize+32(FP), R3
475 DISPATCH(runtime·call16, 16)
476 DISPATCH(runtime·call32, 32)
477 DISPATCH(runtime·call64, 64)
478 DISPATCH(runtime·call128, 128)
479 DISPATCH(runtime·call256, 256)
480 DISPATCH(runtime·call512, 512)
481 DISPATCH(runtime·call1024, 1024)
482 DISPATCH(runtime·call2048, 2048)
483 DISPATCH(runtime·call4096, 4096)
484 DISPATCH(runtime·call8192, 8192)
485 DISPATCH(runtime·call16384, 16384)
486 DISPATCH(runtime·call32768, 32768)
487 DISPATCH(runtime·call65536, 65536)
488 DISPATCH(runtime·call131072, 131072)
489 DISPATCH(runtime·call262144, 262144)
490 DISPATCH(runtime·call524288, 524288)
491 DISPATCH(runtime·call1048576, 1048576)
492 DISPATCH(runtime·call2097152, 2097152)
493 DISPATCH(runtime·call4194304, 4194304)
494 DISPATCH(runtime·call8388608, 8388608)
495 DISPATCH(runtime·call16777216, 16777216)
496 DISPATCH(runtime·call33554432, 33554432)
497 DISPATCH(runtime·call67108864, 67108864)
498 DISPATCH(runtime·call134217728, 134217728)
499 DISPATCH(runtime·call268435456, 268435456)
500 DISPATCH(runtime·call536870912, 536870912)
501 DISPATCH(runtime·call1073741824, 1073741824)
502 MOVD $runtime·badreflectcall(SB), R12
503 MOVD R12, CTR
504 BR (CTR)
505
506#define CALLFN(NAME,MAXSIZE) \
507TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
508 NO_LOCAL_POINTERS; \
509 /* copy arguments to stack */ \
510 MOVD stackArgs+16(FP), R3; \
511 MOVWZ stackArgsSize+24(FP), R4; \
512 MOVD R1, R5; \
513 CMP R4, $8; \
514 BLT tailsetup; \
515 /* copy 8 at a time if possible */ \
516 ADD $(FIXED_FRAME-8), R5; \
517 SUB $8, R3; \
518top: \
519 MOVDU 8(R3), R7; \
520 MOVDU R7, 8(R5); \
521 SUB $8, R4; \
522 CMP R4, $8; \
523 BGE top; \
524 /* handle remaining bytes */ \
525 CMP $0, R4; \
526 BEQ callfn; \
527 ADD $7, R3; \
528 ADD $7, R5; \
529 BR tail; \
530tailsetup: \
531 CMP $0, R4; \
532 BEQ callfn; \
533 ADD $(FIXED_FRAME-1), R5; \
534 SUB $1, R3; \
535tail: \
536 MOVBU 1(R3), R6; \
537 MOVBU R6, 1(R5); \
538 SUB $1, R4; \
539 CMP $0, R4; \
540 BGT tail; \
541callfn: \
542 /* call function */ \
543 MOVD f+8(FP), R11; \
544#ifdef GOOS_aix \
545 /* AIX won't trigger a SIGSEGV if R11 = nil */ \
546 /* So it manually triggers it */ \
547 CMP R11, $0 \
548 BNE 2(PC) \
549 MOVD R0, 0(R0) \
550#endif \
551 MOVD regArgs+40(FP), R20; \
552 BL runtime·unspillArgs(SB); \
553 MOVD (R11), R12; \
554 MOVD R12, CTR; \
555 PCDATA $PCDATA_StackMapIndex, $0; \
556 BL (CTR); \
557#ifndef GOOS_aix \
558 MOVD 24(R1), R2; \
559#endif \
560 /* copy return values back */ \
561 MOVD regArgs+40(FP), R20; \
562 BL runtime·spillArgs(SB); \
563 MOVD stackArgsType+0(FP), R7; \
564 MOVD stackArgs+16(FP), R3; \
565 MOVWZ stackArgsSize+24(FP), R4; \
566 MOVWZ stackRetOffset+28(FP), R6; \
567 ADD $FIXED_FRAME, R1, R5; \
568 ADD R6, R5; \
569 ADD R6, R3; \
570 SUB R6, R4; \
571 BL callRet<>(SB); \
572 RET
573
574// callRet copies return values back at the end of call*. This is a
575// separate function so it can allocate stack space for the arguments
576// to reflectcallmove. It does not follow the Go ABI; it expects its
577// arguments in registers.
578TEXT callRet<>(SB), NOSPLIT, $40-0
579 NO_LOCAL_POINTERS
580 MOVD R7, FIXED_FRAME+0(R1)
581 MOVD R3, FIXED_FRAME+8(R1)
582 MOVD R5, FIXED_FRAME+16(R1)
583 MOVD R4, FIXED_FRAME+24(R1)
584 MOVD R20, FIXED_FRAME+32(R1)
585 BL runtime·reflectcallmove(SB)
586 RET
587
588CALLFN(·call16, 16)
589CALLFN(·call32, 32)
590CALLFN(·call64, 64)
591CALLFN(·call128, 128)
592CALLFN(·call256, 256)
593CALLFN(·call512, 512)
594CALLFN(·call1024, 1024)
595CALLFN(·call2048, 2048)
596CALLFN(·call4096, 4096)
597CALLFN(·call8192, 8192)
598CALLFN(·call16384, 16384)
599CALLFN(·call32768, 32768)
600CALLFN(·call65536, 65536)
601CALLFN(·call131072, 131072)
602CALLFN(·call262144, 262144)
603CALLFN(·call524288, 524288)
604CALLFN(·call1048576, 1048576)
605CALLFN(·call2097152, 2097152)
606CALLFN(·call4194304, 4194304)
607CALLFN(·call8388608, 8388608)
608CALLFN(·call16777216, 16777216)
609CALLFN(·call33554432, 33554432)
610CALLFN(·call67108864, 67108864)
611CALLFN(·call134217728, 134217728)
612CALLFN(·call268435456, 268435456)
613CALLFN(·call536870912, 536870912)
614CALLFN(·call1073741824, 1073741824)
615
616TEXT runtime·procyieldAsm(SB),NOSPLIT|NOFRAME,$0-4
617 MOVW cycles+0(FP), R7
618 CMP $0, R7
619 BEQ done
620 // POWER does not have a pause/yield instruction equivalent.
621 // Instead, we can lower the program priority by setting the
622 // Program Priority Register prior to the wait loop and set it
623 // back to default afterwards. On Linux, the default priority is
624 // medium-low. For details, see page 837 of the ISA 3.0.
625 OR R1, R1, R1 // Set PPR priority to low
626again:
627 SUB $1, R7
628 CMP $0, R7
629 BNE again
630 OR R6, R6, R6 // Set PPR priority back to medium-low
631done:
632 RET
633
634// Save state of caller into g->sched,
635// but using fake PC from systemstack_switch.
636// Must only be called from functions with no locals ($0)
637// or else unwinding from systemstack_switch is incorrect.
638// Smashes R31.
639TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
640 MOVD $runtime·systemstack_switch(SB), R31
641 ADD $16, R31 // get past prologue (including r2-setting instructions when they're there)
642 MOVD R31, (g_sched+gobuf_pc)(g)
643 MOVD R1, (g_sched+gobuf_sp)(g)
644 MOVD R0, (g_sched+gobuf_lr)(g)
645 // Assert ctxt is zero. See func save.
646 MOVD (g_sched+gobuf_ctxt)(g), R31
647 CMP R31, $0
648 BEQ 2(PC)
649 BL runtime·abort(SB)
650 RET
651
652#ifdef GOOS_aix
653#define asmcgocallSaveOffset cgoCalleeStackSize + 8
654#else
655#define asmcgocallSaveOffset cgoCalleeStackSize
656#endif
657
658// func asmcgocall_no_g(fn, arg unsafe.Pointer)
659// Call fn(arg) aligned appropriately for the gcc ABI.
660// Called on a system stack, and there may be no g yet (during needm).
661TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
662 MOVD fn+0(FP), R3
663 MOVD arg+8(FP), R4
664
665 MOVD R1, R15
666 SUB $(asmcgocallSaveOffset+8), R1
667 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI
668 MOVD R15, asmcgocallSaveOffset(R1)
669
670 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
671
672 // This is a "global call", so put the global entry point in r12
673 MOVD R3, R12
674
675#ifdef GO_PPC64X_HAS_FUNCDESC
676 // Load the real entry address from the first slot of the function descriptor.
677 MOVD 8(R12), R2
678 MOVD (R12), R12
679#endif
680 MOVD R12, CTR
681 MOVD R4, R3 // arg in r3
682 BL (CTR)
683
684 // C code can clobber R0, so set it back to 0. F27-F31 are
685 // callee save, so we don't need to recover those.
686 XOR R0, R0
687
688 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
689#ifndef GOOS_aix
690 MOVD 24(R1), R2
691#endif
692
693 RET
694
695// func asmcgocall(fn, arg unsafe.Pointer) int32
696// Call fn(arg) on the scheduler stack,
697// aligned appropriately for the gcc ABI.
698// See cgocall.go for more details.
699TEXT ·asmcgocall<ABIInternal>(SB),NOSPLIT,$0-20
700 // R3 = fn
701 // R4 = arg
702
703 MOVD R1, R7 // save original stack pointer
704 CMP $0, g
705 BEQ nosave
706 MOVD g, R5
707
708 // Figure out if we need to switch to m->g0 stack.
709 // We get called to create new OS threads too, and those
710 // come in on the m->g0 stack already. Or we might already
711 // be on the m->gsignal stack.
712 MOVD g_m(g), R8
713 MOVD m_gsignal(R8), R6
714 CMP R6, g
715 BEQ nosave
716 MOVD m_g0(R8), R6
717 CMP R6, g
718 BEQ nosave
719
720 BL gosave_systemstack_switch<>(SB)
721 MOVD R6, g
722 BL runtime·save_g(SB)
723 MOVD (g_sched+gobuf_sp)(g), R1
724
725 // Now on a scheduling stack (a pthread-created stack).
726#ifdef GOOS_aix
727 // Create a fake LR to improve backtrace.
728 MOVD $runtime·asmcgocall(SB), R6
729 MOVD R6, 16(R1)
730 // AIX also saves one argument on the stack.
731 SUB $8, R1
732#endif
733 // Save room for two of our pointers, plus the callee
734 // save area that lives on the caller stack.
735 // Do arithmetics in R10 to hide from the assembler
736 // counting it as SP delta, which is irrelevant as we are
737 // on the system stack.
738 SUB $(asmcgocallSaveOffset+16), R1, R10
739 RLDCR $0, R10, $~15, R1 // 16-byte alignment for gcc ABI
740 MOVD R5, (asmcgocallSaveOffset+8)(R1) // save old g on stack
741 MOVD (g_stack+stack_hi)(R5), R5
742 SUB R7, R5
743 MOVD R5, asmcgocallSaveOffset(R1) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
744#ifdef GOOS_aix
745 MOVD R7, 0(R1) // Save frame pointer to allow manual backtrace with gdb
746#else
747 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
748#endif
749 // This is a "global call", so put the global entry point in r12
750 MOVD R3, R12
751
752#ifdef GO_PPC64X_HAS_FUNCDESC
753 // Load the real entry address from the first slot of the function descriptor.
754 MOVD 8(R12), R2
755 MOVD (R12), R12
756#endif
757 MOVD R12, CTR
758 MOVD R4, R3 // arg in r3
759 BL (CTR)
760
761 // Reinitialise zero value register.
762 XOR R0, R0
763
764 // Restore g, stack pointer, toc pointer.
765 // R3 is errno, so don't touch it
766 MOVD (asmcgocallSaveOffset+8)(R1), g
767 MOVD (g_stack+stack_hi)(g), R5
768 MOVD asmcgocallSaveOffset(R1), R6
769 SUB R6, R5
770#ifndef GOOS_aix
771 MOVD 24(R5), R2
772#endif
773 MOVD R5, R1
774 BL runtime·save_g(SB)
775
776 // ret = R3
777 RET
778
779nosave:
780 // Running on a system stack, perhaps even without a g.
781 // Having no g can happen during thread creation or thread teardown.
782 // This code is like the above sequence but without saving/restoring g
783 // and without worrying about the stack moving out from under us
784 // (because we're on a system stack, not a goroutine stack).
785 // The above code could be used directly if already on a system stack,
786 // but then the only path through this code would be a rare case.
787 // Using this code for all "already on system stack" calls exercises it more,
788 // which should help keep it correct.
789
790 SUB $(asmcgocallSaveOffset+8), R1, R10
791 RLDCR $0, R10, $~15, R1 // 16-byte alignment for gcc ABI
792 MOVD R7, asmcgocallSaveOffset(R1) // Save original stack pointer.
793
794 MOVD R3, R12 // fn
795#ifdef GO_PPC64X_HAS_FUNCDESC
796 // Load the real entry address from the first slot of the function descriptor.
797 MOVD 8(R12), R2
798 MOVD (R12), R12
799#endif
800 MOVD R12, CTR
801 MOVD R4, R3 // arg
802 BL (CTR)
803
804 // Reinitialise zero value register.
805 XOR R0, R0
806
807 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
808#ifndef GOOS_aix
809 MOVD 24(R1), R2
810#endif
811 // ret = R3
812 RET
813
814// func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
815// See cgocall.go for more details.
816TEXT ·cgocallback(SB),NOSPLIT,$24-24
817 NO_LOCAL_POINTERS
818
819 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
820 // It is used to dropm while thread is exiting.
821 MOVD fn+0(FP), R5
822 CMP R5, $0
823 BNE loadg
824 // Restore the g from frame.
825 MOVD frame+8(FP), g
826 BR dropm
827
828loadg:
829 // Load m and g from thread-local storage.
830#ifndef GOOS_openbsd
831 MOVBZ runtime·iscgo(SB), R3
832 CMP R3, $0
833 BEQ nocgo
834#endif
835 BL runtime·load_g(SB)
836nocgo:
837
838 // If g is nil, Go did not create the current thread,
839 // or if this thread never called into Go on pthread platforms.
840 // Call needm to obtain one for temporary use.
841 // In this case, we're running on the thread stack, so there's
842 // lots of space, but the linker doesn't know. Hide the call from
843 // the linker analysis by using an indirect call.
844 CMP g, $0
845 BEQ needm
846
847 MOVD g_m(g), R8
848 MOVD R8, savedm-8(SP)
849 BR havem
850
851needm:
852 MOVD g, savedm-8(SP) // g is zero, so is m.
853 MOVD $runtime·needAndBindM(SB), R12
854 MOVD R12, CTR
855 BL (CTR)
856
857 // Set m->sched.sp = SP, so that if a panic happens
858 // during the function we are about to execute, it will
859 // have a valid SP to run on the g0 stack.
860 // The next few lines (after the havem label)
861 // will save this SP onto the stack and then write
862 // the same SP back to m->sched.sp. That seems redundant,
863 // but if an unrecovered panic happens, unwindm will
864 // restore the g->sched.sp from the stack location
865 // and then systemstack will try to use it. If we don't set it here,
866 // that restored SP will be uninitialized (typically 0) and
867 // will not be usable.
868 MOVD g_m(g), R8
869 MOVD m_g0(R8), R3
870 MOVD R1, (g_sched+gobuf_sp)(R3)
871
872havem:
873 // Now there's a valid m, and we're running on its m->g0.
874 // Save current m->g0->sched.sp on stack and then set it to SP.
875 // Save current sp in m->g0->sched.sp in preparation for
876 // switch back to m->curg stack.
877 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
878 MOVD m_g0(R8), R3
879 MOVD (g_sched+gobuf_sp)(R3), R4
880 MOVD R4, savedsp-24(SP) // must match frame size
881 MOVD R1, (g_sched+gobuf_sp)(R3)
882
883 // Switch to m->curg stack and call runtime.cgocallbackg.
884 // Because we are taking over the execution of m->curg
885 // but *not* resuming what had been running, we need to
886 // save that information (m->curg->sched) so we can restore it.
887 // We can restore m->curg->sched.sp easily, because calling
888 // runtime.cgocallbackg leaves SP unchanged upon return.
889 // To save m->curg->sched.pc, we push it onto the curg stack and
890 // open a frame the same size as cgocallback's g0 frame.
891 // Once we switch to the curg stack, the pushed PC will appear
892 // to be the return PC of cgocallback, so that the traceback
893 // will seamlessly trace back into the earlier calls.
894 MOVD m_curg(R8), g
895 BL runtime·save_g(SB)
896 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
897 MOVD (g_sched+gobuf_pc)(g), R5
898 MOVD R5, -(24+FIXED_FRAME)(R4) // "saved LR"; must match frame size
899 // Gather our arguments into registers.
900 MOVD fn+0(FP), R5
901 MOVD frame+8(FP), R6
902 MOVD ctxt+16(FP), R7
903 MOVD $-(24+FIXED_FRAME)(R4), R1 // switch stack; must match frame size
904 MOVD R5, FIXED_FRAME+0(R1)
905 MOVD R6, FIXED_FRAME+8(R1)
906 MOVD R7, FIXED_FRAME+16(R1)
907
908 MOVD $runtime·cgocallbackg(SB), R12
909 MOVD R12, CTR
910 CALL (CTR) // indirect call to bypass nosplit check. We're on a different stack now.
911
912 // Restore g->sched (== m->curg->sched) from saved values.
913 MOVD 0(R1), R5
914 MOVD R5, (g_sched+gobuf_pc)(g)
915 MOVD $(24+FIXED_FRAME)(R1), R4 // must match frame size
916 MOVD R4, (g_sched+gobuf_sp)(g)
917
918 // Switch back to m->g0's stack and restore m->g0->sched.sp.
919 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
920 // so we do not have to restore it.)
921 MOVD g_m(g), R8
922 MOVD m_g0(R8), g
923 BL runtime·save_g(SB)
924 MOVD (g_sched+gobuf_sp)(g), R1
925 MOVD savedsp-24(SP), R4 // must match frame size
926 MOVD R4, (g_sched+gobuf_sp)(g)
927
928 // If the m on entry was nil, we called needm above to borrow an m,
929 // 1. for the duration of the call on non-pthread platforms,
930 // 2. or the duration of the C thread alive on pthread platforms.
931 // If the m on entry wasn't nil,
932 // 1. the thread might be a Go thread,
933 // 2. or it wasn't the first call from a C thread on pthread platforms,
934 // since then we skip dropm to reuse the m in the first call.
935 MOVD savedm-8(SP), R6
936 CMP R6, $0
937 BNE droppedm
938
939 // Skip dropm to reuse it in the next call, when a pthread key has been created.
940 MOVD _cgo_pthread_key_created(SB), R6
941 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
942 CMP R6, $0
943 BEQ dropm
944 MOVD (R6), R6
945 CMP R6, $0
946 BNE droppedm
947
948dropm:
949 MOVD $runtime·dropm(SB), R12
950 MOVD R12, CTR
951 BL (CTR)
952droppedm:
953
954 // Done!
955 RET
956
957// void setg(G*); set g. for use by needm.
958TEXT runtime·setg(SB), NOSPLIT, $0-8
959 MOVD gg+0(FP), g
960 // This only happens if iscgo, so jump straight to save_g
961 BL runtime·save_g(SB)
962 RET
963
964#ifdef GO_PPC64X_HAS_FUNCDESC
965DEFINE_PPC64X_FUNCDESC(setg_gcc<>, _setg_gcc<>)
966TEXT _setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
967#else
968TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
969#endif
970 // The standard prologue clobbers R31, which is callee-save in
971 // the C ABI, so we have to use $-8-0 and save LR ourselves.
972 MOVD LR, R4
973 // Also save g and R31, since they're callee-save in C ABI
974 MOVD R31, R5
975 MOVD g, R6
976
977 MOVD R3, g
978 BL runtime·save_g(SB)
979
980 MOVD R6, g
981 MOVD R5, R31
982 MOVD R4, LR
983 RET
984
985TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
986 MOVW (R0), R0
987 UNDEF
988
989#define TBR 268
990
991// int64 runtime·cputicks(void)
992TEXT runtime·cputicks(SB),NOSPLIT,$0-8
993 MOVD SPR(TBR), R3
994 MOVD R3, ret+0(FP)
995 RET
996
997// spillArgs stores return values from registers to a *internal/abi.RegArgs in R20.
998TEXT runtime·spillArgs(SB),NOSPLIT,$0-0
999 MOVD R3, 0(R20)
1000 MOVD R4, 8(R20)
1001 MOVD R5, 16(R20)
1002 MOVD R6, 24(R20)
1003 MOVD R7, 32(R20)
1004 MOVD R8, 40(R20)
1005 MOVD R9, 48(R20)
1006 MOVD R10, 56(R20)
1007 MOVD R14, 64(R20)
1008 MOVD R15, 72(R20)
1009 MOVD R16, 80(R20)
1010 MOVD R17, 88(R20)
1011 FMOVD F1, 96(R20)
1012 FMOVD F2, 104(R20)
1013 FMOVD F3, 112(R20)
1014 FMOVD F4, 120(R20)
1015 FMOVD F5, 128(R20)
1016 FMOVD F6, 136(R20)
1017 FMOVD F7, 144(R20)
1018 FMOVD F8, 152(R20)
1019 FMOVD F9, 160(R20)
1020 FMOVD F10, 168(R20)
1021 FMOVD F11, 176(R20)
1022 FMOVD F12, 184(R20)
1023 RET
1024
1025// unspillArgs loads args into registers from a *internal/abi.RegArgs in R20.
1026TEXT runtime·unspillArgs(SB),NOSPLIT,$0-0
1027 MOVD 0(R20), R3
1028 MOVD 8(R20), R4
1029 MOVD 16(R20), R5
1030 MOVD 24(R20), R6
1031 MOVD 32(R20), R7
1032 MOVD 40(R20), R8
1033 MOVD 48(R20), R9
1034 MOVD 56(R20), R10
1035 MOVD 64(R20), R14
1036 MOVD 72(R20), R15
1037 MOVD 80(R20), R16
1038 MOVD 88(R20), R17
1039 FMOVD 96(R20), F1
1040 FMOVD 104(R20), F2
1041 FMOVD 112(R20), F3
1042 FMOVD 120(R20), F4
1043 FMOVD 128(R20), F5
1044 FMOVD 136(R20), F6
1045 FMOVD 144(R20), F7
1046 FMOVD 152(R20), F8
1047 FMOVD 160(R20), F9
1048 FMOVD 168(R20), F10
1049 FMOVD 176(R20), F11
1050 FMOVD 184(R20), F12
1051 RET
1052
1053// AES hashing not implemented for ppc64
1054TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
1055 JMP runtime·memhashFallback<ABIInternal>(SB)
1056TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
1057 JMP runtime·strhashFallback<ABIInternal>(SB)
1058TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
1059 JMP runtime·memhash32Fallback<ABIInternal>(SB)
1060TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
1061 JMP runtime·memhash64Fallback<ABIInternal>(SB)
1062
1063// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
1064// Must obey the gcc calling convention.
1065#ifdef GOOS_aix
1066// On AIX, _cgo_topofstack is defined in runtime/cgo, because it must
1067// be a longcall in order to prevent trampolines from ld.
1068TEXT __cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
1069#else
1070TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
1071#endif
1072 // g (R30) and R31 are callee-save in the C ABI, so save them
1073 MOVD g, R4
1074 MOVD R31, R5
1075 MOVD LR, R6
1076
1077 BL runtime·load_g(SB) // clobbers g (R30), R31
1078 MOVD g_m(g), R3
1079 MOVD m_curg(R3), R3
1080 MOVD (g_stack+stack_hi)(R3), R3
1081
1082 MOVD R4, g
1083 MOVD R5, R31
1084 MOVD R6, LR
1085 RET
1086
1087// The top-most function running on a goroutine
1088// returns to goexit+PCQuantum.
1089//
1090// When dynamically linking Go, it can be returned to from a function
1091// implemented in a different module and so needs to reload the TOC pointer
1092// from the stack (although this function declares that it does not set up x-a
1093// frame, newproc1 does in fact allocate one for goexit and saves the TOC
1094// pointer in the correct place).
1095// goexit+_PCQuantum is halfway through the usual global entry point prologue
1096// that derives r2 from r12 which is a bit silly, but not harmful.
1097TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
1098 MOVD 24(R1), R2
1099 BL runtime·goexit1(SB) // does not return
1100 // traceback from goexit1 must hit code range of goexit
1101 MOVD R0, R0 // NOP
1102
1103// prepGoExitFrame saves the current TOC pointer (i.e. the TOC pointer for the
1104// module containing runtime) to the frame that goexit will execute in when
1105// the goroutine exits. It's implemented in assembly mainly because that's the
1106// easiest way to get access to R2.
1107TEXT runtime·prepGoExitFrame(SB),NOSPLIT,$0-8
1108 MOVD sp+0(FP), R3
1109 MOVD R2, 24(R3)
1110 RET
1111
1112TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
1113 ADD $-8, R1
1114 MOVD R31, 0(R1)
1115 MOVD runtime·lastmoduledatap(SB), R4
1116 MOVD R3, moduledata_next(R4)
1117 MOVD R3, runtime·lastmoduledatap(SB)
1118 MOVD 0(R1), R31
1119 ADD $8, R1
1120 RET
1121
1122TEXT ·checkASM(SB),NOSPLIT,$0-1
1123 MOVW $1, R3
1124 MOVB R3, ret+0(FP)
1125 RET
1126
1127// gcWriteBarrier informs the GC about heap pointer writes.
1128//
1129// gcWriteBarrier does NOT follow the Go ABI. It accepts the
1130// number of bytes of buffer needed in R29, and returns a pointer
1131// to the buffer space in R29.
1132// It clobbers condition codes.
1133// It does not clobber R0 through R17 (except special registers),
1134// but may clobber any other register, *including* R31.
1135TEXT gcWriteBarrier<>(SB),NOSPLIT,$120
1136 // The standard prologue clobbers R31.
1137 // We use R18, R19, and R31 as scratch registers.
1138retry:
1139 MOVD g_m(g), R18
1140 MOVD m_p(R18), R18
1141 MOVD (p_wbBuf+wbBuf_next)(R18), R19
1142 MOVD (p_wbBuf+wbBuf_end)(R18), R31
1143 // Increment wbBuf.next position.
1144 ADD R29, R19
1145 // Is the buffer full?
1146 CMPU R31, R19
1147 BLT flush
1148 // Commit to the larger buffer.
1149 MOVD R19, (p_wbBuf+wbBuf_next)(R18)
1150 // Make return value (the original next position)
1151 SUB R29, R19, R29
1152 RET
1153
1154flush:
1155 // Save registers R0 through R15 since these were not saved by the caller.
1156 // We don't save all registers on ppc64 because it takes too much space.
1157 MOVD R20, (FIXED_FRAME+0)(R1)
1158 MOVD R21, (FIXED_FRAME+8)(R1)
1159 // R0 is always 0, so no need to spill.
1160 // R1 is SP.
1161 // R2 is SB.
1162 MOVD R3, (FIXED_FRAME+16)(R1)
1163 MOVD R4, (FIXED_FRAME+24)(R1)
1164 MOVD R5, (FIXED_FRAME+32)(R1)
1165 MOVD R6, (FIXED_FRAME+40)(R1)
1166 MOVD R7, (FIXED_FRAME+48)(R1)
1167 MOVD R8, (FIXED_FRAME+56)(R1)
1168 MOVD R9, (FIXED_FRAME+64)(R1)
1169 MOVD R10, (FIXED_FRAME+72)(R1)
1170 // R11, R12 may be clobbered by external-linker-inserted trampoline
1171 // R13 is REGTLS
1172 MOVD R14, (FIXED_FRAME+80)(R1)
1173 MOVD R15, (FIXED_FRAME+88)(R1)
1174 MOVD R16, (FIXED_FRAME+96)(R1)
1175 MOVD R17, (FIXED_FRAME+104)(R1)
1176 MOVD R29, (FIXED_FRAME+112)(R1)
1177
1178 CALL runtime·wbBufFlush(SB)
1179
1180 MOVD (FIXED_FRAME+0)(R1), R20
1181 MOVD (FIXED_FRAME+8)(R1), R21
1182 MOVD (FIXED_FRAME+16)(R1), R3
1183 MOVD (FIXED_FRAME+24)(R1), R4
1184 MOVD (FIXED_FRAME+32)(R1), R5
1185 MOVD (FIXED_FRAME+40)(R1), R6
1186 MOVD (FIXED_FRAME+48)(R1), R7
1187 MOVD (FIXED_FRAME+56)(R1), R8
1188 MOVD (FIXED_FRAME+64)(R1), R9
1189 MOVD (FIXED_FRAME+72)(R1), R10
1190 MOVD (FIXED_FRAME+80)(R1), R14
1191 MOVD (FIXED_FRAME+88)(R1), R15
1192 MOVD (FIXED_FRAME+96)(R1), R16
1193 MOVD (FIXED_FRAME+104)(R1), R17
1194 MOVD (FIXED_FRAME+112)(R1), R29
1195 JMP retry
1196
1197TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
1198 MOVD $8, R29
1199 JMP gcWriteBarrier<>(SB)
1200TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
1201 MOVD $16, R29
1202 JMP gcWriteBarrier<>(SB)
1203TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
1204 MOVD $24, R29
1205 JMP gcWriteBarrier<>(SB)
1206TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
1207 MOVD $32, R29
1208 JMP gcWriteBarrier<>(SB)
1209TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
1210 MOVD $40, R29
1211 JMP gcWriteBarrier<>(SB)
1212TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
1213 MOVD $48, R29
1214 JMP gcWriteBarrier<>(SB)
1215TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
1216 MOVD $56, R29
1217 JMP gcWriteBarrier<>(SB)
1218TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
1219 MOVD $64, R29
1220 JMP gcWriteBarrier<>(SB)
1221
1222DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
1223GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
1224
1225// debugCallV2 is the entry point for debugger-injected function
1226// calls on running goroutines. It informs the runtime that a
1227// debug call has been injected and creates a call frame for the
1228// debugger to fill in.
1229//
1230// To inject a function call, a debugger should:
1231// 1. Check that the goroutine is in state _Grunning and that
1232// there are at least 320 bytes free on the stack.
1233// 2. Set SP as SP-32.
1234// 3. Store the current LR in (SP) (using the SP after step 2).
1235// 4. Store the current PC in the LR register.
1236// 5. Write the desired argument frame size at SP-32
1237// 6. Save all machine registers (including flags and floating point registers)
1238// so they can be restored later by the debugger.
1239// 7. Set the PC to debugCallV2 and resume execution.
1240//
1241// If the goroutine is in state _Grunnable, then it's not generally
1242// safe to inject a call because it may return out via other runtime
1243// operations. Instead, the debugger should unwind the stack to find
1244// the return to non-runtime code, add a temporary breakpoint there,
1245// and inject the call once that breakpoint is hit.
1246//
1247// If the goroutine is in any other state, it's not safe to inject a call.
1248//
1249// This function communicates back to the debugger by setting R20 and
1250// invoking TW to raise a breakpoint signal. Note that the signal PC of
1251// the signal triggered by the TW instruction is the PC where the signal
1252// is trapped, not the next PC, so to resume execution, the debugger needs
1253// to set the signal PC to PC+4. See the comments in the implementation for
1254// the protocol the debugger is expected to follow. InjectDebugCall in the
1255// runtime tests demonstrates this protocol.
1256// The debugger must ensure that any pointers passed to the function
1257// obey escape analysis requirements. Specifically, it must not pass
1258// a stack pointer to an escaping argument. debugCallV2 cannot check
1259// this invariant.
1260//
1261// This is ABIInternal because Go code injects its PC directly into new
1262// goroutine stacks.
1263#ifdef GOARCH_ppc64le
1264TEXT runtime·debugCallV2<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-0
1265 // save scratch register R31 first
1266 MOVD R31, -184(R1)
1267 MOVD 0(R1), R31
1268 // save caller LR
1269 MOVD R31, -304(R1)
1270 MOVD -32(R1), R31
1271 // save argument frame size
1272 MOVD R31, -192(R1)
1273 MOVD LR, R31
1274 MOVD R31, -320(R1)
1275 ADD $-320, R1
1276 // save all registers that can contain pointers
1277 // and the CR register
1278 MOVW CR, R31
1279 MOVD R31, 8(R1)
1280 MOVD R2, 24(R1)
1281 MOVD R3, 56(R1)
1282 MOVD R4, 64(R1)
1283 MOVD R5, 72(R1)
1284 MOVD R6, 80(R1)
1285 MOVD R7, 88(R1)
1286 MOVD R8, 96(R1)
1287 MOVD R9, 104(R1)
1288 MOVD R10, 112(R1)
1289 MOVD R11, 120(R1)
1290 MOVD R12, 144(R1)
1291 MOVD R13, 152(R1)
1292 MOVD R14, 160(R1)
1293 MOVD R15, 168(R1)
1294 MOVD R16, 176(R1)
1295 MOVD R17, 184(R1)
1296 MOVD R18, 192(R1)
1297 MOVD R19, 200(R1)
1298 MOVD R20, 208(R1)
1299 MOVD R21, 216(R1)
1300 MOVD R22, 224(R1)
1301 MOVD R23, 232(R1)
1302 MOVD R24, 240(R1)
1303 MOVD R25, 248(R1)
1304 MOVD R26, 256(R1)
1305 MOVD R27, 264(R1)
1306 MOVD R28, 272(R1)
1307 MOVD R29, 280(R1)
1308 MOVD g, 288(R1)
1309 MOVD LR, R31
1310 MOVD R31, 32(R1)
1311 CALL runtime·debugCallCheck(SB)
1312 MOVD 40(R1), R22
1313 XOR R0, R0
1314 CMP R22, $0
1315 BEQ good
1316 MOVD 48(R1), R22
1317 MOVD $8, R20
1318 TW $31, R0, R0
1319
1320 BR restore
1321
1322good:
1323#define DEBUG_CALL_DISPATCH(NAME,MAXSIZE) \
1324 MOVD $MAXSIZE, R23; \
1325 CMP R26, R23; \
1326 BGT 5(PC); \
1327 MOVD $NAME(SB), R26; \
1328 MOVD R26, 32(R1); \
1329 CALL runtime·debugCallWrap(SB); \
1330 BR restore
1331
1332 // the argument frame size
1333 MOVD 128(R1), R26
1334
1335 DEBUG_CALL_DISPATCH(debugCall32<>, 32)
1336 DEBUG_CALL_DISPATCH(debugCall64<>, 64)
1337 DEBUG_CALL_DISPATCH(debugCall128<>, 128)
1338 DEBUG_CALL_DISPATCH(debugCall256<>, 256)
1339 DEBUG_CALL_DISPATCH(debugCall512<>, 512)
1340 DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)
1341 DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)
1342 DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)
1343 DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)
1344 DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)
1345 DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)
1346 DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)
1347 // The frame size is too large. Report the error.
1348 MOVD $debugCallFrameTooLarge<>(SB), R22
1349 MOVD R22, 32(R1)
1350 MOVD $20, R22
1351 // length of debugCallFrameTooLarge string
1352 MOVD R22, 40(R1)
1353 MOVD $8, R20
1354 TW $31, R0, R0
1355 BR restore
1356restore:
1357 MOVD $16, R20
1358 TW $31, R0, R0
1359 // restore all registers that can contain
1360 // pointers including CR
1361 MOVD 8(R1), R31
1362 MOVW R31, CR
1363 MOVD 24(R1), R2
1364 MOVD 56(R1), R3
1365 MOVD 64(R1), R4
1366 MOVD 72(R1), R5
1367 MOVD 80(R1), R6
1368 MOVD 88(R1), R7
1369 MOVD 96(R1), R8
1370 MOVD 104(R1), R9
1371 MOVD 112(R1), R10
1372 MOVD 120(R1), R11
1373 MOVD 144(R1), R12
1374 MOVD 152(R1), R13
1375 MOVD 160(R1), R14
1376 MOVD 168(R1), R15
1377 MOVD 176(R1), R16
1378 MOVD 184(R1), R17
1379 MOVD 192(R1), R18
1380 MOVD 200(R1), R19
1381 MOVD 208(R1), R20
1382 MOVD 216(R1), R21
1383 MOVD 224(R1), R22
1384 MOVD 232(R1), R23
1385 MOVD 240(R1), R24
1386 MOVD 248(R1), R25
1387 MOVD 256(R1), R26
1388 MOVD 264(R1), R27
1389 MOVD 272(R1), R28
1390 MOVD 280(R1), R29
1391 MOVD 288(R1), g
1392 MOVD 16(R1), R31
1393 // restore old LR
1394 MOVD R31, LR
1395 // restore caller PC
1396 MOVD 0(R1), CTR
1397 MOVD 136(R1), R31
1398 // Add 32 bytes more to compensate for SP change in saveSigContext
1399 ADD $352, R1
1400 JMP (CTR)
1401#endif
1402#define DEBUG_CALL_FN(NAME,MAXSIZE) \
1403TEXT NAME(SB),WRAPPER,$MAXSIZE-0; \
1404 NO_LOCAL_POINTERS; \
1405 MOVD $0, R20; \
1406 TW $31, R0, R0 \
1407 MOVD $1, R20; \
1408 TW $31, R0, R0 \
1409 RET
1410DEBUG_CALL_FN(debugCall32<>, 32)
1411DEBUG_CALL_FN(debugCall64<>, 64)
1412DEBUG_CALL_FN(debugCall128<>, 128)
1413DEBUG_CALL_FN(debugCall256<>, 256)
1414DEBUG_CALL_FN(debugCall512<>, 512)
1415DEBUG_CALL_FN(debugCall1024<>, 1024)
1416DEBUG_CALL_FN(debugCall2048<>, 2048)
1417DEBUG_CALL_FN(debugCall4096<>, 4096)
1418DEBUG_CALL_FN(debugCall8192<>, 8192)
1419DEBUG_CALL_FN(debugCall16384<>, 16384)
1420DEBUG_CALL_FN(debugCall32768<>, 32768)
1421DEBUG_CALL_FN(debugCall65536<>, 65536)
1422
1423#ifdef GOARCH_ppc64le
1424// func debugCallPanicked(val interface{})
1425TEXT runtime·debugCallPanicked(SB),NOSPLIT,$32-16
1426 // Copy the panic value to the top of stack at SP+32.
1427 MOVD val_type+0(FP), R31
1428 MOVD R31, 32(R1)
1429 MOVD val_data+8(FP), R31
1430 MOVD R31, 40(R1)
1431 MOVD $2, R20
1432 TW $31, R0, R0
1433 RET
1434#endif
1435
1436TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$88-0
1437 // Note: frame size is 16 bytes larger than necessary
1438 // in order to pacify vet. Vet doesn't understand ppc64
1439 // layout properly.
1440 NO_LOCAL_POINTERS
1441 // Save all 7 int registers that could have an index in them.
1442 // They may be pointers, but if so they are dead.
1443 // Skip R0 aka ZERO, R1 aka SP, R2 aka SB
1444 MOVD R3, 48(R1)
1445 MOVD R4, 56(R1)
1446 MOVD R5, 64(R1)
1447 MOVD R6, 72(R1)
1448 MOVD R7, 80(R1)
1449 MOVD R8, 88(R1)
1450 MOVD R9, 96(R1)
1451 // Note: we only save 7 registers to keep under nosplit stack limit
1452 // Also, R11 is clobbered in dynamic linking situations
1453
1454 MOVD LR, R3 // PC immediately after call to panicBounds
1455 ADD $48, R1, R4 // pointer to save area
1456 CALL runtime·panicBounds64<ABIInternal>(SB)
1457 RET
1458
1459// These functions are used when internal linking cgo with external
1460// objects compiled with the -Os on gcc. They reduce prologue/epilogue
1461// size by deferring preservation of callee-save registers to a shared
1462// function. These are defined in PPC64 ELFv2 2.3.3 (but also present
1463// in ELFv1)
1464//
1465// These appear unused, but the linker will redirect calls to functions
1466// like _savegpr0_14 or _restgpr1_14 to runtime.elf_savegpr0 or
1467// runtime.elf_restgpr1 with an appropriate offset based on the number
1468// register operations required when linking external objects which
1469// make these calls. For GPR/FPR saves, the minimum register value is
1470// 14, for VR it is 20.
1471//
1472// These are only used when linking such cgo code internally. Note, R12
1473// and R0 may be used in different ways than regular ELF compliant
1474// functions.
1475TEXT runtime·elf_savegpr0(SB),NOSPLIT|NOFRAME,$0
1476 // R0 holds the LR of the caller's caller, R1 holds save location
1477 MOVD R14, -144(R1)
1478 MOVD R15, -136(R1)
1479 MOVD R16, -128(R1)
1480 MOVD R17, -120(R1)
1481 MOVD R18, -112(R1)
1482 MOVD R19, -104(R1)
1483 MOVD R20, -96(R1)
1484 MOVD R21, -88(R1)
1485 MOVD R22, -80(R1)
1486 MOVD R23, -72(R1)
1487 MOVD R24, -64(R1)
1488 MOVD R25, -56(R1)
1489 MOVD R26, -48(R1)
1490 MOVD R27, -40(R1)
1491 MOVD R28, -32(R1)
1492 MOVD R29, -24(R1)
1493 MOVD g, -16(R1)
1494 MOVD R31, -8(R1)
1495 MOVD R0, 16(R1)
1496 RET
1497TEXT runtime·elf_restgpr0(SB),NOSPLIT|NOFRAME,$0
1498 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1499 MOVD -144(R1), R14
1500 MOVD -136(R1), R15
1501 MOVD -128(R1), R16
1502 MOVD -120(R1), R17
1503 MOVD -112(R1), R18
1504 MOVD -104(R1), R19
1505 MOVD -96(R1), R20
1506 MOVD -88(R1), R21
1507 MOVD -80(R1), R22
1508 MOVD -72(R1), R23
1509 MOVD -64(R1), R24
1510 MOVD -56(R1), R25
1511 MOVD -48(R1), R26
1512 MOVD -40(R1), R27
1513 MOVD -32(R1), R28
1514 MOVD -24(R1), R29
1515 MOVD -16(R1), g
1516 MOVD -8(R1), R31
1517 MOVD 16(R1), R0 // Load and return to saved LR
1518 MOVD R0, LR
1519 RET
1520TEXT runtime·elf_savegpr1(SB),NOSPLIT|NOFRAME,$0
1521 // R12 holds the save location
1522 MOVD R14, -144(R12)
1523 MOVD R15, -136(R12)
1524 MOVD R16, -128(R12)
1525 MOVD R17, -120(R12)
1526 MOVD R18, -112(R12)
1527 MOVD R19, -104(R12)
1528 MOVD R20, -96(R12)
1529 MOVD R21, -88(R12)
1530 MOVD R22, -80(R12)
1531 MOVD R23, -72(R12)
1532 MOVD R24, -64(R12)
1533 MOVD R25, -56(R12)
1534 MOVD R26, -48(R12)
1535 MOVD R27, -40(R12)
1536 MOVD R28, -32(R12)
1537 MOVD R29, -24(R12)
1538 MOVD g, -16(R12)
1539 MOVD R31, -8(R12)
1540 RET
1541TEXT runtime·elf_restgpr1(SB),NOSPLIT|NOFRAME,$0
1542 // R12 holds the save location
1543 MOVD -144(R12), R14
1544 MOVD -136(R12), R15
1545 MOVD -128(R12), R16
1546 MOVD -120(R12), R17
1547 MOVD -112(R12), R18
1548 MOVD -104(R12), R19
1549 MOVD -96(R12), R20
1550 MOVD -88(R12), R21
1551 MOVD -80(R12), R22
1552 MOVD -72(R12), R23
1553 MOVD -64(R12), R24
1554 MOVD -56(R12), R25
1555 MOVD -48(R12), R26
1556 MOVD -40(R12), R27
1557 MOVD -32(R12), R28
1558 MOVD -24(R12), R29
1559 MOVD -16(R12), g
1560 MOVD -8(R12), R31
1561 RET
1562TEXT runtime·elf_savefpr(SB),NOSPLIT|NOFRAME,$0
1563 // R0 holds the LR of the caller's caller, R1 holds save location
1564 FMOVD F14, -144(R1)
1565 FMOVD F15, -136(R1)
1566 FMOVD F16, -128(R1)
1567 FMOVD F17, -120(R1)
1568 FMOVD F18, -112(R1)
1569 FMOVD F19, -104(R1)
1570 FMOVD F20, -96(R1)
1571 FMOVD F21, -88(R1)
1572 FMOVD F22, -80(R1)
1573 FMOVD F23, -72(R1)
1574 FMOVD F24, -64(R1)
1575 FMOVD F25, -56(R1)
1576 FMOVD F26, -48(R1)
1577 FMOVD F27, -40(R1)
1578 FMOVD F28, -32(R1)
1579 FMOVD F29, -24(R1)
1580 FMOVD F30, -16(R1)
1581 FMOVD F31, -8(R1)
1582 MOVD R0, 16(R1)
1583 RET
1584TEXT runtime·elf_restfpr(SB),NOSPLIT|NOFRAME,$0
1585 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1586 FMOVD -144(R1), F14
1587 FMOVD -136(R1), F15
1588 FMOVD -128(R1), F16
1589 FMOVD -120(R1), F17
1590 FMOVD -112(R1), F18
1591 FMOVD -104(R1), F19
1592 FMOVD -96(R1), F20
1593 FMOVD -88(R1), F21
1594 FMOVD -80(R1), F22
1595 FMOVD -72(R1), F23
1596 FMOVD -64(R1), F24
1597 FMOVD -56(R1), F25
1598 FMOVD -48(R1), F26
1599 FMOVD -40(R1), F27
1600 FMOVD -32(R1), F28
1601 FMOVD -24(R1), F29
1602 FMOVD -16(R1), F30
1603 FMOVD -8(R1), F31
1604 MOVD 16(R1), R0 // Load and return to saved LR
1605 MOVD R0, LR
1606 RET
1607TEXT runtime·elf_savevr(SB),NOSPLIT|NOFRAME,$0
1608 // R0 holds the save location, R12 is clobbered
1609 MOVD $-192, R12
1610 STVX V20, (R0+R12)
1611 MOVD $-176, R12
1612 STVX V21, (R0+R12)
1613 MOVD $-160, R12
1614 STVX V22, (R0+R12)
1615 MOVD $-144, R12
1616 STVX V23, (R0+R12)
1617 MOVD $-128, R12
1618 STVX V24, (R0+R12)
1619 MOVD $-112, R12
1620 STVX V25, (R0+R12)
1621 MOVD $-96, R12
1622 STVX V26, (R0+R12)
1623 MOVD $-80, R12
1624 STVX V27, (R0+R12)
1625 MOVD $-64, R12
1626 STVX V28, (R0+R12)
1627 MOVD $-48, R12
1628 STVX V29, (R0+R12)
1629 MOVD $-32, R12
1630 STVX V30, (R0+R12)
1631 MOVD $-16, R12
1632 STVX V31, (R0+R12)
1633 RET
1634TEXT runtime·elf_restvr(SB),NOSPLIT|NOFRAME,$0
1635 // R0 holds the save location, R12 is clobbered
1636 MOVD $-192, R12
1637 LVX (R0+R12), V20
1638 MOVD $-176, R12
1639 LVX (R0+R12), V21
1640 MOVD $-160, R12
1641 LVX (R0+R12), V22
1642 MOVD $-144, R12
1643 LVX (R0+R12), V23
1644 MOVD $-128, R12
1645 LVX (R0+R12), V24
1646 MOVD $-112, R12
1647 LVX (R0+R12), V25
1648 MOVD $-96, R12
1649 LVX (R0+R12), V26
1650 MOVD $-80, R12
1651 LVX (R0+R12), V27
1652 MOVD $-64, R12
1653 LVX (R0+R12), V28
1654 MOVD $-48, R12
1655 LVX (R0+R12), V29
1656 MOVD $-32, R12
1657 LVX (R0+R12), V30
1658 MOVD $-16, R12
1659 LVX (R0+R12), V31
1660 RET
View as plain text