Text file
src/runtime/asm_riscv64.s
Documentation: runtime
1// Copyright 2017 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "funcdata.h"
7#include "textflag.h"
8
9
10// When building with -buildmode=c-shared, this symbol is called when the shared
11// library is loaded.
12TEXT _rt0_riscv64_lib(SB),NOSPLIT,$224
13 // Preserve callee-save registers, along with X1 (LR).
14 MOV X1, (8*3)(X2)
15 MOV X8, (8*4)(X2)
16 MOV X9, (8*5)(X2)
17 MOV X18, (8*6)(X2)
18 MOV X19, (8*7)(X2)
19 MOV X20, (8*8)(X2)
20 MOV X21, (8*9)(X2)
21 MOV X22, (8*10)(X2)
22 MOV X23, (8*11)(X2)
23 MOV X24, (8*12)(X2)
24 MOV X25, (8*13)(X2)
25 MOV X26, (8*14)(X2)
26 MOV g, (8*15)(X2)
27 MOVD F8, (8*16)(X2)
28 MOVD F9, (8*17)(X2)
29 MOVD F18, (8*18)(X2)
30 MOVD F19, (8*19)(X2)
31 MOVD F20, (8*20)(X2)
32 MOVD F21, (8*21)(X2)
33 MOVD F22, (8*22)(X2)
34 MOVD F23, (8*23)(X2)
35 MOVD F24, (8*24)(X2)
36 MOVD F25, (8*25)(X2)
37 MOVD F26, (8*26)(X2)
38 MOVD F27, (8*27)(X2)
39
40 // Initialize g as nil in case of using g later e.g. sigaction in cgo_sigaction.go
41 MOV X0, g
42
43 MOV A0, _rt0_riscv64_lib_argc<>(SB)
44 MOV A1, _rt0_riscv64_lib_argv<>(SB)
45
46 // Synchronous initialization.
47 MOV $runtime·libpreinit(SB), T0
48 JALR RA, T0
49
50 // Create a new thread to do the runtime initialization and return.
51 MOV _cgo_sys_thread_create(SB), T0
52 BEQZ T0, nocgo
53 MOV $_rt0_riscv64_lib_go(SB), A0
54 MOV $0, A1
55 JALR RA, T0
56 JMP restore
57
58nocgo:
59 MOV $0x800000, A0 // stacksize = 8192KB
60 MOV $_rt0_riscv64_lib_go(SB), A1
61 MOV A0, 8(X2)
62 MOV A1, 16(X2)
63 MOV $runtime·newosproc0(SB), T0
64 JALR RA, T0
65
66restore:
67 // Restore callee-save registers, along with X1 (LR).
68 MOV (8*3)(X2), X1
69 MOV (8*4)(X2), X8
70 MOV (8*5)(X2), X9
71 MOV (8*6)(X2), X18
72 MOV (8*7)(X2), X19
73 MOV (8*8)(X2), X20
74 MOV (8*9)(X2), X21
75 MOV (8*10)(X2), X22
76 MOV (8*11)(X2), X23
77 MOV (8*12)(X2), X24
78 MOV (8*13)(X2), X25
79 MOV (8*14)(X2), X26
80 MOV (8*15)(X2), g
81 MOVD (8*16)(X2), F8
82 MOVD (8*17)(X2), F9
83 MOVD (8*18)(X2), F18
84 MOVD (8*19)(X2), F19
85 MOVD (8*20)(X2), F20
86 MOVD (8*21)(X2), F21
87 MOVD (8*22)(X2), F22
88 MOVD (8*23)(X2), F23
89 MOVD (8*24)(X2), F24
90 MOVD (8*25)(X2), F25
91 MOVD (8*26)(X2), F26
92 MOVD (8*27)(X2), F27
93
94 RET
95
96TEXT _rt0_riscv64_lib_go(SB),NOSPLIT,$0
97 MOV _rt0_riscv64_lib_argc<>(SB), A0
98 MOV _rt0_riscv64_lib_argv<>(SB), A1
99 MOV $runtime·rt0_go(SB), T0
100 JALR ZERO, T0
101
102DATA _rt0_riscv64_lib_argc<>(SB)/8, $0
103GLOBL _rt0_riscv64_lib_argc<>(SB),NOPTR, $8
104DATA _rt0_riscv64_lib_argv<>(SB)/8, $0
105GLOBL _rt0_riscv64_lib_argv<>(SB),NOPTR, $8
106
107// func rt0_go()
108TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
109 // X2 = stack; A0 = argc; A1 = argv
110 SUB $24, X2
111 MOV A0, 8(X2) // argc
112 MOV A1, 16(X2) // argv
113
114 // create istack out of the given (operating system) stack.
115 // _cgo_init may update stackguard.
116 MOV $runtime·g0(SB), g
117 MOV $(-64*1024), T0
118 ADD T0, X2, T1
119 MOV T1, g_stackguard0(g)
120 MOV T1, g_stackguard1(g)
121 MOV T1, (g_stack+stack_lo)(g)
122 MOV X2, (g_stack+stack_hi)(g)
123
124 // if there is a _cgo_init, call it using the gcc ABI.
125 MOV _cgo_init(SB), T0
126 BEQ T0, ZERO, nocgo
127
128 MOV ZERO, A3 // arg 3: not used
129 MOV ZERO, A2 // arg 2: not used
130 MOV $setg_gcc<>(SB), A1 // arg 1: setg
131 MOV g, A0 // arg 0: G
132 JALR RA, T0
133
134nocgo:
135 // update stackguard after _cgo_init
136 MOV (g_stack+stack_lo)(g), T0
137 ADD $const_stackGuard, T0
138 MOV T0, g_stackguard0(g)
139 MOV T0, g_stackguard1(g)
140
141 // set the per-goroutine and per-mach "registers"
142 MOV $runtime·m0(SB), T0
143
144 // save m->g0 = g0
145 MOV g, m_g0(T0)
146 // save m0 to g0->m
147 MOV T0, g_m(g)
148
149 CALL runtime·check(SB)
150
151 // args are already prepared
152 CALL runtime·args(SB)
153 CALL runtime·osinit(SB)
154 CALL runtime·schedinit(SB)
155
156 // create a new goroutine to start program
157 MOV $runtime·mainPC(SB), T0 // entry
158 SUB $16, X2
159 MOV T0, 8(X2)
160 MOV ZERO, 0(X2)
161 CALL runtime·newproc(SB)
162 ADD $16, X2
163
164 // start this M
165 CALL runtime·mstart(SB)
166
167 WORD $0 // crash if reached
168 RET
169
170TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
171 CALL runtime·mstart0(SB)
172 RET // not reached
173
174// void setg_gcc(G*); set g called from gcc with g in A0
175TEXT setg_gcc<>(SB),NOSPLIT,$0-0
176 MOV A0, g
177 CALL runtime·save_g(SB)
178 RET
179
180// func cputicks() int64
181TEXT runtime·cputicks<ABIInternal>(SB),NOSPLIT,$0-0
182 // RDTIME to emulate cpu ticks
183 // RDCYCLE reads counter that is per HART(core) based
184 // according to the riscv manual, see issue 46737
185 RDTIME X10
186 RET
187
188// systemstack_switch is a dummy routine that systemstack leaves at the bottom
189// of the G stack. We need to distinguish the routine that
190// lives at the bottom of the G stack from the one that lives
191// at the top of the system stack because the one at the top of
192// the system stack terminates the stack walk (see topofstack()).
193TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
194 UNDEF
195 JALR RA, ZERO // make sure this function is not leaf
196 RET
197
198// func systemstack(fn func())
199TEXT runtime·systemstack(SB), NOSPLIT, $0-8
200 MOV fn+0(FP), CTXT // CTXT = fn
201 MOV g_m(g), T0 // T0 = m
202
203 MOV m_gsignal(T0), T1 // T1 = gsignal
204 BEQ g, T1, noswitch
205
206 MOV m_g0(T0), T1 // T1 = g0
207 BEQ g, T1, noswitch
208
209 MOV m_curg(T0), T2
210 BEQ g, T2, switch
211
212 // Bad: g is not gsignal, not g0, not curg. What is it?
213 // Hide call from linker nosplit analysis.
214 MOV $runtime·badsystemstack(SB), T1
215 JALR RA, T1
216
217switch:
218 // save our state in g->sched. Pretend to
219 // be systemstack_switch if the G stack is scanned.
220 CALL gosave_systemstack_switch<>(SB)
221
222 // switch to g0
223 MOV T1, g
224 CALL runtime·save_g(SB)
225 MOV (g_sched+gobuf_sp)(g), T0
226 MOV T0, X2
227
228 // call target function
229 MOV 0(CTXT), T1 // code pointer
230 JALR RA, T1
231
232 // switch back to g
233 MOV g_m(g), T0
234 MOV m_curg(T0), g
235 CALL runtime·save_g(SB)
236 MOV (g_sched+gobuf_sp)(g), X2
237 MOV ZERO, (g_sched+gobuf_sp)(g)
238 RET
239
240noswitch:
241 // already on m stack, just call directly
242 // Using a tail call here cleans up tracebacks since we won't stop
243 // at an intermediate systemstack.
244 MOV 0(CTXT), T1 // code pointer
245 ADD $8, X2
246 JMP (T1)
247
248// func switchToCrashStack0(fn func())
249TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
250 MOV X10, CTXT // context register
251 MOV g_m(g), X11 // curm
252
253 // set g to gcrash
254 MOV $runtime·gcrash(SB), g // g = &gcrash
255 CALL runtime·save_g(SB) // clobbers X31
256 MOV X11, g_m(g) // g.m = curm
257 MOV g, m_g0(X11) // curm.g0 = g
258
259 // switch to crashstack
260 MOV (g_stack+stack_hi)(g), X11
261 SUB $(4*8), X11
262 MOV X11, X2
263
264 // call target function
265 MOV 0(CTXT), X10
266 JALR X1, X10
267
268 // should never return
269 CALL runtime·abort(SB)
270 UNDEF
271
272/*
273 * support for morestack
274 */
275
276// Called during function prolog when more stack is needed.
277// Called with return address (i.e. caller's PC) in X5 (aka T0),
278// and the LR register contains the caller's LR.
279//
280// The traceback routines see morestack on a g0 as being
281// the top of a stack (for example, morestack calling newstack
282// calling the scheduler calling newm calling gc), so we must
283// record an argument size. For that purpose, it has no arguments.
284
285// func morestack()
286TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
287 // Called from f.
288 // Set g->sched to context in f.
289 MOV X2, (g_sched+gobuf_sp)(g)
290 MOV T0, (g_sched+gobuf_pc)(g)
291 MOV RA, (g_sched+gobuf_lr)(g)
292 MOV CTXT, (g_sched+gobuf_ctxt)(g)
293
294 // Cannot grow scheduler stack (m->g0).
295 MOV g_m(g), A0
296 MOV m_g0(A0), A1
297 BNE g, A1, 3(PC)
298 CALL runtime·badmorestackg0(SB)
299 CALL runtime·abort(SB)
300
301 // Cannot grow signal stack (m->gsignal).
302 MOV m_gsignal(A0), A1
303 BNE g, A1, 3(PC)
304 CALL runtime·badmorestackgsignal(SB)
305 CALL runtime·abort(SB)
306
307 // Called from f.
308 // Set m->morebuf to f's caller.
309 MOV RA, (m_morebuf+gobuf_pc)(A0) // f's caller's PC
310 MOV X2, (m_morebuf+gobuf_sp)(A0) // f's caller's SP
311 MOV g, (m_morebuf+gobuf_g)(A0)
312
313 // Call newstack on m->g0's stack.
314 MOV m_g0(A0), g
315 CALL runtime·save_g(SB)
316 MOV (g_sched+gobuf_sp)(g), X2
317 // Create a stack frame on g0 to call newstack.
318 MOV ZERO, -8(X2) // Zero saved LR in frame
319 SUB $8, X2
320 CALL runtime·newstack(SB)
321
322 // Not reached, but make sure the return PC from the call to newstack
323 // is still in this function, and not the beginning of the next.
324 UNDEF
325
326// func morestack_noctxt()
327TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
328 // Force SPWRITE. This function doesn't actually write SP,
329 // but it is called with a special calling convention where
330 // the caller doesn't save LR on stack but passes it as a
331 // register, and the unwinder currently doesn't understand.
332 // Make it SPWRITE to stop unwinding. (See issue 54332)
333 MOV X2, X2
334
335 MOV ZERO, CTXT
336 JMP runtime·morestack(SB)
337
338// AES hashing not implemented for riscv64
339TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
340 JMP runtime·memhashFallback<ABIInternal>(SB)
341TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
342 JMP runtime·strhashFallback<ABIInternal>(SB)
343TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
344 JMP runtime·memhash32Fallback<ABIInternal>(SB)
345TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
346 JMP runtime·memhash64Fallback<ABIInternal>(SB)
347
348// restore state from Gobuf; longjmp
349
350// func gogo(buf *gobuf)
351TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
352 MOV buf+0(FP), T0
353 MOV gobuf_g(T0), T1
354 MOV 0(T1), ZERO // make sure g != nil
355 JMP gogo<>(SB)
356
357TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
358 MOV T1, g
359 CALL runtime·save_g(SB)
360
361 MOV gobuf_sp(T0), X2
362 MOV gobuf_lr(T0), RA
363 MOV gobuf_ctxt(T0), CTXT
364 MOV ZERO, gobuf_sp(T0)
365 MOV ZERO, gobuf_lr(T0)
366 MOV ZERO, gobuf_ctxt(T0)
367 MOV gobuf_pc(T0), T0
368 JALR ZERO, T0
369
370// func procyieldAsm(cycles uint32)
371TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0
372 RET
373
374// Switch to m->g0's stack, call fn(g).
375// Fn must never return. It should gogo(&g->sched)
376// to keep running g.
377
378// func mcall(fn func(*g))
379TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
380 MOV X10, CTXT
381
382 // Save caller state in g->sched
383 MOV X2, (g_sched+gobuf_sp)(g)
384 MOV RA, (g_sched+gobuf_pc)(g)
385 MOV ZERO, (g_sched+gobuf_lr)(g)
386
387 // Switch to m->g0 & its stack, call fn.
388 MOV g, X10
389 MOV g_m(g), T1
390 MOV m_g0(T1), g
391 CALL runtime·save_g(SB)
392 BNE g, X10, 2(PC)
393 JMP runtime·badmcall(SB)
394 MOV 0(CTXT), T1 // code pointer
395 MOV (g_sched+gobuf_sp)(g), X2 // sp = m->g0->sched.sp
396 // we don't need special macro for regabi since arg0(X10) = g
397 SUB $16, X2
398 MOV X10, 8(X2) // setup g
399 MOV ZERO, 0(X2) // clear return address
400 JALR RA, T1
401 JMP runtime·badmcall2(SB)
402
403// Save state of caller into g->sched,
404// but using fake PC from systemstack_switch.
405// Must only be called from functions with no locals ($0)
406// or else unwinding from systemstack_switch is incorrect.
407// Smashes X31.
408TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
409 MOV $runtime·systemstack_switch(SB), X31
410 ADD $8, X31 // get past prologue
411 MOV X31, (g_sched+gobuf_pc)(g)
412 MOV X2, (g_sched+gobuf_sp)(g)
413 MOV ZERO, (g_sched+gobuf_lr)(g)
414 // Assert ctxt is zero. See func save.
415 MOV (g_sched+gobuf_ctxt)(g), X31
416 BEQ ZERO, X31, 2(PC)
417 CALL runtime·abort(SB)
418 RET
419
420// func asmcgocall_no_g(fn, arg unsafe.Pointer)
421// Call fn(arg) aligned appropriately for the gcc ABI.
422// Called on a system stack, and there may be no g yet (during needm).
423TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
424 MOV fn+0(FP), X5
425 MOV arg+8(FP), X10
426 JALR RA, (X5)
427 RET
428
429// func asmcgocall(fn, arg unsafe.Pointer) int32
430// Call fn(arg) on the scheduler stack,
431// aligned appropriately for the gcc ABI.
432// See cgocall.go for more details.
433TEXT ·asmcgocall(SB),NOSPLIT,$0-20
434 MOV fn+0(FP), X5
435 MOV arg+8(FP), X10
436
437 MOV X2, X8 // save original stack pointer
438 MOV g, X9
439
440 // Figure out if we need to switch to m->g0 stack.
441 // We get called to create new OS threads too, and those
442 // come in on the m->g0 stack already. Or we might already
443 // be on the m->gsignal stack.
444 MOV g_m(g), X6
445 MOV m_gsignal(X6), X7
446 BEQ X7, g, g0
447 MOV m_g0(X6), X7
448 BEQ X7, g, g0
449
450 CALL gosave_systemstack_switch<>(SB)
451 MOV X7, g
452 CALL runtime·save_g(SB)
453 MOV (g_sched+gobuf_sp)(g), X2
454
455 // Now on a scheduling stack (a pthread-created stack).
456g0:
457 // Save room for two of our pointers.
458 SUB $16, X2
459 MOV X9, 0(X2) // save old g on stack
460 MOV (g_stack+stack_hi)(X9), X9
461 SUB X8, X9, X8
462 MOV X8, 8(X2) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
463
464 JALR RA, (X5)
465
466 // Restore g, stack pointer. X10 is return value.
467 MOV 0(X2), g
468 CALL runtime·save_g(SB)
469 MOV (g_stack+stack_hi)(g), X5
470 MOV 8(X2), X6
471 SUB X6, X5, X6
472 MOV X6, X2
473
474 MOVW X10, ret+16(FP)
475 RET
476
477// func asminit()
478TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
479 RET
480
481// reflectcall: call a function with the given argument list
482// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
483// we don't have variable-sized frames, so we use a small number
484// of constant-sized-frame functions to encode a few bits of size in the pc.
485// Caution: ugly multiline assembly macros in your future!
486
487#define DISPATCH(NAME,MAXSIZE) \
488 MOV $MAXSIZE, T1 \
489 BLTU T1, T0, 3(PC) \
490 MOV $NAME(SB), T2; \
491 JALR ZERO, T2
492// Note: can't just "BR NAME(SB)" - bad inlining results.
493
494// func call(stackArgsType *rtype, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
495TEXT reflect·call(SB), NOSPLIT, $0-0
496 JMP ·reflectcall(SB)
497
498// func call(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
499TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
500 MOVWU frameSize+32(FP), T0
501 DISPATCH(runtime·call16, 16)
502 DISPATCH(runtime·call32, 32)
503 DISPATCH(runtime·call64, 64)
504 DISPATCH(runtime·call128, 128)
505 DISPATCH(runtime·call256, 256)
506 DISPATCH(runtime·call512, 512)
507 DISPATCH(runtime·call1024, 1024)
508 DISPATCH(runtime·call2048, 2048)
509 DISPATCH(runtime·call4096, 4096)
510 DISPATCH(runtime·call8192, 8192)
511 DISPATCH(runtime·call16384, 16384)
512 DISPATCH(runtime·call32768, 32768)
513 DISPATCH(runtime·call65536, 65536)
514 DISPATCH(runtime·call131072, 131072)
515 DISPATCH(runtime·call262144, 262144)
516 DISPATCH(runtime·call524288, 524288)
517 DISPATCH(runtime·call1048576, 1048576)
518 DISPATCH(runtime·call2097152, 2097152)
519 DISPATCH(runtime·call4194304, 4194304)
520 DISPATCH(runtime·call8388608, 8388608)
521 DISPATCH(runtime·call16777216, 16777216)
522 DISPATCH(runtime·call33554432, 33554432)
523 DISPATCH(runtime·call67108864, 67108864)
524 DISPATCH(runtime·call134217728, 134217728)
525 DISPATCH(runtime·call268435456, 268435456)
526 DISPATCH(runtime·call536870912, 536870912)
527 DISPATCH(runtime·call1073741824, 1073741824)
528 MOV $runtime·badreflectcall(SB), T2
529 JALR ZERO, T2
530
531#define CALLFN(NAME,MAXSIZE) \
532TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
533 NO_LOCAL_POINTERS; \
534 /* copy arguments to stack */ \
535 MOV stackArgs+16(FP), A1; \
536 MOVWU stackArgsSize+24(FP), A2; \
537 MOV X2, A3; \
538 ADD $8, A3; \
539 ADD A3, A2; \
540 BEQ A3, A2, 6(PC); \
541 MOVBU (A1), A4; \
542 ADD $1, A1; \
543 MOVB A4, (A3); \
544 ADD $1, A3; \
545 JMP -5(PC); \
546 /* set up argument registers */ \
547 MOV regArgs+40(FP), X25; \
548 CALL ·unspillArgs(SB); \
549 /* call function */ \
550 MOV f+8(FP), CTXT; \
551 MOV (CTXT), X25; \
552 PCDATA $PCDATA_StackMapIndex, $0; \
553 JALR RA, X25; \
554 /* copy return values back */ \
555 MOV regArgs+40(FP), X25; \
556 CALL ·spillArgs(SB); \
557 MOV stackArgsType+0(FP), A5; \
558 MOV stackArgs+16(FP), A1; \
559 MOVWU stackArgsSize+24(FP), A2; \
560 MOVWU stackRetOffset+28(FP), A4; \
561 ADD $8, X2, A3; \
562 ADD A4, A3; \
563 ADD A4, A1; \
564 SUB A4, A2; \
565 CALL callRet<>(SB); \
566 RET
567
568// callRet copies return values back at the end of call*. This is a
569// separate function so it can allocate stack space for the arguments
570// to reflectcallmove. It does not follow the Go ABI; it expects its
571// arguments in registers.
572TEXT callRet<>(SB), NOSPLIT, $40-0
573 NO_LOCAL_POINTERS
574 MOV A5, 8(X2)
575 MOV A1, 16(X2)
576 MOV A3, 24(X2)
577 MOV A2, 32(X2)
578 MOV X25, 40(X2)
579 CALL runtime·reflectcallmove(SB)
580 RET
581
582CALLFN(·call16, 16)
583CALLFN(·call32, 32)
584CALLFN(·call64, 64)
585CALLFN(·call128, 128)
586CALLFN(·call256, 256)
587CALLFN(·call512, 512)
588CALLFN(·call1024, 1024)
589CALLFN(·call2048, 2048)
590CALLFN(·call4096, 4096)
591CALLFN(·call8192, 8192)
592CALLFN(·call16384, 16384)
593CALLFN(·call32768, 32768)
594CALLFN(·call65536, 65536)
595CALLFN(·call131072, 131072)
596CALLFN(·call262144, 262144)
597CALLFN(·call524288, 524288)
598CALLFN(·call1048576, 1048576)
599CALLFN(·call2097152, 2097152)
600CALLFN(·call4194304, 4194304)
601CALLFN(·call8388608, 8388608)
602CALLFN(·call16777216, 16777216)
603CALLFN(·call33554432, 33554432)
604CALLFN(·call67108864, 67108864)
605CALLFN(·call134217728, 134217728)
606CALLFN(·call268435456, 268435456)
607CALLFN(·call536870912, 536870912)
608CALLFN(·call1073741824, 1073741824)
609
610// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
611// Must obey the gcc calling convention.
612TEXT _cgo_topofstack(SB),NOSPLIT,$8
613 // g (X27) and REG_TMP (X31) might be clobbered by load_g.
614 // X27 is callee-save in the gcc calling convention, so save it.
615 MOV g, savedX27-8(SP)
616
617 CALL runtime·load_g(SB)
618 MOV g_m(g), X5
619 MOV m_curg(X5), X5
620 MOV (g_stack+stack_hi)(X5), X10 // return value in X10
621
622 MOV savedX27-8(SP), g
623 RET
624
625// func goexit(neverCallThisFunction)
626// The top-most function running on a goroutine, returns to goexit+PCQuantum*2.
627// Note that the NOPs are written in a manner that will not be compressed,
628// since the offset must be known by the runtime.
629TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
630 WORD $0x00000013 // NOP
631 JMP runtime·goexit1(SB) // does not return
632 // traceback from goexit1 must hit code range of goexit
633 WORD $0x00000013 // NOP
634
635// This is called from .init_array and follows the platform, not the Go ABI.
636TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
637 // Use X31 as it is a scratch register in both the Go ABI and psABI.
638 MOV runtime·lastmoduledatap(SB), X31
639 MOV X10, moduledata_next(X31)
640 MOV X10, runtime·lastmoduledatap(SB)
641 RET
642
643// func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
644// See cgocall.go for more details.
645TEXT ·cgocallback(SB),NOSPLIT,$24-24
646 NO_LOCAL_POINTERS
647
648 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
649 // It is used to dropm while thread is exiting.
650 MOV fn+0(FP), X7
651 BNE ZERO, X7, loadg
652 // Restore the g from frame.
653 MOV frame+8(FP), g
654 JMP dropm
655
656loadg:
657 // Load m and g from thread-local storage.
658 MOVBU runtime·iscgo(SB), X5
659 BEQ ZERO, X5, nocgo
660 CALL runtime·load_g(SB)
661nocgo:
662
663 // If g is nil, Go did not create the current thread,
664 // or if this thread never called into Go on pthread platforms.
665 // Call needm to obtain one for temporary use.
666 // In this case, we're running on the thread stack, so there's
667 // lots of space, but the linker doesn't know. Hide the call from
668 // the linker analysis by using an indirect call.
669 BEQ ZERO, g, needm
670
671 MOV g_m(g), X5
672 MOV X5, savedm-8(SP)
673 JMP havem
674
675needm:
676 MOV g, savedm-8(SP) // g is zero, so is m.
677 MOV $runtime·needAndBindM(SB), X6
678 JALR RA, X6
679
680 // Set m->sched.sp = SP, so that if a panic happens
681 // during the function we are about to execute, it will
682 // have a valid SP to run on the g0 stack.
683 // The next few lines (after the havem label)
684 // will save this SP onto the stack and then write
685 // the same SP back to m->sched.sp. That seems redundant,
686 // but if an unrecovered panic happens, unwindm will
687 // restore the g->sched.sp from the stack location
688 // and then systemstack will try to use it. If we don't set it here,
689 // that restored SP will be uninitialized (typically 0) and
690 // will not be usable.
691 MOV g_m(g), X5
692 MOV m_g0(X5), X6
693 MOV X2, (g_sched+gobuf_sp)(X6)
694
695havem:
696 // Now there's a valid m, and we're running on its m->g0.
697 // Save current m->g0->sched.sp on stack and then set it to SP.
698 // Save current sp in m->g0->sched.sp in preparation for
699 // switch back to m->curg stack.
700 // NOTE: unwindm knows that the saved g->sched.sp is at 8(X2) aka savedsp-24(SP).
701 MOV m_g0(X5), X6
702 MOV (g_sched+gobuf_sp)(X6), X7
703 MOV X7, savedsp-24(SP) // must match frame size
704 MOV X2, (g_sched+gobuf_sp)(X6)
705
706 // Switch to m->curg stack and call runtime.cgocallbackg.
707 // Because we are taking over the execution of m->curg
708 // but *not* resuming what had been running, we need to
709 // save that information (m->curg->sched) so we can restore it.
710 // We can restore m->curg->sched.sp easily, because calling
711 // runtime.cgocallbackg leaves SP unchanged upon return.
712 // To save m->curg->sched.pc, we push it onto the curg stack and
713 // open a frame the same size as cgocallback's g0 frame.
714 // Once we switch to the curg stack, the pushed PC will appear
715 // to be the return PC of cgocallback, so that the traceback
716 // will seamlessly trace back into the earlier calls.
717 MOV m_curg(X5), g
718 CALL runtime·save_g(SB)
719 MOV (g_sched+gobuf_sp)(g), X6 // prepare stack as X6
720 MOV (g_sched+gobuf_pc)(g), X7
721 MOV X7, -(24+8)(X6) // "saved LR"; must match frame size
722 // Gather our arguments into registers.
723 MOV fn+0(FP), X7
724 MOV frame+8(FP), X8
725 MOV ctxt+16(FP), X9
726 MOV $-(24+8)(X6), X2 // switch stack; must match frame size
727 MOV X7, 8(X2)
728 MOV X8, 16(X2)
729 MOV X9, 24(X2)
730 CALL runtime·cgocallbackg(SB)
731
732 // Restore g->sched (== m->curg->sched) from saved values.
733 MOV 0(X2), X7
734 MOV X7, (g_sched+gobuf_pc)(g)
735 MOV $(24+8)(X2), X6 // must match frame size
736 MOV X6, (g_sched+gobuf_sp)(g)
737
738 // Switch back to m->g0's stack and restore m->g0->sched.sp.
739 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
740 // so we do not have to restore it.)
741 MOV g_m(g), X5
742 MOV m_g0(X5), g
743 CALL runtime·save_g(SB)
744 MOV (g_sched+gobuf_sp)(g), X2
745 MOV savedsp-24(SP), X6 // must match frame size
746 MOV X6, (g_sched+gobuf_sp)(g)
747
748 // If the m on entry was nil, we called needm above to borrow an m,
749 // 1. for the duration of the call on non-pthread platforms,
750 // 2. or the duration of the C thread alive on pthread platforms.
751 // If the m on entry wasn't nil,
752 // 1. the thread might be a Go thread,
753 // 2. or it wasn't the first call from a C thread on pthread platforms,
754 // since then we skip dropm to reuse the m in the first call.
755 MOV savedm-8(SP), X5
756 BNE ZERO, X5, droppedm
757
758 // Skip dropm to reuse it in the next call, when a pthread key has been created.
759 MOV _cgo_pthread_key_created(SB), X5
760 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
761 BEQ ZERO, X5, dropm
762 MOV (X5), X5
763 BNE ZERO, X5, droppedm
764
765dropm:
766 MOV $runtime·dropm(SB), X6
767 JALR RA, X6
768droppedm:
769
770 // Done!
771 RET
772
773TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
774 EBREAK
775 RET
776
777TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
778 EBREAK
779 RET
780
781// void setg(G*); set g. for use by needm.
782TEXT runtime·setg(SB), NOSPLIT, $0-8
783 MOV gg+0(FP), g
784 // This only happens if iscgo, so jump straight to save_g
785 CALL runtime·save_g(SB)
786 RET
787
788TEXT ·checkASM(SB),NOSPLIT,$0-1
789 MOV $1, T0
790 MOV T0, ret+0(FP)
791 RET
792
793// spillArgs stores return values from registers to a *internal/abi.RegArgs in X25.
794TEXT ·spillArgs(SB),NOSPLIT,$0-0
795 MOV X10, (0*8)(X25)
796 MOV X11, (1*8)(X25)
797 MOV X12, (2*8)(X25)
798 MOV X13, (3*8)(X25)
799 MOV X14, (4*8)(X25)
800 MOV X15, (5*8)(X25)
801 MOV X16, (6*8)(X25)
802 MOV X17, (7*8)(X25)
803 MOV X8, (8*8)(X25)
804 MOV X9, (9*8)(X25)
805 MOV X18, (10*8)(X25)
806 MOV X19, (11*8)(X25)
807 MOV X20, (12*8)(X25)
808 MOV X21, (13*8)(X25)
809 MOV X22, (14*8)(X25)
810 MOV X23, (15*8)(X25)
811 MOVD F10, (16*8)(X25)
812 MOVD F11, (17*8)(X25)
813 MOVD F12, (18*8)(X25)
814 MOVD F13, (19*8)(X25)
815 MOVD F14, (20*8)(X25)
816 MOVD F15, (21*8)(X25)
817 MOVD F16, (22*8)(X25)
818 MOVD F17, (23*8)(X25)
819 MOVD F8, (24*8)(X25)
820 MOVD F9, (25*8)(X25)
821 MOVD F18, (26*8)(X25)
822 MOVD F19, (27*8)(X25)
823 MOVD F20, (28*8)(X25)
824 MOVD F21, (29*8)(X25)
825 MOVD F22, (30*8)(X25)
826 MOVD F23, (31*8)(X25)
827 RET
828
829// unspillArgs loads args into registers from a *internal/abi.RegArgs in X25.
830TEXT ·unspillArgs(SB),NOSPLIT,$0-0
831 MOV (0*8)(X25), X10
832 MOV (1*8)(X25), X11
833 MOV (2*8)(X25), X12
834 MOV (3*8)(X25), X13
835 MOV (4*8)(X25), X14
836 MOV (5*8)(X25), X15
837 MOV (6*8)(X25), X16
838 MOV (7*8)(X25), X17
839 MOV (8*8)(X25), X8
840 MOV (9*8)(X25), X9
841 MOV (10*8)(X25), X18
842 MOV (11*8)(X25), X19
843 MOV (12*8)(X25), X20
844 MOV (13*8)(X25), X21
845 MOV (14*8)(X25), X22
846 MOV (15*8)(X25), X23
847 MOVD (16*8)(X25), F10
848 MOVD (17*8)(X25), F11
849 MOVD (18*8)(X25), F12
850 MOVD (19*8)(X25), F13
851 MOVD (20*8)(X25), F14
852 MOVD (21*8)(X25), F15
853 MOVD (22*8)(X25), F16
854 MOVD (23*8)(X25), F17
855 MOVD (24*8)(X25), F8
856 MOVD (25*8)(X25), F9
857 MOVD (26*8)(X25), F18
858 MOVD (27*8)(X25), F19
859 MOVD (28*8)(X25), F20
860 MOVD (29*8)(X25), F21
861 MOVD (30*8)(X25), F22
862 MOVD (31*8)(X25), F23
863 RET
864
865// gcWriteBarrier informs the GC about heap pointer writes.
866//
867// gcWriteBarrier does NOT follow the Go ABI. It accepts the
868// number of bytes of buffer needed in X24, and returns a pointer
869// to the buffer space in X24.
870// It clobbers X31 aka T6 (the linker temp register - REG_TMP).
871// The act of CALLing gcWriteBarrier will clobber RA (LR).
872// It does not clobber any other general-purpose registers,
873// but may clobber others (e.g., floating point registers).
874TEXT gcWriteBarrier<>(SB),NOSPLIT,$208
875 // Save the registers clobbered by the fast path.
876 MOV A0, 24*8(X2)
877 MOV A1, 25*8(X2)
878retry:
879 MOV g_m(g), A0
880 MOV m_p(A0), A0
881 MOV (p_wbBuf+wbBuf_next)(A0), A1
882 MOV (p_wbBuf+wbBuf_end)(A0), T6 // T6 is linker temp register (REG_TMP)
883 // Increment wbBuf.next position.
884 ADD X24, A1
885 // Is the buffer full?
886 BLTU T6, A1, flush
887 // Commit to the larger buffer.
888 MOV A1, (p_wbBuf+wbBuf_next)(A0)
889 // Make the return value (the original next position)
890 SUB X24, A1, X24
891 // Restore registers.
892 MOV 24*8(X2), A0
893 MOV 25*8(X2), A1
894 RET
895
896flush:
897 // Save all general purpose registers since these could be
898 // clobbered by wbBufFlush and were not saved by the caller.
899 MOV T0, 1*8(X2)
900 MOV T1, 2*8(X2)
901 // X0 is zero register
902 // X1 is LR, saved by prologue
903 // X2 is SP
904 // X3 is GP
905 // X4 is TP
906 MOV X7, 3*8(X2)
907 MOV X8, 4*8(X2)
908 MOV X9, 5*8(X2)
909 // X10 already saved (A0)
910 // X11 already saved (A1)
911 MOV X12, 6*8(X2)
912 MOV X13, 7*8(X2)
913 MOV X14, 8*8(X2)
914 MOV X15, 9*8(X2)
915 MOV X16, 10*8(X2)
916 MOV X17, 11*8(X2)
917 MOV X18, 12*8(X2)
918 MOV X19, 13*8(X2)
919 MOV X20, 14*8(X2)
920 MOV X21, 15*8(X2)
921 MOV X22, 16*8(X2)
922 MOV X23, 17*8(X2)
923 MOV X24, 18*8(X2)
924 MOV X25, 19*8(X2)
925 MOV X26, 20*8(X2)
926 // X27 is g.
927 MOV X28, 21*8(X2)
928 MOV X29, 22*8(X2)
929 MOV X30, 23*8(X2)
930 // X31 is tmp register.
931
932 CALL runtime·wbBufFlush(SB)
933
934 MOV 1*8(X2), T0
935 MOV 2*8(X2), T1
936 MOV 3*8(X2), X7
937 MOV 4*8(X2), X8
938 MOV 5*8(X2), X9
939 MOV 6*8(X2), X12
940 MOV 7*8(X2), X13
941 MOV 8*8(X2), X14
942 MOV 9*8(X2), X15
943 MOV 10*8(X2), X16
944 MOV 11*8(X2), X17
945 MOV 12*8(X2), X18
946 MOV 13*8(X2), X19
947 MOV 14*8(X2), X20
948 MOV 15*8(X2), X21
949 MOV 16*8(X2), X22
950 MOV 17*8(X2), X23
951 MOV 18*8(X2), X24
952 MOV 19*8(X2), X25
953 MOV 20*8(X2), X26
954 MOV 21*8(X2), X28
955 MOV 22*8(X2), X29
956 MOV 23*8(X2), X30
957
958 JMP retry
959
960TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
961 MOV $8, X24
962 JMP gcWriteBarrier<>(SB)
963TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
964 MOV $16, X24
965 JMP gcWriteBarrier<>(SB)
966TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
967 MOV $24, X24
968 JMP gcWriteBarrier<>(SB)
969TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
970 MOV $32, X24
971 JMP gcWriteBarrier<>(SB)
972TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
973 MOV $40, X24
974 JMP gcWriteBarrier<>(SB)
975TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
976 MOV $48, X24
977 JMP gcWriteBarrier<>(SB)
978TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
979 MOV $56, X24
980 JMP gcWriteBarrier<>(SB)
981TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
982 MOV $64, X24
983 JMP gcWriteBarrier<>(SB)
984
985TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$144-0
986 NO_LOCAL_POINTERS
987 // Save all 16 int registers that could have an index in them.
988 // They may be pointers, but if they are they are dead.
989 // Skip X0 aka ZERO, X1 aka LR, X2 aka SP, X3 aka GP, X4 aka TP.
990 MOV X5, 24(X2)
991 MOV X6, 32(X2)
992 MOV X7, 40(X2)
993 MOV X8, 48(X2)
994 MOV X9, 56(X2)
995 MOV X10, 64(X2)
996 MOV X11, 72(X2)
997 MOV X12, 80(X2)
998 MOV X13, 88(X2)
999 MOV X14, 96(X2)
1000 MOV X15, 104(X2)
1001 MOV X16, 112(X2)
1002 MOV X17, 120(X2)
1003 MOV X18, 128(X2)
1004 MOV X19, 136(X2)
1005 MOV X20, 144(X2)
1006
1007 MOV X1, X10 // PC immediately after call to panicBounds
1008 ADD $24, X2, X11 // pointer to save area
1009 CALL runtime·panicBounds64<ABIInternal>(SB)
1010 RET
1011
1012DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
1013GLOBL runtime·mainPC(SB),RODATA,$8
View as plain text