Text file src/runtime/asm_riscv64.s

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  #include "go_asm.h"
     6  #include "funcdata.h"
     7  #include "textflag.h"
     8  
     9  // func rt0_go()
    10  TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
    11  	// X2 = stack; A0 = argc; A1 = argv
    12  	SUB	$24, X2
    13  	MOV	A0, 8(X2)	// argc
    14  	MOV	A1, 16(X2)	// argv
    15  
    16  	// create istack out of the given (operating system) stack.
    17  	// _cgo_init may update stackguard.
    18  	MOV	$runtime·g0(SB), g
    19  	MOV	$(-64*1024), T0
    20  	ADD	T0, X2, T1
    21  	MOV	T1, g_stackguard0(g)
    22  	MOV	T1, g_stackguard1(g)
    23  	MOV	T1, (g_stack+stack_lo)(g)
    24  	MOV	X2, (g_stack+stack_hi)(g)
    25  
    26  	// if there is a _cgo_init, call it using the gcc ABI.
    27  	MOV	_cgo_init(SB), T0
    28  	BEQ	T0, ZERO, nocgo
    29  
    30  	MOV	ZERO, A3		// arg 3: not used
    31  	MOV	ZERO, A2		// arg 2: not used
    32  	MOV	$setg_gcc<>(SB), A1	// arg 1: setg
    33  	MOV	g, A0			// arg 0: G
    34  	JALR	RA, T0
    35  
    36  nocgo:
    37  	// update stackguard after _cgo_init
    38  	MOV	(g_stack+stack_lo)(g), T0
    39  	ADD	$const_stackGuard, T0
    40  	MOV	T0, g_stackguard0(g)
    41  	MOV	T0, g_stackguard1(g)
    42  
    43  	// set the per-goroutine and per-mach "registers"
    44  	MOV	$runtime·m0(SB), T0
    45  
    46  	// save m->g0 = g0
    47  	MOV	g, m_g0(T0)
    48  	// save m0 to g0->m
    49  	MOV	T0, g_m(g)
    50  
    51  	CALL	runtime·check(SB)
    52  
    53  	// args are already prepared
    54  	CALL	runtime·args(SB)
    55  	CALL	runtime·osinit(SB)
    56  	CALL	runtime·schedinit(SB)
    57  
    58  	// create a new goroutine to start program
    59  	MOV	$runtime·mainPC(SB), T0		// entry
    60  	SUB	$16, X2
    61  	MOV	T0, 8(X2)
    62  	MOV	ZERO, 0(X2)
    63  	CALL	runtime·newproc(SB)
    64  	ADD	$16, X2
    65  
    66  	// start this M
    67  	CALL	runtime·mstart(SB)
    68  
    69  	WORD $0 // crash if reached
    70  	RET
    71  
    72  TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
    73  	CALL	runtime·mstart0(SB)
    74  	RET // not reached
    75  
    76  // void setg_gcc(G*); set g called from gcc with g in A0
    77  TEXT setg_gcc<>(SB),NOSPLIT,$0-0
    78  	MOV	A0, g
    79  	CALL	runtime·save_g(SB)
    80  	RET
    81  
    82  // func cputicks() int64
    83  TEXT runtime·cputicks(SB),NOSPLIT,$0-8
    84  	// RDTIME to emulate cpu ticks
    85  	// RDCYCLE reads counter that is per HART(core) based
    86  	// according to the riscv manual, see issue 46737
    87  	RDTIME	A0
    88  	MOV	A0, ret+0(FP)
    89  	RET
    90  
    91  // systemstack_switch is a dummy routine that systemstack leaves at the bottom
    92  // of the G stack. We need to distinguish the routine that
    93  // lives at the bottom of the G stack from the one that lives
    94  // at the top of the system stack because the one at the top of
    95  // the system stack terminates the stack walk (see topofstack()).
    96  TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
    97  	UNDEF
    98  	JALR	RA, ZERO	// make sure this function is not leaf
    99  	RET
   100  
   101  // func systemstack(fn func())
   102  TEXT runtime·systemstack(SB), NOSPLIT, $0-8
   103  	MOV	fn+0(FP), CTXT	// CTXT = fn
   104  	MOV	g_m(g), T0	// T0 = m
   105  
   106  	MOV	m_gsignal(T0), T1	// T1 = gsignal
   107  	BEQ	g, T1, noswitch
   108  
   109  	MOV	m_g0(T0), T1	// T1 = g0
   110  	BEQ	g, T1, noswitch
   111  
   112  	MOV	m_curg(T0), T2
   113  	BEQ	g, T2, switch
   114  
   115  	// Bad: g is not gsignal, not g0, not curg. What is it?
   116  	// Hide call from linker nosplit analysis.
   117  	MOV	$runtime·badsystemstack(SB), T1
   118  	JALR	RA, T1
   119  
   120  switch:
   121  	// save our state in g->sched. Pretend to
   122  	// be systemstack_switch if the G stack is scanned.
   123  	CALL	gosave_systemstack_switch<>(SB)
   124  
   125  	// switch to g0
   126  	MOV	T1, g
   127  	CALL	runtime·save_g(SB)
   128  	MOV	(g_sched+gobuf_sp)(g), T0
   129  	MOV	T0, X2
   130  
   131  	// call target function
   132  	MOV	0(CTXT), T1	// code pointer
   133  	JALR	RA, T1
   134  
   135  	// switch back to g
   136  	MOV	g_m(g), T0
   137  	MOV	m_curg(T0), g
   138  	CALL	runtime·save_g(SB)
   139  	MOV	(g_sched+gobuf_sp)(g), X2
   140  	MOV	ZERO, (g_sched+gobuf_sp)(g)
   141  	RET
   142  
   143  noswitch:
   144  	// already on m stack, just call directly
   145  	// Using a tail call here cleans up tracebacks since we won't stop
   146  	// at an intermediate systemstack.
   147  	MOV	0(CTXT), T1	// code pointer
   148  	ADD	$8, X2
   149  	JMP	(T1)
   150  
   151  // func switchToCrashStack0(fn func())
   152  TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
   153  	MOV	X10, CTXT			// context register
   154  	MOV	g_m(g), X11			// curm
   155  
   156  	// set g to gcrash
   157  	MOV	$runtime·gcrash(SB), g	// g = &gcrash
   158  	CALL	runtime·save_g(SB)	// clobbers X31
   159  	MOV	X11, g_m(g)			// g.m = curm
   160  	MOV	g, m_g0(X11)			// curm.g0 = g
   161  
   162  	// switch to crashstack
   163  	MOV	(g_stack+stack_hi)(g), X11
   164  	SUB	$(4*8), X11
   165  	MOV	X11, X2
   166  
   167  	// call target function
   168  	MOV	0(CTXT), X10
   169  	JALR	X1, X10
   170  
   171  	// should never return
   172  	CALL	runtime·abort(SB)
   173  	UNDEF
   174  
   175  /*
   176   * support for morestack
   177   */
   178  
   179  // Called during function prolog when more stack is needed.
   180  // Called with return address (i.e. caller's PC) in X5 (aka T0),
   181  // and the LR register contains the caller's LR.
   182  //
   183  // The traceback routines see morestack on a g0 as being
   184  // the top of a stack (for example, morestack calling newstack
   185  // calling the scheduler calling newm calling gc), so we must
   186  // record an argument size. For that purpose, it has no arguments.
   187  
   188  // func morestack()
   189  TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
   190  	// Called from f.
   191  	// Set g->sched to context in f.
   192  	MOV	X2, (g_sched+gobuf_sp)(g)
   193  	MOV	T0, (g_sched+gobuf_pc)(g)
   194  	MOV	RA, (g_sched+gobuf_lr)(g)
   195  	MOV	CTXT, (g_sched+gobuf_ctxt)(g)
   196  
   197  	// Cannot grow scheduler stack (m->g0).
   198  	MOV	g_m(g), A0
   199  	MOV	m_g0(A0), A1
   200  	BNE	g, A1, 3(PC)
   201  	CALL	runtime·badmorestackg0(SB)
   202  	CALL	runtime·abort(SB)
   203  
   204  	// Cannot grow signal stack (m->gsignal).
   205  	MOV	m_gsignal(A0), A1
   206  	BNE	g, A1, 3(PC)
   207  	CALL	runtime·badmorestackgsignal(SB)
   208  	CALL	runtime·abort(SB)
   209  
   210  	// Called from f.
   211  	// Set m->morebuf to f's caller.
   212  	MOV	RA, (m_morebuf+gobuf_pc)(A0)	// f's caller's PC
   213  	MOV	X2, (m_morebuf+gobuf_sp)(A0)	// f's caller's SP
   214  	MOV	g, (m_morebuf+gobuf_g)(A0)
   215  
   216  	// Call newstack on m->g0's stack.
   217  	MOV	m_g0(A0), g
   218  	CALL	runtime·save_g(SB)
   219  	MOV	(g_sched+gobuf_sp)(g), X2
   220  	// Create a stack frame on g0 to call newstack.
   221  	MOV	ZERO, -8(X2)	// Zero saved LR in frame
   222  	SUB	$8, X2
   223  	CALL	runtime·newstack(SB)
   224  
   225  	// Not reached, but make sure the return PC from the call to newstack
   226  	// is still in this function, and not the beginning of the next.
   227  	UNDEF
   228  
   229  // func morestack_noctxt()
   230  TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
   231  	// Force SPWRITE. This function doesn't actually write SP,
   232  	// but it is called with a special calling convention where
   233  	// the caller doesn't save LR on stack but passes it as a
   234  	// register, and the unwinder currently doesn't understand.
   235  	// Make it SPWRITE to stop unwinding. (See issue 54332)
   236  	MOV	X2, X2
   237  
   238  	MOV	ZERO, CTXT
   239  	JMP	runtime·morestack(SB)
   240  
   241  // AES hashing not implemented for riscv64
   242  TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
   243  	JMP	runtime·memhashFallback<ABIInternal>(SB)
   244  TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
   245  	JMP	runtime·strhashFallback<ABIInternal>(SB)
   246  TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
   247  	JMP	runtime·memhash32Fallback<ABIInternal>(SB)
   248  TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
   249  	JMP	runtime·memhash64Fallback<ABIInternal>(SB)
   250  
   251  // func return0()
   252  TEXT runtime·return0(SB), NOSPLIT, $0
   253  	MOV	$0, A0
   254  	RET
   255  
   256  // restore state from Gobuf; longjmp
   257  
   258  // func gogo(buf *gobuf)
   259  TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
   260  	MOV	buf+0(FP), T0
   261  	MOV	gobuf_g(T0), T1
   262  	MOV	0(T1), ZERO // make sure g != nil
   263  	JMP	gogo<>(SB)
   264  
   265  TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
   266  	MOV	T1, g
   267  	CALL	runtime·save_g(SB)
   268  
   269  	MOV	gobuf_sp(T0), X2
   270  	MOV	gobuf_lr(T0), RA
   271  	MOV	gobuf_ret(T0), A0
   272  	MOV	gobuf_ctxt(T0), CTXT
   273  	MOV	ZERO, gobuf_sp(T0)
   274  	MOV	ZERO, gobuf_ret(T0)
   275  	MOV	ZERO, gobuf_lr(T0)
   276  	MOV	ZERO, gobuf_ctxt(T0)
   277  	MOV	gobuf_pc(T0), T0
   278  	JALR	ZERO, T0
   279  
   280  // func procyield(cycles uint32)
   281  TEXT runtime·procyield(SB),NOSPLIT,$0-0
   282  	RET
   283  
   284  // Switch to m->g0's stack, call fn(g).
   285  // Fn must never return. It should gogo(&g->sched)
   286  // to keep running g.
   287  
   288  // func mcall(fn func(*g))
   289  TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
   290  	MOV	X10, CTXT
   291  
   292  	// Save caller state in g->sched
   293  	MOV	X2, (g_sched+gobuf_sp)(g)
   294  	MOV	RA, (g_sched+gobuf_pc)(g)
   295  	MOV	ZERO, (g_sched+gobuf_lr)(g)
   296  
   297  	// Switch to m->g0 & its stack, call fn.
   298  	MOV	g, X10
   299  	MOV	g_m(g), T1
   300  	MOV	m_g0(T1), g
   301  	CALL	runtime·save_g(SB)
   302  	BNE	g, X10, 2(PC)
   303  	JMP	runtime·badmcall(SB)
   304  	MOV	0(CTXT), T1			// code pointer
   305  	MOV	(g_sched+gobuf_sp)(g), X2	// sp = m->g0->sched.sp
   306  	// we don't need special macro for regabi since arg0(X10) = g
   307  	SUB	$16, X2
   308  	MOV	X10, 8(X2)			// setup g
   309  	MOV	ZERO, 0(X2)			// clear return address
   310  	JALR	RA, T1
   311  	JMP	runtime·badmcall2(SB)
   312  
   313  // Save state of caller into g->sched,
   314  // but using fake PC from systemstack_switch.
   315  // Must only be called from functions with no locals ($0)
   316  // or else unwinding from systemstack_switch is incorrect.
   317  // Smashes X31.
   318  TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
   319  	MOV	$runtime·systemstack_switch(SB), X31
   320  	ADD	$8, X31	// get past prologue
   321  	MOV	X31, (g_sched+gobuf_pc)(g)
   322  	MOV	X2, (g_sched+gobuf_sp)(g)
   323  	MOV	ZERO, (g_sched+gobuf_lr)(g)
   324  	MOV	ZERO, (g_sched+gobuf_ret)(g)
   325  	// Assert ctxt is zero. See func save.
   326  	MOV	(g_sched+gobuf_ctxt)(g), X31
   327  	BEQ	ZERO, X31, 2(PC)
   328  	CALL	runtime·abort(SB)
   329  	RET
   330  
   331  // func asmcgocall_no_g(fn, arg unsafe.Pointer)
   332  // Call fn(arg) aligned appropriately for the gcc ABI.
   333  // Called on a system stack, and there may be no g yet (during needm).
   334  TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
   335  	MOV	fn+0(FP), X5
   336  	MOV	arg+8(FP), X10
   337  	JALR	RA, (X5)
   338  	RET
   339  
   340  // func asmcgocall(fn, arg unsafe.Pointer) int32
   341  // Call fn(arg) on the scheduler stack,
   342  // aligned appropriately for the gcc ABI.
   343  // See cgocall.go for more details.
   344  TEXT ·asmcgocall(SB),NOSPLIT,$0-20
   345  	MOV	fn+0(FP), X5
   346  	MOV	arg+8(FP), X10
   347  
   348  	MOV	X2, X8	// save original stack pointer
   349  	MOV	g, X9
   350  
   351  	// Figure out if we need to switch to m->g0 stack.
   352  	// We get called to create new OS threads too, and those
   353  	// come in on the m->g0 stack already. Or we might already
   354  	// be on the m->gsignal stack.
   355  	MOV	g_m(g), X6
   356  	MOV	m_gsignal(X6), X7
   357  	BEQ	X7, g, g0
   358  	MOV	m_g0(X6), X7
   359  	BEQ	X7, g, g0
   360  
   361  	CALL	gosave_systemstack_switch<>(SB)
   362  	MOV	X7, g
   363  	CALL	runtime·save_g(SB)
   364  	MOV	(g_sched+gobuf_sp)(g), X2
   365  
   366  	// Now on a scheduling stack (a pthread-created stack).
   367  g0:
   368  	// Save room for two of our pointers.
   369  	SUB	$16, X2
   370  	MOV	X9, 0(X2)	// save old g on stack
   371  	MOV	(g_stack+stack_hi)(X9), X9
   372  	SUB	X8, X9, X8
   373  	MOV	X8, 8(X2)	// save depth in old g stack (can't just save SP, as stack might be copied during a callback)
   374  
   375  	JALR	RA, (X5)
   376  
   377  	// Restore g, stack pointer. X10 is return value.
   378  	MOV	0(X2), g
   379  	CALL	runtime·save_g(SB)
   380  	MOV	(g_stack+stack_hi)(g), X5
   381  	MOV	8(X2), X6
   382  	SUB	X6, X5, X6
   383  	MOV	X6, X2
   384  
   385  	MOVW	X10, ret+16(FP)
   386  	RET
   387  
   388  // func asminit()
   389  TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
   390  	RET
   391  
   392  // reflectcall: call a function with the given argument list
   393  // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
   394  // we don't have variable-sized frames, so we use a small number
   395  // of constant-sized-frame functions to encode a few bits of size in the pc.
   396  // Caution: ugly multiline assembly macros in your future!
   397  
   398  #define DISPATCH(NAME,MAXSIZE)	\
   399  	MOV	$MAXSIZE, T1	\
   400  	BLTU	T1, T0, 3(PC)	\
   401  	MOV	$NAME(SB), T2;	\
   402  	JALR	ZERO, T2
   403  // Note: can't just "BR NAME(SB)" - bad inlining results.
   404  
   405  // func call(stackArgsType *rtype, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
   406  TEXT reflect·call(SB), NOSPLIT, $0-0
   407  	JMP	·reflectcall(SB)
   408  
   409  // func call(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
   410  TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
   411  	MOVWU	frameSize+32(FP), T0
   412  	DISPATCH(runtime·call16, 16)
   413  	DISPATCH(runtime·call32, 32)
   414  	DISPATCH(runtime·call64, 64)
   415  	DISPATCH(runtime·call128, 128)
   416  	DISPATCH(runtime·call256, 256)
   417  	DISPATCH(runtime·call512, 512)
   418  	DISPATCH(runtime·call1024, 1024)
   419  	DISPATCH(runtime·call2048, 2048)
   420  	DISPATCH(runtime·call4096, 4096)
   421  	DISPATCH(runtime·call8192, 8192)
   422  	DISPATCH(runtime·call16384, 16384)
   423  	DISPATCH(runtime·call32768, 32768)
   424  	DISPATCH(runtime·call65536, 65536)
   425  	DISPATCH(runtime·call131072, 131072)
   426  	DISPATCH(runtime·call262144, 262144)
   427  	DISPATCH(runtime·call524288, 524288)
   428  	DISPATCH(runtime·call1048576, 1048576)
   429  	DISPATCH(runtime·call2097152, 2097152)
   430  	DISPATCH(runtime·call4194304, 4194304)
   431  	DISPATCH(runtime·call8388608, 8388608)
   432  	DISPATCH(runtime·call16777216, 16777216)
   433  	DISPATCH(runtime·call33554432, 33554432)
   434  	DISPATCH(runtime·call67108864, 67108864)
   435  	DISPATCH(runtime·call134217728, 134217728)
   436  	DISPATCH(runtime·call268435456, 268435456)
   437  	DISPATCH(runtime·call536870912, 536870912)
   438  	DISPATCH(runtime·call1073741824, 1073741824)
   439  	MOV	$runtime·badreflectcall(SB), T2
   440  	JALR	ZERO, T2
   441  
   442  #define CALLFN(NAME,MAXSIZE)			\
   443  TEXT NAME(SB), WRAPPER, $MAXSIZE-48;		\
   444  	NO_LOCAL_POINTERS;			\
   445  	/* copy arguments to stack */		\
   446  	MOV	stackArgs+16(FP), A1;			\
   447  	MOVWU	stackArgsSize+24(FP), A2;		\
   448  	MOV	X2, A3;				\
   449  	ADD	$8, A3;				\
   450  	ADD	A3, A2;				\
   451  	BEQ	A3, A2, 6(PC);			\
   452  	MOVBU	(A1), A4;			\
   453  	ADD	$1, A1;				\
   454  	MOVB	A4, (A3);			\
   455  	ADD	$1, A3;				\
   456  	JMP	-5(PC);				\
   457  	/* set up argument registers */		\
   458  	MOV	regArgs+40(FP), X25;		\
   459  	CALL	·unspillArgs(SB);		\
   460  	/* call function */			\
   461  	MOV	f+8(FP), CTXT;			\
   462  	MOV	(CTXT), X25;			\
   463  	PCDATA  $PCDATA_StackMapIndex, $0;	\
   464  	JALR	RA, X25;				\
   465  	/* copy return values back */		\
   466  	MOV	regArgs+40(FP), X25;		\
   467  	CALL	·spillArgs(SB);		\
   468  	MOV	stackArgsType+0(FP), A5;		\
   469  	MOV	stackArgs+16(FP), A1;			\
   470  	MOVWU	stackArgsSize+24(FP), A2;			\
   471  	MOVWU	stackRetOffset+28(FP), A4;		\
   472  	ADD	$8, X2, A3;			\
   473  	ADD	A4, A3; 			\
   474  	ADD	A4, A1;				\
   475  	SUB	A4, A2;				\
   476  	CALL	callRet<>(SB);			\
   477  	RET
   478  
   479  // callRet copies return values back at the end of call*. This is a
   480  // separate function so it can allocate stack space for the arguments
   481  // to reflectcallmove. It does not follow the Go ABI; it expects its
   482  // arguments in registers.
   483  TEXT callRet<>(SB), NOSPLIT, $40-0
   484  	NO_LOCAL_POINTERS
   485  	MOV	A5, 8(X2)
   486  	MOV	A1, 16(X2)
   487  	MOV	A3, 24(X2)
   488  	MOV	A2, 32(X2)
   489  	MOV	X25, 40(X2)
   490  	CALL	runtime·reflectcallmove(SB)
   491  	RET
   492  
   493  CALLFN(·call16, 16)
   494  CALLFN(·call32, 32)
   495  CALLFN(·call64, 64)
   496  CALLFN(·call128, 128)
   497  CALLFN(·call256, 256)
   498  CALLFN(·call512, 512)
   499  CALLFN(·call1024, 1024)
   500  CALLFN(·call2048, 2048)
   501  CALLFN(·call4096, 4096)
   502  CALLFN(·call8192, 8192)
   503  CALLFN(·call16384, 16384)
   504  CALLFN(·call32768, 32768)
   505  CALLFN(·call65536, 65536)
   506  CALLFN(·call131072, 131072)
   507  CALLFN(·call262144, 262144)
   508  CALLFN(·call524288, 524288)
   509  CALLFN(·call1048576, 1048576)
   510  CALLFN(·call2097152, 2097152)
   511  CALLFN(·call4194304, 4194304)
   512  CALLFN(·call8388608, 8388608)
   513  CALLFN(·call16777216, 16777216)
   514  CALLFN(·call33554432, 33554432)
   515  CALLFN(·call67108864, 67108864)
   516  CALLFN(·call134217728, 134217728)
   517  CALLFN(·call268435456, 268435456)
   518  CALLFN(·call536870912, 536870912)
   519  CALLFN(·call1073741824, 1073741824)
   520  
   521  // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
   522  // Must obey the gcc calling convention.
   523  TEXT _cgo_topofstack(SB),NOSPLIT,$8
   524  	// g (X27) and REG_TMP (X31) might be clobbered by load_g.
   525  	// X27 is callee-save in the gcc calling convention, so save it.
   526  	MOV	g, savedX27-8(SP)
   527  
   528  	CALL	runtime·load_g(SB)
   529  	MOV	g_m(g), X5
   530  	MOV	m_curg(X5), X5
   531  	MOV	(g_stack+stack_hi)(X5), X10 // return value in X10
   532  
   533  	MOV	savedX27-8(SP), g
   534  	RET
   535  
   536  // func goexit(neverCallThisFunction)
   537  // The top-most function running on a goroutine
   538  // returns to goexit+PCQuantum.
   539  TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
   540  	MOV	ZERO, ZERO	// NOP
   541  	JMP	runtime·goexit1(SB)	// does not return
   542  	// traceback from goexit1 must hit code range of goexit
   543  	MOV	ZERO, ZERO	// NOP
   544  
   545  // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
   546  // See cgocall.go for more details.
   547  TEXT ·cgocallback(SB),NOSPLIT,$24-24
   548  	NO_LOCAL_POINTERS
   549  
   550  	// Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
   551  	// It is used to dropm while thread is exiting.
   552  	MOV	fn+0(FP), X7
   553  	BNE	ZERO, X7, loadg
   554  	// Restore the g from frame.
   555  	MOV	frame+8(FP), g
   556  	JMP	dropm
   557  
   558  loadg:
   559  	// Load m and g from thread-local storage.
   560  	MOVBU	runtime·iscgo(SB), X5
   561  	BEQ	ZERO, X5, nocgo
   562  	CALL	runtime·load_g(SB)
   563  nocgo:
   564  
   565  	// If g is nil, Go did not create the current thread,
   566  	// or if this thread never called into Go on pthread platforms.
   567  	// Call needm to obtain one for temporary use.
   568  	// In this case, we're running on the thread stack, so there's
   569  	// lots of space, but the linker doesn't know. Hide the call from
   570  	// the linker analysis by using an indirect call.
   571  	BEQ	ZERO, g, needm
   572  
   573  	MOV	g_m(g), X5
   574  	MOV	X5, savedm-8(SP)
   575  	JMP	havem
   576  
   577  needm:
   578  	MOV	g, savedm-8(SP) // g is zero, so is m.
   579  	MOV	$runtime·needAndBindM(SB), X6
   580  	JALR	RA, X6
   581  
   582  	// Set m->sched.sp = SP, so that if a panic happens
   583  	// during the function we are about to execute, it will
   584  	// have a valid SP to run on the g0 stack.
   585  	// The next few lines (after the havem label)
   586  	// will save this SP onto the stack and then write
   587  	// the same SP back to m->sched.sp. That seems redundant,
   588  	// but if an unrecovered panic happens, unwindm will
   589  	// restore the g->sched.sp from the stack location
   590  	// and then systemstack will try to use it. If we don't set it here,
   591  	// that restored SP will be uninitialized (typically 0) and
   592  	// will not be usable.
   593  	MOV	g_m(g), X5
   594  	MOV	m_g0(X5), X6
   595  	MOV	X2, (g_sched+gobuf_sp)(X6)
   596  
   597  havem:
   598  	// Now there's a valid m, and we're running on its m->g0.
   599  	// Save current m->g0->sched.sp on stack and then set it to SP.
   600  	// Save current sp in m->g0->sched.sp in preparation for
   601  	// switch back to m->curg stack.
   602  	// NOTE: unwindm knows that the saved g->sched.sp is at 8(X2) aka savedsp-24(SP).
   603  	MOV	m_g0(X5), X6
   604  	MOV	(g_sched+gobuf_sp)(X6), X7
   605  	MOV	X7, savedsp-24(SP)	// must match frame size
   606  	MOV	X2, (g_sched+gobuf_sp)(X6)
   607  
   608  	// Switch to m->curg stack and call runtime.cgocallbackg.
   609  	// Because we are taking over the execution of m->curg
   610  	// but *not* resuming what had been running, we need to
   611  	// save that information (m->curg->sched) so we can restore it.
   612  	// We can restore m->curg->sched.sp easily, because calling
   613  	// runtime.cgocallbackg leaves SP unchanged upon return.
   614  	// To save m->curg->sched.pc, we push it onto the curg stack and
   615  	// open a frame the same size as cgocallback's g0 frame.
   616  	// Once we switch to the curg stack, the pushed PC will appear
   617  	// to be the return PC of cgocallback, so that the traceback
   618  	// will seamlessly trace back into the earlier calls.
   619  	MOV	m_curg(X5), g
   620  	CALL	runtime·save_g(SB)
   621  	MOV	(g_sched+gobuf_sp)(g), X6 // prepare stack as X6
   622  	MOV	(g_sched+gobuf_pc)(g), X7
   623  	MOV	X7, -(24+8)(X6)		// "saved LR"; must match frame size
   624  	// Gather our arguments into registers.
   625  	MOV	fn+0(FP), X7
   626  	MOV	frame+8(FP), X8
   627  	MOV	ctxt+16(FP), X9
   628  	MOV	$-(24+8)(X6), X2	// switch stack; must match frame size
   629  	MOV	X7, 8(X2)
   630  	MOV	X8, 16(X2)
   631  	MOV	X9, 24(X2)
   632  	CALL	runtime·cgocallbackg(SB)
   633  
   634  	// Restore g->sched (== m->curg->sched) from saved values.
   635  	MOV	0(X2), X7
   636  	MOV	X7, (g_sched+gobuf_pc)(g)
   637  	MOV	$(24+8)(X2), X6		// must match frame size
   638  	MOV	X6, (g_sched+gobuf_sp)(g)
   639  
   640  	// Switch back to m->g0's stack and restore m->g0->sched.sp.
   641  	// (Unlike m->curg, the g0 goroutine never uses sched.pc,
   642  	// so we do not have to restore it.)
   643  	MOV	g_m(g), X5
   644  	MOV	m_g0(X5), g
   645  	CALL	runtime·save_g(SB)
   646  	MOV	(g_sched+gobuf_sp)(g), X2
   647  	MOV	savedsp-24(SP), X6	// must match frame size
   648  	MOV	X6, (g_sched+gobuf_sp)(g)
   649  
   650  	// If the m on entry was nil, we called needm above to borrow an m,
   651  	// 1. for the duration of the call on non-pthread platforms,
   652  	// 2. or the duration of the C thread alive on pthread platforms.
   653  	// If the m on entry wasn't nil,
   654  	// 1. the thread might be a Go thread,
   655  	// 2. or it wasn't the first call from a C thread on pthread platforms,
   656  	//    since then we skip dropm to reuse the m in the first call.
   657  	MOV	savedm-8(SP), X5
   658  	BNE	ZERO, X5, droppedm
   659  
   660  	// Skip dropm to reuse it in the next call, when a pthread key has been created.
   661  	MOV	_cgo_pthread_key_created(SB), X5
   662  	// It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
   663  	BEQ	ZERO, X5, dropm
   664  	MOV	(X5), X5
   665  	BNE	ZERO, X5, droppedm
   666  
   667  dropm:
   668  	MOV	$runtime·dropm(SB), X6
   669  	JALR	RA, X6
   670  droppedm:
   671  
   672  	// Done!
   673  	RET
   674  
   675  TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
   676  	EBREAK
   677  	RET
   678  
   679  TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
   680  	EBREAK
   681  	RET
   682  
   683  // void setg(G*); set g. for use by needm.
   684  TEXT runtime·setg(SB), NOSPLIT, $0-8
   685  	MOV	gg+0(FP), g
   686  	// This only happens if iscgo, so jump straight to save_g
   687  	CALL	runtime·save_g(SB)
   688  	RET
   689  
   690  TEXT ·checkASM(SB),NOSPLIT,$0-1
   691  	MOV	$1, T0
   692  	MOV	T0, ret+0(FP)
   693  	RET
   694  
   695  // spillArgs stores return values from registers to a *internal/abi.RegArgs in X25.
   696  TEXT ·spillArgs(SB),NOSPLIT,$0-0
   697  	MOV	X10, (0*8)(X25)
   698  	MOV	X11, (1*8)(X25)
   699  	MOV	X12, (2*8)(X25)
   700  	MOV	X13, (3*8)(X25)
   701  	MOV	X14, (4*8)(X25)
   702  	MOV	X15, (5*8)(X25)
   703  	MOV	X16, (6*8)(X25)
   704  	MOV	X17, (7*8)(X25)
   705  	MOV	X8,  (8*8)(X25)
   706  	MOV	X9,  (9*8)(X25)
   707  	MOV	X18, (10*8)(X25)
   708  	MOV	X19, (11*8)(X25)
   709  	MOV	X20, (12*8)(X25)
   710  	MOV	X21, (13*8)(X25)
   711  	MOV	X22, (14*8)(X25)
   712  	MOV	X23, (15*8)(X25)
   713  	MOVD	F10, (16*8)(X25)
   714  	MOVD	F11, (17*8)(X25)
   715  	MOVD	F12, (18*8)(X25)
   716  	MOVD	F13, (19*8)(X25)
   717  	MOVD	F14, (20*8)(X25)
   718  	MOVD	F15, (21*8)(X25)
   719  	MOVD	F16, (22*8)(X25)
   720  	MOVD	F17, (23*8)(X25)
   721  	MOVD	F8,  (24*8)(X25)
   722  	MOVD	F9,  (25*8)(X25)
   723  	MOVD	F18, (26*8)(X25)
   724  	MOVD	F19, (27*8)(X25)
   725  	MOVD	F20, (28*8)(X25)
   726  	MOVD	F21, (29*8)(X25)
   727  	MOVD	F22, (30*8)(X25)
   728  	MOVD	F23, (31*8)(X25)
   729  	RET
   730  
   731  // unspillArgs loads args into registers from a *internal/abi.RegArgs in X25.
   732  TEXT ·unspillArgs(SB),NOSPLIT,$0-0
   733  	MOV	(0*8)(X25), X10
   734  	MOV	(1*8)(X25), X11
   735  	MOV	(2*8)(X25), X12
   736  	MOV	(3*8)(X25), X13
   737  	MOV	(4*8)(X25), X14
   738  	MOV	(5*8)(X25), X15
   739  	MOV	(6*8)(X25), X16
   740  	MOV	(7*8)(X25), X17
   741  	MOV	(8*8)(X25), X8
   742  	MOV	(9*8)(X25), X9
   743  	MOV	(10*8)(X25), X18
   744  	MOV	(11*8)(X25), X19
   745  	MOV	(12*8)(X25), X20
   746  	MOV	(13*8)(X25), X21
   747  	MOV	(14*8)(X25), X22
   748  	MOV	(15*8)(X25), X23
   749  	MOVD	(16*8)(X25), F10
   750  	MOVD	(17*8)(X25), F11
   751  	MOVD	(18*8)(X25), F12
   752  	MOVD	(19*8)(X25), F13
   753  	MOVD	(20*8)(X25), F14
   754  	MOVD	(21*8)(X25), F15
   755  	MOVD	(22*8)(X25), F16
   756  	MOVD	(23*8)(X25), F17
   757  	MOVD	(24*8)(X25), F8
   758  	MOVD	(25*8)(X25), F9
   759  	MOVD	(26*8)(X25), F18
   760  	MOVD	(27*8)(X25), F19
   761  	MOVD	(28*8)(X25), F20
   762  	MOVD	(29*8)(X25), F21
   763  	MOVD	(30*8)(X25), F22
   764  	MOVD	(31*8)(X25), F23
   765  	RET
   766  
   767  // gcWriteBarrier informs the GC about heap pointer writes.
   768  //
   769  // gcWriteBarrier does NOT follow the Go ABI. It accepts the
   770  // number of bytes of buffer needed in X24, and returns a pointer
   771  // to the buffer space in X24.
   772  // It clobbers X31 aka T6 (the linker temp register - REG_TMP).
   773  // The act of CALLing gcWriteBarrier will clobber RA (LR).
   774  // It does not clobber any other general-purpose registers,
   775  // but may clobber others (e.g., floating point registers).
   776  TEXT gcWriteBarrier<>(SB),NOSPLIT,$208
   777  	// Save the registers clobbered by the fast path.
   778  	MOV	A0, 24*8(X2)
   779  	MOV	A1, 25*8(X2)
   780  retry:
   781  	MOV	g_m(g), A0
   782  	MOV	m_p(A0), A0
   783  	MOV	(p_wbBuf+wbBuf_next)(A0), A1
   784  	MOV	(p_wbBuf+wbBuf_end)(A0), T6 // T6 is linker temp register (REG_TMP)
   785  	// Increment wbBuf.next position.
   786  	ADD	X24, A1
   787  	// Is the buffer full?
   788  	BLTU	T6, A1, flush
   789  	// Commit to the larger buffer.
   790  	MOV	A1, (p_wbBuf+wbBuf_next)(A0)
   791  	// Make the return value (the original next position)
   792  	SUB	X24, A1, X24
   793  	// Restore registers.
   794  	MOV	24*8(X2), A0
   795  	MOV	25*8(X2), A1
   796  	RET
   797  
   798  flush:
   799  	// Save all general purpose registers since these could be
   800  	// clobbered by wbBufFlush and were not saved by the caller.
   801  	MOV	T0, 1*8(X2)
   802  	MOV	T1, 2*8(X2)
   803  	// X0 is zero register
   804  	// X1 is LR, saved by prologue
   805  	// X2 is SP
   806  	// X3 is GP
   807  	// X4 is TP
   808  	MOV	X7, 3*8(X2)
   809  	MOV	X8, 4*8(X2)
   810  	MOV	X9, 5*8(X2)
   811  	// X10 already saved (A0)
   812  	// X11 already saved (A1)
   813  	MOV	X12, 6*8(X2)
   814  	MOV	X13, 7*8(X2)
   815  	MOV	X14, 8*8(X2)
   816  	MOV	X15, 9*8(X2)
   817  	MOV	X16, 10*8(X2)
   818  	MOV	X17, 11*8(X2)
   819  	MOV	X18, 12*8(X2)
   820  	MOV	X19, 13*8(X2)
   821  	MOV	X20, 14*8(X2)
   822  	MOV	X21, 15*8(X2)
   823  	MOV	X22, 16*8(X2)
   824  	MOV	X23, 17*8(X2)
   825  	MOV	X24, 18*8(X2)
   826  	MOV	X25, 19*8(X2)
   827  	MOV	X26, 20*8(X2)
   828  	// X27 is g.
   829  	MOV	X28, 21*8(X2)
   830  	MOV	X29, 22*8(X2)
   831  	MOV	X30, 23*8(X2)
   832  	// X31 is tmp register.
   833  
   834  	CALL	runtime·wbBufFlush(SB)
   835  
   836  	MOV	1*8(X2), T0
   837  	MOV	2*8(X2), T1
   838  	MOV	3*8(X2), X7
   839  	MOV	4*8(X2), X8
   840  	MOV	5*8(X2), X9
   841  	MOV	6*8(X2), X12
   842  	MOV	7*8(X2), X13
   843  	MOV	8*8(X2), X14
   844  	MOV	9*8(X2), X15
   845  	MOV	10*8(X2), X16
   846  	MOV	11*8(X2), X17
   847  	MOV	12*8(X2), X18
   848  	MOV	13*8(X2), X19
   849  	MOV	14*8(X2), X20
   850  	MOV	15*8(X2), X21
   851  	MOV	16*8(X2), X22
   852  	MOV	17*8(X2), X23
   853  	MOV	18*8(X2), X24
   854  	MOV	19*8(X2), X25
   855  	MOV	20*8(X2), X26
   856  	MOV	21*8(X2), X28
   857  	MOV	22*8(X2), X29
   858  	MOV	23*8(X2), X30
   859  
   860  	JMP	retry
   861  
   862  TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
   863  	MOV	$8, X24
   864  	JMP	gcWriteBarrier<>(SB)
   865  TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
   866  	MOV	$16, X24
   867  	JMP	gcWriteBarrier<>(SB)
   868  TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
   869  	MOV	$24, X24
   870  	JMP	gcWriteBarrier<>(SB)
   871  TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
   872  	MOV	$32, X24
   873  	JMP	gcWriteBarrier<>(SB)
   874  TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
   875  	MOV	$40, X24
   876  	JMP	gcWriteBarrier<>(SB)
   877  TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
   878  	MOV	$48, X24
   879  	JMP	gcWriteBarrier<>(SB)
   880  TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
   881  	MOV	$56, X24
   882  	JMP	gcWriteBarrier<>(SB)
   883  TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
   884  	MOV	$64, X24
   885  	JMP	gcWriteBarrier<>(SB)
   886  
   887  // Note: these functions use a special calling convention to save generated code space.
   888  // Arguments are passed in registers (ssa/gen/RISCV64Ops.go), but the space for those
   889  // arguments are allocated in the caller's stack frame.
   890  // These stubs write the args into that stack space and then tail call to the
   891  // corresponding runtime handler.
   892  // The tail call makes these stubs disappear in backtraces.
   893  TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
   894  	MOV	T0, X10
   895  	MOV	T1, X11
   896  	JMP	runtime·goPanicIndex<ABIInternal>(SB)
   897  TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
   898  	MOV	T0, X10
   899  	MOV	T1, X11
   900  	JMP	runtime·goPanicIndexU<ABIInternal>(SB)
   901  TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
   902  	MOV	T1, X10
   903  	MOV	T2, X11
   904  	JMP	runtime·goPanicSliceAlen<ABIInternal>(SB)
   905  TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
   906  	MOV	T1, X10
   907  	MOV	T2, X11
   908  	JMP	runtime·goPanicSliceAlenU<ABIInternal>(SB)
   909  TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
   910  	MOV	T1, X10
   911  	MOV	T2, X11
   912  	JMP	runtime·goPanicSliceAcap<ABIInternal>(SB)
   913  TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
   914  	MOV	T1, X10
   915  	MOV	T2, X11
   916  	JMP	runtime·goPanicSliceAcapU<ABIInternal>(SB)
   917  TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
   918  	MOV	T0, X10
   919  	MOV	T1, X11
   920  	JMP	runtime·goPanicSliceB<ABIInternal>(SB)
   921  TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
   922  	MOV	T0, X10
   923  	MOV	T1, X11
   924  	JMP	runtime·goPanicSliceBU<ABIInternal>(SB)
   925  TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
   926  	MOV	T2, X10
   927  	MOV	T3, X11
   928  	JMP	runtime·goPanicSlice3Alen<ABIInternal>(SB)
   929  TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
   930  	MOV	T2, X10
   931  	MOV	T3, X11
   932  	JMP	runtime·goPanicSlice3AlenU<ABIInternal>(SB)
   933  TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
   934  	MOV	T2, X10
   935  	MOV	T3, X11
   936  	JMP	runtime·goPanicSlice3Acap<ABIInternal>(SB)
   937  TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
   938  	MOV	T2, X10
   939  	MOV	T3, X11
   940  	JMP	runtime·goPanicSlice3AcapU<ABIInternal>(SB)
   941  TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
   942  	MOV	T1, X10
   943  	MOV	T2, X11
   944  	JMP	runtime·goPanicSlice3B<ABIInternal>(SB)
   945  TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
   946  	MOV	T1, X10
   947  	MOV	T2, X11
   948  	JMP	runtime·goPanicSlice3BU<ABIInternal>(SB)
   949  TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
   950  	MOV	T0, X10
   951  	MOV	T1, X11
   952  	JMP	runtime·goPanicSlice3C<ABIInternal>(SB)
   953  TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
   954  	MOV	T0, X10
   955  	MOV	T1, X11
   956  	JMP	runtime·goPanicSlice3CU<ABIInternal>(SB)
   957  TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16
   958  	MOV	T2, X10
   959  	MOV	T3, X11
   960  	JMP	runtime·goPanicSliceConvert<ABIInternal>(SB)
   961  
   962  DATA	runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
   963  GLOBL	runtime·mainPC(SB),RODATA,$8
   964  

View as plain text