Source file src/runtime/stubs.go
1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "internal/abi" 9 "unsafe" 10 ) 11 12 // Should be a built-in for unsafe.Pointer? 13 // 14 // add should be an internal detail, 15 // but widely used packages access it using linkname. 16 // Notable members of the hall of shame include: 17 // - fortio.org/log 18 // 19 // Do not remove or change the type signature. 20 // See go.dev/issue/67401. 21 // 22 //go:linkname add 23 //go:nosplit 24 func add(p unsafe.Pointer, x uintptr) unsafe.Pointer { 25 return unsafe.Pointer(uintptr(p) + x) 26 } 27 28 // getg returns the pointer to the current g. 29 // The compiler rewrites calls to this function into instructions 30 // that fetch the g directly (from TLS or from the dedicated register). 31 func getg() *g 32 33 // mcall switches from the g to the g0 stack and invokes fn(g), 34 // where g is the goroutine that made the call. 35 // mcall saves g's current PC/SP in g->sched so that it can be restored later. 36 // It is up to fn to arrange for that later execution, typically by recording 37 // g in a data structure, causing something to call ready(g) later. 38 // mcall returns to the original goroutine g later, when g has been rescheduled. 39 // fn must not return at all; typically it ends by calling schedule, to let the m 40 // run other goroutines. 41 // 42 // mcall can only be called from g stacks (not g0, not gsignal). 43 // 44 // This must NOT be go:noescape: if fn is a stack-allocated closure, 45 // fn puts g on a run queue, and g executes before fn returns, the 46 // closure will be invalidated while it is still executing. 47 func mcall(fn func(*g)) 48 49 // systemstack runs fn on a system stack. 50 // If systemstack is called from the per-OS-thread (g0) stack, or 51 // if systemstack is called from the signal handling (gsignal) stack, 52 // systemstack calls fn directly and returns. 53 // Otherwise, systemstack is being called from the limited stack 54 // of an ordinary goroutine. In this case, systemstack switches 55 // to the per-OS-thread stack, calls fn, and switches back. 56 // It is common to use a func literal as the argument, in order 57 // to share inputs and outputs with the code around the call 58 // to system stack: 59 // 60 // ... set up y ... 61 // systemstack(func() { 62 // x = bigcall(y) 63 // }) 64 // ... use x ... 65 // 66 //go:noescape 67 func systemstack(fn func()) 68 69 //go:nosplit 70 //go:nowritebarrierrec 71 func badsystemstack() { 72 writeErrStr("fatal: systemstack called from unexpected goroutine") 73 } 74 75 // memclrNoHeapPointers clears n bytes starting at ptr. 76 // 77 // Usually you should use typedmemclr. memclrNoHeapPointers should be 78 // used only when the caller knows that *ptr contains no heap pointers 79 // because either: 80 // 81 // *ptr is initialized memory and its type is pointer-free, or 82 // 83 // *ptr is uninitialized memory (e.g., memory that's being reused 84 // for a new allocation) and hence contains only "junk". 85 // 86 // memclrNoHeapPointers ensures that if ptr is pointer-aligned, and n 87 // is a multiple of the pointer size, then any pointer-aligned, 88 // pointer-sized portion is cleared atomically. Despite the function 89 // name, this is necessary because this function is the underlying 90 // implementation of typedmemclr and memclrHasPointers. See the doc of 91 // memmove for more details. 92 // 93 // The (CPU-specific) implementations of this function are in memclr_*.s. 94 // 95 // memclrNoHeapPointers should be an internal detail, 96 // but widely used packages access it using linkname. 97 // Notable members of the hall of shame include: 98 // - github.com/bytedance/sonic 99 // - github.com/chenzhuoyu/iasm 100 // - github.com/dgraph-io/ristretto 101 // - github.com/outcaste-io/ristretto 102 // 103 // Do not remove or change the type signature. 104 // See go.dev/issue/67401. 105 // 106 //go:linkname memclrNoHeapPointers 107 //go:noescape 108 func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) 109 110 //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers 111 func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) { 112 memclrNoHeapPointers(ptr, n) 113 } 114 115 // memmove copies n bytes from "from" to "to". 116 // 117 // memmove ensures that any pointer in "from" is written to "to" with 118 // an indivisible write, so that racy reads cannot observe a 119 // half-written pointer. This is necessary to prevent the garbage 120 // collector from observing invalid pointers, and differs from memmove 121 // in unmanaged languages. However, memmove is only required to do 122 // this if "from" and "to" may contain pointers, which can only be the 123 // case if "from", "to", and "n" are all be word-aligned. 124 // 125 // Implementations are in memmove_*.s. 126 // 127 // Outside assembly calls memmove. 128 // 129 // memmove should be an internal detail, 130 // but widely used packages access it using linkname. 131 // Notable members of the hall of shame include: 132 // - github.com/bytedance/sonic 133 // - github.com/cloudwego/dynamicgo 134 // - github.com/ebitengine/purego 135 // - github.com/tetratelabs/wazero 136 // - github.com/ugorji/go/codec 137 // - gvisor.dev/gvisor 138 // - github.com/sagernet/gvisor 139 // 140 // Do not remove or change the type signature. 141 // See go.dev/issue/67401. 142 // 143 //go:linkname memmove 144 //go:noescape 145 func memmove(to, from unsafe.Pointer, n uintptr) 146 147 //go:linkname reflect_memmove reflect.memmove 148 func reflect_memmove(to, from unsafe.Pointer, n uintptr) { 149 memmove(to, from, n) 150 } 151 152 // exported value for testing 153 const hashLoad = float32(loadFactorNum) / float32(loadFactorDen) 154 155 // in internal/bytealg/equal_*.s 156 // 157 // memequal should be an internal detail, 158 // but widely used packages access it using linkname. 159 // Notable members of the hall of shame include: 160 // - github.com/bytedance/sonic 161 // 162 // Do not remove or change the type signature. 163 // See go.dev/issue/67401. 164 // 165 //go:linkname memequal 166 //go:noescape 167 func memequal(a, b unsafe.Pointer, size uintptr) bool 168 169 // noescape hides a pointer from escape analysis. noescape is 170 // the identity function but escape analysis doesn't think the 171 // output depends on the input. noescape is inlined and currently 172 // compiles down to zero instructions. 173 // USE CAREFULLY! 174 // 175 // noescape should be an internal detail, 176 // but widely used packages access it using linkname. 177 // Notable members of the hall of shame include: 178 // - github.com/bytedance/gopkg 179 // - github.com/ebitengine/purego 180 // - github.com/hamba/avro/v2 181 // - github.com/puzpuzpuz/xsync/v3 182 // - github.com/songzhibin97/gkit 183 // 184 // Do not remove or change the type signature. 185 // See go.dev/issue/67401. 186 // 187 //go:linkname noescape 188 //go:nosplit 189 func noescape(p unsafe.Pointer) unsafe.Pointer { 190 x := uintptr(p) 191 return unsafe.Pointer(x ^ 0) 192 } 193 194 // noEscapePtr hides a pointer from escape analysis. See noescape. 195 // USE CAREFULLY! 196 // 197 //go:nosplit 198 func noEscapePtr[T any](p *T) *T { 199 x := uintptr(unsafe.Pointer(p)) 200 return (*T)(unsafe.Pointer(x ^ 0)) 201 } 202 203 // Not all cgocallback frames are actually cgocallback, 204 // so not all have these arguments. Mark them uintptr so that the GC 205 // does not misinterpret memory when the arguments are not present. 206 // cgocallback is not called from Go, only from crosscall2. 207 // This in turn calls cgocallbackg, which is where we'll find 208 // pointer-declared arguments. 209 // 210 // When fn is nil (frame is saved g), call dropm instead, 211 // this is used when the C thread is exiting. 212 func cgocallback(fn, frame, ctxt uintptr) 213 214 func gogo(buf *gobuf) 215 216 func asminit() 217 func setg(gg *g) 218 func breakpoint() 219 220 // reflectcall calls fn with arguments described by stackArgs, stackArgsSize, 221 // frameSize, and regArgs. 222 // 223 // Arguments passed on the stack and space for return values passed on the stack 224 // must be laid out at the space pointed to by stackArgs (with total length 225 // stackArgsSize) according to the ABI. 226 // 227 // stackRetOffset must be some value <= stackArgsSize that indicates the 228 // offset within stackArgs where the return value space begins. 229 // 230 // frameSize is the total size of the argument frame at stackArgs and must 231 // therefore be >= stackArgsSize. It must include additional space for spilling 232 // register arguments for stack growth and preemption. 233 // 234 // TODO(mknyszek): Once we don't need the additional spill space, remove frameSize, 235 // since frameSize will be redundant with stackArgsSize. 236 // 237 // Arguments passed in registers must be laid out in regArgs according to the ABI. 238 // regArgs will hold any return values passed in registers after the call. 239 // 240 // reflectcall copies stack arguments from stackArgs to the goroutine stack, and 241 // then copies back stackArgsSize-stackRetOffset bytes back to the return space 242 // in stackArgs once fn has completed. It also "unspills" argument registers from 243 // regArgs before calling fn, and spills them back into regArgs immediately 244 // following the call to fn. If there are results being returned on the stack, 245 // the caller should pass the argument frame type as stackArgsType so that 246 // reflectcall can execute appropriate write barriers during the copy. 247 // 248 // reflectcall expects regArgs.ReturnIsPtr to be populated indicating which 249 // registers on the return path will contain Go pointers. It will then store 250 // these pointers in regArgs.Ptrs such that they are visible to the GC. 251 // 252 // Package reflect passes a frame type. In package runtime, there is only 253 // one call that copies results back, in callbackWrap in syscall_windows.go, and it 254 // does NOT pass a frame type, meaning there are no write barriers invoked. See that 255 // call site for justification. 256 // 257 // Package reflect accesses this symbol through a linkname. 258 // 259 // Arguments passed through to reflectcall do not escape. The type is used 260 // only in a very limited callee of reflectcall, the stackArgs are copied, and 261 // regArgs is only used in the reflectcall frame. 262 // 263 //go:noescape 264 func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 265 266 // procyield should be an internal detail, 267 // but widely used packages access it using linkname. 268 // Notable members of the hall of shame include: 269 // - github.com/sagernet/sing-tun 270 // - github.com/slackhq/nebula 271 // - golang.zx2c4.com/wireguard 272 // 273 // Do not remove or change the type signature. 274 // See go.dev/issue/67401. 275 // 276 //go:linkname procyield 277 func procyield(cycles uint32) 278 279 type neverCallThisFunction struct{} 280 281 // goexit is the return stub at the top of every goroutine call stack. 282 // Each goroutine stack is constructed as if goexit called the 283 // goroutine's entry point function, so that when the entry point 284 // function returns, it will return to goexit, which will call goexit1 285 // to perform the actual exit. 286 // 287 // This function must never be called directly. Call goexit1 instead. 288 // gentraceback assumes that goexit terminates the stack. A direct 289 // call on the stack will cause gentraceback to stop walking the stack 290 // prematurely and if there is leftover state it may panic. 291 func goexit(neverCallThisFunction) 292 293 // publicationBarrier performs a store/store barrier (a "publication" 294 // or "export" barrier). Some form of synchronization is required 295 // between initializing an object and making that object accessible to 296 // another processor. Without synchronization, the initialization 297 // writes and the "publication" write may be reordered, allowing the 298 // other processor to follow the pointer and observe an uninitialized 299 // object. In general, higher-level synchronization should be used, 300 // such as locking or an atomic pointer write. publicationBarrier is 301 // for when those aren't an option, such as in the implementation of 302 // the memory manager. 303 // 304 // There's no corresponding barrier for the read side because the read 305 // side naturally has a data dependency order. All architectures that 306 // Go supports or seems likely to ever support automatically enforce 307 // data dependency ordering. 308 func publicationBarrier() 309 310 //go:noescape 311 func asmcgocall(fn, arg unsafe.Pointer) int32 312 313 func morestack() 314 315 // morestack_noctxt should be an internal detail, 316 // but widely used packages access it using linkname. 317 // Notable members of the hall of shame include: 318 // - github.com/bytedance/sonic 319 // 320 // Do not remove or change the type signature. 321 // See go.dev/issues/67401. 322 // See go.dev/issues/71672. 323 // 324 //go:linkname morestack_noctxt 325 func morestack_noctxt() 326 327 func rt0_go() 328 329 // return0 is a stub used to return 0 from deferproc. 330 // It is called at the very end of deferproc to signal 331 // the calling Go function that it should not jump 332 // to deferreturn. 333 // in asm_*.s 334 func return0() 335 336 // in asm_*.s 337 // not called directly; definitions here supply type information for traceback. 338 // These must have the same signature (arg pointer map) as reflectcall. 339 func call16(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 340 func call32(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 341 func call64(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 342 func call128(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 343 func call256(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 344 func call512(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 345 func call1024(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 346 func call2048(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 347 func call4096(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 348 func call8192(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 349 func call16384(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 350 func call32768(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 351 func call65536(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 352 func call131072(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 353 func call262144(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 354 func call524288(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 355 func call1048576(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 356 func call2097152(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 357 func call4194304(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 358 func call8388608(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 359 func call16777216(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 360 func call33554432(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 361 func call67108864(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 362 func call134217728(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 363 func call268435456(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 364 func call536870912(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 365 func call1073741824(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 366 367 func systemstack_switch() 368 369 // alignUp rounds n up to a multiple of a. a must be a power of 2. 370 // 371 //go:nosplit 372 func alignUp(n, a uintptr) uintptr { 373 return (n + a - 1) &^ (a - 1) 374 } 375 376 // alignDown rounds n down to a multiple of a. a must be a power of 2. 377 // 378 //go:nosplit 379 func alignDown(n, a uintptr) uintptr { 380 return n &^ (a - 1) 381 } 382 383 // divRoundUp returns ceil(n / a). 384 // 385 //go:nosplit 386 func divRoundUp(n, a uintptr) uintptr { 387 // a is generally a power of two. This will get inlined and 388 // the compiler will optimize the division. 389 return (n + a - 1) / a 390 } 391 392 // checkASM reports whether assembly runtime checks have passed. 393 func checkASM() bool 394 395 func memequal_varlen(a, b unsafe.Pointer) bool 396 397 // bool2int returns 0 if x is false or 1 if x is true. 398 func bool2int(x bool) int { 399 // Avoid branches. In the SSA compiler, this compiles to 400 // exactly what you would want it to. 401 return int(*(*uint8)(unsafe.Pointer(&x))) 402 } 403 404 // abort crashes the runtime in situations where even throw might not 405 // work. In general it should do something a debugger will recognize 406 // (e.g., an INT3 on x86). A crash in abort is recognized by the 407 // signal handler, which will attempt to tear down the runtime 408 // immediately. 409 func abort() 410 411 // Called from compiled code; declared for vet; do NOT call from Go. 412 func gcWriteBarrier1() 413 414 // gcWriteBarrier2 should be an internal detail, 415 // but widely used packages access it using linkname. 416 // Notable members of the hall of shame include: 417 // - github.com/bytedance/sonic 418 // 419 // Do not remove or change the type signature. 420 // See go.dev/issue/67401. 421 // 422 //go:linkname gcWriteBarrier2 423 func gcWriteBarrier2() 424 425 func gcWriteBarrier3() 426 func gcWriteBarrier4() 427 func gcWriteBarrier5() 428 func gcWriteBarrier6() 429 func gcWriteBarrier7() 430 func gcWriteBarrier8() 431 func duffzero() 432 func duffcopy() 433 434 // Called from linker-generated .initarray; declared for go vet; do NOT call from Go. 435 func addmoduledata() 436 437 // Injected by the signal handler for panicking signals. 438 // Initializes any registers that have fixed meaning at calls but 439 // are scratch in bodies and calls sigpanic. 440 // On many platforms it just jumps to sigpanic. 441 func sigpanic0() 442 443 // intArgRegs is used by the various register assignment 444 // algorithm implementations in the runtime. These include:. 445 // - Finalizers (mfinal.go) 446 // - Windows callbacks (syscall_windows.go) 447 // 448 // Both are stripped-down versions of the algorithm since they 449 // only have to deal with a subset of cases (finalizers only 450 // take a pointer or interface argument, Go Windows callbacks 451 // don't support floating point). 452 // 453 // It should be modified with care and are generally only 454 // modified when testing this package. 455 // 456 // It should never be set higher than its internal/abi 457 // constant counterparts, because the system relies on a 458 // structure that is at least large enough to hold the 459 // registers the system supports. 460 // 461 // Protected by finlock. 462 var intArgRegs = abi.IntArgRegs 463