Source file src/runtime/runtime2.go
1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "internal/abi" 9 "internal/chacha8rand" 10 "internal/goarch" 11 "internal/goexperiment" 12 "internal/runtime/atomic" 13 "internal/runtime/sys" 14 "unsafe" 15 ) 16 17 // defined constants 18 const ( 19 // G status 20 // 21 // Beyond indicating the general state of a G, the G status 22 // acts like a lock on the goroutine's stack (and hence its 23 // ability to execute user code). 24 // 25 // If you add to this list, add to the list 26 // of "okay during garbage collection" status 27 // in mgcmark.go too. 28 // 29 // TODO(austin): The _Gscan bit could be much lighter-weight. 30 // For example, we could choose not to run _Gscanrunnable 31 // goroutines found in the run queue, rather than CAS-looping 32 // until they become _Grunnable. And transitions like 33 // _Gscanwaiting -> _Gscanrunnable are actually okay because 34 // they don't affect stack ownership. 35 36 // _Gidle means this goroutine was just allocated and has not 37 // yet been initialized. 38 _Gidle = iota // 0 39 40 // _Grunnable means this goroutine is on a run queue. It is 41 // not currently executing user code. The stack is not owned. 42 _Grunnable // 1 43 44 // _Grunning means this goroutine may execute user code. The 45 // stack is owned by this goroutine. It is not on a run queue. 46 // It is assigned an M and a P (g.m and g.m.p are valid). 47 _Grunning // 2 48 49 // _Gsyscall means this goroutine is executing a system call. 50 // It is not executing user code. The stack is owned by this 51 // goroutine. It is not on a run queue. It is assigned an M. 52 _Gsyscall // 3 53 54 // _Gwaiting means this goroutine is blocked in the runtime. 55 // It is not executing user code. It is not on a run queue, 56 // but should be recorded somewhere (e.g., a channel wait 57 // queue) so it can be ready()d when necessary. The stack is 58 // not owned *except* that a channel operation may read or 59 // write parts of the stack under the appropriate channel 60 // lock. Otherwise, it is not safe to access the stack after a 61 // goroutine enters _Gwaiting (e.g., it may get moved). 62 _Gwaiting // 4 63 64 // _Gmoribund_unused is currently unused, but hardcoded in gdb 65 // scripts. 66 _Gmoribund_unused // 5 67 68 // _Gdead means this goroutine is currently unused. It may be 69 // just exited, on a free list, or just being initialized. It 70 // is not executing user code. It may or may not have a stack 71 // allocated. The G and its stack (if any) are owned by the M 72 // that is exiting the G or that obtained the G from the free 73 // list. 74 _Gdead // 6 75 76 // _Genqueue_unused is currently unused. 77 _Genqueue_unused // 7 78 79 // _Gcopystack means this goroutine's stack is being moved. It 80 // is not executing user code and is not on a run queue. The 81 // stack is owned by the goroutine that put it in _Gcopystack. 82 _Gcopystack // 8 83 84 // _Gpreempted means this goroutine stopped itself for a 85 // suspendG preemption. It is like _Gwaiting, but nothing is 86 // yet responsible for ready()ing it. Some suspendG must CAS 87 // the status to _Gwaiting to take responsibility for 88 // ready()ing this G. 89 _Gpreempted // 9 90 91 // _Gscan combined with one of the above states other than 92 // _Grunning indicates that GC is scanning the stack. The 93 // goroutine is not executing user code and the stack is owned 94 // by the goroutine that set the _Gscan bit. 95 // 96 // _Gscanrunning is different: it is used to briefly block 97 // state transitions while GC signals the G to scan its own 98 // stack. This is otherwise like _Grunning. 99 // 100 // atomicstatus&~Gscan gives the state the goroutine will 101 // return to when the scan completes. 102 _Gscan = 0x1000 103 _Gscanrunnable = _Gscan + _Grunnable // 0x1001 104 _Gscanrunning = _Gscan + _Grunning // 0x1002 105 _Gscansyscall = _Gscan + _Gsyscall // 0x1003 106 _Gscanwaiting = _Gscan + _Gwaiting // 0x1004 107 _Gscanpreempted = _Gscan + _Gpreempted // 0x1009 108 ) 109 110 const ( 111 // P status 112 113 // _Pidle means a P is not being used to run user code or the 114 // scheduler. Typically, it's on the idle P list and available 115 // to the scheduler, but it may just be transitioning between 116 // other states. 117 // 118 // The P is owned by the idle list or by whatever is 119 // transitioning its state. Its run queue is empty. 120 _Pidle = iota 121 122 // _Prunning means a P is owned by an M and is being used to 123 // run user code or the scheduler. Only the M that owns this P 124 // is allowed to change the P's status from _Prunning. The M 125 // may transition the P to _Pidle (if it has no more work to 126 // do), _Psyscall (when entering a syscall), or _Pgcstop (to 127 // halt for the GC). The M may also hand ownership of the P 128 // off directly to another M (e.g., to schedule a locked G). 129 _Prunning 130 131 // _Psyscall means a P is not running user code. It has 132 // affinity to an M in a syscall but is not owned by it and 133 // may be stolen by another M. This is similar to _Pidle but 134 // uses lightweight transitions and maintains M affinity. 135 // 136 // Leaving _Psyscall must be done with a CAS, either to steal 137 // or retake the P. Note that there's an ABA hazard: even if 138 // an M successfully CASes its original P back to _Prunning 139 // after a syscall, it must understand the P may have been 140 // used by another M in the interim. 141 _Psyscall 142 143 // _Pgcstop means a P is halted for STW and owned by the M 144 // that stopped the world. The M that stopped the world 145 // continues to use its P, even in _Pgcstop. Transitioning 146 // from _Prunning to _Pgcstop causes an M to release its P and 147 // park. 148 // 149 // The P retains its run queue and startTheWorld will restart 150 // the scheduler on Ps with non-empty run queues. 151 _Pgcstop 152 153 // _Pdead means a P is no longer used (GOMAXPROCS shrank). We 154 // reuse Ps if GOMAXPROCS increases. A dead P is mostly 155 // stripped of its resources, though a few things remain 156 // (e.g., trace buffers). 157 _Pdead 158 ) 159 160 // Mutual exclusion locks. In the uncontended case, 161 // as fast as spin locks (just a few user-level instructions), 162 // but on the contention path they sleep in the kernel. 163 // A zeroed Mutex is unlocked (no need to initialize each lock). 164 // Initialization is helpful for static lock ranking, but not required. 165 type mutex struct { 166 // Empty struct if lock ranking is disabled, otherwise includes the lock rank 167 lockRankStruct 168 // Futex-based impl treats it as uint32 key, 169 // while sema-based impl as M* waitm. 170 // Used to be a union, but unions break precise GC. 171 key uintptr 172 } 173 174 type funcval struct { 175 fn uintptr 176 // variable-size, fn-specific data here 177 } 178 179 type iface struct { 180 tab *itab 181 data unsafe.Pointer 182 } 183 184 type eface struct { 185 _type *_type 186 data unsafe.Pointer 187 } 188 189 func efaceOf(ep *any) *eface { 190 return (*eface)(unsafe.Pointer(ep)) 191 } 192 193 // The guintptr, muintptr, and puintptr are all used to bypass write barriers. 194 // It is particularly important to avoid write barriers when the current P has 195 // been released, because the GC thinks the world is stopped, and an 196 // unexpected write barrier would not be synchronized with the GC, 197 // which can lead to a half-executed write barrier that has marked the object 198 // but not queued it. If the GC skips the object and completes before the 199 // queuing can occur, it will incorrectly free the object. 200 // 201 // We tried using special assignment functions invoked only when not 202 // holding a running P, but then some updates to a particular memory 203 // word went through write barriers and some did not. This breaks the 204 // write barrier shadow checking mode, and it is also scary: better to have 205 // a word that is completely ignored by the GC than to have one for which 206 // only a few updates are ignored. 207 // 208 // Gs and Ps are always reachable via true pointers in the 209 // allgs and allp lists or (during allocation before they reach those lists) 210 // from stack variables. 211 // 212 // Ms are always reachable via true pointers either from allm or 213 // freem. Unlike Gs and Ps we do free Ms, so it's important that 214 // nothing ever hold an muintptr across a safe point. 215 216 // A guintptr holds a goroutine pointer, but typed as a uintptr 217 // to bypass write barriers. It is used in the Gobuf goroutine state 218 // and in scheduling lists that are manipulated without a P. 219 // 220 // The Gobuf.g goroutine pointer is almost always updated by assembly code. 221 // In one of the few places it is updated by Go code - func save - it must be 222 // treated as a uintptr to avoid a write barrier being emitted at a bad time. 223 // Instead of figuring out how to emit the write barriers missing in the 224 // assembly manipulation, we change the type of the field to uintptr, 225 // so that it does not require write barriers at all. 226 // 227 // Goroutine structs are published in the allg list and never freed. 228 // That will keep the goroutine structs from being collected. 229 // There is never a time that Gobuf.g's contain the only references 230 // to a goroutine: the publishing of the goroutine in allg comes first. 231 // Goroutine pointers are also kept in non-GC-visible places like TLS, 232 // so I can't see them ever moving. If we did want to start moving data 233 // in the GC, we'd need to allocate the goroutine structs from an 234 // alternate arena. Using guintptr doesn't make that problem any worse. 235 // Note that pollDesc.rg, pollDesc.wg also store g in uintptr form, 236 // so they would need to be updated too if g's start moving. 237 type guintptr uintptr 238 239 //go:nosplit 240 func (gp guintptr) ptr() *g { return (*g)(unsafe.Pointer(gp)) } 241 242 //go:nosplit 243 func (gp *guintptr) set(g *g) { *gp = guintptr(unsafe.Pointer(g)) } 244 245 //go:nosplit 246 func (gp *guintptr) cas(old, new guintptr) bool { 247 return atomic.Casuintptr((*uintptr)(unsafe.Pointer(gp)), uintptr(old), uintptr(new)) 248 } 249 250 //go:nosplit 251 func (gp *g) guintptr() guintptr { 252 return guintptr(unsafe.Pointer(gp)) 253 } 254 255 // setGNoWB performs *gp = new without a write barrier. 256 // For times when it's impractical to use a guintptr. 257 // 258 //go:nosplit 259 //go:nowritebarrier 260 func setGNoWB(gp **g, new *g) { 261 (*guintptr)(unsafe.Pointer(gp)).set(new) 262 } 263 264 type puintptr uintptr 265 266 //go:nosplit 267 func (pp puintptr) ptr() *p { return (*p)(unsafe.Pointer(pp)) } 268 269 //go:nosplit 270 func (pp *puintptr) set(p *p) { *pp = puintptr(unsafe.Pointer(p)) } 271 272 // muintptr is a *m that is not tracked by the garbage collector. 273 // 274 // Because we do free Ms, there are some additional constrains on 275 // muintptrs: 276 // 277 // 1. Never hold an muintptr locally across a safe point. 278 // 279 // 2. Any muintptr in the heap must be owned by the M itself so it can 280 // ensure it is not in use when the last true *m is released. 281 type muintptr uintptr 282 283 //go:nosplit 284 func (mp muintptr) ptr() *m { return (*m)(unsafe.Pointer(mp)) } 285 286 //go:nosplit 287 func (mp *muintptr) set(m *m) { *mp = muintptr(unsafe.Pointer(m)) } 288 289 // setMNoWB performs *mp = new without a write barrier. 290 // For times when it's impractical to use an muintptr. 291 // 292 //go:nosplit 293 //go:nowritebarrier 294 func setMNoWB(mp **m, new *m) { 295 (*muintptr)(unsafe.Pointer(mp)).set(new) 296 } 297 298 type gobuf struct { 299 // The offsets of sp, pc, and g are known to (hard-coded in) libmach. 300 // 301 // ctxt is unusual with respect to GC: it may be a 302 // heap-allocated funcval, so GC needs to track it, but it 303 // needs to be set and cleared from assembly, where it's 304 // difficult to have write barriers. However, ctxt is really a 305 // saved, live register, and we only ever exchange it between 306 // the real register and the gobuf. Hence, we treat it as a 307 // root during stack scanning, which means assembly that saves 308 // and restores it doesn't need write barriers. It's still 309 // typed as a pointer so that any other writes from Go get 310 // write barriers. 311 sp uintptr 312 pc uintptr 313 g guintptr 314 ctxt unsafe.Pointer 315 ret uintptr 316 lr uintptr 317 bp uintptr // for framepointer-enabled architectures 318 } 319 320 // sudog (pseudo-g) represents a g in a wait list, such as for sending/receiving 321 // on a channel. 322 // 323 // sudog is necessary because the g ↔ synchronization object relation 324 // is many-to-many. A g can be on many wait lists, so there may be 325 // many sudogs for one g; and many gs may be waiting on the same 326 // synchronization object, so there may be many sudogs for one object. 327 // 328 // sudogs are allocated from a special pool. Use acquireSudog and 329 // releaseSudog to allocate and free them. 330 type sudog struct { 331 // The following fields are protected by the hchan.lock of the 332 // channel this sudog is blocking on. shrinkstack depends on 333 // this for sudogs involved in channel ops. 334 335 g *g 336 337 next *sudog 338 prev *sudog 339 elem unsafe.Pointer // data element (may point to stack) 340 341 // The following fields are never accessed concurrently. 342 // For channels, waitlink is only accessed by g. 343 // For semaphores, all fields (including the ones above) 344 // are only accessed when holding a semaRoot lock. 345 346 acquiretime int64 347 releasetime int64 348 ticket uint32 349 350 // isSelect indicates g is participating in a select, so 351 // g.selectDone must be CAS'd to win the wake-up race. 352 isSelect bool 353 354 // success indicates whether communication over channel c 355 // succeeded. It is true if the goroutine was awoken because a 356 // value was delivered over channel c, and false if awoken 357 // because c was closed. 358 success bool 359 360 // waiters is a count of semaRoot waiting list other than head of list, 361 // clamped to a uint16 to fit in unused space. 362 // Only meaningful at the head of the list. 363 // (If we wanted to be overly clever, we could store a high 16 bits 364 // in the second entry in the list.) 365 waiters uint16 366 367 parent *sudog // semaRoot binary tree 368 waitlink *sudog // g.waiting list or semaRoot 369 waittail *sudog // semaRoot 370 c *hchan // channel 371 } 372 373 type libcall struct { 374 fn uintptr 375 n uintptr // number of parameters 376 args uintptr // parameters 377 r1 uintptr // return values 378 r2 uintptr 379 err uintptr // error number 380 } 381 382 // Stack describes a Go execution stack. 383 // The bounds of the stack are exactly [lo, hi), 384 // with no implicit data structures on either side. 385 type stack struct { 386 lo uintptr 387 hi uintptr 388 } 389 390 // heldLockInfo gives info on a held lock and the rank of that lock 391 type heldLockInfo struct { 392 lockAddr uintptr 393 rank lockRank 394 } 395 396 type g struct { 397 // Stack parameters. 398 // stack describes the actual stack memory: [stack.lo, stack.hi). 399 // stackguard0 is the stack pointer compared in the Go stack growth prologue. 400 // It is stack.lo+StackGuard normally, but can be StackPreempt to trigger a preemption. 401 // stackguard1 is the stack pointer compared in the //go:systemstack stack growth prologue. 402 // It is stack.lo+StackGuard on g0 and gsignal stacks. 403 // It is ~0 on other goroutine stacks, to trigger a call to morestackc (and crash). 404 stack stack // offset known to runtime/cgo 405 stackguard0 uintptr // offset known to liblink 406 stackguard1 uintptr // offset known to liblink 407 408 _panic *_panic // innermost panic - offset known to liblink 409 _defer *_defer // innermost defer 410 m *m // current m; offset known to arm liblink 411 sched gobuf 412 syscallsp uintptr // if status==Gsyscall, syscallsp = sched.sp to use during gc 413 syscallpc uintptr // if status==Gsyscall, syscallpc = sched.pc to use during gc 414 syscallbp uintptr // if status==Gsyscall, syscallbp = sched.bp to use in fpTraceback 415 stktopsp uintptr // expected sp at top of stack, to check in traceback 416 // param is a generic pointer parameter field used to pass 417 // values in particular contexts where other storage for the 418 // parameter would be difficult to find. It is currently used 419 // in four ways: 420 // 1. When a channel operation wakes up a blocked goroutine, it sets param to 421 // point to the sudog of the completed blocking operation. 422 // 2. By gcAssistAlloc1 to signal back to its caller that the goroutine completed 423 // the GC cycle. It is unsafe to do so in any other way, because the goroutine's 424 // stack may have moved in the meantime. 425 // 3. By debugCallWrap to pass parameters to a new goroutine because allocating a 426 // closure in the runtime is forbidden. 427 // 4. When a panic is recovered and control returns to the respective frame, 428 // param may point to a savedOpenDeferState. 429 param unsafe.Pointer 430 atomicstatus atomic.Uint32 431 stackLock uint32 // sigprof/scang lock; TODO: fold in to atomicstatus 432 goid uint64 433 schedlink guintptr 434 waitsince int64 // approx time when the g become blocked 435 waitreason waitReason // if status==Gwaiting 436 437 preempt bool // preemption signal, duplicates stackguard0 = stackpreempt 438 preemptStop bool // transition to _Gpreempted on preemption; otherwise, just deschedule 439 preemptShrink bool // shrink stack at synchronous safe point 440 441 // asyncSafePoint is set if g is stopped at an asynchronous 442 // safe point. This means there are frames on the stack 443 // without precise pointer information. 444 asyncSafePoint bool 445 446 paniconfault bool // panic (instead of crash) on unexpected fault address 447 gcscandone bool // g has scanned stack; protected by _Gscan bit in status 448 throwsplit bool // must not split stack 449 // activeStackChans indicates that there are unlocked channels 450 // pointing into this goroutine's stack. If true, stack 451 // copying needs to acquire channel locks to protect these 452 // areas of the stack. 453 activeStackChans bool 454 // parkingOnChan indicates that the goroutine is about to 455 // park on a chansend or chanrecv. Used to signal an unsafe point 456 // for stack shrinking. 457 parkingOnChan atomic.Bool 458 // inMarkAssist indicates whether the goroutine is in mark assist. 459 // Used by the execution tracer. 460 inMarkAssist bool 461 coroexit bool // argument to coroswitch_m 462 463 raceignore int8 // ignore race detection events 464 nocgocallback bool // whether disable callback from C 465 tracking bool // whether we're tracking this G for sched latency statistics 466 trackingSeq uint8 // used to decide whether to track this G 467 trackingStamp int64 // timestamp of when the G last started being tracked 468 runnableTime int64 // the amount of time spent runnable, cleared when running, only used when tracking 469 lockedm muintptr 470 fipsIndicator uint8 471 sig uint32 472 writebuf []byte 473 sigcode0 uintptr 474 sigcode1 uintptr 475 sigpc uintptr 476 parentGoid uint64 // goid of goroutine that created this goroutine 477 gopc uintptr // pc of go statement that created this goroutine 478 ancestors *[]ancestorInfo // ancestor information goroutine(s) that created this goroutine (only used if debug.tracebackancestors) 479 startpc uintptr // pc of goroutine function 480 racectx uintptr 481 waiting *sudog // sudog structures this g is waiting on (that have a valid elem ptr); in lock order 482 cgoCtxt []uintptr // cgo traceback context 483 labels unsafe.Pointer // profiler labels 484 timer *timer // cached timer for time.Sleep 485 sleepWhen int64 // when to sleep until 486 selectDone atomic.Uint32 // are we participating in a select and did someone win the race? 487 488 // goroutineProfiled indicates the status of this goroutine's stack for the 489 // current in-progress goroutine profile 490 goroutineProfiled goroutineProfileStateHolder 491 492 coroarg *coro // argument during coroutine transfers 493 syncGroup *synctestGroup 494 495 // Per-G tracer state. 496 trace gTraceState 497 498 // Per-G GC state 499 500 // gcAssistBytes is this G's GC assist credit in terms of 501 // bytes allocated. If this is positive, then the G has credit 502 // to allocate gcAssistBytes bytes without assisting. If this 503 // is negative, then the G must correct this by performing 504 // scan work. We track this in bytes to make it fast to update 505 // and check for debt in the malloc hot path. The assist ratio 506 // determines how this corresponds to scan work debt. 507 gcAssistBytes int64 508 } 509 510 // gTrackingPeriod is the number of transitions out of _Grunning between 511 // latency tracking runs. 512 const gTrackingPeriod = 8 513 514 const ( 515 // tlsSlots is the number of pointer-sized slots reserved for TLS on some platforms, 516 // like Windows. 517 tlsSlots = 6 518 tlsSize = tlsSlots * goarch.PtrSize 519 ) 520 521 // Values for m.freeWait. 522 const ( 523 freeMStack = 0 // M done, free stack and reference. 524 freeMRef = 1 // M done, free reference. 525 freeMWait = 2 // M still in use. 526 ) 527 528 type m struct { 529 g0 *g // goroutine with scheduling stack 530 morebuf gobuf // gobuf arg to morestack 531 divmod uint32 // div/mod denominator for arm - known to liblink 532 _ uint32 // align next field to 8 bytes 533 534 // Fields not known to debuggers. 535 procid uint64 // for debuggers, but offset not hard-coded 536 gsignal *g // signal-handling g 537 goSigStack gsignalStack // Go-allocated signal handling stack 538 sigmask sigset // storage for saved signal mask 539 tls [tlsSlots]uintptr // thread-local storage (for x86 extern register) 540 mstartfn func() 541 curg *g // current running goroutine 542 caughtsig guintptr // goroutine running during fatal signal 543 p puintptr // attached p for executing go code (nil if not executing go code) 544 nextp puintptr 545 oldp puintptr // the p that was attached before executing a syscall 546 id int64 547 mallocing int32 548 throwing throwType 549 preemptoff string // if != "", keep curg running on this m 550 locks int32 551 dying int32 552 profilehz int32 553 spinning bool // m is out of work and is actively looking for work 554 blocked bool // m is blocked on a note 555 newSigstack bool // minit on C thread called sigaltstack 556 printlock int8 557 incgo bool // m is executing a cgo call 558 isextra bool // m is an extra m 559 isExtraInC bool // m is an extra m that is not executing Go code 560 isExtraInSig bool // m is an extra m in a signal handler 561 freeWait atomic.Uint32 // Whether it is safe to free g0 and delete m (one of freeMRef, freeMStack, freeMWait) 562 needextram bool 563 g0StackAccurate bool // whether the g0 stack has accurate bounds 564 traceback uint8 565 ncgocall uint64 // number of cgo calls in total 566 ncgo int32 // number of cgo calls currently in progress 567 cgoCallersUse atomic.Uint32 // if non-zero, cgoCallers in use temporarily 568 cgoCallers *cgoCallers // cgo traceback if crashing in cgo call 569 park note 570 alllink *m // on allm 571 schedlink muintptr 572 lockedg guintptr 573 createstack [32]uintptr // stack that created this thread, it's used for StackRecord.Stack0, so it must align with it. 574 lockedExt uint32 // tracking for external LockOSThread 575 lockedInt uint32 // tracking for internal lockOSThread 576 mWaitList mWaitList // list of runtime lock waiters 577 578 mLockProfile mLockProfile // fields relating to runtime.lock contention 579 profStack []uintptr // used for memory/block/mutex stack traces 580 581 // wait* are used to carry arguments from gopark into park_m, because 582 // there's no stack to put them on. That is their sole purpose. 583 waitunlockf func(*g, unsafe.Pointer) bool 584 waitlock unsafe.Pointer 585 waitTraceSkip int 586 waitTraceBlockReason traceBlockReason 587 588 syscalltick uint32 589 freelink *m // on sched.freem 590 trace mTraceState 591 592 // these are here because they are too large to be on the stack 593 // of low-level NOSPLIT functions. 594 libcall libcall 595 libcallpc uintptr // for cpu profiler 596 libcallsp uintptr 597 libcallg guintptr 598 winsyscall winlibcall // stores syscall parameters on windows 599 600 vdsoSP uintptr // SP for traceback while in VDSO call (0 if not in call) 601 vdsoPC uintptr // PC for traceback while in VDSO call 602 603 // preemptGen counts the number of completed preemption 604 // signals. This is used to detect when a preemption is 605 // requested, but fails. 606 preemptGen atomic.Uint32 607 608 // Whether this is a pending preemption signal on this M. 609 signalPending atomic.Uint32 610 611 // pcvalue lookup cache 612 pcvalueCache pcvalueCache 613 614 dlogPerM 615 616 mOS 617 618 chacha8 chacha8rand.State 619 cheaprand uint64 620 621 // Up to 10 locks held by this m, maintained by the lock ranking code. 622 locksHeldLen int 623 locksHeld [10]heldLockInfo 624 625 // Size the runtime.m structure so it fits in the 2048-byte size class, and 626 // not in the next-smallest (1792-byte) size class. That leaves the 11 low 627 // bits of muintptr values available for flags, as required for 628 // GOEXPERIMENT=spinbitmutex. 629 _ [goexperiment.SpinbitMutexInt * 700 * (2 - goarch.PtrSize/4)]byte 630 } 631 632 type p struct { 633 id int32 634 status uint32 // one of pidle/prunning/... 635 link puintptr 636 schedtick uint32 // incremented on every scheduler call 637 syscalltick uint32 // incremented on every system call 638 sysmontick sysmontick // last tick observed by sysmon 639 m muintptr // back-link to associated m (nil if idle) 640 mcache *mcache 641 pcache pageCache 642 raceprocctx uintptr 643 644 deferpool []*_defer // pool of available defer structs (see panic.go) 645 deferpoolbuf [32]*_defer 646 647 // Cache of goroutine ids, amortizes accesses to runtime·sched.goidgen. 648 goidcache uint64 649 goidcacheend uint64 650 651 // Queue of runnable goroutines. Accessed without lock. 652 runqhead uint32 653 runqtail uint32 654 runq [256]guintptr 655 // runnext, if non-nil, is a runnable G that was ready'd by 656 // the current G and should be run next instead of what's in 657 // runq if there's time remaining in the running G's time 658 // slice. It will inherit the time left in the current time 659 // slice. If a set of goroutines is locked in a 660 // communicate-and-wait pattern, this schedules that set as a 661 // unit and eliminates the (potentially large) scheduling 662 // latency that otherwise arises from adding the ready'd 663 // goroutines to the end of the run queue. 664 // 665 // Note that while other P's may atomically CAS this to zero, 666 // only the owner P can CAS it to a valid G. 667 runnext guintptr 668 669 // Available G's (status == Gdead) 670 gFree struct { 671 gList 672 n int32 673 } 674 675 sudogcache []*sudog 676 sudogbuf [128]*sudog 677 678 // Cache of mspan objects from the heap. 679 mspancache struct { 680 // We need an explicit length here because this field is used 681 // in allocation codepaths where write barriers are not allowed, 682 // and eliminating the write barrier/keeping it eliminated from 683 // slice updates is tricky, more so than just managing the length 684 // ourselves. 685 len int 686 buf [128]*mspan 687 } 688 689 // Cache of a single pinner object to reduce allocations from repeated 690 // pinner creation. 691 pinnerCache *pinner 692 693 trace pTraceState 694 695 palloc persistentAlloc // per-P to avoid mutex 696 697 // Per-P GC state 698 gcAssistTime int64 // Nanoseconds in assistAlloc 699 gcFractionalMarkTime int64 // Nanoseconds in fractional mark worker (atomic) 700 701 // limiterEvent tracks events for the GC CPU limiter. 702 limiterEvent limiterEvent 703 704 // gcMarkWorkerMode is the mode for the next mark worker to run in. 705 // That is, this is used to communicate with the worker goroutine 706 // selected for immediate execution by 707 // gcController.findRunnableGCWorker. When scheduling other goroutines, 708 // this field must be set to gcMarkWorkerNotWorker. 709 gcMarkWorkerMode gcMarkWorkerMode 710 // gcMarkWorkerStartTime is the nanotime() at which the most recent 711 // mark worker started. 712 gcMarkWorkerStartTime int64 713 714 // gcw is this P's GC work buffer cache. The work buffer is 715 // filled by write barriers, drained by mutator assists, and 716 // disposed on certain GC state transitions. 717 gcw gcWork 718 719 // wbBuf is this P's GC write barrier buffer. 720 // 721 // TODO: Consider caching this in the running G. 722 wbBuf wbBuf 723 724 runSafePointFn uint32 // if 1, run sched.safePointFn at next safe point 725 726 // statsSeq is a counter indicating whether this P is currently 727 // writing any stats. Its value is even when not, odd when it is. 728 statsSeq atomic.Uint32 729 730 // Timer heap. 731 timers timers 732 733 // maxStackScanDelta accumulates the amount of stack space held by 734 // live goroutines (i.e. those eligible for stack scanning). 735 // Flushed to gcController.maxStackScan once maxStackScanSlack 736 // or -maxStackScanSlack is reached. 737 maxStackScanDelta int64 738 739 // gc-time statistics about current goroutines 740 // Note that this differs from maxStackScan in that this 741 // accumulates the actual stack observed to be used at GC time (hi - sp), 742 // not an instantaneous measure of the total stack size that might need 743 // to be scanned (hi - lo). 744 scannedStackSize uint64 // stack size of goroutines scanned by this P 745 scannedStacks uint64 // number of goroutines scanned by this P 746 747 // preempt is set to indicate that this P should be enter the 748 // scheduler ASAP (regardless of what G is running on it). 749 preempt bool 750 751 // gcStopTime is the nanotime timestamp that this P last entered _Pgcstop. 752 gcStopTime int64 753 754 // Padding is no longer needed. False sharing is now not a worry because p is large enough 755 // that its size class is an integer multiple of the cache line size (for any of our architectures). 756 } 757 758 type schedt struct { 759 goidgen atomic.Uint64 760 lastpoll atomic.Int64 // time of last network poll, 0 if currently polling 761 pollUntil atomic.Int64 // time to which current poll is sleeping 762 763 lock mutex 764 765 // When increasing nmidle, nmidlelocked, nmsys, or nmfreed, be 766 // sure to call checkdead(). 767 768 midle muintptr // idle m's waiting for work 769 nmidle int32 // number of idle m's waiting for work 770 nmidlelocked int32 // number of locked m's waiting for work 771 mnext int64 // number of m's that have been created and next M ID 772 maxmcount int32 // maximum number of m's allowed (or die) 773 nmsys int32 // number of system m's not counted for deadlock 774 nmfreed int64 // cumulative number of freed m's 775 776 ngsys atomic.Int32 // number of system goroutines 777 778 pidle puintptr // idle p's 779 npidle atomic.Int32 780 nmspinning atomic.Int32 // See "Worker thread parking/unparking" comment in proc.go. 781 needspinning atomic.Uint32 // See "Delicate dance" comment in proc.go. Boolean. Must hold sched.lock to set to 1. 782 783 // Global runnable queue. 784 runq gQueue 785 runqsize int32 786 787 // disable controls selective disabling of the scheduler. 788 // 789 // Use schedEnableUser to control this. 790 // 791 // disable is protected by sched.lock. 792 disable struct { 793 // user disables scheduling of user goroutines. 794 user bool 795 runnable gQueue // pending runnable Gs 796 n int32 // length of runnable 797 } 798 799 // Global cache of dead G's. 800 gFree struct { 801 lock mutex 802 stack gList // Gs with stacks 803 noStack gList // Gs without stacks 804 n int32 805 } 806 807 // Central cache of sudog structs. 808 sudoglock mutex 809 sudogcache *sudog 810 811 // Central pool of available defer structs. 812 deferlock mutex 813 deferpool *_defer 814 815 // freem is the list of m's waiting to be freed when their 816 // m.exited is set. Linked through m.freelink. 817 freem *m 818 819 gcwaiting atomic.Bool // gc is waiting to run 820 stopwait int32 821 stopnote note 822 sysmonwait atomic.Bool 823 sysmonnote note 824 825 // safePointFn should be called on each P at the next GC 826 // safepoint if p.runSafePointFn is set. 827 safePointFn func(*p) 828 safePointWait int32 829 safePointNote note 830 831 profilehz int32 // cpu profiling rate 832 833 procresizetime int64 // nanotime() of last change to gomaxprocs 834 totaltime int64 // ∫gomaxprocs dt up to procresizetime 835 836 // sysmonlock protects sysmon's actions on the runtime. 837 // 838 // Acquire and hold this mutex to block sysmon from interacting 839 // with the rest of the runtime. 840 sysmonlock mutex 841 842 // timeToRun is a distribution of scheduling latencies, defined 843 // as the sum of time a G spends in the _Grunnable state before 844 // it transitions to _Grunning. 845 timeToRun timeHistogram 846 847 // idleTime is the total CPU time Ps have "spent" idle. 848 // 849 // Reset on each GC cycle. 850 idleTime atomic.Int64 851 852 // totalMutexWaitTime is the sum of time goroutines have spent in _Gwaiting 853 // with a waitreason of the form waitReasonSync{RW,}Mutex{R,}Lock. 854 totalMutexWaitTime atomic.Int64 855 856 // stwStoppingTimeGC/Other are distributions of stop-the-world stopping 857 // latencies, defined as the time taken by stopTheWorldWithSema to get 858 // all Ps to stop. stwStoppingTimeGC covers all GC-related STWs, 859 // stwStoppingTimeOther covers the others. 860 stwStoppingTimeGC timeHistogram 861 stwStoppingTimeOther timeHistogram 862 863 // stwTotalTimeGC/Other are distributions of stop-the-world total 864 // latencies, defined as the total time from stopTheWorldWithSema to 865 // startTheWorldWithSema. This is a superset of 866 // stwStoppingTimeGC/Other. stwTotalTimeGC covers all GC-related STWs, 867 // stwTotalTimeOther covers the others. 868 stwTotalTimeGC timeHistogram 869 stwTotalTimeOther timeHistogram 870 871 // totalRuntimeLockWaitTime (plus the value of lockWaitTime on each M in 872 // allm) is the sum of time goroutines have spent in _Grunnable and with an 873 // M, but waiting for locks within the runtime. This field stores the value 874 // for Ms that have exited. 875 totalRuntimeLockWaitTime atomic.Int64 876 } 877 878 // Values for the flags field of a sigTabT. 879 const ( 880 _SigNotify = 1 << iota // let signal.Notify have signal, even if from kernel 881 _SigKill // if signal.Notify doesn't take it, exit quietly 882 _SigThrow // if signal.Notify doesn't take it, exit loudly 883 _SigPanic // if the signal is from the kernel, panic 884 _SigDefault // if the signal isn't explicitly requested, don't monitor it 885 _SigGoExit // cause all runtime procs to exit (only used on Plan 9). 886 _SigSetStack // Don't explicitly install handler, but add SA_ONSTACK to existing libc handler 887 _SigUnblock // always unblock; see blockableSig 888 _SigIgn // _SIG_DFL action is to ignore the signal 889 ) 890 891 // Layout of in-memory per-function information prepared by linker 892 // See https://golang.org/s/go12symtab. 893 // Keep in sync with linker (../cmd/link/internal/ld/pcln.go:/pclntab) 894 // and with package debug/gosym and with symtab.go in package runtime. 895 type _func struct { 896 sys.NotInHeap // Only in static data 897 898 entryOff uint32 // start pc, as offset from moduledata.text/pcHeader.textStart 899 nameOff int32 // function name, as index into moduledata.funcnametab. 900 901 args int32 // in/out args size 902 deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any. 903 904 pcsp uint32 905 pcfile uint32 906 pcln uint32 907 npcdata uint32 908 cuOffset uint32 // runtime.cutab offset of this function's CU 909 startLine int32 // line number of start of function (func keyword/TEXT directive) 910 funcID abi.FuncID // set for certain special runtime functions 911 flag abi.FuncFlag 912 _ [1]byte // pad 913 nfuncdata uint8 // must be last, must end on a uint32-aligned boundary 914 915 // The end of the struct is followed immediately by two variable-length 916 // arrays that reference the pcdata and funcdata locations for this 917 // function. 918 919 // pcdata contains the offset into moduledata.pctab for the start of 920 // that index's table. e.g., 921 // &moduledata.pctab[_func.pcdata[_PCDATA_UnsafePoint]] is the start of 922 // the unsafe point table. 923 // 924 // An offset of 0 indicates that there is no table. 925 // 926 // pcdata [npcdata]uint32 927 928 // funcdata contains the offset past moduledata.gofunc which contains a 929 // pointer to that index's funcdata. e.g., 930 // *(moduledata.gofunc + _func.funcdata[_FUNCDATA_ArgsPointerMaps]) is 931 // the argument pointer map. 932 // 933 // An offset of ^uint32(0) indicates that there is no entry. 934 // 935 // funcdata [nfuncdata]uint32 936 } 937 938 // Pseudo-Func that is returned for PCs that occur in inlined code. 939 // A *Func can be either a *_func or a *funcinl, and they are distinguished 940 // by the first uintptr. 941 // 942 // TODO(austin): Can we merge this with inlinedCall? 943 type funcinl struct { 944 ones uint32 // set to ^0 to distinguish from _func 945 entry uintptr // entry of the real (the "outermost") frame 946 name string 947 file string 948 line int32 949 startLine int32 950 } 951 952 type itab = abi.ITab 953 954 // Lock-free stack node. 955 // Also known to export_test.go. 956 type lfnode struct { 957 next uint64 958 pushcnt uintptr 959 } 960 961 type forcegcstate struct { 962 lock mutex 963 g *g 964 idle atomic.Bool 965 } 966 967 // A _defer holds an entry on the list of deferred calls. 968 // If you add a field here, add code to clear it in deferProcStack. 969 // This struct must match the code in cmd/compile/internal/ssagen/ssa.go:deferstruct 970 // and cmd/compile/internal/ssagen/ssa.go:(*state).call. 971 // Some defers will be allocated on the stack and some on the heap. 972 // All defers are logically part of the stack, so write barriers to 973 // initialize them are not required. All defers must be manually scanned, 974 // and for heap defers, marked. 975 type _defer struct { 976 heap bool 977 rangefunc bool // true for rangefunc list 978 sp uintptr // sp at time of defer 979 pc uintptr // pc at time of defer 980 fn func() // can be nil for open-coded defers 981 link *_defer // next defer on G; can point to either heap or stack! 982 983 // If rangefunc is true, *head is the head of the atomic linked list 984 // during a range-over-func execution. 985 head *atomic.Pointer[_defer] 986 } 987 988 // A _panic holds information about an active panic. 989 // 990 // A _panic value must only ever live on the stack. 991 // 992 // The argp and link fields are stack pointers, but don't need special 993 // handling during stack growth: because they are pointer-typed and 994 // _panic values only live on the stack, regular stack pointer 995 // adjustment takes care of them. 996 type _panic struct { 997 argp unsafe.Pointer // pointer to arguments of deferred call run during panic; cannot move - known to liblink 998 arg any // argument to panic 999 link *_panic // link to earlier panic 1000 1001 // startPC and startSP track where _panic.start was called. 1002 startPC uintptr 1003 startSP unsafe.Pointer 1004 1005 // The current stack frame that we're running deferred calls for. 1006 sp unsafe.Pointer 1007 lr uintptr 1008 fp unsafe.Pointer 1009 1010 // retpc stores the PC where the panic should jump back to, if the 1011 // function last returned by _panic.next() recovers the panic. 1012 retpc uintptr 1013 1014 // Extra state for handling open-coded defers. 1015 deferBitsPtr *uint8 1016 slotsPtr unsafe.Pointer 1017 1018 recovered bool // whether this panic has been recovered 1019 goexit bool 1020 deferreturn bool 1021 } 1022 1023 // savedOpenDeferState tracks the extra state from _panic that's 1024 // necessary for deferreturn to pick up where gopanic left off, 1025 // without needing to unwind the stack. 1026 type savedOpenDeferState struct { 1027 retpc uintptr 1028 deferBitsOffset uintptr 1029 slotsOffset uintptr 1030 } 1031 1032 // ancestorInfo records details of where a goroutine was started. 1033 type ancestorInfo struct { 1034 pcs []uintptr // pcs from the stack of this goroutine 1035 goid uint64 // goroutine id of this goroutine; original goroutine possibly dead 1036 gopc uintptr // pc of go statement that created this goroutine 1037 } 1038 1039 // A waitReason explains why a goroutine has been stopped. 1040 // See gopark. Do not re-use waitReasons, add new ones. 1041 type waitReason uint8 1042 1043 const ( 1044 waitReasonZero waitReason = iota // "" 1045 waitReasonGCAssistMarking // "GC assist marking" 1046 waitReasonIOWait // "IO wait" 1047 waitReasonChanReceiveNilChan // "chan receive (nil chan)" 1048 waitReasonChanSendNilChan // "chan send (nil chan)" 1049 waitReasonDumpingHeap // "dumping heap" 1050 waitReasonGarbageCollection // "garbage collection" 1051 waitReasonGarbageCollectionScan // "garbage collection scan" 1052 waitReasonPanicWait // "panicwait" 1053 waitReasonSelect // "select" 1054 waitReasonSelectNoCases // "select (no cases)" 1055 waitReasonGCAssistWait // "GC assist wait" 1056 waitReasonGCSweepWait // "GC sweep wait" 1057 waitReasonGCScavengeWait // "GC scavenge wait" 1058 waitReasonChanReceive // "chan receive" 1059 waitReasonChanSend // "chan send" 1060 waitReasonFinalizerWait // "finalizer wait" 1061 waitReasonForceGCIdle // "force gc (idle)" 1062 waitReasonSemacquire // "semacquire" 1063 waitReasonSleep // "sleep" 1064 waitReasonSyncCondWait // "sync.Cond.Wait" 1065 waitReasonSyncMutexLock // "sync.Mutex.Lock" 1066 waitReasonSyncRWMutexRLock // "sync.RWMutex.RLock" 1067 waitReasonSyncRWMutexLock // "sync.RWMutex.Lock" 1068 waitReasonSyncWaitGroupWait // "sync.WaitGroup.Wait" 1069 waitReasonTraceReaderBlocked // "trace reader (blocked)" 1070 waitReasonWaitForGCCycle // "wait for GC cycle" 1071 waitReasonGCWorkerIdle // "GC worker (idle)" 1072 waitReasonGCWorkerActive // "GC worker (active)" 1073 waitReasonPreempted // "preempted" 1074 waitReasonDebugCall // "debug call" 1075 waitReasonGCMarkTermination // "GC mark termination" 1076 waitReasonStoppingTheWorld // "stopping the world" 1077 waitReasonFlushProcCaches // "flushing proc caches" 1078 waitReasonTraceGoroutineStatus // "trace goroutine status" 1079 waitReasonTraceProcStatus // "trace proc status" 1080 waitReasonPageTraceFlush // "page trace flush" 1081 waitReasonCoroutine // "coroutine" 1082 waitReasonGCWeakToStrongWait // "GC weak to strong wait" 1083 waitReasonSynctestRun // "synctest.Run" 1084 waitReasonSynctestWait // "synctest.Wait" 1085 waitReasonSynctestChanReceive // "chan receive (synctest)" 1086 waitReasonSynctestChanSend // "chan send (synctest)" 1087 waitReasonSynctestSelect // "select (synctest)" 1088 ) 1089 1090 var waitReasonStrings = [...]string{ 1091 waitReasonZero: "", 1092 waitReasonGCAssistMarking: "GC assist marking", 1093 waitReasonIOWait: "IO wait", 1094 waitReasonChanReceiveNilChan: "chan receive (nil chan)", 1095 waitReasonChanSendNilChan: "chan send (nil chan)", 1096 waitReasonDumpingHeap: "dumping heap", 1097 waitReasonGarbageCollection: "garbage collection", 1098 waitReasonGarbageCollectionScan: "garbage collection scan", 1099 waitReasonPanicWait: "panicwait", 1100 waitReasonSelect: "select", 1101 waitReasonSelectNoCases: "select (no cases)", 1102 waitReasonGCAssistWait: "GC assist wait", 1103 waitReasonGCSweepWait: "GC sweep wait", 1104 waitReasonGCScavengeWait: "GC scavenge wait", 1105 waitReasonChanReceive: "chan receive", 1106 waitReasonChanSend: "chan send", 1107 waitReasonFinalizerWait: "finalizer wait", 1108 waitReasonForceGCIdle: "force gc (idle)", 1109 waitReasonSemacquire: "semacquire", 1110 waitReasonSleep: "sleep", 1111 waitReasonSyncCondWait: "sync.Cond.Wait", 1112 waitReasonSyncMutexLock: "sync.Mutex.Lock", 1113 waitReasonSyncRWMutexRLock: "sync.RWMutex.RLock", 1114 waitReasonSyncRWMutexLock: "sync.RWMutex.Lock", 1115 waitReasonSyncWaitGroupWait: "sync.WaitGroup.Wait", 1116 waitReasonTraceReaderBlocked: "trace reader (blocked)", 1117 waitReasonWaitForGCCycle: "wait for GC cycle", 1118 waitReasonGCWorkerIdle: "GC worker (idle)", 1119 waitReasonGCWorkerActive: "GC worker (active)", 1120 waitReasonPreempted: "preempted", 1121 waitReasonDebugCall: "debug call", 1122 waitReasonGCMarkTermination: "GC mark termination", 1123 waitReasonStoppingTheWorld: "stopping the world", 1124 waitReasonFlushProcCaches: "flushing proc caches", 1125 waitReasonTraceGoroutineStatus: "trace goroutine status", 1126 waitReasonTraceProcStatus: "trace proc status", 1127 waitReasonPageTraceFlush: "page trace flush", 1128 waitReasonCoroutine: "coroutine", 1129 waitReasonGCWeakToStrongWait: "GC weak to strong wait", 1130 waitReasonSynctestRun: "synctest.Run", 1131 waitReasonSynctestWait: "synctest.Wait", 1132 waitReasonSynctestChanReceive: "chan receive (synctest)", 1133 waitReasonSynctestChanSend: "chan send (synctest)", 1134 waitReasonSynctestSelect: "select (synctest)", 1135 } 1136 1137 func (w waitReason) String() string { 1138 if w < 0 || w >= waitReason(len(waitReasonStrings)) { 1139 return "unknown wait reason" 1140 } 1141 return waitReasonStrings[w] 1142 } 1143 1144 func (w waitReason) isMutexWait() bool { 1145 return w == waitReasonSyncMutexLock || 1146 w == waitReasonSyncRWMutexRLock || 1147 w == waitReasonSyncRWMutexLock 1148 } 1149 1150 func (w waitReason) isWaitingForGC() bool { 1151 return isWaitingForGC[w] 1152 } 1153 1154 // isWaitingForGC indicates that a goroutine is only entering _Gwaiting and 1155 // setting a waitReason because it needs to be able to let the GC take ownership 1156 // of its stack. The G is always actually executing on the system stack, in 1157 // these cases. 1158 // 1159 // TODO(mknyszek): Consider replacing this with a new dedicated G status. 1160 var isWaitingForGC = [len(waitReasonStrings)]bool{ 1161 waitReasonStoppingTheWorld: true, 1162 waitReasonGCMarkTermination: true, 1163 waitReasonGarbageCollection: true, 1164 waitReasonGarbageCollectionScan: true, 1165 waitReasonTraceGoroutineStatus: true, 1166 waitReasonTraceProcStatus: true, 1167 waitReasonPageTraceFlush: true, 1168 waitReasonGCAssistMarking: true, 1169 waitReasonGCWorkerActive: true, 1170 waitReasonFlushProcCaches: true, 1171 } 1172 1173 func (w waitReason) isIdleInSynctest() bool { 1174 return isIdleInSynctest[w] 1175 } 1176 1177 // isIdleInSynctest indicates that a goroutine is considered idle by synctest.Wait. 1178 var isIdleInSynctest = [len(waitReasonStrings)]bool{ 1179 waitReasonChanReceiveNilChan: true, 1180 waitReasonChanSendNilChan: true, 1181 waitReasonSelectNoCases: true, 1182 waitReasonSleep: true, 1183 waitReasonSyncCondWait: true, 1184 waitReasonSyncWaitGroupWait: true, 1185 waitReasonCoroutine: true, 1186 waitReasonSynctestRun: true, 1187 waitReasonSynctestWait: true, 1188 waitReasonSynctestChanReceive: true, 1189 waitReasonSynctestChanSend: true, 1190 waitReasonSynctestSelect: true, 1191 } 1192 1193 var ( 1194 allm *m 1195 gomaxprocs int32 1196 ncpu int32 1197 forcegc forcegcstate 1198 sched schedt 1199 newprocs int32 1200 ) 1201 1202 var ( 1203 // allpLock protects P-less reads and size changes of allp, idlepMask, 1204 // and timerpMask, and all writes to allp. 1205 allpLock mutex 1206 1207 // len(allp) == gomaxprocs; may change at safe points, otherwise 1208 // immutable. 1209 allp []*p 1210 1211 // Bitmask of Ps in _Pidle list, one bit per P. Reads and writes must 1212 // be atomic. Length may change at safe points. 1213 // 1214 // Each P must update only its own bit. In order to maintain 1215 // consistency, a P going idle must the idle mask simultaneously with 1216 // updates to the idle P list under the sched.lock, otherwise a racing 1217 // pidleget may clear the mask before pidleput sets the mask, 1218 // corrupting the bitmap. 1219 // 1220 // N.B., procresize takes ownership of all Ps in stopTheWorldWithSema. 1221 idlepMask pMask 1222 1223 // Bitmask of Ps that may have a timer, one bit per P. Reads and writes 1224 // must be atomic. Length may change at safe points. 1225 // 1226 // Ideally, the timer mask would be kept immediately consistent on any timer 1227 // operations. Unfortunately, updating a shared global data structure in the 1228 // timer hot path adds too much overhead in applications frequently switching 1229 // between no timers and some timers. 1230 // 1231 // As a compromise, the timer mask is updated only on pidleget / pidleput. A 1232 // running P (returned by pidleget) may add a timer at any time, so its mask 1233 // must be set. An idle P (passed to pidleput) cannot add new timers while 1234 // idle, so if it has no timers at that time, its mask may be cleared. 1235 // 1236 // Thus, we get the following effects on timer-stealing in findrunnable: 1237 // 1238 // - Idle Ps with no timers when they go idle are never checked in findrunnable 1239 // (for work- or timer-stealing; this is the ideal case). 1240 // - Running Ps must always be checked. 1241 // - Idle Ps whose timers are stolen must continue to be checked until they run 1242 // again, even after timer expiration. 1243 // 1244 // When the P starts running again, the mask should be set, as a timer may be 1245 // added at any time. 1246 // 1247 // TODO(prattmic): Additional targeted updates may improve the above cases. 1248 // e.g., updating the mask when stealing a timer. 1249 timerpMask pMask 1250 ) 1251 1252 // goarmsoftfp is used by runtime/cgo assembly. 1253 // 1254 //go:linkname goarmsoftfp 1255 1256 var ( 1257 // Pool of GC parked background workers. Entries are type 1258 // *gcBgMarkWorkerNode. 1259 gcBgMarkWorkerPool lfstack 1260 1261 // Total number of gcBgMarkWorker goroutines. Protected by worldsema. 1262 gcBgMarkWorkerCount int32 1263 1264 // Information about what cpu features are available. 1265 // Packages outside the runtime should not use these 1266 // as they are not an external api. 1267 // Set on startup in asm_{386,amd64}.s 1268 processorVersionInfo uint32 1269 isIntel bool 1270 ) 1271 1272 // set by cmd/link on arm systems 1273 // accessed using linkname by internal/runtime/atomic. 1274 // 1275 // goarm should be an internal detail, 1276 // but widely used packages access it using linkname. 1277 // Notable members of the hall of shame include: 1278 // - github.com/creativeprojects/go-selfupdate 1279 // 1280 // Do not remove or change the type signature. 1281 // See go.dev/issue/67401. 1282 // 1283 //go:linkname goarm 1284 var ( 1285 goarm uint8 1286 goarmsoftfp uint8 1287 ) 1288 1289 // Set by the linker so the runtime can determine the buildmode. 1290 var ( 1291 islibrary bool // -buildmode=c-shared 1292 isarchive bool // -buildmode=c-archive 1293 ) 1294 1295 // Must agree with internal/buildcfg.FramePointerEnabled. 1296 const framepointer_enabled = GOARCH == "amd64" || GOARCH == "arm64" 1297 1298 // getcallerfp returns the frame pointer of the caller of the caller 1299 // of this function. 1300 // 1301 //go:nosplit 1302 //go:noinline 1303 func getcallerfp() uintptr { 1304 fp := getfp() // This frame's FP. 1305 if fp != 0 { 1306 fp = *(*uintptr)(unsafe.Pointer(fp)) // The caller's FP. 1307 fp = *(*uintptr)(unsafe.Pointer(fp)) // The caller's caller's FP. 1308 } 1309 return fp 1310 } 1311