Source file src/runtime/runtime.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/abi"
     9  	"internal/runtime/atomic"
    10  	"unsafe"
    11  )
    12  
    13  //go:generate go run wincallback.go
    14  //go:generate go run mkduff.go
    15  //go:generate go run mkfastlog2table.go
    16  //go:generate go run mklockrank.go -o lockrank.go
    17  
    18  var ticks ticksType
    19  
    20  type ticksType struct {
    21  	// lock protects access to start* and val.
    22  	lock       mutex
    23  	startTicks int64
    24  	startTime  int64
    25  	val        atomic.Int64
    26  }
    27  
    28  // init initializes ticks to maximize the chance that we have a good ticksPerSecond reference.
    29  //
    30  // Must not run concurrently with ticksPerSecond.
    31  func (t *ticksType) init() {
    32  	lock(&ticks.lock)
    33  	t.startTime = nanotime()
    34  	t.startTicks = cputicks()
    35  	unlock(&ticks.lock)
    36  }
    37  
    38  // minTimeForTicksPerSecond is the minimum elapsed time we require to consider our ticksPerSecond
    39  // measurement to be of decent enough quality for profiling.
    40  //
    41  // There's a linear relationship here between minimum time and error from the true value.
    42  // The error from the true ticks-per-second in a linux/amd64 VM seems to be:
    43  // -   1 ms -> ~0.02% error
    44  // -   5 ms -> ~0.004% error
    45  // -  10 ms -> ~0.002% error
    46  // -  50 ms -> ~0.0003% error
    47  // - 100 ms -> ~0.0001% error
    48  //
    49  // We're willing to take 0.004% error here, because ticksPerSecond is intended to be used for
    50  // converting durations, not timestamps. Durations are usually going to be much larger, and so
    51  // the tiny error doesn't matter. The error is definitely going to be a problem when trying to
    52  // use this for timestamps, as it'll make those timestamps much less likely to line up.
    53  const minTimeForTicksPerSecond = 5_000_000*(1-osHasLowResClockInt) + 100_000_000*osHasLowResClockInt
    54  
    55  // ticksPerSecond returns a conversion rate between the cputicks clock and the nanotime clock.
    56  //
    57  // Note: Clocks are hard. Using this as an actual conversion rate for timestamps is ill-advised
    58  // and should be avoided when possible. Use only for durations, where a tiny error term isn't going
    59  // to make a meaningful difference in even a 1ms duration. If an accurate timestamp is needed,
    60  // use nanotime instead. (The entire Windows platform is a broad exception to this rule, where nanotime
    61  // produces timestamps on such a coarse granularity that the error from this conversion is actually
    62  // preferable.)
    63  //
    64  // The strategy for computing the conversion rate is to write down nanotime and cputicks as
    65  // early in process startup as possible. From then, we just need to wait until we get values
    66  // from nanotime that we can use (some platforms have a really coarse system time granularity).
    67  // We require some amount of time to pass to ensure that the conversion rate is fairly accurate
    68  // in aggregate. But because we compute this rate lazily, there's a pretty good chance a decent
    69  // amount of time has passed by the time we get here.
    70  //
    71  // Must be called from a normal goroutine context (running regular goroutine with a P).
    72  //
    73  // Called by runtime/pprof in addition to runtime code.
    74  //
    75  // TODO(mknyszek): This doesn't account for things like CPU frequency scaling. Consider
    76  // a more sophisticated and general approach in the future.
    77  func ticksPerSecond() int64 {
    78  	// Get the conversion rate if we've already computed it.
    79  	r := ticks.val.Load()
    80  	if r != 0 {
    81  		return r
    82  	}
    83  
    84  	// Compute the conversion rate.
    85  	for {
    86  		lock(&ticks.lock)
    87  		r = ticks.val.Load()
    88  		if r != 0 {
    89  			unlock(&ticks.lock)
    90  			return r
    91  		}
    92  
    93  		// Grab the current time in both clocks.
    94  		nowTime := nanotime()
    95  		nowTicks := cputicks()
    96  
    97  		// See if we can use these times.
    98  		if nowTicks > ticks.startTicks && nowTime-ticks.startTime > minTimeForTicksPerSecond {
    99  			// Perform the calculation with floats. We don't want to risk overflow.
   100  			r = int64(float64(nowTicks-ticks.startTicks) * 1e9 / float64(nowTime-ticks.startTime))
   101  			if r == 0 {
   102  				// Zero is both a sentinel value and it would be bad if callers used this as
   103  				// a divisor. We tried out best, so just make it 1.
   104  				r++
   105  			}
   106  			ticks.val.Store(r)
   107  			unlock(&ticks.lock)
   108  			break
   109  		}
   110  		unlock(&ticks.lock)
   111  
   112  		// Sleep in one millisecond increments until we have a reliable time.
   113  		timeSleep(1_000_000)
   114  	}
   115  	return r
   116  }
   117  
   118  var envs []string
   119  var argslice []string
   120  
   121  //go:linkname syscall_runtime_envs syscall.runtime_envs
   122  func syscall_runtime_envs() []string { return append([]string{}, envs...) }
   123  
   124  //go:linkname syscall_Getpagesize syscall.Getpagesize
   125  func syscall_Getpagesize() int { return int(physPageSize) }
   126  
   127  //go:linkname os_runtime_args os.runtime_args
   128  func os_runtime_args() []string { return append([]string{}, argslice...) }
   129  
   130  //go:linkname syscall_Exit syscall.Exit
   131  //go:nosplit
   132  func syscall_Exit(code int) {
   133  	exit(int32(code))
   134  }
   135  
   136  var godebugDefault string
   137  var godebugUpdate atomic.Pointer[func(string, string)]
   138  var godebugEnv atomic.Pointer[string] // set by parsedebugvars
   139  var godebugNewIncNonDefault atomic.Pointer[func(string) func()]
   140  
   141  //go:linkname godebug_setUpdate internal/godebug.setUpdate
   142  func godebug_setUpdate(update func(string, string)) {
   143  	p := new(func(string, string))
   144  	*p = update
   145  	godebugUpdate.Store(p)
   146  	godebugNotify(false)
   147  }
   148  
   149  //go:linkname godebug_setNewIncNonDefault internal/godebug.setNewIncNonDefault
   150  func godebug_setNewIncNonDefault(newIncNonDefault func(string) func()) {
   151  	p := new(func(string) func())
   152  	*p = newIncNonDefault
   153  	godebugNewIncNonDefault.Store(p)
   154  }
   155  
   156  // A godebugInc provides access to internal/godebug's IncNonDefault function
   157  // for a given GODEBUG setting.
   158  // Calls before internal/godebug registers itself are dropped on the floor.
   159  type godebugInc struct {
   160  	name string
   161  	inc  atomic.Pointer[func()]
   162  }
   163  
   164  func (g *godebugInc) IncNonDefault() {
   165  	inc := g.inc.Load()
   166  	if inc == nil {
   167  		newInc := godebugNewIncNonDefault.Load()
   168  		if newInc == nil {
   169  			return
   170  		}
   171  		inc = new(func())
   172  		*inc = (*newInc)(g.name)
   173  		if raceenabled {
   174  			racereleasemerge(unsafe.Pointer(&g.inc))
   175  		}
   176  		if !g.inc.CompareAndSwap(nil, inc) {
   177  			inc = g.inc.Load()
   178  		}
   179  	}
   180  	if raceenabled {
   181  		raceacquire(unsafe.Pointer(&g.inc))
   182  	}
   183  	(*inc)()
   184  }
   185  
   186  func godebugNotify(envChanged bool) {
   187  	update := godebugUpdate.Load()
   188  	var env string
   189  	if p := godebugEnv.Load(); p != nil {
   190  		env = *p
   191  	}
   192  	if envChanged {
   193  		reparsedebugvars(env)
   194  	}
   195  	if update != nil {
   196  		(*update)(godebugDefault, env)
   197  	}
   198  }
   199  
   200  //go:linkname syscall_runtimeSetenv syscall.runtimeSetenv
   201  func syscall_runtimeSetenv(key, value string) {
   202  	setenv_c(key, value)
   203  	if key == "GODEBUG" {
   204  		p := new(string)
   205  		*p = value
   206  		godebugEnv.Store(p)
   207  		godebugNotify(true)
   208  	}
   209  }
   210  
   211  //go:linkname syscall_runtimeUnsetenv syscall.runtimeUnsetenv
   212  func syscall_runtimeUnsetenv(key string) {
   213  	unsetenv_c(key)
   214  	if key == "GODEBUG" {
   215  		godebugEnv.Store(nil)
   216  		godebugNotify(true)
   217  	}
   218  }
   219  
   220  // writeErrStr writes a string to descriptor 2.
   221  // If SetCrashOutput(f) was called, it also writes to f.
   222  //
   223  //go:nosplit
   224  func writeErrStr(s string) {
   225  	writeErrData(unsafe.StringData(s), int32(len(s)))
   226  }
   227  
   228  // writeErrData is the common parts of writeErr{,Str}.
   229  //
   230  //go:nosplit
   231  func writeErrData(data *byte, n int32) {
   232  	write(2, unsafe.Pointer(data), n)
   233  
   234  	// If crashing, print a copy to the SetCrashOutput fd.
   235  	gp := getg()
   236  	if gp != nil && gp.m.dying > 0 ||
   237  		gp == nil && panicking.Load() > 0 {
   238  		if fd := crashFD.Load(); fd != ^uintptr(0) {
   239  			write(fd, unsafe.Pointer(data), n)
   240  		}
   241  	}
   242  }
   243  
   244  // crashFD is an optional file descriptor to use for fatal panics, as
   245  // set by debug.SetCrashOutput (see #42888). If it is a valid fd (not
   246  // all ones), writeErr and related functions write to it in addition
   247  // to standard error.
   248  //
   249  // Initialized to -1 in schedinit.
   250  var crashFD atomic.Uintptr
   251  
   252  //go:linkname setCrashFD
   253  func setCrashFD(fd uintptr) uintptr {
   254  	// Don't change the crash FD if a crash is already in progress.
   255  	//
   256  	// Unlike the case below, this is not required for correctness, but it
   257  	// is generally nicer to have all of the crash output go to the same
   258  	// place rather than getting split across two different FDs.
   259  	if panicking.Load() > 0 {
   260  		return ^uintptr(0)
   261  	}
   262  
   263  	old := crashFD.Swap(fd)
   264  
   265  	// If we are panicking, don't return the old FD to runtime/debug for
   266  	// closing. writeErrData may have already read the old FD from crashFD
   267  	// before the swap and closing it would cause the write to be lost [1].
   268  	// The old FD will never be closed, but we are about to crash anyway.
   269  	//
   270  	// On the writeErrData thread, panicking.Add(1) happens-before
   271  	// crashFD.Load() [2].
   272  	//
   273  	// On this thread, swapping old FD for new in crashFD happens-before
   274  	// panicking.Load() > 0.
   275  	//
   276  	// Therefore, if panicking.Load() == 0 here (old FD will be closed), it
   277  	// is impossible for the writeErrData thread to observe
   278  	// crashFD.Load() == old FD.
   279  	//
   280  	// [1] Or, if really unlucky, another concurrent open could reuse the
   281  	// FD, sending the write into an unrelated file.
   282  	//
   283  	// [2] If gp != nil, it occurs when incrementing gp.m.dying in
   284  	// startpanic_m. If gp == nil, we read panicking.Load() > 0, so an Add
   285  	// must have happened-before.
   286  	if panicking.Load() > 0 {
   287  		return ^uintptr(0)
   288  	}
   289  	return old
   290  }
   291  
   292  // auxv is populated on relevant platforms but defined here for all platforms
   293  // so x/sys/cpu can assume the getAuxv symbol exists without keeping its list
   294  // of auxv-using GOOS build tags in sync.
   295  //
   296  // It contains an even number of elements, (tag, value) pairs.
   297  var auxv []uintptr
   298  
   299  // golang.org/x/sys/cpu uses getAuxv via linkname.
   300  // Do not remove or change the type signature.
   301  // (See go.dev/issue/57336.)
   302  //
   303  // getAuxv should be an internal detail,
   304  // but widely used packages access it using linkname.
   305  // Notable members of the hall of shame include:
   306  //   - github.com/cilium/ebpf
   307  //
   308  // Do not remove or change the type signature.
   309  // See go.dev/issue/67401.
   310  //
   311  //go:linkname getAuxv
   312  func getAuxv() []uintptr { return auxv }
   313  
   314  // zeroVal is used by reflect via linkname.
   315  //
   316  // zeroVal should be an internal detail,
   317  // but widely used packages access it using linkname.
   318  // Notable members of the hall of shame include:
   319  //   - github.com/ugorji/go/codec
   320  //
   321  // Do not remove or change the type signature.
   322  // See go.dev/issue/67401.
   323  //
   324  //go:linkname zeroVal
   325  var zeroVal [abi.ZeroValSize]byte
   326  

View as plain text