aboutsummaryrefslogtreecommitdiff
path: root/src/runtime/stack.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/runtime/stack.c')
-rw-r--r--src/runtime/stack.c874
1 files changed, 0 insertions, 874 deletions
diff --git a/src/runtime/stack.c b/src/runtime/stack.c
deleted file mode 100644
index ffae73a2ab..0000000000
--- a/src/runtime/stack.c
+++ /dev/null
@@ -1,874 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-#include "runtime.h"
-#include "arch_GOARCH.h"
-#include "malloc.h"
-#include "stack.h"
-#include "funcdata.h"
-#include "typekind.h"
-#include "type.h"
-#include "race.h"
-#include "mgc0.h"
-#include "textflag.h"
-
-enum
-{
- // StackDebug == 0: no logging
- // == 1: logging of per-stack operations
- // == 2: logging of per-frame operations
- // == 3: logging of per-word updates
- // == 4: logging of per-word reads
- StackDebug = 0,
- StackFromSystem = 0, // allocate stacks from system memory instead of the heap
- StackFaultOnFree = 0, // old stacks are mapped noaccess to detect use after free
- StackPoisonCopy = 0, // fill stack that should not be accessed with garbage, to detect bad dereferences during copy
-
- StackCache = 1,
-};
-
-// Global pool of spans that have free stacks.
-// Stacks are assigned an order according to size.
-// order = log_2(size/FixedStack)
-// There is a free list for each order.
-MSpan runtime·stackpool[NumStackOrders];
-Mutex runtime·stackpoolmu;
-// TODO: one lock per order?
-
-static Stack stackfreequeue;
-
-void
-runtime·stackinit(void)
-{
- int32 i;
-
- if((StackCacheSize & PageMask) != 0)
- runtime·throw("cache size must be a multiple of page size");
-
- for(i = 0; i < NumStackOrders; i++)
- runtime·MSpanList_Init(&runtime·stackpool[i]);
-}
-
-// Allocates a stack from the free pool. Must be called with
-// stackpoolmu held.
-static MLink*
-poolalloc(uint8 order)
-{
- MSpan *list;
- MSpan *s;
- MLink *x;
- uintptr i;
-
- list = &runtime·stackpool[order];
- s = list->next;
- if(s == list) {
- // no free stacks. Allocate another span worth.
- s = runtime·MHeap_AllocStack(&runtime·mheap, StackCacheSize >> PageShift);
- if(s == nil)
- runtime·throw("out of memory");
- if(s->ref != 0)
- runtime·throw("bad ref");
- if(s->freelist != nil)
- runtime·throw("bad freelist");
- for(i = 0; i < StackCacheSize; i += FixedStack << order) {
- x = (MLink*)((s->start << PageShift) + i);
- x->next = s->freelist;
- s->freelist = x;
- }
- runtime·MSpanList_Insert(list, s);
- }
- x = s->freelist;
- if(x == nil)
- runtime·throw("span has no free stacks");
- s->freelist = x->next;
- s->ref++;
- if(s->freelist == nil) {
- // all stacks in s are allocated.
- runtime·MSpanList_Remove(s);
- }
- return x;
-}
-
-// Adds stack x to the free pool. Must be called with stackpoolmu held.
-static void
-poolfree(MLink *x, uint8 order)
-{
- MSpan *s;
-
- s = runtime·MHeap_Lookup(&runtime·mheap, x);
- if(s->state != MSpanStack)
- runtime·throw("freeing stack not in a stack span");
- if(s->freelist == nil) {
- // s will now have a free stack
- runtime·MSpanList_Insert(&runtime·stackpool[order], s);
- }
- x->next = s->freelist;
- s->freelist = x;
- s->ref--;
- if(s->ref == 0) {
- // span is completely free - return to heap
- runtime·MSpanList_Remove(s);
- s->freelist = nil;
- runtime·MHeap_FreeStack(&runtime·mheap, s);
- }
-}
-
-// stackcacherefill/stackcacherelease implement a global pool of stack segments.
-// The pool is required to prevent unlimited growth of per-thread caches.
-static void
-stackcacherefill(MCache *c, uint8 order)
-{
- MLink *x, *list;
- uintptr size;
-
- if(StackDebug >= 1)
- runtime·printf("stackcacherefill order=%d\n", order);
-
- // Grab some stacks from the global cache.
- // Grab half of the allowed capacity (to prevent thrashing).
- list = nil;
- size = 0;
- runtime·lock(&runtime·stackpoolmu);
- while(size < StackCacheSize/2) {
- x = poolalloc(order);
- x->next = list;
- list = x;
- size += FixedStack << order;
- }
- runtime·unlock(&runtime·stackpoolmu);
-
- c->stackcache[order].list = list;
- c->stackcache[order].size = size;
-}
-
-static void
-stackcacherelease(MCache *c, uint8 order)
-{
- MLink *x, *y;
- uintptr size;
-
- if(StackDebug >= 1)
- runtime·printf("stackcacherelease order=%d\n", order);
- x = c->stackcache[order].list;
- size = c->stackcache[order].size;
- runtime·lock(&runtime·stackpoolmu);
- while(size > StackCacheSize/2) {
- y = x->next;
- poolfree(x, order);
- x = y;
- size -= FixedStack << order;
- }
- runtime·unlock(&runtime·stackpoolmu);
- c->stackcache[order].list = x;
- c->stackcache[order].size = size;
-}
-
-void
-runtime·stackcache_clear(MCache *c)
-{
- uint8 order;
- MLink *x, *y;
-
- if(StackDebug >= 1)
- runtime·printf("stackcache clear\n");
- runtime·lock(&runtime·stackpoolmu);
- for(order = 0; order < NumStackOrders; order++) {
- x = c->stackcache[order].list;
- while(x != nil) {
- y = x->next;
- poolfree(x, order);
- x = y;
- }
- c->stackcache[order].list = nil;
- c->stackcache[order].size = 0;
- }
- runtime·unlock(&runtime·stackpoolmu);
-}
-
-Stack
-runtime·stackalloc(uint32 n)
-{
- uint8 order;
- uint32 n2;
- void *v;
- MLink *x;
- MSpan *s;
- MCache *c;
-
- // Stackalloc must be called on scheduler stack, so that we
- // never try to grow the stack during the code that stackalloc runs.
- // Doing so would cause a deadlock (issue 1547).
- if(g != g->m->g0)
- runtime·throw("stackalloc not on scheduler stack");
- if((n & (n-1)) != 0)
- runtime·throw("stack size not a power of 2");
- if(StackDebug >= 1)
- runtime·printf("stackalloc %d\n", n);
-
- if(runtime·debug.efence || StackFromSystem) {
- v = runtime·sysAlloc(ROUND(n, PageSize), &mstats.stacks_sys);
- if(v == nil)
- runtime·throw("out of memory (stackalloc)");
- return (Stack){(uintptr)v, (uintptr)v+n};
- }
-
- // Small stacks are allocated with a fixed-size free-list allocator.
- // If we need a stack of a bigger size, we fall back on allocating
- // a dedicated span.
- if(StackCache && n < FixedStack << NumStackOrders && n < StackCacheSize) {
- order = 0;
- n2 = n;
- while(n2 > FixedStack) {
- order++;
- n2 >>= 1;
- }
- c = g->m->mcache;
- if(c == nil || g->m->gcing || g->m->helpgc) {
- // c == nil can happen in the guts of exitsyscall or
- // procresize. Just get a stack from the global pool.
- // Also don't touch stackcache during gc
- // as it's flushed concurrently.
- runtime·lock(&runtime·stackpoolmu);
- x = poolalloc(order);
- runtime·unlock(&runtime·stackpoolmu);
- } else {
- x = c->stackcache[order].list;
- if(x == nil) {
- stackcacherefill(c, order);
- x = c->stackcache[order].list;
- }
- c->stackcache[order].list = x->next;
- c->stackcache[order].size -= n;
- }
- v = (byte*)x;
- } else {
- s = runtime·MHeap_AllocStack(&runtime·mheap, ROUND(n, PageSize) >> PageShift);
- if(s == nil)
- runtime·throw("out of memory");
- v = (byte*)(s->start<<PageShift);
- }
-
- if(raceenabled)
- runtime·racemalloc(v, n);
- if(StackDebug >= 1)
- runtime·printf(" allocated %p\n", v);
- return (Stack){(uintptr)v, (uintptr)v+n};
-}
-
-void
-runtime·stackfree(Stack stk)
-{
- uint8 order;
- uintptr n, n2;
- MSpan *s;
- MLink *x;
- MCache *c;
- void *v;
-
- n = stk.hi - stk.lo;
- v = (void*)stk.lo;
- if(n & (n-1))
- runtime·throw("stack not a power of 2");
- if(StackDebug >= 1) {
- runtime·printf("stackfree %p %d\n", v, (int32)n);
- runtime·memclr(v, n); // for testing, clobber stack data
- }
- if(runtime·debug.efence || StackFromSystem) {
- if(runtime·debug.efence || StackFaultOnFree)
- runtime·SysFault(v, n);
- else
- runtime·SysFree(v, n, &mstats.stacks_sys);
- return;
- }
- if(StackCache && n < FixedStack << NumStackOrders && n < StackCacheSize) {
- order = 0;
- n2 = n;
- while(n2 > FixedStack) {
- order++;
- n2 >>= 1;
- }
- x = (MLink*)v;
- c = g->m->mcache;
- if(c == nil || g->m->gcing || g->m->helpgc) {
- runtime·lock(&runtime·stackpoolmu);
- poolfree(x, order);
- runtime·unlock(&runtime·stackpoolmu);
- } else {
- if(c->stackcache[order].size >= StackCacheSize)
- stackcacherelease(c, order);
- x->next = c->stackcache[order].list;
- c->stackcache[order].list = x;
- c->stackcache[order].size += n;
- }
- } else {
- s = runtime·MHeap_Lookup(&runtime·mheap, v);
- if(s->state != MSpanStack) {
- runtime·printf("%p %p\n", s->start<<PageShift, v);
- runtime·throw("bad span state");
- }
- runtime·MHeap_FreeStack(&runtime·mheap, s);
- }
-}
-
-uintptr runtime·maxstacksize = 1<<20; // enough until runtime.main sets it for real
-
-static uint8*
-mapnames[] = {
- (uint8*)"---",
- (uint8*)"scalar",
- (uint8*)"ptr",
- (uint8*)"multi",
-};
-
-// Stack frame layout
-//
-// (x86)
-// +------------------+
-// | args from caller |
-// +------------------+ <- frame->argp
-// | return address |
-// +------------------+ <- frame->varp
-// | locals |
-// +------------------+
-// | args to callee |
-// +------------------+ <- frame->sp
-//
-// (arm)
-// +------------------+
-// | args from caller |
-// +------------------+ <- frame->argp
-// | caller's retaddr |
-// +------------------+ <- frame->varp
-// | locals |
-// +------------------+
-// | args to callee |
-// +------------------+
-// | return address |
-// +------------------+ <- frame->sp
-
-void runtime·main(void);
-void runtime·switchtoM(void(*)(void));
-
-typedef struct AdjustInfo AdjustInfo;
-struct AdjustInfo {
- Stack old;
- uintptr delta; // ptr distance from old to new stack (newbase - oldbase)
-};
-
-// Adjustpointer checks whether *vpp is in the old stack described by adjinfo.
-// If so, it rewrites *vpp to point into the new stack.
-static void
-adjustpointer(AdjustInfo *adjinfo, void *vpp)
-{
- byte **pp, *p;
-
- pp = vpp;
- p = *pp;
- if(StackDebug >= 4)
- runtime·printf(" %p:%p\n", pp, p);
- if(adjinfo->old.lo <= (uintptr)p && (uintptr)p < adjinfo->old.hi) {
- *pp = p + adjinfo->delta;
- if(StackDebug >= 3)
- runtime·printf(" adjust ptr %p: %p -> %p\n", pp, p, *pp);
- }
-}
-
-// bv describes the memory starting at address scanp.
-// Adjust any pointers contained therein.
-static void
-adjustpointers(byte **scanp, BitVector *bv, AdjustInfo *adjinfo, Func *f)
-{
- uintptr delta;
- int32 num, i;
- byte *p, *minp, *maxp;
-
- minp = (byte*)adjinfo->old.lo;
- maxp = (byte*)adjinfo->old.hi;
- delta = adjinfo->delta;
- num = bv->n / BitsPerPointer;
- for(i = 0; i < num; i++) {
- if(StackDebug >= 4)
- runtime·printf(" %p:%s:%p\n", &scanp[i], mapnames[bv->bytedata[i / (8 / BitsPerPointer)] >> (i * BitsPerPointer & 7) & 3], scanp[i]);
- switch(bv->bytedata[i / (8 / BitsPerPointer)] >> (i * BitsPerPointer & 7) & 3) {
- case BitsDead:
- if(runtime·debug.gcdead)
- scanp[i] = (byte*)PoisonStack;
- break;
- case BitsScalar:
- break;
- case BitsPointer:
- p = scanp[i];
- if(f != nil && (byte*)0 < p && (p < (byte*)PageSize && runtime·invalidptr || (uintptr)p == PoisonGC || (uintptr)p == PoisonStack)) {
- // Looks like a junk value in a pointer slot.
- // Live analysis wrong?
- g->m->traceback = 2;
- runtime·printf("runtime: bad pointer in frame %s at %p: %p\n", runtime·funcname(f), &scanp[i], p);
- runtime·throw("invalid stack pointer");
- }
- if(minp <= p && p < maxp) {
- if(StackDebug >= 3)
- runtime·printf("adjust ptr %p %s\n", p, runtime·funcname(f));
- scanp[i] = p + delta;
- }
- break;
- case BitsMultiWord:
- runtime·throw("adjustpointers: unexpected garbage collection bits");
- }
- }
-}
-
-// Note: the argument/return area is adjusted by the callee.
-static bool
-adjustframe(Stkframe *frame, void *arg)
-{
- AdjustInfo *adjinfo;
- Func *f;
- StackMap *stackmap;
- int32 pcdata;
- BitVector bv;
- uintptr targetpc, size, minsize;
-
- adjinfo = arg;
- targetpc = frame->continpc;
- if(targetpc == 0) {
- // Frame is dead.
- return true;
- }
- f = frame->fn;
- if(StackDebug >= 2)
- runtime·printf(" adjusting %s frame=[%p,%p] pc=%p continpc=%p\n", runtime·funcname(f), frame->sp, frame->fp, frame->pc, frame->continpc);
- if(f->entry == (uintptr)runtime·switchtoM) {
- // A special routine at the bottom of stack of a goroutine that does an onM call.
- // We will allow it to be copied even though we don't
- // have full GC info for it (because it is written in asm).
- return true;
- }
- if(targetpc != f->entry)
- targetpc--;
- pcdata = runtime·pcdatavalue(f, PCDATA_StackMapIndex, targetpc);
- if(pcdata == -1)
- pcdata = 0; // in prologue
-
- // Adjust local variables if stack frame has been allocated.
- size = frame->varp - frame->sp;
- if(thechar != '6' && thechar != '8')
- minsize = sizeof(uintptr);
- else
- minsize = 0;
- if(size > minsize) {
- stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps);
- if(stackmap == nil || stackmap->n <= 0) {
- runtime·printf("runtime: frame %s untyped locals %p+%p\n", runtime·funcname(f), (byte*)(frame->varp-size), size);
- runtime·throw("missing stackmap");
- }
- // Locals bitmap information, scan just the pointers in locals.
- if(pcdata < 0 || pcdata >= stackmap->n) {
- // don't know where we are
- runtime·printf("runtime: pcdata is %d and %d locals stack map entries for %s (targetpc=%p)\n",
- pcdata, stackmap->n, runtime·funcname(f), targetpc);
- runtime·throw("bad symbol table");
- }
- bv = runtime·stackmapdata(stackmap, pcdata);
- size = (bv.n * PtrSize) / BitsPerPointer;
- if(StackDebug >= 3)
- runtime·printf(" locals\n");
- adjustpointers((byte**)(frame->varp - size), &bv, adjinfo, f);
- }
-
- // Adjust arguments.
- if(frame->arglen > 0) {
- if(frame->argmap != nil) {
- bv = *frame->argmap;
- } else {
- stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps);
- if(stackmap == nil || stackmap->n <= 0) {
- runtime·printf("runtime: frame %s untyped args %p+%p\n", runtime·funcname(f), frame->argp, (uintptr)frame->arglen);
- runtime·throw("missing stackmap");
- }
- if(pcdata < 0 || pcdata >= stackmap->n) {
- // don't know where we are
- runtime·printf("runtime: pcdata is %d and %d args stack map entries for %s (targetpc=%p)\n",
- pcdata, stackmap->n, runtime·funcname(f), targetpc);
- runtime·throw("bad symbol table");
- }
- bv = runtime·stackmapdata(stackmap, pcdata);
- }
- if(StackDebug >= 3)
- runtime·printf(" args\n");
- adjustpointers((byte**)frame->argp, &bv, adjinfo, nil);
- }
-
- return true;
-}
-
-static void
-adjustctxt(G *gp, AdjustInfo *adjinfo)
-{
- adjustpointer(adjinfo, &gp->sched.ctxt);
-}
-
-static void
-adjustdefers(G *gp, AdjustInfo *adjinfo)
-{
- Defer *d;
- bool (*cb)(Stkframe*, void*);
-
- // Adjust defer argument blocks the same way we adjust active stack frames.
- cb = adjustframe;
- runtime·tracebackdefers(gp, &cb, adjinfo);
-
- // Adjust pointers in the Defer structs.
- // Defer structs themselves are never on the stack.
- for(d = gp->defer; d != nil; d = d->link) {
- adjustpointer(adjinfo, &d->fn);
- adjustpointer(adjinfo, &d->argp);
- adjustpointer(adjinfo, &d->panic);
- }
-}
-
-static void
-adjustpanics(G *gp, AdjustInfo *adjinfo)
-{
- // Panics are on stack and already adjusted.
- // Update pointer to head of list in G.
- adjustpointer(adjinfo, &gp->panic);
-}
-
-static void
-adjustsudogs(G *gp, AdjustInfo *adjinfo)
-{
- SudoG *s;
-
- // the data elements pointed to by a SudoG structure
- // might be in the stack.
- for(s = gp->waiting; s != nil; s = s->waitlink) {
- adjustpointer(adjinfo, &s->elem);
- adjustpointer(adjinfo, &s->selectdone);
- }
-}
-
-// Copies gp's stack to a new stack of a different size.
-// Caller must have changed gp status to Gcopystack.
-static void
-copystack(G *gp, uintptr newsize)
-{
- Stack old, new;
- uintptr used;
- AdjustInfo adjinfo;
- bool (*cb)(Stkframe*, void*);
- byte *p, *ep;
-
- if(gp->syscallsp != 0)
- runtime·throw("stack growth not allowed in system call");
- old = gp->stack;
- if(old.lo == 0)
- runtime·throw("nil stackbase");
- used = old.hi - gp->sched.sp;
-
- // allocate new stack
- new = runtime·stackalloc(newsize);
- if(StackPoisonCopy) {
- p = (byte*)new.lo;
- ep = (byte*)new.hi;
- while(p < ep)
- *p++ = 0xfd;
- }
-
- if(StackDebug >= 1)
- runtime·printf("copystack gp=%p [%p %p %p]/%d -> [%p %p %p]/%d\n", gp, old.lo, old.hi-used, old.hi, (int32)(old.hi-old.lo), new.lo, new.hi-used, new.hi, (int32)newsize);
-
- // adjust pointers in the to-be-copied frames
- adjinfo.old = old;
- adjinfo.delta = new.hi - old.hi;
- cb = adjustframe;
- runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, &cb, &adjinfo, 0);
-
- // adjust other miscellaneous things that have pointers into stacks.
- adjustctxt(gp, &adjinfo);
- adjustdefers(gp, &adjinfo);
- adjustpanics(gp, &adjinfo);
- adjustsudogs(gp, &adjinfo);
-
- // copy the stack to the new location
- if(StackPoisonCopy) {
- p = (byte*)new.lo;
- ep = (byte*)new.hi;
- while(p < ep)
- *p++ = 0xfb;
- }
- runtime·memmove((byte*)new.hi - used, (byte*)old.hi - used, used);
-
- // Swap out old stack for new one
- gp->stack = new;
- gp->stackguard0 = new.lo + StackGuard; // NOTE: might clobber a preempt request
- gp->sched.sp = new.hi - used;
-
- // free old stack
- if(StackPoisonCopy) {
- p = (byte*)old.lo;
- ep = (byte*)old.hi;
- while(p < ep)
- *p++ = 0xfc;
- }
- if(newsize > old.hi-old.lo) {
- // growing, free stack immediately
- runtime·stackfree(old);
- } else {
- // shrinking, queue up free operation. We can't actually free the stack
- // just yet because we might run into the following situation:
- // 1) GC starts, scans a SudoG but does not yet mark the SudoG.elem pointer
- // 2) The stack that pointer points to is shrunk
- // 3) The old stack is freed
- // 4) The containing span is marked free
- // 5) GC attempts to mark the SudoG.elem pointer. The marking fails because
- // the pointer looks like a pointer into a free span.
- // By not freeing, we prevent step #4 until GC is done.
- runtime·lock(&runtime·stackpoolmu);
- *(Stack*)old.lo = stackfreequeue;
- stackfreequeue = old;
- runtime·unlock(&runtime·stackpoolmu);
- }
-}
-
-// round x up to a power of 2.
-int32
-runtime·round2(int32 x)
-{
- int32 s;
-
- s = 0;
- while((1 << s) < x)
- s++;
- return 1 << s;
-}
-
-// Called from runtime·morestack when more stack is needed.
-// Allocate larger stack and relocate to new stack.
-// Stack growth is multiplicative, for constant amortized cost.
-//
-// g->atomicstatus will be Grunning or Gscanrunning upon entry.
-// If the GC is trying to stop this g then it will set preemptscan to true.
-void
-runtime·newstack(void)
-{
- int32 oldsize, newsize;
- uint32 oldstatus;
- uintptr sp;
- G *gp;
- Gobuf morebuf;
-
- if(g->m->morebuf.g->stackguard0 == (uintptr)StackFork)
- runtime·throw("stack growth after fork");
- if(g->m->morebuf.g != g->m->curg) {
- runtime·printf("runtime: newstack called from g=%p\n"
- "\tm=%p m->curg=%p m->g0=%p m->gsignal=%p\n",
- g->m->morebuf.g, g->m, g->m->curg, g->m->g0, g->m->gsignal);
- morebuf = g->m->morebuf;
- runtime·traceback(morebuf.pc, morebuf.sp, morebuf.lr, morebuf.g);
- runtime·throw("runtime: wrong goroutine in newstack");
- }
- if(g->m->curg->throwsplit)
- runtime·throw("runtime: stack split at bad time");
-
- // The goroutine must be executing in order to call newstack,
- // so it must be Grunning or Gscanrunning.
-
- gp = g->m->curg;
- morebuf = g->m->morebuf;
- g->m->morebuf.pc = (uintptr)nil;
- g->m->morebuf.lr = (uintptr)nil;
- g->m->morebuf.sp = (uintptr)nil;
- g->m->morebuf.g = (G*)nil;
-
- runtime·casgstatus(gp, Grunning, Gwaiting);
- gp->waitreason = runtime·gostringnocopy((byte*)"stack growth");
-
- runtime·rewindmorestack(&gp->sched);
-
- if(gp->stack.lo == 0)
- runtime·throw("missing stack in newstack");
- sp = gp->sched.sp;
- if(thechar == '6' || thechar == '8') {
- // The call to morestack cost a word.
- sp -= sizeof(uintreg);
- }
- if(StackDebug >= 1 || sp < gp->stack.lo) {
- runtime·printf("runtime: newstack sp=%p stack=[%p, %p]\n"
- "\tmorebuf={pc:%p sp:%p lr:%p}\n"
- "\tsched={pc:%p sp:%p lr:%p ctxt:%p}\n",
- sp, gp->stack.lo, gp->stack.hi,
- g->m->morebuf.pc, g->m->morebuf.sp, g->m->morebuf.lr,
- gp->sched.pc, gp->sched.sp, gp->sched.lr, gp->sched.ctxt);
- }
- if(sp < gp->stack.lo) {
- runtime·printf("runtime: gp=%p, gp->status=%d\n ", (void*)gp, runtime·readgstatus(gp));
- runtime·printf("runtime: split stack overflow: %p < %p\n", sp, gp->stack.lo);
- runtime·throw("runtime: split stack overflow");
- }
-
- if(gp->sched.ctxt != nil) {
- // morestack wrote sched.ctxt on its way in here,
- // without a write barrier. Run the write barrier now.
- // It is not possible to be preempted between then
- // and now, so it's okay.
- runtime·writebarrierptr_nostore(&gp->sched.ctxt, gp->sched.ctxt);
- }
-
- if(gp->stackguard0 == (uintptr)StackPreempt) {
- if(gp == g->m->g0)
- runtime·throw("runtime: preempt g0");
- if(g->m->p == nil && g->m->locks == 0)
- runtime·throw("runtime: g is running but p is not");
- if(gp->preemptscan) {
- runtime·gcphasework(gp);
- runtime·casgstatus(gp, Gwaiting, Grunning);
- gp->stackguard0 = gp->stack.lo + StackGuard;
- gp->preempt = false;
- gp->preemptscan = false; // Tells the GC premption was successful.
- runtime·gogo(&gp->sched); // never return
- }
-
- // Be conservative about where we preempt.
- // We are interested in preempting user Go code, not runtime code.
- if(g->m->locks || g->m->mallocing || g->m->gcing || g->m->p->status != Prunning) {
- // Let the goroutine keep running for now.
- // gp->preempt is set, so it will be preempted next time.
- gp->stackguard0 = gp->stack.lo + StackGuard;
- runtime·casgstatus(gp, Gwaiting, Grunning);
- runtime·gogo(&gp->sched); // never return
- }
- // Act like goroutine called runtime.Gosched.
- runtime·casgstatus(gp, Gwaiting, Grunning);
- runtime·gosched_m(gp); // never return
- }
-
- // Allocate a bigger segment and move the stack.
- oldsize = gp->stack.hi - gp->stack.lo;
- newsize = oldsize * 2;
- if(newsize > runtime·maxstacksize) {
- runtime·printf("runtime: goroutine stack exceeds %D-byte limit\n", (uint64)runtime·maxstacksize);
- runtime·throw("stack overflow");
- }
-
- oldstatus = runtime·readgstatus(gp);
- oldstatus &= ~Gscan;
- runtime·casgstatus(gp, oldstatus, Gcopystack); // oldstatus is Gwaiting or Grunnable
- // The concurrent GC will not scan the stack while we are doing the copy since
- // the gp is in a Gcopystack status.
- copystack(gp, newsize);
- if(StackDebug >= 1)
- runtime·printf("stack grow done\n");
- runtime·casgstatus(gp, Gcopystack, Grunning);
- runtime·gogo(&gp->sched);
-}
-
-#pragma textflag NOSPLIT
-void
-runtime·nilfunc(void)
-{
- *(byte*)0 = 0;
-}
-
-// adjust Gobuf as if it executed a call to fn
-// and then did an immediate gosave.
-void
-runtime·gostartcallfn(Gobuf *gobuf, FuncVal *fv)
-{
- void *fn;
-
- if(fv != nil)
- fn = fv->fn;
- else
- fn = runtime·nilfunc;
- runtime·gostartcall(gobuf, fn, fv);
-}
-
-// Maybe shrink the stack being used by gp.
-// Called at garbage collection time.
-void
-runtime·shrinkstack(G *gp)
-{
- uintptr used, oldsize, newsize;
- uint32 oldstatus;
-
- if(runtime·readgstatus(gp) == Gdead) {
- if(gp->stack.lo != 0) {
- // Free whole stack - it will get reallocated
- // if G is used again.
- runtime·stackfree(gp->stack);
- gp->stack.lo = 0;
- gp->stack.hi = 0;
- }
- return;
- }
- if(gp->stack.lo == 0)
- runtime·throw("missing stack in shrinkstack");
-
- oldsize = gp->stack.hi - gp->stack.lo;
- newsize = oldsize / 2;
- if(newsize < FixedStack)
- return; // don't shrink below the minimum-sized stack
- used = gp->stack.hi - gp->sched.sp;
- if(used >= oldsize / 4)
- return; // still using at least 1/4 of the segment.
-
- // We can't copy the stack if we're in a syscall.
- // The syscall might have pointers into the stack.
- if(gp->syscallsp != 0)
- return;
-
-#ifdef GOOS_windows
- if(gp->m != nil && gp->m->libcallsp != 0)
- return;
-#endif
- if(StackDebug > 0)
- runtime·printf("shrinking stack %D->%D\n", (uint64)oldsize, (uint64)newsize);
- // This is being done in a Gscan state and was initiated by the GC so no need to move to
- // the Gcopystate.
- // The world is stopped, so the goroutine must be Gwaiting or Grunnable,
- // and what it is is not changing underfoot.
-
- oldstatus = runtime·readgstatus(gp);
- oldstatus &= ~Gscan;
- if(oldstatus != Gwaiting && oldstatus != Grunnable)
- runtime·throw("status is not Gwaiting or Grunnable");
- runtime·casgstatus(gp, oldstatus, Gcopystack);
- copystack(gp, newsize);
- runtime·casgstatus(gp, Gcopystack, oldstatus);
- }
-
-// Do any delayed stack freeing that was queued up during GC.
-void
-runtime·shrinkfinish(void)
-{
- Stack s, t;
-
- runtime·lock(&runtime·stackpoolmu);
- s = stackfreequeue;
- stackfreequeue = (Stack){0,0};
- runtime·unlock(&runtime·stackpoolmu);
- while(s.lo != 0) {
- t = *(Stack*)s.lo;
- runtime·stackfree(s);
- s = t;
- }
-}
-
-static void badc(void);
-
-#pragma textflag NOSPLIT
-void
-runtime·morestackc(void)
-{
- void (*fn)(void);
-
- fn = badc;
- runtime·onM(&fn);
-}
-
-static void
-badc(void)
-{
- runtime·throw("attempt to execute C code on Go stack");
-}
generated by cgit v1.3 (git 2.53.0) at 2026-04-15 23:03:50 +0000