From f64654dca32977e6be0ef599f112557aa2aa9124 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Wed, 29 Jul 2026 18:57:53 +0800 Subject: [PATCH 01/22] cl: make frontend options request local --- cl/caller_frame_test.go | 12 ++---- cl/compile.go | 69 +++++++++++++++++++++++++------ cl/debug_compile_test.go | 13 ++---- cl/import.go | 4 +- cl/instr.go | 12 +----- internal/build/build.go | 88 ++++++++++++++++++++++++---------------- 6 files changed, 120 insertions(+), 78 deletions(-) diff --git a/cl/caller_frame_test.go b/cl/caller_frame_test.go index 84b835adcc..ad47f5e71d 100644 --- a/cl/caller_frame_test.go +++ b/cl/caller_frame_test.go @@ -499,9 +499,7 @@ func TestRuntimeFrameNameNormalization(t *testing.T) { } func TestCompileRuntimeCallerFrameInstrumentation(t *testing.T) { - old := emitShadowStackInstrumentation - emitShadowStackInstrumentation = true - defer func() { emitShadowStackInstrumentation = old }() + t.Setenv("LLGO_SHADOW_STACK", "1") ssapkg, files := buildCallerFrameSSAPackage(t, "example.com/foo", `package foo import "runtime/debug" @@ -744,9 +742,7 @@ func top() { } func TestCompileRuntimeCallerFrameUsesGoNameForLinkname(t *testing.T) { - old := emitShadowStackInstrumentation - emitShadowStackInstrumentation = true - defer func() { emitShadowStackInstrumentation = old }() + t.Setenv("LLGO_SHADOW_STACK", "1") ssapkg, files := buildCallerFrameSSAPackage(t, "command-line-arguments", `package main import "runtime" @@ -825,9 +821,7 @@ func f() { _ = runtime.FuncForPC(0) } } func TestCompileRuntimeCallerLocationOnlyForRuntimePaths(t *testing.T) { - old := emitShadowStackInstrumentation - emitShadowStackInstrumentation = true - defer func() { emitShadowStackInstrumentation = old }() + t.Setenv("LLGO_SHADOW_STACK", "1") ssapkg, files := buildCallerFrameSSAPackage(t, "example.com/foo", `package foo import "runtime" diff --git a/cl/compile.go b/cl/compile.go index 3c65a41d39..f0f06e64ce 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -65,6 +65,27 @@ var ( enableExportRename bool ) +// Options contains frontend behavior for one package compilation. Drivers that +// may host multiple builds in one process should pass Options explicitly +// instead of changing the legacy package-level Enable* settings. +type Options struct { + Debug bool + DebugSymbols bool + Trace bool + ExportRename bool + ShadowStack bool +} + +func legacyOptions() Options { + return Options{ + Debug: enableDbg, + DebugSymbols: enableDbgSyms, + Trace: enableCallTracing, + ExportRename: enableExportRename, + ShadowStack: os.Getenv("LLGO_SHADOW_STACK") == "1", + } +} + // SetDebug sets debug flags. func SetDebug(dbgFlags dbgFlags) { debugInstr = (dbgFlags & DbgFlagInstruction) != 0 @@ -115,20 +136,27 @@ func dbgGoSSAln(args ...any) { } } +// EnableDebug changes the legacy process-wide default. +// Deprecated: pass Options to NewPackageExWithEmbedMetaOptions. func EnableDebug(b bool) { enableDbg = b } +// EnableDbgSyms changes the legacy process-wide default. +// Deprecated: pass Options to NewPackageExWithEmbedMetaOptions. func EnableDbgSyms(b bool) { enableDbgSyms = b } +// EnableTrace changes the legacy process-wide default. +// Deprecated: pass Options to NewPackageExWithEmbedMetaOptions. func EnableTrace(b bool) { enableCallTracing = b } // EnableExportRename enables or disables //export with different C symbol names. // This is enabled when using -target flag for TinyGo compatibility. +// Deprecated: pass Options to NewPackageExWithEmbedMetaOptions. func EnableExportRename(b bool) { enableExportRename = b } @@ -180,6 +208,8 @@ type context struct { debugAllocVars map[*ssa.Alloc]*types.Var runtimeCallerFuncs map[*ssa.Function]bool pcLineSeq uint64 + options Options + optionsSet bool patches Patches blkInfos []blocks.Info @@ -213,6 +243,13 @@ type context struct { locality localityLowering } +func (p *context) frontendOptions() Options { + if p != nil && p.optionsSet { + return p.options + } + return legacyOptions() +} + func (p *context) rewriteValue(name string) (string, bool) { if p.rewrites == nil { return "", false @@ -601,8 +638,8 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun if f.Recover != nil { // set recover block fn.SetRecover(fn.Block(f.Recover.Index)) } - dbgEnabled := enableDbg - dbgSymsEnabled := enableDbgSyms && (f == nil || f.Origin() == nil) + dbgEnabled := p.frontendOptions().Debug + dbgSymsEnabled := p.frontendOptions().DebugSymbols && (f == nil || f.Origin() == nil) p.inits = append(p.inits, func() { oldFn, oldGoFn, oldMethodNilDerefChecks, oldCallerFrameMark := p.fn, p.goFn, p.methodNilDerefChecks, p.callerFrameMark oldLocalityFunction := p.locality.function @@ -857,11 +894,11 @@ func (p *context) compileBlock(b llssa.Builder, block *ssa.BasicBlock, n int, do if block.Index == 0 && p.shouldTrackCallerFrames() { p.pushCallerLocationFrame(b, block.Parent()) } - if block.Index == 0 && enableCallTracing && !strings.HasPrefix(fn.Name(), "github.com/goplus/llgo/runtime/internal/runtime.Print") { + if block.Index == 0 && p.frontendOptions().Trace && !strings.HasPrefix(fn.Name(), "github.com/goplus/llgo/runtime/internal/runtime.Print") { b.Printf("call " + fn.Name() + "\n\x00") } // place here to avoid wrong current-block - if enableDbgSyms && block.Parent().Origin() == nil && block.Index == 0 { + if p.frontendOptions().DebugSymbols && block.Parent().Origin() == nil && block.Index == 0 { p.debugParams(b, block.Parent()) } @@ -1647,7 +1684,7 @@ func (p *context) compileInstr(b llssa.Builder, instr ssa.Instruction) { if _, ok := p.staticInitInstrs[instr]; ok { return } - if enableDbg && instr.Parent().Origin() == nil { + if p.frontendOptions().Debug && instr.Parent().Origin() == nil { if _, isDebugRef := instr.(*ssa.DebugRef); !isDebugRef { scope := p.getDebugLocScope(instr.Parent(), instr.Pos()) if scope != nil { @@ -1764,7 +1801,7 @@ func (p *context) compileInstr(b llssa.Builder, instr ssa.Instruction) { p.recordPanicLocation(b, v.Pos()) b.Send(ch, x) case *ssa.DebugRef: - if enableDbgSyms && v.Parent().Origin() == nil { + if p.frontendOptions().DebugSymbols && v.Parent().Origin() == nil { p.debugRef(b, v) } default: @@ -1827,7 +1864,7 @@ func (p *context) compileValue(b llssa.Builder, v ssa.Value) llssa.Expr { if isCgoVar(varName) { p.cgoSymbols = append(p.cgoSymbols, val.Name()) } - if enableDbgSyms && p.localityAllowsGlobalDebug(v) { + if p.frontendOptions().DebugSymbols && p.localityAllowsGlobalDebug(v) { pos := p.fset.Position(v.Pos()) b.DIGlobal(val, v.Name(), pos) } @@ -2081,7 +2118,7 @@ func NewPackage(prog llssa.Program, pkg *ssa.Package, files []*ast.File) (ret ll // The rewrites map uses short variable names (without package qualifier) and // only affects string-typed globals defined in the current package. func NewPackageEx(prog llssa.Program, patches Patches, rewrites map[string]string, pkg *ssa.Package, files []*ast.File) (ret llssa.Package, externs []string, err error) { - return newPackageEx(prog, nil, patches, rewrites, pkg, files, nil, false) + return newPackageEx(prog, nil, patches, rewrites, pkg, files, nil, false, legacyOptions()) } // NewPackageExWithEmbed compiles a package using pre-loaded go:embed metadata. @@ -2092,14 +2129,20 @@ func NewPackageEx(prog llssa.Program, patches Patches, rewrites map[string]strin // of one compilation (like patches). nil means one-shot: a fresh // instance is created for this call. func NewPackageExWithEmbed(prog llssa.Program, ct *CallerTracking, patches Patches, rewrites map[string]string, pkg *ssa.Package, files []*ast.File, embedMap goembed.VarMap) (ret llssa.Package, externs []string, err error) { - return newPackageEx(prog, ct, patches, rewrites, pkg, files, &embedMap, false) + return newPackageEx(prog, ct, patches, rewrites, pkg, files, &embedMap, false, legacyOptions()) } func NewPackageExWithEmbedMeta(prog llssa.Program, ct *CallerTracking, patches Patches, rewrites map[string]string, pkg *ssa.Package, files []*ast.File, embedMap goembed.VarMap, metaCollect bool) (ret llssa.Package, externs []string, err error) { - return newPackageEx(prog, ct, patches, rewrites, pkg, files, &embedMap, metaCollect) + return newPackageEx(prog, ct, patches, rewrites, pkg, files, &embedMap, metaCollect, legacyOptions()) +} + +// NewPackageExWithEmbedMetaOptions is NewPackageExWithEmbedMeta with explicit +// per-package frontend options. +func NewPackageExWithEmbedMetaOptions(prog llssa.Program, ct *CallerTracking, patches Patches, rewrites map[string]string, pkg *ssa.Package, files []*ast.File, embedMap goembed.VarMap, metaCollect bool, options Options) (ret llssa.Package, externs []string, err error) { + return newPackageEx(prog, ct, patches, rewrites, pkg, files, &embedMap, metaCollect, options) } -func newPackageEx(prog llssa.Program, ct *CallerTracking, patches Patches, rewrites map[string]string, pkg *ssa.Package, files []*ast.File, embedMap *goembed.VarMap, metaCollect bool) (ret llssa.Package, externs []string, err error) { +func newPackageEx(prog llssa.Program, ct *CallerTracking, patches Patches, rewrites map[string]string, pkg *ssa.Package, files []*ast.File, embedMap *goembed.VarMap, metaCollect bool, options Options) (ret llssa.Package, externs []string, err error) { pkgProg := pkg.Prog pkgTypes := pkg.Pkg oldTypes := pkgTypes @@ -2123,7 +2166,7 @@ func newPackageEx(prog llssa.Program, ct *CallerTracking, patches Patches, rewri prog.SetRuntime(pkgTypes) } ret = prog.NewPackageEx(pkgName, pkgPath, metaCollect) - if enableDbg { + if options.Debug { ret.InitDebug(pkgName, pkgPath, pkgProg.Fset) defer ret.FinalizeDebug() } @@ -2139,6 +2182,8 @@ func newPackageEx(prog llssa.Program, ct *CallerTracking, patches Patches, rewri goTyps: pkgTypes, goPkg: pkg, patches: patches, + options: options, + optionsSet: true, skips: make(map[string]none), vargs: make(map[*ssa.Alloc][]llssa.Expr), funcs: make(map[*ssa.Function]llssa.Function), diff --git a/cl/debug_compile_test.go b/cl/debug_compile_test.go index 2f177005b2..2d5ba3f153 100644 --- a/cl/debug_compile_test.go +++ b/cl/debug_compile_test.go @@ -55,21 +55,16 @@ var anonymous = func(seed int) int { t.Fatal(err) } - oldDebug, oldDebugSyms := enableDbg, enableDbgSyms - EnableDebug(true) - EnableDbgSyms(true) - defer func() { - EnableDebug(oldDebug) - EnableDbgSyms(oldDebugSyms) - }() - prog := newLLSSAProgForTarget(t, &llssa.Target{ GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, OptLevel: optlevel.O0, }) defer prog.Dispose() - pkg, err := NewPackage(prog, ssaPkg, []*ast.File{file}) + pkg, _, err := newPackageEx(prog, nil, nil, nil, ssaPkg, []*ast.File{file}, nil, false, Options{ + Debug: true, + DebugSymbols: true, + }) if err != nil { t.Fatal(err) } diff --git a/cl/import.go b/cl/import.go index a1d284f6c3..85e4cfc9fd 100644 --- a/cl/import.go +++ b/cl/import.go @@ -296,7 +296,7 @@ func (p *context) processLinknameByDoc(doc *ast.CommentGroup, fullName, inPkgNam for n := len(doc.List) - 1; n >= 0; n-- { line := doc.List[n].Text ret := p.initLinkname(line, allowExport, func(name string, isExport bool) (_ string, _, ok bool) { - return fullName, isVar, name == inPkgName || (isExport && enableExportRename) + return fullName, isVar, name == inPkgName || (isExport && p.frontendOptions().ExportRename) }) if ret != unknownDirective { return ret == hasLinkname @@ -370,7 +370,7 @@ func (p *context) initLink(line string, prefix int, export bool, f func(inPkgNam } } else { // Export with different names already processed by initLinknameByDoc - if export && enableExportRename { + if export && p.frontendOptions().ExportRename { return } if export { diff --git a/cl/instr.go b/cl/instr.go index 678c4afc6d..7e091eee82 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -1481,16 +1481,8 @@ func (p *context) runtimeCallerFrameName() string { return "" } -// emitShadowStackInstrumentation gates the legacy shadow-stack calls -// (PushCallerLocationFrame / RecordCallerLocation / RecordPanicLocation). -// The FP-chain unwinder supersedes them: physical pcs resolve through the -// prebuilt ftab and pcline labels, so tracked functions keep only noinline, -// no-tail-call and the label records. The emitters stay for one release as -// an escape hatch (LLGO_SHADOW_STACK=1). -var emitShadowStackInstrumentation = os.Getenv("LLGO_SHADOW_STACK") == "1" - func (p *context) pushCallerLocationFrame(b llssa.Builder, fn *ssa.Function) { - if !emitShadowStackInstrumentation { + if !p.frontendOptions().ShadowStack { return } if fn == nil { @@ -1516,7 +1508,7 @@ func (p *context) recordPanicLocation(b llssa.Builder, pos token.Pos) { } func (p *context) recordRuntimeLocation(b llssa.Builder, pos token.Pos, fn string) { - if !emitShadowStackInstrumentation || !p.shouldTrackCallerFrames() { + if !p.frontendOptions().ShadowStack || !p.shouldTrackCallerFrames() { return } position := p.fset.Position(pos) diff --git a/internal/build/build.go b/internal/build/build.go index a54e7a5ae7..ff6e7686c9 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -359,6 +359,8 @@ const ( loadSyntax = loadTypes | packages.NeedSyntax | packages.NeedTypesInfo ) +var llssaInitOnce sync.Once + func Do(args []string, conf *Config) ([]Package, error) { return Build(Invocation{Args: args, Config: conf}) } @@ -396,11 +398,6 @@ func Build(inv Invocation) ([]Package, error) { if err := validateLinkOptions(conf, &export); err != nil { return nil, err } - // Enable different export names for TinyGo compatibility when using -target - if conf.Target != "" { - cl.EnableExportRename(true) - } - verbose := conf.Verbose patterns := slices.Clone(inv.Args) tags := defaultBuildTags(conf.Goarch, conf.Target) @@ -435,10 +432,16 @@ func Build(inv Invocation) ([]Package, error) { abi.SetRewriteMainPrefix(conf.RewriteMainPrefix) emitDebugInfo := shouldEmitDebugInfo(conf, &export) - cl.EnableDebug(emitDebugInfo) - cl.EnableDbgSyms(emitDebugInfo) - cl.EnableTrace(IsTraceEnabled()) - llssa.Initialize(llssa.InitAll) + frontendOptions := cl.Options{ + Debug: emitDebugInfo, + DebugSymbols: emitDebugInfo, + Trace: IsTraceEnabled(), + ExportRename: conf.Target != "", + ShadowStack: isEnvOn(llgoShadowStack, false), + } + llssaInitOnce.Do(func() { + llssa.Initialize(llssa.InitAll) + }) target := &llssa.Target{ GOOS: conf.Goos, @@ -605,15 +608,16 @@ func Build(inv Invocation) ([]Package, error) { ctx := &context{conf: cfg, progSSA: progSSA, prog: prog, dedup: dedup, patches: patches, callerTracking: cl.NewCallerTracking(), built: make(map[string]none), initial: initial, mode: mode, - fingerprinting: make(map[string]bool), - pkgs: map[*packages.Package]Package{}, - pkgByID: map[string]Package{}, - output: output, - passOpt: passOpt, - buildConf: conf, - crossCompile: export, - commands: commands, - cTransformer: cabi.NewTransformer(prog, export.LLVMTarget, export.TargetABI, conf.AbiMode, cabiOptimize), + fingerprinting: make(map[string]bool), + pkgs: map[*packages.Package]Package{}, + pkgByID: map[string]Package{}, + output: output, + passOpt: passOpt, + buildConf: conf, + crossCompile: export, + commands: commands, + frontendOptions: frontendOptions, + cTransformer: cabi.NewTransformer(prog, export.LLVMTarget, export.TargetABI, conf.AbiMode, cabiOptimize), } defer ctx.closePackageMetas() @@ -868,9 +872,10 @@ type context struct { output bool passOpt bool - buildConf *Config - crossCompile crosscompile.Export - commands commandEnv + buildConf *Config + crossCompile crosscompile.Export + commands commandEnv + frontendOptions cl.Options cTransformer *cabi.Transformer @@ -893,6 +898,11 @@ type context struct { pclnExternal *pclnmap.Data } +// frontendDebugMu protects the legacy cl/ssa instruction-debug switches. +// Code-generation options are invocation-local; this process-wide lock remains +// only for verbose diagnostic logging until those helpers accept a logger. +var frontendDebugMu sync.RWMutex + // closePackageMetas releases metadata mappings owned by this build. Metadata // remains available to hooks and whole-program consumers until Do returns. func (c *context) closePackageMetas() { @@ -1754,22 +1764,27 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { syntax = append(syntax, altPkg.Syntax...) } showDetail := verbose && pkgExists(ctx.initial, pkg) - if showDetail { - llssa.SetDebug(llssa.DbgFlagAll) - cl.SetDebug(cl.DbgFlagAll) - defer func() { - llssa.SetDebug(0) - cl.SetDebug(0) - }() - } - - embedMap, err := goembed.LoadDirectives(ctx.conf.Fset, syntax) - if err != nil { - return fmt.Errorf("load go:embed directives for %s failed: %w", pkgPath, err) - } - needMeta := !aPkg.CacheHit && ctx.buildConf.packageMetaEnabled() - ret, externs, err := cl.NewPackageExWithEmbedMeta(ctx.prog, ctx.callerTracking, ctx.patches, aPkg.rewriteVars, aPkg.SSA, syntax, embedMap, needMeta) + ret, externs, err := func() (llssa.Package, []string, error) { + if showDetail { + frontendDebugMu.Lock() + defer frontendDebugMu.Unlock() + llssa.SetDebug(llssa.DbgFlagAll) + cl.SetDebug(cl.DbgFlagAll) + defer func() { + llssa.SetDebug(0) + cl.SetDebug(0) + }() + } else { + frontendDebugMu.RLock() + defer frontendDebugMu.RUnlock() + } + embedMap, err := goembed.LoadDirectives(ctx.conf.Fset, syntax) + if err != nil { + return nil, nil, fmt.Errorf("load go:embed directives for %s failed: %w", pkgPath, err) + } + return cl.NewPackageExWithEmbedMetaOptions(ctx.prog, ctx.callerTracking, ctx.patches, aPkg.rewriteVars, aPkg.SSA, syntax, embedMap, needMeta, ctx.frontendOptions) + }() check(err) aPkg.LPkg = ret @@ -2369,6 +2384,7 @@ const llgoWasiThreads = "LLGO_WASI_THREADS" const llgoStdioNobuf = "LLGO_STDIO_NOBUF" const llgoFullRpath = "LLGO_FULL_RPATH" const llgoBuildCache = "LLGO_BUILD_CACHE" +const llgoShadowStack = "LLGO_SHADOW_STACK" // for Plan9 asm translation debug const llgoPlan9ASMPkgs = "LLGO_PLAN9ASM_PKGS" From b6fc1811c96c3e6b5ff16add2b5b3987c9f66f7d Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Wed, 29 Jul 2026 18:59:05 +0800 Subject: [PATCH 02/22] ssa: introduce compilation symbol namers --- ssa/abi/abi.go | 86 +++++++++++++++++++++++++++++++-------------- ssa/abi/abi_test.go | 6 ++++ 2 files changed, 66 insertions(+), 26 deletions(-) diff --git a/ssa/abi/abi.go b/ssa/abi/abi.go index f178cecb54..ab08b665c3 100644 --- a/ssa/abi/abi.go +++ b/ssa/abi/abi.go @@ -201,33 +201,47 @@ func (b *Builder) TypeName(t types.Type) (ret string, pub bool) { return } -func NamedName(t *types.Named) string { +// Namer owns the package-path policy used to construct symbols for one +// compilation. +type Namer struct { + RewriteMainPrefix bool +} + +func (n Namer) NamedName(t *types.Named) string { if targs := t.TypeArgs(); targs != nil { - n := targs.Len() - infos := make([]string, n) - for i := 0; i < n; i++ { - infos[i] = typeArgString(targs.At(i)) + count := targs.Len() + infos := make([]string, count) + for i := 0; i < count; i++ { + infos[i] = n.typeArgString(targs.At(i)) } return t.Obj().Name() + "[" + strings.Join(infos, ",") + "]" } return t.Obj().Name() } -func TypeArgs(typeArgs []types.Type) string { +func NamedName(t *types.Named) string { + return processNamer().NamedName(t) +} + +func (n Namer) TypeArgs(typeArgs []types.Type) string { targs := make([]string, len(typeArgs)) for i, t := range typeArgs { - targs[i] = typeArgString(t) + targs[i] = n.typeArgString(t) } return "[" + strings.Join(targs, ",") + "]" } -func namedLikeTypeArgString(obj types.Object, targs *types.TypeList) string { +func TypeArgs(typeArgs []types.Type) string { + return processNamer().TypeArgs(typeArgs) +} + +func (n Namer) namedLikeTypeArgString(obj types.Object, targs *types.TypeList) string { name := obj.Name() if targs != nil { - n := targs.Len() - infos := make([]string, n) - for i := 0; i < n; i++ { - infos[i] = typeArgString(targs.At(i)) + count := targs.Len() + infos := make([]string, count) + for i := 0; i < count; i++ { + infos[i] = n.typeArgString(targs.At(i)) } name += "[" + strings.Join(infos, ",") + "]" } @@ -235,31 +249,35 @@ func namedLikeTypeArgString(obj types.Object, targs *types.TypeList) string { // tests). Disambiguate them in symbol names using stable scope indices. name += scopeIndices(obj) if pkg := obj.Pkg(); pkg != nil { - return PathOf(pkg) + "." + name + return n.PathOf(pkg) + "." + name } return name } -func typeArgString(t types.Type) string { +func namedLikeTypeArgString(obj types.Object, targs *types.TypeList) string { + return processNamer().namedLikeTypeArgString(obj, targs) +} + +func (n Namer) typeArgString(t types.Type) string { switch t := t.(type) { case *types.Alias: - return typeArgString(types.Unalias(t)) + return n.typeArgString(types.Unalias(t)) case *types.Basic: return t.String() case *types.Named: - return namedLikeTypeArgString(t.Obj(), t.TypeArgs()) + return n.namedLikeTypeArgString(t.Obj(), t.TypeArgs()) case *types.Pointer: - return "*" + typeArgString(t.Elem()) + return "*" + n.typeArgString(t.Elem()) case *types.Slice: - return "[]" + typeArgString(t.Elem()) + return "[]" + n.typeArgString(t.Elem()) case *types.Array: - return fmt.Sprintf("[%v]%s", t.Len(), typeArgString(t.Elem())) + return fmt.Sprintf("[%v]%s", t.Len(), n.typeArgString(t.Elem())) case *types.Map: - return fmt.Sprintf("map[%s]%s", typeArgString(t.Key()), typeArgString(t.Elem())) + return fmt.Sprintf("map[%s]%s", n.typeArgString(t.Key()), n.typeArgString(t.Elem())) case *types.Chan: _, s := ChanDir(t.Dir()) elem := t.Elem() - elemStr := typeArgString(elem) + elemStr := n.typeArgString(elem) // Keep canonical channel formatting for nested directional channels. // Example: chan (<-chan int), not "chan <-chan int" (ambiguous). if t.Dir() == types.SendRecv { @@ -271,10 +289,14 @@ func typeArgString(t types.Type) string { default: // Fallback for rare type arguments (e.g. signature/interface/struct). // Collisions are mainly caused by local named types, handled above. - return types.TypeString(t, PathOf) + return types.TypeString(t, n.PathOf) } } +func typeArgString(t types.Type) string { + return processNamer().typeArgString(t) +} + const ( PatchPathPrefix = env.LLGoRuntimePkg + "/internal/lib/" ) @@ -288,23 +310,35 @@ func SetRewriteMainPrefix(b bool) { var rewriteMainPrefix bool +func processNamer() Namer { + return Namer{RewriteMainPrefix: rewriteMainPrefix} +} + // PathOf returns the package path of the specified package. -func PathOf(pkg *types.Package) string { +func (n Namer) PathOf(pkg *types.Package) string { if pkg == nil { return "" } - if rewriteMainPrefix && pkg.Name() == "main" { + if n.RewriteMainPrefix && pkg.Name() == "main" { return "main" } return strings.TrimPrefix(pkg.Path(), PatchPathPrefix) } +func PathOf(pkg *types.Package) string { + return processNamer().PathOf(pkg) +} + // FullName returns the full name of a package member. -func FullName(pkg *types.Package, name string) string { +func (n Namer) FullName(pkg *types.Package, name string) string { if pkg == nil { return name } - return PathOf(pkg) + "." + name + return n.PathOf(pkg) + "." + name +} + +func FullName(pkg *types.Package, name string) string { + return processNamer().FullName(pkg, name) } // BasicName returns the ABI type name for the specified basic type. diff --git a/ssa/abi/abi_test.go b/ssa/abi/abi_test.go index bc0a0b93ea..56d97a20d8 100644 --- a/ssa/abi/abi_test.go +++ b/ssa/abi/abi_test.go @@ -417,5 +417,11 @@ func TestRewriteMainPrefix(t *testing.T) { if path := abi.PathOf(pkg); path != "main" { t.Fatalf("error %v", path) } + if path := (abi.Namer{}).PathOf(pkg); path != "example.com/foo/pkg" { + t.Fatalf("request namer inherited process setting: %v", path) + } + if path := (abi.Namer{RewriteMainPrefix: true}).PathOf(pkg); path != "main" { + t.Fatalf("request namer did not rewrite main path: %v", path) + } abi.SetRewriteMainPrefix(false) } From 7b3736a7f2b7f478a4f920f921f0d011c3ae3438 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Wed, 29 Jul 2026 19:01:58 +0800 Subject: [PATCH 03/22] ssa: make main prefix rewriting program local --- cl/compile.go | 14 +++++----- cl/import.go | 27 +++++++++++++------ cl/instr.go | 2 +- cl/locality_lower.go | 4 +-- internal/build/build.go | 11 ++++---- internal/build/build_test.go | 52 ++++++++++++++++++++++++++++++++++++ ssa/abi/abi.go | 28 ++++++------------- ssa/abi/abi_test.go | 12 +++------ ssa/abi/type.go | 12 ++++++--- ssa/abitype.go | 20 +++++++------- ssa/decl.go | 2 +- ssa/globaldce.go | 14 +++++++--- ssa/interface.go | 4 +-- ssa/package.go | 5 ++-- ssa/ssa_test.go | 33 +++++++++++++++++++++++ ssa/target.go | 11 ++++---- ssa/type.go | 43 ++++++++++++++++++++++++----- 17 files changed, 209 insertions(+), 85 deletions(-) diff --git a/cl/compile.go b/cl/compile.go index f0f06e64ce..84ac759888 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -391,7 +391,7 @@ func (p *context) compileType(pkg llssa.Package, t *ssa.Type) { } tnName := tn.Name() typ := tn.Type() - name := llssa.FullName(tn.Pkg(), tnName) + name := p.prog.FullName(tn.Pkg(), tnName) dbgInstrln("==> NewType", name, typ) p.compileMethods(pkg, typ) p.compileMethods(pkg, types.NewPointer(typ)) @@ -612,7 +612,7 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun if p.prog.FuncInfoMetadataEnabled() { goName := fn.Name() if pkgTypes != nil { - goName = funcName(pkgTypes, f, false) + goName = funcNameWithProgram(p.prog, pkgTypes, f, false) } pos := p.funcInfoPosition(f) pkg.EmitFuncInfo(fn.Name(), funcInfoDisplayName(pkgTypes, goName), pos.Filename, pos.Line, pos.Column) @@ -2146,7 +2146,7 @@ func newPackageEx(prog llssa.Program, ct *CallerTracking, patches Patches, rewri pkgProg := pkg.Prog pkgTypes := pkg.Pkg oldTypes := pkgTypes - pkgName, pkgPath := pkgTypes.Name(), llssa.PathOf(pkgTypes) + pkgName, pkgPath := pkgTypes.Name(), prog.PathOf(pkgTypes) patch, hasPatch := patches[pkgPath] if hasPatch { pkgTypes = patch.Types @@ -2479,7 +2479,7 @@ func (p *context) typeArgName(t types.Type) string { case *types.Named: name := p.localNamedName(t, p.isLocalType(t.Obj())) if pkg := t.Obj().Pkg(); pkg != nil { - return reflectTypeArgPkgPath(pkg) + "." + name + return p.reflectTypeArgPkgPath(pkg) + "." + name } return name case *types.Pointer: @@ -2500,7 +2500,7 @@ func (p *context) typeArgName(t types.Type) string { } return fmt.Sprintf("%s %s", s, elem) default: - return types.TypeString(t, reflectTypeArgPkgPath) + return types.TypeString(t, p.reflectTypeArgPkgPath) } } @@ -2517,14 +2517,14 @@ func chanDirName(dir types.ChanDir) string { } } -func reflectTypeArgPkgPath(pkg *types.Package) string { +func (p *context) reflectTypeArgPkgPath(pkg *types.Package) string { if pkg == nil { return "" } if pkg.Path() == "command-line-arguments" && pkg.Name() != "" { return pkg.Name() } - return llssa.PathOf(pkg) + return p.prog.PathOf(pkg) } func (p *context) isGenericLocalType(obj types.Object) bool { diff --git a/cl/import.go b/cl/import.go index 85e4cfc9fd..816f21de5b 100644 --- a/cl/import.go +++ b/cl/import.go @@ -131,7 +131,7 @@ func pkgKindByScope(scope *types.Scope) (int, string) { } func (p *context) importPkg(pkg *types.Package, i *pkgInfo) { - pkgPath := llssa.PathOf(pkg) + pkgPath := p.prog.PathOf(pkg) scope := pkg.Scope() kind, _ := pkgKindByScope(scope) if kind == PkgNormal { @@ -446,6 +446,10 @@ func typesFuncName(pkgPath string, fn *types.Func) (fullName, inPkgName string) // - func: pkg.name // - method: pkg.(T).name, pkg.(*T).name func funcName(pkg *types.Package, fn *ssa.Function, org bool) string { + return funcNameWithProgram(nil, pkg, fn, org) +} + +func funcNameWithProgram(prog llssa.Program, pkg *types.Package, fn *ssa.Function, org bool) string { // Closures in methods can be nested (closure inside closure inside method). // Walking only one Parent() loses the receiver for deeper nests, producing // names like "pkg.marshal$1$1" that can collide across receiver types. @@ -472,11 +476,18 @@ func funcName(pkg *types.Package, fn *ssa.Function, org bool) string { if org := fn.Origin(); org != nil { fnName = org.Name() if fn.Signature.Recv() == nil { - fnName += llssa.TypeArgs(fn.TypeArgs()) + if prog != nil { + fnName += prog.TypeArgs(fn.TypeArgs()) + } else { + fnName += llssa.TypeArgs(fn.TypeArgs()) + } } } else { fnName = fn.Name() } + if prog != nil { + return prog.FuncName(pkg, fnName, recv, org) + } return llssa.FuncName(pkg, fnName, recv, org) } @@ -621,7 +632,7 @@ func (p *context) funcName(fn *ssa.Function) (*types.Package, string, int) { if origin := fn.Origin(); origin != nil { pkg = origin.Pkg.Pkg p.ensureLoaded(pkg) - orgName = funcName(pkg, origin, true) + orgName = funcNameWithProgram(p.prog, pkg, origin, true) } else { fname := fn.Name() if checkCgo(fname) && !cgoIgnored(fname) { @@ -647,7 +658,7 @@ func (p *context) funcName(fn *ssa.Function) (*types.Package, string, int) { pkg = p.goTyps } p.ensureLoaded(pkg) - orgName = funcName(pkg, fn, false) + orgName = funcNameWithProgram(p.prog, pkg, fn, false) } if v, ok := p.prog.Linkname(orgName); ok { if strings.HasPrefix(v, "C.") { @@ -672,7 +683,7 @@ func (p *context) funcName(fn *ssa.Function) (*types.Package, string, int) { if instr, ok := syncAtomicIntrinsicMap[orgName]; ok { return nil, instr, llgoInstr } - return pkg, funcName(pkg, fn, false), goFunc + return pkg, funcNameWithProgram(p.prog, pkg, fn, false), goFunc } const ( @@ -683,12 +694,12 @@ const ( ) func (p *context) varName(pkg *types.Package, v *ssa.Global) (vName string, vtype int, define bool) { - name := llssa.FullName(pkg, v.Name()) + name := p.prog.FullName(pkg, v.Name()) // TODO(lijie): need a bettery way to process linkname (maybe alias) if !isCgoCfpvar(v.Name()) && !isCgoVar(v.Name()) { if v, ok := p.prog.Linkname(name); ok { if strings.HasPrefix(v, "go:") { - if llssa.PathOf(pkg) == "runtime" { + if p.prog.PathOf(pkg) == "runtime" { return v, goVar, true } return v, goVar, false @@ -761,7 +772,7 @@ func ParsePkgSyntax(prog llssa.Program, fset *token.FileSet, pkg *types.Package, return nil } ctx := &context{prog: prog} - pkgPath := llssa.PathOf(pkg) + pkgPath := prog.PathOf(pkg) for _, file := range files { for _, decl := range file.Decls { switch decl := decl.(type) { diff --git a/cl/instr.go b/cl/instr.go index 7e091eee82..fa139f2126 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -1473,7 +1473,7 @@ func (p *context) runtimeCallerFrameName() string { return "" } if p.goFn != nil && p.goFn.Pkg != nil && p.goFn.Pkg.Pkg != nil { - return runtimeFrameName(funcName(p.goFn.Pkg.Pkg, p.goFn, false)) + return runtimeFrameName(funcNameWithProgram(p.prog, p.goFn.Pkg.Pkg, p.goFn, false)) } if p.fn != nil { return runtimeFrameName(p.fn.Name()) diff --git a/cl/locality_lower.go b/cl/locality_lower.go index 7ca857f770..5bf4183c26 100644 --- a/cl/locality_lower.go +++ b/cl/locality_lower.go @@ -88,7 +88,7 @@ func (p *context) prepareExportedLocalContext(f *ssa.Function) { if !p.prog.NeedsLocalContext() || f == nil || f.Pkg == nil { return } - fullName := funcName(f.Pkg.Pkg, f, false) + fullName := funcNameWithProgram(p.prog, f.Pkg.Pkg, f, false) if _, exported := p.pkg.ExportFuncs()[fullName]; !exported { return } @@ -185,7 +185,7 @@ func (p *context) localTypesPackage(fullName string) *types.Package { if pkg == nil { return false } - prefix := llssa.PathOf(pkg) + "." + prefix := p.prog.PathOf(pkg) + "." if !strings.HasPrefix(fullName, prefix) { return false } diff --git a/internal/build/build.go b/internal/build/build.go index ff6e7686c9..978e9ae417 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -429,8 +429,6 @@ func Build(inv Invocation) ([]Package, error) { if conf.Mode == ModeTest { cfg.Mode |= packages.NeedForTest } - abi.SetRewriteMainPrefix(conf.RewriteMainPrefix) - emitDebugInfo := shouldEmitDebugInfo(conf, &export) frontendOptions := cl.Options{ Debug: emitDebugInfo, @@ -444,10 +442,11 @@ func Build(inv Invocation) ([]Package, error) { }) target := &llssa.Target{ - GOOS: conf.Goos, - GOARCH: conf.Goarch, - Target: conf.Target, - OptLevel: conf.OptLevel, + GOOS: conf.Goos, + GOARCH: conf.Goarch, + Target: conf.Target, + OptLevel: conf.OptLevel, + RewriteMainPrefix: conf.RewriteMainPrefix, } prog := llssa.NewProgram(target) diff --git a/internal/build/build_test.go b/internal/build/build_test.go index 0985167064..e92e41f9bf 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -168,6 +168,58 @@ func TestInvocationUsesExplicitWorkingDirectory(t *testing.T) { pkgs[0].LPkg.Prog.Dispose() } +func TestConcurrentInvocationsIsolateRewriteMainPrefix(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module example.com/rewrite\n\ngo 1.24\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "main.go"), []byte("package main\n\nfunc F() {}\n"), 0o644); err != nil { + t.Fatal(err) + } + t.Setenv(llgoBuildCache, "0") + + type result struct { + rewrite bool + pkgs []Package + err error + } + results := make(chan result, 2) + for _, rewrite := range []bool{false, true} { + conf := NewDefaultConf(ModeGen) + conf.RewriteMainPrefix = rewrite + go func() { + pkgs, err := Build(Invocation{ + Args: []string{"."}, + Config: conf, + Dir: dir, + }) + results <- result{rewrite: rewrite, pkgs: pkgs, err: err} + }() + } + var built []result + for range 2 { + got := <-results + if got.err != nil { + t.Fatal(got.err) + } + if len(got.pkgs) != 1 || got.pkgs[0].LPkg == nil { + t.Fatalf("Build returned packages = %+v, want one compiled package", got.pkgs) + } + t.Cleanup(got.pkgs[0].LPkg.Prog.Dispose) + built = append(built, got) + } + for _, got := range built { + ir := got.pkgs[0].LPkg.String() + want, notWant := "example.com/rewrite.F", "main.F" + if got.rewrite { + want, notWant = notWant, want + } + if !strings.Contains(ir, want) || strings.Contains(ir, notWant) { + t.Fatalf("RewriteMainPrefix=%v produced unexpected symbols; want %q and not %q:\n%s", got.rewrite, want, notWant, ir) + } + } +} + func TestResolveOutputsUsesInvocationDirectory(t *testing.T) { dir := t.TempDir() out := &OutFmtDetails{ diff --git a/ssa/abi/abi.go b/ssa/abi/abi.go index ab08b665c3..69a4c9c970 100644 --- a/ssa/abi/abi.go +++ b/ssa/abi/abi.go @@ -135,6 +135,7 @@ type Builder struct { buf []byte PtrSize uintptr Sizes types.Sizes + Namer Namer } // New creates a new ABI type Builder. @@ -172,7 +173,7 @@ func (b *Builder) TypeName(t types.Type) (ret string, pub bool) { o := t.Obj() pkg := o.Pkg() ids := scopeIndices(o) - return "_llgo_" + FullName(pkg, NamedName(t)+ids), (pkg == nil || o.Exported() && ids == "") + return "_llgo_" + b.Namer.FullName(pkg, b.Namer.NamedName(t)+ids), (pkg == nil || o.Exported() && ids == "") case *types.Interface: if t.Empty() { return "_llgo_any", true @@ -220,7 +221,7 @@ func (n Namer) NamedName(t *types.Named) string { } func NamedName(t *types.Named) string { - return processNamer().NamedName(t) + return (Namer{}).NamedName(t) } func (n Namer) TypeArgs(typeArgs []types.Type) string { @@ -232,7 +233,7 @@ func (n Namer) TypeArgs(typeArgs []types.Type) string { } func TypeArgs(typeArgs []types.Type) string { - return processNamer().TypeArgs(typeArgs) + return (Namer{}).TypeArgs(typeArgs) } func (n Namer) namedLikeTypeArgString(obj types.Object, targs *types.TypeList) string { @@ -255,7 +256,7 @@ func (n Namer) namedLikeTypeArgString(obj types.Object, targs *types.TypeList) s } func namedLikeTypeArgString(obj types.Object, targs *types.TypeList) string { - return processNamer().namedLikeTypeArgString(obj, targs) + return (Namer{}).namedLikeTypeArgString(obj, targs) } func (n Namer) typeArgString(t types.Type) string { @@ -294,26 +295,13 @@ func (n Namer) typeArgString(t types.Type) string { } func typeArgString(t types.Type) string { - return processNamer().typeArgString(t) + return (Namer{}).typeArgString(t) } const ( PatchPathPrefix = env.LLGoRuntimePkg + "/internal/lib/" ) -// SetRewriteMainPrefix controls whether symbols in the main package -// use "main." as their package path prefix instead of the actual -// import path. When true, pkgpath.sym is rewritten to main.sym. -func SetRewriteMainPrefix(b bool) { - rewriteMainPrefix = b -} - -var rewriteMainPrefix bool - -func processNamer() Namer { - return Namer{RewriteMainPrefix: rewriteMainPrefix} -} - // PathOf returns the package path of the specified package. func (n Namer) PathOf(pkg *types.Package) string { if pkg == nil { @@ -326,7 +314,7 @@ func (n Namer) PathOf(pkg *types.Package) string { } func PathOf(pkg *types.Package) string { - return processNamer().PathOf(pkg) + return (Namer{}).PathOf(pkg) } // FullName returns the full name of a package member. @@ -338,7 +326,7 @@ func (n Namer) FullName(pkg *types.Package, name string) string { } func FullName(pkg *types.Package, name string) string { - return processNamer().FullName(pkg, name) + return (Namer{}).FullName(pkg, name) } // BasicName returns the ABI type name for the specified basic type. diff --git a/ssa/abi/abi_test.go b/ssa/abi/abi_test.go index 56d97a20d8..0ba6035fd1 100644 --- a/ssa/abi/abi_test.go +++ b/ssa/abi/abi_test.go @@ -413,15 +413,11 @@ func TestRewriteMainPrefix(t *testing.T) { if path := abi.PathOf(pkg); path != "example.com/foo/pkg" { t.Fatalf("error %v", path) } - abi.SetRewriteMainPrefix(true) - if path := abi.PathOf(pkg); path != "main" { + namer := abi.Namer{RewriteMainPrefix: true} + if path := namer.PathOf(pkg); path != "main" { t.Fatalf("error %v", path) } - if path := (abi.Namer{}).PathOf(pkg); path != "example.com/foo/pkg" { - t.Fatalf("request namer inherited process setting: %v", path) - } - if path := (abi.Namer{RewriteMainPrefix: true}).PathOf(pkg); path != "main" { - t.Fatalf("request namer did not rewrite main path: %v", path) + if path := abi.PathOf(pkg); path != "example.com/foo/pkg" { + t.Fatalf("default namer changed to %v", path) } - abi.SetRewriteMainPrefix(false) } diff --git a/ssa/abi/type.go b/ssa/abi/type.go index d3b072b82b..100c7fa590 100644 --- a/ssa/abi/type.go +++ b/ssa/abi/type.go @@ -120,7 +120,7 @@ func (b *Builder) reflectTypeArgBaseString(t types.Type) string { case *types.Named: name := b.namedStr(t) if pkg := t.Obj().Pkg(); pkg != nil { - return reflectTypeArgPkgPath(pkg) + "." + name + return b.reflectTypeArgPkgPath(pkg) + "." + name } return name case *types.Interface: @@ -141,17 +141,21 @@ func (b *Builder) reflectTypeArgBaseString(t types.Type) string { _, s := ChanDir(t.Dir()) return s + " " + b.reflectTypeArgString(t.Elem()) } - return types.TypeString(t, reflectTypeArgPkgPath) + return types.TypeString(t, b.reflectTypeArgPkgPath) } -func reflectTypeArgPkgPath(pkg *types.Package) string { +func (b *Builder) reflectTypeArgPkgPath(pkg *types.Package) string { if pkg == nil { return "" } if pkg.Path() == "command-line-arguments" && pkg.Name() != "" { return pkg.Name() } - return PathOf(pkg) + return b.Namer.PathOf(pkg) +} + +func reflectTypeArgPkgPath(pkg *types.Package) string { + return (&Builder{}).reflectTypeArgPkgPath(pkg) } func (b *Builder) structStr(t *types.Struct) string { diff --git a/ssa/abitype.go b/ssa/abitype.go index 8d5e728c59..9b1c85bfca 100644 --- a/ssa/abitype.go +++ b/ssa/abitype.go @@ -214,7 +214,7 @@ func (b Builder) abiInterfaceImethods(t *types.Interface, name string) llvm.Valu var values []llvm.Value name := f.Name() if !token.IsExported(name) { - name = abi.FullName(f.Pkg(), name) + name = prog.FullName(f.Pkg(), name) } values = append(values, b.Str(name).impl) ftyp := funcType(prog, f.Type()) @@ -320,7 +320,7 @@ func (b Builder) abiExtendedFields(t types.Type, name string, global llvm.Value) for i := 0; i < n; i++ { if f := t.Field(i); !f.Exported() { if pkg := f.Pkg(); pkg != nil { - pkgPath = reflectPkgPath(pkg) + pkgPath = prog.reflectPkgPath(pkg) break } } @@ -405,19 +405,19 @@ retry: goto retry case *types.Named: pkg := typ.Obj().Pkg() - return pkg, reflectPkgPath(pkg) + return pkg, b.Prog.reflectPkgPath(pkg) } return nil, b.Pkg.Path() } -func reflectPkgPath(pkg *types.Package) string { +func (p Program) reflectPkgPath(pkg *types.Package) string { if pkg == nil { return "" } if pkg.Path() == "command-line-arguments" && pkg.Name() != "" { return pkg.Name() } - return abi.PathOf(pkg) + return p.PathOf(pkg) } func (b Builder) abiUncommonMethodSet(t types.Type) (mset *types.MethodSet, ok bool) { @@ -498,7 +498,7 @@ func (b Builder) abiUncommonMethods(t types.Type, methods []*types.Selection) ll m := methods[i] obj := m.Obj() mName := obj.Name() - fullName := abiMethodName(obj) + fullName := prog.abiMethodName(obj) name := b.Str(fullName).impl mSig := m.Type().(*types.Signature) var tfn, ifn llvm.Value @@ -546,12 +546,12 @@ func funcType(prog Program, typ types.Type) types.Type { return ftyp.raw.Type.(*types.Struct).Field(0).Type() } -func abiMethodName(obj types.Object) string { +func (p Program) abiMethodName(obj types.Object) string { name := obj.Name() if token.IsExported(name) { return name } - return abi.FullName(obj.Pkg(), name) + return p.FullName(obj.Pkg(), name) } func methodExprSignature(sig *types.Signature) *types.Signature { @@ -572,9 +572,9 @@ func methodExprSignature(sig *types.Signature) *types.Signature { func (b Builder) abiMethodFunc(anonymous bool, mPkg *types.Package, mName string, mSig *types.Signature) Function { var fullName string if anonymous { - fullName = b.Pkg.Path() + "." + types.TypeString(mSig.Recv().Type(), abi.PathOf) + "." + mName + fullName = b.Pkg.Path() + "." + types.TypeString(mSig.Recv().Type(), b.Prog.PathOf) + "." + mName } else { - fullName = FuncName(mPkg, mName, mSig.Recv(), false) + fullName = b.Prog.FuncName(mPkg, mName, mSig.Recv(), false) } if b.Pkg.fnlink != nil { fullName = b.Pkg.fnlink(fullName) diff --git a/ssa/decl.go b/ssa/decl.go index a575dd1bee..d4a7bb383c 100644 --- a/ssa/decl.go +++ b/ssa/decl.go @@ -142,7 +142,7 @@ func (p Package) doNewVarEx(name string, t Type, threadLocal bool) Global { } if rt != nil { // Do not redirect the runtime's own zero-sized allocation sentinel. - zeroName := FullName(rt, runtimeZeroSizedAllocSymbol) + zeroName := p.Prog.FullName(rt, runtimeZeroSizedAllocSymbol) if name != zeroName && p.ownsGlobal(name) { zero := p.moduleZeroSizedAlloc(p.Prog.Elem(t)) alias := llvm.AddAlias(p.mod, typ, 0, zero.impl, name) diff --git a/ssa/globaldce.go b/ssa/globaldce.go index cb83c2a2bc..a0ea9f66c7 100644 --- a/ssa/globaldce.go +++ b/ssa/globaldce.go @@ -30,12 +30,16 @@ type ReflectMethodCheck struct { } func methodCapabilitySig(sig *types.Signature) string { + return methodCapabilitySigWithPath(sig, PathOf) +} + +func methodCapabilitySigWithPath(sig *types.Signature, pathOf func(*types.Package) string) string { canon := types.NewSignatureType(nil, nil, nil, methodCapabilityTuple(sig.Params()), methodCapabilityTuple(sig.Results()), sig.Variadic()) return types.TypeString(canon, func(pkg *types.Package) string { if pkg == nil { return "" } - return PathOf(pkg) + return pathOf(pkg) }) } @@ -113,7 +117,11 @@ func methodCapabilityType(t types.Type) types.Type { } func methodCapabilityKey(method *types.Func) string { - return "go.method." + methodCapabilityName(method) + ":" + methodCapabilitySig(method.Type().(*types.Signature)) + return methodCapabilityKeyWithPath(method, PathOf) +} + +func methodCapabilityKeyWithPath(method *types.Func, pathOf func(*types.Package) string) string { + return "go.method." + methodCapabilityName(method) + ":" + methodCapabilitySigWithPath(method.Type().(*types.Signature), pathOf) } func methodCapabilityName(method *types.Func) string { @@ -289,7 +297,7 @@ func (p Program) addMethodTypeMetadata(global llvm.Value, fullType Type, methods methodStride := p.SizeOf(methodType) for i, sel := range methods { baseOffset := methodArrayOffset + uint64(i)*methodStride - p.addTypeMetadata(global, baseOffset+ifnOffset, methodCapabilityKey(sel.Obj().(*types.Func))) + p.addTypeMetadata(global, baseOffset+ifnOffset, methodCapabilityKeyWithPath(sel.Obj().(*types.Func), p.PathOf)) if sel.Obj().Exported() { name := sel.Obj().Name() p.addTypeMetadata(global, baseOffset+ifnOffset, reflectValueMethodTypeID) diff --git a/ssa/interface.go b/ssa/interface.go index 253df5c64b..f9aee874c6 100644 --- a/ssa/interface.go +++ b/ssa/interface.go @@ -91,7 +91,7 @@ func (b Builder) Imethod(intf Expr, method *types.Func) Expr { if prog.enableGoGlobalDCE { fnType := prog.Elem(pfn.Type) fn = Expr{ - prog.methodCheckedLoad(b.impl, pfn.impl, methodCapabilityKey(method)), + prog.methodCheckedLoad(b.impl, pfn.impl, methodCapabilityKeyWithPath(method, prog.PathOf)), fnType, } } else { @@ -212,7 +212,7 @@ func (b Builder) recordInterfaceInfo(t *types.Interface, typeName string) { for i := 0; i < t.NumMethods(); i++ { f := t.Method(i) ftypName, _ := prog.abi.TypeName(funcType(prog, f.Type())) - mb.AddIfaceMethod(intfSym, abiMethodName(f), mb.Sym(ftypName)) + mb.AddIfaceMethod(intfSym, prog.abiMethodName(f), mb.Sym(ftypName)) } } diff --git a/ssa/package.go b/ssa/package.go index db41bae4ab..7abed72b69 100644 --- a/ssa/package.go +++ b/ssa/package.go @@ -320,6 +320,7 @@ func NewProgram(target *Target) Program { debugInfoOptimized: target.effectiveOptLevel() != optlevel.O0, } prog.abi.Init(uintptr(prog.ptrSize), (*goProgram)(unsafe.Pointer(prog))) + prog.abi.Namer.RewriteMainPrefix = target.RewriteMainPrefix return prog } @@ -393,7 +394,7 @@ func (p Program) isNoInterfaceMethod(fn *types.Func) bool { if !ok || sig.Recv() == nil { return false } - _, ok = p.noInterface[FuncName(fn.Pkg(), fn.Name(), sig.Recv(), true)] + _, ok = p.noInterface[p.FuncName(fn.Pkg(), fn.Name(), sig.Recv(), true)] return ok } @@ -892,7 +893,7 @@ func (p Package) MaterializePreserveSyms() { func (p Package) rtFunc(fnName string) Expr { p.NeedRuntime = true fn := p.Prog.runtime().Scope().Lookup(fnName).(*types.Func) - name := FullName(fn.Pkg(), fnName) + name := p.Prog.FullName(fn.Pkg(), fnName) if p.fnlink != nil { name = p.fnlink(name) } diff --git a/ssa/ssa_test.go b/ssa/ssa_test.go index 38085a627b..b0f6cdbdac 100644 --- a/ssa/ssa_test.go +++ b/ssa/ssa_test.go @@ -38,6 +38,39 @@ import ( "github.com/xgo-dev/llvm" ) +func TestProgramRewriteMainPrefixIsInvocationScoped(t *testing.T) { + pkg := types.NewPackage("example.com/rewrite", "main") + typeName := types.NewTypeName(token.NoPos, pkg, "T", nil) + named := types.NewNamed(typeName, types.Typ[types.Int], nil) + + plain := NewProgram(nil) + defer plain.Dispose() + rewritten := NewProgram(&Target{RewriteMainPrefix: true}) + defer rewritten.Dispose() + + if got := plain.FullName(pkg, "F"); got != "example.com/rewrite.F" { + t.Fatalf("plain FullName = %q, want example.com/rewrite.F", got) + } + if got := rewritten.FullName(pkg, "F"); got != "main.F" { + t.Fatalf("rewritten FullName = %q, want main.F", got) + } + if got := plain.NameOf(named); got != "example.com/rewrite.T" { + t.Fatalf("plain NameOf = %q, want example.com/rewrite.T", got) + } + if got := rewritten.NameOf(named); got != "main.T" { + t.Fatalf("rewritten NameOf = %q, want main.T", got) + } + if got, _ := plain.abi.TypeName(named); got != "_llgo_example.com/rewrite.T" { + t.Fatalf("plain ABI TypeName = %q, want _llgo_example.com/rewrite.T", got) + } + if got, _ := rewritten.abi.TypeName(named); got != "_llgo_main.T" { + t.Fatalf("rewritten ABI TypeName = %q, want _llgo_main.T", got) + } + if got := FullName(pkg, "F"); got != "example.com/rewrite.F" { + t.Fatalf("default FullName changed to %q", got) + } +} + func TestEndDefer(t *testing.T) { prog := NewProgram(nil) pkg := prog.NewPackage("foo", "foo") diff --git a/ssa/target.go b/ssa/target.go index a352b477fd..2e3f767cbc 100644 --- a/ssa/target.go +++ b/ssa/target.go @@ -27,11 +27,12 @@ import ( // ----------------------------------------------------------------------------- type Target struct { - GOOS string - GOARCH string - GOARM string // "5", "6", "7" (default) - Target string // target name from -target flag (e.g., "esp32", "arm7tdmi", "wasi") - OptLevel optlevel.Level + GOOS string + GOARCH string + GOARM string // "5", "6", "7" (default) + Target string // target name from -target flag (e.g., "esp32", "arm7tdmi", "wasi") + OptLevel optlevel.Level + RewriteMainPrefix bool // use "main" as the symbol prefix for packages named main } func (p *Target) targetInfo() (llvm.TargetData, llvm.TargetMachine) { diff --git a/ssa/type.go b/ssa/type.go index 98c9fc4848..5f5c5c351d 100644 --- a/ssa/type.go +++ b/ssa/type.go @@ -521,7 +521,7 @@ func (p Program) retType(raw *types.Signature) Type { } func (p Program) llvmNameOf(named *types.Named) (name string) { - name = NameOf(named) + name = p.NameOf(named) if obj := named.Obj(); obj != nil { parent := obj.Parent() pkg := obj.Pkg() @@ -634,20 +634,41 @@ func NameOf(typ *types.Named) string { return abi.FullName(typ.Obj().Pkg(), abi.NamedName(typ)) } +func (p Program) NameOf(typ *types.Named) string { + namer := p.namer() + return namer.FullName(typ.Obj().Pkg(), namer.NamedName(typ)) +} + // FullName returns the full name of a package member. func FullName(pkg *types.Package, name string) string { return abi.FullName(pkg, name) } +func (p Program) FullName(pkg *types.Package, name string) string { + return p.namer().FullName(pkg, name) +} + // PathOf returns the package path of the specified package. func PathOf(pkg *types.Package) string { return abi.PathOf(pkg) } +func (p Program) PathOf(pkg *types.Package) string { + return p.namer().PathOf(pkg) +} + // FuncName: // - func: pkg.name // - method: pkg.T.name, pkg.(*T).name func FuncName(pkg *types.Package, name string, recv *types.Var, org bool) string { + return funcName(abi.Namer{}, pkg, name, recv, org) +} + +func (p Program) FuncName(pkg *types.Package, name string, recv *types.Var, org bool) string { + return funcName(p.namer(), pkg, name, recv, org) +} + +func funcName(namer abi.Namer, pkg *types.Package, name string, recv *types.Var, org bool) string { if recv != nil { named, ptr := recvNamed(recv.Type()) var tName string @@ -655,18 +676,17 @@ func FuncName(pkg *types.Package, name string, recv *types.Var, org bool) string if org { tName = named.Obj().Name() } else { - tName = abi.NamedName(named) + tName = namer.NamedName(named) } if ptr { tName = "(*" + tName + ")" } } else { - tName = types.TypeString(recv.Type(), PathOf) + tName = types.TypeString(recv.Type(), namer.PathOf) } - return PathOf(pkg) + "." + tName + "." + name + return namer.PathOf(pkg) + "." + tName + "." + name } - ret := FullName(pkg, name) - return ret + return namer.FullName(pkg, name) } func recvNamed(t types.Type) (typ *types.Named, ptr bool) { @@ -689,4 +709,15 @@ func TypeArgs(typeArgs []types.Type) string { return abi.TypeArgs(typeArgs) } +func (p Program) TypeArgs(typeArgs []types.Type) string { + return p.namer().TypeArgs(typeArgs) +} + +func (p Program) namer() abi.Namer { + if p == nil { + return abi.Namer{} + } + return p.abi.Namer +} + // ----------------------------------------------------------------------------- From 05b1efd14ee11d8be42ceb5c8967da66fc1cbe12 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Thu, 30 Jul 2026 11:33:56 +0800 Subject: [PATCH 04/22] ssa: keep type equivalence within program naming --- cl/compile.go | 13 +++++++++++-- ssa/abi/abi.go | 2 ++ ssa/type.go | 17 ++++++++++++----- ssa/type_patch_test.go | 28 +++++++++++++++++++++++----- 4 files changed, 48 insertions(+), 12 deletions(-) diff --git a/cl/compile.go b/cl/compile.go index 84ac759888..da72415026 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -2096,7 +2096,9 @@ type Patch struct { // Patches is patches of some packages. type Patches = map[string]Patch -// NewPackage compiles a Go package to LLVM IR package. +// NewPackage compiles a Go package to LLVM IR package. Concurrent build +// drivers should use NewPackageExWithEmbedMetaOptions to pass frontend options +// explicitly. func NewPackage(prog llssa.Program, pkg *ssa.Package, files []*ast.File) (ret llssa.Package, err error) { ret, _, err = NewPackageEx(prog, nil, nil, pkg, files) return @@ -2106,7 +2108,9 @@ func NewPackage(prog llssa.Program, pkg *ssa.Package, files []*ast.File) (ret ll // call gets fresh caller-tracking memoization. Multi-package drivers // use NewPackageExWithEmbed with a shared CallerTracking instead. -// NewPackageEx compiles a Go package to LLVM IR package. +// NewPackageEx compiles a Go package to LLVM IR package. Concurrent build +// drivers should use NewPackageExWithEmbedMetaOptions to pass frontend options +// explicitly. // // Parameters: // - prog: target LLVM SSA program context @@ -2122,6 +2126,8 @@ func NewPackageEx(prog llssa.Program, patches Patches, rewrites map[string]strin } // NewPackageExWithEmbed compiles a package using pre-loaded go:embed metadata. +// Concurrent build drivers should use NewPackageExWithEmbedMetaOptions to pass +// frontend options explicitly. // // This avoids re-scanning directives when the caller already loaded them. // ct carries the compilation-scoped caller-tracking memoization; drivers @@ -2132,6 +2138,9 @@ func NewPackageExWithEmbed(prog llssa.Program, ct *CallerTracking, patches Patch return newPackageEx(prog, ct, patches, rewrites, pkg, files, &embedMap, false, legacyOptions()) } +// NewPackageExWithEmbedMeta compiles a package with optional metadata +// collection. Concurrent build drivers should use +// NewPackageExWithEmbedMetaOptions to pass frontend options explicitly. func NewPackageExWithEmbedMeta(prog llssa.Program, ct *CallerTracking, patches Patches, rewrites map[string]string, pkg *ssa.Package, files []*ast.File, embedMap goembed.VarMap, metaCollect bool) (ret llssa.Package, externs []string, err error) { return newPackageEx(prog, ct, patches, rewrites, pkg, files, &embedMap, metaCollect, legacyOptions()) } diff --git a/ssa/abi/abi.go b/ssa/abi/abi.go index 69a4c9c970..e07f42e1d4 100644 --- a/ssa/abi/abi.go +++ b/ssa/abi/abi.go @@ -208,6 +208,7 @@ type Namer struct { RewriteMainPrefix bool } +// NamedName returns the symbol component for a named type. func (n Namer) NamedName(t *types.Named) string { if targs := t.TypeArgs(); targs != nil { count := targs.Len() @@ -224,6 +225,7 @@ func NamedName(t *types.Named) string { return (Namer{}).NamedName(t) } +// TypeArgs formats type arguments using this Namer's package-path policy. func (n Namer) TypeArgs(typeArgs []types.Type) string { targs := make([]string, len(typeArgs)) for i, t := range typeArgs { diff --git a/ssa/type.go b/ssa/type.go index 5f5c5c351d..f0d7a74249 100644 --- a/ssa/type.go +++ b/ssa/type.go @@ -550,7 +550,7 @@ func isPkgScope(parent, pkgScope *types.Scope) bool { func (p Program) toNamed(raw *types.Named) Type { name := p.llvmNameOf(raw) if typ, ok := p.named[name]; ok { - if namedTypeEquivalent(typ.raw.Type, raw) || p.namedStructLayoutEquivalent(typ, raw) { + if p.namedTypeEquivalent(typ.raw.Type, raw) || p.namedStructLayoutEquivalent(typ, raw) { return typ } // Some toolchains produce distinct instantiated named types that share @@ -560,7 +560,7 @@ func (p Program) toNamed(raw *types.Named) Type { for i := 0; ; i++ { name = fmt.Sprintf("%s#%d", base, i) if typ2, ok2 := p.named[name]; ok2 { - if namedTypeEquivalent(typ2.raw.Type, raw) || p.namedStructLayoutEquivalent(typ2, raw) { + if p.namedTypeEquivalent(typ2.raw.Type, raw) || p.namedStructLayoutEquivalent(typ2, raw) { return typ2 } continue @@ -581,7 +581,7 @@ func (p Program) toNamed(raw *types.Named) Type { } } -func namedTypeEquivalent(a, b types.Type) bool { +func (p Program) namedTypeEquivalent(a, b types.Type) bool { na, okA := types.Unalias(a).(*types.Named) nb, okB := types.Unalias(b).(*types.Named) if !okA || !okB { @@ -590,7 +590,7 @@ func namedTypeEquivalent(a, b types.Type) bool { if types.Identical(na, nb) { return true } - if NameOf(na) != NameOf(nb) { + if p.NameOf(na) != p.NameOf(nb) { return false } _, sigA := na.Underlying().(*types.Signature) @@ -609,7 +609,7 @@ func (p Program) namedStructLayoutEquivalent(existing Type, raw *types.Named) bo return false } en, ok := types.Unalias(existing.raw.Type).(*types.Named) - if !ok || NameOf(en) != NameOf(raw) { + if !ok || p.NameOf(en) != p.NameOf(raw) { return false } rs, ok := raw.Underlying().(*types.Struct) @@ -634,6 +634,7 @@ func NameOf(typ *types.Named) string { return abi.FullName(typ.Obj().Pkg(), abi.NamedName(typ)) } +// NameOf returns the full name of typ using this Program's symbol-naming policy. func (p Program) NameOf(typ *types.Named) string { namer := p.namer() return namer.FullName(typ.Obj().Pkg(), namer.NamedName(typ)) @@ -644,6 +645,8 @@ func FullName(pkg *types.Package, name string) string { return abi.FullName(pkg, name) } +// FullName returns the full name of a package member using this Program's +// symbol-naming policy. func (p Program) FullName(pkg *types.Package, name string) string { return p.namer().FullName(pkg, name) } @@ -653,6 +656,7 @@ func PathOf(pkg *types.Package) string { return abi.PathOf(pkg) } +// PathOf returns the package path using this Program's symbol-naming policy. func (p Program) PathOf(pkg *types.Package) string { return p.namer().PathOf(pkg) } @@ -664,6 +668,8 @@ func FuncName(pkg *types.Package, name string, recv *types.Var, org bool) string return funcName(abi.Namer{}, pkg, name, recv, org) } +// FuncName returns a function or method name using this Program's +// symbol-naming policy. func (p Program) FuncName(pkg *types.Package, name string, recv *types.Var, org bool) string { return funcName(p.namer(), pkg, name, recv, org) } @@ -709,6 +715,7 @@ func TypeArgs(typeArgs []types.Type) string { return abi.TypeArgs(typeArgs) } +// TypeArgs formats type arguments using this Program's symbol-naming policy. func (p Program) TypeArgs(typeArgs []types.Type) string { return p.namer().TypeArgs(typeArgs) } diff --git a/ssa/type_patch_test.go b/ssa/type_patch_test.go index aa85bec573..cc8c1e9b52 100644 --- a/ssa/type_patch_test.go +++ b/ssa/type_patch_test.go @@ -54,6 +54,8 @@ func TestTypeStringWithPkgAndIsPkgScope(t *testing.T) { } func TestNamedTypeEquivalent(t *testing.T) { + prog := NewProgram(nil) + defer prog.Dispose() pkg1 := types.NewPackage("example.com/p", "p") pkg2 := types.NewPackage("example.com/p", "p") obj1 := types.NewTypeName(token.NoPos, pkg1, "T", nil) @@ -64,19 +66,21 @@ func TestNamedTypeEquivalent(t *testing.T) { b := types.NewNamed(obj2, types.NewStruct([]*types.Var{ types.NewField(token.NoPos, nil, "A", types.Typ[types.Int], false), }, nil), nil) - if !namedTypeEquivalent(a, b) { + if !prog.namedTypeEquivalent(a, b) { t.Fatalf("namedTypeEquivalent should be true for equivalent named structs") } c := types.NewNamed(types.NewTypeName(token.NoPos, pkg2, "T", nil), types.NewStruct([]*types.Var{ types.NewField(token.NoPos, nil, "A", types.Typ[types.String], false), }, nil), nil) - if namedTypeEquivalent(a, c) { + if prog.namedTypeEquivalent(a, c) { t.Fatalf("namedTypeEquivalent should be false for different underlying types") } } func TestNamedTypeEquivalentRecursiveSignature(t *testing.T) { + prog := NewProgram(nil) + defer prog.Dispose() pkg1 := types.NewPackage("example.com/p", "p") pkg2 := types.NewPackage("example.com/p", "p") a := types.NewNamed(types.NewTypeName(token.NoPos, pkg1, "F", nil), nil, nil) @@ -89,12 +93,14 @@ func TestNamedTypeEquivalentRecursiveSignature(t *testing.T) { types.NewTuple(types.NewParam(token.NoPos, nil, "", b)), types.NewTuple(types.NewParam(token.NoPos, nil, "", b)), false)) - if namedTypeEquivalent(a, b) { + if prog.namedTypeEquivalent(a, b) { t.Fatalf("namedTypeEquivalent should be false for recursive function signatures") } } func TestNamedTypeEquivalentRejectsAnySignature(t *testing.T) { + prog := NewProgram(nil) + defer prog.Dispose() pkg1 := types.NewPackage("example.com/p", "p") pkg2 := types.NewPackage("example.com/p", "p") sigNamed := types.NewNamed(types.NewTypeName(token.NoPos, pkg1, "F", nil), @@ -103,14 +109,26 @@ func TestNamedTypeEquivalentRejectsAnySignature(t *testing.T) { types.NewStruct([]*types.Var{ types.NewField(token.NoPos, nil, "A", types.Typ[types.Int], false), }, nil), nil) - if namedTypeEquivalent(sigNamed, structNamed) { + if prog.namedTypeEquivalent(sigNamed, structNamed) { t.Fatalf("namedTypeEquivalent should be false when only one side is a signature") } - if namedTypeEquivalent(structNamed, sigNamed) { + if prog.namedTypeEquivalent(structNamed, sigNamed) { t.Fatalf("namedTypeEquivalent should be false when only one side is a signature") } } +func TestNamedTypeEquivalentUsesProgramNamingPolicy(t *testing.T) { + prog := NewProgram(&Target{RewriteMainPrefix: true}) + defer prog.Dispose() + pkg1 := types.NewPackage("example.com/first", "main") + pkg2 := types.NewPackage("example.com/second", "main") + a := types.NewNamed(types.NewTypeName(token.NoPos, pkg1, "T", nil), types.NewStruct(nil, nil), nil) + b := types.NewNamed(types.NewTypeName(token.NoPos, pkg2, "T", nil), types.NewStruct(nil, nil), nil) + if !prog.namedTypeEquivalent(a, b) { + t.Fatal("equivalent main-package types should use the Program naming policy") + } +} + func TestToLLVMFuncPtrUsesVoidPtr(t *testing.T) { prog := NewProgram(nil) sig := types.NewSignatureType(nil, nil, nil, nil, nil, false) From e018a21c0ecdde5fbd08cb62c00ff6958aaa5634 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 10:47:16 +0800 Subject: [PATCH 05/22] test: cover legacy symbol naming wrappers --- ssa/abi/abi_patch_test.go | 4 ++++ ssa/ssa_test.go | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/ssa/abi/abi_patch_test.go b/ssa/abi/abi_patch_test.go index e256502d34..a205447da5 100644 --- a/ssa/abi/abi_patch_test.go +++ b/ssa/abi/abi_patch_test.go @@ -28,6 +28,10 @@ func testGenericNamed(pkg *types.Package, name string) *types.Named { } func TestTypeArgs_FormatsCommonKinds(t *testing.T) { + if got := typeArgString(types.Typ[types.Int]); got != "int" { + t.Fatalf("typeArgString(int) = %q, want int", got) + } + local := testLocalNamed() pkg := types.NewPackage("example.com/p", "p") aliasObj := types.NewTypeName(token.NoPos, pkg, "AliasInt", nil) diff --git a/ssa/ssa_test.go b/ssa/ssa_test.go index b0f6cdbdac..2a5ab7d4f6 100644 --- a/ssa/ssa_test.go +++ b/ssa/ssa_test.go @@ -69,6 +69,13 @@ func TestProgramRewriteMainPrefixIsInvocationScoped(t *testing.T) { if got := FullName(pkg, "F"); got != "example.com/rewrite.F" { t.Fatalf("default FullName changed to %q", got) } + if got := FuncName(pkg, "F", nil, false); got != "example.com/rewrite.F" { + t.Fatalf("default FuncName changed to %q", got) + } + var nilProgram Program + if got := nilProgram.TypeArgs([]types.Type{types.Typ[types.Int]}); got != "[int]" { + t.Fatalf("nil Program TypeArgs = %q, want [int]", got) + } } func TestEndDefer(t *testing.T) { From fe35ab3c093497aee7daa402d5cc00fc5fe67d3f Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Thu, 30 Jul 2026 08:06:38 +0800 Subject: [PATCH 06/22] build: split package compilation stages --- internal/build/build.go | 289 +++++++++++++++++---------- internal/build/collect.go | 43 ++-- internal/build/dependencies.go | 50 +++++ internal/build/dependencies_test.go | 46 +++++ internal/build/package_build.go | 64 ++++++ internal/build/package_build_test.go | 67 +++++++ 6 files changed, 440 insertions(+), 119 deletions(-) create mode 100644 internal/build/dependencies.go create mode 100644 internal/build/dependencies_test.go create mode 100644 internal/build/package_build.go create mode 100644 internal/build/package_build_test.go diff --git a/internal/build/build.go b/internal/build/build.go index 978e9ae417..eded0bcd49 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -350,6 +350,13 @@ func (c *Config) packageMetaEnabled() bool { return c.CollectPackageMeta || c.deadcodeDropEnabled() } +func (c *Config) parallelism() int { + if c != nil && c.BuildParallelism > 0 { + return c.BuildParallelism + } + return max(1, runtime.GOMAXPROCS(0)) +} + // ----------------------------------------------------------------------------- const ( @@ -601,7 +608,7 @@ func Build(inv Invocation) ([]Package, error) { } progSSA := ssa.NewProgram(initial[0].Fset, buildMode) patches := make(cl.Patches, len(altPkgPaths)) - altSSAPkgs(progSSA, patches, altPkgs[1:], conf, verbose) + altEntries := registerAltSSAPkgs(progSSA, patches, altPkgs[1:], conf, verbose) output := conf.OutFile != "" ctx := &context{conf: cfg, progSSA: progSSA, prog: prog, dedup: dedup, @@ -623,14 +630,15 @@ func Build(inv Invocation) ([]Package, error) { // default runtime globals must be registered before packages are built addGlobalString(conf, "runtime.defaultGOROOT="+runtime.GOROOT(), nil) addGlobalString(conf, "runtime.buildVersion="+runtime.Version(), nil) - pkgs, err := buildSSAPkgs(ctx, initial, verbose) + pkgs, pkgEntries, err := registerSSAPkgs(ctx, initial, verbose) if err != nil { return nil, err } - depPkgs, err := buildSSAPkgs(ctx, altPkgs, verbose) + depPkgs, depEntries, err := registerSSAPkgs(ctx, altPkgs, verbose) if err != nil { return nil, err } + buildSSAPkgs(ctx, append(append(altEntries, pkgEntries...), depEntries...)) allPkgs := append([]*aPackage{}, pkgs...) allPkgs = append(allPkgs, depPkgs...) @@ -863,6 +871,7 @@ type context struct { callerTracking *cl.CallerTracking built map[string]none fingerprinting map[string]bool + cacheDisabled map[string]none initial []*packages.Package pkgs map[*packages.Package]Package // cache for lookup pkgByID map[string]Package // cache for lookup by pkg.ID @@ -978,106 +987,34 @@ func normalizeToArchive(ctx *context, aPkg *aPackage, verbose bool) error { } func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, error) { - built := ctx.built - // Split packages into runtime tree vs others so we can defer runtime build. - var runtimePkgs []*aPackage - var normalPkgs []*aPackage + var runtimePkgs []packageBuildSpec + var normalPkgs []packageBuildSpec for _, p := range pkgs { - if isRuntimePkg(p.PkgPath) { - runtimePkgs = append(runtimePkgs, p) + spec := newPackageBuildSpec(p) + if spec.runtime { + runtimePkgs = append(runtimePkgs, spec) } else { - normalPkgs = append(normalPkgs, p) + normalPkgs = append(normalPkgs, spec) } } var needRuntime, needPyInit bool - buildOne := func(aPkg *aPackage) error { - pkg := aPkg.Package - if _, ok := built[pkg.ID]; ok { - // Already built, skip but keep ExportFile for linking - return nil - } - built[pkg.ID] = none{} - - switch kind, param := cl.PkgKindOf(pkg.Types); kind { - case cl.PkgDeclOnly: - pkg.ExportFile = "" - case cl.PkgLinkIR, cl.PkgLinkExtern, cl.PkgPyModule: - if len(pkg.GoFiles) > 0 { - if err := ctx.collectFingerprint(aPkg); err != nil { - return err - } - ctx.tryLoadFromCache(aPkg) - if verbose { - if aPkg.CacheHit { - fmt.Fprintf(os.Stderr, "CACHE HIT: %s\n", pkg.PkgPath) - } else { - fmt.Fprintf(os.Stderr, "CACHE MISS: %s\n", pkg.PkgPath) - } - } - if err := buildPkg(ctx, aPkg, verbose); err != nil { - return err - } - if !aPkg.CacheHit { - if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { - return err - } - if kind == cl.PkgLinkExtern { - appendExternalLinkArgs(ctx, aPkg, param) - } - if err := ctx.saveToCache(aPkg); err != nil && verbose { - fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", pkg.PkgPath, err) - } - } - } else { - pkg.ExportFile = "" - if kind == cl.PkgLinkExtern { - appendExternalLinkArgs(ctx, aPkg, param) - } - } - default: - if err := ctx.collectFingerprint(aPkg); err != nil { - return err - } - ctx.tryLoadFromCache(aPkg) - if verbose { - if aPkg.CacheHit { - fmt.Fprintf(os.Stderr, "CACHE HIT: %s\n", pkg.PkgPath) - } else { - fmt.Fprintf(os.Stderr, "CACHE MISS: %s\n", pkg.PkgPath) - } - } - if err := buildPkg(ctx, aPkg, verbose); err != nil { - return err - } - aPkg.setNeedRuntimeOrPyInit(aPkg.LPkg.NeedRuntime, aPkg.LPkg.NeedPyInit) - needRuntime = needRuntime || aPkg.NeedRt - needPyInit = needPyInit || aPkg.NeedPyInit - if !aPkg.CacheHit { - if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { - return err - } - if err := ctx.saveToCache(aPkg); err != nil && verbose { - fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", pkg.PkgPath, err) - } - } - } - return nil - } - // Build non-runtime packages first, so we know whether runtime is actually needed. - for _, p := range normalPkgs { - if err := buildOne(p); err != nil { + for _, spec := range normalPkgs { + result, err := buildOnePackage(ctx, spec, verbose) + if err != nil { return nil, err } + needRuntime = needRuntime || result.needRuntime + needPyInit = needPyInit || result.needPyInit } // Only build runtime packages when required (or host build with empty Target). if needRuntime || needPyInit || ctx.buildConf.Target == "" { - for _, p := range runtimePkgs { - if err := buildOne(p); err != nil { + for _, spec := range runtimePkgs { + if _, err := buildOnePackage(ctx, spec, verbose); err != nil { return nil, err } } @@ -1086,6 +1023,85 @@ func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, er return pkgs, nil } +// buildOnePackage is the serial package pipeline. Its explicit stages are the +// contract used by later package workers; this commit deliberately preserves +// serial LLVM execution. +func buildOnePackage(ctx *context, spec packageBuildSpec, verbose bool) (packageBuildResult, error) { + skip, err := preflightPackageBuild(ctx, spec, verbose) + if err != nil || skip { + return packageBuildResultFor(spec), err + } + if err := executePackageBuild(ctx, spec, verbose); err != nil { + return packageBuildResultFor(spec), err + } + return finalizePackageBuild(ctx, spec, verbose) +} + +// preflightPackageBuild performs classification, fingerprinting, and cache +// lookup without creating or transforming an LLVM module. +func preflightPackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (skip bool, err error) { + aPkg := spec.pkg + pkg := aPkg.Package + if _, ok := ctx.built[pkg.ID]; ok { + return true, nil + } + ctx.built[pkg.ID] = none{} + if spec.isDeclOnly() { + pkg.ExportFile = "" + return true, nil + } + if spec.isLinkOnly() && !spec.hasSource() { + pkg.ExportFile = "" + if spec.kind == cl.PkgLinkExtern { + appendExternalLinkArgs(ctx, aPkg, spec.kindParam) + } + return true, nil + } + if err := ctx.collectFingerprint(aPkg); err != nil { + return false, err + } + ctx.tryLoadFromCache(aPkg) + if verbose { + status := "MISS" + if aPkg.CacheHit { + status = "HIT" + } + fmt.Fprintf(os.Stderr, "CACHE %s: %s\n", status, pkg.PkgPath) + } + return false, nil +} + +// executePackageBuild creates the package module and runs its LLVM backend. +func executePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) error { + aPkg := spec.pkg + if err := buildPkg(ctx, aPkg, verbose); err != nil { + return err + } + if spec.needsRuntimeSignals() { + aPkg.setNeedRuntimeOrPyInit(aPkg.LPkg.NeedRuntime, aPkg.LPkg.NeedPyInit) + } + return nil +} + +// finalizePackageBuild publishes the archive and cache metadata. Cache hits +// already carry both and therefore require no publication. +func finalizePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (packageBuildResult, error) { + aPkg := spec.pkg + if aPkg.CacheHit { + return packageBuildResultFor(spec), nil + } + if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { + return packageBuildResultFor(spec), err + } + if spec.kind == cl.PkgLinkExtern { + appendExternalLinkArgs(ctx, aPkg, spec.kindParam) + } + if err := ctx.saveToCache(aPkg); err != nil && verbose { + fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", aPkg.PkgPath, err) + } + return packageBuildResultFor(spec), nil +} + func appendExternalLinkArgs(ctx *context, aPkg *aPackage, spec string) { // need to be linked with external library // format: ';' separated alternative link methods. e.g. @@ -1747,6 +1763,15 @@ func is32Bits(goarch string) bool { } func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { + externs, err := preparePackageModule(ctx, aPkg, verbose) + if err != nil || aPkg.CacheHit || aPkg.LPkg == nil { + return err + } + return compilePackageModule(ctx, aPkg, externs, verbose) +} + +// preparePackageModule runs the frontend and creates the package LLVM module. +func preparePackageModule(ctx *context, aPkg *aPackage, verbose bool) ([]string, error) { pkg := aPkg.Package pkgPath := pkg.PkgPath if debugBuild || verbose { @@ -1756,7 +1781,7 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { } if llruntime.SkipToBuild(pkgPath) { pkg.ExportFile = "" - return nil + return nil, nil } var syntax = pkg.Syntax if altPkg := aPkg.AltPkg; altPkg != nil { @@ -1796,8 +1821,16 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { // If cache hit, we only needed to register types - skip compilation if aPkg.CacheHit { - return nil + return nil, nil } + return externs, nil +} + +// compilePackageModule applies LLVM transforms and emits package objects. +func compilePackageModule(ctx *context, aPkg *aPackage, externs []string, verbose bool) error { + pkg := aPkg.Package + pkgPath := pkg.PkgPath + ret := aPkg.LPkg ctx.cTransformer.SetSkipFuncs(cabiSkipFuncsForPlan9Asm(ctx, pkgPath, ret.Module())) llabi.LowerLargeAggregates(ctx.prog.TargetData(), ret.Module()) @@ -1811,7 +1844,7 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { mod.SetTarget(ctx.prog.Target().Spec().Triple) pbo := gllvm.NewPassBuilderOptions() defer pbo.Dispose() - if err = gllvm.VerifyModule(mod, gllvm.ReturnStatusAction); err != nil { + if err := gllvm.VerifyModule(mod, gllvm.ReturnStatusAction); err != nil { return fmt.Errorf("verify LLVM module for %v failed: %w", pkgPath, err) } if err := mod.RunPasses(llvmPassPipeline(ctx.buildConf.OptLevel, ctx.buildConf.ltoMode()), ctx.prog.TargetMachine(), pbo); err != nil { @@ -2096,13 +2129,21 @@ func prepareLocalVariables(prog llssa.Program, groups ...[]*packages.Package) er return nil } -func altSSAPkgs(prog *ssa.Program, patches cl.Patches, alts []*packages.Package, conf *Config, verbose bool) { +type ssaBuildEntry struct { + pkg *ssa.Package + syntax []*ast.File + fixOrder bool +} + +func registerAltSSAPkgs(prog *ssa.Program, patches cl.Patches, alts []*packages.Package, conf *Config, verbose bool) []ssaBuildEntry { + var entries []ssaBuildEntry packages.Visit(alts, nil, func(p *packages.Package) { if typs := p.Types; typs != nil && !p.IllTyped { if debugBuild || verbose { log.Println("==> BuildSSA", p.ID) } pkgSSA := prog.CreatePackage(typs, p.Syntax, p.TypesInfo, true) + entries = append(entries, ssaBuildEntry{pkg: pkgSSA, syntax: p.Syntax}) if strings.HasPrefix(p.ID, altPkgPathPrefix) { path := p.ID[len(altPkgPathPrefix):] // Even if an alt package exists and is pulled in as a dependency of other @@ -2118,7 +2159,7 @@ func altSSAPkgs(prog *ssa.Program, patches cl.Patches, alts []*packages.Package, } } }) - prog.Build() + return entries } type aPackage struct { @@ -2144,9 +2185,10 @@ type aPackage struct { type Package = *aPackage -func buildSSAPkgs(ctx *context, initial []*packages.Package, verbose bool) ([]*aPackage, error) { +func registerSSAPkgs(ctx *context, initial []*packages.Package, verbose bool) ([]*aPackage, []ssaBuildEntry, error) { prog := ctx.progSSA var all []*aPackage + var entries []ssaBuildEntry var errs []*packages.Package packages.Visit(initial, nil, func(p *packages.Package) { if p.Types != nil && !p.IllTyped { @@ -2156,7 +2198,10 @@ func buildSSAPkgs(ctx *context, initial []*packages.Package, verbose bool) ([]*a return } var altPkg *packages.Cached - var ssaPkg = createSSAPkg(ctx, prog, p, verbose) + ssaPkg, created := createSSAPkg(ctx, prog, p, verbose) + if created { + entries = append(entries, ssaBuildEntry{pkg: ssaPkg, syntax: p.Syntax, fixOrder: true}) + } if ctx.hasAltPkg(pkgPath) { if altPkg = ctx.dedup.Check(altPkgPathPrefix + pkgPath); altPkg == nil { return @@ -2188,9 +2233,51 @@ func buildSSAPkgs(ctx *context, initial []*packages.Package, verbose bool) ([]*a } fmt.Fprintln(os.Stderr, "cannot build SSA for package", errPkg) } - return nil, fmt.Errorf("cannot build SSA for packages") + return nil, nil, fmt.Errorf("cannot build SSA for packages") + } + return all, entries, nil +} + +// buildSSAPkgs builds registered packages with the requested bound, then +// performs ordering repair serially because it mutates instruction slices. +func buildSSAPkgs(ctx *context, entries []ssaBuildEntry) { + if len(entries) == 0 { + return + } + unique := make([]ssaBuildEntry, 0, len(entries)) + index := make(map[*ssa.Package]int, len(entries)) + for _, entry := range entries { + if entry.pkg == nil { + continue + } + if i, ok := index[entry.pkg]; ok { + unique[i].fixOrder = unique[i].fixOrder || entry.fixOrder + continue + } + index[entry.pkg] = len(unique) + unique = append(unique, entry) + } + jobs := make(chan ssaBuildEntry, len(unique)) + var wg sync.WaitGroup + for range min(ctx.buildConf.parallelism(), len(unique)) { + wg.Add(1) + go func() { + defer wg.Done() + for entry := range jobs { + entry.pkg.Build() + } + }() + } + for _, entry := range unique { + jobs <- entry + } + close(jobs) + wg.Wait() + for _, entry := range unique { + if entry.fixOrder { + fixSSAOrder(entry.pkg, entry.syntax) + } } - return all, nil } func formatPackageError(err packages.Error, noColumn bool) string { @@ -2338,7 +2425,7 @@ func applyPatches(ctx *context, p *packages.Package, verbose bool) { } } -func createSSAPkg(ctx *context, prog *ssa.Program, p *packages.Package, verbose bool) *ssa.Package { +func createSSAPkg(ctx *context, prog *ssa.Program, p *packages.Package, verbose bool) (*ssa.Package, bool) { pkgSSA := prog.ImportedPackage(p.ID) if pkgSSA == nil { if debugBuild || verbose { @@ -2346,11 +2433,9 @@ func createSSAPkg(ctx *context, prog *ssa.Program, p *packages.Package, verbose } applyPatches(ctx, p, verbose) pkgSSA = prog.CreatePackage(p.Types, p.Syntax, p.TypesInfo, true) - pkgSSA.Build() // TODO(xsw): build concurrently - // Apply local SSA fixups once when package SSA is first built. - fixSSAOrder(pkgSSA, p.Syntax) + return pkgSSA, true } - return pkgSSA + return pkgSSA, false } /* diff --git a/internal/build/collect.go b/internal/build/collect.go index 66da887c21..d8dcee7b1f 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -23,7 +23,6 @@ import ( "os/exec" "path/filepath" "runtime" - "sort" "strconv" "strings" @@ -42,7 +41,11 @@ func (c *context) collectFingerprint(pkg *aPackage) error { c.fingerprinting = make(map[string]bool) } if c.fingerprinting[pkg.ID] { - return fmt.Errorf("fingerprint cycle detected for %s", pkg.ID) + // Alternate runtime replacements can close a cycle after the complete + // SSA graph is registered. Such a cycle has no stable per-package key, + // so rebuild its members instead of reporting a false cache hit. + c.disablePackageCache(c.fingerprinting) + return nil } c.fingerprinting[pkg.ID] = true defer delete(c.fingerprinting, pkg.ID) @@ -70,6 +73,20 @@ func (c *context) collectFingerprint(pkg *aPackage) error { return nil } +func (c *context) disablePackageCache(pkgs map[string]bool) { + if c.cacheDisabled == nil { + c.cacheDisabled = make(map[string]none, len(pkgs)) + } + for id := range pkgs { + c.cacheDisabled[id] = none{} + } +} + +func (c *context) packageCacheDisabled(id string) bool { + _, disabled := c.cacheDisabled[id] + return disabled +} + // collectEnvInputs collects environment-related inputs. func (c *context) collectEnvInputs(m *manifestBuilder) { m.env.Goos = c.buildConf.Goos @@ -197,21 +214,7 @@ func (c *context) collectPackageInputs(m *manifestBuilder, pkg *aPackage) error // collectDependencyInputs adds dependency fingerprints/versions into manifest. func (c *context) collectDependencyInputs(m *manifestBuilder, pkg *aPackage) error { - if len(pkg.Imports) == 0 { - return nil - } - - deps := make([]*packages.Package, 0, len(pkg.Imports)) - for _, dep := range pkg.Imports { - if dep == nil || dep.ID == pkg.ID { - continue - } - deps = append(deps, dep) - } - - sort.Slice(deps, func(i, j int) bool { return deps[i].ID < deps[j].ID }) - - for _, dep := range deps { + for _, dep := range effectiveDependencies(pkg) { depEntry, err := c.dependencyFingerprint(dep) if err != nil { return err @@ -328,6 +331,9 @@ func (c *context) tryLoadFromCache(pkg *aPackage) bool { if !cacheEnabled() { return false } + if c.packageCacheDisabled(pkg.ID) { + return false + } // Main packages are intentionally not written to the build cache because // each executable's entry module is linked against the current main archive. @@ -453,6 +459,9 @@ func (c *context) saveToCache(pkg *aPackage) error { if !cacheEnabled() { return nil } + if c.packageCacheDisabled(pkg.ID) { + return nil + } if pkg.Fingerprint == "" || pkg.Manifest == "" { return nil diff --git a/internal/build/dependencies.go b/internal/build/dependencies.go new file mode 100644 index 0000000000..39f5233217 --- /dev/null +++ b/internal/build/dependencies.go @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "sort" + + "github.com/goplus/llgo/internal/packages" +) + +// effectiveDependencies returns every package whose source contributes to an +// aPackage. Alternate packages may add imports absent from the original graph. +func effectiveDependencies(pkg *aPackage) []*packages.Package { + if pkg == nil || pkg.Package == nil { + return nil + } + deps := make(map[string]*packages.Package) + add := func(imports map[string]*packages.Package) { + for _, dep := range imports { + if dep == nil || dep.ID == pkg.ID || (pkg.AltPkg != nil && dep.ID == pkg.AltPkg.ID) { + continue + } + deps[dep.ID] = dep + } + } + add(pkg.Imports) + if pkg.AltPkg != nil { + add(pkg.AltPkg.Imports) + } + ret := make([]*packages.Package, 0, len(deps)) + for _, dep := range deps { + ret = append(ret, dep) + } + sort.Slice(ret, func(i, j int) bool { return ret[i].ID < ret[j].ID }) + return ret +} diff --git a/internal/build/dependencies_test.go b/internal/build/dependencies_test.go new file mode 100644 index 0000000000..7f083eb13d --- /dev/null +++ b/internal/build/dependencies_test.go @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "reflect" + "testing" + + "github.com/goplus/llgo/internal/packages" +) + +func TestEffectiveDependenciesIncludesAlternateImports(t *testing.T) { + base := &packages.Package{ID: "base"} + shared := &packages.Package{ID: "shared"} + altOnly := &packages.Package{ID: "alt-only"} + alt := &packages.Package{ + ID: "alt", + Imports: map[string]*packages.Package{"shared": shared, "alt-only": altOnly}, + } + pkg := &aPackage{ + Package: &packages.Package{ID: "pkg", Imports: map[string]*packages.Package{"base": base, "shared": shared}}, + AltPkg: &packages.Cached{Package: alt}, + } + deps := effectiveDependencies(pkg) + got := make([]string, len(deps)) + for i, dep := range deps { + got[i] = dep.ID + } + if want := []string{"alt-only", "base", "shared"}; !reflect.DeepEqual(got, want) { + t.Fatalf("effectiveDependencies = %v, want %v", got, want) + } +} diff --git a/internal/build/package_build.go b/internal/build/package_build.go new file mode 100644 index 0000000000..a5ccc0ab38 --- /dev/null +++ b/internal/build/package_build.go @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import "github.com/goplus/llgo/cl" + +type packageBuildSpec struct { + pkg *aPackage + kind int + kindParam string + runtime bool +} + +func newPackageBuildSpec(pkg *aPackage) packageBuildSpec { + kind, kindParam := cl.PkgKindOf(pkg.Types) + return packageBuildSpec{ + pkg: pkg, + kind: kind, + kindParam: kindParam, + runtime: isRuntimePkg(pkg.PkgPath), + } +} + +func (s packageBuildSpec) isDeclOnly() bool { + return s.kind == cl.PkgDeclOnly +} + +func (s packageBuildSpec) isLinkOnly() bool { + return s.kind == cl.PkgLinkIR || s.kind == cl.PkgLinkExtern || s.kind == cl.PkgPyModule +} + +func (s packageBuildSpec) hasSource() bool { + return len(s.pkg.GoFiles) > 0 +} + +func (s packageBuildSpec) needsRuntimeSignals() bool { + return !s.isLinkOnly() && !s.isDeclOnly() +} + +type packageBuildResult struct { + needRuntime bool + needPyInit bool +} + +func packageBuildResultFor(spec packageBuildSpec) packageBuildResult { + return packageBuildResult{ + needRuntime: spec.pkg.NeedRt, + needPyInit: spec.pkg.NeedPyInit, + } +} diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go new file mode 100644 index 0000000000..e3cf97a055 --- /dev/null +++ b/internal/build/package_build_test.go @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "go/types" + "testing" + + "github.com/goplus/llgo/internal/env" + "github.com/goplus/llgo/internal/packages" +) + +func TestPackageBuildSpecAndResult(t *testing.T) { + pkg := &aPackage{Package: &packages.Package{ + PkgPath: "example.com/p", + GoFiles: []string{"p.go"}, + Types: types.NewPackage("example.com/p", "p"), + }, NeedRt: true, NeedPyInit: true} + spec := newPackageBuildSpec(pkg) + if spec.isDeclOnly() || spec.isLinkOnly() || !spec.hasSource() || spec.runtime || !spec.needsRuntimeSignals() { + t.Fatalf("unexpected normal package spec: %+v", spec) + } + result := packageBuildResultFor(spec) + if !result.needRuntime || !result.needPyInit { + t.Fatalf("unexpected package result: %+v", result) + } +} + +func TestPackageBuildSpecSpecialKinds(t *testing.T) { + decl := newPackageBuildSpec(&aPackage{Package: &packages.Package{ + PkgPath: "unsafe", + Types: types.Unsafe, + }}) + if !decl.isDeclOnly() || decl.needsRuntimeSignals() { + t.Fatalf("unexpected declaration-only spec: %+v", decl) + } + runtime := newPackageBuildSpec(&aPackage{Package: &packages.Package{ + PkgPath: env.LLGoRuntimePkg, + Types: types.NewPackage(env.LLGoRuntimePkg, "runtime"), + }}) + if !runtime.runtime { + t.Fatalf("runtime package was not marked runtime: %+v", runtime) + } +} + +func TestConfigParallelism(t *testing.T) { + if got := (&Config{BuildParallelism: 3}).parallelism(); got != 3 { + t.Fatalf("parallelism = %d, want 3", got) + } + if got := (&Config{}).parallelism(); got < 1 { + t.Fatalf("default parallelism = %d, want positive value", got) + } +} From df990fcdc6e5af7e253cd0a64b343b40028e9316 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 00:39:38 +0800 Subject: [PATCH 07/22] test: cover package build stage boundaries --- internal/build/collect_test.go | 17 ++++++++ internal/build/package_build_test.go | 58 ++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/internal/build/collect_test.go b/internal/build/collect_test.go index d6a8c585d5..5b6294f1f4 100644 --- a/internal/build/collect_test.go +++ b/internal/build/collect_test.go @@ -135,6 +135,23 @@ func TestCollectFingerprintDeterminism(t *testing.T) { } } +func TestDisablePackageCache(t *testing.T) { + ctx := &context{} + ctx.disablePackageCache(map[string]bool{ + "example.com/a": true, + "example.com/b": false, + }) + + for _, id := range []string{"example.com/a", "example.com/b"} { + if !ctx.packageCacheDisabled(id) { + t.Fatalf("package cache for %q remains enabled", id) + } + } + if ctx.packageCacheDisabled("example.com/c") { + t.Fatal("unlisted package cache was disabled") + } +} + func TestCollectFingerprintIncludesEmitDWARF(t *testing.T) { td := t.TempDir() goFile := filepath.Join(td, "main.go") diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go index e3cf97a055..a9a940beef 100644 --- a/internal/build/package_build_test.go +++ b/internal/build/package_build_test.go @@ -65,3 +65,61 @@ func TestConfigParallelism(t *testing.T) { t.Fatalf("default parallelism = %d, want positive value", got) } } + +func TestBuildOnePackageSkipsAlreadyBuiltPackage(t *testing.T) { + pkg := &aPackage{Package: &packages.Package{ + ID: "example.com/already-built", + PkgPath: "example.com/already-built", + GoFiles: []string{"already.go"}, + Types: types.NewPackage("example.com/already-built", "already"), + }, NeedRt: true} + ctx := &context{built: map[string]none{pkg.ID: {}}} + + result, err := buildOnePackage(ctx, newPackageBuildSpec(pkg), false) + if err != nil { + t.Fatal(err) + } + if !result.needRuntime { + t.Fatalf("build result = %+v, want runtime requirement preserved", result) + } +} + +func TestPreflightPackageBuildSkipsDeclarationOnlyPackage(t *testing.T) { + pkg := &aPackage{Package: &packages.Package{ + ID: "unsafe", + PkgPath: "unsafe", + Types: types.Unsafe, + ExportFile: "stale.a", + }} + ctx := &context{built: make(map[string]none)} + + skip, err := preflightPackageBuild(ctx, newPackageBuildSpec(pkg), false) + if err != nil { + t.Fatal(err) + } + if !skip { + t.Fatal("declaration-only package was not skipped") + } + if pkg.ExportFile != "" { + t.Fatalf("ExportFile = %q, want empty", pkg.ExportFile) + } + if _, ok := ctx.built[pkg.ID]; !ok { + t.Fatal("declaration-only package was not recorded as built") + } +} + +func TestFinalizePackageBuildReturnsCachedResult(t *testing.T) { + pkg := &aPackage{Package: &packages.Package{ + ID: "example.com/cached", + PkgPath: "example.com/cached", + Types: types.NewPackage("example.com/cached", "cached"), + }, CacheHit: true, NeedPyInit: true} + + result, err := finalizePackageBuild(&context{}, newPackageBuildSpec(pkg), false) + if err != nil { + t.Fatal(err) + } + if !result.needPyInit { + t.Fatalf("build result = %+v, want Python initialization requirement preserved", result) + } +} From b273adf5a161949523226dcbfe56a3eb04a35a57 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 01:20:51 +0800 Subject: [PATCH 08/22] test: cover empty package dependencies --- internal/build/dependencies_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/build/dependencies_test.go b/internal/build/dependencies_test.go index 7f083eb13d..78fa352e56 100644 --- a/internal/build/dependencies_test.go +++ b/internal/build/dependencies_test.go @@ -44,3 +44,9 @@ func TestEffectiveDependenciesIncludesAlternateImports(t *testing.T) { t.Fatalf("effectiveDependencies = %v, want %v", got, want) } } + +func TestEffectiveDependenciesHandlesNilPackage(t *testing.T) { + if deps := effectiveDependencies(nil); deps != nil { + t.Fatalf("effectiveDependencies(nil) = %v, want nil", deps) + } +} From 7687a22e775e96532f631312d7190dd7e6cd2515 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 04:13:51 +0800 Subject: [PATCH 09/22] test: cover cgo command environment parsing --- internal/build/cgo_test.go | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/internal/build/cgo_test.go b/internal/build/cgo_test.go index 91fba3f534..8ccbb79d0d 100644 --- a/internal/build/cgo_test.go +++ b/internal/build/cgo_test.go @@ -77,6 +77,52 @@ func TestParseCgoDeclFlags(t *testing.T) { } } +func TestParseCgoDeclWithCommandEnvBranches(t *testing.T) { + commands := commandEnv{dir: t.TempDir(), environ: []string{"CGO_TEST_MARKER=1"}} + tests := []struct { + name string + line string + want []cgoDecl + wantErr string + }{ + { + name: "LDFLAGS with tag", + line: "#cgo darwin LDFLAGS: -framework CoreFoundation -lz", + want: []cgoDecl{{tag: "darwin", ldflags: []string{"-framework CoreFoundation", "-lz"}}}, + }, + {name: "missing colon", line: "#cgo CFLAGS -I/missing", wantErr: "invalid cgo format"}, + {name: "missing directive", line: "CFLAGS: -I/missing", wantErr: "invalid cgo directive"}, + {name: "unsupported flag", line: "#cgo FOOFLAGS: -unsupported", wantErr: "unsupported cgo flag type"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseCgoDeclWithCommandEnv(commands, tt.line) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("parseCgoDeclWithCommandEnv(%q) error = %v, want %q", tt.line, err, tt.wantErr) + } + return + } + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("parseCgoDeclWithCommandEnv(%q) = %#v, want %#v", tt.line, got, tt.want) + } + }) + } +} + +func TestParseCgoPreambleDelegatesToCommandEnvParser(t *testing.T) { + preamble, decls, err := parseCgoPreamble(token.Position{Filename: "request.go", Line: 7}, "#cgo CFLAGS: -I/request/include") + if err != nil { + t.Fatal(err) + } + if preamble.goFile != "request.go" || preamble.src == "" || !reflect.DeepEqual(decls, []cgoDecl{{cflags: []string{"-I/request/include"}}}) { + t.Fatalf("parseCgoPreamble() = %#v, %#v", preamble, decls) + } +} + func TestCollectCgoSymbolsStripsPackagePrefix(t *testing.T) { externs := []string{ "command-line-arguments._cgo_96608f8de8c8_Cfunc_fputs", From b8fd8985506a2739601cf8173485db12fa42a7e2 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 05:10:44 +0800 Subject: [PATCH 10/22] test: cover cgo pkg-config invocation --- internal/build/cgo_test.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/internal/build/cgo_test.go b/internal/build/cgo_test.go index 8ccbb79d0d..9127c6f92a 100644 --- a/internal/build/cgo_test.go +++ b/internal/build/cgo_test.go @@ -113,6 +113,40 @@ func TestParseCgoDeclWithCommandEnvBranches(t *testing.T) { } } +func TestParseCgoDeclWithCommandEnvPkgConfig(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("test helper uses a shell script") + } + + dir := t.TempDir() + tool := filepath.Join(dir, "pkg-config") + script := `#!/bin/sh +if [ "$1" = "--libs" ]; then + printf '%s\n' '-L/request/lib -lrequest' + exit 0 +fi +printf '%s\n' '-I/request/include -DREQUEST="request value"' +` + if err := os.WriteFile(tool, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir) + + commands := commandEnv{dir: dir, environ: []string{"PATH=" + dir}} + got, err := parseCgoDeclWithCommandEnv(commands, "#cgo linux pkg-config: request") + if err != nil { + t.Fatal(err) + } + want := []cgoDecl{{ + tag: "linux", + cflags: []string{"-I/request/include", `-DREQUEST="request value"`}, + ldflags: []string{"-L/request/lib", "-lrequest"}, + }} + if !reflect.DeepEqual(got, want) { + t.Fatalf("parseCgoDeclWithCommandEnv(pkg-config) = %#v, want %#v", got, want) + } +} + func TestParseCgoPreambleDelegatesToCommandEnvParser(t *testing.T) { preamble, decls, err := parseCgoPreamble(token.Position{Filename: "request.go", Line: 7}, "#cgo CFLAGS: -I/request/include") if err != nil { From 93f7d8a59d7dcaaca4dd48ccb2821648acf0b81c Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 08:07:45 +0800 Subject: [PATCH 11/22] test: cover cgo pkg-config failures --- internal/build/cgo_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/build/cgo_test.go b/internal/build/cgo_test.go index 9127c6f92a..926b30b518 100644 --- a/internal/build/cgo_test.go +++ b/internal/build/cgo_test.go @@ -121,6 +121,9 @@ func TestParseCgoDeclWithCommandEnvPkgConfig(t *testing.T) { dir := t.TempDir() tool := filepath.Join(dir, "pkg-config") script := `#!/bin/sh +if [ "$PKG_CONFIG_TEST_FAIL" = "$1" ]; then + exit 1 +fi if [ "$1" = "--libs" ]; then printf '%s\n' '-L/request/lib -lrequest' exit 0 @@ -145,6 +148,15 @@ printf '%s\n' '-I/request/include -DREQUEST="request value"' if !reflect.DeepEqual(got, want) { t.Fatalf("parseCgoDeclWithCommandEnv(pkg-config) = %#v, want %#v", got, want) } + + for _, arg := range []string{"--libs", "--cflags"} { + t.Run("failed "+arg, func(t *testing.T) { + commands := commandEnv{dir: dir, environ: []string{"PATH=" + dir, "PKG_CONFIG_TEST_FAIL=" + arg}} + if _, err := parseCgoDeclWithCommandEnv(commands, "#cgo pkg-config: request"); err == nil || !strings.Contains(err.Error(), "pkg-config") { + t.Fatalf("parseCgoDeclWithCommandEnv(pkg-config) error = %v, want pkg-config failure", err) + } + }) + } } func TestParseCgoPreambleDelegatesToCommandEnvParser(t *testing.T) { From a3ebe54f4aa193d414cca377bdeb657e0b1a04e7 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 16:55:18 +0800 Subject: [PATCH 12/22] build: archive LLVM package objects in memory --- go.mod | 2 + go.sum | 4 +- internal/build/build.go | 58 ++++++++++-- internal/build/package_archive.go | 99 ++++++++++++++++++++ internal/build/package_archive_test.go | 123 +++++++++++++++++++++++++ 5 files changed, 275 insertions(+), 11 deletions(-) create mode 100644 internal/build/package_archive.go create mode 100644 internal/build/package_archive_test.go diff --git a/go.mod b/go.mod index c55f42ffd4..23ba621674 100644 --- a/go.mod +++ b/go.mod @@ -26,3 +26,5 @@ require ( ) replace github.com/goplus/llgo/runtime => ./runtime + +replace github.com/xgo-dev/llvm => github.com/zhouguangyuan0718/go-llvm v0.0.0-20260731084524-d8f8a2aa851e diff --git a/go.sum b/go.sum index db139c2acc..fa2dd992cd 100644 --- a/go.sum +++ b/go.sum @@ -22,10 +22,10 @@ github.com/qiniu/x v1.18.0 h1:iMfc7Gqy1au+akr+Tl5Z40px7TR8VBLLkJsIeajKIbc= github.com/qiniu/x v1.18.0/go.mod h1:Sx3Wy+0GI9OsX4a53mYj6A0o7mHJ94PUvraqGYb4EIs= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/xgo-dev/llvm v0.9.5 h1:RijI/6vGu7DGw5ldlaqvpK2uFcg9OXY9OXB/RmywtEk= -github.com/xgo-dev/llvm v0.9.5/go.mod h1:42vav2/cI5BAIcL543DZSMO9do8/aCK2z7JERH+AE+M= github.com/xgo-dev/plan9asm v0.3.5 h1:886BmpjMK6JfJ03VWA3nPK01jkVZA1a7/mZia3BOsdg= github.com/xgo-dev/plan9asm v0.3.5/go.mod h1:0yM4CCIp2PyT8h+Ro3Ukro3lHL8ji9mzHEv5yfhOckc= +github.com/zhouguangyuan0718/go-llvm v0.0.0-20260731084524-d8f8a2aa851e h1:O9Hz3E3WoZv6XXL4+4Gn1EkaEqN9+/F61WiwOJA5F5c= +github.com/zhouguangyuan0718/go-llvm v0.0.0-20260731084524-d8f8a2aa851e/go.mod h1:42vav2/cI5BAIcL543DZSMO9do8/aCK2z7JERH+AE+M= go.bug.st/serial v1.6.4 h1:7FmqNPgVp3pu2Jz5PoPtbZ9jJO5gnEnZIvnI1lzve8A= go.bug.st/serial v1.6.4/go.mod h1:nofMJxTeNVny/m6+KaafC6vJGj3miwQZ6vW4BZUGJPI= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= diff --git a/internal/build/build.go b/internal/build/build.go index eded0bcd49..14a2e4bbac 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -626,6 +626,7 @@ func Build(inv Invocation) ([]Package, error) { cTransformer: cabi.NewTransformer(prog, export.LLVMTarget, export.TargetABI, conf.AbiMode, cabiOptimize), } defer ctx.closePackageMetas() + defer ctx.closePackageArchiveBuffers() // default runtime globals must be registered before packages are built addGlobalString(conf, "runtime.defaultGOROOT="+runtime.GOROOT(), nil) @@ -962,12 +963,13 @@ func (c *context) hasAltPkg(pkgPath string) bool { return hasAltPkgForTarget(c.buildConf, pkgPath) } -// normalizeToArchive creates an archive from object files and sets ArchiveFile. +// normalizeToArchive creates an archive from file and memory members and sets ArchiveFile. // This ensures the link step always consumes .a archives regardless of cache state. func normalizeToArchive(ctx *context, aPkg *aPackage, verbose bool) error { - if len(aPkg.ObjFiles) == 0 { + if len(aPkg.ObjFiles) == 0 && len(aPkg.ObjBuffers) == 0 { return nil } + defer aPkg.disposeArchiveBuffers() archiveFile, err := os.CreateTemp("", "pkg-*.a") if err != nil { @@ -976,7 +978,7 @@ func normalizeToArchive(ctx *context, aPkg *aPackage, verbose bool) error { archiveFile.Close() archivePath := archiveFile.Name() - if err := ctx.createArchiveFile(archivePath, aPkg.ObjFiles, verbose); err != nil { + if err := ctx.createPackageArchiveFile(archivePath, aPkg, verbose); err != nil { os.Remove(archivePath) return fmt.Errorf("create archive for %s: %w", aPkg.PkgPath, err) } @@ -1896,11 +1898,15 @@ func compilePackageModule(ctx *context, aPkg *aPackage, externs []string, verbos aPkg.LinkArgs = append(aPkg.LinkArgs, goCgoLinkArgs(ctx.buildConf.Goos, aPkg.AltPkg.Syntax)...) } if pkg.ExportFile != "" { - exportFile, err := exportObject(ctx, pkg.PkgPath, pkg.ExportFile, ret) + exportFile, exportBuffer, err := exportPackageObject(ctx, pkg.PkgPath, pkg.ExportFile, ret) if err != nil { return fmt.Errorf("export object of %v failed: %v", pkgPath, err) } - aPkg.ObjFiles = append(aPkg.ObjFiles, exportFile) + if exportFile != "" { + aPkg.ObjFiles = append(aPkg.ObjFiles, exportFile) + } else { + aPkg.ObjBuffers = append(aPkg.ObjBuffers, exportBuffer) + } if debugBuild || verbose { fmt.Fprintf(os.Stderr, "==> Export %s: %s\n", aPkg.PkgPath, pkg.ExportFile) } @@ -1921,6 +1927,28 @@ func exportObject(ctx *context, pkgPath string, exportFile string, pkg llssa.Pac return exportObjectWithClang(ctx, pkgPath, exportFile, []byte(pkg.String())) } +func exportPackageObject(ctx *context, pkgPath string, exportFile string, pkg llssa.Package) (string, packageArchiveBuffer, error) { + if !useInMemoryNativeCodegen(ctx) { + path, err := exportObjectWithClang(ctx, pkgPath, exportFile, []byte(pkg.String())) + return path, packageArchiveBuffer{}, err + } + if ctx.buildConf.CheckLLFiles || ctx.buildConf.GenLL { + if err := dumpLLVMIRIfNeeded(ctx, pkgPath, exportFile, pkg.String()); err != nil { + return "", packageArchiveBuffer{}, err + } + } + buf, kind, err := emitObjectToMemoryBuffer(ctx, pkg) + if err != nil { + return "", packageArchiveBuffer{}, err + } + name := filepath.Base(exportFile) + ".o" + if ctx.shouldPrintCommands(false) { + fmt.Fprintf(os.Stderr, "# compiling archive member %s for pkg: %s\n", name, pkgPath) + fmt.Fprintf(os.Stderr, "# using %s\n", kind) + } + return "", packageArchiveBuffer{name: name, buffer: buf}, nil +} + func useInMemoryNativeCodegen(ctx *context) bool { return useInMemoryNativeCodegenConf(ctx.buildConf) } @@ -1977,6 +2005,15 @@ func exportObjectInMemory(ctx *context, pkgPath string, exportFile string, pkg l return "", err } } + buf, kind, err := emitObjectToMemoryBuffer(ctx, pkg) + if err != nil { + return "", err + } + defer buf.Dispose() + return writeObjectBufferToFile(ctx, pkgPath, exportFile, buf, kind) +} + +func emitObjectToMemoryBuffer(ctx *context, pkg llssa.Package) (gllvm.MemoryBuffer, string, error) { ltoMode := ctx.buildConf.ltoMode() var ( buf gllvm.MemoryBuffer @@ -1995,11 +2032,13 @@ func exportObjectInMemory(ctx *context, pkgPath string, exportFile string, pkg l default: buf, err = ctx.prog.TargetMachine().EmitToMemoryBuffer(pkg.Module(), gllvm.ObjectFile) if err != nil { - return "", err + return gllvm.MemoryBuffer{}, "", err } } - defer buf.Dispose() + return buf, kind, nil +} +func writeObjectBufferToFile(ctx *context, pkgPath, exportFile string, buf gllvm.MemoryBuffer, kind string) (string, error) { base := filepath.Base(exportFile) objFile, err := os.CreateTemp("", base+"-*.o") if err != nil { @@ -2172,8 +2211,9 @@ type aPackage struct { NeedPyInit bool LinkArgs []string - ObjFiles []string // object files: .o or .ll (output of compiler, input to archiver) - ArchiveFile string // archive file: .a (output of archiver, used for linking) + ObjFiles []string // file-backed archive members: .o or .ll + ObjBuffers []packageArchiveBuffer // LLVM-produced in-memory archive members + ArchiveFile string // archive file: .a (output of archiver, used for linking) Meta *meta.PackageMeta rewriteVars map[string]string diff --git a/internal/build/package_archive.go b/internal/build/package_archive.go new file mode 100644 index 0000000000..87b732d7f5 --- /dev/null +++ b/internal/build/package_archive.go @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "fmt" + "os" + "path/filepath" + + gllvm "github.com/xgo-dev/llvm" +) + +// packageArchiveBuffer owns an LLVM-produced archive member until package +// publication. +type packageArchiveBuffer struct { + name string + buffer gllvm.MemoryBuffer +} + +func (p *aPackage) disposeArchiveBuffers() { + for i := range p.ObjBuffers { + member := &p.ObjBuffers[i] + if member.buffer.IsNil() { + continue + } + member.buffer.Dispose() + member.buffer = gllvm.MemoryBuffer{} + } + p.ObjBuffers = nil +} + +func (c *context) closePackageArchiveBuffers() { + for _, pkg := range c.pkgs { + pkg.disposeArchiveBuffers() + } +} + +// createPackageArchiveFile writes path-backed auxiliary objects and +// LLVM-produced memory buffers into one archive. LLVM performs archive symbol +// indexing in-process; no temporary object is needed for memory members. +func (c *context) createPackageArchiveFile(archivePath string, pkg *aPackage, verbose bool) error { + if len(pkg.ObjFiles) == 0 && len(pkg.ObjBuffers) == 0 { + return fmt.Errorf("no object files provided for archive %s", archivePath) + } + if len(pkg.ObjBuffers) == 0 { + return c.createArchiveFile(archivePath, pkg.ObjFiles, verbose) + } + + if err := os.MkdirAll(filepath.Dir(archivePath), 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(archivePath), filepath.Base(archivePath)+".tmp-*") + if err != nil { + return err + } + tmpName := tmp.Name() + if err := tmp.Close(); err != nil { + os.Remove(tmpName) + return err + } + if err := os.Remove(tmpName); err != nil { + return err + } + + members := make([]gllvm.ArchiveMember, 0, len(pkg.ObjFiles)+len(pkg.ObjBuffers)) + for _, path := range pkg.ObjFiles { + members = append(members, gllvm.NewArchiveMemberFromFile(path)) + } + for _, member := range pkg.ObjBuffers { + members = append(members, gllvm.NewArchiveMemberFromMemoryBuffer(member.name, member.buffer)) + } + if c.shouldPrintCommands(verbose) { + fmt.Fprintf(os.Stderr, "# llvm archive %s (%d file members, %d memory members)\n", + tmpName, len(pkg.ObjFiles), len(pkg.ObjBuffers)) + } + if err := gllvm.WriteArchive(tmpName, c.targetTriple(), members); err != nil { + os.Remove(tmpName) + return fmt.Errorf("create archive %s: %w", archivePath, err) + } + if err := os.Rename(tmpName, archivePath); err != nil { + os.Remove(tmpName) + return fmt.Errorf("publish archive %s: %w", archivePath, err) + } + return nil +} diff --git a/internal/build/package_archive_test.go b/internal/build/package_archive_test.go new file mode 100644 index 0000000000..9668a9c650 --- /dev/null +++ b/internal/build/package_archive_test.go @@ -0,0 +1,123 @@ +//go:build !llgo +// +build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/goplus/llgo/internal/crosscompile" + "github.com/goplus/llgo/internal/packages" + gllvm "github.com/xgo-dev/llvm" +) + +func TestNormalizeToArchiveUsesMemoryBuffer(t *testing.T) { + llvmCtx := gllvm.NewContext() + defer llvmCtx.Dispose() + mod := llvmCtx.NewModule("archive") + defer mod.Dispose() + mod.SetTarget("x86_64-unknown-linux-gnu") + gllvm.AddFunction(mod, "memory_symbol", gllvm.FunctionType(llvmCtx.Int32Type(), nil, false)) + + memoryBuf := gllvm.WriteBitcodeToMemoryBuffer(mod) + ctx := &context{ + buildConf: &Config{Goos: "linux", Goarch: "amd64"}, + crossCompile: crosscompile.Export{ + LLVMTarget: "x86_64-unknown-linux-gnu", + }, + } + fileBuf := gllvm.WriteBitcodeToMemoryBuffer(mod) + defer fileBuf.Dispose() + filePath := filepath.Join(t.TempDir(), "file-member.bc") + if err := os.WriteFile(filePath, fileBuf.Bytes(), 0o644); err != nil { + memoryBuf.Dispose() + t.Fatal(err) + } + + pkg := &aPackage{ + Package: &packages.Package{PkgPath: "example.com/archive"}, + ObjFiles: []string{filePath}, + ObjBuffers: []packageArchiveBuffer{{ + name: "memory-member.bc", + buffer: memoryBuf, + }}, + } + if err := normalizeToArchive(ctx, pkg, false); err != nil { + t.Fatal(err) + } + defer os.Remove(pkg.ArchiveFile) + + if len(pkg.ObjFiles) != 0 || len(pkg.ObjBuffers) != 0 { + t.Fatalf("package members were not released: files=%v buffers=%v", pkg.ObjFiles, pkg.ObjBuffers) + } + data, err := os.ReadFile(pkg.ArchiveFile) + if err != nil { + t.Fatal(err) + } + if !bytes.HasPrefix(data, []byte("!\n")) { + t.Fatalf("archive has invalid magic: %q", data[:8]) + } + for _, name := range []string{"file-member.bc", "memory-member.bc"} { + if !bytes.Contains(data, []byte(name)) { + t.Errorf("archive does not contain member %q", name) + } + } +} + +func TestNormalizeToArchiveFailsWithoutObjectFallback(t *testing.T) { + llvmCtx := gllvm.NewContext() + defer llvmCtx.Dispose() + mod := llvmCtx.NewModule("archive-error") + defer mod.Dispose() + mod.SetTarget("x86_64-unknown-linux-gnu") + gllvm.AddFunction(mod, "memory_symbol", gllvm.FunctionType(llvmCtx.Int32Type(), nil, false)) + + memoryBuf := gllvm.WriteBitcodeToMemoryBuffer(mod) + ctx := &context{ + buildConf: &Config{Goos: "linux", Goarch: "amd64"}, + crossCompile: crosscompile.Export{ + LLVMTarget: "x86_64-unknown-linux-gnu", + }, + } + pkg := &aPackage{ + Package: &packages.Package{PkgPath: "example.com/archive-error"}, + ObjFiles: []string{filepath.Join(t.TempDir(), "missing.o")}, + ObjBuffers: []packageArchiveBuffer{{ + name: "memory-member.bc", + buffer: memoryBuf, + }}, + } + err := normalizeToArchive(ctx, pkg, false) + if err == nil { + t.Fatal("normalizeToArchive succeeded with a missing member") + } + if !strings.Contains(err.Error(), "missing.o") { + t.Fatalf("normalizeToArchive error = %v, want missing member", err) + } + if len(pkg.ObjBuffers) != 0 { + t.Fatal("memory buffer was retained after archive failure") + } + if pkg.ArchiveFile != "" { + t.Fatalf("ArchiveFile = %q after failure", pkg.ArchiveFile) + } +} From dcb891d70c60c7a563dca13fc0361ce7898c4455 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Thu, 30 Jul 2026 08:09:16 +0800 Subject: [PATCH 13/22] build: snapshot package linker state --- internal/build/build.go | 99 ++++++----- internal/build/collect.go | 19 +- internal/build/collect_test.go | 86 +++++++++- internal/build/fingerprint.go | 7 +- internal/build/funcinfo_table.go | 37 ++-- internal/build/package_build_test.go | 26 ++- internal/build/package_summary.go | 229 +++++++++++++++++++++++++ internal/build/package_summary_test.go | 93 ++++++++++ 8 files changed, 529 insertions(+), 67 deletions(-) create mode 100644 internal/build/package_summary.go create mode 100644 internal/build/package_summary_test.go diff --git a/internal/build/build.go b/internal/build/build.go index 14a2e4bbac..6617d5233c 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -1050,14 +1050,16 @@ func preflightPackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (s ctx.built[pkg.ID] = none{} if spec.isDeclOnly() { pkg.ExportFile = "" - return true, nil + aPkg.Summary = summarizePackage(aPkg) + return true, ctx.collectFingerprint(aPkg) } if spec.isLinkOnly() && !spec.hasSource() { pkg.ExportFile = "" if spec.kind == cl.PkgLinkExtern { appendExternalLinkArgs(ctx, aPkg, spec.kindParam) } - return true, nil + aPkg.Summary = summarizePackage(aPkg) + return true, ctx.collectFingerprint(aPkg) } if err := ctx.collectFingerprint(aPkg); err != nil { return false, err @@ -1098,6 +1100,7 @@ func finalizePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (pa if spec.kind == cl.PkgLinkExtern { appendExternalLinkArgs(ctx, aPkg, spec.kindParam) } + aPkg.Summary = summarizePackage(aPkg) if err := ctx.saveToCache(aPkg); err != nil && verbose { fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", aPkg.PkgPath, err) } @@ -1348,39 +1351,47 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa aPkg = ctx.pkgByID[p.ID] } if p.ExportFile != "" && aPkg != nil { // skip packages that only contain declarations + if aPkg.Summary == nil && isRuntimePkg(aPkg.PkgPath) { + return + } linkedPkgs[p.ID] = true linkedOrder = append(linkedOrder, aPkg) } }) + linkedSummaries := make([]*PackageSummary, len(linkedOrder)) + for i, aPkg := range linkedOrder { + if aPkg.Summary == nil { + return fmt.Errorf("package %s has no linker summary", aPkg.PkgPath) + } + linkedSummaries[i] = aPkg.Summary + } // packages.Visit with a post callback yields dependencies before importers. // Reverse that order so static archives are linked after the objects that use them. for i := len(linkedOrder) - 1; i >= 0; i-- { - aPkg := linkedOrder[i] - p := aPkg.Package + summary := linkedSummaries[i] // Defer linking runtime packages unless we actually need the runtime. - if isRuntimePkg(p.PkgPath) { - rtLinkArgs = append(rtLinkArgs, aPkg.LinkArgs...) - if aPkg.ArchiveFile != "" { - rtLinkInputs = append(rtLinkInputs, aPkg.ArchiveFile) + if isRuntimePkg(summary.PkgPath) { + rtLinkArgs = append(rtLinkArgs, summary.LinkArgs...) + if summary.ArchiveFile != "" { + rtLinkInputs = append(rtLinkInputs, summary.ArchiveFile) } continue } // Only let non-runtime packages influence whether runtime is needed. - need1, need2 := aPkg.isNeedRuntimeOrPyInit() - needRuntime = needRuntime || need1 - needPyInit = needPyInit || need2 - needAbiInit |= aPkg.LPkg.NeedAbiInit - for k, _ := range aPkg.LPkg.MethodByIndex { - methodByIndex[k] = none{} + needRuntime = needRuntime || summary.NeedRuntime + needPyInit = needPyInit || summary.NeedPyInit + needAbiInit |= summary.NeedAbiInit + for _, method := range summary.MethodByIndex { + methodByIndex[method] = none{} } - for k, _ := range aPkg.LPkg.MethodByName { - methodByName[k] = none{} + for _, method := range summary.MethodByName { + methodByName[method] = none{} } - linkArgs = append(linkArgs, aPkg.LinkArgs...) - if aPkg.ArchiveFile != "" { - archiveInputs = append(archiveInputs, aPkg.ArchiveFile) + linkArgs = append(linkArgs, summary.LinkArgs...) + if summary.ArchiveFile != "" { + archiveInputs = append(archiveInputs, summary.ArchiveFile) } } @@ -1397,9 +1408,9 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa var pcLineInfo []pcLineRecord var funcInfoStubs []funcInfoStubRecord if ctx.buildConf.PCLNMode != PCLNNone { - funcInfo = prepareFuncInfoTableRecords(collectFuncInfo(linkedOrder), nil) - pcLineInfo = collectPCLineInfo(linkedOrder) - funcInfoStubs = collectFuncInfoStubRecords(linkedOrder, funcInfo) + funcInfo = prepareFuncInfoTableRecords(collectFuncInfoSummaries(linkedSummaries), nil) + pcLineInfo = collectPCLineInfoSummaries(linkedSummaries) + funcInfoStubs = collectFuncInfoStubRecordsSummaries(linkedSummaries, funcInfo) } entryPkg := genMainModule(ctx, llssa.PkgRuntime, pkg, &genConfig{ rtInit: needRuntime, @@ -1407,7 +1418,7 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa abiInit: needAbiInit, methodByIndex: methodByIndex, methodByName: methodByName, - abiSymbols: linkedModuleGlobals(linkedOrder), + abiSymbols: linkedPackageGlobals(linkedSummaries), funcInfo: funcInfo, pcLineInfo: pcLineInfo, funcInfoStubs: funcInfoStubs, @@ -1442,7 +1453,7 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa } } } - linkArgs = append(linkArgs, cSharedExportArgs(ctx, linkedOrder)...) + linkArgs = append(linkArgs, cSharedExportArgsSummaries(ctx, linkedSummaries)...) err = linkObjFiles(ctx, outputPath, linkInputs, linkArgs, verbose) if err != nil { @@ -1453,19 +1464,22 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa } func linkedModuleGlobals(pkgs []Package) map[string]none { - if len(pkgs) == 0 { + return linkedPackageGlobals(summariesForPackages(pkgs)) +} + +func linkedPackageGlobals(summaries []*PackageSummary) map[string]none { + if len(summaries) == 0 { return nil } seen := make(map[string]none) - for _, pkg := range pkgs { - if pkg == nil || pkg.LPkg == nil { + for _, summary := range summaries { + if summary == nil { continue } - for g := pkg.LPkg.Module().FirstGlobal(); !g.IsNil(); g = gllvm.NextGlobal(g) { - if g.IsDeclaration() { - continue + for _, name := range summary.GlobalSymbols { + if name != "" { + seen[name] = none{} } - seen[g.Name()] = none{} } } return seen @@ -1544,22 +1558,26 @@ func linkObjFiles(ctx *context, app string, objFiles, linkArgs []string, verbose // shared-library link roots. They live in package archives and otherwise remain // unreferenced, so the linker can omit both their object files and symbols. func cSharedExportArgs(ctx *context, pkgs []*aPackage) []string { + return cSharedExportArgsSummaries(ctx, summariesForPackages(pkgs)) +} + +func cSharedExportArgsSummaries(ctx *context, summaries []*PackageSummary) []string { if ctx == nil || ctx.buildConf == nil || ctx.buildConf.BuildMode != BuildModeCShared { return nil } exports := make(map[string]none) - for _, pkg := range pkgs { - if pkg == nil || pkg.LPkg == nil { + for _, summary := range summaries { + if summary == nil { continue } - for _, name := range pkg.LPkg.ExportFuncs() { + for _, name := range summary.CSharedExports { if name != "" { exports[name] = none{} } } - if ctx.mode == ModeTest && pkg.Package != nil && pkg.Name == "main" && strings.HasSuffix(pkg.PkgPath, ".test") { - exports[pkg.PkgPath+".init"] = none{} - exports[pkg.PkgPath+".main"] = none{} + if ctx.mode == ModeTest && summary.Name == "main" && strings.HasSuffix(summary.PkgPath, ".test") { + exports[summary.PkgPath+".init"] = none{} + exports[summary.PkgPath+".main"] = none{} } } names := make([]string, 0, len(exports)) @@ -2203,9 +2221,10 @@ func registerAltSSAPkgs(prog *ssa.Program, patches cl.Patches, alts []*packages. type aPackage struct { *packages.Package - SSA *ssa.Package - AltPkg *packages.Cached - LPkg llssa.Package + SSA *ssa.Package + AltPkg *packages.Cached + LPkg llssa.Package + Summary *PackageSummary NeedRt bool NeedPyInit bool diff --git a/internal/build/collect.go b/internal/build/collect.go index d8dcee7b1f..7d3c65166b 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -334,6 +334,9 @@ func (c *context) tryLoadFromCache(pkg *aPackage) bool { if c.packageCacheDisabled(pkg.ID) { return false } + if c.buildConf != nil && (c.buildConf.BuildMode == BuildModeCArchive || c.buildConf.BuildMode == BuildModeCShared) { + return false + } // Main packages are intentionally not written to the build cache because // each executable's entry module is linked against the current main archive. @@ -377,6 +380,9 @@ func (c *context) tryLoadFromCache(pkg *aPackage) bool { if err != nil { return false } + if meta.Summary == nil { + return false + } // Use the .a archive directly for linking (no extraction needed) pkg.ArchiveFile = paths.Archive @@ -385,6 +391,7 @@ func (c *context) tryLoadFromCache(pkg *aPackage) bool { pkg.NeedPyInit = meta.NeedPyInit pkg.Meta = pkgMeta pkg.CacheHit = true + pkg.Summary = summaryFromMetadata(pkg, meta) return true } @@ -399,6 +406,7 @@ func parseManifestMetadata(content string) (*cacheArchiveMetadata, error) { meta.LinkArgs = append([]string(nil), data.Metadata.LinkArgs...) meta.NeedRt = data.Metadata.NeedRt meta.NeedPyInit = data.Metadata.NeedPyInit + meta.Summary = data.Metadata.Summary } return meta, nil } @@ -452,6 +460,7 @@ type cacheArchiveMetadata struct { LinkArgs []string NeedRt bool NeedPyInit bool + Summary *packageSummaryMetadata } // saveToCache saves a built package to cache. @@ -511,16 +520,16 @@ func (c *context) saveToCache(pkg *aPackage) error { return fmt.Errorf("decode manifest: %w", err) } + if pkg.Summary == nil { + pkg.Summary = summarizePackage(pkg) + } meta := &manifestMetadata{ LinkArgs: append([]string(nil), pkg.LinkArgs...), NeedRt: pkg.NeedRt, NeedPyInit: pkg.NeedPyInit, + Summary: pkg.Summary.metadata(), } - if len(meta.LinkArgs) == 0 && !meta.NeedRt && !meta.NeedPyInit { - data.Metadata = nil - } else { - data.Metadata = meta - } + data.Metadata = meta manifestWithMeta, err := buildManifestYAML(data) if err != nil { diff --git a/internal/build/collect_test.go b/internal/build/collect_test.go index 5b6294f1f4..879b4b6511 100644 --- a/internal/build/collect_test.go +++ b/internal/build/collect_test.go @@ -22,6 +22,7 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "runtime" "strings" "testing" @@ -1031,8 +1032,8 @@ func TestSaveToCache_Success(t *testing.T) { if data.Env.Goos != "darwin" { t.Errorf("manifest should contain original env content") } - if data.Metadata != nil { - t.Errorf("metadata should be empty when no link args/runtime flags") + if data.Metadata == nil || data.Metadata.Summary == nil { + t.Errorf("metadata should preserve an empty linker summary") } // Check archive exists @@ -1206,6 +1207,7 @@ func TestTryLoadFromCacheIgnoresMetaWhenPackageMetaDisabled(t *testing.T) { m := newManifestBuilder() m.env.Goos = "darwin" m.pkg.PkgPath = "example.com/nometa" + m.meta = &manifestMetadata{Summary: &packageSummaryMetadata{}} if err := writeManifest(paths.Manifest, m.Build()); err != nil { t.Fatal(err) } @@ -1221,6 +1223,86 @@ func TestTryLoadFromCacheIgnoresMetaWhenPackageMetaDisabled(t *testing.T) { } } +func TestTryLoadFromCacheRestoresPackageSummary(t *testing.T) { + td := t.TempDir() + oldFunc := cacheRootFunc + cacheRootFunc = func() string { return td } + defer func() { cacheRootFunc = oldFunc }() + + ctx := &context{ + conf: &packages.Config{}, + buildConf: &Config{Goos: "darwin", Goarch: "arm64"}, + crossCompile: crosscompile.Export{ + LLVMTarget: "arm64-apple-darwin", + }, + } + pkg := &aPackage{ + Package: &packages.Package{ + ID: "example.com/cached", + PkgPath: "example.com/cached", + Name: "cached", + }, + Fingerprint: "summary-test", + Manifest: func() string { + m := newManifestBuilder() + m.env.Goos = "darwin" + m.pkg.PkgPath = "example.com/cached" + return m.Build() + }(), + NeedRt: true, + NeedPyInit: true, + LinkArgs: []string{"-lcached"}, + Summary: &PackageSummary{ + ID: "example.com/cached", + PkgPath: "example.com/cached", + Name: "cached", + LinkArgs: []string{"-lcached"}, + NeedRuntime: true, + NeedPyInit: true, + NeedAbiInit: 3, + MethodByIndex: []int{1}, + MethodByName: []string{"Method"}, + GlobalSymbols: []string{"example.com/cached.global"}, + FuncInfo: []funcInfoRecord{{symbol: "example.com/cached.fn", name: "Fn", file: "p.go", line: 7}}, + PCLineInfo: []pcLineRecord{{id: 9, symbol: "example.com/cached.fn", file: "p.go", line: 8}}, + FuncInfoStubs: []string{closureStubPrefix + "example.com/cached.fn"}, + CSharedExports: []string{"Cached"}, + }, + } + want := *pkg.Summary + obj, err := os.CreateTemp(td, "cached-*.o") + if err != nil { + t.Fatal(err) + } + if _, err := obj.WriteString("object"); err != nil { + t.Fatal(err) + } + if err := obj.Close(); err != nil { + t.Fatal(err) + } + pkg.ObjFiles = []string{obj.Name()} + if err := ctx.saveToCache(pkg); err != nil { + t.Fatalf("saveToCache: %v", err) + } + + pkg.ObjFiles = nil + pkg.ArchiveFile = "" + pkg.LinkArgs = nil + pkg.NeedRt = false + pkg.NeedPyInit = false + pkg.Summary = nil + if !ctx.tryLoadFromCache(pkg) { + t.Fatal("tryLoadFromCache = false, want summary cache hit") + } + if pkg.Summary == nil { + t.Fatal("cache hit did not restore package summary") + } + want.ArchiveFile = pkg.ArchiveFile + if !reflect.DeepEqual(pkg.Summary, &want) { + t.Fatalf("restored summary = %#v, want %#v", pkg.Summary, &want) + } +} + func TestGetLLVMVersion(t *testing.T) { ctx := &context{ crossCompile: crosscompile.Export{}, diff --git a/internal/build/fingerprint.go b/internal/build/fingerprint.go index ecc55298ab..781ec180f6 100644 --- a/internal/build/fingerprint.go +++ b/internal/build/fingerprint.go @@ -37,9 +37,10 @@ type depEntry struct { // manifestMetadata stores metadata produced during build but not part of the fingerprint. type manifestMetadata struct { - LinkArgs []string `yaml:"link_args,omitempty"` - NeedRt bool `yaml:"need_rt,omitempty"` - NeedPyInit bool `yaml:"need_py_init,omitempty"` + LinkArgs []string `yaml:"link_args,omitempty"` + NeedRt bool `yaml:"need_rt,omitempty"` + NeedPyInit bool `yaml:"need_py_init,omitempty"` + Summary *packageSummaryMetadata `yaml:"summary,omitempty"` } // manifestData is the structured representation of manifest content. diff --git a/internal/build/funcinfo_table.go b/internal/build/funcinfo_table.go index b36aa29654..f86fd972a1 100644 --- a/internal/build/funcinfo_table.go +++ b/internal/build/funcinfo_table.go @@ -86,12 +86,16 @@ type funcInfoSymbolIndexRecord struct { } func collectFuncInfo(pkgs []Package) []funcInfoRecord { + return collectFuncInfoSummaries(summariesForPackages(pkgs)) +} + +func collectFuncInfoSummaries(summaries []*PackageSummary) []funcInfoRecord { seen := make(map[string]funcInfoRecord) - for _, pkg := range pkgs { - if pkg == nil || pkg.LPkg == nil { + for _, summary := range summaries { + if summary == nil { continue } - for _, rec := range readFuncInfo(pkg.LPkg.Module()) { + for _, rec := range summary.FuncInfo { if rec.symbol == "" { continue } @@ -114,13 +118,17 @@ func collectFuncInfo(pkgs []Package) []funcInfoRecord { } func collectPCLineInfo(pkgs []Package) []pcLineRecord { + return collectPCLineInfoSummaries(summariesForPackages(pkgs)) +} + +func collectPCLineInfoSummaries(summaries []*PackageSummary) []pcLineRecord { var out []pcLineRecord seen := make(map[uint64]none) - for _, pkg := range pkgs { - if pkg == nil || pkg.LPkg == nil { + for _, summary := range summaries { + if summary == nil { continue } - for _, rec := range readPCLineInfo(pkg.LPkg.Module()) { + for _, rec := range summary.PCLineInfo { if rec.id == 0 || rec.symbol == "" { continue } @@ -144,6 +152,10 @@ func collectPCLineInfo(pkgs []Package) []pcLineRecord { } func collectFuncInfoStubRecords(pkgs []Package, records []funcInfoRecord) []funcInfoStubRecord { + return collectFuncInfoStubRecordsSummaries(summariesForPackages(pkgs), records) +} + +func collectFuncInfoStubRecordsSummaries(summaries []*PackageSummary, records []funcInfoRecord) []funcInfoStubRecord { if len(records) == 0 { return nil } @@ -154,23 +166,16 @@ func collectFuncInfoStubRecords(pkgs []Package, records []funcInfoRecord) []func } } seen := make(map[string]funcInfoStubRecord) - for _, pkg := range pkgs { - if pkg == nil || pkg.LPkg == nil { + for _, summary := range summaries { + if summary == nil { continue } - fn := pkg.LPkg.Module().FirstFunction() - for !fn.IsNil() { - if fn.IsDeclaration() || fn.BasicBlocksCount() == 0 { - fn = llvm.NextFunction(fn) - continue - } - name := fn.Name() + for _, name := range summary.FuncInfoStubs { if target, ok := strings.CutPrefix(name, closureStubPrefix); ok { if idx := recordBySymbol[target]; idx != 0 { seen[name] = funcInfoStubRecord{symbol: name, funcIndex: idx} } } - fn = llvm.NextFunction(fn) } } if len(seen) == 0 { diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go index a9a940beef..ad97feab3f 100644 --- a/internal/build/package_build_test.go +++ b/internal/build/package_build_test.go @@ -91,7 +91,11 @@ func TestPreflightPackageBuildSkipsDeclarationOnlyPackage(t *testing.T) { Types: types.Unsafe, ExportFile: "stale.a", }} - ctx := &context{built: make(map[string]none)} + ctx := &context{ + conf: &packages.Config{}, + buildConf: &Config{Goos: "linux", Goarch: "amd64", ForceRebuild: true}, + built: make(map[string]none), + } skip, err := preflightPackageBuild(ctx, newPackageBuildSpec(pkg), false) if err != nil { @@ -123,3 +127,23 @@ func TestFinalizePackageBuildReturnsCachedResult(t *testing.T) { t.Fatalf("build result = %+v, want Python initialization requirement preserved", result) } } + +func TestPreflightFingerprintsSkippedPackage(t *testing.T) { + pkg := &aPackage{Package: &packages.Package{ + ID: "unsafe", + PkgPath: "unsafe", + Types: types.Unsafe, + }} + ctx := &context{ + conf: &packages.Config{}, + buildConf: &Config{Goos: "linux", Goarch: "amd64", ForceRebuild: true}, + built: make(map[string]none), + } + skip, err := preflightPackageBuild(ctx, newPackageBuildSpec(pkg), false) + if err != nil { + t.Fatal(err) + } + if !skip || pkg.Fingerprint == "" || pkg.Manifest == "" || pkg.Summary == nil { + t.Fatalf("skipped package was not fully prepared: skip=%v fingerprint=%q manifest=%q summary=%#v", skip, pkg.Fingerprint, pkg.Manifest, pkg.Summary) + } +} diff --git a/internal/build/package_summary.go b/internal/build/package_summary.go new file mode 100644 index 0000000000..76c1becca6 --- /dev/null +++ b/internal/build/package_summary.go @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "sort" + "strings" + + "github.com/xgo-dev/llvm" +) + +// PackageSummary is the immutable, LLVM-free output one package contributes +// to the final link. A backend can emit its object, capture this summary, and +// release its Program and LLVM context before whole-program linking starts. +// +// C archive/shared header declarations still consume LPkg and therefore stay +// on the serial compatibility path. +type PackageSummary struct { + ID string + PkgPath string + Name string + + LinkArgs []string + ArchiveFile string + NeedRuntime bool + NeedPyInit bool + + NeedAbiInit int + MethodByIndex []int + MethodByName []string + GlobalSymbols []string + + FuncInfo []funcInfoRecord + PCLineInfo []pcLineRecord + FuncInfoStubs []string + CSharedExports []string +} + +type packageSummaryMetadata struct { + NeedAbiInit int `yaml:"need_abi_init,omitempty"` + MethodByIndex []int `yaml:"method_by_index,omitempty"` + MethodByName []string `yaml:"method_by_name,omitempty"` + GlobalSymbols []string `yaml:"global_symbols,omitempty"` + + FuncInfo []funcInfoMetadata `yaml:"func_info,omitempty"` + PCLineInfo []pcLineMetadata `yaml:"pcline_info,omitempty"` + FuncInfoStubs []string `yaml:"func_info_stubs,omitempty"` + CSharedExports []string `yaml:"c_shared_exports,omitempty"` +} + +type funcInfoMetadata struct { + Symbol string `yaml:"symbol"` + Name string `yaml:"name,omitempty"` + File string `yaml:"file,omitempty"` + Line uint32 `yaml:"line,omitempty"` + Column uint32 `yaml:"column,omitempty"` +} + +type pcLineMetadata struct { + ID uint64 `yaml:"id"` + Symbol string `yaml:"symbol"` + File string `yaml:"file,omitempty"` + Line uint32 `yaml:"line,omitempty"` + Column uint32 `yaml:"column,omitempty"` +} + +func summarizePackage(pkg *aPackage) *PackageSummary { + if pkg == nil { + return nil + } + summary := &PackageSummary{ + LinkArgs: append([]string(nil), pkg.LinkArgs...), + ArchiveFile: pkg.ArchiveFile, + NeedRuntime: pkg.NeedRt, + NeedPyInit: pkg.NeedPyInit, + } + if pkg.Package != nil { + summary.ID = pkg.ID + summary.PkgPath = pkg.PkgPath + summary.Name = pkg.Name + } + if pkg.LPkg == nil { + return summary + } + + lpkg := pkg.LPkg + if summary.PkgPath == "" { + summary.PkgPath = lpkg.Path() + } + summary.NeedAbiInit = lpkg.NeedAbiInit + for method := range lpkg.MethodByIndex { + summary.MethodByIndex = append(summary.MethodByIndex, method) + } + sort.Ints(summary.MethodByIndex) + for method := range lpkg.MethodByName { + summary.MethodByName = append(summary.MethodByName, method) + } + sort.Strings(summary.MethodByName) + + mod := lpkg.Module() + for global := mod.FirstGlobal(); !global.IsNil(); global = llvm.NextGlobal(global) { + if !global.IsDeclaration() { + summary.GlobalSymbols = append(summary.GlobalSymbols, global.Name()) + } + } + sort.Strings(summary.GlobalSymbols) + summary.FuncInfo = readFuncInfo(mod) + summary.PCLineInfo = readPCLineInfo(mod) + for fn := mod.FirstFunction(); !fn.IsNil(); fn = llvm.NextFunction(fn) { + if fn.IsDeclaration() || fn.BasicBlocksCount() == 0 { + continue + } + if _, ok := strings.CutPrefix(fn.Name(), closureStubPrefix); ok { + summary.FuncInfoStubs = append(summary.FuncInfoStubs, fn.Name()) + } + } + sort.Strings(summary.FuncInfoStubs) + for _, name := range lpkg.ExportFuncs() { + if name != "" { + summary.CSharedExports = append(summary.CSharedExports, name) + } + } + sort.Strings(summary.CSharedExports) + return summary +} + +func (s *PackageSummary) metadata() *packageSummaryMetadata { + if s == nil { + return nil + } + meta := &packageSummaryMetadata{ + NeedAbiInit: s.NeedAbiInit, + MethodByIndex: append([]int(nil), s.MethodByIndex...), + MethodByName: append([]string(nil), s.MethodByName...), + GlobalSymbols: append([]string(nil), s.GlobalSymbols...), + FuncInfoStubs: append([]string(nil), s.FuncInfoStubs...), + CSharedExports: append([]string(nil), s.CSharedExports...), + } + for _, rec := range s.FuncInfo { + meta.FuncInfo = append(meta.FuncInfo, funcInfoMetadata{ + Symbol: rec.symbol, + Name: rec.name, + File: rec.file, + Line: rec.line, + Column: rec.column, + }) + } + for _, rec := range s.PCLineInfo { + meta.PCLineInfo = append(meta.PCLineInfo, pcLineMetadata{ + ID: rec.id, + Symbol: rec.symbol, + File: rec.file, + Line: rec.line, + Column: rec.column, + }) + } + return meta +} + +func summaryFromMetadata(pkg *aPackage, meta *cacheArchiveMetadata) *PackageSummary { + if pkg == nil || pkg.Package == nil || meta == nil || meta.Summary == nil { + return nil + } + summary := &PackageSummary{ + ID: pkg.ID, + PkgPath: pkg.PkgPath, + Name: pkg.Name, + LinkArgs: append([]string(nil), pkg.LinkArgs...), + ArchiveFile: pkg.ArchiveFile, + NeedRuntime: pkg.NeedRt, + NeedPyInit: pkg.NeedPyInit, + NeedAbiInit: meta.Summary.NeedAbiInit, + MethodByIndex: append([]int(nil), meta.Summary.MethodByIndex...), + MethodByName: append([]string(nil), meta.Summary.MethodByName...), + GlobalSymbols: append([]string(nil), meta.Summary.GlobalSymbols...), + FuncInfoStubs: append([]string(nil), meta.Summary.FuncInfoStubs...), + CSharedExports: append([]string(nil), meta.Summary.CSharedExports...), + } + for _, rec := range meta.Summary.FuncInfo { + summary.FuncInfo = append(summary.FuncInfo, funcInfoRecord{ + symbol: rec.Symbol, + name: rec.Name, + file: rec.File, + line: rec.Line, + column: rec.Column, + }) + } + for _, rec := range meta.Summary.PCLineInfo { + summary.PCLineInfo = append(summary.PCLineInfo, pcLineRecord{ + id: rec.ID, + symbol: rec.Symbol, + file: rec.File, + line: rec.Line, + column: rec.Column, + }) + } + return summary +} + +func summariesForPackages(pkgs []Package) []*PackageSummary { + summaries := make([]*PackageSummary, 0, len(pkgs)) + for _, pkg := range pkgs { + if pkg == nil { + continue + } + summary := pkg.Summary + if summary == nil { + summary = summarizePackage(pkg) + } + if summary != nil { + summaries = append(summaries, summary) + } + } + return summaries +} diff --git a/internal/build/package_summary_test.go b/internal/build/package_summary_test.go new file mode 100644 index 0000000000..0e27ad6f56 --- /dev/null +++ b/internal/build/package_summary_test.go @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "go/types" + "reflect" + "testing" + + "github.com/xgo-dev/llvm" + + "github.com/goplus/llgo/internal/packages" + llssa "github.com/goplus/llgo/ssa" +) + +func TestPackageSummaryCapturesLinkerFacts(t *testing.T) { + prog := llssa.NewProgram(nil) + defer prog.Dispose() + lpkg := prog.NewPackage("p", "example.com/p") + lpkg.NeedAbiInit = 3 + lpkg.RecordReflectMethodByIndex("example.com/p.Method", 4) + lpkg.RecordReflectMethodByIndex("example.com/p.Method", 1) + lpkg.RecordReflectMethodByName("example.com/p.MethodByName", "B") + lpkg.RecordReflectMethodByName("example.com/p.MethodByName", "A") + lpkg.SetExport("example.com/p.Export", "Export") + lpkg.EmitFuncInfo("example.com/p.live", "example.com/p.Live", "p.go", 17, 2) + lpkg.EmitPCLineInfo(42, "example.com/p.live", "p.go", 18, 3) + lpkg.NewFunc(closureStubPrefix+"example.com/p.live", llssa.NoArgsNoRet, llssa.InGo).MakeBody(1).Return() + + i32 := lpkg.Module().Context().Int32Type() + defined := llvm.AddGlobal(lpkg.Module(), i32, "example.com/p.defined") + defined.SetInitializer(llvm.ConstInt(i32, 1, false)) + llvm.AddGlobal(lpkg.Module(), i32, "example.com/p.declared") + + pkg := &aPackage{ + Package: &packages.Package{ + ID: "example.com/p", + PkgPath: "example.com/p", + Name: "p", + Types: types.NewPackage("example.com/p", "p"), + }, + LPkg: lpkg, + NeedRt: true, + NeedPyInit: true, + LinkArgs: []string{"-lp"}, + ArchiveFile: "p.a", + } + summary := summarizePackage(pkg) + if got, want := summary.MethodByIndex, []int{1, 4}; !reflect.DeepEqual(got, want) { + t.Fatalf("MethodByIndex = %v, want %v", got, want) + } + if got, want := summary.MethodByName, []string{"A", "B"}; !reflect.DeepEqual(got, want) { + t.Fatalf("MethodByName = %v, want %v", got, want) + } + if got, want := summary.GlobalSymbols, []string{"example.com/p.defined"}; !reflect.DeepEqual(got, want) { + t.Fatalf("GlobalSymbols = %v, want %v", got, want) + } + if got, want := summary.FuncInfoStubs, []string{closureStubPrefix + "example.com/p.live"}; !reflect.DeepEqual(got, want) { + t.Fatalf("FuncInfoStubs = %v, want %v", got, want) + } + if got := collectFuncInfoSummaries([]*PackageSummary{summary}); len(got) != 1 || got[0].symbol != "example.com/p.live" { + t.Fatalf("func info from summary = %+v, want live record", got) + } + if got := linkedPackageGlobals([]*PackageSummary{summary}); len(got) != 1 { + t.Fatalf("globals from summary = %#v, want one defined global", got) + } + + loadedPkg := &aPackage{ + Package: pkg.Package, + LinkArgs: []string{"-lp"}, + ArchiveFile: "p.a", + NeedRt: true, + NeedPyInit: true, + } + loaded := summaryFromMetadata(loadedPkg, &cacheArchiveMetadata{Summary: summary.metadata()}) + if !reflect.DeepEqual(loaded, summary) { + t.Fatalf("cache summary round trip = %#v, want %#v", loaded, summary) + } +} From fb7f717cfb96363de47a1833852171c3b8ab52e5 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Thu, 30 Jul 2026 08:12:34 +0800 Subject: [PATCH 14/22] build: isolate package backend sessions --- cl/caller_tracking_precompute_test.go | 76 +++++++++ cl/instr.go | 56 ++++++- internal/build/backend_program_test.go | 57 +++++++ internal/build/build.go | 204 ++++++++++++++++++++++--- 4 files changed, 373 insertions(+), 20 deletions(-) create mode 100644 cl/caller_tracking_precompute_test.go create mode 100644 internal/build/backend_program_test.go diff --git a/cl/caller_tracking_precompute_test.go b/cl/caller_tracking_precompute_test.go new file mode 100644 index 0000000000..de78c7dcb9 --- /dev/null +++ b/cl/caller_tracking_precompute_test.go @@ -0,0 +1,76 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "sync" + "testing" + + gossa "golang.org/x/tools/go/ssa" +) + +func TestCallerTrackingPrecomputeFreezesConcurrentReads(t *testing.T) { + dep, root := buildCallerFrameSSAProgram(t, + "example.com/dep", `package dep +import "runtime" +func Where() { runtime.Caller(0) } +`, + "example.com/root", `package root +import "example.com/dep" +func Logs() { dep.Where() } +`) + tracking := NewCallerTracking() + tracking.Precompute([]*gossa.Package{root}) + if !tracking.frozen { + t.Fatal("CallerTracking was not frozen after precomputation") + } + if !runtimeCallerBaseSet(tracking, dep)[dep.Func("Where")] { + t.Fatal("precomputed base set lost runtime caller function") + } + if !runtimeCallerFuncSet(tracking, root)[root.Func("Logs")] { + t.Fatal("precomputed extended set lost cross-package caller") + } + + var wg sync.WaitGroup + errs := make(chan struct{}, 32) + for range 32 { + wg.Add(1) + go func() { + defer wg.Done() + if !runtimeCallerBaseSet(tracking, dep)[dep.Func("Where")] || + !runtimeCallerFuncSet(tracking, root)[root.Func("Logs")] { + errs <- struct{}{} + } + }() + } + wg.Wait() + close(errs) + if len(errs) != 0 { + t.Fatal("concurrent read lost precomputed caller tracking data") + } + + delete(tracking.base, dep) + if got := runtimeCallerBaseSet(tracking, dep); got != nil { + t.Fatalf("frozen base lookup for unknown package = %v, want nil", got) + } + delete(tracking.extended, root) + if got := runtimeCallerFuncSet(tracking, root); got != nil { + t.Fatalf("frozen extended lookup for unknown package = %v, want nil", got) + } +} diff --git a/cl/instr.go b/cl/instr.go index fa139f2126..1a85a81e26 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -25,6 +25,7 @@ import ( "log" "os" "regexp" + "sort" "strings" "golang.org/x/tools/go/ssa" @@ -924,6 +925,9 @@ func runtimeCallerFuncSet(c *CallerTracking, pkg *ssa.Package) map[*ssa.Function if set, ok := c.extended[pkg]; ok { return set } + if c.frozen { + return nil + } base := runtimeCallerBaseSet(c, pkg) out := make(map[*ssa.Function]bool, len(base)) for fn := range base { @@ -977,12 +981,55 @@ func runtimeCallerFuncSet(c *CallerTracking, pkg *ssa.Package) map[*ssa.Function // queries (criterion 2 below) hit the memoization. It must not outlive // the compilation — the maps are keyed by *ssa.Package with // *ssa.Function values, so anything longer-lived would pin every -// compiled package's go/types and go/ssa graphs. Plain maps are enough: -// packages of one compilation are compiled sequentially (the LLVM -// context is not thread-safe). +// compiled package's go/types and go/ssa graphs. Concurrent drivers call +// Precompute and share only the resulting frozen, read-only maps. type CallerTracking struct { base map[*ssa.Package]map[*ssa.Function]bool extended map[*ssa.Package]map[*ssa.Function]bool + frozen bool +} + +// Precompute resolves every caller-tracking query before backend workers +// start, then freezes the maps for concurrent read-only access. +func (c *CallerTracking) Precompute(pkgs []*ssa.Package) { + if c == nil || c.frozen { + return + } + all := make(map[*ssa.Package]bool) + for _, pkg := range pkgs { + if pkg == nil { + continue + } + all[pkg] = true + if pkg.Prog != nil { + for _, programPkg := range pkg.Prog.AllPackages() { + if programPkg != nil { + all[programPkg] = true + } + } + } + } + ordered := make([]*ssa.Package, 0, len(all)) + for pkg := range all { + ordered = append(ordered, pkg) + } + sort.Slice(ordered, func(i, j int) bool { + left, right := "", "" + if ordered[i].Pkg != nil { + left = ordered[i].Pkg.Path() + } + if ordered[j].Pkg != nil { + right = ordered[j].Pkg.Path() + } + return left < right + }) + for _, pkg := range ordered { + runtimeCallerBaseSet(c, pkg) + } + for _, pkg := range ordered { + runtimeCallerFuncSet(c, pkg) + } + c.frozen = true } // NewCallerTracking creates the caller-tracking memoization for one @@ -1012,6 +1059,9 @@ func runtimeCallerBaseSet(c *CallerTracking, pkg *ssa.Package) map[*ssa.Function if set, ok := c.base[pkg]; ok { return set } + if c.frozen { + return nil + } set := computeRuntimeCallerBaseSet(pkg) c.base[pkg] = set return set diff --git a/internal/build/backend_program_test.go b/internal/build/backend_program_test.go new file mode 100644 index 0000000000..0756aa47e7 --- /dev/null +++ b/internal/build/backend_program_test.go @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "testing" + + llssa "github.com/goplus/llgo/ssa" +) + +func TestBackendProgramTemplateCreatesIsolatedSessions(t *testing.T) { + conf := &Config{Goos: "linux", Goarch: "amd64", RewriteMainPrefix: true} + template := newBackendProgramTemplate( + &llssa.Target{GOOS: conf.Goos, GOARCH: conf.Goarch, RewriteMainPrefix: conf.RewriteMainPrefix}, + conf, + true, + true, + ) + first, err := template.newSession() + if err != nil { + t.Fatal(err) + } + defer first.prog.Dispose() + second, err := template.newSession() + if err != nil { + t.Fatal(err) + } + defer second.prog.Dispose() + if first.transformer == nil || second.transformer == nil { + t.Fatal("backend session missing C ABI transformer") + } + if !first.prog.FuncInfoMetadataEnabled() || !first.prog.FuncInfoSitesEnabled() { + t.Fatal("backend template did not preserve funcinfo configuration") + } + if !first.prog.Target().RewriteMainPrefix || !second.prog.Target().RewriteMainPrefix { + t.Fatal("backend template did not preserve request-local symbol naming") + } + firstModule := first.prog.NewPackage("first", "example.com/first").Module() + secondModule := second.prog.NewPackage("second", "example.com/second").Module() + if firstModule.Context().C == secondModule.Context().C { + t.Fatal("backend sessions share an LLVM context") + } +} diff --git a/internal/build/build.go b/internal/build/build.go index 6617d5233c..8d31670271 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -456,8 +456,14 @@ func Build(inv Invocation) ([]Package, error) { RewriteMainPrefix: conf.RewriteMainPrefix, } - prog := llssa.NewProgram(target) - prog.DisableBoundsChecks(conf.DisableBoundsChecks) + funcInfo := conf.Mode != ModeGen && conf.PCLNMode != PCLNNone + backendTemplate := newBackendProgramTemplate( + target, + conf, + funcInfo, + shouldEnablePCLNSites(conf, funcInfo, emitDebugInfo), + ) + prog := backendTemplate.newProgram() if conf.Mode != ModeGen { // ModeGen callers (llgen and the golden suites) read LPkg.String() // after Do returns and dispose the program themselves; every other @@ -467,24 +473,17 @@ func Build(inv Invocation) ([]Package, error) { // harness) otherwise accumulate every compile's C++-side memory. defer prog.Dispose() } - prog.EnableGoGlobalDCE(conf.goGlobalDCEEnabled()) - prog.EnableDeadcodeDrop(conf.deadcodeDropEnabled()) - if conf.PthreadStackSize > 0 { - prog.SetPthreadStackSize(uint64(conf.PthreadStackSize)) - } - prog.EnableLTOPluginMarkers(conf.LTOPlugin.Enabled()) - funcInfo := conf.Mode != ModeGen && conf.PCLNMode != PCLNNone - prog.EnableFuncInfoMetadata(funcInfo) - // Site records are inline-asm fragments inside function bodies. Darwin - // DWARF builds avoid them because they disturb LLDB lexical scopes; Linux - // still needs them because its restricted dynamic symbol table cannot - // reconstruct every Go entry PC through dlsym. External mode always needs - // final-PC sites for sidecar construction. - prog.EnableFuncInfoSites(shouldEnablePCLNSites(conf, funcInfo, emitDebugInfo)) + var backendTypeSizes types.Sizes + var backendTypeSizesMu sync.Mutex sizes := func(sizes types.Sizes, compiler, arch string) types.Sizes { if arch == "wasm" { sizes = &types.StdSizes{WordSize: 4, MaxAlign: 4} } + backendTypeSizesMu.Lock() + if backendTypeSizes == nil { + backendTypeSizes = sizes + } + backendTypeSizesMu.Unlock() return prog.TypeSizes(sizes) } dedup := packages.NewDeduper() @@ -587,11 +586,22 @@ func Build(inv Invocation) ([]Package, error) { return altPkgs[0].Types }) prog.SetPython(func() *types.Package { - return dedup.Check(llssa.PkgPython).Types + if pkg := dedup.Check(llssa.PkgPython); pkg != nil { + return pkg.Types + } + return nil }) if err := prepareLocalVariables(prog, initial, altPkgs); err != nil { return nil, err } + backendTemplate.typeSizes = backendTypeSizes + backendTemplate.runtimePackage = altPkgs[0].Types + if pkg := dedup.Check(llssa.PkgPython); pkg != nil { + backendTemplate.pythonPackage = pkg.Types + } + backendTemplate.inputs = collectBackendProgramInputs(initial, altPkgs) + backendTemplate.llvmTarget = export.LLVMTarget + backendTemplate.targetABI = export.TargetABI buildMode := ssaBuildMode cabiOptimize := true @@ -606,9 +616,11 @@ func Build(inv Invocation) ([]Package, error) { if !IsOptimizeEnabled() { buildMode |= ssa.NaiveForm } + backendTemplate.cabiOptimize = cabiOptimize progSSA := ssa.NewProgram(initial[0].Fset, buildMode) patches := make(cl.Patches, len(altPkgPaths)) altEntries := registerAltSSAPkgs(progSSA, patches, altPkgs[1:], conf, verbose) + appendPatchedBackendInputs(&backendTemplate, patches, dedup) output := conf.OutFile != "" ctx := &context{conf: cfg, progSSA: progSSA, prog: prog, dedup: dedup, @@ -624,6 +636,7 @@ func Build(inv Invocation) ([]Package, error) { commands: commands, frontendOptions: frontendOptions, cTransformer: cabi.NewTransformer(prog, export.LLVMTarget, export.TargetABI, conf.AbiMode, cabiOptimize), + backend: backendTemplate, } defer ctx.closePackageMetas() defer ctx.closePackageArchiveBuffers() @@ -640,6 +653,7 @@ func Build(inv Invocation) ([]Package, error) { return nil, err } buildSSAPkgs(ctx, append(append(altEntries, pkgEntries...), depEntries...)) + ctx.callerTracking.Precompute(ctx.progSSA.AllPackages()) allPkgs := append([]*aPackage{}, pkgs...) allPkgs = append(allPkgs, depPkgs...) @@ -887,6 +901,7 @@ type context struct { frontendOptions cl.Options cTransformer *cabi.Transformer + backend backendProgramTemplate testFail bool @@ -924,6 +939,161 @@ func (c *context) closePackageMetas() { } } +type backendProgramInput struct { + fset *token.FileSet + pkg *types.Package + info *types.Info + files []*ast.File + parseSyntax bool + prepareLocal bool +} + +// backendProgramTemplate contains only immutable build-local inputs. Creating +// a session allocates a new llssa.Program, LLVM context, TargetMachine, and C +// ABI transformer; no LLVM-owned state is shared between sessions. +type backendProgramTemplate struct { + target *llssa.Target + disableBoundsChecks bool + typeSizes types.Sizes + goGlobalDCE bool + deadcodeDrop bool + pthreadStackSize int64 + ltoPluginMarkers bool + funcInfoMetadata bool + funcInfoSites bool + runtimePackage *types.Package + pythonPackage *types.Package + inputs []backendProgramInput + llvmTarget string + targetABI string + abiMode cabi.Mode + cabiOptimize bool +} + +type backendSession struct { + prog llssa.Program + transformer *cabi.Transformer +} + +func newBackendProgramTemplate(target *llssa.Target, conf *Config, funcInfoMetadata, funcInfoSites bool) backendProgramTemplate { + var targetCopy *llssa.Target + if target != nil { + copy := *target + targetCopy = © + } + return backendProgramTemplate{ + target: targetCopy, + disableBoundsChecks: conf.DisableBoundsChecks, + goGlobalDCE: conf.goGlobalDCEEnabled(), + deadcodeDrop: conf.deadcodeDropEnabled(), + pthreadStackSize: conf.PthreadStackSize, + ltoPluginMarkers: conf.LTOPlugin.Enabled(), + funcInfoMetadata: funcInfoMetadata, + funcInfoSites: funcInfoSites, + abiMode: conf.AbiMode, + } +} + +func (t backendProgramTemplate) newProgram() llssa.Program { + var target *llssa.Target + if t.target != nil { + copy := *t.target + target = © + } + prog := llssa.NewProgram(target) + prog.DisableBoundsChecks(t.disableBoundsChecks) + if t.typeSizes != nil { + prog.TypeSizes(t.typeSizes) + } + prog.EnableGoGlobalDCE(t.goGlobalDCE) + prog.EnableDeadcodeDrop(t.deadcodeDrop) + if t.pthreadStackSize > 0 { + prog.SetPthreadStackSize(uint64(t.pthreadStackSize)) + } + prog.EnableLTOPluginMarkers(t.ltoPluginMarkers) + prog.EnableFuncInfoMetadata(t.funcInfoMetadata) + prog.EnableFuncInfoSites(t.funcInfoSites) + if t.runtimePackage != nil { + prog.SetRuntime(t.runtimePackage) + } + if t.pythonPackage != nil { + prog.SetPython(t.pythonPackage) + } + return prog +} + +func (t backendProgramTemplate) newSession() (backendSession, error) { + prog := t.newProgram() + if err := t.replayProgramState(prog); err != nil { + prog.Dispose() + return backendSession{}, err + } + return backendSession{ + prog: prog, + transformer: cabi.NewTransformer(prog, t.llvmTarget, t.targetABI, t.abiMode, t.cabiOptimize), + }, nil +} + +func (t backendProgramTemplate) replayProgramState(prog llssa.Program) error { + for _, input := range t.inputs { + if input.parseSyntax { + if err := cl.ParsePkgSyntax(prog, input.fset, input.pkg, input.files); err != nil { + return err + } + } + } + for _, input := range t.inputs { + if input.prepareLocal { + if err := cl.PrepareLocalVariables(prog, input.fset, input.pkg, input.info, input.files); err != nil { + return err + } + } + } + return nil +} + +func collectBackendProgramInputs(groups ...[]*packages.Package) []backendProgramInput { + seen := make(map[*types.Package]bool) + var inputs []backendProgramInput + for _, roots := range groups { + packages.Visit(roots, nil, func(pkg *packages.Package) { + if pkg == nil || pkg.Types == nil || pkg.IllTyped || seen[pkg.Types] { + return + } + seen[pkg.Types] = true + inputs = append(inputs, backendProgramInput{ + fset: pkg.Fset, + pkg: pkg.Types, + info: pkg.TypesInfo, + files: slices.Clone(pkg.Syntax), + parseSyntax: !llruntime.SkipToBuild(pkg.PkgPath), + prepareLocal: true, + }) + }) + } + return inputs +} + +func appendPatchedBackendInputs(template *backendProgramTemplate, patches cl.Patches, dedup packages.Deduper) { + paths := make([]string, 0, len(patches)) + for pkgPath := range patches { + paths = append(paths, pkgPath) + } + slices.Sort(paths) + for _, pkgPath := range paths { + alt := dedup.Check(altPkgPathPrefix + pkgPath) + if alt == nil || len(alt.Syntax) == 0 { + continue + } + template.inputs = append(template.inputs, backendProgramInput{ + fset: alt.Fset, + pkg: types.NewPackage(pkgPath, ""), + files: slices.Clone(alt.Syntax), + parseSyntax: true, + }) + } +} + func (c *context) compiler() *clang.Cmd { config := clang.NewConfig( c.crossCompile.CC, From 431411a88e5a8ada802dffa28506f986756ff7c2 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Thu, 30 Jul 2026 23:07:29 +0800 Subject: [PATCH 15/22] build: run package backends in parallel --- cl/import.go | 10 +- cl/import_coverage_test.go | 39 ++++ internal/build/backend_program_test.go | 35 ++- internal/build/build.go | 99 +++++---- internal/build/main_module.go | 2 + internal/build/package_build.go | 284 ++++++++++++++++++++++++- internal/build/package_build_test.go | 159 ++++++++++++++ internal/build/package_summary.go | 30 +++ internal/build/package_summary_test.go | 14 ++ internal/build/plan9asm.go | 103 +++++---- ssa/abitype.go | 74 +++++++ ssa/package.go | 7 + ssa/ssa_test.go | 51 +++++ xtool/env/env.go | 71 ++++++- xtool/env/env_test.go | 42 ++++ 15 files changed, 921 insertions(+), 99 deletions(-) create mode 100644 xtool/env/env_test.go diff --git a/cl/import.go b/cl/import.go index 816f21de5b..0caa1a96f3 100644 --- a/cl/import.go +++ b/cl/import.go @@ -276,7 +276,7 @@ func (p *context) collectSkip(line string, prefix int) { } } -func collectLinknameByDoc(prog llssa.Program, doc *ast.CommentGroup, fullName, inPkgName string) { +func collectLinknameByDoc(prog llssa.Program, doc *ast.CommentGroup, fullName, inPkgName string) bool { directives := directive.ParseGroup(doc) for n := len(directives) - 1; n >= 0; n-- { directive := directives[n] @@ -286,9 +286,10 @@ func collectLinknameByDoc(prog llssa.Program, doc *ast.CommentGroup, fullName, i fields := strings.Fields(directive.Args) if len(fields) >= 2 && fields[0] == inPkgName { prog.SetLinkname(fullName, strings.Join(fields[1:], " ")) - return + return true } } + return false } func (p *context) processLinknameByDoc(doc *ast.CommentGroup, fullName, inPkgName string, isVar, allowExport bool) bool { @@ -784,7 +785,10 @@ func ParsePkgSyntax(prog llssa.Program, fset *token.FileSet, pkg *types.Package, return err } fullName, inPkgName := astFuncName(pkgPath, decl) - collectLinknameByDoc(prog, decl.Doc, fullName, inPkgName) + hasLinkname := collectLinknameByDoc(prog, decl.Doc, fullName, inPkgName) + if !hasLinkname && pkg.Name() == "C" && decl.Recv == nil && token.IsExported(inPkgName) { + prog.SetLinkname(fullName, strings.TrimPrefix(inPkgName, "X")) + } ctx.processNoInterfaceByDoc(decl.Doc, fullName) case *ast.GenDecl: if decl.Tok == token.VAR { diff --git a/cl/import_coverage_test.go b/cl/import_coverage_test.go index 4afa5b0110..4f9848b34d 100644 --- a/cl/import_coverage_test.go +++ b/cl/import_coverage_test.go @@ -257,6 +257,45 @@ func TestParsePkgSyntaxCollectsLinknames(t *testing.T) { } } +func TestParsePkgSyntaxCollectsCPackageExports(t *testing.T) { + const src = `package C + +func Xadd(a, b int) int { return a + b } +func Double(x float64) float64 { return 2 * x } +func hidden() {} + +//go:linkname Xnamed explicit_name +func Xnamed() +` + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "c.go", src, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + prog := llssa.NewProgram(nil) + pkg := types.NewPackage("example.com/c", "C") + if err := ParsePkgSyntax(prog, fset, pkg, []*ast.File{file}); err != nil { + t.Fatal(err) + } + tests := []struct { + name string + want string + ok bool + }{ + {name: "Xadd", want: "add", ok: true}, + {name: "Double", want: "Double", ok: true}, + {name: "hidden"}, + {name: "Xnamed", want: "explicit_name", ok: true}, + } + for _, tt := range tests { + fullName := pkg.Path() + "." + tt.name + got, ok := prog.Linkname(fullName) + if got != tt.want || ok != tt.ok { + t.Errorf("Linkname(%q) = (%q, %v), want (%q, %v)", fullName, got, ok, tt.want, tt.ok) + } + } +} + func TestCollectLinknameByDocIgnoresOtherDirectives(t *testing.T) { prog := llssa.NewProgram(nil) doc := &ast.CommentGroup{List: []*ast.Comment{ diff --git a/internal/build/backend_program_test.go b/internal/build/backend_program_test.go index 0756aa47e7..4eba591e55 100644 --- a/internal/build/backend_program_test.go +++ b/internal/build/backend_program_test.go @@ -17,6 +17,10 @@ package build import ( + "go/ast" + "go/parser" + "go/token" + "go/types" "testing" llssa "github.com/goplus/llgo/ssa" @@ -47,7 +51,7 @@ func TestBackendProgramTemplateCreatesIsolatedSessions(t *testing.T) { t.Fatal("backend template did not preserve funcinfo configuration") } if !first.prog.Target().RewriteMainPrefix || !second.prog.Target().RewriteMainPrefix { - t.Fatal("backend template did not preserve request-local symbol naming") + t.Fatal("backend template did not preserve invocation-local symbol naming") } firstModule := first.prog.NewPackage("first", "example.com/first").Module() secondModule := second.prog.NewPackage("second", "example.com/second").Module() @@ -55,3 +59,32 @@ func TestBackendProgramTemplateCreatesIsolatedSessions(t *testing.T) { t.Fatal("backend sessions share an LLVM context") } } + +func TestBackendProgramTemplateReplaysCPackageExports(t *testing.T) { + const src = `package C +func Xadd(a, b int) int { return a + b } +` + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "c.go", src, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + pkg := types.NewPackage("example.com/c", "C") + template := backendProgramTemplate{ + inputs: []backendProgramInput{{ + fset: fset, + pkg: pkg, + files: []*ast.File{file}, + parseSyntax: true, + }}, + } + session, err := template.newSession() + if err != nil { + t.Fatal(err) + } + defer session.prog.Dispose() + const fullName = "example.com/c.Xadd" + if got, ok := session.prog.Linkname(fullName); !ok || got != "add" { + t.Fatalf("replayed Linkname(%q) = (%q, %v), want (%q, true)", fullName, got, ok, "add") + } +} diff --git a/internal/build/build.go b/internal/build/build.go index 8d31670271..16d4c19c59 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -910,23 +910,21 @@ type context struct { llvmVersion string // go list derived file lists (SFiles, etc.) - sfilesCache map[string][]string // pkg.ID -> absolute .s/.S file paths + sfilesCache map[string][]string // pkg.ID -> absolute .s/.S file paths + sfilesFrozen bool // plan9asm package policy parsed from env. - plan9asmOnce sync.Once - plan9asmMode plan9asmPkgsEnvMode - plan9asmPkgs map[string]bool + plan9asmOnce sync.Once + plan9asmReady bool + plan9asmMode plan9asmPkgsEnvMode + plan9asmPkgs map[string]bool + plan9asmSigs map[string]map[string]struct{} // pclnExternal is populated while generating the synthetic main module // and completed with final linked PCs by the post-link externalizer. pclnExternal *pclnmap.Data } -// frontendDebugMu protects the legacy cl/ssa instruction-debug switches. -// Code-generation options are invocation-local; this process-wide lock remains -// only for verbose diagnostic logging until those helpers accept a logger. -var frontendDebugMu sync.RWMutex - // closePackageMetas releases metadata mappings owned by this build. Metadata // remains available to hooks and whole-program consumers until Do returns. func (c *context) closePackageMetas() { @@ -1159,36 +1157,34 @@ func normalizeToArchive(ctx *context, aPkg *aPackage, verbose bool) error { } func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, error) { - // Split packages into runtime tree vs others so we can defer runtime build. - var runtimePkgs []packageBuildSpec - var normalPkgs []packageBuildSpec + specs := make([]packageBuildSpec, 0, len(pkgs)) for _, p := range pkgs { - spec := newPackageBuildSpec(p) - if spec.runtime { - runtimePkgs = append(runtimePkgs, spec) - } else { - normalPkgs = append(normalPkgs, spec) - } + specs = append(specs, newPackageBuildSpec(p)) } + preflights, err := preflightPackageBuilds(ctx, specs, verbose) + if err != nil { + return nil, err + } + ctx.sfilesFrozen = true var needRuntime, needPyInit bool // Build non-runtime packages first, so we know whether runtime is actually needed. - for _, spec := range normalPkgs { - result, err := buildOnePackage(ctx, spec, verbose) - if err != nil { - return nil, err - } + normalResults, err := buildPreflightedPackageGroup( + ctx, packageBuildSpecsForRuntime(specs, false), preflights, verbose) + if err != nil { + return nil, err + } + for _, result := range normalResults { needRuntime = needRuntime || result.needRuntime needPyInit = needPyInit || result.needPyInit } // Only build runtime packages when required (or host build with empty Target). if needRuntime || needPyInit || ctx.buildConf.Target == "" { - for _, spec := range runtimePkgs { - if _, err := buildOnePackage(ctx, spec, verbose); err != nil { - return nil, err - } + if _, err := buildPreflightedPackageGroup( + ctx, packageBuildSpecsForRuntime(specs, true), preflights, verbose); err != nil { + return nil, err } } @@ -1235,6 +1231,9 @@ func preflightPackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (s return false, err } ctx.tryLoadFromCache(aPkg) + if err := preparePackageSFiles(ctx, aPkg); err != nil { + return false, err + } if verbose { status := "MISS" if aPkg.CacheHit { @@ -1251,7 +1250,7 @@ func executePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) erro if err := buildPkg(ctx, aPkg, verbose); err != nil { return err } - if spec.needsRuntimeSignals() { + if spec.needsRuntimeSignals() && aPkg.LPkg != nil { aPkg.setNeedRuntimeOrPyInit(aPkg.LPkg.NeedRuntime, aPkg.LPkg.NeedPyInit) } return nil @@ -1270,7 +1269,14 @@ func finalizePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (pa if spec.kind == cl.PkgLinkExtern { appendExternalLinkArgs(ctx, aPkg, spec.kindParam) } - aPkg.Summary = summarizePackage(aPkg) + if aPkg.Summary == nil { + aPkg.Summary = summarizePackage(aPkg) + } else { + aPkg.Summary.LinkArgs = append(aPkg.Summary.LinkArgs[:0], aPkg.LinkArgs...) + aPkg.Summary.ArchiveFile = aPkg.ArchiveFile + aPkg.Summary.NeedRuntime = aPkg.NeedRt + aPkg.Summary.NeedPyInit = aPkg.NeedPyInit + } if err := ctx.saveToCache(aPkg); err != nil && verbose { fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", aPkg.PkgPath, err) } @@ -1286,7 +1292,7 @@ func appendExternalLinkArgs(ctx *context, aPkg *aPackage, spec string) { for _, alt := range altParts { alt = strings.TrimSpace(alt) if strings.ContainsRune(alt, '$') { - expdArgs = append(expdArgs, xenv.ExpandEnvToArgs(alt)...) + expdArgs = append(expdArgs, xenv.ExpandEnvToArgsWith(alt, ctx.commands.dir, ctx.commands.environ)...) atomic.AddInt32(&ctx.nLibdir, 1) } else { fields := strings.Fields(alt) @@ -1589,6 +1595,7 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa methodByIndex: methodByIndex, methodByName: methodByName, abiSymbols: linkedPackageGlobals(linkedSummaries), + abiTypes: abiTypesForSummaries(linkedSummaries), funcInfo: funcInfo, pcLineInfo: pcLineInfo, funcInfoStubs: funcInfoStubs, @@ -1979,26 +1986,16 @@ func preparePackageModule(ctx *context, aPkg *aPackage, verbose bool) ([]string, } showDetail := verbose && pkgExists(ctx.initial, pkg) needMeta := !aPkg.CacheHit && ctx.buildConf.packageMetaEnabled() - ret, externs, err := func() (llssa.Package, []string, error) { - if showDetail { - frontendDebugMu.Lock() - defer frontendDebugMu.Unlock() - llssa.SetDebug(llssa.DbgFlagAll) - cl.SetDebug(cl.DbgFlagAll) - defer func() { - llssa.SetDebug(0) - cl.SetDebug(0) - }() - } else { - frontendDebugMu.RLock() - defer frontendDebugMu.RUnlock() - } - embedMap, err := goembed.LoadDirectives(ctx.conf.Fset, syntax) - if err != nil { - return nil, nil, fmt.Errorf("load go:embed directives for %s failed: %w", pkgPath, err) - } - return cl.NewPackageExWithEmbedMetaOptions(ctx.prog, ctx.callerTracking, ctx.patches, aPkg.rewriteVars, aPkg.SSA, syntax, embedMap, needMeta, ctx.frontendOptions) - }() + if showDetail { + fmt.Fprintf(os.Stderr, "==> Compile %s\n", pkgPath) + } + embedMap, err := goembed.LoadDirectives(ctx.conf.Fset, syntax) + if err != nil { + return nil, fmt.Errorf("load go:embed directives for %s failed: %w", pkgPath, err) + } + ret, externs, err := cl.NewPackageExWithEmbedMetaOptions( + ctx.prog, ctx.callerTracking, ctx.patches, aPkg.rewriteVars, + aPkg.SSA, syntax, embedMap, needMeta, ctx.frontendOptions) check(err) aPkg.LPkg = ret @@ -2812,7 +2809,7 @@ func clFiles(ctx *context, files string, pkg *packages.Package, procFile func(li args := make([]string, 0, 16) if strings.HasPrefix(files, "$") { // has cflags if pos := strings.IndexByte(files, ':'); pos > 0 { - cflags := xenv.ExpandEnvToArgs(files[:pos]) + cflags := xenv.ExpandEnvToArgsWith(files[:pos], ctx.commands.dir, ctx.commands.environ) files = files[pos+1:] args = append(args, cflags...) } diff --git a/internal/build/main_module.go b/internal/build/main_module.go index cb0c2a418b..8d493ab46c 100644 --- a/internal/build/main_module.go +++ b/internal/build/main_module.go @@ -42,6 +42,7 @@ type genConfig struct { methodByIndex map[int]none methodByName map[string]none abiSymbols map[string]none + abiTypes []llssa.AbiTypeInfo funcInfo []funcInfoRecord pcLineInfo []pcLineRecord funcInfoStubs []funcInfoStubRecord @@ -94,6 +95,7 @@ func genMainModule(ctx *context, rtPkgPath string, pkg *packages.Package, cfg *g var abiInit llssa.Function if cfg.abiInit != 0 { + mainPkg.RegisterAbiTypes(cfg.abiTypes) abiInit = mainPkg.InitAbiTypesFor("init$abitypes", func(sym *llssa.AbiSymbol) bool { if _, ok := cfg.abiSymbols[sym.Name]; !ok { return false diff --git a/internal/build/package_build.go b/internal/build/package_build.go index a5ccc0ab38..694077f3f9 100644 --- a/internal/build/package_build.go +++ b/internal/build/package_build.go @@ -16,7 +16,13 @@ package build -import "github.com/goplus/llgo/cl" +import ( + "fmt" + "sync" + + "github.com/goplus/llgo/cl" + "github.com/goplus/llgo/internal/packages" +) type packageBuildSpec struct { pkg *aPackage @@ -56,6 +62,282 @@ type packageBuildResult struct { needPyInit bool } +type packagePreflight struct { + spec packageBuildSpec + skip bool +} + +func preflightPackageBuilds(ctx *context, specs []packageBuildSpec, verbose bool) (map[*aPackage]packagePreflight, error) { + preflights := make(map[*aPackage]packagePreflight, len(specs)) + for _, spec := range specs { + skip, err := preflightPackageBuild(ctx, spec, verbose) + if err != nil { + return nil, err + } + preflights[spec.pkg] = packagePreflight{spec: spec, skip: skip} + } + return preflights, nil +} + +func packageBuildSpecsForRuntime(specs []packageBuildSpec, runtime bool) []packageBuildSpec { + filtered := make([]packageBuildSpec, 0, len(specs)) + for _, spec := range specs { + if spec.runtime == runtime { + filtered = append(filtered, spec) + } + } + return filtered +} + +// buildPreflightedPackageGroup executes package backends first and then +// publishes archives/cache entries serially in deterministic package order. +func buildPreflightedPackageGroup(ctx *context, specs []packageBuildSpec, preflights map[*aPackage]packagePreflight, verbose bool) ([]packageBuildResult, error) { + if len(specs) == 0 { + return nil, nil + } + + patched, coordinator, isolated, err := partitionPackageExecutions(ctx, specs) + if err != nil { + return nil, err + } + for _, index := range patched { + if err := executePreflightedPackage(ctx, preflights[specs[index].pkg], verbose); err != nil { + return nil, err + } + } + for _, index := range coordinator { + preflight := preflights[specs[index].pkg] + if preflight.skip { + continue + } + if err := executePackageBuild(ctx, preflight.spec, verbose); err != nil { + return nil, err + } + } + if err := executeIsolatedPackages(ctx, specs, isolated, preflights, verbose); err != nil { + return nil, err + } + + results := make([]packageBuildResult, len(specs)) + for i, spec := range specs { + preflight := preflights[spec.pkg] + if preflight.skip { + results[i] = packageBuildResultFor(spec) + continue + } + result, err := finalizePackageBuild(ctx, spec, verbose) + if err != nil { + return nil, err + } + results[i] = result + } + return results, nil +} + +func partitionPackageExecutions(ctx *context, specs []packageBuildSpec) (patched, coordinator, isolated []int, err error) { + for i, spec := range specs { + if _, ok := ctx.patches[spec.pkg.PkgPath]; ok { + patched = append(patched, i) + continue + } + serial, serialErr := ctx.packageRequiresCoordinator(spec) + if serialErr != nil { + return nil, nil, nil, serialErr + } + if serial { + coordinator = append(coordinator, i) + } else { + isolated = append(isolated, i) + } + } + return +} + +func executePreflightedPackage(ctx *context, preflight packagePreflight, verbose bool) error { + if preflight.skip { + return nil + } + serial, err := ctx.packageRequiresCoordinator(preflight.spec) + if err != nil { + return err + } + if serial { + return executePackageBuild(ctx, preflight.spec, verbose) + } + return ctx.executeIsolatedPackage(preflight.spec, verbose) +} + +func executeIsolatedPackages(ctx *context, specs []packageBuildSpec, indexes []int, preflights map[*aPackage]packagePreflight, verbose bool) error { + if len(indexes) == 0 { + return nil + } + return runBoundedPackageJobs(ctx.buildConf.parallelism(), indexes, func(index int) error { + preflight := preflights[specs[index].pkg] + if preflight.skip { + return nil + } + return ctx.executeIsolatedPackage(preflight.spec, verbose) + }) +} + +func runBoundedPackageJobs(parallelism int, indexes []int, run func(index int) error) error { + if len(indexes) == 0 { + return nil + } + workers := min(max(1, parallelism), len(indexes)) + jobs := make(chan int, len(indexes)) + errs := make(map[int]error, len(indexes)) + var errsMu sync.Mutex + var wg sync.WaitGroup + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + for index := range jobs { + if err := runPackageJob(index, run); err != nil { + errsMu.Lock() + errs[index] = err + errsMu.Unlock() + } + } + }() + } + for _, index := range indexes { + jobs <- index + } + close(jobs) + wg.Wait() + for _, index := range indexes { + if err := errs[index]; err != nil { + return err + } + } + return nil +} + +func runPackageJob(index int, run func(index int) error) (err error) { + defer func() { + if value := recover(); value != nil { + if recovered, ok := value.(error); ok { + err = recovered + } else { + err = fmt.Errorf("package job %d panicked: %v", index, value) + } + } + }() + return run(index) +} + +func (ctx *context) canUseIsolatedBackend() bool { + return ctx.mode != ModeGen && + ctx.buildConf.BuildMode == BuildModeExe && + ctx.buildConf.ModuleHook == nil +} + +func (ctx *context) packageRequiresCoordinator(spec packageBuildSpec) (bool, error) { + if !ctx.canUseIsolatedBackend() { + return true, nil + } + usesPlan9, err := ctx.packageUsesPlan9Asm(spec.pkg) + if err != nil { + return false, err + } + return usesPlan9, nil +} + +func (ctx *context) packageUsesPlan9Asm(pkg *aPackage) (bool, error) { + check := func(p *packages.Package) (bool, error) { + if p == nil { + return false, nil + } + sfiles, err := pkgSFiles(ctx, p) + if err != nil { + return false, err + } + return len(sfiles) != 0 && ctx.plan9asmEnabled(p.PkgPath), nil + } + if yes, err := check(pkg.Package); err != nil || yes { + return yes, err + } + if pkg.AltPkg != nil { + return check(pkg.AltPkg.Package) + } + return false, nil +} + +func (ctx *context) executeIsolatedPackage(spec packageBuildSpec, verbose bool) error { + session, err := ctx.backend.newSession() + if err != nil { + return fmt.Errorf("create backend session for %s: %w", spec.pkg.PkgPath, err) + } + defer session.prog.Dispose() + task := ctx.newBackendTask(session) + defer func() { spec.pkg.LPkg = nil }() + + if err := buildPkg(task, spec.pkg, verbose); err != nil { + return err + } + if spec.pkg.LPkg == nil { + if spec.pkg.Summary == nil { + spec.pkg.Summary = summarizePackage(spec.pkg) + } + return nil + } + if !spec.pkg.CacheHit { + if spec.needsRuntimeSignals() { + spec.pkg.setNeedRuntimeOrPyInit(spec.pkg.LPkg.NeedRuntime, spec.pkg.LPkg.NeedPyInit) + } + spec.pkg.Summary = summarizePackage(spec.pkg) + } else { + spec.pkg.Summary.AbiTypes = session.prog.AbiTypes() + } + if spec.pkg.Summary == nil { + return fmt.Errorf("package %s produced no linker summary", spec.pkg.PkgPath) + } + return nil +} + +func preparePackageSFiles(ctx *context, pkg *aPackage) error { + if _, err := pkgSFiles(ctx, pkg.Package); err != nil { + return err + } + if pkg.AltPkg != nil { + if _, err := pkgSFiles(ctx, pkg.AltPkg.Package); err != nil { + return err + } + } + return nil +} + +func (ctx *context) newBackendTask(session backendSession) *context { + return &context{ + conf: ctx.conf, + progSSA: ctx.progSSA, + prog: session.prog, + dedup: ctx.dedup, + patches: ctx.patches, + callerTracking: ctx.callerTracking, + initial: ctx.initial, + pkgs: ctx.pkgs, + pkgByID: ctx.pkgByID, + mode: ctx.mode, + output: ctx.output, + passOpt: ctx.passOpt, + buildConf: ctx.buildConf, + crossCompile: ctx.crossCompile, + commands: ctx.commands, + frontendOptions: ctx.frontendOptions, + cTransformer: session.transformer, + backend: ctx.backend, + sfilesCache: ctx.sfilesCache, + sfilesFrozen: true, + plan9asmReady: true, + plan9asmMode: ctx.plan9asmMode, + plan9asmPkgs: ctx.plan9asmPkgs, + plan9asmSigs: make(map[string]map[string]struct{}), + } +} + func packageBuildResultFor(spec packageBuildSpec) packageBuildResult { return packageBuildResult{ needRuntime: spec.pkg.NeedRt, diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go index ad97feab3f..b34863b4ea 100644 --- a/internal/build/package_build_test.go +++ b/internal/build/package_build_test.go @@ -17,9 +17,15 @@ package build import ( + "errors" "go/types" + "os" + "path/filepath" + "sync/atomic" "testing" + "time" + "github.com/goplus/llgo/cl" "github.com/goplus/llgo/internal/env" "github.com/goplus/llgo/internal/packages" ) @@ -147,3 +153,156 @@ func TestPreflightFingerprintsSkippedPackage(t *testing.T) { t.Fatalf("skipped package was not fully prepared: skip=%v fingerprint=%q manifest=%q summary=%#v", skip, pkg.Fingerprint, pkg.Manifest, pkg.Summary) } } + +func TestCanUseIsolatedBackend(t *testing.T) { + ctx := &context{ + mode: ModeBuild, + buildConf: &Config{BuildMode: BuildModeExe}, + } + if !ctx.canUseIsolatedBackend() { + t.Fatal("normal executable build should use isolated backends") + } + ctx.mode = ModeGen + if ctx.canUseIsolatedBackend() { + t.Fatal("generation mode should remain on the coordinator") + } + ctx.mode = ModeTest + if !ctx.canUseIsolatedBackend() { + t.Fatal("test mode should use isolated executable backends") + } + ctx.buildConf.BuildMode = BuildModeCShared + if ctx.canUseIsolatedBackend() { + t.Fatal("c-shared mode should remain on the coordinator") + } + ctx.buildConf.BuildMode = BuildModeExe + ctx.buildConf.ModuleHook = func(*aPackage) {} + if ctx.canUseIsolatedBackend() { + t.Fatal("module hooks should remain on the coordinator") + } +} + +func TestPartitionPackageExecutions(t *testing.T) { + patchedPkg := &aPackage{Package: &packages.Package{ + ID: "example.com/patched", + PkgPath: "example.com/patched", + }} + normalPkg := &aPackage{Package: &packages.Package{ + ID: "example.com/normal", + PkgPath: "example.com/normal", + }} + ctx := &context{ + mode: ModeBuild, + buildConf: &Config{BuildMode: BuildModeExe}, + patches: cl.Patches{"example.com/patched": {}}, + sfilesCache: make(map[string][]string), + } + patched, coordinator, isolated, err := partitionPackageExecutions(ctx, []packageBuildSpec{ + {pkg: patchedPkg}, + {pkg: normalPkg}, + }) + if err != nil { + t.Fatal(err) + } + if len(patched) != 1 || patched[0] != 0 { + t.Fatalf("patched = %v, want [0]", patched) + } + if len(coordinator) != 0 { + t.Fatalf("coordinator = %v, want empty", coordinator) + } + if len(isolated) != 1 || isolated[0] != 1 { + t.Fatalf("isolated = %v, want [1]", isolated) + } +} + +func TestPkgSFilesRejectsUnpreparedBackendRead(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "asm.s"), []byte("TEXT ·f(SB),$0-0\n\tRET\n"), 0o644); err != nil { + t.Fatal(err) + } + ctx := &context{ + sfilesCache: make(map[string][]string), + sfilesFrozen: true, + } + _, err := pkgSFiles(ctx, &packages.Package{ + ID: "example.com/asm", + PkgPath: "example.com/asm", + Dir: dir, + }) + if err == nil { + t.Fatal("expected frozen SFiles cache to reject an unprepared package") + } +} + +func TestRunBoundedPackageJobs(t *testing.T) { + started := make(chan int, 4) + release := make(chan struct{}) + done := make(chan error, 1) + var active atomic.Int32 + var maximum atomic.Int32 + go func() { + done <- runBoundedPackageJobs(2, []int{0, 1, 2, 3}, func(index int) error { + current := active.Add(1) + for { + old := maximum.Load() + if current <= old || maximum.CompareAndSwap(old, current) { + break + } + } + started <- index + <-release + active.Add(-1) + return nil + }) + }() + + for range 2 { + select { + case <-started: + case <-time.After(5 * time.Second): + close(release) + t.Fatal("two package workers did not start concurrently") + } + } + if got := maximum.Load(); got != 2 { + close(release) + t.Fatalf("maximum concurrent jobs = %d, want 2", got) + } + close(release) + if err := <-done; err != nil { + t.Fatal(err) + } + if got := maximum.Load(); got != 2 { + t.Fatalf("maximum concurrent jobs = %d after completion, want 2", got) + } +} + +func TestRunBoundedPackageJobsReturnsFirstOrderedError(t *testing.T) { + first := errors.New("first") + second := errors.New("second") + err := runBoundedPackageJobs(3, []int{3, 1, 2}, func(index int) error { + switch index { + case 3: + return first + case 1: + return second + default: + return nil + } + }) + if !errors.Is(err, first) { + t.Fatalf("error = %v, want first submitted error", err) + } +} + +func TestRunBoundedPackageJobsConvertsPanicToError(t *testing.T) { + boom := errors.New("boom") + err := runBoundedPackageJobs(2, []int{0, 1}, func(index int) error { + if index == 0 { + panic(boom) + } + return nil + }) + if !errors.Is(err, boom) { + t.Fatalf("error = %v, want recovered panic %v", err, boom) + } +} diff --git a/internal/build/package_summary.go b/internal/build/package_summary.go index 76c1becca6..4cc7a9d53a 100644 --- a/internal/build/package_summary.go +++ b/internal/build/package_summary.go @@ -20,6 +20,7 @@ import ( "sort" "strings" + llssa "github.com/goplus/llgo/ssa" "github.com/xgo-dev/llvm" ) @@ -48,6 +49,7 @@ type PackageSummary struct { PCLineInfo []pcLineRecord FuncInfoStubs []string CSharedExports []string + AbiTypes []llssa.AbiTypeInfo } type packageSummaryMetadata struct { @@ -120,6 +122,7 @@ func summarizePackage(pkg *aPackage) *PackageSummary { sort.Strings(summary.GlobalSymbols) summary.FuncInfo = readFuncInfo(mod) summary.PCLineInfo = readPCLineInfo(mod) + summary.AbiTypes = lpkg.Prog.AbiTypes() for fn := mod.FirstFunction(); !fn.IsNil(); fn = llvm.NextFunction(fn) { if fn.IsDeclaration() || fn.BasicBlocksCount() == 0 { continue @@ -227,3 +230,30 @@ func summariesForPackages(pkgs []Package) []*PackageSummary { } return summaries } + +func abiTypesForSummaries(summaries []*PackageSummary) []llssa.AbiTypeInfo { + byName := make(map[string]llssa.AbiTypeInfo) + for _, summary := range summaries { + if summary == nil { + continue + } + for _, info := range summary.AbiTypes { + if info.Name == "" || info.Raw == nil { + continue + } + if _, ok := byName[info.Name]; !ok { + byName[info.Name] = info + } + } + } + names := make([]string, 0, len(byName)) + for name := range byName { + names = append(names, name) + } + sort.Strings(names) + ret := make([]llssa.AbiTypeInfo, 0, len(names)) + for _, name := range names { + ret = append(ret, byName[name]) + } + return ret +} diff --git a/internal/build/package_summary_test.go b/internal/build/package_summary_test.go index 0e27ad6f56..5c3e538190 100644 --- a/internal/build/package_summary_test.go +++ b/internal/build/package_summary_test.go @@ -91,3 +91,17 @@ func TestPackageSummaryCapturesLinkerFacts(t *testing.T) { t.Fatalf("cache summary round trip = %#v, want %#v", loaded, summary) } } + +func TestAbiTypesForSummariesDeduplicatesBySymbol(t *testing.T) { + first := llssa.AbiTypeInfo{Name: "_llgo_int", Raw: types.Typ[types.Int]} + second := llssa.AbiTypeInfo{Name: "_llgo_string", Raw: types.Typ[types.String]} + got := abiTypesForSummaries([]*PackageSummary{ + nil, + {AbiTypes: []llssa.AbiTypeInfo{second, first}}, + {AbiTypes: []llssa.AbiTypeInfo{first, {}}}, + }) + want := []llssa.AbiTypeInfo{first, second} + if !reflect.DeepEqual(got, want) { + t.Fatalf("ABI types = %#v, want %#v", got, want) + } +} diff --git a/internal/build/plan9asm.go b/internal/build/plan9asm.go index 30077f5f72..6f7703b67f 100644 --- a/internal/build/plan9asm.go +++ b/internal/build/plan9asm.go @@ -8,7 +8,6 @@ import ( "os/exec" "path/filepath" "strings" - "sync" llabi "github.com/goplus/llgo/internal/abi" "github.com/goplus/llgo/internal/cabi" @@ -166,13 +165,6 @@ func shouldSkipDarwinDynimportTrampolineAsm(enabled bool, sfile string, src []by bytes.Contains(src, []byte("_trampoline_addr(SB)")) } -type plan9AsmSigCacheKey struct { - ctx *context - pkgPath string -} - -var plan9AsmSigCache sync.Map // key: plan9AsmSigCacheKey, value: map[string]struct{} - func archSupportsPlan9AsmDefaults(goarch string) bool { return goarch == "arm64" || goarch == "amd64" } @@ -225,18 +217,20 @@ func plan9asmSigsForPkg(ctx *context, pkgPath string) (map[string]struct{}, erro if ctx == nil || pkgPath == "" { return nil, nil } - key := plan9AsmSigCacheKey{ctx: ctx, pkgPath: pkgPath} - if v, ok := plan9AsmSigCache.Load(key); ok { - return v.(map[string]struct{}), nil + if ctx.plan9asmSigs == nil { + ctx.plan9asmSigs = make(map[string]map[string]struct{}) + } + if sigs, ok := ctx.plan9asmSigs[pkgPath]; ok { + return sigs, nil } sigs := make(map[string]struct{}) if !ctx.plan9asmEnabled(pkgPath) { - plan9AsmSigCache.Store(key, sigs) + ctx.plan9asmSigs[pkgPath] = sigs return sigs, nil } - if hasAltPkgForTarget(ctx.buildConf, pkgPath) && !llruntime.HasAdditiveAltPkgForGOARCH(pkgPath, ctx.buildConf.Goarch) { - plan9AsmSigCache.Store(key, sigs) + if ctx.hasAltPkgWithResolvedPlan9(pkgPath) && !llruntime.HasAdditiveAltPkgForGOARCH(pkgPath, ctx.buildConf.Goarch) { + ctx.plan9asmSigs[pkgPath] = sigs return sigs, nil } @@ -248,7 +242,7 @@ func plan9asmSigsForPkg(ctx *context, pkgPath string) (map[string]struct{}, erro } } if pkg == nil { - plan9AsmSigCache.Store(key, sigs) + ctx.plan9asmSigs[pkgPath] = sigs return sigs, nil } @@ -272,7 +266,7 @@ func plan9asmSigsForPkg(ctx *context, pkgPath string) (map[string]struct{}, erro sigs[name] = struct{}{} } } - plan9AsmSigCache.Store(key, sigs) + ctx.plan9asmSigs[pkgPath] = sigs return sigs, nil } @@ -306,19 +300,22 @@ func cabiSkipFuncsForPlan9Asm(ctx *context, pkgPath string, mod gllvm.Module) [] } func (ctx *context) plan9asmEnabled(pkgPath string) bool { - ctx.plan9asmOnce.Do(func() { - cfg := parsePlan9AsmPkgsEnv(Plan9ASMPkgs()) - ctx.plan9asmMode = cfg.mode - switch cfg.mode { - case plan9asmEnvSelected: - ctx.plan9asmPkgs = make(map[string]bool, len(cfg.pkgs)) - for p := range cfg.pkgs { - ctx.plan9asmPkgs[p] = true + if !ctx.plan9asmReady { + ctx.plan9asmOnce.Do(func() { + cfg := parsePlan9AsmPkgsEnv(Plan9ASMPkgs()) + ctx.plan9asmMode = cfg.mode + switch cfg.mode { + case plan9asmEnvSelected: + ctx.plan9asmPkgs = make(map[string]bool, len(cfg.pkgs)) + for p := range cfg.pkgs { + ctx.plan9asmPkgs[p] = true + } + default: + ctx.plan9asmPkgs = make(map[string]bool) } - default: - ctx.plan9asmPkgs = make(map[string]bool) - } - }) + ctx.plan9asmReady = true + }) + } switch ctx.plan9asmMode { case plan9asmEnvAll: @@ -334,6 +331,30 @@ func (ctx *context) plan9asmEnabled(pkgPath string) bool { } } +func (ctx *context) hasAltPkgWithResolvedPlan9(pkgPath string) bool { + conf := ctx.buildConf + if conf == nil || !llruntime.HasAltPkgForGOARCH(pkgPath, conf.Goarch) { + return false + } + if llruntime.HasAdditiveAltPkgForGOARCH(pkgPath, conf.Goarch) { + return true + } + if plan9asmEnabledByDefault(conf, pkgPath) && ctx.plan9asmMode != plan9asmEnvNone { + return false + } + if conf.AbiMode != cabi.ModeAllFunc { + switch ctx.plan9asmMode { + case plan9asmEnvAll: + return false + case plan9asmEnvSelected: + if ctx.plan9asmPkgs[pkgPath] { + return false + } + } + } + return true +} + func hasAltPkgForTarget(conf *Config, pkgPath string) bool { if conf == nil || !llruntime.HasAltPkgForGOARCH(pkgPath, conf.Goarch) { return false @@ -379,26 +400,32 @@ func pkgSFiles(ctx *context, pkg *packages.Package) ([]string, error) { if pkg == nil || pkg.PkgPath == "" { return nil, nil } + if ctx.sfilesCache == nil { + if ctx.sfilesFrozen { + return nil, fmt.Errorf("package %s assembly files were not prepared before backend execution", pkg.PkgPath) + } + ctx.sfilesCache = make(map[string][]string) + } + if v, ok := ctx.sfilesCache[pkg.ID]; ok { + return v, nil + } // Some unit tests construct synthetic packages that are not loadable via // `go list` (PkgPath not in any module, and Dir/Standard/Goroot unset). // In that case, treat the package as having no selected .s files. if pkg.Dir == "" { + ctx.sfilesCache[pkg.ID] = nil return nil, nil } // Fast path: if directory has no .s/.S at all, skip `go list`. - if pkg.Dir != "" { - if ss, _ := filepath.Glob(filepath.Join(pkg.Dir, "*.s")); len(ss) == 0 { - if ss, _ := filepath.Glob(filepath.Join(pkg.Dir, "*.S")); len(ss) == 0 { - return nil, nil - } + if ss, _ := filepath.Glob(filepath.Join(pkg.Dir, "*.s")); len(ss) == 0 { + if ss, _ := filepath.Glob(filepath.Join(pkg.Dir, "*.S")); len(ss) == 0 { + ctx.sfilesCache[pkg.ID] = nil + return nil, nil } } - if ctx.sfilesCache == nil { - ctx.sfilesCache = make(map[string][]string) - } - if v, ok := ctx.sfilesCache[pkg.ID]; ok { - return v, nil + if ctx.sfilesFrozen { + return nil, fmt.Errorf("package %s assembly files were not prepared before backend execution", pkg.PkgPath) } args := []string{"list", "-json"} diff --git a/ssa/abitype.go b/ssa/abitype.go index 9b1c85bfca..9c0b8eb18f 100644 --- a/ssa/abitype.go +++ b/ssa/abitype.go @@ -445,6 +445,21 @@ func (b Builder) abiUncommonMethodSet(t types.Type) (mset *types.MethodSet, ok b return } +func abiUncommonMethodSetForDeclaration(t types.Type) (mset *types.MethodSet, ok bool) { + switch t := types.Unalias(t).(type) { + case *types.Named: + if _, isInterface := t.Underlying().(*types.Interface); isInterface { + return &types.MethodSet{}, true + } + return types.NewMethodSet(t), true + case *types.Struct, *types.Pointer: + if mset := types.NewMethodSet(t); mset.Len() != 0 { + return mset, true + } + } + return +} + /* type UncommonType struct { PkgPath_ string // import path; empty for built-in types like int, string @@ -659,6 +674,65 @@ func (b Builder) abiType(t types.Type) Expr { return ret } +// AbiTypes snapshots the runtime type descriptors materialized by this +// Program without retaining any LLVM-owned values. +func (p Program) AbiTypes() []AbiTypeInfo { + if len(p.abiSymbol) == 0 { + return nil + } + names := make([]string, 0, len(p.abiSymbol)) + for name := range p.abiSymbol { + names = append(names, name) + } + sort.Strings(names) + ret := make([]AbiTypeInfo, 0, len(names)) + for _, name := range names { + sym := p.abiSymbol[name] + ret = append(ret, AbiTypeInfo{Name: name, Raw: sym.Raw}) + } + return ret +} + +// RegisterAbiTypes declares runtime type descriptors owned by other Programs. +// It records only Go type identity and target-local LLVM types; descriptor +// definitions remain in the package modules that materialized them. +func (p Package) RegisterAbiTypes(infos []AbiTypeInfo) { + builder := &aBuilder{Prog: p.Prog, Pkg: p} + for _, info := range infos { + if info.Raw == nil { + continue + } + if _, ok := p.Prog.abiSymbol[info.Name]; ok { + continue + } + + t := info.Raw + mset, hasUncommon := abiUncommonMethodSetForDeclaration(t) + var methods []*types.Selection + if hasUncommon { + methods = builder.abiInterfaceMethods(mset) + } + rt := p.Prog.rtNamed(p.Prog.abi.RuntimeName(t)) + var typ types.Type = rt + if hasUncommon { + ut := p.Prog.rtNamed("uncommonType") + mt := p.Prog.rtNamed("Method") + typ = types.NewStruct([]*types.Var{ + types.NewVar(token.NoPos, nil, "T", rt), + types.NewVar(token.NoPos, nil, "U", ut), + types.NewVar(token.NoPos, nil, "M", types.NewArray(mt, int64(len(methods)))), + }, nil) + } + p.Prog.abiSymbol[info.Name] = &AbiSymbol{ + Name: info.Name, + PkgPath: p.Path(), + Raw: t, + Typ: p.Prog.Type(types.NewPointer(typ), InGo), + MSet: mset, + } + } +} + func (p Package) getAbiTypesFor(name string, filter func(sym *AbiSymbol) bool) Expr { prog := p.Prog var names []string diff --git a/ssa/package.go b/ssa/package.go index 7abed72b69..8880099c91 100644 --- a/ssa/package.go +++ b/ssa/package.go @@ -254,6 +254,13 @@ type AbiSymbol struct { MSet *types.MethodSet } +// AbiTypeInfo is the LLVM-free identity needed to declare one runtime type +// descriptor in another Program. +type AbiTypeInfo struct { + Name string + Raw types.Type +} + // A Program presents a program. type Program = *aProgram diff --git a/ssa/ssa_test.go b/ssa/ssa_test.go index 2a5ab7d4f6..db128b56d1 100644 --- a/ssa/ssa_test.go +++ b/ssa/ssa_test.go @@ -2870,6 +2870,57 @@ func TestInitAbiTypesForSubset(t *testing.T) { } } +func TestRegisterAbiTypesAcrossPrograms(t *testing.T) { + runtimePkg, err := importer.For("source", nil).Import(PkgRuntime) + if err != nil { + t.Fatal(err) + } + newProgram := func() Program { + prog := NewProgram(nil) + prog.sizes = types.SizesFor("gc", runtime.GOARCH) + prog.SetRuntime(runtimePkg) + return prog + } + + source := newProgram() + defer source.Dispose() + sourcePkg := source.NewPackage("source", "example.com/source") + sourceFn := sourcePkg.NewFunc("source", NoArgsNoRet, InC) + sourceBuilder := sourceFn.MakeBody(1) + sourceBuilder.abiType(types.NewSlice(types.Typ[types.Int])) + named := types.NewNamed( + types.NewTypeName(token.NoPos, types.NewPackage("example.com/source", "source"), "Named", nil), + types.NewStruct(nil, nil), + nil, + ) + sourceBuilder.abiType(named) + sourceBuilder.Return() + infos := source.AbiTypes() + if len(infos) == 0 { + t.Fatal("source Program produced no ABI type snapshot") + } + + target := newProgram() + defer target.Dispose() + targetPkg := target.NewPackage("target", "example.com/target") + targetPkg.RegisterAbiTypes(infos) + targetPkg.RegisterAbiTypes(append(infos, AbiTypeInfo{})) + for _, info := range infos { + if _, ok := target.abiSymbol[info.Name]; !ok { + t.Fatalf("target Program did not recreate ABI type %q", info.Name) + } + } + if targetPkg.InitAbiTypes("init$abitypes") == nil { + t.Fatal("recreated ABI types did not produce typelist initializer") + } + for _, info := range infos { + global := targetPkg.Module().NamedGlobal(info.Name) + if global.IsNil() || !global.IsDeclaration() { + t.Fatalf("target descriptor %q is not an external declaration", info.Name) + } + } +} + func TestInitAbiTypesForEmptySelection(t *testing.T) { prog := NewProgram(nil) pkg := prog.NewPackage("bar", "foo/bar") diff --git a/xtool/env/env.go b/xtool/env/env.go index 469b7e97f2..1c14c07121 100644 --- a/xtool/env/env.go +++ b/xtool/env/env.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "regexp" "strings" @@ -32,7 +33,19 @@ var ( ) func ExpandEnvToArgs(s string) []string { - r, config := expandEnvWithCmd(s) + r, config := expandEnvWithCmd(s, "", nil) + return expandedArgs(r, config) +} + +// ExpandEnvToArgsWith expands variables and supported helper commands using +// the supplied request directory and environment. A non-nil environ prevents +// subprocesses and variable expansion from consulting process-global state. +func ExpandEnvToArgsWith(s, dir string, environ []string) []string { + r, config := expandEnvWithCmd(s, dir, environ) + return expandedArgs(r, config) +} + +func expandedArgs(r string, config bool) []string { if r == "" { return nil } @@ -43,11 +56,11 @@ func ExpandEnvToArgs(s string) []string { } func ExpandEnv(s string) string { - r, _ := expandEnvWithCmd(s) + r, _ := expandEnvWithCmd(s, "", nil) return r } -func expandEnvWithCmd(s string) (string, bool) { +func expandEnvWithCmd(s, dir string, environ []string) (string, bool) { var config bool expanded := reSubcmd.ReplaceAllStringFunc(s, func(m string) string { subcmd := strings.TrimSpace(m[2 : len(m)-1]) @@ -61,7 +74,16 @@ func expandEnvWithCmd(s string) (string, bool) { var out []byte var err error - out, err = exec.Command(cmd, args[1:]...).Output() + executable := cmd + if environ != nil { + executable = lookPathInEnvironment(cmd, dir, environ) + } + command := exec.Command(executable, args[1:]...) + command.Dir = dir + if environ != nil { + command.Env = append([]string(nil), environ...) + } + out, err = command.Output() if err != nil { // TODO(kindy): log in verbose mode @@ -70,7 +92,46 @@ func expandEnvWithCmd(s string) (string, bool) { return strings.Replace(strings.TrimSpace(string(out)), "\n", " ", -1) }) - return strings.TrimSpace(os.Expand(expanded, os.Getenv)), config + lookup := os.Getenv + if environ != nil { + lookup = func(key string) string { + prefix := key + "=" + for i := len(environ) - 1; i >= 0; i-- { + if strings.HasPrefix(environ[i], prefix) { + return strings.TrimPrefix(environ[i], prefix) + } + } + return "" + } + } + return strings.TrimSpace(os.Expand(expanded, lookup)), config +} + +func lookPathInEnvironment(name, dir string, environ []string) string { + if strings.ContainsRune(name, filepath.Separator) { + return name + } + path := "" + prefix := "PATH=" + for i := len(environ) - 1; i >= 0; i-- { + if strings.HasPrefix(environ[i], prefix) { + path = strings.TrimPrefix(environ[i], prefix) + break + } + } + for _, entry := range filepath.SplitList(path) { + if entry == "" { + entry = "." + } + if !filepath.IsAbs(entry) && dir != "" { + entry = filepath.Join(dir, entry) + } + candidate := filepath.Join(entry, name) + if info, err := os.Stat(candidate); err == nil && !info.IsDir() && info.Mode()&0o111 != 0 { + return candidate + } + } + return name } func parseSubcmd(s string) []string { diff --git a/xtool/env/env_test.go b/xtool/env/env_test.go new file mode 100644 index 0000000000..953fddf6e1 --- /dev/null +++ b/xtool/env/env_test.go @@ -0,0 +1,42 @@ +package env + +import ( + "os" + "path/filepath" + "reflect" + "runtime" + "testing" +) + +func TestExpandEnvToArgsWithUsesExplicitEnvironment(t *testing.T) { + t.Setenv("LLGO_ENV_TEST", "ambient") + got := ExpandEnvToArgsWith("$LLGO_ENV_TEST", "", []string{"LLGO_ENV_TEST=request"}) + if want := []string{"request"}; !reflect.DeepEqual(got, want) { + t.Fatalf("ExpandEnvToArgsWith = %q, want %q", got, want) + } +} + +func TestExpandEnvToArgsWithConfiguresSubprocess(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("shell fixture is Unix-only") + } + dir := t.TempDir() + tool := filepath.Join(dir, "pkg-config") + script := "#!/bin/sh\nprintf '%s' \"-L$LLGO_ENV_TEST -I$PWD\"\n" + if err := os.WriteFile(tool, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + got := ExpandEnvToArgsWith( + "$(pkg-config --libs fixture)", + dir, + []string{"PATH=" + dir, "LLGO_ENV_TEST=request"}, + ) + resolvedDir, err := filepath.EvalSymlinks(dir) + if err != nil { + t.Fatal(err) + } + want := []string{"-Lrequest", "-I" + resolvedDir} + if !reflect.DeepEqual(got, want) { + t.Fatalf("ExpandEnvToArgsWith = %q, want %q", got, want) + } +} From 466c31adc77bb71c1d0e120556e5da8c9d787e31 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 00:45:54 +0800 Subject: [PATCH 16/22] test: cover parallel backend task setup --- internal/build/package_build_test.go | 72 ++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go index b34863b4ea..a62ed9141d 100644 --- a/internal/build/package_build_test.go +++ b/internal/build/package_build_test.go @@ -21,6 +21,7 @@ import ( "go/types" "os" "path/filepath" + "strings" "sync/atomic" "testing" "time" @@ -306,3 +307,74 @@ func TestRunBoundedPackageJobsConvertsPanicToError(t *testing.T) { t.Fatalf("error = %v, want recovered panic %v", err, boom) } } +func TestPackageBuildStageEmptyAndSkippedInputs(t *testing.T) { + runtimePkg := &aPackage{Package: &packages.Package{PkgPath: env.LLGoRuntimePkg}} + normalPkg := &aPackage{Package: &packages.Package{PkgPath: "example.com/normal"}} + specs := []packageBuildSpec{ + {pkg: runtimePkg, runtime: true}, + {pkg: normalPkg}, + } + if got := packageBuildSpecsForRuntime(specs, true); len(got) != 1 || got[0].pkg != runtimePkg { + t.Fatalf("runtime specs = %+v, want runtime package", got) + } + if got := packageBuildSpecsForRuntime(specs, false); len(got) != 1 || got[0].pkg != normalPkg { + t.Fatalf("non-runtime specs = %+v, want normal package", got) + } + + ctx := &context{buildConf: &Config{}} + preflights, err := preflightPackageBuilds(ctx, nil, false) + if err != nil || len(preflights) != 0 { + t.Fatalf("empty preflights = %#v, %v", preflights, err) + } + results, err := buildPreflightedPackageGroup(ctx, nil, preflights, false) + if err != nil || results != nil { + t.Fatalf("empty package group = %#v, %v", results, err) + } + if err := executePreflightedPackage(ctx, packagePreflight{skip: true}, false); err != nil { + t.Fatal(err) + } + if err := executeIsolatedPackages(ctx, nil, nil, nil, false); err != nil { + t.Fatal(err) + } + if err := runBoundedPackageJobs(0, nil, func(int) error { + t.Fatal("empty package jobs invoked callback") + return nil + }); err != nil { + t.Fatal(err) + } + if err := runPackageJob(7, func(int) error { panic("boom") }); err == nil || !strings.Contains(err.Error(), "package job 7 panicked: boom") { + t.Fatalf("non-error panic = %v", err) + } +} + +func TestNewBackendTaskUsesPackageLocalState(t *testing.T) { + coordinator := &context{ + conf: &packages.Config{}, + mode: ModeBuild, + buildConf: &Config{BuildMode: BuildModeExe}, + commands: commandEnv{dir: t.TempDir()}, + frontendOptions: cl.Options{Debug: true}, + sfilesCache: map[string][]string{"example.com/p": {"asm.s"}}, + plan9asmReady: true, + plan9asmMode: plan9asmEnvSelected, + plan9asmPkgs: map[string]bool{"example.com/p": true}, + } + + task := coordinator.newBackendTask(backendSession{}) + if task == coordinator { + t.Fatal("backend task aliases coordinator") + } + if task.buildConf != coordinator.buildConf || task.conf != coordinator.conf { + t.Fatal("backend task did not retain immutable build inputs") + } + if !task.sfilesFrozen || !task.plan9asmReady || task.plan9asmMode != plan9asmEnvSelected { + t.Fatalf("backend task state = %+v", task) + } + if !task.frontendOptions.Debug || task.commands.dir != coordinator.commands.dir { + t.Fatal("backend task lost invocation settings") + } + task.plan9asmSigs["example.com/p"] = map[string]struct{}{"f": {}} + if coordinator.plan9asmSigs != nil { + t.Fatal("backend task signature cache aliases coordinator") + } +} From 66d3e9f5bda44a86999e8a51b968a2bbb397de06 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 02:37:14 +0800 Subject: [PATCH 17/22] test: cover package scheduling boundaries --- internal/build/package_build_test.go | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go index a62ed9141d..00d1ce3963 100644 --- a/internal/build/package_build_test.go +++ b/internal/build/package_build_test.go @@ -378,3 +378,37 @@ func TestNewBackendTaskUsesPackageLocalState(t *testing.T) { t.Fatal("backend task signature cache aliases coordinator") } } + +func TestPreflightPackageBuildsRecordsSkippedPackages(t *testing.T) { + pkg := &aPackage{Package: &packages.Package{ + ID: "example.com/already-built", + PkgPath: "example.com/already-built", + }} + ctx := &context{built: map[string]none{pkg.ID: {}}} + preflights, err := preflightPackageBuilds(ctx, []packageBuildSpec{{pkg: pkg}}, false) + if err != nil { + t.Fatal(err) + } + preflight, ok := preflights[pkg] + if !ok || !preflight.skip || preflight.spec.pkg != pkg { + t.Fatalf("preflights = %#v, want skipped package", preflights) + } +} + +func TestPackageSchedulingHandlesNonBackendPackages(t *testing.T) { + spec := packageBuildSpec{pkg: &aPackage{}} + ctx := &context{mode: ModeGen, buildConf: &Config{BuildMode: BuildModeExe}} + serial, err := ctx.packageRequiresCoordinator(spec) + if err != nil || !serial { + t.Fatalf("generation package coordinator = %v, %v; want true, nil", serial, err) + } + + usesPlan9, err := (&context{}).packageUsesPlan9Asm(spec.pkg) + if err != nil || usesPlan9 { + t.Fatalf("nil package Plan9 asm = %v, %v; want false, nil", usesPlan9, err) + } + patched, coordinator, isolated, err := partitionPackageExecutions(ctx, nil) + if err != nil || patched != nil || coordinator != nil || isolated != nil { + t.Fatalf("empty partition = %v, %v, %v, %v", patched, coordinator, isolated, err) + } +} From 8cf97185b6a83ffaa417b79beb5e3ba2d91bbf26 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 03:36:18 +0800 Subject: [PATCH 18/22] test: cover Plan9 asm package selection --- internal/build/plan9asm_env_test.go | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 internal/build/plan9asm_env_test.go diff --git a/internal/build/plan9asm_env_test.go b/internal/build/plan9asm_env_test.go new file mode 100644 index 0000000000..cb017e747b --- /dev/null +++ b/internal/build/plan9asm_env_test.go @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import "testing" + +func TestParsePlan9AsmPkgsEnv(t *testing.T) { + tests := []struct { + raw string + mode plan9asmPkgsEnvMode + pkgs map[string]bool + }{ + {raw: "", mode: plan9asmEnvDefaults}, + {raw: " off ", mode: plan9asmEnvNone}, + {raw: "FALSE", mode: plan9asmEnvNone}, + {raw: "*", mode: plan9asmEnvAll}, + {raw: "On", mode: plan9asmEnvAll}, + {raw: "runtime, net ; crypto\t net\n", mode: plan9asmEnvSelected, pkgs: map[string]bool{"runtime": true, "net": true, "crypto": true}}, + } + for _, tt := range tests { + got := parsePlan9AsmPkgsEnv(tt.raw) + if got.mode != tt.mode { + t.Fatalf("parsePlan9AsmPkgsEnv(%q).mode = %v, want %v", tt.raw, got.mode, tt.mode) + } + if len(got.pkgs) != len(tt.pkgs) { + t.Fatalf("parsePlan9AsmPkgsEnv(%q).pkgs = %v, want %v", tt.raw, got.pkgs, tt.pkgs) + } + for pkg := range tt.pkgs { + if !got.pkgs[pkg] { + t.Fatalf("parsePlan9AsmPkgsEnv(%q) did not retain %q", tt.raw, pkg) + } + } + } +} From 4501ff42fe730a3a9f83b9ab823c44d6500c19e6 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 08:39:28 +0800 Subject: [PATCH 19/22] build: replay locality state in backend sessions --- internal/build/build.go | 37 ++++++++++------------ ssa/locality.go | 69 +++++++++++++++++++++++++++++++++++++++++ ssa/locality_test.go | 28 +++++++++++++++++ 3 files changed, 113 insertions(+), 21 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index 16d4c19c59..cbcbf61fad 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -599,7 +599,8 @@ func Build(inv Invocation) ([]Package, error) { if pkg := dedup.Check(llssa.PkgPython); pkg != nil { backendTemplate.pythonPackage = pkg.Types } - backendTemplate.inputs = collectBackendProgramInputs(initial, altPkgs) + backendTemplate.inputs = collectBackendProgramInputs(prog, initial, altPkgs) + backendTemplate.localities = prog.SnapshotLocalityState() backendTemplate.llvmTarget = export.LLVMTarget backendTemplate.targetABI = export.TargetABI @@ -938,12 +939,11 @@ func (c *context) closePackageMetas() { } type backendProgramInput struct { - fset *token.FileSet - pkg *types.Package - info *types.Info - files []*ast.File - parseSyntax bool - prepareLocal bool + fset *token.FileSet + pkg *types.Package + info *types.Info + files []*ast.File + parseSyntax bool } // backendProgramTemplate contains only immutable build-local inputs. Creating @@ -962,6 +962,7 @@ type backendProgramTemplate struct { runtimePackage *types.Package pythonPackage *types.Package inputs []backendProgramInput + localities llssa.LocalityState llvmTarget string targetABI string abiMode cabi.Mode @@ -1040,17 +1041,11 @@ func (t backendProgramTemplate) replayProgramState(prog llssa.Program) error { } } } - for _, input := range t.inputs { - if input.prepareLocal { - if err := cl.PrepareLocalVariables(prog, input.fset, input.pkg, input.info, input.files); err != nil { - return err - } - } - } + prog.RestoreLocalityState(t.localities) return nil } -func collectBackendProgramInputs(groups ...[]*packages.Package) []backendProgramInput { +func collectBackendProgramInputs(prog llssa.Program, groups ...[]*packages.Package) []backendProgramInput { seen := make(map[*types.Package]bool) var inputs []backendProgramInput for _, roots := range groups { @@ -1059,13 +1054,13 @@ func collectBackendProgramInputs(groups ...[]*packages.Package) []backendProgram return } seen[pkg.Types] = true + parsed := prog.PackageSyntaxParsed(pkg.Types) inputs = append(inputs, backendProgramInput{ - fset: pkg.Fset, - pkg: pkg.Types, - info: pkg.TypesInfo, - files: slices.Clone(pkg.Syntax), - parseSyntax: !llruntime.SkipToBuild(pkg.PkgPath), - prepareLocal: true, + fset: pkg.Fset, + pkg: pkg.Types, + info: pkg.TypesInfo, + files: slices.Clone(pkg.Syntax), + parseSyntax: parsed && !llruntime.SkipToBuild(pkg.PkgPath), }) }) } diff --git a/ssa/locality.go b/ssa/locality.go index 0e9b41e6cd..cd3402b773 100644 --- a/ssa/locality.go +++ b/ssa/locality.go @@ -61,6 +61,17 @@ type localityInfos struct { parsedPackages map[*types.Package]struct{} } +// LocalityState is a copyable snapshot of the Go-side locality metadata owned +// by one Program. It contains no LLVM objects and can therefore be replayed +// safely into an independent backend Program. +type LocalityState struct { + entries map[string]VariableLocality + ownerlessEntries map[string]VariableLocality + declarationEntries map[string]map[string]VariableLocality + activePackages map[string]struct{} + parsedPackages map[*types.Package]struct{} +} + func newLocalityInfos() *localityInfos { return &localityInfos{ entries: make(map[string]VariableLocality), @@ -71,6 +82,64 @@ func newLocalityInfos() *localityInfos { } } +// SnapshotLocalityState returns an independent snapshot of p's locality +// metadata. Callers may use it after preflight has completed to seed isolated +// backend sessions without sharing mutable Program state. +func (p Program) SnapshotLocalityState() LocalityState { + p.localities.mu.RLock() + defer p.localities.mu.RUnlock() + state := LocalityState{ + entries: cloneLocalityEntries(p.localities.entries), + ownerlessEntries: cloneLocalityEntries(p.localities.ownerlessEntries), + activePackages: cloneLocalitySet(p.localities.activePackages), + parsedPackages: cloneParsedPackages(p.localities.parsedPackages), + } + state.declarationEntries = make(map[string]map[string]VariableLocality, len(p.localities.declarationEntries)) + for name, entries := range p.localities.declarationEntries { + state.declarationEntries[name] = cloneLocalityEntries(entries) + } + return state +} + +// RestoreLocalityState replaces p's locality metadata with an independent +// copy of state. The copied maps keep backend sessions independent. +func (p Program) RestoreLocalityState(state LocalityState) { + p.localities.mu.Lock() + p.localities.entries = cloneLocalityEntries(state.entries) + p.localities.ownerlessEntries = cloneLocalityEntries(state.ownerlessEntries) + p.localities.activePackages = cloneLocalitySet(state.activePackages) + p.localities.parsedPackages = cloneParsedPackages(state.parsedPackages) + p.localities.declarationEntries = make(map[string]map[string]VariableLocality, len(state.declarationEntries)) + for name, entries := range state.declarationEntries { + p.localities.declarationEntries[name] = cloneLocalityEntries(entries) + } + p.localities.mu.Unlock() +} + +func cloneLocalityEntries(entries map[string]VariableLocality) map[string]VariableLocality { + ret := make(map[string]VariableLocality, len(entries)) + for name, entry := range entries { + ret[name] = entry + } + return ret +} + +func cloneLocalitySet(entries map[string]struct{}) map[string]struct{} { + ret := make(map[string]struct{}, len(entries)) + for name := range entries { + ret[name] = struct{}{} + } + return ret +} + +func cloneParsedPackages(entries map[*types.Package]struct{}) map[*types.Package]struct{} { + ret := make(map[*types.Package]struct{}, len(entries)) + for pkg := range entries { + ret[pkg] = struct{}{} + } + return ret +} + func (p *localityInfos) update(name string, update func(*VariableLocality)) { p.mu.Lock() info := p.entries[name] diff --git a/ssa/locality_test.go b/ssa/locality_test.go index 1d6ad63cb5..b2a79f0ad1 100644 --- a/ssa/locality_test.go +++ b/ssa/locality_test.go @@ -63,6 +63,34 @@ func TestLocalityInfos(t *testing.T) { } } +func TestLocalityStateSnapshotIsIndependent(t *testing.T) { + source := NewProgram(nil) + defer source.Dispose() + pkg := types.NewPackage("example.com/p", "p") + name := "example.com/p.state" + source.DeclareLocality(pkg, "state", LocalityInfo{Locality: GoroutineLocal, HasInitializer: true}) + source.SetLocalStorageFor(pkg, name, LocalStoragePackage) + source.ActivateLocalitiesFor(pkg) + source.MarkPackageSyntaxParsed(pkg) + + dest := NewProgram(nil) + defer dest.Dispose() + dest.RestoreLocalityState(source.SnapshotLocalityState()) + got, ok := dest.VariableLocalityFor(pkg, name) + if !ok || got.Locality != GoroutineLocal || got.LocalStorage != LocalStoragePackage { + t.Fatalf("restored locality = %+v, %v", got, ok) + } + if !dest.NeedsLocalContext() || !dest.PackageSyntaxParsed(pkg) { + t.Fatal("restored locality state lost active or parsed metadata") + } + + source.SetLocalStorageFor(pkg, name, LocalStorageNativeTLS) + got, ok = dest.VariableLocalityFor(pkg, name) + if !ok || got.LocalStorage != LocalStoragePackage { + t.Fatalf("restored locality aliased source state: %+v, %v", got, ok) + } +} + func TestPackageLocalitiesRetainDeclarationOwners(t *testing.T) { prog := NewProgram(nil) std := types.NewPackage("runtime", "runtime") From b7c5539f3a2068064d79bbe3b6bd546e612b38ee Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 09:48:11 +0800 Subject: [PATCH 20/22] build: pipeline package SSA into backends --- cl/caller_tracking_precompute_test.go | 35 ++ cl/compile.go | 24 +- cl/import.go | 28 +- cl/instr.go | 53 ++- cl/patch_prepare_test.go | 96 +++++ internal/build/build.go | 17 +- internal/build/build_test.go | 58 ++- internal/build/package_build.go | 18 +- internal/build/package_pipeline.go | 539 ++++++++++++++++++++++++ internal/build/package_pipeline_test.go | 113 +++++ internal/typepatch/patch.go | 7 + internal/typepatch/patch_test.go | 81 ++++ 12 files changed, 1050 insertions(+), 19 deletions(-) create mode 100644 cl/patch_prepare_test.go create mode 100644 internal/build/package_pipeline.go create mode 100644 internal/build/package_pipeline_test.go create mode 100644 internal/typepatch/patch_test.go diff --git a/cl/caller_tracking_precompute_test.go b/cl/caller_tracking_precompute_test.go index de78c7dcb9..c70eb8859d 100644 --- a/cl/caller_tracking_precompute_test.go +++ b/cl/caller_tracking_precompute_test.go @@ -74,3 +74,38 @@ func Logs() { dep.Where() } t.Fatalf("frozen extended lookup for unknown package = %v, want nil", got) } } + +func TestNewPackageCallerTrackingMatchesWholeProgramPrecompute(t *testing.T) { + dep, root := buildCallerFrameSSAProgram(t, + "example.com/dep", `package dep +import "runtime" +func Where() { runtime.Caller(0) } +`, + "example.com/root", `package root +import "example.com/dep" +func Logs() { dep.Where() } +`) + + whole := NewCallerTracking() + whole.Precompute([]*gossa.Package{root}) + local := NewPackageCallerTracking( + root, + SummarizeCallerTracking(dep), + SummarizeCallerTracking(root), + ) + if !local.frozen { + t.Fatal("package caller tracking was not frozen") + } + if got, want := local.base[dep][dep.Func("Where")], whole.base[dep][dep.Func("Where")]; got != want { + t.Fatalf("dependency base tracking = %v, want %v", got, want) + } + if got, want := local.extended[root][root.Func("Logs")], whole.extended[root][root.Func("Logs")]; got != want { + t.Fatalf("root extended tracking = %v, want %v", got, want) + } + if len(local.extended) != 1 { + t.Fatalf("package snapshot computed %d extended package sets, want 1", len(local.extended)) + } + if len(local.base) != 2 { + t.Fatalf("package snapshot retained %d base package sets, want 2", len(local.base)) + } +} diff --git a/cl/compile.go b/cl/compile.go index da72415026..c44729ddab 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -2091,11 +2091,27 @@ func (p *context) compileValues(b llssa.Builder, vals []ssa.Value, hasVArg int) type Patch struct { Alt *ssa.Package Types *types.Package + + skips map[string]none + skipall bool + prepared bool } // Patches is patches of some packages. type Patches = map[string]Patch +// PreparePatch creates the immutable merged type view imported by packages +// compiled before the patched package's own LLVM backend runs. +func PreparePatch(patch Patch, original *types.Package, files []*ast.File) Patch { + if patch.prepared || patch.Types == nil || original == nil { + return patch + } + patch.skips, patch.skipall = collectPatchSkips(files) + typepatch.MergePrepared(patch.Types, original, patch.skips, patch.skipall) + patch.prepared = true + return patch +} + // NewPackage compiles a Go package to LLVM IR package. Concurrent build // drivers should use NewPackageExWithEmbedMetaOptions to pass frontend options // explicitly. @@ -2223,6 +2239,10 @@ func newPackageEx(prog llssa.Program, ct *CallerTracking, patches Patches, rewri if hasPatch { skips := ctx.skips + if patch.prepared { + skips = patch.skips + ctx.skipall = patch.skipall + } typepatch.Merge(pkgTypes, oldTypes, skips, ctx.skipall) ctx.skips = nil ctx.state = pkgInPatch @@ -2378,8 +2398,8 @@ func (p *context) _patchType(typ types.Type) (types.Type, bool) { return t, true } o := typ.Obj() - if pkg := o.Pkg(); typepatch.IsPatched(pkg) { - if patch, ok := p.patches[pkg.Path()]; ok { + if pkg := o.Pkg(); pkg != nil { + if patch, ok := p.patches[pkg.Path()]; ok && patch.Types != nil { if obj := patch.Types.Scope().Lookup(o.Name()); obj != nil { raw := p.prog.Type(instantiate(obj.Type(), typ), llssa.InGo).RawType() return raw, typ != raw diff --git a/cl/import.go b/cl/import.go index 0caa1a96f3..48e4500e85 100644 --- a/cl/import.go +++ b/cl/import.go @@ -136,7 +136,12 @@ func (p *context) importPkg(pkg *types.Package, i *pkgInfo) { kind, _ := pkgKindByScope(scope) if kind == PkgNormal { if patch, ok := p.patches[pkgPath]; ok { - pkg = patch.Alt.Pkg + switch { + case patch.Types != nil: + pkg = patch.Types + case patch.Alt != nil: + pkg = patch.Alt.Pkg + } scope = pkg.Scope() if kind, _ = pkgKindByScope(scope); kind != PkgNormal { goto start @@ -220,6 +225,27 @@ func (p *context) initFiles(pkgPath string, files []*ast.File, cPkg bool) { } } +func collectPatchSkips(files []*ast.File) (map[string]none, bool) { + ctx := &context{skips: make(map[string]none)} + for _, file := range files { + for _, decl := range file.Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok { + continue + } + switch gen.Tok { + case token.CONST, token.TYPE: + ctx.collectSkipNamesByDoc(gen.Doc) + case token.IMPORT: + if gen.Doc != nil && len(gen.Doc.List) != 0 { + ctx.collectSkipNames(gen.Doc.List[len(gen.Doc.List)-1].Text) + } + } + } + } + return ctx.skips, ctx.skipall +} + // Collect skip names and skip other annotations, such as go: and llgo: // llgo:skip symbol1 symbol2 ... // llgo:skipall diff --git a/cl/instr.go b/cl/instr.go index 1a85a81e26..33f91e3f07 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -974,15 +974,13 @@ func runtimeCallerFuncSet(c *CallerTracking, pkg *ssa.Package) map[*ssa.Function return out } -// CallerTracking memoizes the per-package caller-tracking sets for one -// compilation. Like Patches, it is compilation-scoped state owned by the -// driver: create one per compilation and pass it to every -// NewPackageExWithEmbed call of that compilation, so cross-package -// queries (criterion 2 below) hit the memoization. It must not outlive -// the compilation — the maps are keyed by *ssa.Package with -// *ssa.Function values, so anything longer-lived would pin every -// compiled package's go/types and go/ssa graphs. Concurrent drivers call -// Precompute and share only the resulting frozen, read-only maps. +// CallerTracking memoizes per-package caller-tracking sets. A two-phase driver +// may create one compilation-scoped instance, call Precompute, and share the +// frozen result. A pipelined driver instead creates one instance per backend +// with NewPackageCallerTrackingForPackages, seeded by immutable summaries +// produced as SSA packages finish. Neither form may outlive the compilation: +// the maps are keyed by *ssa.Package with *ssa.Function values, so longer-lived +// state would pin every compiled package's go/types and go/ssa graphs. type CallerTracking struct { base map[*ssa.Package]map[*ssa.Function]bool extended map[*ssa.Package]map[*ssa.Function]bool @@ -1032,6 +1030,43 @@ func (c *CallerTracking) Precompute(pkgs []*ssa.Package) { c.frozen = true } +// CallerTrackingSummary is the immutable caller-tracking state computed when +// one package finishes SSA construction. +type CallerTrackingSummary struct { + pkg *ssa.Package + base map[*ssa.Function]bool +} + +// SummarizeCallerTracking computes the package-local caller base set once. +func SummarizeCallerTracking(pkg *ssa.Package) CallerTrackingSummary { + return CallerTrackingSummary{pkg: pkg, base: computeRuntimeCallerBaseSet(pkg)} +} + +// NewPackageCallerTracking computes and freezes the extended caller-tracking +// state needed to compile pkg. summaries should contain pkg and its effective +// dependencies, allowing package backends to reuse the package-local analysis +// performed as each package completed SSA construction. +func NewPackageCallerTracking(pkg *ssa.Package, summaries ...CallerTrackingSummary) *CallerTracking { + return NewPackageCallerTrackingForPackages([]*ssa.Package{pkg}, summaries...) +} + +// NewPackageCallerTrackingForPackages is NewPackageCallerTracking for a +// backend that compiles more than one SSA package, such as an alternate-package +// patch. +func NewPackageCallerTrackingForPackages(pkgs []*ssa.Package, summaries ...CallerTrackingSummary) *CallerTracking { + c := NewCallerTracking() + for _, summary := range summaries { + if summary.pkg != nil { + c.base[summary.pkg] = summary.base + } + } + for _, pkg := range pkgs { + runtimeCallerFuncSet(c, pkg) + } + c.frozen = true + return c +} + // NewCallerTracking creates the caller-tracking memoization for one // compilation. func NewCallerTracking() *CallerTracking { diff --git a/cl/patch_prepare_test.go b/cl/patch_prepare_test.go new file mode 100644 index 0000000000..ecabbd9828 --- /dev/null +++ b/cl/patch_prepare_test.go @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "go/ast" + "go/parser" + "go/token" + "go/types" + "testing" + + "github.com/goplus/llgo/internal/typepatch" +) + +func TestPreparePatchBuildsImmutableMergedTypes(t *testing.T) { + original := types.NewPackage("example.com/p", "p") + original.Scope().Insert(types.NewVar(token.NoPos, original, "Keep", types.Typ[types.Int])) + original.Scope().Insert(types.NewVar(token.NoPos, original, "Skip", types.Typ[types.Int])) + alternate := types.NewPackage("example.com/p", "p") + alternate.Scope().Insert(types.NewVar(token.NoPos, alternate, "Alt", types.Typ[types.Int])) + + file, err := parser.ParseFile(token.NewFileSet(), "p.go", `package p +//llgo:skip Skip +type T int +`, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + prepared := PreparePatch(Patch{Types: typepatch.Clone(alternate)}, original, []*ast.File{file}) + if !prepared.prepared { + t.Fatal("patch was not marked prepared") + } + for _, name := range []string{"Alt", "Keep"} { + if prepared.Types.Scope().Lookup(name) == nil { + t.Fatalf("prepared patch is missing %s", name) + } + } + if prepared.Types.Scope().Lookup("Skip") != nil { + t.Fatal("prepared patch retained skipped declaration") + } + if typepatch.IsPatched(original) { + t.Fatal("preparing a patch modified the original types package") + } + if original.Scope().Lookup("Keep") == nil || original.Scope().Lookup("Skip") == nil { + t.Fatal("preparing a patch modified the original package scope") + } +} + +func TestPreparePatchHandlesSkipAllAndNoopInputs(t *testing.T) { + original := types.NewPackage("example.com/p", "p") + original.Scope().Insert(types.NewVar(token.NoPos, original, "Keep", types.Typ[types.Int])) + alternate := types.NewPackage("example.com/p", "p") + alternate.Scope().Insert(types.NewVar(token.NoPos, alternate, "Alt", types.Typ[types.Int])) + + file, err := parser.ParseFile(token.NewFileSet(), "p.go", `package p +//llgo:skipall +import "unsafe" +var V int +func F() {} +`, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + prepared := PreparePatch(Patch{Types: typepatch.Clone(alternate)}, original, []*ast.File{file}) + if !prepared.prepared || !prepared.skipall { + t.Fatalf("prepared patch flags = (prepared=%v, skipall=%v), want true, true", prepared.prepared, prepared.skipall) + } + if prepared.Types.Scope().Lookup("Keep") != nil { + t.Fatal("skipall merged an original declaration") + } + + alreadyPrepared := Patch{Types: prepared.Types, prepared: true} + if got := PreparePatch(alreadyPrepared, original, nil); !got.prepared || got.Types != alreadyPrepared.Types { + t.Fatal("PreparePatch changed an already prepared patch") + } + if got := PreparePatch(Patch{}, original, nil); got.prepared { + t.Fatal("PreparePatch prepared a patch without a type package") + } + if got := PreparePatch(Patch{Types: prepared.Types}, nil, nil); got.prepared { + t.Fatal("PreparePatch prepared a patch without an original package") + } +} diff --git a/internal/build/build.go b/internal/build/build.go index cbcbf61fad..5a1b23af8c 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -208,6 +208,10 @@ type Config struct { GlobalRewrites map[string]Rewrites ModuleHook ModuleHook Overlay map[string][]byte + + // packagePipelineObserver is used by focused scheduler tests. It must be + // safe for concurrent calls. + packagePipelineObserver func(packagePipelineStage, string, bool) } type Rewrites map[string]string @@ -653,12 +657,17 @@ func Build(inv Invocation) ([]Package, error) { if err != nil { return nil, err } - buildSSAPkgs(ctx, append(append(altEntries, pkgEntries...), depEntries...)) - ctx.callerTracking.Precompute(ctx.progSSA.AllPackages()) - allPkgs := append([]*aPackage{}, pkgs...) allPkgs = append(allPkgs, depPkgs...) - allPkgs, err = buildAllPkgs(ctx, allPkgs, verbose) + ssaEntries := append(append(altEntries, pkgEntries...), depEntries...) + if ctx.canUseIsolatedBackend() { + preparePackagePatches(ctx, allPkgs) + allPkgs, err = buildPackagePipeline(ctx, ssaEntries, allPkgs, verbose) + } else { + buildSSAPkgs(ctx, ssaEntries) + ctx.callerTracking.Precompute(ctx.progSSA.AllPackages()) + allPkgs, err = buildAllPkgs(ctx, allPkgs, verbose) + } if err != nil { return nil, err } diff --git a/internal/build/build_test.go b/internal/build/build_test.go index e92e41f9bf..346b040e23 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -21,6 +21,7 @@ import ( "slices" "strconv" "strings" + "sync" "testing" "github.com/goplus/llgo/cl" @@ -520,7 +521,62 @@ func TestRun(t *testing.T) { func TestTest(t *testing.T) { // FIXME(zzy): with builtin package test in a llgo test ./... will cause duplicate symbol error - mockRun([]string{"../../cl/_testgo/runtest"}, &Config{Mode: ModeTest}) + var mu sync.Mutex + var events []struct { + stage packagePipelineStage + start bool + } + active, maximum := 0, 0 + exclusive, exclusiveViolation := false, false + conf := &Config{Mode: ModeTest, BuildParallelism: 2} + conf.packagePipelineObserver = func(stage packagePipelineStage, _ string, start bool) { + mu.Lock() + defer mu.Unlock() + events = append(events, struct { + stage packagePipelineStage + start bool + }{stage: stage, start: start}) + if start { + if exclusive || (stage == pipelineStagePatchedBackend && active != 0) { + exclusiveViolation = true + } + active++ + maximum = max(maximum, active) + if stage == pipelineStagePatchedBackend { + exclusive = true + } + } else { + active-- + if stage == pipelineStagePatchedBackend { + exclusive = false + } + } + } + mockRun([]string{"../../cl/_testgo/runtest"}, conf) + + mu.Lock() + defer mu.Unlock() + if active != 0 { + t.Fatalf("pipeline left %d active tasks", active) + } + if maximum > conf.BuildParallelism { + t.Fatalf("pipeline active tasks = %d, want at most %d", maximum, conf.BuildParallelism) + } + if exclusiveViolation { + t.Fatal("patched backend overlapped another pipeline task") + } + firstBackend, lastSSA := -1, -1 + for i, event := range events { + if event.stage != pipelineStageSSA && event.start && firstBackend < 0 { + firstBackend = i + } + if event.stage == pipelineStageSSA && !event.start { + lastSSA = i + } + } + if firstBackend < 0 || lastSSA < 0 || firstBackend >= lastSSA { + t.Fatalf("pipeline did not overlap SSA and backend work: first backend event %d, last SSA event %d", firstBackend, lastSSA) + } } func TestExtest(t *testing.T) { diff --git a/internal/build/package_build.go b/internal/build/package_build.go index 694077f3f9..372b6c6f31 100644 --- a/internal/build/package_build.go +++ b/internal/build/package_build.go @@ -266,12 +266,16 @@ func (ctx *context) packageUsesPlan9Asm(pkg *aPackage) (bool, error) { } func (ctx *context) executeIsolatedPackage(spec packageBuildSpec, verbose bool) error { + return ctx.executeIsolatedPackageWithCallerTracking(spec, ctx.callerTracking, verbose) +} + +func (ctx *context) executeIsolatedPackageWithCallerTracking(spec packageBuildSpec, tracking *cl.CallerTracking, verbose bool) error { session, err := ctx.backend.newSession() if err != nil { return fmt.Errorf("create backend session for %s: %w", spec.pkg.PkgPath, err) } defer session.prog.Dispose() - task := ctx.newBackendTask(session) + task := ctx.newBackendTaskWithCallerTracking(session, tracking) defer func() { spec.pkg.LPkg = nil }() if err := buildPkg(task, spec.pkg, verbose); err != nil { @@ -310,13 +314,17 @@ func preparePackageSFiles(ctx *context, pkg *aPackage) error { } func (ctx *context) newBackendTask(session backendSession) *context { + return ctx.newBackendTaskWithCallerTracking(session, ctx.callerTracking) +} + +func (ctx *context) newBackendTaskWithCallerTracking(session backendSession, tracking *cl.CallerTracking) *context { return &context{ conf: ctx.conf, progSSA: ctx.progSSA, prog: session.prog, dedup: ctx.dedup, patches: ctx.patches, - callerTracking: ctx.callerTracking, + callerTracking: tracking, initial: ctx.initial, pkgs: ctx.pkgs, pkgByID: ctx.pkgByID, @@ -338,6 +346,12 @@ func (ctx *context) newBackendTask(session backendSession) *context { } } +func (ctx *context) executeCoordinatorPackageWithCallerTracking(spec packageBuildSpec, tracking *cl.CallerTracking, verbose bool) error { + task := *ctx + task.callerTracking = tracking + return executePackageBuild(&task, spec, verbose) +} + func packageBuildResultFor(spec packageBuildSpec) packageBuildResult { return packageBuildResult{ needRuntime: spec.pkg.NeedRt, diff --git a/internal/build/package_pipeline.go b/internal/build/package_pipeline.go new file mode 100644 index 0000000000..9021ab4bc5 --- /dev/null +++ b/internal/build/package_pipeline.go @@ -0,0 +1,539 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "fmt" + "go/ast" + + "github.com/goplus/llgo/cl" + "golang.org/x/tools/go/ssa" +) + +func preparePackagePatches(ctx *context, pkgs []*aPackage) { + for _, pkg := range pkgs { + patch, ok := ctx.patches[pkg.PkgPath] + if !ok { + continue + } + files := append([]*ast.File(nil), pkg.Syntax...) + if pkg.AltPkg != nil { + files = append(files, pkg.AltPkg.Syntax...) + } + ctx.patches[pkg.PkgPath] = cl.PreparePatch(patch, pkg.Types, files) + } +} + +type packagePipelineClass uint8 + +const ( + pipelineIsolated packagePipelineClass = iota + pipelineCoordinator +) + +type packagePipelineStage uint8 + +const ( + pipelineStageSSA packagePipelineStage = iota + pipelineStageBackend + pipelineStagePatchedBackend +) + +type packagePipelineSSA struct { + entry ssaBuildEntry + order int + done bool + callerSummary cl.CallerTrackingSummary +} + +type packagePipelineTask struct { + index int + spec packageBuildSpec + preflight packagePreflight + class packagePipelineClass + ownSSA *packagePipelineSSA + ssaDeps []*packagePipelineSSA + callerRoots []*ssa.Package + callerNodes []*packagePipelineSSA + enabled bool + running bool + done bool + executed bool + patched bool +} + +type packagePipelineEvent struct { + ssa *packagePipelineSSA + task *packagePipelineTask + err error +} + +// buildPackagePipeline overlaps completed Go SSA packages with isolated LLVM +// backends. SSA and backend work share one concurrency budget, so -p remains a +// bound on CPU-heavy package work rather than creating two independent pools. +func buildPackagePipeline(ctx *context, entries []ssaBuildEntry, pkgs []*aPackage, verbose bool) ([]*aPackage, error) { + specs := make([]packageBuildSpec, len(pkgs)) + for i, pkg := range pkgs { + specs[i] = newPackageBuildSpec(pkg) + } + preflights, err := preflightPackageBuilds(ctx, specs, verbose) + if err != nil { + return nil, err + } + ctx.sfilesFrozen = true + + ssaNodes, ssaByPackage := newPackagePipelineSSANodes(entries) + tasks, serialOrder, err := newPackagePipelineTasks(ctx, specs, preflights, ssaByPackage) + if err != nil { + return nil, err + } + ssaNodes = orderPackagePipelineSSA(ssaNodes, tasks) + + hostRuntime := ctx.buildConf.Target == "" + normalRemaining := 0 + backendFailed := false + var needRuntime, needPyInit bool + for _, task := range tasks { + task.enabled = !task.spec.runtime || hostRuntime + if task.preflight.skip { + task.done = true + if !task.spec.runtime { + result := packageBuildResultFor(task.spec) + needRuntime = needRuntime || result.needRuntime + needPyInit = needPyInit || result.needPyInit + } + continue + } + if !task.spec.runtime { + normalRemaining++ + } + } + + runtimeResolved := hostRuntime + resolveRuntime := func() { + if runtimeResolved || normalRemaining != 0 { + return + } + runtimeResolved = true + buildRuntime := !backendFailed && (needRuntime || needPyInit) + for _, task := range tasks { + if !task.spec.runtime || task.done { + continue + } + task.enabled = buildRuntime + if !buildRuntime { + task.done = true + } + } + } + resolveRuntime() + + parallelism := ctx.buildConf.parallelism() + events := make(chan packagePipelineEvent, parallelism) + active := 0 + ssaActive := 0 + nextSSA := 0 + ssaDone := 0 + serialPos := 0 + serialActive := false + exclusiveActive := false + var ssaErr error + backendErrs := make(map[int]error) + + advanceSerial := func() { + for serialPos < len(serialOrder) && serialOrder[serialPos].done { + serialPos++ + } + } + advanceSerial() + + ready := func(task *packagePipelineTask) bool { + if task == nil || !task.enabled || task.running || task.done { + return false + } + // Legacy patch compilation rewrites shared go/ssa package type pointers. + // Its immutable merged type view is prepared up front for ordinary + // backends, but the patched package itself still runs exclusively after + // SSA construction has stopped mutating the program. + if task.patched && ssaDone != len(ssaNodes) { + return false + } + if task.ownSSA != nil && !task.ownSSA.done { + return false + } + for _, dep := range task.ssaDeps { + if !dep.done { + return false + } + } + return true + } + nextBackend := func() *packagePipelineTask { + advanceSerial() + if !serialActive && serialPos < len(serialOrder) { + task := serialOrder[serialPos] + if task.patched && ssaDone == len(ssaNodes) { + if active == 0 && ready(task) { + return task + } + return nil + } + if ready(task) { + return task + } + } + for _, task := range tasks { + if task.class == pipelineIsolated && ready(task) { + return task + } + } + return nil + } + allBackendsDone := func() bool { + for _, task := range tasks { + if !task.done { + return false + } + } + return true + } + launchSSA := func() bool { + if nextSSA >= len(ssaNodes) { + return false + } + node := ssaNodes[nextSSA] + nextSSA++ + active++ + ssaActive++ + go func() { + pkgPath := packagePipelineSSAPath(node.entry.pkg) + observePackagePipeline(ctx, pipelineStageSSA, pkgPath, true) + runErr := buildPackagePipelineSSA(node.entry) + observePackagePipeline(ctx, pipelineStageSSA, pkgPath, false) + events <- packagePipelineEvent{ssa: node, err: runErr} + }() + return true + } + + for ssaDone != len(ssaNodes) || !allBackendsDone() || active != 0 { + for ssaErr == nil && active < parallelism { + if exclusiveActive { + break + } + // Keep at least one SSA producer active while packages remain. At + // -p=1 this preserves the legacy all-SSA-then-backend order; with + // more workers, the remaining slots consume ready backend work. + if ssaActive == 0 && launchSSA() { + continue + } + if task := nextBackend(); task != nil { + task.running = true + if task.class == pipelineCoordinator { + serialActive = true + } + if task.patched { + exclusiveActive = true + } + active++ + go func() { + stage := pipelineStageBackend + if task.patched { + stage = pipelineStagePatchedBackend + } + observePackagePipeline(ctx, stage, task.spec.pkg.PkgPath, true) + runErr := runPackageJob(task.index, func(int) error { + summaries := make([]cl.CallerTrackingSummary, 0, len(task.callerNodes)) + for _, node := range task.callerNodes { + summaries = append(summaries, node.callerSummary) + } + tracking := cl.NewPackageCallerTrackingForPackages(task.callerRoots, summaries...) + if task.class == pipelineCoordinator { + return ctx.executeCoordinatorPackageWithCallerTracking(task.spec, tracking, verbose) + } + return ctx.executeIsolatedPackageWithCallerTracking(task.spec, tracking, verbose) + }) + observePackagePipeline(ctx, stage, task.spec.pkg.PkgPath, false) + events <- packagePipelineEvent{task: task, err: runErr} + }() + continue + } + if launchSSA() { + continue + } + break + } + + if active == 0 { + if ssaErr != nil { + return nil, ssaErr + } + resolveRuntime() + if allBackendsDone() && ssaDone == len(ssaNodes) { + break + } + return nil, fmt.Errorf("package pipeline stalled with %d/%d SSA packages complete", ssaDone, len(ssaNodes)) + } + + event := <-events + active-- + switch { + case event.ssa != nil: + ssaActive-- + if event.err == nil { + event.ssa.callerSummary, event.err = finishPackagePipelineSSA(event.ssa.entry) + } + if event.err != nil { + if ssaErr == nil { + ssaErr = event.err + } + continue + } + event.ssa.done = true + ssaDone++ + case event.task != nil: + task := event.task + task.running = false + task.done = true + task.executed = event.err == nil + if task.class == pipelineCoordinator { + serialActive = false + } + if task.patched { + exclusiveActive = false + } + if event.err != nil { + backendErrs[task.index] = event.err + backendFailed = true + } + if !task.spec.runtime { + normalRemaining-- + } + if event.err == nil && !task.spec.runtime { + result := packageBuildResultFor(task.spec) + needRuntime = needRuntime || result.needRuntime + needPyInit = needPyInit || result.needPyInit + } + resolveRuntime() + advanceSerial() + } + } + + if ssaErr != nil { + return nil, ssaErr + } + for i := range tasks { + if err := backendErrs[i]; err != nil { + return nil, err + } + } + for _, task := range tasks { + if !task.executed { + continue + } + if _, err := finalizePackageBuild(ctx, task.spec, verbose); err != nil { + return nil, err + } + } + return pkgs, nil +} + +func observePackagePipeline(ctx *context, stage packagePipelineStage, pkgPath string, start bool) { + if observer := ctx.buildConf.packagePipelineObserver; observer != nil { + observer(stage, pkgPath, start) + } +} + +func packagePipelineSSAPath(pkg *ssa.Package) string { + if pkg == nil || pkg.Pkg == nil { + return "" + } + return pkg.Pkg.Path() +} + +func newPackagePipelineSSANodes(entries []ssaBuildEntry) ([]*packagePipelineSSA, map[*ssa.Package]*packagePipelineSSA) { + nodes := make([]*packagePipelineSSA, 0, len(entries)) + byPackage := make(map[*ssa.Package]*packagePipelineSSA, len(entries)) + for _, entry := range entries { + if entry.pkg == nil { + continue + } + if node := byPackage[entry.pkg]; node != nil { + node.entry.fixOrder = node.entry.fixOrder || entry.fixOrder + continue + } + node := &packagePipelineSSA{entry: entry, order: len(nodes)} + nodes = append(nodes, node) + byPackage[entry.pkg] = node + } + return nodes, byPackage +} + +func newPackagePipelineTasks( + ctx *context, + specs []packageBuildSpec, + preflights map[*aPackage]packagePreflight, + ssaByPackage map[*ssa.Package]*packagePipelineSSA, +) ([]*packagePipelineTask, []*packagePipelineTask, error) { + tasks := make([]*packagePipelineTask, len(specs)) + var patchedNormal, coordinatorNormal, patchedRuntime, coordinatorRuntime []*packagePipelineTask + for i, spec := range specs { + task := &packagePipelineTask{ + index: i, + spec: spec, + preflight: preflights[spec.pkg], + ownSSA: ssaByPackage[spec.pkg.SSA], + } + if spec.pkg.SSA != nil { + task.callerRoots = append(task.callerRoots, spec.pkg.SSA) + } + seenCallerNodes := make(map[*packagePipelineSSA]bool) + addCallerNode := func(pkg *ssa.Package) { + node := ssaByPackage[pkg] + if node == nil || seenCallerNodes[node] { + return + } + seenCallerNodes[node] = true + task.callerNodes = append(task.callerNodes, node) + } + addCallerNode(spec.pkg.SSA) + seen := make(map[*packagePipelineSSA]bool) + addDep := func(pkg *ssa.Package) { + node := ssaByPackage[pkg] + if node == nil || node == task.ownSSA || seen[node] { + return + } + seen[node] = true + task.ssaDeps = append(task.ssaDeps, node) + } + visitedDeps := make(map[string]bool) + var addPackageDeps func(*aPackage) + addPackageDeps = func(pkg *aPackage) { + if pkg == nil || visitedDeps[pkg.ID] { + return + } + visitedDeps[pkg.ID] = true + addDep(pkg.SSA) + for _, dep := range effectiveDependencies(pkg) { + addPackageDeps(ctx.pkgByID[dep.ID]) + } + } + for _, dep := range effectiveDependencies(spec.pkg) { + depPkg := ctx.pkgByID[dep.ID] + if depPkg != nil { + addCallerNode(depPkg.SSA) + } + addPackageDeps(depPkg) + } + if patch, ok := ctx.patches[spec.pkg.PkgPath]; ok { + addDep(patch.Alt) + addCallerNode(patch.Alt) + if patch.Alt != nil && patch.Alt != spec.pkg.SSA { + task.callerRoots = append(task.callerRoots, patch.Alt) + } + } + + patched := false + if _, ok := ctx.patches[spec.pkg.PkgPath]; ok { + patched = true + task.patched = true + task.class = pipelineCoordinator + } else { + serial, err := ctx.packageRequiresCoordinator(spec) + if err != nil { + return nil, nil, err + } + if serial { + task.class = pipelineCoordinator + } + } + if task.class == pipelineCoordinator && !task.preflight.skip { + switch { + case !spec.runtime && patched: + patchedNormal = append(patchedNormal, task) + case !spec.runtime: + coordinatorNormal = append(coordinatorNormal, task) + case patched: + patchedRuntime = append(patchedRuntime, task) + default: + coordinatorRuntime = append(coordinatorRuntime, task) + } + } + tasks[i] = task + } + serial := append(patchedNormal, coordinatorNormal...) + serial = append(serial, patchedRuntime...) + serial = append(serial, coordinatorRuntime...) + return tasks, serial, nil +} + +func orderPackagePipelineSSA(nodes []*packagePipelineSSA, tasks []*packagePipelineTask) []*packagePipelineSSA { + deps := make(map[*packagePipelineSSA][]*packagePipelineSSA) + for _, task := range tasks { + if task.ownSSA == nil { + continue + } + deps[task.ownSSA] = append(deps[task.ownSSA], task.ssaDeps...) + } + ordered := make([]*packagePipelineSSA, 0, len(nodes)) + state := make(map[*packagePipelineSSA]uint8, len(nodes)) + var visit func(*packagePipelineSSA) + visit = func(node *packagePipelineSSA) { + if node == nil || state[node] == 2 { + return + } + if state[node] == 1 { + return + } + state[node] = 1 + for _, dep := range deps[node] { + visit(dep) + } + state[node] = 2 + ordered = append(ordered, node) + } + for _, node := range nodes { + visit(node) + } + return ordered +} + +func buildPackagePipelineSSA(entry ssaBuildEntry) (err error) { + pkgPath := packagePipelineSSAPath(entry.pkg) + defer func() { + if value := recover(); value != nil { + err = fmt.Errorf("build SSA for %s: %v", pkgPath, value) + } + }() + if entry.pkg == nil { + return fmt.Errorf("build SSA for %s: nil package", pkgPath) + } + entry.pkg.Build() + return nil +} + +func finishPackagePipelineSSA(entry ssaBuildEntry) (summary cl.CallerTrackingSummary, err error) { + pkgPath := packagePipelineSSAPath(entry.pkg) + defer func() { + if value := recover(); value != nil { + err = fmt.Errorf("fix SSA order for %s: %v", pkgPath, value) + } + }() + if entry.fixOrder { + fixSSAOrder(entry.pkg, entry.syntax) + } + return cl.SummarizeCallerTracking(entry.pkg), nil +} diff --git a/internal/build/package_pipeline_test.go b/internal/build/package_pipeline_test.go new file mode 100644 index 0000000000..6df093db81 --- /dev/null +++ b/internal/build/package_pipeline_test.go @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "go/token" + "go/types" + "testing" + + "golang.org/x/tools/go/ssa" +) + +func TestNewPackagePipelineSSANodesDeduplicates(t *testing.T) { + first := new(ssa.Package) + second := new(ssa.Package) + nodes, byPackage := newPackagePipelineSSANodes([]ssaBuildEntry{ + {pkg: first}, + {pkg: nil, fixOrder: true}, + {pkg: second}, + {pkg: first, fixOrder: true}, + }) + + if len(nodes) != 2 { + t.Fatalf("node count = %d, want 2", len(nodes)) + } + if nodes[0] != byPackage[first] || nodes[1] != byPackage[second] { + t.Fatal("package lookup does not preserve first-seen order") + } + if !nodes[0].entry.fixOrder { + t.Fatal("duplicate package did not retain fixOrder") + } + if nodes[0].order != 0 || nodes[1].order != 1 { + t.Fatalf("node order = (%d, %d), want (0, 1)", nodes[0].order, nodes[1].order) + } +} + +func TestOrderPackagePipelineSSADependenciesFirst(t *testing.T) { + first := &packagePipelineSSA{order: 0} + second := &packagePipelineSSA{order: 1} + third := &packagePipelineSSA{order: 2} + tasks := []*packagePipelineTask{ + {ownSSA: first, ssaDeps: []*packagePipelineSSA{second}}, + {ownSSA: second, ssaDeps: []*packagePipelineSSA{third}}, + } + + got := orderPackagePipelineSSA([]*packagePipelineSSA{first, second, third}, tasks) + want := []*packagePipelineSSA{third, second, first} + if len(got) != len(want) { + t.Fatalf("ordered node count = %d, want %d", len(got), len(want)) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("ordered node %d = %p, want %p", i, got[i], want[i]) + } + } +} + +func TestOrderPackagePipelineSSAToleratesCyclesAndEmptyTasks(t *testing.T) { + first := &packagePipelineSSA{order: 0} + second := &packagePipelineSSA{order: 1} + tasks := []*packagePipelineTask{ + {}, + {ownSSA: first, ssaDeps: []*packagePipelineSSA{second}}, + {ownSSA: second, ssaDeps: []*packagePipelineSSA{first}}, + } + + got := orderPackagePipelineSSA([]*packagePipelineSSA{nil, first, second, first}, tasks) + if len(got) != 2 { + t.Fatalf("ordered node count = %d, want 2", len(got)) + } + seen := map[*packagePipelineSSA]bool{} + for _, node := range got { + if seen[node] { + t.Fatalf("SSA node %p was returned more than once", node) + } + seen[node] = true + } + if !seen[first] || !seen[second] { + t.Fatalf("ordered nodes = %v, want both cycle members", got) + } +} + +func TestBuildPackagePipelineSSARejectsNilPackage(t *testing.T) { + if err := buildPackagePipelineSSA(ssaBuildEntry{}); err == nil { + t.Fatal("nil SSA package was accepted") + } +} + +func TestBuildPackagePipelineSSAConvertsPanicsToErrors(t *testing.T) { + prog := ssa.NewProgram(token.NewFileSet(), 0) + pkg := prog.CreatePackage(types.NewPackage("example.com/p", "p"), nil, &types.Info{}, true) + pkg.Prog = nil + if err := buildPackagePipelineSSA(ssaBuildEntry{pkg: pkg}); err == nil { + t.Fatal("SSA build panic was not converted to an error") + } + if _, err := finishPackagePipelineSSA(ssaBuildEntry{}); err == nil { + t.Fatal("SSA finalization panic was not converted to an error") + } +} diff --git a/internal/typepatch/patch.go b/internal/typepatch/patch.go index 1f3cc12b00..020e9215e0 100644 --- a/internal/typepatch/patch.go +++ b/internal/typepatch/patch.go @@ -82,6 +82,13 @@ func Clone(alt *types.Package) *types.Package { func Merge(alt, pkg *types.Package, skips map[string]struct{}, skipall bool) { setPatched(pkg) + MergePrepared(alt, pkg, skips, skipall) +} + +// MergePrepared merges pkg into the cloned alternate package without +// modifying pkg. It is used by build preflight to publish an immutable patch +// type view before any LLVM backend starts. +func MergePrepared(alt, pkg *types.Package, skips map[string]struct{}, skipall bool) { if skipall { return } diff --git a/internal/typepatch/patch_test.go b/internal/typepatch/patch_test.go new file mode 100644 index 0000000000..bb0b2c700b --- /dev/null +++ b/internal/typepatch/patch_test.go @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package typepatch + +import ( + "go/token" + "go/types" + "testing" +) + +func packageWithVars(path string, names ...string) *types.Package { + pkg := types.NewPackage(path, "p") + for _, name := range names { + pkg.Scope().Insert(types.NewVar(token.NoPos, pkg, name, types.Typ[types.Int])) + } + return pkg +} + +func TestMergePreparedDoesNotMarkOriginal(t *testing.T) { + original := packageWithVars("example.com/original", "Keep", "Skip") + merged := Clone(packageWithVars("example.com/alternate", "Alt")) + + MergePrepared(merged, original, map[string]struct{}{"Skip": {}}, false) + + for _, name := range []string{"Alt", "Keep"} { + if merged.Scope().Lookup(name) == nil { + t.Fatalf("merged package is missing %s", name) + } + } + if merged.Scope().Lookup("Skip") != nil { + t.Fatal("merged package retained skipped declaration") + } + if IsPatched(original) { + t.Fatal("MergePrepared marked the original package as patched") + } + if original.Scope().Lookup("Keep") == nil || original.Scope().Lookup("Skip") == nil { + t.Fatal("MergePrepared modified the original package scope") + } +} + +func TestMergePreparedSkipAll(t *testing.T) { + original := packageWithVars("example.com/original", "Keep") + merged := Clone(packageWithVars("example.com/alternate", "Alt")) + + MergePrepared(merged, original, nil, true) + + if merged.Scope().Lookup("Keep") != nil { + t.Fatal("skipall merged an original declaration") + } + if merged.Scope().Lookup("Alt") == nil { + t.Fatal("skipall removed an alternate declaration") + } +} + +func TestMergeMarksOriginal(t *testing.T) { + original := packageWithVars("example.com/original", "Keep") + merged := Clone(packageWithVars("example.com/alternate", "Alt")) + + Merge(merged, original, nil, false) + + if !IsPatched(original) { + t.Fatal("Merge did not mark the original package as patched") + } + if merged.Scope().Lookup("Keep") == nil { + t.Fatal("Merge did not include the original declaration") + } +} From 25c203a261f320fd6e8106bed451b316b2785789 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 12:47:00 +0800 Subject: [PATCH 21/22] build: add Chrome scheduler trace --- cmd/internal/build/build.go | 2 + cmd/internal/build/build_test.go | 10 + cmd/internal/flags/flags.go | 9 + internal/build/build.go | 54 +++++- internal/build/build_test.go | 21 ++- internal/build/build_trace.go | 283 +++++++++++++++++++++++++++++ internal/build/build_trace_test.go | 145 +++++++++++++++ internal/build/package_build.go | 20 +- internal/build/package_pipeline.go | 20 ++ 9 files changed, 557 insertions(+), 7 deletions(-) create mode 100644 internal/build/build_trace.go create mode 100644 internal/build/build_trace_test.go diff --git a/cmd/internal/build/build.go b/cmd/internal/build/build.go index 5547577b29..76ef359e73 100644 --- a/cmd/internal/build/build.go +++ b/cmd/internal/build/build.go @@ -42,6 +42,7 @@ func init() { flags.AddCommonFlags(&Cmd.Flag) flags.AddCompilerVerboseFlag(&Cmd.Flag) flags.AddBuildFlags(&Cmd.Flag) + flags.AddBuildTraceFlag(&Cmd.Flag) flags.AddBuildModeFlags(&Cmd.Flag) flags.AddEmulatorFlags(&Cmd.Flag) flags.AddEmbeddedFlags(&Cmd.Flag) @@ -59,6 +60,7 @@ func runCmd(cmd *base.Command, args []string) { fmt.Fprintln(os.Stderr, err) mockable.Exit(1) } + conf.BuildTrace = flags.BuildTrace if err := flags.ApplyGoBuildFlags(conf, goBuildFlags.Args); err != nil { fmt.Fprintln(os.Stderr, err) mockable.Exit(1) diff --git a/cmd/internal/build/build_test.go b/cmd/internal/build/build_test.go index 22d71b0f43..36e64938ac 100644 --- a/cmd/internal/build/build_test.go +++ b/cmd/internal/build/build_test.go @@ -57,3 +57,13 @@ func TestRunCmdPassesGoBuildFlags(t *testing.T) { t.Fatalf("stderr = %q, want missing-package diagnostic", data) } } + +func TestBuildCommandHasSchedulerTraceFlag(t *testing.T) { + flag := Cmd.Flag.Lookup("debug-trace") + if flag == nil { + t.Fatal("llgo build has no -debug-trace flag") + } + if !strings.Contains(flag.Usage, "Chrome/Perfetto") { + t.Fatalf("-debug-trace usage = %q", flag.Usage) + } +} diff --git a/cmd/internal/flags/flags.go b/cmd/internal/flags/flags.go index 4a315e0b0d..c1310f9609 100644 --- a/cmd/internal/flags/flags.go +++ b/cmd/internal/flags/flags.go @@ -48,6 +48,7 @@ var SizeFormat string var SizeLevel string var ForceRebuild bool var PrintCommands bool +var BuildTrace string var DeadcodeDrop bool var PthreadStackSize byteSizeFlag var OptLevel optlevel.Level @@ -238,6 +239,14 @@ func AddBuildFlags(fs *flag.FlagSet) { fs.StringVar(&SizeLevel, "size-level", "", "Size report aggregation level (full,module,package). Default module.") } +// AddBuildTraceFlag adds the build-scheduler trace flag. It is intentionally +// separate from AddBuildFlags because only "llgo build" owns a single trace +// output file; test and run may coordinate multiple child invocations. +func AddBuildTraceFlag(fs *flag.FlagSet) { + BuildTrace = "" + fs.StringVar(&BuildTrace, "debug-trace", "", "Write a Chrome/Perfetto build trace to file") +} + func AddBuildModeFlags(fs *flag.FlagSet) { fs.StringVar(&BuildMode, "buildmode", "exe", "Build mode (exe, c-archive, c-shared)") } diff --git a/internal/build/build.go b/internal/build/build.go index 5a1b23af8c..d46187b804 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -173,7 +173,10 @@ type Config struct { // BuildParallelism is the package-level concurrency requested by Go's -p // build flag for llgo test. Zero uses the Go default, GOMAXPROCS. BuildParallelism int - LinkOptions LinkOptions + // BuildTrace is an optional Chrome Trace Event JSON output path. Relative + // paths are resolved from the build invocation directory. + BuildTrace string + LinkOptions LinkOptions // OmitDWARFByDefault controls linked builds only when -w was not // explicitly specified. Explicit -w and -w=false always win. OmitDWARFByDefault bool @@ -377,7 +380,7 @@ func Do(args []string, conf *Config) ([]Package, error) { } // Build executes one build invocation. -func Build(inv Invocation) ([]Package, error) { +func Build(inv Invocation) (result []Package, resultErr error) { dir := inv.Dir if dir == "" { var err error @@ -392,6 +395,20 @@ func Build(inv Invocation) ([]Package, error) { if err != nil { return nil, err } + buildTrace, err := startBuildTrace(conf.BuildTrace, dir, conf.parallelism()) + if err != nil { + return nil, fmt.Errorf("start build trace: %w", err) + } + buildSpan := buildTrace.startCoordinator("build", map[string]any{ + "packages": slices.Clone(inv.Args), + }) + defer func() { + buildSpan.done() + if closeErr := buildTrace.close(); closeErr != nil && resultErr == nil { + result = nil + resultErr = fmt.Errorf("write build trace: %w", closeErr) + } + }() // Handle crosscompile configuration first to set correct GOOS/GOARCH forceEspClang := conf.ForceEspClang || conf.Target != "" export, err := crosscompile.Use(conf.Goos, conf.Goarch, conf.Target, IsWasiThreadsEnabled(), forceEspClang, conf.OptLevel, conf.ltoMode(), conf.goGlobalDCEEnabled()) @@ -541,7 +558,11 @@ func Build(inv Invocation) ([]Package, error) { return parser.ParseFile(fset, filename, src, mode) } + loadSpan := buildTrace.startCoordinator("load packages", map[string]any{ + "patterns": slices.Clone(patterns), + }) initial, err := packages.LoadExWithGoVersion(dedup, sizes, cfg, conf.GoVersion, patterns...) + loadSpan.done() if err != nil { return nil, err } @@ -578,7 +599,11 @@ func Build(inv Invocation) ([]Package, error) { altPkgPaths := altPkgs(initial, conf, llssa.PkgRuntime) altCfg := *cfg altCfg.Dir = env.LLGoRuntimeDir() + loadAltSpan := buildTrace.startCoordinator("load runtime packages", map[string]any{ + "packages": slices.Clone(altPkgPaths), + }) altPkgs, err := packages.LoadEx(dedup, sizes, &altCfg, altPkgPaths...) + loadAltSpan.done() if err != nil { return nil, err } @@ -642,6 +667,7 @@ func Build(inv Invocation) ([]Package, error) { frontendOptions: frontendOptions, cTransformer: cabi.NewTransformer(prog, export.LLVMTarget, export.TargetABI, conf.AbiMode, cabiOptimize), backend: backendTemplate, + buildTrace: buildTrace, } defer ctx.closePackageMetas() defer ctx.closePackageArchiveBuffers() @@ -693,7 +719,12 @@ func Build(inv Invocation) ([]Package, error) { resolveOutputs(ctx.commands.dir, outFmts) // Link main package using the output path from buildOutFmts + linkSpan := buildTrace.startCoordinator("link "+pkg.PkgPath, map[string]any{ + "package": pkg.PkgPath, + "output": outFmts.Out, + }) err = linkMainPkg(ctx, pkg, allPkgs, outFmts.Out, verbose) + linkSpan.done() if err != nil { return nil, err } @@ -933,6 +964,8 @@ type context struct { // pclnExternal is populated while generating the synthetic main module // and completed with final linked PCs by the post-link externalizer. pclnExternal *pclnmap.Data + + buildTrace *buildTracer } // closePackageMetas releases metadata mappings owned by this build. Metadata @@ -1203,7 +1236,11 @@ func buildOnePackage(ctx *context, spec packageBuildSpec, verbose bool) (package if err != nil || skip { return packageBuildResultFor(spec), err } - if err := executePackageBuild(ctx, spec, verbose); err != nil { + backendSpan := ctx.buildTrace.startWorker("backend", spec.pkg.PkgPath) + backendSpan.setArg("class", "coordinator") + err = executePackageBuild(ctx, spec, verbose) + backendSpan.done() + if err != nil { return packageBuildResultFor(spec), err } return finalizePackageBuild(ctx, spec, verbose) @@ -1214,6 +1251,12 @@ func buildOnePackage(ctx *context, spec packageBuildSpec, verbose bool) (package func preflightPackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (skip bool, err error) { aPkg := spec.pkg pkg := aPkg.Package + traceSpan := ctx.buildTrace.startPackageCoordinator("preflight", pkg.PkgPath) + defer func() { + traceSpan.setArg("cache_hit", aPkg.CacheHit) + traceSpan.setArg("skip", skip) + traceSpan.done() + }() if _, ok := ctx.built[pkg.ID]; ok { return true, nil } @@ -1264,6 +1307,9 @@ func executePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) erro // already carry both and therefore require no publication. func finalizePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (packageBuildResult, error) { aPkg := spec.pkg + traceSpan := ctx.buildTrace.startPackageCoordinator("publish", aPkg.PkgPath) + traceSpan.setArg("cache_hit", aPkg.CacheHit) + defer traceSpan.done() if aPkg.CacheHit { return packageBuildResultFor(spec), nil } @@ -2494,7 +2540,9 @@ func buildSSAPkgs(ctx *context, entries []ssaBuildEntry) { go func() { defer wg.Done() for entry := range jobs { + traceSpan := ctx.buildTrace.startWorker("ssa", packagePipelineSSAPath(entry.pkg)) entry.pkg.Build() + traceSpan.done() } }() } diff --git a/internal/build/build_test.go b/internal/build/build_test.go index 346b040e23..e9e49220b0 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -6,6 +6,7 @@ package build import ( "bytes" "debug/macho" + "encoding/json" "fmt" "go/ast" gobuild "go/build" @@ -528,7 +529,8 @@ func TestTest(t *testing.T) { } active, maximum := 0, 0 exclusive, exclusiveViolation := false, false - conf := &Config{Mode: ModeTest, BuildParallelism: 2} + tracePath := filepath.Join(t.TempDir(), "build-trace.json") + conf := &Config{Mode: ModeTest, BuildParallelism: 2, BuildTrace: tracePath} conf.packagePipelineObserver = func(stage packagePipelineStage, _ string, start bool) { mu.Lock() defer mu.Unlock() @@ -554,6 +556,23 @@ func TestTest(t *testing.T) { } mockRun([]string{"../../cl/_testgo/runtest"}, conf) + traceData, err := os.ReadFile(tracePath) + if err != nil { + t.Fatal(err) + } + var traceEvents []buildTraceEvent + if err := json.Unmarshal(traceData, &traceEvents); err != nil { + t.Fatalf("invalid build trace: %v", err) + } + var tracedSSA, tracedBackend bool + for _, event := range traceEvents { + tracedSSA = tracedSSA || event.Phase == "X" && event.Category == "llgo.ssa" + tracedBackend = tracedBackend || event.Phase == "X" && strings.HasPrefix(event.Category, "llgo.backend") + } + if !tracedSSA || !tracedBackend { + t.Fatalf("build trace stages: ssa=%v backend=%v", tracedSSA, tracedBackend) + } + mu.Lock() defer mu.Unlock() if active != 0 { diff --git a/internal/build/build_trace.go b/internal/build/build_trace.go new file mode 100644 index 0000000000..f687b9c0e3 --- /dev/null +++ b/internal/build/build_trace.go @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" + "time" +) + +const buildTracePID = 1 + +// buildTraceEvent follows the Chrome Trace Event format accepted by both +// chrome://tracing and Perfetto. +type buildTraceEvent struct { + Name string `json:"name"` + Category string `json:"cat,omitempty"` + Phase string `json:"ph"` + Time int64 `json:"ts,omitempty"` + Duration int64 `json:"dur,omitempty"` + PID int `json:"pid"` + TID int `json:"tid"` + ID uint64 `json:"id,omitempty"` + Bind string `json:"bp,omitempty"` + Args map[string]any `json:"args,omitempty"` +} + +// buildTracer is owned by one Build invocation. The lane semaphore mirrors +// the build's -p limit, so overlapping worker-lane events visualize the same +// concurrency budget used by SSA and LLVM backend work. +type buildTracer struct { + mu sync.Mutex + file *os.File + writer *bufio.Writer + first bool + writeErr error + + lanes chan int + + nextFlowID uint64 + closeOnce sync.Once + closeErr error +} + +type buildTraceSpan struct { + tracer *buildTracer + name string + cat string + args map[string]any + start time.Time + lane int + worker bool + + end time.Time + once sync.Once +} + +func startBuildTrace(path, dir string, parallelism int) (*buildTracer, error) { + if path == "" { + return nil, nil + } + if !filepath.IsAbs(path) { + path = filepath.Join(dir, path) + } + if filepath.Ext(path) == ".go" { + return nil, fmt.Errorf("refusing to overwrite Go source file %s", path) + } + file, err := os.Create(path) + if err != nil { + return nil, err + } + tracer := &buildTracer{ + file: file, + writer: bufio.NewWriter(file), + first: true, + lanes: make(chan int, max(1, parallelism)), + } + for lane := 1; lane <= cap(tracer.lanes); lane++ { + tracer.lanes <- lane + } + if _, err := tracer.writer.WriteString("[\n"); err != nil { + _ = file.Close() + return nil, err + } + tracer.writeEvent(buildTraceEvent{ + Name: "process_name", + Phase: "M", + PID: buildTracePID, + Args: map[string]any{"name": "llgo build"}, + }) + tracer.writeEvent(buildTraceEvent{ + Name: "thread_name", + Phase: "M", + PID: buildTracePID, + TID: 0, + Args: map[string]any{"name": "coordinator"}, + }) + for lane := 1; lane <= cap(tracer.lanes); lane++ { + tracer.writeEvent(buildTraceEvent{ + Name: "thread_name", + Phase: "M", + PID: buildTracePID, + TID: lane, + Args: map[string]any{"name": fmt.Sprintf("worker %d", lane)}, + }) + } + return tracer, nil +} + +func (t *buildTracer) startCoordinator(name string, args map[string]any) *buildTraceSpan { + if t == nil { + return nil + } + return &buildTraceSpan{ + tracer: t, + name: name, + cat: "llgo.build", + args: args, + start: time.Now(), + } +} + +func (t *buildTracer) startPackageCoordinator(stage, pkgPath string) *buildTraceSpan { + if t == nil { + return nil + } + return t.startCoordinator(stage+" "+pkgPath, map[string]any{ + "package": pkgPath, + "stage": stage, + }) +} + +func (t *buildTracer) startWorker(stage, pkgPath string) *buildTraceSpan { + if t == nil { + return nil + } + lane := <-t.lanes + return &buildTraceSpan{ + tracer: t, + name: stage + " " + pkgPath, + cat: "llgo." + stage, + args: map[string]any{ + "package": pkgPath, + "stage": stage, + }, + start: time.Now(), + lane: lane, + worker: true, + } +} + +func (s *buildTraceSpan) setArg(name string, value any) { + if s == nil { + return + } + if s.args == nil { + s.args = make(map[string]any) + } + s.args[name] = value +} + +func (s *buildTraceSpan) done() { + if s == nil { + return + } + s.once.Do(func() { + s.end = time.Now() + s.tracer.writeEvent(buildTraceEvent{ + Name: s.name, + Category: s.cat, + Phase: "X", + Time: traceMicros(s.start), + Duration: max(0, traceMicros(s.end)-traceMicros(s.start)), + PID: buildTracePID, + TID: s.lane, + Args: s.args, + }) + if s.worker { + s.tracer.lanes <- s.lane + } + }) +} + +// flow records a dependency arrow from a completed producer to a started +// consumer, matching the flow events emitted by Go's build trace. +func (t *buildTracer) flow(from, to *buildTraceSpan) { + if t == nil || from == nil || to == nil || from.end.IsZero() { + return + } + t.mu.Lock() + t.nextFlowID++ + id := t.nextFlowID + t.mu.Unlock() + name := from.name + " -> " + to.name + t.writeEvent(buildTraceEvent{ + Name: name, + Category: "flow", + Phase: "s", + Time: traceMicros(from.end), + PID: buildTracePID, + TID: from.lane, + ID: id, + }) + t.writeEvent(buildTraceEvent{ + Name: name, + Category: "flow", + Phase: "f", + Time: traceMicros(to.start), + PID: buildTracePID, + TID: to.lane, + ID: id, + Bind: "e", + }) +} + +func (t *buildTracer) writeEvent(event buildTraceEvent) { + if t == nil { + return + } + data, err := json.Marshal(event) + t.mu.Lock() + defer t.mu.Unlock() + if t.writeErr != nil { + return + } + if err != nil { + t.writeErr = err + return + } + if !t.first { + if _, err := t.writer.WriteString(",\n"); err != nil { + t.writeErr = err + return + } + } + t.first = false + if _, err := t.writer.Write(data); err != nil { + t.writeErr = err + } +} + +func (t *buildTracer) close() error { + if t == nil { + return nil + } + t.closeOnce.Do(func() { + t.mu.Lock() + defer t.mu.Unlock() + if t.writeErr == nil { + _, t.writeErr = t.writer.WriteString("\n]\n") + } + if err := t.writer.Flush(); t.writeErr == nil { + t.writeErr = err + } + if err := t.file.Close(); t.writeErr == nil { + t.writeErr = err + } + t.closeErr = t.writeErr + }) + return t.closeErr +} + +func traceMicros(t time.Time) int64 { + return t.UnixNano() / int64(time.Microsecond) +} diff --git a/internal/build/build_trace_test.go b/internal/build/build_trace_test.go new file mode 100644 index 0000000000..488b146310 --- /dev/null +++ b/internal/build/build_trace_test.go @@ -0,0 +1,145 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "encoding/json" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestBuildTraceWritesChromeEvents(t *testing.T) { + dir := t.TempDir() + tracer, err := startBuildTrace("trace.json", dir, 2) + if err != nil { + t.Fatal(err) + } + buildSpan := tracer.startCoordinator("build", map[string]any{"packages": []string{"./..."}}) + ssaSpan := tracer.startWorker("ssa", "example.com/p") + ssaSpan.done() + backendSpan := tracer.startWorker("backend", "example.com/p") + backendSpan.setArg("class", "isolated") + tracer.flow(ssaSpan, backendSpan) + backendSpan.done() + buildSpan.done() + if err := tracer.close(); err != nil { + t.Fatal(err) + } + + data, err := os.ReadFile(filepath.Join(dir, "trace.json")) + if err != nil { + t.Fatal(err) + } + var events []buildTraceEvent + if err := json.Unmarshal(data, &events); err != nil { + t.Fatalf("trace is not valid Chrome Trace JSON: %v\n%s", err, data) + } + var metadata, complete, flowStart, flowEnd bool + for _, event := range events { + switch event.Phase { + case "M": + metadata = true + case "X": + complete = true + if event.Name == "backend example.com/p" { + if got := event.Args["class"]; got != "isolated" { + t.Fatalf("backend class = %v, want isolated", got) + } + } + case "s": + flowStart = true + case "f": + flowEnd = event.Bind == "e" + } + } + if !metadata || !complete || !flowStart || !flowEnd { + t.Fatalf("trace phases: metadata=%v complete=%v flowStart=%v flowEnd=%v", metadata, complete, flowStart, flowEnd) + } +} + +func TestBuildTraceWorkerLanesRespectParallelism(t *testing.T) { + tracer, err := startBuildTrace(filepath.Join(t.TempDir(), "trace.json"), "", 2) + if err != nil { + t.Fatal(err) + } + var active atomic.Int32 + var maximum atomic.Int32 + var wg sync.WaitGroup + for i := range 12 { + wg.Add(1) + go func() { + defer wg.Done() + span := tracer.startWorker("backend", string(rune('a'+i))) + current := active.Add(1) + for { + previous := maximum.Load() + if current <= previous || maximum.CompareAndSwap(previous, current) { + break + } + } + time.Sleep(time.Millisecond) + active.Add(-1) + span.done() + }() + } + wg.Wait() + if got := maximum.Load(); got > 2 { + t.Fatalf("maximum active traced workers = %d, want <= 2", got) + } + if err := tracer.close(); err != nil { + t.Fatal(err) + } +} + +func TestBuildTraceRefusesGoSourceOutput(t *testing.T) { + if _, err := startBuildTrace("trace.go", t.TempDir(), 1); err == nil { + t.Fatal("startBuildTrace(trace.go) succeeded") + } +} + +func TestBuildTraceDisabledAndErrors(t *testing.T) { + tracer, err := startBuildTrace("", t.TempDir(), 1) + if err != nil || tracer != nil { + t.Fatalf("disabled trace = (%v, %v), want (nil, nil)", tracer, err) + } + var span *buildTraceSpan + span.setArg("ignored", true) + span.done() + + if _, err := startBuildTrace(filepath.Join(t.TempDir(), "missing", "trace.json"), "", 1); err == nil { + t.Fatal("startBuildTrace in missing directory succeeded") + } + + tracer, err = startBuildTrace(filepath.Join(t.TempDir(), "trace.json"), "", 1) + if err != nil { + t.Fatal(err) + } + tracer.writeEvent(buildTraceEvent{ + Name: "invalid", + Phase: "X", + Args: map[string]any{"function": func() {}}, + }) + if err := tracer.close(); err == nil { + t.Fatal("closing trace after JSON encoding error succeeded") + } +} diff --git a/internal/build/package_build.go b/internal/build/package_build.go index 372b6c6f31..fc4c4d9ca7 100644 --- a/internal/build/package_build.go +++ b/internal/build/package_build.go @@ -101,7 +101,12 @@ func buildPreflightedPackageGroup(ctx *context, specs []packageBuildSpec, prefli return nil, err } for _, index := range patched { - if err := executePreflightedPackage(ctx, preflights[specs[index].pkg], verbose); err != nil { + spec := specs[index] + traceSpan := ctx.buildTrace.startWorker("backend-patched", spec.pkg.PkgPath) + traceSpan.setArg("class", "patched") + err := executePreflightedPackage(ctx, preflights[spec.pkg], verbose) + traceSpan.done() + if err != nil { return nil, err } } @@ -110,7 +115,11 @@ func buildPreflightedPackageGroup(ctx *context, specs []packageBuildSpec, prefli if preflight.skip { continue } - if err := executePackageBuild(ctx, preflight.spec, verbose); err != nil { + traceSpan := ctx.buildTrace.startWorker("backend-coordinator", preflight.spec.pkg.PkgPath) + traceSpan.setArg("class", "coordinator") + err := executePackageBuild(ctx, preflight.spec, verbose) + traceSpan.done() + if err != nil { return nil, err } } @@ -176,7 +185,11 @@ func executeIsolatedPackages(ctx *context, specs []packageBuildSpec, indexes []i if preflight.skip { return nil } - return ctx.executeIsolatedPackage(preflight.spec, verbose) + traceSpan := ctx.buildTrace.startWorker("backend", preflight.spec.pkg.PkgPath) + traceSpan.setArg("class", "isolated") + err := ctx.executeIsolatedPackage(preflight.spec, verbose) + traceSpan.done() + return err }) } @@ -337,6 +350,7 @@ func (ctx *context) newBackendTaskWithCallerTracking(session backendSession, tra frontendOptions: ctx.frontendOptions, cTransformer: session.transformer, backend: ctx.backend, + buildTrace: ctx.buildTrace, sfilesCache: ctx.sfilesCache, sfilesFrozen: true, plan9asmReady: true, diff --git a/internal/build/package_pipeline.go b/internal/build/package_pipeline.go index 9021ab4bc5..8a2c0e14f6 100644 --- a/internal/build/package_pipeline.go +++ b/internal/build/package_pipeline.go @@ -58,6 +58,7 @@ type packagePipelineSSA struct { order int done bool callerSummary cl.CallerTrackingSummary + traceSpan *buildTraceSpan } type packagePipelineTask struct { @@ -221,9 +222,11 @@ func buildPackagePipeline(ctx *context, entries []ssaBuildEntry, pkgs []*aPackag ssaActive++ go func() { pkgPath := packagePipelineSSAPath(node.entry.pkg) + node.traceSpan = ctx.buildTrace.startWorker("ssa", pkgPath) observePackagePipeline(ctx, pipelineStageSSA, pkgPath, true) runErr := buildPackagePipelineSSA(node.entry) observePackagePipeline(ctx, pipelineStageSSA, pkgPath, false) + node.traceSpan.done() events <- packagePipelineEvent{ssa: node, err: runErr} }() return true @@ -251,8 +254,24 @@ func buildPackagePipeline(ctx *context, entries []ssaBuildEntry, pkgs []*aPackag active++ go func() { stage := pipelineStageBackend + traceStage := "backend" if task.patched { stage = pipelineStagePatchedBackend + traceStage = "backend-patched" + } + traceSpan := ctx.buildTrace.startWorker(traceStage, task.spec.pkg.PkgPath) + if task.class == pipelineCoordinator { + traceSpan.setArg("class", "coordinator") + } else { + traceSpan.setArg("class", "isolated") + } + if ctx.buildTrace != nil { + // Draw direct SSA dependencies only. Transitive scheduler + // dependencies are implied by these edges and would make + // traces for large repositories unnecessarily quadratic. + for _, dependency := range task.callerNodes { + ctx.buildTrace.flow(dependency.traceSpan, traceSpan) + } } observePackagePipeline(ctx, stage, task.spec.pkg.PkgPath, true) runErr := runPackageJob(task.index, func(int) error { @@ -267,6 +286,7 @@ func buildPackagePipeline(ctx *context, entries []ssaBuildEntry, pkgs []*aPackag return ctx.executeIsolatedPackageWithCallerTracking(task.spec, tracking, verbose) }) observePackagePipeline(ctx, stage, task.spec.pkg.PkgPath, false) + traceSpan.done() events <- packagePipelineEvent{task: task, err: runErr} }() continue From 35b417025e7dd00d3949fcbb5a5b4f73fb696d9d Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 14:38:21 +0800 Subject: [PATCH 22/22] build: publish package archives concurrently --- internal/build/build.go | 68 ++++++++-- internal/build/build_test.go | 7 +- internal/build/collect.go | 36 ++--- internal/build/collect_test.go | 193 +++++++++++++++++++++++++++ internal/build/package_build_test.go | 94 +++++++++++++ internal/build/package_pipeline.go | 56 +++++++- 6 files changed, 422 insertions(+), 32 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index d46187b804..6b84012ade 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -671,6 +671,7 @@ func Build(inv Invocation) (result []Package, resultErr error) { } defer ctx.closePackageMetas() defer ctx.closePackageArchiveBuffers() + defer ctx.cleanupTemporaryArchives() // default runtime globals must be registered before packages are built addGlobalString(conf, "runtime.defaultGOROOT="+runtime.GOROOT(), nil) @@ -980,6 +981,26 @@ func (c *context) closePackageMetas() { } } +// cleanupTemporaryArchives removes package archives that were created outside +// the persistent cache. Darwin debug maps refer back to linked input archives, +// so preserve those inputs when DWARF remains enabled for later dsymutil use. +// Cached archives remain available for future builds in either case. +func (c *context) cleanupTemporaryArchives() { + if c.buildConf != nil && + c.buildConf.Goos == "darwin" && + shouldEmitDebugInfo(c.buildConf, &c.crossCompile) { + return + } + for _, pkg := range c.pkgs { + if pkg == nil || !pkg.ArchiveTemp || pkg.ArchiveFile == "" { + continue + } + _ = os.Remove(pkg.ArchiveFile) + pkg.ArchiveFile = "" + pkg.ArchiveTemp = false + } +} + type backendProgramInput struct { fset *token.FileSet pkg *types.Package @@ -1176,11 +1197,26 @@ func normalizeToArchive(ctx *context, aPkg *aPackage, verbose bool) error { } defer aPkg.disposeArchiveBuffers() + if paths, ok := ctx.packageCachePathsForWrite(aPkg); ok { + if err := ctx.createPackageArchiveFile(paths.Archive, aPkg, verbose); err == nil { + aPkg.ObjFiles = nil + aPkg.ArchiveFile = paths.Archive + aPkg.ArchiveTemp = false + return nil + } + } + + // Cache writes are best effort. If the direct cache destination is not + // writable, keep the build usable through a temporary archive and let + // saveToCache report the cache failure as a non-fatal warning. archiveFile, err := os.CreateTemp("", "pkg-*.a") if err != nil { return fmt.Errorf("create temp archive: %w", err) } - archiveFile.Close() + if err := archiveFile.Close(); err != nil { + os.Remove(archiveFile.Name()) + return fmt.Errorf("close temp archive: %w", err) + } archivePath := archiveFile.Name() if err := ctx.createPackageArchiveFile(archivePath, aPkg, verbose); err != nil { @@ -1190,6 +1226,7 @@ func normalizeToArchive(ctx *context, aPkg *aPackage, verbose bool) error { aPkg.ObjFiles = nil aPkg.ArchiveFile = archivePath + aPkg.ArchiveTemp = true return nil } @@ -1306,15 +1343,29 @@ func executePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) erro // finalizePackageBuild publishes the archive and cache metadata. Cache hits // already carry both and therefore require no publication. func finalizePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (packageBuildResult, error) { + result, warning, err := publishPackageBuild(ctx, spec, verbose) + if warning != nil && verbose { + fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", spec.pkg.PkgPath, warning) + } + return result, err +} + +// publishPackageBuild creates the package archive and commits its cache +// metadata. Cache failures remain non-fatal and are returned separately so a +// parallel caller can report warnings in deterministic package order. +func publishPackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (result packageBuildResult, warning, err error) { aPkg := spec.pkg - traceSpan := ctx.buildTrace.startPackageCoordinator("publish", aPkg.PkgPath) + traceSpan := ctx.buildTrace.startWorker("publish", aPkg.PkgPath) traceSpan.setArg("cache_hit", aPkg.CacheHit) - defer traceSpan.done() + defer func() { + traceSpan.setArg("cache_warning", warning != nil) + traceSpan.done() + }() if aPkg.CacheHit { - return packageBuildResultFor(spec), nil + return packageBuildResultFor(spec), nil, nil } if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { - return packageBuildResultFor(spec), err + return packageBuildResultFor(spec), nil, err } if spec.kind == cl.PkgLinkExtern { appendExternalLinkArgs(ctx, aPkg, spec.kindParam) @@ -1327,10 +1378,10 @@ func finalizePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (pa aPkg.Summary.NeedRuntime = aPkg.NeedRt aPkg.Summary.NeedPyInit = aPkg.NeedPyInit } - if err := ctx.saveToCache(aPkg); err != nil && verbose { - fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", aPkg.PkgPath, err) + if err := ctx.saveToCache(aPkg); err != nil { + return packageBuildResultFor(spec), err, nil } - return packageBuildResultFor(spec), nil + return packageBuildResultFor(spec), nil, nil } func appendExternalLinkArgs(ctx *context, aPkg *aPackage, spec string) { @@ -2450,6 +2501,7 @@ type aPackage struct { ObjFiles []string // file-backed archive members: .o or .ll ObjBuffers []packageArchiveBuffer // LLVM-produced in-memory archive members ArchiveFile string // archive file: .a (output of archiver, used for linking) + ArchiveTemp bool // ArchiveFile is a build-owned temporary file Meta *meta.PackageMeta rewriteVars map[string]string diff --git a/internal/build/build_test.go b/internal/build/build_test.go index e9e49220b0..7aa3e2dd6e 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -564,13 +564,14 @@ func TestTest(t *testing.T) { if err := json.Unmarshal(traceData, &traceEvents); err != nil { t.Fatalf("invalid build trace: %v", err) } - var tracedSSA, tracedBackend bool + var tracedSSA, tracedBackend, tracedPublish bool for _, event := range traceEvents { tracedSSA = tracedSSA || event.Phase == "X" && event.Category == "llgo.ssa" tracedBackend = tracedBackend || event.Phase == "X" && strings.HasPrefix(event.Category, "llgo.backend") + tracedPublish = tracedPublish || event.Phase == "X" && event.Category == "llgo.publish" } - if !tracedSSA || !tracedBackend { - t.Fatalf("build trace stages: ssa=%v backend=%v", tracedSSA, tracedBackend) + if !tracedSSA || !tracedBackend || !tracedPublish { + t.Fatalf("build trace stages: ssa=%v backend=%v publish=%v", tracedSSA, tracedBackend, tracedPublish) } mu.Lock() diff --git a/internal/build/collect.go b/internal/build/collect.go index 7d3c65166b..f65103ecb6 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -465,24 +465,12 @@ type cacheArchiveMetadata struct { // saveToCache saves a built package to cache. func (c *context) saveToCache(pkg *aPackage) error { - if !cacheEnabled() { - return nil - } - if c.packageCacheDisabled(pkg.ID) { - return nil - } - - if pkg.Fingerprint == "" || pkg.Manifest == "" { - return nil - } - - // Don't cache main packages - if pkg.Name == "main" { + paths, ok := c.packageCachePathsForWrite(pkg) + if !ok { return nil } cm := c.ensureCacheManager() - paths := cm.PackagePaths(c.targetTriple(), pkg.PkgPath, pkg.Fingerprint) // Ensure directory exists if err := cm.EnsureDir(paths); err != nil { @@ -491,7 +479,11 @@ func (c *context) saveToCache(pkg *aPackage) error { // If ArchiveFile is already set (from normalizeToArchive), copy it to cache if pkg.ArchiveFile != "" { - if err := copyFileAtomic(pkg.ArchiveFile, paths.Archive); err != nil { + if filepath.Clean(pkg.ArchiveFile) != filepath.Clean(paths.Archive) { + if err := copyFileAtomic(pkg.ArchiveFile, paths.Archive); err != nil { + return err + } + } else if _, err := os.Stat(paths.Archive); err != nil { return err } } else if len(pkg.ObjFiles) > 0 { @@ -544,6 +536,20 @@ func (c *context) saveToCache(pkg *aPackage) error { return nil } +// packageCachePathsForWrite resolves the stable cache destination for pkg. +// Concurrent publishers must initialize cacheManager on the coordinator +// before entering this helper. +func (c *context) packageCachePathsForWrite(pkg *aPackage) (cachePaths, bool) { + if !cacheEnabled() || c.packageCacheDisabled(pkg.ID) { + return cachePaths{}, false + } + if pkg.Fingerprint == "" || pkg.Manifest == "" || pkg.Name == "main" { + return cachePaths{}, false + } + cm := c.ensureCacheManager() + return cm.PackagePaths(c.targetTriple(), pkg.PkgPath, pkg.Fingerprint), true +} + // copyFileAtomic copies src to dst using a temp file for atomicity. func copyFileAtomic(src, dst string) error { if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { diff --git a/internal/build/collect_test.go b/internal/build/collect_test.go index 879b4b6511..6597f65e23 100644 --- a/internal/build/collect_test.go +++ b/internal/build/collect_test.go @@ -33,6 +33,7 @@ import ( "github.com/goplus/llgo/internal/meta" "github.com/goplus/llgo/internal/packages" llssa "github.com/goplus/llgo/ssa" + gllvm "github.com/xgo-dev/llvm" gopackages "golang.org/x/tools/go/packages" ) @@ -920,6 +921,198 @@ func TestSaveToCache_MainPackage(t *testing.T) { } } +func TestNormalizeToArchivePublishesDirectlyToCache(t *testing.T) { + t.Setenv(llgoBuildCache, "1") + td := t.TempDir() + oldFunc := cacheRootFunc + cacheRootFunc = func() string { return td } + defer func() { cacheRootFunc = oldFunc }() + + llvmCtx := gllvm.NewContext() + defer llvmCtx.Dispose() + mod := llvmCtx.NewModule("direct-cache") + defer mod.Dispose() + mod.SetTarget("x86_64-unknown-linux-gnu") + gllvm.AddFunction(mod, "direct_cache_symbol", gllvm.FunctionType(llvmCtx.Int32Type(), nil, false)) + objBuffer := gllvm.WriteBitcodeToMemoryBuffer(mod) + + pkg := &aPackage{ + Package: &packages.Package{ + ID: "example.com/direct", + PkgPath: "example.com/direct", + Name: "direct", + }, + Fingerprint: "direct123", + Manifest: func() string { + m := newManifestBuilder() + m.env.Goos = "linux" + m.pkg.PkgPath = "example.com/direct" + return m.Build() + }(), + ObjBuffers: []packageArchiveBuffer{{ + name: "member.bc", + buffer: objBuffer, + }}, + } + defer pkg.disposeArchiveBuffers() + ctx := &context{ + buildConf: &Config{Goos: "linux", Goarch: "amd64"}, + crossCompile: crosscompile.Export{ + LLVMTarget: "x86_64-unknown-linux-gnu", + }, + pkgs: map[*packages.Package]Package{pkg.Package: pkg}, + } + paths := ctx.ensureCacheManager().PackagePaths(ctx.targetTriple(), pkg.PkgPath, pkg.Fingerprint) + + if err := normalizeToArchive(ctx, pkg, false); err != nil { + t.Fatal(err) + } + if pkg.ArchiveFile != paths.Archive { + t.Fatalf("ArchiveFile = %q, want cache path %q", pkg.ArchiveFile, paths.Archive) + } + if pkg.ArchiveTemp { + t.Fatal("cache archive was marked temporary") + } + if _, err := os.Stat(paths.Archive); err != nil { + t.Fatalf("cache archive does not exist: %v", err) + } + if err := ctx.saveToCache(pkg); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(paths.Manifest); err != nil { + t.Fatalf("cache manifest does not exist: %v", err) + } + + ctx.cleanupTemporaryArchives() + if _, err := os.Stat(paths.Archive); err != nil { + t.Fatalf("cleanup removed persistent cache archive: %v", err) + } +} + +func TestNormalizeToArchiveFallsBackWhenCacheArchiveIsUnwritable(t *testing.T) { + t.Setenv(llgoBuildCache, "1") + td := t.TempDir() + oldFunc := cacheRootFunc + cacheRootFunc = func() string { return td } + defer func() { cacheRootFunc = oldFunc }() + + objFile := filepath.Join(td, "member.o") + if err := os.WriteFile(objFile, []byte("object payload"), 0o644); err != nil { + t.Fatal(err) + } + pkg := &aPackage{ + Package: &packages.Package{ + ID: "example.com/fallback", + PkgPath: "example.com/fallback", + Name: "fallback", + }, + Fingerprint: "fallback123", + Manifest: func() string { + m := newManifestBuilder() + m.env.Goos = "linux" + m.pkg.PkgPath = "example.com/fallback" + return m.Build() + }(), + ObjFiles: []string{objFile}, + } + ctx := &context{ + buildConf: &Config{Goos: "linux", Goarch: "amd64"}, + pkgs: map[*packages.Package]Package{pkg.Package: pkg}, + } + paths := ctx.ensureCacheManager().PackagePaths(ctx.targetTriple(), pkg.PkgPath, pkg.Fingerprint) + if err := os.MkdirAll(paths.Archive, 0o755); err != nil { + t.Fatal(err) + } + + if err := normalizeToArchive(ctx, pkg, false); err != nil { + t.Fatalf("cache destination failure became fatal: %v", err) + } + if pkg.ArchiveFile == paths.Archive || !pkg.ArchiveTemp { + t.Fatalf("fallback archive state = %q, %v", pkg.ArchiveFile, pkg.ArchiveTemp) + } + if _, err := os.Stat(pkg.ArchiveFile); err != nil { + t.Fatalf("fallback archive does not exist: %v", err) + } + if err := ctx.saveToCache(pkg); err == nil { + t.Fatal("saveToCache unexpectedly succeeded with an invalid cache destination") + } + + archivePath := pkg.ArchiveFile + ctx.cleanupTemporaryArchives() + if _, err := os.Stat(archivePath); !os.IsNotExist(err) { + t.Fatalf("fallback archive was not removed: %v", err) + } +} + +func TestNormalizeToArchiveCleansTemporaryArchive(t *testing.T) { + t.Setenv(llgoBuildCache, "off") + td := t.TempDir() + objFile := filepath.Join(td, "member.o") + if err := os.WriteFile(objFile, []byte("object payload"), 0o644); err != nil { + t.Fatal(err) + } + pkg := &aPackage{ + Package: &packages.Package{ + ID: "example.com/temporary", + PkgPath: "example.com/temporary", + Name: "temporary", + }, + ObjFiles: []string{objFile}, + } + ctx := &context{ + buildConf: &Config{Goos: "linux", Goarch: "amd64"}, + pkgs: map[*packages.Package]Package{pkg.Package: pkg}, + } + + if err := normalizeToArchive(ctx, pkg, false); err != nil { + t.Fatal(err) + } + archivePath := pkg.ArchiveFile + if archivePath == "" || !pkg.ArchiveTemp { + t.Fatalf("temporary archive state = %q, %v", archivePath, pkg.ArchiveTemp) + } + if _, err := os.Stat(archivePath); err != nil { + t.Fatalf("temporary archive does not exist: %v", err) + } + + ctx.cleanupTemporaryArchives() + if _, err := os.Stat(archivePath); !os.IsNotExist(err) { + t.Fatalf("temporary archive was not removed: %v", err) + } + if pkg.ArchiveFile != "" || pkg.ArchiveTemp { + t.Fatalf("temporary archive state was not cleared: %q, %v", pkg.ArchiveFile, pkg.ArchiveTemp) + } +} + +func TestCleanupTemporaryArchivesPreservesDarwinDWARFInputs(t *testing.T) { + archivePath := filepath.Join(t.TempDir(), "main.a") + if err := os.WriteFile(archivePath, []byte("archive"), 0o644); err != nil { + t.Fatal(err) + } + pkg := &aPackage{ + Package: &packages.Package{PkgPath: "main", Name: "main"}, + ArchiveFile: archivePath, + ArchiveTemp: true, + } + ctx := &context{ + buildConf: &Config{ + Mode: ModeBuild, + Goos: "darwin", + Goarch: "arm64", + LinkOptions: LinkOptions{DWARF: DWARFPreserve}, + }, + pkgs: map[*packages.Package]Package{pkg.Package: pkg}, + } + + ctx.cleanupTemporaryArchives() + if _, err := os.Stat(archivePath); err != nil { + t.Fatalf("Darwin DWARF input archive was removed: %v", err) + } + if pkg.ArchiveFile != archivePath || !pkg.ArchiveTemp { + t.Fatalf("Darwin DWARF input state changed: %q, %v", pkg.ArchiveFile, pkg.ArchiveTemp) + } +} + func TestTryLoadFromCache_MainPackage(t *testing.T) { td := t.TempDir() oldFunc := cacheRootFunc diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go index 00d1ce3963..634004d267 100644 --- a/internal/build/package_build_test.go +++ b/internal/build/package_build_test.go @@ -18,6 +18,7 @@ package build import ( "errors" + "fmt" "go/types" "os" "path/filepath" @@ -307,6 +308,99 @@ func TestRunBoundedPackageJobsConvertsPanicToError(t *testing.T) { t.Fatalf("error = %v, want recovered panic %v", err, boom) } } + +func TestPublishPackagePipelineTasksUsesBoundedWorkers(t *testing.T) { + t.Setenv(llgoBuildCache, "off") + tasks := make([]*packagePipelineTask, 5) + for i := range tasks { + pkgPath := fmt.Sprintf("example.com/p%d", i) + tasks[i] = &packagePipelineTask{ + executed: i != 4, + spec: packageBuildSpec{pkg: &aPackage{Package: &packages.Package{ + ID: pkgPath, + PkgPath: pkgPath, + Name: fmt.Sprintf("p%d", i), + }}}, + } + } + + started := make(chan struct{}, len(tasks)) + release := make(chan struct{}) + done := make(chan error, 1) + var active atomic.Int32 + var maximum atomic.Int32 + go func() { + ctx := &context{buildConf: &Config{BuildParallelism: 8}} + done <- publishPackagePipelineTasksWith(ctx, tasks, false, func(_ *context, _ packageBuildSpec, _ bool) (packageBuildResult, error, error) { + current := active.Add(1) + for { + old := maximum.Load() + if current <= old || maximum.CompareAndSwap(old, current) { + break + } + } + started <- struct{}{} + <-release + active.Add(-1) + return packageBuildResult{}, nil, nil + }) + }() + + for range maxConcurrentPackagePublishes { + select { + case <-started: + case <-time.After(5 * time.Second): + close(release) + t.Fatal("publication workers did not start concurrently") + } + } + if got := maximum.Load(); got != maxConcurrentPackagePublishes { + close(release) + t.Fatalf("maximum concurrent publications = %d, want %d", got, maxConcurrentPackagePublishes) + } + close(release) + if err := <-done; err != nil { + t.Fatal(err) + } + if got := len(started); got != 2 { + t.Fatalf("remaining executed publications = %d, want 2", got) + } +} + +func TestPublishPackagePipelineTasksReturnsFirstOrderedError(t *testing.T) { + t.Setenv(llgoBuildCache, "off") + first := errors.New("first") + second := errors.New("second") + tasks := make([]*packagePipelineTask, 3) + for i := range tasks { + tasks[i] = &packagePipelineTask{ + executed: true, + spec: packageBuildSpec{pkg: &aPackage{Package: &packages.Package{ + PkgPath: fmt.Sprintf("example.com/p%d", i), + }}}, + } + } + err := publishPackagePipelineTasksWith( + &context{buildConf: &Config{BuildParallelism: 3}}, + tasks, + false, + func(_ *context, spec packageBuildSpec, _ bool) (packageBuildResult, error, error) { + switch spec.pkg.PkgPath { + case "example.com/p0": + time.Sleep(10 * time.Millisecond) + return packageBuildResult{}, nil, first + case "example.com/p1": + return packageBuildResult{}, nil, second + default: + return packageBuildResult{}, nil, nil + } + }, + ) + if !errors.Is(err, first) { + t.Fatalf("error = %v, want first submitted error", err) + } +} + func TestPackageBuildStageEmptyAndSkippedInputs(t *testing.T) { runtimePkg := &aPackage{Package: &packages.Package{PkgPath: env.LLGoRuntimePkg}} normalPkg := &aPackage{Package: &packages.Package{PkgPath: "example.com/normal"}} diff --git a/internal/build/package_pipeline.go b/internal/build/package_pipeline.go index 8a2c0e14f6..1925f6d501 100644 --- a/internal/build/package_pipeline.go +++ b/internal/build/package_pipeline.go @@ -19,6 +19,7 @@ package build import ( "fmt" "go/ast" + "os" "github.com/goplus/llgo/cl" "golang.org/x/tools/go/ssa" @@ -53,6 +54,8 @@ const ( pipelineStagePatchedBackend ) +const maxConcurrentPackagePublishes = 2 + type packagePipelineSSA struct { entry ssaBuildEntry order int @@ -360,15 +363,56 @@ func buildPackagePipeline(ctx *context, entries []ssaBuildEntry, pkgs []*aPackag return nil, err } } - for _, task := range tasks { - if !task.executed { - continue + if err := publishPackagePipelineTasks(ctx, tasks, verbose); err != nil { + return nil, err + } + return pkgs, nil +} + +type packagePublishFunc func(*context, packageBuildSpec, bool) (packageBuildResult, error, error) + +func publishPackagePipelineTasks(ctx *context, tasks []*packagePipelineTask, verbose bool) error { + return publishPackagePipelineTasksWith(ctx, tasks, verbose, publishPackageBuild) +} + +func publishPackagePipelineTasksWith( + ctx *context, + tasks []*packagePipelineTask, + verbose bool, + publish packagePublishFunc, +) error { + indexes := make([]int, 0, len(tasks)) + for i, task := range tasks { + if task.executed { + indexes = append(indexes, i) } - if _, err := finalizePackageBuild(ctx, task.spec, verbose); err != nil { - return nil, err + } + if len(indexes) == 0 { + return nil + } + + // cacheManager is lazily initialized. Resolve it on the coordinator before + // cacheable packages enter concurrent publication. + if cacheEnabled() { + ctx.ensureCacheManager() + } + + warnings := make([]error, len(tasks)) + // Archive creation is I/O-heavy. More than two concurrent archivers caused + // severe filesystem contention in large builds, so keep publication within + // both the build-wide -p setting and this tighter I/O bound. + parallelism := min(ctx.buildConf.parallelism(), maxConcurrentPackagePublishes) + err := runBoundedPackageJobs(parallelism, indexes, func(index int) error { + _, warning, err := publish(ctx, tasks[index].spec, verbose) + warnings[index] = warning + return err + }) + for _, index := range indexes { + if warning := warnings[index]; warning != nil && verbose { + fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", tasks[index].spec.pkg.PkgPath, warning) } } - return pkgs, nil + return err } func observePackagePipeline(ctx *context, stage packagePipelineStage, pkgPath string, start bool) {