From 9010dfd79bd951ce7d13184df270c3dd70091532 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Mon, 27 Jul 2026 17:39:49 +0800 Subject: [PATCH] build: prepare bounded parallel compilation --- internal/build/build.go | 290 ++++++++++++++++++---------- internal/build/collect.go | 62 ++++-- internal/build/collect_test.go | 40 ++++ internal/build/dependencies.go | 52 +++++ internal/build/dependencies_test.go | 46 +++++ internal/build/fingerprint_test.go | 7 +- internal/build/module_hook_test.go | 1 + internal/goflags/flagfile.go | 14 +- internal/goflags/flagfile_test.go | 11 ++ internal/goflags/gobuild.go | 33 +++- internal/goflags/gobuild_test.go | 25 +++ 11 files changed, 458 insertions(+), 123 deletions(-) create mode 100644 internal/build/dependencies.go create mode 100644 internal/build/dependencies_test.go diff --git a/internal/build/build.go b/internal/build/build.go index bc96bd469b..3bf3bda540 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -183,6 +183,9 @@ type Config struct { // DisableBoundsChecks disables index, slice, and slice-to-array conversion // bounds checks while retaining required integer conversions and nil checks. DisableBoundsChecks bool + // Parallel is the maximum number of LLGo's internally parallel build tasks. + // Zero uses GOMAXPROCS; command-line -p, like go build -p, must be >= 1. + Parallel int // PthreadStackSize sets a custom stack size, in bytes, for pthread-backed // goroutines. A zero value keeps the platform pthread default. @@ -281,7 +284,19 @@ const ( loadSyntax = loadTypes | packages.NeedSyntax | packages.NeedTypesInfo ) +var llssaInitOnce sync.Once + +func (c *Config) parallelism() int { + if c.Parallel > 0 { + return c.Parallel + } + return runtime.GOMAXPROCS(0) +} + func Do(args []string, conf *Config) ([]Package, error) { + if conf.Parallel < 0 { + return nil, fmt.Errorf("parallelism must not be negative: %d", conf.Parallel) + } if conf.Goos == "" { conf.Goos = runtime.GOOS } @@ -372,7 +387,9 @@ func Do(args []string, conf *Config) ([]Package, error) { cl.EnableDebug(emitDebugInfo) cl.EnableDbgSyms(emitDebugInfo) cl.EnableTrace(IsTraceEnabled()) - llssa.Initialize(llssa.InitAll) + llssaInitOnce.Do(func() { + llssa.Initialize(llssa.InitAll) + }) target := &llssa.Target{ GOOS: conf.Goos, @@ -498,7 +515,6 @@ func Do(args []string, conf *Config) ([]Package, error) { } progSSA := ssa.NewProgram(initial[0].Fset, buildMode) patches := make(cl.Patches, len(altPkgPaths)) - altSSAPkgs(progSSA, patches, altPkgs[1:], conf, verbose) env := llvm.New("") os.Setenv("PATH", env.BinDir()+":"+os.Getenv("PATH")) // TODO(xsw): check windows @@ -515,20 +531,24 @@ func Do(args []string, conf *Config) ([]Package, error) { buildConf: conf, crossCompile: export, cTransformer: cabi.NewTransformer(prog, export.LLVMTarget, export.TargetABI, conf.AbiMode, cabiOptimize), + sfilesCache: make(map[string][]string), } defer ctx.closePackageMetas() + ctx.initializePackageBuildState() // default runtime globals must be registered before packages are built addGlobalString(conf, "runtime.defaultGOROOT="+runtime.GOROOT(), nil) addGlobalString(conf, "runtime.buildVersion="+runtime.Version(), nil) - pkgs, err := buildSSAPkgs(ctx, initial, verbose) + altEntries := registerAltSSAPkgs(progSSA, patches, altPkgs[1:], conf, verbose) + pkgs, pkgEntries, err := registerSSAPkgs(ctx, initial, verbose) if err != nil { return nil, err } - depPkgs, err := buildSSAPkgs(ctx, altPkgs, verbose) + depPkgs, depEntries, err := registerSSAPkgs(ctx, altPkgs, verbose) if err != nil { return nil, err } + buildSSAPkgs(ctx, append(append(altEntries, pkgEntries...), depEntries...)) allPkgs := append([]*aPackage{}, pkgs...) allPkgs = append(allPkgs, depPkgs...) @@ -769,8 +789,10 @@ type context struct { testFail bool // Cache related fields - cacheManager *cacheManager - llvmVersion string + cacheManager *cacheManager + cacheDisabled map[string]none + llvmVersion string + llvmVersionReady bool // go list derived file lists (SFiles, etc.) sfilesCache map[string][]string // pkg.ID -> absolute .s/.S file paths @@ -857,8 +879,6 @@ func normalizeToArchive(ctx *context, aPkg *aPackage, verbose bool) error { } func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, error) { - built := ctx.built - // Split packages into runtime tree vs others so we can defer runtime build. var runtimePkgs []*aPackage var normalPkgs []*aPackage @@ -872,91 +892,20 @@ func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, er var needRuntime, needPyInit bool - buildOne := func(aPkg *aPackage) error { - pkg := aPkg.Package - if _, ok := built[pkg.ID]; ok { - // Already built, skip but keep ExportFile for linking - return nil - } - built[pkg.ID] = none{} - - switch kind, param := cl.PkgKindOf(pkg.Types); kind { - case cl.PkgDeclOnly: - pkg.ExportFile = "" - case cl.PkgLinkIR, cl.PkgLinkExtern, cl.PkgPyModule: - if len(pkg.GoFiles) > 0 { - if err := ctx.collectFingerprint(aPkg); err != nil { - return err - } - ctx.tryLoadFromCache(aPkg) - if verbose { - if aPkg.CacheHit { - fmt.Fprintf(os.Stderr, "CACHE HIT: %s\n", pkg.PkgPath) - } else { - fmt.Fprintf(os.Stderr, "CACHE MISS: %s\n", pkg.PkgPath) - } - } - if err := buildPkg(ctx, aPkg, verbose); err != nil { - return err - } - if !aPkg.CacheHit { - if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { - return err - } - if kind == cl.PkgLinkExtern { - appendExternalLinkArgs(ctx, aPkg, param) - } - if err := ctx.saveToCache(aPkg); err != nil && verbose { - fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", pkg.PkgPath, err) - } - } - } else { - pkg.ExportFile = "" - if kind == cl.PkgLinkExtern { - appendExternalLinkArgs(ctx, aPkg, param) - } - } - default: - if err := ctx.collectFingerprint(aPkg); err != nil { - return err - } - ctx.tryLoadFromCache(aPkg) - if verbose { - if aPkg.CacheHit { - fmt.Fprintf(os.Stderr, "CACHE HIT: %s\n", pkg.PkgPath) - } else { - fmt.Fprintf(os.Stderr, "CACHE MISS: %s\n", pkg.PkgPath) - } - } - if err := buildPkg(ctx, aPkg, verbose); err != nil { - return err - } - aPkg.setNeedRuntimeOrPyInit(aPkg.LPkg.NeedRuntime, aPkg.LPkg.NeedPyInit) - needRuntime = needRuntime || aPkg.NeedRt - needPyInit = needPyInit || aPkg.NeedPyInit - if !aPkg.CacheHit { - if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { - return err - } - if err := ctx.saveToCache(aPkg); err != nil && verbose { - fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", pkg.PkgPath, err) - } - } - } - return nil - } - // Build non-runtime packages first, so we know whether runtime is actually needed. for _, p := range normalPkgs { - if err := buildOne(p); err != nil { + result, err := buildOnePackage(ctx, p, verbose) + if err != nil { return nil, err } + needRuntime = needRuntime || result.needRuntime + needPyInit = needPyInit || result.needPyInit } // Only build runtime packages when required (or host build with empty Target). if needRuntime || needPyInit || ctx.buildConf.Target == "" { for _, p := range runtimePkgs { - if err := buildOne(p); err != nil { + if _, err := buildOnePackage(ctx, p, verbose); err != nil { return nil, err } } @@ -965,6 +914,69 @@ func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, er return pkgs, nil } +type packageBuildResult struct { + needRuntime bool + needPyInit bool +} + +// buildOnePackage is the serial package pipeline. The stages are separated so +// a future scheduler can parallelize only stages with isolated package state. +func buildOnePackage(ctx *context, aPkg *aPackage, verbose bool) (packageBuildResult, error) { + pkg := aPkg.Package + if _, ok := ctx.built[pkg.ID]; ok { + return packageBuildResult{}, nil + } + ctx.built[pkg.ID] = none{} + + kind, param := cl.PkgKindOf(pkg.Types) + if kind == cl.PkgDeclOnly { + pkg.ExportFile = "" + return packageBuildResult{}, nil + } + if (kind == cl.PkgLinkIR || kind == cl.PkgLinkExtern || kind == cl.PkgPyModule) && len(pkg.GoFiles) == 0 { + pkg.ExportFile = "" + if kind == cl.PkgLinkExtern { + appendExternalLinkArgs(ctx, aPkg, param) + } + return packageBuildResult{}, nil + } + + if err := ctx.collectFingerprint(aPkg); err != nil { + return packageBuildResult{}, err + } + ctx.tryLoadFromCache(aPkg) + if verbose { + status := "MISS" + if aPkg.CacheHit { + status = "HIT" + } + fmt.Fprintf(os.Stderr, "CACHE %s: %s\n", status, pkg.PkgPath) + } + if err := buildPkg(ctx, aPkg, verbose); err != nil { + return packageBuildResult{}, err + } + + result := packageBuildResult{} + if kind != cl.PkgLinkIR && kind != cl.PkgLinkExtern && kind != cl.PkgPyModule { + aPkg.setNeedRuntimeOrPyInit(aPkg.LPkg.NeedRuntime, aPkg.LPkg.NeedPyInit) + result.needRuntime = aPkg.NeedRt + result.needPyInit = aPkg.NeedPyInit + } + if aPkg.CacheHit { + return result, nil + } + if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { + return packageBuildResult{}, err + } + if kind == cl.PkgLinkExtern { + appendExternalLinkArgs(ctx, aPkg, param) + } + if err := ctx.saveToCache(aPkg); err != nil && verbose { + fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", pkg.PkgPath, err) + } + return result, nil +} + func appendExternalLinkArgs(ctx *context, aPkg *aPackage, spec string) { // need to be linked with external library // format: ';' separated alternative link methods. e.g. @@ -1618,6 +1630,16 @@ func is32Bits(goarch string) bool { } func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { + externs, err := preparePackageModule(ctx, aPkg, verbose) + if err != nil || aPkg.CacheHit || aPkg.LPkg == nil { + return err + } + return compilePackageModule(ctx, aPkg, externs, verbose) +} + +// preparePackageModule runs the frontend and creates the package LLVM module. +// This stage remains serial because it updates Program-wide registration state. +func preparePackageModule(ctx *context, aPkg *aPackage, verbose bool) ([]string, error) { pkg := aPkg.Package pkgPath := pkg.PkgPath if debugBuild || verbose { @@ -1627,7 +1649,7 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { } if llruntime.SkipToBuild(pkgPath) { pkg.ExportFile = "" - return nil + return nil, nil } var syntax = pkg.Syntax if altPkg := aPkg.AltPkg; altPkg != nil { @@ -1645,7 +1667,7 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { embedMap, err := goembed.LoadDirectives(ctx.conf.Fset, syntax) if err != nil { - return fmt.Errorf("load go:embed directives for %s failed: %w", pkgPath, err) + return nil, fmt.Errorf("load go:embed directives for %s failed: %w", pkgPath, err) } needMeta := !aPkg.CacheHit && ctx.buildConf.packageMetaEnabled() @@ -1662,8 +1684,18 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { // If cache hit, we only needed to register types - skip compilation if aPkg.CacheHit { - return nil + return nil, nil } + return externs, nil +} + +// compilePackageModule applies LLVM transforms and emits package objects. +// It receives a module owned by aPkg and is intentionally kept separate from +// frontend setup so later PRs can give it a worker-local backend context. +func compilePackageModule(ctx *context, aPkg *aPackage, externs []string, verbose bool) error { + pkg := aPkg.Package + pkgPath := pkg.PkgPath + ret := aPkg.LPkg ctx.cTransformer.SetSkipFuncs(cabiSkipFuncsForPlan9Asm(ctx, pkgPath, ret.Module())) llabi.LowerLargeAggregates(ctx.prog.TargetData(), ret.Module()) @@ -1677,7 +1709,7 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { mod.SetTarget(ctx.prog.Target().Spec().Triple) pbo := gllvm.NewPassBuilderOptions() defer pbo.Dispose() - if err = gllvm.VerifyModule(mod, gllvm.ReturnStatusAction); err != nil { + if err := gllvm.VerifyModule(mod, gllvm.ReturnStatusAction); err != nil { return fmt.Errorf("verify LLVM module for %v failed: %w", pkgPath, err) } if err := mod.RunPasses(llvmPassPipeline(ctx.buildConf.OptLevel, ctx.buildConf.ltoMode()), ctx.prog.TargetMachine(), pbo); err != nil { @@ -1933,13 +1965,24 @@ func preCollectRuntimeLinknames(prog llssa.Program, pkgs []*packages.Package) { } } -func altSSAPkgs(prog *ssa.Program, patches cl.Patches, alts []*packages.Package, conf *Config, verbose bool) { +type ssaBuildEntry struct { + pkg *ssa.Package + syntax []*ast.File + fixOrder bool +} + +// registerAltSSAPkgs creates the alternate packages and patch table before any +// SSA package is built. Building is deliberately deferred so every package can +// be built under the same bounded worker pool. +func registerAltSSAPkgs(prog *ssa.Program, patches cl.Patches, alts []*packages.Package, conf *Config, verbose bool) []ssaBuildEntry { + var entries []ssaBuildEntry packages.Visit(alts, nil, func(p *packages.Package) { if typs := p.Types; typs != nil && !p.IllTyped { if debugBuild || verbose { log.Println("==> BuildSSA", p.ID) } pkgSSA := prog.CreatePackage(typs, p.Syntax, p.TypesInfo, true) + entries = append(entries, ssaBuildEntry{pkg: pkgSSA, syntax: p.Syntax}) if strings.HasPrefix(p.ID, altPkgPathPrefix) { path := p.ID[len(altPkgPathPrefix):] // Even if an alt package exists and is pulled in as a dependency of other @@ -1955,7 +1998,7 @@ func altSSAPkgs(prog *ssa.Program, patches cl.Patches, alts []*packages.Package, } } }) - prog.Build() + return entries } type aPackage struct { @@ -1981,9 +2024,13 @@ type aPackage struct { type Package = *aPackage -func buildSSAPkgs(ctx *context, initial []*packages.Package, verbose bool) ([]*aPackage, error) { +// registerSSAPkgs creates all ordinary SSA packages and returns their build +// entries. It intentionally performs no Package.Build calls: applying patches +// and registering the full package graph must remain serial. +func registerSSAPkgs(ctx *context, initial []*packages.Package, verbose bool) ([]*aPackage, []ssaBuildEntry, error) { prog := ctx.progSSA var all []*aPackage + var entries []ssaBuildEntry var errs []*packages.Package packages.Visit(initial, nil, func(p *packages.Package) { if p.Types != nil && !p.IllTyped { @@ -1993,7 +2040,10 @@ func buildSSAPkgs(ctx *context, initial []*packages.Package, verbose bool) ([]*a return } var altPkg *packages.Cached - var ssaPkg = createSSAPkg(ctx, prog, p, verbose) + ssaPkg, created := createSSAPkg(ctx, prog, p, verbose) + if created { + entries = append(entries, ssaBuildEntry{pkg: ssaPkg, syntax: p.Syntax, fixOrder: true}) + } if ctx.hasAltPkg(pkgPath) { if altPkg = ctx.dedup.Check(altPkgPathPrefix + pkgPath); altPkg == nil { return @@ -2025,9 +2075,53 @@ func buildSSAPkgs(ctx *context, initial []*packages.Package, verbose bool) ([]*a } fmt.Fprintln(os.Stderr, "cannot build SSA for package", errPkg) } - return nil, fmt.Errorf("cannot build SSA for packages") + return nil, nil, fmt.Errorf("cannot build SSA for packages") + } + return all, entries, nil +} + +// buildSSAPkgs builds registered packages with the configured bound, then +// performs ordering repair serially because it mutates package instruction +// slices. go/ssa.Package.Build is documented as safe for concurrent calls. +func buildSSAPkgs(ctx *context, entries []ssaBuildEntry) { + if len(entries) == 0 { + return + } + unique := make([]ssaBuildEntry, 0, len(entries)) + entryIndex := make(map[*ssa.Package]int, len(entries)) + for _, entry := range entries { + if entry.pkg == nil { + continue + } + if index, exists := entryIndex[entry.pkg]; exists { + unique[index].fixOrder = unique[index].fixOrder || entry.fixOrder + continue + } + entryIndex[entry.pkg] = len(unique) + unique = append(unique, entry) + } + workers := min(ctx.buildConf.parallelism(), len(unique)) + jobs := make(chan ssaBuildEntry, len(unique)) + var wg sync.WaitGroup + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + for entry := range jobs { + entry.pkg.Build() + } + }() + } + for _, entry := range unique { + jobs <- entry + } + close(jobs) + wg.Wait() + for _, entry := range unique { + if entry.fixOrder { + fixSSAOrder(entry.pkg, entry.syntax) + } } - return all, nil } func formatPackageError(err packages.Error, noColumn bool) string { @@ -2175,7 +2269,7 @@ func applyPatches(ctx *context, p *packages.Package, verbose bool) { } } -func createSSAPkg(ctx *context, prog *ssa.Program, p *packages.Package, verbose bool) *ssa.Package { +func createSSAPkg(ctx *context, prog *ssa.Program, p *packages.Package, verbose bool) (*ssa.Package, bool) { pkgSSA := prog.ImportedPackage(p.ID) if pkgSSA == nil { if debugBuild || verbose { @@ -2183,11 +2277,9 @@ func createSSAPkg(ctx *context, prog *ssa.Program, p *packages.Package, verbose } applyPatches(ctx, p, verbose) pkgSSA = prog.CreatePackage(p.Types, p.Syntax, p.TypesInfo, true) - pkgSSA.Build() // TODO(xsw): build concurrently - // Apply local SSA fixups once when package SSA is first built. - fixSSAOrder(pkgSSA, p.Syntax) + return pkgSSA, true } - return pkgSSA + return pkgSSA, false } /* diff --git a/internal/build/collect.go b/internal/build/collect.go index e89db4d1c5..b4e0790eca 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -23,7 +23,6 @@ import ( "os/exec" "path/filepath" "runtime" - "sort" "strconv" "strings" @@ -42,7 +41,12 @@ func (c *context) collectFingerprint(pkg *aPackage) error { c.fingerprinting = make(map[string]bool) } if c.fingerprinting[pkg.ID] { - return fmt.Errorf("fingerprint cycle detected for %s", pkg.ID) + // Alternate packages can intentionally close a cycle in the runtime + // replacement graph after all packages have been built into SSA. A + // cycle cannot have a stable per-package cache key, so compile every + // member rather than returning an incorrect cache hit. + c.disablePackageCache(c.fingerprinting) + return nil } c.fingerprinting[pkg.ID] = true defer delete(c.fingerprinting, pkg.ID) @@ -70,6 +74,20 @@ func (c *context) collectFingerprint(pkg *aPackage) error { return nil } +func (c *context) disablePackageCache(pkgs map[string]bool) { + if c.cacheDisabled == nil { + c.cacheDisabled = make(map[string]none, len(pkgs)) + } + for id := range pkgs { + c.cacheDisabled[id] = none{} + } +} + +func (c *context) packageCacheDisabled(id string) bool { + _, disabled := c.cacheDisabled[id] + return disabled +} + // collectEnvInputs collects environment-related inputs. func (c *context) collectEnvInputs(m *manifestBuilder) { m.env.Goos = c.buildConf.Goos @@ -196,21 +214,7 @@ func (c *context) collectPackageInputs(m *manifestBuilder, pkg *aPackage) error // collectDependencyInputs adds dependency fingerprints/versions into manifest. func (c *context) collectDependencyInputs(m *manifestBuilder, pkg *aPackage) error { - if len(pkg.Imports) == 0 { - return nil - } - - deps := make([]*packages.Package, 0, len(pkg.Imports)) - for _, dep := range pkg.Imports { - if dep == nil || dep.ID == pkg.ID { - continue - } - deps = append(deps, dep) - } - - sort.Slice(deps, func(i, j int) bool { return deps[i].ID < deps[j].ID }) - - for _, dep := range deps { + for _, dep := range effectiveDependencies(pkg) { depEntry, err := c.dependencyFingerprint(dep) if err != nil { return err @@ -221,6 +225,21 @@ func (c *context) collectDependencyInputs(m *manifestBuilder, pkg *aPackage) err return nil } +// initializePackageBuildState initializes the mutable state used by the +// package pipeline before any scheduler is introduced. This makes ownership +// explicit and avoids lazy first-use writes becoming data races later. +func (c *context) initializePackageBuildState() { + if c.sfilesCache == nil { + c.sfilesCache = make(map[string][]string) + } + if !cacheEnabled() { + return + } + c.cacheManager = newCacheManager() + c.llvmVersion = detectLLVMVersion(c) + c.llvmVersionReady = true +} + func (c *context) dependencyFingerprint(dep *packages.Package) (depEntry, error) { entry := depEntry{ID: dep.ID} if v := moduleVersion(dep.Module); v != "" { @@ -264,10 +283,11 @@ func moduleVersion(mod *gopackages.Module) string { // getLLVMVersion returns the cached LLVM version or detects it. func (c *context) getLLVMVersion() string { - if c.llvmVersion != "" { + if c.llvmVersionReady { return c.llvmVersion } c.llvmVersion = detectLLVMVersion(c) + c.llvmVersionReady = true return c.llvmVersion } @@ -326,6 +346,9 @@ func (c *context) tryLoadFromCache(pkg *aPackage) bool { if !cacheEnabled() { return false } + if c.packageCacheDisabled(pkg.ID) { + return false + } // Main packages are intentionally not written to the build cache because // each executable's entry module is linked against the current main archive. @@ -451,6 +474,9 @@ func (c *context) saveToCache(pkg *aPackage) error { if !cacheEnabled() { return nil } + if c.packageCacheDisabled(pkg.ID) { + return nil + } if pkg.Fingerprint == "" || pkg.Manifest == "" { return nil diff --git a/internal/build/collect_test.go b/internal/build/collect_test.go index 0dd9d7ca4d..7cc6e85de1 100644 --- a/internal/build/collect_test.go +++ b/internal/build/collect_test.go @@ -90,6 +90,46 @@ func TestCollectFingerprint(t *testing.T) { } } +func TestCollectFingerprintAltDependencyCycleDisablesCache(t *testing.T) { + runtimePkg := &packages.Package{ID: "runtime", PkgPath: "runtime"} + osPkg := &packages.Package{ID: "runtime/internal/clite/os", PkgPath: "runtime/internal/clite/os"} + syscallPkg := &packages.Package{ID: "runtime/internal/clite/syscall", PkgPath: "runtime/internal/clite/syscall"} + runtime := &aPackage{ + Package: runtimePkg, + AltPkg: &packages.Cached{Package: &packages.Package{ + ID: "runtime/alt", + Imports: map[string]*packages.Package{"os": osPkg}, + }}, + } + os := &aPackage{Package: osPkg} + syscall := &aPackage{Package: syscallPkg} + os.Imports = map[string]*packages.Package{"syscall": syscallPkg} + syscall.Imports = map[string]*packages.Package{"runtime": runtimePkg} + ctx := &context{ + conf: &packages.Config{}, + buildConf: &Config{Goos: "js", Goarch: "wasm"}, + crossCompile: crosscompile.Export{ + LLVMTarget: "wasm32-unknown-unknown", + }, + pkgByID: map[string]Package{ + runtimePkg.ID: runtime, + osPkg.ID: os, + syscallPkg.ID: syscall, + }, + } + if err := ctx.collectFingerprint(runtime); err != nil { + t.Fatalf("collectFingerprint: %v", err) + } + for _, pkg := range []*aPackage{runtime, os, syscall} { + if !ctx.packageCacheDisabled(pkg.ID) { + t.Fatalf("cache for %s was not disabled after fingerprint cycle", pkg.ID) + } + if pkg.Fingerprint == "" || pkg.Manifest == "" { + t.Fatalf("fingerprint state for %s was not completed", pkg.ID) + } + } +} + func TestCollectFingerprintDeterminism(t *testing.T) { td := t.TempDir() diff --git a/internal/build/dependencies.go b/internal/build/dependencies.go new file mode 100644 index 0000000000..a2197623d0 --- /dev/null +++ b/internal/build/dependencies.go @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "sort" + + "github.com/goplus/llgo/internal/packages" +) + +// effectiveDependencies returns the package graph that contributes code to an +// aPackage. An alternate package can add imports beyond the original package, +// so using only Package.Imports would allow stale cache entries after an alt +// dependency changes. +func effectiveDependencies(pkg *aPackage) []*packages.Package { + if pkg == nil || pkg.Package == nil { + return nil + } + deps := make(map[string]*packages.Package) + add := func(imports map[string]*packages.Package) { + for _, dep := range imports { + if dep == nil || dep.ID == pkg.ID || (pkg.AltPkg != nil && dep.ID == pkg.AltPkg.ID) { + continue + } + deps[dep.ID] = dep + } + } + add(pkg.Imports) + if pkg.AltPkg != nil { + add(pkg.AltPkg.Imports) + } + ret := make([]*packages.Package, 0, len(deps)) + for _, dep := range deps { + ret = append(ret, dep) + } + sort.Slice(ret, func(i, j int) bool { return ret[i].ID < ret[j].ID }) + return ret +} diff --git a/internal/build/dependencies_test.go b/internal/build/dependencies_test.go new file mode 100644 index 0000000000..7f083eb13d --- /dev/null +++ b/internal/build/dependencies_test.go @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "reflect" + "testing" + + "github.com/goplus/llgo/internal/packages" +) + +func TestEffectiveDependenciesIncludesAlternateImports(t *testing.T) { + base := &packages.Package{ID: "base"} + shared := &packages.Package{ID: "shared"} + altOnly := &packages.Package{ID: "alt-only"} + alt := &packages.Package{ + ID: "alt", + Imports: map[string]*packages.Package{"shared": shared, "alt-only": altOnly}, + } + pkg := &aPackage{ + Package: &packages.Package{ID: "pkg", Imports: map[string]*packages.Package{"base": base, "shared": shared}}, + AltPkg: &packages.Cached{Package: alt}, + } + deps := effectiveDependencies(pkg) + got := make([]string, len(deps)) + for i, dep := range deps { + got[i] = dep.ID + } + if want := []string{"alt-only", "base", "shared"}; !reflect.DeepEqual(got, want) { + t.Fatalf("effectiveDependencies = %v, want %v", got, want) + } +} diff --git a/internal/build/fingerprint_test.go b/internal/build/fingerprint_test.go index 697f9ef8af..e45342a731 100644 --- a/internal/build/fingerprint_test.go +++ b/internal/build/fingerprint_test.go @@ -158,10 +158,9 @@ func TestManifestBuilder_EmptySections(t *testing.T) { m := newManifestBuilder() content := m.Build() - // Empty sections should not be written - expected := `` - if content != expected { - t.Errorf("unexpected empty manifest:\ngot:\n%s\nwant:\n%s", content, expected) + // Empty sections should not be written. + if content != "" { + t.Errorf("unexpected empty manifest:\ngot:\n%s", content) } // Should still produce a valid fingerprint diff --git a/internal/build/module_hook_test.go b/internal/build/module_hook_test.go index 99d2bde828..d91def525d 100644 --- a/internal/build/module_hook_test.go +++ b/internal/build/module_hook_test.go @@ -9,6 +9,7 @@ import ( func TestModuleHookReceivesMainPackageModule(t *testing.T) { conf := NewDefaultConf(ModeGen) + conf.Parallel = 2 counts := make(map[string]int) snapshots := make(map[string]string) diff --git a/internal/goflags/flagfile.go b/internal/goflags/flagfile.go index e8e4adf975..48ad661fcd 100644 --- a/internal/goflags/flagfile.go +++ b/internal/goflags/flagfile.go @@ -23,6 +23,18 @@ import ( ) var argumentListFlagNames = [...]string{ + "asmflags", + "gcflags", + "gccgoflags", + "ldflags", + "p", + "toolexec", +} + +// wholeLineValueFlagNames is the subset whose unquoted values can themselves +// contain arbitrary flag-like words. Scalar flags such as -p still belong to +// argumentListFlagNames for normalization, but must not consume a whole line. +var wholeLineValueFlagNames = [...]string{ "asmflags", "gcflags", "gccgoflags", @@ -62,7 +74,7 @@ func ParseFlagFile(data string) ([]string, error) { } func wholeLineValueFlag(line string) (flag string, ok bool) { - for _, name := range argumentListFlagNames { + for _, name := range wholeLineValueFlagNames { for _, prefix := range []string{"-" + name + "=", "--" + name + "="} { value, found := strings.CutPrefix(line, prefix) if !found { diff --git a/internal/goflags/flagfile_test.go b/internal/goflags/flagfile_test.go index d7938f3e93..efda442073 100644 --- a/internal/goflags/flagfile_test.go +++ b/internal/goflags/flagfile_test.go @@ -107,3 +107,14 @@ func TestParseFlagFileErrors(t *testing.T) { } } } + +func TestParseFlagFileKeepsScalarParallelFlagSeparate(t *testing.T) { + got, err := ParseFlagFile("-p=4 -trimpath -tags=fast\n") + if err != nil { + t.Fatal(err) + } + want := []string{"-p=4", "-trimpath", "-tags=fast"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("ParseFlagFile() = %#v, want %#v", got, want) + } +} diff --git a/internal/goflags/gobuild.go b/internal/goflags/gobuild.go index 6f38f42d9f..7d10074f1f 100644 --- a/internal/goflags/gobuild.go +++ b/internal/goflags/gobuild.go @@ -16,7 +16,13 @@ package goflags -import "github.com/goplus/llgo/internal/build" +import ( + "fmt" + "strconv" + "strings" + + "github.com/goplus/llgo/internal/build" +) // ApplyBuildFlags validates and appends normalized Go build flags, and maps // the supported compiler and linker semantics into typed build configuration. @@ -34,12 +40,37 @@ func ApplyBuildFlags(conf *build.Config, args []string) error { if err != nil { return err } + parallel, parallelSet, err := parseBuildParallel(all) + if err != nil { + return err + } next := *conf next.GoBuildFlags = all applyFrontendGCFlags(&next) + if parallelSet { + next.Parallel = parallel + } if linkFlags.Present { next.LinkOptions = linkFlags.Options } *conf = next return nil } + +// parseBuildParallel extracts Go's -p build concurrency flag after it has +// been normalized. Keeping it in GoBuildFlags still lets go/packages apply the +// same setting while Config.Parallel controls LLGo's own build stages. +func parseBuildParallel(flags []string) (parallel int, present bool, err error) { + for _, flag := range flags { + value, ok := strings.CutPrefix(flag, "-p=") + if !ok { + continue + } + parallel, err = strconv.Atoi(value) + if err != nil || parallel <= 0 { + return 0, false, fmt.Errorf("-p must be a positive integer, got %q", value) + } + present = true + } + return parallel, present, nil +} diff --git a/internal/goflags/gobuild_test.go b/internal/goflags/gobuild_test.go index f901e22f72..de809b7557 100644 --- a/internal/goflags/gobuild_test.go +++ b/internal/goflags/gobuild_test.go @@ -105,3 +105,28 @@ func TestApplyBuildFlagsFrontendGCFlagSemantics(t *testing.T) { }) } } + +func TestApplyBuildFlagsParallelism(t *testing.T) { + conf := &build.Config{} + if err := ApplyBuildFlags(conf, []string{"--p", "3"}); err != nil { + t.Fatal(err) + } + if conf.Parallel != 3 { + t.Fatalf("Parallel = %d, want 3", conf.Parallel) + } + if !reflect.DeepEqual(conf.GoBuildFlags, []string{"-p=3"}) { + t.Fatalf("GoBuildFlags = %#v, want [-p=3]", conf.GoBuildFlags) + } +} + +func TestApplyBuildFlagsRejectsInvalidParallelismAtomically(t *testing.T) { + conf := &build.Config{Parallel: 2, GoBuildFlags: []string{"-tags=existing"}} + want := *conf + want.GoBuildFlags = append([]string(nil), conf.GoBuildFlags...) + if err := ApplyBuildFlags(conf, []string{"-p=0"}); err == nil { + t.Fatal("ApplyBuildFlags succeeded, want error") + } + if !reflect.DeepEqual(*conf, want) { + t.Fatalf("configuration changed on error:\n got %+v\nwant %+v", *conf, want) + } +}