From aff2665cd7e863e77f110fdc1b010f43367de0b6 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Thu, 30 Jul 2026 08:06:38 +0800 Subject: [PATCH 01/14] build: split package compilation stages --- internal/build/build.go | 291 +++++++++++++++++---------- internal/build/collect.go | 43 ++-- internal/build/dependencies.go | 50 +++++ internal/build/dependencies_test.go | 46 +++++ internal/build/package_build.go | 64 ++++++ internal/build/package_build_test.go | 101 ++++++++++ 6 files changed, 475 insertions(+), 120 deletions(-) create mode 100644 internal/build/dependencies.go create mode 100644 internal/build/dependencies_test.go create mode 100644 internal/build/package_build.go create mode 100644 internal/build/package_build_test.go diff --git a/internal/build/build.go b/internal/build/build.go index 9607c9d00a..4e4d78d5bc 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -347,6 +347,13 @@ func (c *Config) packageMetaEnabled() bool { return c.CollectPackageMeta || c.deadcodeDropEnabled() } +func (c *Config) parallelism() int { + if c != nil && c.BuildParallelism > 0 { + return c.BuildParallelism + } + return max(1, runtime.GOMAXPROCS(0)) +} + // ----------------------------------------------------------------------------- const ( @@ -597,7 +604,7 @@ func Build(inv Invocation) ([]Package, error) { } progSSA := ssa.NewProgram(initial[0].Fset, buildMode) patches := make(cl.Patches, len(altPkgPaths)) - altSSAPkgs(progSSA, patches, altPkgs[1:], conf, verbose) + altEntries := registerAltSSAPkgs(progSSA, patches, altPkgs[1:], conf, verbose) output := conf.OutFile != "" ctx := &context{conf: cfg, progSSA: progSSA, prog: prog, dedup: dedup, @@ -619,14 +626,15 @@ func Build(inv Invocation) ([]Package, error) { // default runtime globals must be registered before packages are built addGlobalString(conf, "runtime.defaultGOROOT="+runtime.GOROOT(), nil) addGlobalString(conf, "runtime.buildVersion="+runtime.Version(), nil) - pkgs, err := buildSSAPkgs(ctx, initial, verbose) + pkgs, pkgEntries, err := registerSSAPkgs(ctx, initial, verbose) if err != nil { return nil, err } - depPkgs, err := buildSSAPkgs(ctx, altPkgs, verbose) + depPkgs, depEntries, err := registerSSAPkgs(ctx, altPkgs, verbose) if err != nil { return nil, err } + buildSSAPkgs(ctx, append(append(altEntries, pkgEntries...), depEntries...)) allPkgs := append([]*aPackage{}, pkgs...) allPkgs = append(allPkgs, depPkgs...) @@ -859,6 +867,7 @@ type context struct { callerTracking *cl.CallerTracking built map[string]none fingerprinting map[string]bool + cacheDisabled map[string]none initial []*packages.Package pkgs map[*packages.Package]Package // cache for lookup pkgByID map[string]Package // cache for lookup by pkg.ID @@ -969,106 +978,34 @@ func normalizeToArchive(ctx *context, aPkg *aPackage, verbose bool) error { } func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, error) { - built := ctx.built - // Split packages into runtime tree vs others so we can defer runtime build. - var runtimePkgs []*aPackage - var normalPkgs []*aPackage + var runtimePkgs []packageBuildSpec + var normalPkgs []packageBuildSpec for _, p := range pkgs { - if isRuntimePkg(p.PkgPath) { - runtimePkgs = append(runtimePkgs, p) + spec := newPackageBuildSpec(p) + if spec.runtime { + runtimePkgs = append(runtimePkgs, spec) } else { - normalPkgs = append(normalPkgs, p) + normalPkgs = append(normalPkgs, spec) } } var needRuntime, needPyInit bool - buildOne := func(aPkg *aPackage) error { - pkg := aPkg.Package - if _, ok := built[pkg.ID]; ok { - // Already built, skip but keep ExportFile for linking - return nil - } - built[pkg.ID] = none{} - - switch kind, param := cl.PkgKindOf(pkg.Types); kind { - case cl.PkgDeclOnly: - pkg.ExportFile = "" - case cl.PkgLinkIR, cl.PkgLinkExtern, cl.PkgPyModule: - if len(pkg.GoFiles) > 0 { - if err := ctx.collectFingerprint(aPkg); err != nil { - return err - } - ctx.tryLoadFromCache(aPkg) - if verbose { - if aPkg.CacheHit { - fmt.Fprintf(os.Stderr, "CACHE HIT: %s\n", pkg.PkgPath) - } else { - fmt.Fprintf(os.Stderr, "CACHE MISS: %s\n", pkg.PkgPath) - } - } - if err := buildPkg(ctx, aPkg, verbose); err != nil { - return err - } - if !aPkg.CacheHit { - if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { - return err - } - if kind == cl.PkgLinkExtern { - appendExternalLinkArgs(ctx, aPkg, param) - } - if err := ctx.saveToCache(aPkg); err != nil && verbose { - fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", pkg.PkgPath, err) - } - } - } else { - pkg.ExportFile = "" - if kind == cl.PkgLinkExtern { - appendExternalLinkArgs(ctx, aPkg, param) - } - } - default: - if err := ctx.collectFingerprint(aPkg); err != nil { - return err - } - ctx.tryLoadFromCache(aPkg) - if verbose { - if aPkg.CacheHit { - fmt.Fprintf(os.Stderr, "CACHE HIT: %s\n", pkg.PkgPath) - } else { - fmt.Fprintf(os.Stderr, "CACHE MISS: %s\n", pkg.PkgPath) - } - } - if err := buildPkg(ctx, aPkg, verbose); err != nil { - return err - } - aPkg.setNeedRuntimeOrPyInit(aPkg.LPkg.NeedRuntime, aPkg.LPkg.NeedPyInit) - needRuntime = needRuntime || aPkg.NeedRt - needPyInit = needPyInit || aPkg.NeedPyInit - if !aPkg.CacheHit { - if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { - return err - } - if err := ctx.saveToCache(aPkg); err != nil && verbose { - fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", pkg.PkgPath, err) - } - } - } - return nil - } - // Build non-runtime packages first, so we know whether runtime is actually needed. - for _, p := range normalPkgs { - if err := buildOne(p); err != nil { + for _, spec := range normalPkgs { + result, err := buildOnePackage(ctx, spec, verbose) + if err != nil { return nil, err } + needRuntime = needRuntime || result.needRuntime + needPyInit = needPyInit || result.needPyInit } // Only build runtime packages when required (or host build with empty Target). if needRuntime || needPyInit || ctx.buildConf.Target == "" { - for _, p := range runtimePkgs { - if err := buildOne(p); err != nil { + for _, spec := range runtimePkgs { + if _, err := buildOnePackage(ctx, spec, verbose); err != nil { return nil, err } } @@ -1077,6 +1014,85 @@ func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, er return pkgs, nil } +// buildOnePackage is the serial package pipeline. Its explicit stages are the +// contract used by later package workers; this commit deliberately preserves +// serial LLVM execution. +func buildOnePackage(ctx *context, spec packageBuildSpec, verbose bool) (packageBuildResult, error) { + skip, err := preflightPackageBuild(ctx, spec, verbose) + if err != nil || skip { + return packageBuildResultFor(spec), err + } + if err := executePackageBuild(ctx, spec, verbose); err != nil { + return packageBuildResultFor(spec), err + } + return finalizePackageBuild(ctx, spec, verbose) +} + +// preflightPackageBuild performs classification, fingerprinting, and cache +// lookup without creating or transforming an LLVM module. +func preflightPackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (skip bool, err error) { + aPkg := spec.pkg + pkg := aPkg.Package + if _, ok := ctx.built[pkg.ID]; ok { + return true, nil + } + ctx.built[pkg.ID] = none{} + if spec.isDeclOnly() { + pkg.ExportFile = "" + return true, nil + } + if spec.isLinkOnly() && !spec.hasSource() { + pkg.ExportFile = "" + if spec.kind == cl.PkgLinkExtern { + appendExternalLinkArgs(ctx, aPkg, spec.kindParam) + } + return true, nil + } + if err := ctx.collectFingerprint(aPkg); err != nil { + return false, err + } + ctx.tryLoadFromCache(aPkg) + if verbose { + status := "MISS" + if aPkg.CacheHit { + status = "HIT" + } + fmt.Fprintf(os.Stderr, "CACHE %s: %s\n", status, pkg.PkgPath) + } + return false, nil +} + +// executePackageBuild creates the package module and runs its LLVM backend. +func executePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) error { + aPkg := spec.pkg + if err := buildPkg(ctx, aPkg, verbose); err != nil { + return err + } + if spec.needsRuntimeSignals() { + aPkg.setNeedRuntimeOrPyInit(aPkg.LPkg.NeedRuntime, aPkg.LPkg.NeedPyInit) + } + return nil +} + +// finalizePackageBuild publishes the archive and cache metadata. Cache hits +// already carry both and therefore require no publication. +func finalizePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (packageBuildResult, error) { + aPkg := spec.pkg + if aPkg.CacheHit { + return packageBuildResultFor(spec), nil + } + if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { + return packageBuildResultFor(spec), err + } + if spec.kind == cl.PkgLinkExtern { + appendExternalLinkArgs(ctx, aPkg, spec.kindParam) + } + if err := ctx.saveToCache(aPkg); err != nil && verbose { + fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", aPkg.PkgPath, err) + } + return packageBuildResultFor(spec), nil +} + func appendExternalLinkArgs(ctx *context, aPkg *aPackage, spec string) { // need to be linked with external library // format: ';' separated alternative link methods. e.g. @@ -1790,6 +1806,15 @@ func is32Bits(goarch string) bool { } func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { + externs, err := preparePackageModule(ctx, aPkg, verbose) + if err != nil || aPkg.CacheHit || aPkg.LPkg == nil { + return err + } + return compilePackageModule(ctx, aPkg, externs, verbose) +} + +// preparePackageModule runs the frontend and creates the package LLVM module. +func preparePackageModule(ctx *context, aPkg *aPackage, verbose bool) ([]string, error) { pkg := aPkg.Package pkgPath := pkg.PkgPath if debugBuild || verbose { @@ -1799,7 +1824,7 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { } if llruntime.SkipToBuild(pkgPath) { pkg.ExportFile = "" - return nil + return nil, nil } var syntax = pkg.Syntax if altPkg := aPkg.AltPkg; altPkg != nil { @@ -1812,7 +1837,7 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { } embedMap, err := goembed.LoadDirectives(ctx.conf.Fset, syntax) if err != nil { - return fmt.Errorf("load go:embed directives for %s failed: %w", pkgPath, err) + return nil, fmt.Errorf("load go:embed directives for %s failed: %w", pkgPath, err) } ret, externs, err := cl.NewPackageExWithEmbedMetaOptions( ctx.prog, ctx.callerTracking, ctx.patches, aPkg.rewriteVars, @@ -1829,8 +1854,16 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { // If cache hit, we only needed to register types - skip compilation if aPkg.CacheHit { - return nil + return nil, nil } + return externs, nil +} + +// compilePackageModule applies LLVM transforms and emits package objects. +func compilePackageModule(ctx *context, aPkg *aPackage, externs []string, verbose bool) error { + pkg := aPkg.Package + pkgPath := pkg.PkgPath + ret := aPkg.LPkg ctx.cTransformer.SetSkipFuncs(cabiSkipFuncsForPlan9Asm(ctx, pkgPath, ret.Module())) llabi.LowerLargeAggregates(ctx.prog.TargetData(), ret.Module()) @@ -1844,7 +1877,7 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { mod.SetTarget(ctx.prog.Target().Spec().Triple) pbo := gllvm.NewPassBuilderOptions() defer pbo.Dispose() - if err = gllvm.VerifyModule(mod, gllvm.ReturnStatusAction); err != nil { + if err := gllvm.VerifyModule(mod, gllvm.ReturnStatusAction); err != nil { return fmt.Errorf("verify LLVM module for %v failed: %w", pkgPath, err) } if err := mod.RunPasses(llvmPassPipeline(ctx.buildConf.OptLevel, ctx.buildConf.ltoMode()), ctx.prog.TargetMachine(), pbo); err != nil { @@ -2129,13 +2162,21 @@ func prepareLocalVariables(prog llssa.Program, groups ...[]*packages.Package) er return nil } -func altSSAPkgs(prog *ssa.Program, patches cl.Patches, alts []*packages.Package, conf *Config, verbose bool) { +type ssaBuildEntry struct { + pkg *ssa.Package + syntax []*ast.File + fixOrder bool +} + +func registerAltSSAPkgs(prog *ssa.Program, patches cl.Patches, alts []*packages.Package, conf *Config, verbose bool) []ssaBuildEntry { + var entries []ssaBuildEntry packages.Visit(alts, nil, func(p *packages.Package) { if typs := p.Types; typs != nil && !p.IllTyped { if debugBuild || verbose { log.Println("==> BuildSSA", p.ID) } pkgSSA := prog.CreatePackage(typs, p.Syntax, p.TypesInfo, true) + entries = append(entries, ssaBuildEntry{pkg: pkgSSA, syntax: p.Syntax}) if strings.HasPrefix(p.ID, altPkgPathPrefix) { path := p.ID[len(altPkgPathPrefix):] // Even if an alt package exists and is pulled in as a dependency of other @@ -2151,7 +2192,7 @@ func altSSAPkgs(prog *ssa.Program, patches cl.Patches, alts []*packages.Package, } } }) - prog.Build() + return entries } type aPackage struct { @@ -2177,9 +2218,10 @@ type aPackage struct { type Package = *aPackage -func buildSSAPkgs(ctx *context, initial []*packages.Package, verbose bool) ([]*aPackage, error) { +func registerSSAPkgs(ctx *context, initial []*packages.Package, verbose bool) ([]*aPackage, []ssaBuildEntry, error) { prog := ctx.progSSA var all []*aPackage + var entries []ssaBuildEntry var errs []*packages.Package packages.Visit(initial, nil, func(p *packages.Package) { if p.Types != nil && !p.IllTyped { @@ -2189,7 +2231,10 @@ func buildSSAPkgs(ctx *context, initial []*packages.Package, verbose bool) ([]*a return } var altPkg *packages.Cached - var ssaPkg = createSSAPkg(ctx, prog, p, verbose) + ssaPkg, created := createSSAPkg(ctx, prog, p, verbose) + if created { + entries = append(entries, ssaBuildEntry{pkg: ssaPkg, syntax: p.Syntax, fixOrder: true}) + } if ctx.hasAltPkg(pkgPath) { if altPkg = ctx.dedup.Check(altPkgPathPrefix + pkgPath); altPkg == nil { return @@ -2221,9 +2266,51 @@ func buildSSAPkgs(ctx *context, initial []*packages.Package, verbose bool) ([]*a } fmt.Fprintln(os.Stderr, "cannot build SSA for package", errPkg) } - return nil, fmt.Errorf("cannot build SSA for packages") + return nil, nil, fmt.Errorf("cannot build SSA for packages") + } + return all, entries, nil +} + +// buildSSAPkgs builds registered packages with the requested bound, then +// performs ordering repair serially because it mutates instruction slices. +func buildSSAPkgs(ctx *context, entries []ssaBuildEntry) { + if len(entries) == 0 { + return + } + unique := make([]ssaBuildEntry, 0, len(entries)) + index := make(map[*ssa.Package]int, len(entries)) + for _, entry := range entries { + if entry.pkg == nil { + continue + } + if i, ok := index[entry.pkg]; ok { + unique[i].fixOrder = unique[i].fixOrder || entry.fixOrder + continue + } + index[entry.pkg] = len(unique) + unique = append(unique, entry) + } + jobs := make(chan ssaBuildEntry, len(unique)) + var wg sync.WaitGroup + for range min(ctx.buildConf.parallelism(), len(unique)) { + wg.Add(1) + go func() { + defer wg.Done() + for entry := range jobs { + entry.pkg.Build() + } + }() + } + for _, entry := range unique { + jobs <- entry + } + close(jobs) + wg.Wait() + for _, entry := range unique { + if entry.fixOrder { + fixSSAOrder(entry.pkg, entry.syntax) + } } - return all, nil } func formatPackageError(err packages.Error, noColumn bool) string { @@ -2371,7 +2458,7 @@ func applyPatches(ctx *context, p *packages.Package, verbose bool) { } } -func createSSAPkg(ctx *context, prog *ssa.Program, p *packages.Package, verbose bool) *ssa.Package { +func createSSAPkg(ctx *context, prog *ssa.Program, p *packages.Package, verbose bool) (*ssa.Package, bool) { pkgSSA := prog.ImportedPackage(p.ID) if pkgSSA == nil { if debugBuild || verbose { @@ -2379,11 +2466,9 @@ func createSSAPkg(ctx *context, prog *ssa.Program, p *packages.Package, verbose } applyPatches(ctx, p, verbose) pkgSSA = prog.CreatePackage(p.Types, p.Syntax, p.TypesInfo, true) - pkgSSA.Build() // TODO(xsw): build concurrently - // Apply local SSA fixups once when package SSA is first built. - fixSSAOrder(pkgSSA, p.Syntax) + return pkgSSA, true } - return pkgSSA + return pkgSSA, false } /* diff --git a/internal/build/collect.go b/internal/build/collect.go index 66da887c21..d8dcee7b1f 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -23,7 +23,6 @@ import ( "os/exec" "path/filepath" "runtime" - "sort" "strconv" "strings" @@ -42,7 +41,11 @@ func (c *context) collectFingerprint(pkg *aPackage) error { c.fingerprinting = make(map[string]bool) } if c.fingerprinting[pkg.ID] { - return fmt.Errorf("fingerprint cycle detected for %s", pkg.ID) + // Alternate runtime replacements can close a cycle after the complete + // SSA graph is registered. Such a cycle has no stable per-package key, + // so rebuild its members instead of reporting a false cache hit. + c.disablePackageCache(c.fingerprinting) + return nil } c.fingerprinting[pkg.ID] = true defer delete(c.fingerprinting, pkg.ID) @@ -70,6 +73,20 @@ func (c *context) collectFingerprint(pkg *aPackage) error { return nil } +func (c *context) disablePackageCache(pkgs map[string]bool) { + if c.cacheDisabled == nil { + c.cacheDisabled = make(map[string]none, len(pkgs)) + } + for id := range pkgs { + c.cacheDisabled[id] = none{} + } +} + +func (c *context) packageCacheDisabled(id string) bool { + _, disabled := c.cacheDisabled[id] + return disabled +} + // collectEnvInputs collects environment-related inputs. func (c *context) collectEnvInputs(m *manifestBuilder) { m.env.Goos = c.buildConf.Goos @@ -197,21 +214,7 @@ func (c *context) collectPackageInputs(m *manifestBuilder, pkg *aPackage) error // collectDependencyInputs adds dependency fingerprints/versions into manifest. func (c *context) collectDependencyInputs(m *manifestBuilder, pkg *aPackage) error { - if len(pkg.Imports) == 0 { - return nil - } - - deps := make([]*packages.Package, 0, len(pkg.Imports)) - for _, dep := range pkg.Imports { - if dep == nil || dep.ID == pkg.ID { - continue - } - deps = append(deps, dep) - } - - sort.Slice(deps, func(i, j int) bool { return deps[i].ID < deps[j].ID }) - - for _, dep := range deps { + for _, dep := range effectiveDependencies(pkg) { depEntry, err := c.dependencyFingerprint(dep) if err != nil { return err @@ -328,6 +331,9 @@ func (c *context) tryLoadFromCache(pkg *aPackage) bool { if !cacheEnabled() { return false } + if c.packageCacheDisabled(pkg.ID) { + return false + } // Main packages are intentionally not written to the build cache because // each executable's entry module is linked against the current main archive. @@ -453,6 +459,9 @@ func (c *context) saveToCache(pkg *aPackage) error { if !cacheEnabled() { return nil } + if c.packageCacheDisabled(pkg.ID) { + return nil + } if pkg.Fingerprint == "" || pkg.Manifest == "" { return nil diff --git a/internal/build/dependencies.go b/internal/build/dependencies.go new file mode 100644 index 0000000000..39f5233217 --- /dev/null +++ b/internal/build/dependencies.go @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "sort" + + "github.com/goplus/llgo/internal/packages" +) + +// effectiveDependencies returns every package whose source contributes to an +// aPackage. Alternate packages may add imports absent from the original graph. +func effectiveDependencies(pkg *aPackage) []*packages.Package { + if pkg == nil || pkg.Package == nil { + return nil + } + deps := make(map[string]*packages.Package) + add := func(imports map[string]*packages.Package) { + for _, dep := range imports { + if dep == nil || dep.ID == pkg.ID || (pkg.AltPkg != nil && dep.ID == pkg.AltPkg.ID) { + continue + } + deps[dep.ID] = dep + } + } + add(pkg.Imports) + if pkg.AltPkg != nil { + add(pkg.AltPkg.Imports) + } + ret := make([]*packages.Package, 0, len(deps)) + for _, dep := range deps { + ret = append(ret, dep) + } + sort.Slice(ret, func(i, j int) bool { return ret[i].ID < ret[j].ID }) + return ret +} diff --git a/internal/build/dependencies_test.go b/internal/build/dependencies_test.go new file mode 100644 index 0000000000..7f083eb13d --- /dev/null +++ b/internal/build/dependencies_test.go @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "reflect" + "testing" + + "github.com/goplus/llgo/internal/packages" +) + +func TestEffectiveDependenciesIncludesAlternateImports(t *testing.T) { + base := &packages.Package{ID: "base"} + shared := &packages.Package{ID: "shared"} + altOnly := &packages.Package{ID: "alt-only"} + alt := &packages.Package{ + ID: "alt", + Imports: map[string]*packages.Package{"shared": shared, "alt-only": altOnly}, + } + pkg := &aPackage{ + Package: &packages.Package{ID: "pkg", Imports: map[string]*packages.Package{"base": base, "shared": shared}}, + AltPkg: &packages.Cached{Package: alt}, + } + deps := effectiveDependencies(pkg) + got := make([]string, len(deps)) + for i, dep := range deps { + got[i] = dep.ID + } + if want := []string{"alt-only", "base", "shared"}; !reflect.DeepEqual(got, want) { + t.Fatalf("effectiveDependencies = %v, want %v", got, want) + } +} diff --git a/internal/build/package_build.go b/internal/build/package_build.go new file mode 100644 index 0000000000..a5ccc0ab38 --- /dev/null +++ b/internal/build/package_build.go @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import "github.com/goplus/llgo/cl" + +type packageBuildSpec struct { + pkg *aPackage + kind int + kindParam string + runtime bool +} + +func newPackageBuildSpec(pkg *aPackage) packageBuildSpec { + kind, kindParam := cl.PkgKindOf(pkg.Types) + return packageBuildSpec{ + pkg: pkg, + kind: kind, + kindParam: kindParam, + runtime: isRuntimePkg(pkg.PkgPath), + } +} + +func (s packageBuildSpec) isDeclOnly() bool { + return s.kind == cl.PkgDeclOnly +} + +func (s packageBuildSpec) isLinkOnly() bool { + return s.kind == cl.PkgLinkIR || s.kind == cl.PkgLinkExtern || s.kind == cl.PkgPyModule +} + +func (s packageBuildSpec) hasSource() bool { + return len(s.pkg.GoFiles) > 0 +} + +func (s packageBuildSpec) needsRuntimeSignals() bool { + return !s.isLinkOnly() && !s.isDeclOnly() +} + +type packageBuildResult struct { + needRuntime bool + needPyInit bool +} + +func packageBuildResultFor(spec packageBuildSpec) packageBuildResult { + return packageBuildResult{ + needRuntime: spec.pkg.NeedRt, + needPyInit: spec.pkg.NeedPyInit, + } +} diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go new file mode 100644 index 0000000000..c2c160ceeb --- /dev/null +++ b/internal/build/package_build_test.go @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "go/ast" + "go/parser" + "go/token" + "go/types" + "path/filepath" + "strings" + "testing" + + "github.com/goplus/llgo/internal/env" + "github.com/goplus/llgo/internal/packages" +) + +func TestPackageBuildSpecAndResult(t *testing.T) { + pkg := &aPackage{Package: &packages.Package{ + PkgPath: "example.com/p", + GoFiles: []string{"p.go"}, + Types: types.NewPackage("example.com/p", "p"), + }, NeedRt: true, NeedPyInit: true} + spec := newPackageBuildSpec(pkg) + if spec.isDeclOnly() || spec.isLinkOnly() || !spec.hasSource() || spec.runtime || !spec.needsRuntimeSignals() { + t.Fatalf("unexpected normal package spec: %+v", spec) + } + result := packageBuildResultFor(spec) + if !result.needRuntime || !result.needPyInit { + t.Fatalf("unexpected package result: %+v", result) + } +} + +func TestPackageBuildSpecSpecialKinds(t *testing.T) { + decl := newPackageBuildSpec(&aPackage{Package: &packages.Package{ + PkgPath: "unsafe", + Types: types.Unsafe, + }}) + if !decl.isDeclOnly() || decl.needsRuntimeSignals() { + t.Fatalf("unexpected declaration-only spec: %+v", decl) + } + runtime := newPackageBuildSpec(&aPackage{Package: &packages.Package{ + PkgPath: env.LLGoRuntimePkg, + Types: types.NewPackage(env.LLGoRuntimePkg, "runtime"), + }}) + if !runtime.runtime { + t.Fatalf("runtime package was not marked runtime: %+v", runtime) + } +} + +func TestConfigParallelism(t *testing.T) { + if got := (&Config{BuildParallelism: 3}).parallelism(); got != 3 { + t.Fatalf("parallelism = %d, want 3", got) + } + if got := (&Config{}).parallelism(); got < 1 { + t.Fatalf("default parallelism = %d, want positive value", got) + } +} + +func TestPreparePackageModuleReturnsEmbedError(t *testing.T) { + fset := token.NewFileSet() + filename := filepath.Join(t.TempDir(), "p.go") + file, err := parser.ParseFile(fset, filename, `package p + +//go:embed missing.txt +var content string +`, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + pkg := &aPackage{Package: &packages.Package{ + ID: "example.com/p", + PkgPath: "example.com/p", + Syntax: []*ast.File{file}, + }} + ctx := &context{ + conf: &packages.Config{Fset: fset}, + buildConf: &Config{}, + } + externs, err := preparePackageModule(ctx, pkg, true) + if err == nil || !strings.Contains(err.Error(), "only allowed in Go files that import") { + t.Fatalf("preparePackageModule embed error = %v", err) + } + if externs != nil { + t.Fatalf("preparePackageModule externs = %v, want nil", externs) + } +} From 36de60c18a87b9b2a80447f7ee3fad08659e4c3a Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 00:39:38 +0800 Subject: [PATCH 02/14] test: cover package build stage boundaries --- internal/build/collect_test.go | 17 ++++++++ internal/build/package_build_test.go | 58 ++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/internal/build/collect_test.go b/internal/build/collect_test.go index d6a8c585d5..5b6294f1f4 100644 --- a/internal/build/collect_test.go +++ b/internal/build/collect_test.go @@ -135,6 +135,23 @@ func TestCollectFingerprintDeterminism(t *testing.T) { } } +func TestDisablePackageCache(t *testing.T) { + ctx := &context{} + ctx.disablePackageCache(map[string]bool{ + "example.com/a": true, + "example.com/b": false, + }) + + for _, id := range []string{"example.com/a", "example.com/b"} { + if !ctx.packageCacheDisabled(id) { + t.Fatalf("package cache for %q remains enabled", id) + } + } + if ctx.packageCacheDisabled("example.com/c") { + t.Fatal("unlisted package cache was disabled") + } +} + func TestCollectFingerprintIncludesEmitDWARF(t *testing.T) { td := t.TempDir() goFile := filepath.Join(td, "main.go") diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go index c2c160ceeb..84988cd718 100644 --- a/internal/build/package_build_test.go +++ b/internal/build/package_build_test.go @@ -99,3 +99,61 @@ var content string t.Fatalf("preparePackageModule externs = %v, want nil", externs) } } + +func TestBuildOnePackageSkipsAlreadyBuiltPackage(t *testing.T) { + pkg := &aPackage{Package: &packages.Package{ + ID: "example.com/already-built", + PkgPath: "example.com/already-built", + GoFiles: []string{"already.go"}, + Types: types.NewPackage("example.com/already-built", "already"), + }, NeedRt: true} + ctx := &context{built: map[string]none{pkg.ID: {}}} + + result, err := buildOnePackage(ctx, newPackageBuildSpec(pkg), false) + if err != nil { + t.Fatal(err) + } + if !result.needRuntime { + t.Fatalf("build result = %+v, want runtime requirement preserved", result) + } +} + +func TestPreflightPackageBuildSkipsDeclarationOnlyPackage(t *testing.T) { + pkg := &aPackage{Package: &packages.Package{ + ID: "unsafe", + PkgPath: "unsafe", + Types: types.Unsafe, + ExportFile: "stale.a", + }} + ctx := &context{built: make(map[string]none)} + + skip, err := preflightPackageBuild(ctx, newPackageBuildSpec(pkg), false) + if err != nil { + t.Fatal(err) + } + if !skip { + t.Fatal("declaration-only package was not skipped") + } + if pkg.ExportFile != "" { + t.Fatalf("ExportFile = %q, want empty", pkg.ExportFile) + } + if _, ok := ctx.built[pkg.ID]; !ok { + t.Fatal("declaration-only package was not recorded as built") + } +} + +func TestFinalizePackageBuildReturnsCachedResult(t *testing.T) { + pkg := &aPackage{Package: &packages.Package{ + ID: "example.com/cached", + PkgPath: "example.com/cached", + Types: types.NewPackage("example.com/cached", "cached"), + }, CacheHit: true, NeedPyInit: true} + + result, err := finalizePackageBuild(&context{}, newPackageBuildSpec(pkg), false) + if err != nil { + t.Fatal(err) + } + if !result.needPyInit { + t.Fatalf("build result = %+v, want Python initialization requirement preserved", result) + } +} From 0be401463bbf0caa62f74a10fffee0aca5320674 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 01:20:51 +0800 Subject: [PATCH 03/14] test: cover empty package dependencies --- internal/build/dependencies_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/build/dependencies_test.go b/internal/build/dependencies_test.go index 7f083eb13d..78fa352e56 100644 --- a/internal/build/dependencies_test.go +++ b/internal/build/dependencies_test.go @@ -44,3 +44,9 @@ func TestEffectiveDependenciesIncludesAlternateImports(t *testing.T) { t.Fatalf("effectiveDependencies = %v, want %v", got, want) } } + +func TestEffectiveDependenciesHandlesNilPackage(t *testing.T) { + if deps := effectiveDependencies(nil); deps != nil { + t.Fatalf("effectiveDependencies(nil) = %v, want nil", deps) + } +} From d84ca29700cd063919cc7c65e0c677164eaf0671 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 04:13:51 +0800 Subject: [PATCH 04/14] test: cover cgo command environment parsing --- internal/build/cgo_test.go | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/internal/build/cgo_test.go b/internal/build/cgo_test.go index 91fba3f534..8ccbb79d0d 100644 --- a/internal/build/cgo_test.go +++ b/internal/build/cgo_test.go @@ -77,6 +77,52 @@ func TestParseCgoDeclFlags(t *testing.T) { } } +func TestParseCgoDeclWithCommandEnvBranches(t *testing.T) { + commands := commandEnv{dir: t.TempDir(), environ: []string{"CGO_TEST_MARKER=1"}} + tests := []struct { + name string + line string + want []cgoDecl + wantErr string + }{ + { + name: "LDFLAGS with tag", + line: "#cgo darwin LDFLAGS: -framework CoreFoundation -lz", + want: []cgoDecl{{tag: "darwin", ldflags: []string{"-framework CoreFoundation", "-lz"}}}, + }, + {name: "missing colon", line: "#cgo CFLAGS -I/missing", wantErr: "invalid cgo format"}, + {name: "missing directive", line: "CFLAGS: -I/missing", wantErr: "invalid cgo directive"}, + {name: "unsupported flag", line: "#cgo FOOFLAGS: -unsupported", wantErr: "unsupported cgo flag type"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseCgoDeclWithCommandEnv(commands, tt.line) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("parseCgoDeclWithCommandEnv(%q) error = %v, want %q", tt.line, err, tt.wantErr) + } + return + } + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("parseCgoDeclWithCommandEnv(%q) = %#v, want %#v", tt.line, got, tt.want) + } + }) + } +} + +func TestParseCgoPreambleDelegatesToCommandEnvParser(t *testing.T) { + preamble, decls, err := parseCgoPreamble(token.Position{Filename: "request.go", Line: 7}, "#cgo CFLAGS: -I/request/include") + if err != nil { + t.Fatal(err) + } + if preamble.goFile != "request.go" || preamble.src == "" || !reflect.DeepEqual(decls, []cgoDecl{{cflags: []string{"-I/request/include"}}}) { + t.Fatalf("parseCgoPreamble() = %#v, %#v", preamble, decls) + } +} + func TestCollectCgoSymbolsStripsPackagePrefix(t *testing.T) { externs := []string{ "command-line-arguments._cgo_96608f8de8c8_Cfunc_fputs", From 6d29a370e5847a56b4ffdc422424f22d3abe7f27 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 05:10:44 +0800 Subject: [PATCH 05/14] test: cover cgo pkg-config invocation --- internal/build/cgo_test.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/internal/build/cgo_test.go b/internal/build/cgo_test.go index 8ccbb79d0d..9127c6f92a 100644 --- a/internal/build/cgo_test.go +++ b/internal/build/cgo_test.go @@ -113,6 +113,40 @@ func TestParseCgoDeclWithCommandEnvBranches(t *testing.T) { } } +func TestParseCgoDeclWithCommandEnvPkgConfig(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("test helper uses a shell script") + } + + dir := t.TempDir() + tool := filepath.Join(dir, "pkg-config") + script := `#!/bin/sh +if [ "$1" = "--libs" ]; then + printf '%s\n' '-L/request/lib -lrequest' + exit 0 +fi +printf '%s\n' '-I/request/include -DREQUEST="request value"' +` + if err := os.WriteFile(tool, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir) + + commands := commandEnv{dir: dir, environ: []string{"PATH=" + dir}} + got, err := parseCgoDeclWithCommandEnv(commands, "#cgo linux pkg-config: request") + if err != nil { + t.Fatal(err) + } + want := []cgoDecl{{ + tag: "linux", + cflags: []string{"-I/request/include", `-DREQUEST="request value"`}, + ldflags: []string{"-L/request/lib", "-lrequest"}, + }} + if !reflect.DeepEqual(got, want) { + t.Fatalf("parseCgoDeclWithCommandEnv(pkg-config) = %#v, want %#v", got, want) + } +} + func TestParseCgoPreambleDelegatesToCommandEnvParser(t *testing.T) { preamble, decls, err := parseCgoPreamble(token.Position{Filename: "request.go", Line: 7}, "#cgo CFLAGS: -I/request/include") if err != nil { From d24eb12a1dfc8389023036c2b93d5e5df547d0c5 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 08:07:45 +0800 Subject: [PATCH 06/14] test: cover cgo pkg-config failures --- internal/build/cgo_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/build/cgo_test.go b/internal/build/cgo_test.go index 9127c6f92a..926b30b518 100644 --- a/internal/build/cgo_test.go +++ b/internal/build/cgo_test.go @@ -121,6 +121,9 @@ func TestParseCgoDeclWithCommandEnvPkgConfig(t *testing.T) { dir := t.TempDir() tool := filepath.Join(dir, "pkg-config") script := `#!/bin/sh +if [ "$PKG_CONFIG_TEST_FAIL" = "$1" ]; then + exit 1 +fi if [ "$1" = "--libs" ]; then printf '%s\n' '-L/request/lib -lrequest' exit 0 @@ -145,6 +148,15 @@ printf '%s\n' '-I/request/include -DREQUEST="request value"' if !reflect.DeepEqual(got, want) { t.Fatalf("parseCgoDeclWithCommandEnv(pkg-config) = %#v, want %#v", got, want) } + + for _, arg := range []string{"--libs", "--cflags"} { + t.Run("failed "+arg, func(t *testing.T) { + commands := commandEnv{dir: dir, environ: []string{"PATH=" + dir, "PKG_CONFIG_TEST_FAIL=" + arg}} + if _, err := parseCgoDeclWithCommandEnv(commands, "#cgo pkg-config: request"); err == nil || !strings.Contains(err.Error(), "pkg-config") { + t.Fatalf("parseCgoDeclWithCommandEnv(pkg-config) error = %v, want pkg-config failure", err) + } + }) + } } func TestParseCgoPreambleDelegatesToCommandEnvParser(t *testing.T) { From e67a38418f41833dc1a0bd9afae719f9f3d65f25 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Fri, 31 Jul 2026 16:55:18 +0800 Subject: [PATCH 07/14] build: archive LLVM package objects in memory --- go.mod | 2 + go.sum | 4 +- internal/build/build.go | 58 ++++++++++-- internal/build/package_archive.go | 99 ++++++++++++++++++++ internal/build/package_archive_test.go | 123 +++++++++++++++++++++++++ 5 files changed, 275 insertions(+), 11 deletions(-) create mode 100644 internal/build/package_archive.go create mode 100644 internal/build/package_archive_test.go diff --git a/go.mod b/go.mod index c55f42ffd4..23ba621674 100644 --- a/go.mod +++ b/go.mod @@ -26,3 +26,5 @@ require ( ) replace github.com/goplus/llgo/runtime => ./runtime + +replace github.com/xgo-dev/llvm => github.com/zhouguangyuan0718/go-llvm v0.0.0-20260731084524-d8f8a2aa851e diff --git a/go.sum b/go.sum index db139c2acc..fa2dd992cd 100644 --- a/go.sum +++ b/go.sum @@ -22,10 +22,10 @@ github.com/qiniu/x v1.18.0 h1:iMfc7Gqy1au+akr+Tl5Z40px7TR8VBLLkJsIeajKIbc= github.com/qiniu/x v1.18.0/go.mod h1:Sx3Wy+0GI9OsX4a53mYj6A0o7mHJ94PUvraqGYb4EIs= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/xgo-dev/llvm v0.9.5 h1:RijI/6vGu7DGw5ldlaqvpK2uFcg9OXY9OXB/RmywtEk= -github.com/xgo-dev/llvm v0.9.5/go.mod h1:42vav2/cI5BAIcL543DZSMO9do8/aCK2z7JERH+AE+M= github.com/xgo-dev/plan9asm v0.3.5 h1:886BmpjMK6JfJ03VWA3nPK01jkVZA1a7/mZia3BOsdg= github.com/xgo-dev/plan9asm v0.3.5/go.mod h1:0yM4CCIp2PyT8h+Ro3Ukro3lHL8ji9mzHEv5yfhOckc= +github.com/zhouguangyuan0718/go-llvm v0.0.0-20260731084524-d8f8a2aa851e h1:O9Hz3E3WoZv6XXL4+4Gn1EkaEqN9+/F61WiwOJA5F5c= +github.com/zhouguangyuan0718/go-llvm v0.0.0-20260731084524-d8f8a2aa851e/go.mod h1:42vav2/cI5BAIcL543DZSMO9do8/aCK2z7JERH+AE+M= go.bug.st/serial v1.6.4 h1:7FmqNPgVp3pu2Jz5PoPtbZ9jJO5gnEnZIvnI1lzve8A= go.bug.st/serial v1.6.4/go.mod h1:nofMJxTeNVny/m6+KaafC6vJGj3miwQZ6vW4BZUGJPI= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= diff --git a/internal/build/build.go b/internal/build/build.go index 4e4d78d5bc..44ab45aa75 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -622,6 +622,7 @@ func Build(inv Invocation) ([]Package, error) { cTransformer: cabi.NewTransformer(prog, export.LLVMTarget, export.TargetABI, conf.AbiMode, cabiOptimize), } defer ctx.closePackageMetas() + defer ctx.closePackageArchiveBuffers() // default runtime globals must be registered before packages are built addGlobalString(conf, "runtime.defaultGOROOT="+runtime.GOROOT(), nil) @@ -953,12 +954,13 @@ func (c *context) hasAltPkg(pkgPath string) bool { return hasAltPkgForTarget(c.buildConf, pkgPath) } -// normalizeToArchive creates an archive from object files and sets ArchiveFile. +// normalizeToArchive creates an archive from file and memory members and sets ArchiveFile. // This ensures the link step always consumes .a archives regardless of cache state. func normalizeToArchive(ctx *context, aPkg *aPackage, verbose bool) error { - if len(aPkg.ObjFiles) == 0 { + if len(aPkg.ObjFiles) == 0 && len(aPkg.ObjBuffers) == 0 { return nil } + defer aPkg.disposeArchiveBuffers() archiveFile, err := os.CreateTemp("", "pkg-*.a") if err != nil { @@ -967,7 +969,7 @@ func normalizeToArchive(ctx *context, aPkg *aPackage, verbose bool) error { archiveFile.Close() archivePath := archiveFile.Name() - if err := ctx.createArchiveFile(archivePath, aPkg.ObjFiles, verbose); err != nil { + if err := ctx.createPackageArchiveFile(archivePath, aPkg, verbose); err != nil { os.Remove(archivePath) return fmt.Errorf("create archive for %s: %w", aPkg.PkgPath, err) } @@ -1929,11 +1931,15 @@ func compilePackageModule(ctx *context, aPkg *aPackage, externs []string, verbos aPkg.LinkArgs = append(aPkg.LinkArgs, goCgoLinkArgs(ctx.buildConf.Goos, aPkg.AltPkg.Syntax)...) } if pkg.ExportFile != "" { - exportFile, err := exportObject(ctx, pkg.PkgPath, pkg.ExportFile, ret) + exportFile, exportBuffer, err := exportPackageObject(ctx, pkg.PkgPath, pkg.ExportFile, ret) if err != nil { return fmt.Errorf("export object of %v failed: %v", pkgPath, err) } - aPkg.ObjFiles = append(aPkg.ObjFiles, exportFile) + if exportFile != "" { + aPkg.ObjFiles = append(aPkg.ObjFiles, exportFile) + } else { + aPkg.ObjBuffers = append(aPkg.ObjBuffers, exportBuffer) + } if debugBuild || verbose { fmt.Fprintf(os.Stderr, "==> Export %s: %s\n", aPkg.PkgPath, pkg.ExportFile) } @@ -1954,6 +1960,28 @@ func exportObject(ctx *context, pkgPath string, exportFile string, pkg llssa.Pac return exportObjectWithClang(ctx, pkgPath, exportFile, []byte(pkg.String())) } +func exportPackageObject(ctx *context, pkgPath string, exportFile string, pkg llssa.Package) (string, packageArchiveBuffer, error) { + if !useInMemoryNativeCodegen(ctx) { + path, err := exportObjectWithClang(ctx, pkgPath, exportFile, []byte(pkg.String())) + return path, packageArchiveBuffer{}, err + } + if ctx.buildConf.CheckLLFiles || ctx.buildConf.GenLL { + if err := dumpLLVMIRIfNeeded(ctx, pkgPath, exportFile, pkg.String()); err != nil { + return "", packageArchiveBuffer{}, err + } + } + buf, kind, err := emitObjectToMemoryBuffer(ctx, pkg) + if err != nil { + return "", packageArchiveBuffer{}, err + } + name := filepath.Base(exportFile) + ".o" + if ctx.shouldPrintCommands(false) { + fmt.Fprintf(os.Stderr, "# compiling archive member %s for pkg: %s\n", name, pkgPath) + fmt.Fprintf(os.Stderr, "# using %s\n", kind) + } + return "", packageArchiveBuffer{name: name, buffer: buf}, nil +} + func useInMemoryNativeCodegen(ctx *context) bool { return useInMemoryNativeCodegenConf(ctx.buildConf) } @@ -2010,6 +2038,15 @@ func exportObjectInMemory(ctx *context, pkgPath string, exportFile string, pkg l return "", err } } + buf, kind, err := emitObjectToMemoryBuffer(ctx, pkg) + if err != nil { + return "", err + } + defer buf.Dispose() + return writeObjectBufferToFile(ctx, pkgPath, exportFile, buf, kind) +} + +func emitObjectToMemoryBuffer(ctx *context, pkg llssa.Package) (gllvm.MemoryBuffer, string, error) { ltoMode := ctx.buildConf.ltoMode() var ( buf gllvm.MemoryBuffer @@ -2028,11 +2065,13 @@ func exportObjectInMemory(ctx *context, pkgPath string, exportFile string, pkg l default: buf, err = ctx.prog.TargetMachine().EmitToMemoryBuffer(pkg.Module(), gllvm.ObjectFile) if err != nil { - return "", err + return gllvm.MemoryBuffer{}, "", err } } - defer buf.Dispose() + return buf, kind, nil +} +func writeObjectBufferToFile(ctx *context, pkgPath, exportFile string, buf gllvm.MemoryBuffer, kind string) (string, error) { base := filepath.Base(exportFile) objFile, err := os.CreateTemp("", base+"-*.o") if err != nil { @@ -2205,8 +2244,9 @@ type aPackage struct { NeedPyInit bool LinkArgs []string - ObjFiles []string // object files: .o or .ll (output of compiler, input to archiver) - ArchiveFile string // archive file: .a (output of archiver, used for linking) + ObjFiles []string // file-backed archive members: .o or .ll + ObjBuffers []packageArchiveBuffer // LLVM-produced in-memory archive members + ArchiveFile string // archive file: .a (output of archiver, used for linking) Meta *meta.PackageMeta rewriteVars map[string]string diff --git a/internal/build/package_archive.go b/internal/build/package_archive.go new file mode 100644 index 0000000000..87b732d7f5 --- /dev/null +++ b/internal/build/package_archive.go @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "fmt" + "os" + "path/filepath" + + gllvm "github.com/xgo-dev/llvm" +) + +// packageArchiveBuffer owns an LLVM-produced archive member until package +// publication. +type packageArchiveBuffer struct { + name string + buffer gllvm.MemoryBuffer +} + +func (p *aPackage) disposeArchiveBuffers() { + for i := range p.ObjBuffers { + member := &p.ObjBuffers[i] + if member.buffer.IsNil() { + continue + } + member.buffer.Dispose() + member.buffer = gllvm.MemoryBuffer{} + } + p.ObjBuffers = nil +} + +func (c *context) closePackageArchiveBuffers() { + for _, pkg := range c.pkgs { + pkg.disposeArchiveBuffers() + } +} + +// createPackageArchiveFile writes path-backed auxiliary objects and +// LLVM-produced memory buffers into one archive. LLVM performs archive symbol +// indexing in-process; no temporary object is needed for memory members. +func (c *context) createPackageArchiveFile(archivePath string, pkg *aPackage, verbose bool) error { + if len(pkg.ObjFiles) == 0 && len(pkg.ObjBuffers) == 0 { + return fmt.Errorf("no object files provided for archive %s", archivePath) + } + if len(pkg.ObjBuffers) == 0 { + return c.createArchiveFile(archivePath, pkg.ObjFiles, verbose) + } + + if err := os.MkdirAll(filepath.Dir(archivePath), 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(archivePath), filepath.Base(archivePath)+".tmp-*") + if err != nil { + return err + } + tmpName := tmp.Name() + if err := tmp.Close(); err != nil { + os.Remove(tmpName) + return err + } + if err := os.Remove(tmpName); err != nil { + return err + } + + members := make([]gllvm.ArchiveMember, 0, len(pkg.ObjFiles)+len(pkg.ObjBuffers)) + for _, path := range pkg.ObjFiles { + members = append(members, gllvm.NewArchiveMemberFromFile(path)) + } + for _, member := range pkg.ObjBuffers { + members = append(members, gllvm.NewArchiveMemberFromMemoryBuffer(member.name, member.buffer)) + } + if c.shouldPrintCommands(verbose) { + fmt.Fprintf(os.Stderr, "# llvm archive %s (%d file members, %d memory members)\n", + tmpName, len(pkg.ObjFiles), len(pkg.ObjBuffers)) + } + if err := gllvm.WriteArchive(tmpName, c.targetTriple(), members); err != nil { + os.Remove(tmpName) + return fmt.Errorf("create archive %s: %w", archivePath, err) + } + if err := os.Rename(tmpName, archivePath); err != nil { + os.Remove(tmpName) + return fmt.Errorf("publish archive %s: %w", archivePath, err) + } + return nil +} diff --git a/internal/build/package_archive_test.go b/internal/build/package_archive_test.go new file mode 100644 index 0000000000..9668a9c650 --- /dev/null +++ b/internal/build/package_archive_test.go @@ -0,0 +1,123 @@ +//go:build !llgo +// +build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/goplus/llgo/internal/crosscompile" + "github.com/goplus/llgo/internal/packages" + gllvm "github.com/xgo-dev/llvm" +) + +func TestNormalizeToArchiveUsesMemoryBuffer(t *testing.T) { + llvmCtx := gllvm.NewContext() + defer llvmCtx.Dispose() + mod := llvmCtx.NewModule("archive") + defer mod.Dispose() + mod.SetTarget("x86_64-unknown-linux-gnu") + gllvm.AddFunction(mod, "memory_symbol", gllvm.FunctionType(llvmCtx.Int32Type(), nil, false)) + + memoryBuf := gllvm.WriteBitcodeToMemoryBuffer(mod) + ctx := &context{ + buildConf: &Config{Goos: "linux", Goarch: "amd64"}, + crossCompile: crosscompile.Export{ + LLVMTarget: "x86_64-unknown-linux-gnu", + }, + } + fileBuf := gllvm.WriteBitcodeToMemoryBuffer(mod) + defer fileBuf.Dispose() + filePath := filepath.Join(t.TempDir(), "file-member.bc") + if err := os.WriteFile(filePath, fileBuf.Bytes(), 0o644); err != nil { + memoryBuf.Dispose() + t.Fatal(err) + } + + pkg := &aPackage{ + Package: &packages.Package{PkgPath: "example.com/archive"}, + ObjFiles: []string{filePath}, + ObjBuffers: []packageArchiveBuffer{{ + name: "memory-member.bc", + buffer: memoryBuf, + }}, + } + if err := normalizeToArchive(ctx, pkg, false); err != nil { + t.Fatal(err) + } + defer os.Remove(pkg.ArchiveFile) + + if len(pkg.ObjFiles) != 0 || len(pkg.ObjBuffers) != 0 { + t.Fatalf("package members were not released: files=%v buffers=%v", pkg.ObjFiles, pkg.ObjBuffers) + } + data, err := os.ReadFile(pkg.ArchiveFile) + if err != nil { + t.Fatal(err) + } + if !bytes.HasPrefix(data, []byte("!\n")) { + t.Fatalf("archive has invalid magic: %q", data[:8]) + } + for _, name := range []string{"file-member.bc", "memory-member.bc"} { + if !bytes.Contains(data, []byte(name)) { + t.Errorf("archive does not contain member %q", name) + } + } +} + +func TestNormalizeToArchiveFailsWithoutObjectFallback(t *testing.T) { + llvmCtx := gllvm.NewContext() + defer llvmCtx.Dispose() + mod := llvmCtx.NewModule("archive-error") + defer mod.Dispose() + mod.SetTarget("x86_64-unknown-linux-gnu") + gllvm.AddFunction(mod, "memory_symbol", gllvm.FunctionType(llvmCtx.Int32Type(), nil, false)) + + memoryBuf := gllvm.WriteBitcodeToMemoryBuffer(mod) + ctx := &context{ + buildConf: &Config{Goos: "linux", Goarch: "amd64"}, + crossCompile: crosscompile.Export{ + LLVMTarget: "x86_64-unknown-linux-gnu", + }, + } + pkg := &aPackage{ + Package: &packages.Package{PkgPath: "example.com/archive-error"}, + ObjFiles: []string{filepath.Join(t.TempDir(), "missing.o")}, + ObjBuffers: []packageArchiveBuffer{{ + name: "memory-member.bc", + buffer: memoryBuf, + }}, + } + err := normalizeToArchive(ctx, pkg, false) + if err == nil { + t.Fatal("normalizeToArchive succeeded with a missing member") + } + if !strings.Contains(err.Error(), "missing.o") { + t.Fatalf("normalizeToArchive error = %v, want missing member", err) + } + if len(pkg.ObjBuffers) != 0 { + t.Fatal("memory buffer was retained after archive failure") + } + if pkg.ArchiveFile != "" { + t.Fatalf("ArchiveFile = %q after failure", pkg.ArchiveFile) + } +} From dcf4cca45c13639d9156f7d99eaabce949dd31dc Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 1 Aug 2026 08:06:11 +0800 Subject: [PATCH 08/14] test: cover package stage and archive branches --- internal/build/collect_test.go | 29 ++++++++++++++++ internal/build/package_archive_test.go | 48 +++++++++++++++++++++++++- internal/build/package_build_test.go | 34 ++++++++++++++++++ 3 files changed, 110 insertions(+), 1 deletion(-) diff --git a/internal/build/collect_test.go b/internal/build/collect_test.go index 5b6294f1f4..cc9785518f 100644 --- a/internal/build/collect_test.go +++ b/internal/build/collect_test.go @@ -152,6 +152,35 @@ func TestDisablePackageCache(t *testing.T) { } } +func TestDisabledPackageCacheSkipsLoadAndSave(t *testing.T) { + t.Setenv(llgoBuildCache, "1") + pkg := &aPackage{Package: &packages.Package{ID: "example.com/disabled", PkgPath: "example.com/disabled"}} + ctx := &context{buildConf: &Config{}, cacheDisabled: map[string]none{pkg.ID: {}}} + if ctx.tryLoadFromCache(pkg) { + t.Fatal("tryLoadFromCache loaded a disabled package") + } + if err := ctx.saveToCache(pkg); err != nil { + t.Fatalf("saveToCache disabled package: %v", err) + } + + ctx.cacheDisabled = nil + ctx.buildConf.BuildMode = BuildModeCArchive + if ctx.tryLoadFromCache(pkg) { + t.Fatal("tryLoadFromCache loaded a C archive package") + } +} + +func TestCollectFingerprintDisablesCycles(t *testing.T) { + pkg := &aPackage{Package: &packages.Package{ID: "example.com/cycle", PkgPath: "example.com/cycle"}} + ctx := &context{fingerprinting: map[string]bool{pkg.ID: true}} + if err := ctx.collectFingerprint(pkg); err != nil { + t.Fatal(err) + } + if !ctx.packageCacheDisabled(pkg.ID) { + t.Fatal("fingerprint cycle did not disable package cache") + } +} + func TestCollectFingerprintIncludesEmitDWARF(t *testing.T) { td := t.TempDir() goFile := filepath.Join(td, "main.go") diff --git a/internal/build/package_archive_test.go b/internal/build/package_archive_test.go index 9668a9c650..e0d644a896 100644 --- a/internal/build/package_archive_test.go +++ b/internal/build/package_archive_test.go @@ -62,7 +62,7 @@ func TestNormalizeToArchiveUsesMemoryBuffer(t *testing.T) { buffer: memoryBuf, }}, } - if err := normalizeToArchive(ctx, pkg, false); err != nil { + if err := normalizeToArchive(ctx, pkg, true); err != nil { t.Fatal(err) } defer os.Remove(pkg.ArchiveFile) @@ -84,6 +84,52 @@ func TestNormalizeToArchiveUsesMemoryBuffer(t *testing.T) { } } +func TestPackageArchiveEdgeCases(t *testing.T) { + empty := &aPackage{} + if err := (&context{}).createPackageArchiveFile(filepath.Join(t.TempDir(), "empty.a"), empty, false); err == nil { + t.Fatal("createPackageArchiveFile succeeded without members") + } + fileOnly := &aPackage{ObjFiles: []string{filepath.Join(t.TempDir(), "missing.o")}} + if err := (&context{buildConf: &Config{Goos: "linux", Goarch: "amd64"}}).createPackageArchiveFile(filepath.Join(t.TempDir(), "file-only.a"), fileOnly, false); err == nil { + t.Fatal("file-only archive succeeded with a missing member") + } + empty.ObjBuffers = []packageArchiveBuffer{{}} + empty.disposeArchiveBuffers() + if empty.ObjBuffers != nil { + t.Fatal("disposeArchiveBuffers retained nil members") + } + + llvmCtx := gllvm.NewContext() + defer llvmCtx.Dispose() + mod := llvmCtx.NewModule("archive-errors") + defer mod.Dispose() + mod.SetTarget("x86_64-unknown-linux-gnu") + gllvm.AddFunction(mod, "archive_error_symbol", gllvm.FunctionType(llvmCtx.Int32Type(), nil, false)) + buffer := gllvm.WriteBitcodeToMemoryBuffer(mod) + defer buffer.Dispose() + pkg := &aPackage{ObjBuffers: []packageArchiveBuffer{{name: "member.bc", buffer: buffer}}} + ctx := &context{ + buildConf: &Config{Goos: "linux", Goarch: "amd64"}, + crossCompile: crosscompile.Export{LLVMTarget: "x86_64-unknown-linux-gnu"}, + } + + blocker := filepath.Join(t.TempDir(), "blocker") + if err := os.WriteFile(blocker, []byte("not a directory"), 0o644); err != nil { + t.Fatal(err) + } + if err := ctx.createPackageArchiveFile(filepath.Join(blocker, "archive.a"), pkg, false); err == nil { + t.Fatal("createPackageArchiveFile succeeded below a regular file") + } + + archiveDir := filepath.Join(t.TempDir(), "archive.a") + if err := os.Mkdir(archiveDir, 0o755); err != nil { + t.Fatal(err) + } + if err := ctx.createPackageArchiveFile(archiveDir, pkg, false); err == nil { + t.Fatal("createPackageArchiveFile replaced a directory") + } +} + func TestNormalizeToArchiveFailsWithoutObjectFallback(t *testing.T) { llvmCtx := gllvm.NewContext() defer llvmCtx.Dispose() diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go index 84988cd718..d1ce18f415 100644 --- a/internal/build/package_build_test.go +++ b/internal/build/package_build_test.go @@ -25,8 +25,10 @@ import ( "strings" "testing" + "github.com/goplus/llgo/cl" "github.com/goplus/llgo/internal/env" "github.com/goplus/llgo/internal/packages" + "golang.org/x/tools/go/ssa" ) func TestPackageBuildSpecAndResult(t *testing.T) { @@ -142,6 +144,28 @@ func TestPreflightPackageBuildSkipsDeclarationOnlyPackage(t *testing.T) { } } +func TestPreflightPackageBuildSkipsExternalLinkOnlyPackage(t *testing.T) { + pkg := &aPackage{Package: &packages.Package{ + ID: "example.com/linkonly", + PkgPath: "example.com/linkonly", + Name: "linkonly", + Types: types.NewPackage("example.com/linkonly", "linkonly"), + ExportFile: "stale.a", + }} + ctx := &context{buildConf: &Config{}, built: make(map[string]none)} + spec := packageBuildSpec{pkg: pkg, kind: cl.PkgLinkExtern, kindParam: "-lexample"} + skip, err := preflightPackageBuild(ctx, spec, false) + if err != nil { + t.Fatal(err) + } + if !skip || pkg.ExportFile != "" { + t.Fatalf("external link-only preflight = skip %v, export %q", skip, pkg.ExportFile) + } + if len(pkg.LinkArgs) != 1 || pkg.LinkArgs[0] != "-lexample" { + t.Fatalf("external link args = %q, want [-lexample]", pkg.LinkArgs) + } +} + func TestFinalizePackageBuildReturnsCachedResult(t *testing.T) { pkg := &aPackage{Package: &packages.Package{ ID: "example.com/cached", @@ -157,3 +181,13 @@ func TestFinalizePackageBuildReturnsCachedResult(t *testing.T) { t.Fatalf("build result = %+v, want Python initialization requirement preserved", result) } } + +func TestBuildSSAPkgsEmptyAndNilEntries(t *testing.T) { + ctx := &context{buildConf: &Config{}} + buildSSAPkgs(ctx, nil) + buildSSAPkgs(ctx, []ssaBuildEntry{{}, {fixOrder: true}}) + + prog := ssa.NewProgram(token.NewFileSet(), ssa.SanityCheckFunctions) + pkg := prog.CreatePackage(types.NewPackage("example.com/ssa", "ssa"), nil, nil, true) + buildSSAPkgs(ctx, []ssaBuildEntry{{pkg: pkg}, {pkg: pkg}}) +} From 7b39688faa07f853d5326bbd688f2107c28e4fc7 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sun, 2 Aug 2026 11:19:50 +0800 Subject: [PATCH 09/14] test: cover package pipeline error paths --- internal/build/llvm_emit_test.go | 54 +++++++++++++++++ internal/build/package_archive_test.go | 8 ++- internal/build/package_build_test.go | 81 ++++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 2 deletions(-) diff --git a/internal/build/llvm_emit_test.go b/internal/build/llvm_emit_test.go index 40bfb45bb1..2e30777172 100644 --- a/internal/build/llvm_emit_test.go +++ b/internal/build/llvm_emit_test.go @@ -1,10 +1,15 @@ package build import ( + "os" + "path/filepath" "runtime" + "strings" "testing" + "github.com/goplus/llgo/internal/crosscompile" "github.com/goplus/llgo/internal/lto" + llssa "github.com/goplus/llgo/ssa" ) func TestUseInMemoryNativeCodegenConf(t *testing.T) { @@ -55,3 +60,52 @@ func TestUseInMemoryNativeCodegenConf(t *testing.T) { } }) } + +func TestExportPackageObjectErrors(t *testing.T) { + prog := llssa.NewProgram(nil) + defer prog.Dispose() + pkg := prog.NewPackage("p", "example.com/p") + + t.Run("clang", func(t *testing.T) { + ctx := &context{ + buildConf: &Config{Target: "embedded"}, + crossCompile: crosscompile.Export{ + CC: filepath.Join(t.TempDir(), "missing-clang"), + }, + commands: commandEnv{environ: os.Environ()}, + } + path, member, err := exportPackageObject(ctx, pkg.Path(), "p.o", pkg) + if path != "" { + defer os.Remove(path) + } + if err == nil { + member.buffer.Dispose() + t.Fatal("exportPackageObject succeeded with a missing clang") + } + if !member.buffer.IsNil() { + member.buffer.Dispose() + t.Fatal("clang export returned an in-memory archive member") + } + }) + + t.Run("IR dump", func(t *testing.T) { + ctx := &context{buildConf: &Config{ + Goos: runtime.GOOS, + Goarch: runtime.GOARCH, + CheckLLFiles: true, + }} + exportFile := strings.Repeat("x", 300) + path, member, err := exportPackageObject(ctx, pkg.Path(), exportFile, pkg) + if path != "" { + defer os.Remove(path) + } + if err == nil { + member.buffer.Dispose() + t.Fatal("exportPackageObject succeeded with an overlong IR dump prefix") + } + if !member.buffer.IsNil() { + member.buffer.Dispose() + t.Fatal("failed IR dump returned an in-memory archive member") + } + }) +} diff --git a/internal/build/package_archive_test.go b/internal/build/package_archive_test.go index e0d644a896..477963da86 100644 --- a/internal/build/package_archive_test.go +++ b/internal/build/package_archive_test.go @@ -120,6 +120,10 @@ func TestPackageArchiveEdgeCases(t *testing.T) { if err := ctx.createPackageArchiveFile(filepath.Join(blocker, "archive.a"), pkg, false); err == nil { t.Fatal("createPackageArchiveFile succeeded below a regular file") } + tooLong := filepath.Join(t.TempDir(), strings.Repeat("a", 300)+".a") + if err := ctx.createPackageArchiveFile(tooLong, pkg, false); err == nil { + t.Fatal("createPackageArchiveFile succeeded with an overlong temporary-file prefix") + } archiveDir := filepath.Join(t.TempDir(), "archive.a") if err := os.Mkdir(archiveDir, 0o755); err != nil { @@ -153,9 +157,9 @@ func TestNormalizeToArchiveFailsWithoutObjectFallback(t *testing.T) { buffer: memoryBuf, }}, } - err := normalizeToArchive(ctx, pkg, false) + _, err := finalizePackageBuild(ctx, packageBuildSpec{pkg: pkg}, false) if err == nil { - t.Fatal("normalizeToArchive succeeded with a missing member") + t.Fatal("finalizePackageBuild succeeded with a missing member") } if !strings.Contains(err.Error(), "missing.o") { t.Fatalf("normalizeToArchive error = %v, want missing member", err) diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go index d1ce18f415..5a8711370d 100644 --- a/internal/build/package_build_test.go +++ b/internal/build/package_build_test.go @@ -102,6 +102,87 @@ var content string } } +func TestBuildOnePackageReturnsFrontendError(t *testing.T) { + t.Setenv(llgoBuildCache, "off") + fset := token.NewFileSet() + filename := filepath.Join(t.TempDir(), "p.go") + file, err := parser.ParseFile(fset, filename, `package p + +//go:embed missing.txt +var content string +`, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + pkg := &aPackage{ + Package: &packages.Package{ + ID: "example.com/frontend-error", + PkgPath: "example.com/frontend-error", + GoFiles: []string{filename}, + Syntax: []*ast.File{file}, + Types: types.NewPackage("example.com/frontend-error", "p"), + }, + Manifest: "already fingerprinted", + Fingerprint: "frontend-error", + } + ctx := &context{ + conf: &packages.Config{Fset: fset}, + buildConf: &Config{}, + built: make(map[string]none), + } + + _, err = buildOnePackage(ctx, newPackageBuildSpec(pkg), false) + if err == nil || !strings.Contains(err.Error(), "only allowed in Go files that import") { + t.Fatalf("buildOnePackage frontend error = %v", err) + } +} + +func TestPreflightPackageBuildErrorsAndCacheHit(t *testing.T) { + t.Run("fingerprint error", func(t *testing.T) { + pkg := &aPackage{Package: &packages.Package{ + ID: "example.com/missing-source", + PkgPath: "example.com/missing-source", + GoFiles: []string{filepath.Join(t.TempDir(), "missing.go")}, + Types: types.NewPackage("example.com/missing-source", "missing"), + }} + ctx := &context{ + buildConf: &Config{}, + built: make(map[string]none), + llvmVersion: "test", + } + skip, err := preflightPackageBuild(ctx, newPackageBuildSpec(pkg), false) + if err == nil || !strings.Contains(err.Error(), "digest go files") { + t.Fatalf("preflight fingerprint error = %v", err) + } + if skip { + t.Fatal("preflight skipped package after fingerprint error") + } + }) + + t.Run("cache hit", func(t *testing.T) { + t.Setenv(llgoBuildCache, "off") + pkg := &aPackage{ + Package: &packages.Package{ + ID: "example.com/cache-hit", + PkgPath: "example.com/cache-hit", + GoFiles: []string{"cached.go"}, + Types: types.NewPackage("example.com/cache-hit", "cached"), + }, + Manifest: "already fingerprinted", + Fingerprint: "cache-hit", + CacheHit: true, + } + ctx := &context{buildConf: &Config{}, built: make(map[string]none)} + skip, err := preflightPackageBuild(ctx, newPackageBuildSpec(pkg), true) + if err != nil { + t.Fatal(err) + } + if skip || !pkg.CacheHit { + t.Fatalf("preflight cache hit = skip %v, cache hit %v", skip, pkg.CacheHit) + } + }) +} + func TestBuildOnePackageSkipsAlreadyBuiltPackage(t *testing.T) { pkg := &aPackage{Package: &packages.Package{ ID: "example.com/already-built", From 39ffae9785fefdabb1c21ea6ef487379d48e2e8f Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sun, 2 Aug 2026 13:02:30 +0800 Subject: [PATCH 10/14] build: update LLVM bindings to v0.9.6 --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 23ba621674..4c5087f7a1 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/goplus/mod v0.21.1 github.com/mattn/go-tty v0.0.8 github.com/qiniu/x v1.18.0 - github.com/xgo-dev/llvm v0.9.5 + github.com/xgo-dev/llvm v0.9.6 github.com/xgo-dev/plan9asm v0.3.5 go.bug.st/serial v1.6.4 go.yaml.in/yaml/v3 v3.0.5 @@ -26,5 +26,3 @@ require ( ) replace github.com/goplus/llgo/runtime => ./runtime - -replace github.com/xgo-dev/llvm => github.com/zhouguangyuan0718/go-llvm v0.0.0-20260731084524-d8f8a2aa851e diff --git a/go.sum b/go.sum index fa2dd992cd..435f9955c4 100644 --- a/go.sum +++ b/go.sum @@ -22,10 +22,10 @@ github.com/qiniu/x v1.18.0 h1:iMfc7Gqy1au+akr+Tl5Z40px7TR8VBLLkJsIeajKIbc= github.com/qiniu/x v1.18.0/go.mod h1:Sx3Wy+0GI9OsX4a53mYj6A0o7mHJ94PUvraqGYb4EIs= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/xgo-dev/llvm v0.9.6 h1:DtjcgENgDItbf7LyUukssMwnnWXDpERSzGQ9jRQ1CRM= +github.com/xgo-dev/llvm v0.9.6/go.mod h1:42vav2/cI5BAIcL543DZSMO9do8/aCK2z7JERH+AE+M= github.com/xgo-dev/plan9asm v0.3.5 h1:886BmpjMK6JfJ03VWA3nPK01jkVZA1a7/mZia3BOsdg= github.com/xgo-dev/plan9asm v0.3.5/go.mod h1:0yM4CCIp2PyT8h+Ro3Ukro3lHL8ji9mzHEv5yfhOckc= -github.com/zhouguangyuan0718/go-llvm v0.0.0-20260731084524-d8f8a2aa851e h1:O9Hz3E3WoZv6XXL4+4Gn1EkaEqN9+/F61WiwOJA5F5c= -github.com/zhouguangyuan0718/go-llvm v0.0.0-20260731084524-d8f8a2aa851e/go.mod h1:42vav2/cI5BAIcL543DZSMO9do8/aCK2z7JERH+AE+M= go.bug.st/serial v1.6.4 h1:7FmqNPgVp3pu2Jz5PoPtbZ9jJO5gnEnZIvnI1lzve8A= go.bug.st/serial v1.6.4/go.mod h1:nofMJxTeNVny/m6+KaafC6vJGj3miwQZ6vW4BZUGJPI= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= From 275f153a4797fa23e8095b2a9b90587146e4a565 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sun, 2 Aug 2026 14:08:26 +0800 Subject: [PATCH 11/14] ci: retry macOS docs install From 4b7324f56bc31ea92e5d394f8a180a6a5ad51bc1 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sun, 2 Aug 2026 22:19:44 +0800 Subject: [PATCH 12/14] build: unify package task state --- internal/build/build.go | 80 +++++++++++++------------- internal/build/package_archive_test.go | 2 +- internal/build/package_build.go | 35 +++++------ internal/build/package_build_test.go | 61 ++++++++++---------- 4 files changed, 93 insertions(+), 85 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index 44ab45aa75..db46d50186 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -981,22 +981,22 @@ func normalizeToArchive(ctx *context, aPkg *aPackage, verbose bool) error { func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, error) { // Split packages into runtime tree vs others so we can defer runtime build. - var runtimePkgs []packageBuildSpec - var normalPkgs []packageBuildSpec + var runtimePkgs []*packageBuildTask + var normalPkgs []*packageBuildTask for _, p := range pkgs { - spec := newPackageBuildSpec(p) - if spec.runtime { - runtimePkgs = append(runtimePkgs, spec) + task := newPackageBuildTask(p) + if task.isRuntime() { + runtimePkgs = append(runtimePkgs, task) } else { - normalPkgs = append(normalPkgs, spec) + normalPkgs = append(normalPkgs, task) } } var needRuntime, needPyInit bool // Build non-runtime packages first, so we know whether runtime is actually needed. - for _, spec := range normalPkgs { - result, err := buildOnePackage(ctx, spec, verbose) + for _, task := range normalPkgs { + result, err := buildOnePackage(ctx, task, verbose) if err != nil { return nil, err } @@ -1006,8 +1006,8 @@ func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, er // Only build runtime packages when required (or host build with empty Target). if needRuntime || needPyInit || ctx.buildConf.Target == "" { - for _, spec := range runtimePkgs { - if _, err := buildOnePackage(ctx, spec, verbose); err != nil { + for _, task := range runtimePkgs { + if _, err := buildOnePackage(ctx, task, verbose); err != nil { return nil, err } } @@ -1019,39 +1019,41 @@ func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, er // buildOnePackage is the serial package pipeline. Its explicit stages are the // contract used by later package workers; this commit deliberately preserves // serial LLVM execution. -func buildOnePackage(ctx *context, spec packageBuildSpec, verbose bool) (packageBuildResult, error) { - skip, err := preflightPackageBuild(ctx, spec, verbose) - if err != nil || skip { - return packageBuildResultFor(spec), err +func buildOnePackage(ctx *context, task *packageBuildTask, verbose bool) (packageBuildResult, error) { + if err := prePackageBuild(ctx, task, verbose); err != nil || task.skip { + return packageBuildResultFor(task), err } - if err := executePackageBuild(ctx, spec, verbose); err != nil { - return packageBuildResultFor(spec), err + if err := executePackageBuild(ctx, task, verbose); err != nil { + return packageBuildResultFor(task), err } - return finalizePackageBuild(ctx, spec, verbose) + return finalizePackageBuild(ctx, task, verbose) } -// preflightPackageBuild performs classification, fingerprinting, and cache +// prePackageBuild performs classification, fingerprinting, and cache // lookup without creating or transforming an LLVM module. -func preflightPackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (skip bool, err error) { - aPkg := spec.pkg +func prePackageBuild(ctx *context, task *packageBuildTask, verbose bool) error { + aPkg := task.pkg pkg := aPkg.Package if _, ok := ctx.built[pkg.ID]; ok { - return true, nil + task.skip = true + return nil } ctx.built[pkg.ID] = none{} - if spec.isDeclOnly() { + if task.isDeclOnly() { pkg.ExportFile = "" - return true, nil + task.skip = true + return nil } - if spec.isLinkOnly() && !spec.hasSource() { + if task.isLinkOnly() && !task.hasSource() { pkg.ExportFile = "" - if spec.kind == cl.PkgLinkExtern { - appendExternalLinkArgs(ctx, aPkg, spec.kindParam) + if task.kind == cl.PkgLinkExtern { + appendExternalLinkArgs(ctx, aPkg, task.kindParam) } - return true, nil + task.skip = true + return nil } if err := ctx.collectFingerprint(aPkg); err != nil { - return false, err + return err } ctx.tryLoadFromCache(aPkg) if verbose { @@ -1061,16 +1063,16 @@ func preflightPackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (s } fmt.Fprintf(os.Stderr, "CACHE %s: %s\n", status, pkg.PkgPath) } - return false, nil + return nil } // executePackageBuild creates the package module and runs its LLVM backend. -func executePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) error { - aPkg := spec.pkg +func executePackageBuild(ctx *context, task *packageBuildTask, verbose bool) error { + aPkg := task.pkg if err := buildPkg(ctx, aPkg, verbose); err != nil { return err } - if spec.needsRuntimeSignals() { + if task.needsRuntimeSignals() { aPkg.setNeedRuntimeOrPyInit(aPkg.LPkg.NeedRuntime, aPkg.LPkg.NeedPyInit) } return nil @@ -1078,21 +1080,21 @@ func executePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) erro // finalizePackageBuild publishes the archive and cache metadata. Cache hits // already carry both and therefore require no publication. -func finalizePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (packageBuildResult, error) { - aPkg := spec.pkg +func finalizePackageBuild(ctx *context, task *packageBuildTask, verbose bool) (packageBuildResult, error) { + aPkg := task.pkg if aPkg.CacheHit { - return packageBuildResultFor(spec), nil + return packageBuildResultFor(task), nil } if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { - return packageBuildResultFor(spec), err + return packageBuildResultFor(task), err } - if spec.kind == cl.PkgLinkExtern { - appendExternalLinkArgs(ctx, aPkg, spec.kindParam) + if task.kind == cl.PkgLinkExtern { + appendExternalLinkArgs(ctx, aPkg, task.kindParam) } if err := ctx.saveToCache(aPkg); err != nil && verbose { fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", aPkg.PkgPath, err) } - return packageBuildResultFor(spec), nil + return packageBuildResultFor(task), nil } func appendExternalLinkArgs(ctx *context, aPkg *aPackage, spec string) { diff --git a/internal/build/package_archive_test.go b/internal/build/package_archive_test.go index 477963da86..f76987b88b 100644 --- a/internal/build/package_archive_test.go +++ b/internal/build/package_archive_test.go @@ -157,7 +157,7 @@ func TestNormalizeToArchiveFailsWithoutObjectFallback(t *testing.T) { buffer: memoryBuf, }}, } - _, err := finalizePackageBuild(ctx, packageBuildSpec{pkg: pkg}, false) + _, err := finalizePackageBuild(ctx, &packageBuildTask{pkg: pkg}, false) if err == nil { t.Fatal("finalizePackageBuild succeeded with a missing member") } diff --git a/internal/build/package_build.go b/internal/build/package_build.go index a5ccc0ab38..12942f2c68 100644 --- a/internal/build/package_build.go +++ b/internal/build/package_build.go @@ -18,37 +18,40 @@ package build import "github.com/goplus/llgo/cl" -type packageBuildSpec struct { +type packageBuildTask struct { pkg *aPackage kind int kindParam string - runtime bool + skip bool } -func newPackageBuildSpec(pkg *aPackage) packageBuildSpec { +func newPackageBuildTask(pkg *aPackage) *packageBuildTask { kind, kindParam := cl.PkgKindOf(pkg.Types) - return packageBuildSpec{ + return &packageBuildTask{ pkg: pkg, kind: kind, kindParam: kindParam, - runtime: isRuntimePkg(pkg.PkgPath), } } -func (s packageBuildSpec) isDeclOnly() bool { - return s.kind == cl.PkgDeclOnly +func (t *packageBuildTask) isRuntime() bool { + return isRuntimePkg(t.pkg.PkgPath) } -func (s packageBuildSpec) isLinkOnly() bool { - return s.kind == cl.PkgLinkIR || s.kind == cl.PkgLinkExtern || s.kind == cl.PkgPyModule +func (t *packageBuildTask) isDeclOnly() bool { + return t.kind == cl.PkgDeclOnly } -func (s packageBuildSpec) hasSource() bool { - return len(s.pkg.GoFiles) > 0 +func (t *packageBuildTask) isLinkOnly() bool { + return t.kind == cl.PkgLinkIR || t.kind == cl.PkgLinkExtern || t.kind == cl.PkgPyModule } -func (s packageBuildSpec) needsRuntimeSignals() bool { - return !s.isLinkOnly() && !s.isDeclOnly() +func (t *packageBuildTask) hasSource() bool { + return len(t.pkg.GoFiles) > 0 +} + +func (t *packageBuildTask) needsRuntimeSignals() bool { + return !t.isLinkOnly() && !t.isDeclOnly() } type packageBuildResult struct { @@ -56,9 +59,9 @@ type packageBuildResult struct { needPyInit bool } -func packageBuildResultFor(spec packageBuildSpec) packageBuildResult { +func packageBuildResultFor(task *packageBuildTask) packageBuildResult { return packageBuildResult{ - needRuntime: spec.pkg.NeedRt, - needPyInit: spec.pkg.NeedPyInit, + needRuntime: task.pkg.NeedRt, + needPyInit: task.pkg.NeedPyInit, } } diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go index 5a8711370d..235e77e7cc 100644 --- a/internal/build/package_build_test.go +++ b/internal/build/package_build_test.go @@ -31,35 +31,35 @@ import ( "golang.org/x/tools/go/ssa" ) -func TestPackageBuildSpecAndResult(t *testing.T) { +func TestPackageBuildTaskAndResult(t *testing.T) { pkg := &aPackage{Package: &packages.Package{ PkgPath: "example.com/p", GoFiles: []string{"p.go"}, Types: types.NewPackage("example.com/p", "p"), }, NeedRt: true, NeedPyInit: true} - spec := newPackageBuildSpec(pkg) - if spec.isDeclOnly() || spec.isLinkOnly() || !spec.hasSource() || spec.runtime || !spec.needsRuntimeSignals() { - t.Fatalf("unexpected normal package spec: %+v", spec) + task := newPackageBuildTask(pkg) + if task.isDeclOnly() || task.isLinkOnly() || !task.hasSource() || task.isRuntime() || !task.needsRuntimeSignals() { + t.Fatalf("unexpected normal package task: %+v", task) } - result := packageBuildResultFor(spec) + result := packageBuildResultFor(task) if !result.needRuntime || !result.needPyInit { t.Fatalf("unexpected package result: %+v", result) } } -func TestPackageBuildSpecSpecialKinds(t *testing.T) { - decl := newPackageBuildSpec(&aPackage{Package: &packages.Package{ +func TestPackageBuildTaskSpecialKinds(t *testing.T) { + decl := newPackageBuildTask(&aPackage{Package: &packages.Package{ PkgPath: "unsafe", Types: types.Unsafe, }}) if !decl.isDeclOnly() || decl.needsRuntimeSignals() { - t.Fatalf("unexpected declaration-only spec: %+v", decl) + t.Fatalf("unexpected declaration-only task: %+v", decl) } - runtime := newPackageBuildSpec(&aPackage{Package: &packages.Package{ + runtime := newPackageBuildTask(&aPackage{Package: &packages.Package{ PkgPath: env.LLGoRuntimePkg, Types: types.NewPackage(env.LLGoRuntimePkg, "runtime"), }}) - if !runtime.runtime { + if !runtime.isRuntime() { t.Fatalf("runtime package was not marked runtime: %+v", runtime) } } @@ -131,13 +131,13 @@ var content string built: make(map[string]none), } - _, err = buildOnePackage(ctx, newPackageBuildSpec(pkg), false) + _, err = buildOnePackage(ctx, newPackageBuildTask(pkg), false) if err == nil || !strings.Contains(err.Error(), "only allowed in Go files that import") { t.Fatalf("buildOnePackage frontend error = %v", err) } } -func TestPreflightPackageBuildErrorsAndCacheHit(t *testing.T) { +func TestPrePackageBuildErrorsAndCacheHit(t *testing.T) { t.Run("fingerprint error", func(t *testing.T) { pkg := &aPackage{Package: &packages.Package{ ID: "example.com/missing-source", @@ -150,12 +150,13 @@ func TestPreflightPackageBuildErrorsAndCacheHit(t *testing.T) { built: make(map[string]none), llvmVersion: "test", } - skip, err := preflightPackageBuild(ctx, newPackageBuildSpec(pkg), false) + task := newPackageBuildTask(pkg) + err := prePackageBuild(ctx, task, false) if err == nil || !strings.Contains(err.Error(), "digest go files") { - t.Fatalf("preflight fingerprint error = %v", err) + t.Fatalf("pre fingerprint error = %v", err) } - if skip { - t.Fatal("preflight skipped package after fingerprint error") + if task.skip { + t.Fatal("pre skipped package after fingerprint error") } }) @@ -173,12 +174,13 @@ func TestPreflightPackageBuildErrorsAndCacheHit(t *testing.T) { CacheHit: true, } ctx := &context{buildConf: &Config{}, built: make(map[string]none)} - skip, err := preflightPackageBuild(ctx, newPackageBuildSpec(pkg), true) + task := newPackageBuildTask(pkg) + err := prePackageBuild(ctx, task, true) if err != nil { t.Fatal(err) } - if skip || !pkg.CacheHit { - t.Fatalf("preflight cache hit = skip %v, cache hit %v", skip, pkg.CacheHit) + if task.skip || !pkg.CacheHit { + t.Fatalf("pre cache hit = skip %v, cache hit %v", task.skip, pkg.CacheHit) } }) } @@ -192,7 +194,7 @@ func TestBuildOnePackageSkipsAlreadyBuiltPackage(t *testing.T) { }, NeedRt: true} ctx := &context{built: map[string]none{pkg.ID: {}}} - result, err := buildOnePackage(ctx, newPackageBuildSpec(pkg), false) + result, err := buildOnePackage(ctx, newPackageBuildTask(pkg), false) if err != nil { t.Fatal(err) } @@ -201,7 +203,7 @@ func TestBuildOnePackageSkipsAlreadyBuiltPackage(t *testing.T) { } } -func TestPreflightPackageBuildSkipsDeclarationOnlyPackage(t *testing.T) { +func TestPrePackageBuildSkipsDeclarationOnlyPackage(t *testing.T) { pkg := &aPackage{Package: &packages.Package{ ID: "unsafe", PkgPath: "unsafe", @@ -210,11 +212,12 @@ func TestPreflightPackageBuildSkipsDeclarationOnlyPackage(t *testing.T) { }} ctx := &context{built: make(map[string]none)} - skip, err := preflightPackageBuild(ctx, newPackageBuildSpec(pkg), false) + task := newPackageBuildTask(pkg) + err := prePackageBuild(ctx, task, false) if err != nil { t.Fatal(err) } - if !skip { + if !task.skip { t.Fatal("declaration-only package was not skipped") } if pkg.ExportFile != "" { @@ -225,7 +228,7 @@ func TestPreflightPackageBuildSkipsDeclarationOnlyPackage(t *testing.T) { } } -func TestPreflightPackageBuildSkipsExternalLinkOnlyPackage(t *testing.T) { +func TestPrePackageBuildSkipsExternalLinkOnlyPackage(t *testing.T) { pkg := &aPackage{Package: &packages.Package{ ID: "example.com/linkonly", PkgPath: "example.com/linkonly", @@ -234,13 +237,13 @@ func TestPreflightPackageBuildSkipsExternalLinkOnlyPackage(t *testing.T) { ExportFile: "stale.a", }} ctx := &context{buildConf: &Config{}, built: make(map[string]none)} - spec := packageBuildSpec{pkg: pkg, kind: cl.PkgLinkExtern, kindParam: "-lexample"} - skip, err := preflightPackageBuild(ctx, spec, false) + task := &packageBuildTask{pkg: pkg, kind: cl.PkgLinkExtern, kindParam: "-lexample"} + err := prePackageBuild(ctx, task, false) if err != nil { t.Fatal(err) } - if !skip || pkg.ExportFile != "" { - t.Fatalf("external link-only preflight = skip %v, export %q", skip, pkg.ExportFile) + if !task.skip || pkg.ExportFile != "" { + t.Fatalf("external link-only pre = skip %v, export %q", task.skip, pkg.ExportFile) } if len(pkg.LinkArgs) != 1 || pkg.LinkArgs[0] != "-lexample" { t.Fatalf("external link args = %q, want [-lexample]", pkg.LinkArgs) @@ -254,7 +257,7 @@ func TestFinalizePackageBuildReturnsCachedResult(t *testing.T) { Types: types.NewPackage("example.com/cached", "cached"), }, CacheHit: true, NeedPyInit: true} - result, err := finalizePackageBuild(&context{}, newPackageBuildSpec(pkg), false) + result, err := finalizePackageBuild(&context{}, newPackageBuildTask(pkg), false) if err != nil { t.Fatal(err) } From 1f53a0f54192681efeaa110b75d06223c0652ed1 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Thu, 30 Jul 2026 08:09:16 +0800 Subject: [PATCH 13/14] build: snapshot package linker state --- internal/build/build.go | 99 ++++++----- internal/build/collect.go | 19 +- internal/build/collect_test.go | 86 +++++++++- internal/build/fingerprint.go | 7 +- internal/build/funcinfo_table.go | 37 ++-- internal/build/package_build_test.go | 26 ++- internal/build/package_summary.go | 229 +++++++++++++++++++++++++ internal/build/package_summary_test.go | 93 ++++++++++ 8 files changed, 529 insertions(+), 67 deletions(-) create mode 100644 internal/build/package_summary.go create mode 100644 internal/build/package_summary_test.go diff --git a/internal/build/build.go b/internal/build/build.go index db46d50186..3e3ce4a6f6 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -1042,7 +1042,8 @@ func prePackageBuild(ctx *context, task *packageBuildTask, verbose bool) error { if task.isDeclOnly() { pkg.ExportFile = "" task.skip = true - return nil + aPkg.Summary = summarizePackage(aPkg) + return ctx.collectFingerprint(aPkg) } if task.isLinkOnly() && !task.hasSource() { pkg.ExportFile = "" @@ -1050,7 +1051,8 @@ func prePackageBuild(ctx *context, task *packageBuildTask, verbose bool) error { appendExternalLinkArgs(ctx, aPkg, task.kindParam) } task.skip = true - return nil + aPkg.Summary = summarizePackage(aPkg) + return ctx.collectFingerprint(aPkg) } if err := ctx.collectFingerprint(aPkg); err != nil { return err @@ -1091,6 +1093,7 @@ func finalizePackageBuild(ctx *context, task *packageBuildTask, verbose bool) (p if task.kind == cl.PkgLinkExtern { appendExternalLinkArgs(ctx, aPkg, task.kindParam) } + aPkg.Summary = summarizePackage(aPkg) if err := ctx.saveToCache(aPkg); err != nil && verbose { fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", aPkg.PkgPath, err) } @@ -1341,39 +1344,47 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa aPkg = ctx.pkgByID[p.ID] } if p.ExportFile != "" && aPkg != nil { // skip packages that only contain declarations + if aPkg.Summary == nil && isRuntimePkg(aPkg.PkgPath) { + return + } linkedPkgs[p.ID] = true linkedOrder = append(linkedOrder, aPkg) } }) + linkedSummaries := make([]*PackageSummary, len(linkedOrder)) + for i, aPkg := range linkedOrder { + if aPkg.Summary == nil { + return fmt.Errorf("package %s has no linker summary", aPkg.PkgPath) + } + linkedSummaries[i] = aPkg.Summary + } // packages.Visit with a post callback yields dependencies before importers. // Reverse that order so static archives are linked after the objects that use them. for i := len(linkedOrder) - 1; i >= 0; i-- { - aPkg := linkedOrder[i] - p := aPkg.Package + summary := linkedSummaries[i] // Defer linking runtime packages unless we actually need the runtime. - if isRuntimePkg(p.PkgPath) { - rtLinkArgs = append(rtLinkArgs, aPkg.LinkArgs...) - if aPkg.ArchiveFile != "" { - rtLinkInputs = append(rtLinkInputs, aPkg.ArchiveFile) + if isRuntimePkg(summary.PkgPath) { + rtLinkArgs = append(rtLinkArgs, summary.LinkArgs...) + if summary.ArchiveFile != "" { + rtLinkInputs = append(rtLinkInputs, summary.ArchiveFile) } continue } // Only let non-runtime packages influence whether runtime is needed. - need1, need2 := aPkg.isNeedRuntimeOrPyInit() - needRuntime = needRuntime || need1 - needPyInit = needPyInit || need2 - needAbiInit |= aPkg.LPkg.NeedAbiInit - for k, _ := range aPkg.LPkg.MethodByIndex { - methodByIndex[k] = none{} + needRuntime = needRuntime || summary.NeedRuntime + needPyInit = needPyInit || summary.NeedPyInit + needAbiInit |= summary.NeedAbiInit + for _, method := range summary.MethodByIndex { + methodByIndex[method] = none{} } - for k, _ := range aPkg.LPkg.MethodByName { - methodByName[k] = none{} + for _, method := range summary.MethodByName { + methodByName[method] = none{} } - linkArgs = append(linkArgs, aPkg.LinkArgs...) - if aPkg.ArchiveFile != "" { - archiveInputs = append(archiveInputs, aPkg.ArchiveFile) + linkArgs = append(linkArgs, summary.LinkArgs...) + if summary.ArchiveFile != "" { + archiveInputs = append(archiveInputs, summary.ArchiveFile) } } @@ -1390,9 +1401,9 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa var pcLineInfo []pcLineRecord var funcInfoStubs []funcInfoStubRecord if ctx.buildConf.PCLNMode != PCLNNone { - funcInfo = prepareFuncInfoTableRecords(collectFuncInfo(linkedOrder), nil) - pcLineInfo = collectPCLineInfo(linkedOrder) - funcInfoStubs = collectFuncInfoStubRecords(linkedOrder, funcInfo) + funcInfo = prepareFuncInfoTableRecords(collectFuncInfoSummaries(linkedSummaries), nil) + pcLineInfo = collectPCLineInfoSummaries(linkedSummaries) + funcInfoStubs = collectFuncInfoStubRecordsSummaries(linkedSummaries, funcInfo) } entryPkg := genMainModule(ctx, llssa.PkgRuntime, pkg, &genConfig{ rtInit: needRuntime, @@ -1400,7 +1411,7 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa abiInit: needAbiInit, methodByIndex: methodByIndex, methodByName: methodByName, - abiSymbols: linkedModuleGlobals(linkedOrder), + abiSymbols: linkedPackageGlobals(linkedSummaries), funcInfo: funcInfo, pcLineInfo: pcLineInfo, funcInfoStubs: funcInfoStubs, @@ -1440,7 +1451,7 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa } } } - linkArgs = append(linkArgs, cSharedExportArgs(ctx, linkedOrder)...) + linkArgs = append(linkArgs, cSharedExportArgsSummaries(ctx, linkedSummaries)...) err = linkObjFiles(ctx, outputPath, linkInputs, linkArgs, verbose) if err != nil { @@ -1498,19 +1509,22 @@ func dceEntryRootCandidates(pkgs []Package, needRuntime bool) []string { } func linkedModuleGlobals(pkgs []Package) map[string]none { - if len(pkgs) == 0 { + return linkedPackageGlobals(summariesForPackages(pkgs)) +} + +func linkedPackageGlobals(summaries []*PackageSummary) map[string]none { + if len(summaries) == 0 { return nil } seen := make(map[string]none) - for _, pkg := range pkgs { - if pkg == nil || pkg.LPkg == nil { + for _, summary := range summaries { + if summary == nil { continue } - for g := pkg.LPkg.Module().FirstGlobal(); !g.IsNil(); g = gllvm.NextGlobal(g) { - if g.IsDeclaration() { - continue + for _, name := range summary.GlobalSymbols { + if name != "" { + seen[name] = none{} } - seen[g.Name()] = none{} } } return seen @@ -1589,22 +1603,26 @@ func linkObjFiles(ctx *context, app string, objFiles, linkArgs []string, verbose // shared-library link roots. They live in package archives and otherwise remain // unreferenced, so the linker can omit both their object files and symbols. func cSharedExportArgs(ctx *context, pkgs []*aPackage) []string { + return cSharedExportArgsSummaries(ctx, summariesForPackages(pkgs)) +} + +func cSharedExportArgsSummaries(ctx *context, summaries []*PackageSummary) []string { if ctx == nil || ctx.buildConf == nil || ctx.buildConf.BuildMode != BuildModeCShared { return nil } exports := make(map[string]none) - for _, pkg := range pkgs { - if pkg == nil || pkg.LPkg == nil { + for _, summary := range summaries { + if summary == nil { continue } - for _, name := range pkg.LPkg.ExportFuncs() { + for _, name := range summary.CSharedExports { if name != "" { exports[name] = none{} } } - if ctx.mode == ModeTest && pkg.Package != nil && pkg.Name == "main" && strings.HasSuffix(pkg.PkgPath, ".test") { - exports[pkg.PkgPath+".init"] = none{} - exports[pkg.PkgPath+".main"] = none{} + if ctx.mode == ModeTest && summary.Name == "main" && strings.HasSuffix(summary.PkgPath, ".test") { + exports[summary.PkgPath+".init"] = none{} + exports[summary.PkgPath+".main"] = none{} } } names := make([]string, 0, len(exports)) @@ -2238,9 +2256,10 @@ func registerAltSSAPkgs(prog *ssa.Program, patches cl.Patches, alts []*packages. type aPackage struct { *packages.Package - SSA *ssa.Package - AltPkg *packages.Cached - LPkg llssa.Package + SSA *ssa.Package + AltPkg *packages.Cached + LPkg llssa.Package + Summary *PackageSummary NeedRt bool NeedPyInit bool diff --git a/internal/build/collect.go b/internal/build/collect.go index d8dcee7b1f..7d3c65166b 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -334,6 +334,9 @@ func (c *context) tryLoadFromCache(pkg *aPackage) bool { if c.packageCacheDisabled(pkg.ID) { return false } + if c.buildConf != nil && (c.buildConf.BuildMode == BuildModeCArchive || c.buildConf.BuildMode == BuildModeCShared) { + return false + } // Main packages are intentionally not written to the build cache because // each executable's entry module is linked against the current main archive. @@ -377,6 +380,9 @@ func (c *context) tryLoadFromCache(pkg *aPackage) bool { if err != nil { return false } + if meta.Summary == nil { + return false + } // Use the .a archive directly for linking (no extraction needed) pkg.ArchiveFile = paths.Archive @@ -385,6 +391,7 @@ func (c *context) tryLoadFromCache(pkg *aPackage) bool { pkg.NeedPyInit = meta.NeedPyInit pkg.Meta = pkgMeta pkg.CacheHit = true + pkg.Summary = summaryFromMetadata(pkg, meta) return true } @@ -399,6 +406,7 @@ func parseManifestMetadata(content string) (*cacheArchiveMetadata, error) { meta.LinkArgs = append([]string(nil), data.Metadata.LinkArgs...) meta.NeedRt = data.Metadata.NeedRt meta.NeedPyInit = data.Metadata.NeedPyInit + meta.Summary = data.Metadata.Summary } return meta, nil } @@ -452,6 +460,7 @@ type cacheArchiveMetadata struct { LinkArgs []string NeedRt bool NeedPyInit bool + Summary *packageSummaryMetadata } // saveToCache saves a built package to cache. @@ -511,16 +520,16 @@ func (c *context) saveToCache(pkg *aPackage) error { return fmt.Errorf("decode manifest: %w", err) } + if pkg.Summary == nil { + pkg.Summary = summarizePackage(pkg) + } meta := &manifestMetadata{ LinkArgs: append([]string(nil), pkg.LinkArgs...), NeedRt: pkg.NeedRt, NeedPyInit: pkg.NeedPyInit, + Summary: pkg.Summary.metadata(), } - if len(meta.LinkArgs) == 0 && !meta.NeedRt && !meta.NeedPyInit { - data.Metadata = nil - } else { - data.Metadata = meta - } + data.Metadata = meta manifestWithMeta, err := buildManifestYAML(data) if err != nil { diff --git a/internal/build/collect_test.go b/internal/build/collect_test.go index cc9785518f..2abc1e3c13 100644 --- a/internal/build/collect_test.go +++ b/internal/build/collect_test.go @@ -22,6 +22,7 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "runtime" "strings" "testing" @@ -1060,8 +1061,8 @@ func TestSaveToCache_Success(t *testing.T) { if data.Env.Goos != "darwin" { t.Errorf("manifest should contain original env content") } - if data.Metadata != nil { - t.Errorf("metadata should be empty when no link args/runtime flags") + if data.Metadata == nil || data.Metadata.Summary == nil { + t.Errorf("metadata should preserve an empty linker summary") } // Check archive exists @@ -1235,6 +1236,7 @@ func TestTryLoadFromCacheIgnoresMetaWhenPackageMetaDisabled(t *testing.T) { m := newManifestBuilder() m.env.Goos = "darwin" m.pkg.PkgPath = "example.com/nometa" + m.meta = &manifestMetadata{Summary: &packageSummaryMetadata{}} if err := writeManifest(paths.Manifest, m.Build()); err != nil { t.Fatal(err) } @@ -1250,6 +1252,86 @@ func TestTryLoadFromCacheIgnoresMetaWhenPackageMetaDisabled(t *testing.T) { } } +func TestTryLoadFromCacheRestoresPackageSummary(t *testing.T) { + td := t.TempDir() + oldFunc := cacheRootFunc + cacheRootFunc = func() string { return td } + defer func() { cacheRootFunc = oldFunc }() + + ctx := &context{ + conf: &packages.Config{}, + buildConf: &Config{Goos: "darwin", Goarch: "arm64"}, + crossCompile: crosscompile.Export{ + LLVMTarget: "arm64-apple-darwin", + }, + } + pkg := &aPackage{ + Package: &packages.Package{ + ID: "example.com/cached", + PkgPath: "example.com/cached", + Name: "cached", + }, + Fingerprint: "summary-test", + Manifest: func() string { + m := newManifestBuilder() + m.env.Goos = "darwin" + m.pkg.PkgPath = "example.com/cached" + return m.Build() + }(), + NeedRt: true, + NeedPyInit: true, + LinkArgs: []string{"-lcached"}, + Summary: &PackageSummary{ + ID: "example.com/cached", + PkgPath: "example.com/cached", + Name: "cached", + LinkArgs: []string{"-lcached"}, + NeedRuntime: true, + NeedPyInit: true, + NeedAbiInit: 3, + MethodByIndex: []int{1}, + MethodByName: []string{"Method"}, + GlobalSymbols: []string{"example.com/cached.global"}, + FuncInfo: []funcInfoRecord{{symbol: "example.com/cached.fn", name: "Fn", file: "p.go", line: 7}}, + PCLineInfo: []pcLineRecord{{id: 9, symbol: "example.com/cached.fn", file: "p.go", line: 8}}, + FuncInfoStubs: []string{closureStubPrefix + "example.com/cached.fn"}, + CSharedExports: []string{"Cached"}, + }, + } + want := *pkg.Summary + obj, err := os.CreateTemp(td, "cached-*.o") + if err != nil { + t.Fatal(err) + } + if _, err := obj.WriteString("object"); err != nil { + t.Fatal(err) + } + if err := obj.Close(); err != nil { + t.Fatal(err) + } + pkg.ObjFiles = []string{obj.Name()} + if err := ctx.saveToCache(pkg); err != nil { + t.Fatalf("saveToCache: %v", err) + } + + pkg.ObjFiles = nil + pkg.ArchiveFile = "" + pkg.LinkArgs = nil + pkg.NeedRt = false + pkg.NeedPyInit = false + pkg.Summary = nil + if !ctx.tryLoadFromCache(pkg) { + t.Fatal("tryLoadFromCache = false, want summary cache hit") + } + if pkg.Summary == nil { + t.Fatal("cache hit did not restore package summary") + } + want.ArchiveFile = pkg.ArchiveFile + if !reflect.DeepEqual(pkg.Summary, &want) { + t.Fatalf("restored summary = %#v, want %#v", pkg.Summary, &want) + } +} + func TestGetLLVMVersion(t *testing.T) { ctx := &context{ crossCompile: crosscompile.Export{}, diff --git a/internal/build/fingerprint.go b/internal/build/fingerprint.go index ecc55298ab..781ec180f6 100644 --- a/internal/build/fingerprint.go +++ b/internal/build/fingerprint.go @@ -37,9 +37,10 @@ type depEntry struct { // manifestMetadata stores metadata produced during build but not part of the fingerprint. type manifestMetadata struct { - LinkArgs []string `yaml:"link_args,omitempty"` - NeedRt bool `yaml:"need_rt,omitempty"` - NeedPyInit bool `yaml:"need_py_init,omitempty"` + LinkArgs []string `yaml:"link_args,omitempty"` + NeedRt bool `yaml:"need_rt,omitempty"` + NeedPyInit bool `yaml:"need_py_init,omitempty"` + Summary *packageSummaryMetadata `yaml:"summary,omitempty"` } // manifestData is the structured representation of manifest content. diff --git a/internal/build/funcinfo_table.go b/internal/build/funcinfo_table.go index b36aa29654..f86fd972a1 100644 --- a/internal/build/funcinfo_table.go +++ b/internal/build/funcinfo_table.go @@ -86,12 +86,16 @@ type funcInfoSymbolIndexRecord struct { } func collectFuncInfo(pkgs []Package) []funcInfoRecord { + return collectFuncInfoSummaries(summariesForPackages(pkgs)) +} + +func collectFuncInfoSummaries(summaries []*PackageSummary) []funcInfoRecord { seen := make(map[string]funcInfoRecord) - for _, pkg := range pkgs { - if pkg == nil || pkg.LPkg == nil { + for _, summary := range summaries { + if summary == nil { continue } - for _, rec := range readFuncInfo(pkg.LPkg.Module()) { + for _, rec := range summary.FuncInfo { if rec.symbol == "" { continue } @@ -114,13 +118,17 @@ func collectFuncInfo(pkgs []Package) []funcInfoRecord { } func collectPCLineInfo(pkgs []Package) []pcLineRecord { + return collectPCLineInfoSummaries(summariesForPackages(pkgs)) +} + +func collectPCLineInfoSummaries(summaries []*PackageSummary) []pcLineRecord { var out []pcLineRecord seen := make(map[uint64]none) - for _, pkg := range pkgs { - if pkg == nil || pkg.LPkg == nil { + for _, summary := range summaries { + if summary == nil { continue } - for _, rec := range readPCLineInfo(pkg.LPkg.Module()) { + for _, rec := range summary.PCLineInfo { if rec.id == 0 || rec.symbol == "" { continue } @@ -144,6 +152,10 @@ func collectPCLineInfo(pkgs []Package) []pcLineRecord { } func collectFuncInfoStubRecords(pkgs []Package, records []funcInfoRecord) []funcInfoStubRecord { + return collectFuncInfoStubRecordsSummaries(summariesForPackages(pkgs), records) +} + +func collectFuncInfoStubRecordsSummaries(summaries []*PackageSummary, records []funcInfoRecord) []funcInfoStubRecord { if len(records) == 0 { return nil } @@ -154,23 +166,16 @@ func collectFuncInfoStubRecords(pkgs []Package, records []funcInfoRecord) []func } } seen := make(map[string]funcInfoStubRecord) - for _, pkg := range pkgs { - if pkg == nil || pkg.LPkg == nil { + for _, summary := range summaries { + if summary == nil { continue } - fn := pkg.LPkg.Module().FirstFunction() - for !fn.IsNil() { - if fn.IsDeclaration() || fn.BasicBlocksCount() == 0 { - fn = llvm.NextFunction(fn) - continue - } - name := fn.Name() + for _, name := range summary.FuncInfoStubs { if target, ok := strings.CutPrefix(name, closureStubPrefix); ok { if idx := recordBySymbol[target]; idx != 0 { seen[name] = funcInfoStubRecord{symbol: name, funcIndex: idx} } } - fn = llvm.NextFunction(fn) } } if len(seen) == 0 { diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go index 235e77e7cc..9ee12a983a 100644 --- a/internal/build/package_build_test.go +++ b/internal/build/package_build_test.go @@ -210,7 +210,11 @@ func TestPrePackageBuildSkipsDeclarationOnlyPackage(t *testing.T) { Types: types.Unsafe, ExportFile: "stale.a", }} - ctx := &context{built: make(map[string]none)} + ctx := &context{ + conf: &packages.Config{}, + buildConf: &Config{Goos: "linux", Goarch: "amd64", ForceRebuild: true}, + built: make(map[string]none), + } task := newPackageBuildTask(pkg) err := prePackageBuild(ctx, task, false) @@ -275,3 +279,23 @@ func TestBuildSSAPkgsEmptyAndNilEntries(t *testing.T) { pkg := prog.CreatePackage(types.NewPackage("example.com/ssa", "ssa"), nil, nil, true) buildSSAPkgs(ctx, []ssaBuildEntry{{pkg: pkg}, {pkg: pkg}}) } + +func TestPreflightFingerprintsSkippedPackage(t *testing.T) { + pkg := &aPackage{Package: &packages.Package{ + ID: "unsafe", + PkgPath: "unsafe", + Types: types.Unsafe, + }} + ctx := &context{ + conf: &packages.Config{}, + buildConf: &Config{Goos: "linux", Goarch: "amd64", ForceRebuild: true}, + built: make(map[string]none), + } + skip, err := preflightPackageBuild(ctx, newPackageBuildSpec(pkg), false) + if err != nil { + t.Fatal(err) + } + if !skip || pkg.Fingerprint == "" || pkg.Manifest == "" || pkg.Summary == nil { + t.Fatalf("skipped package was not fully prepared: skip=%v fingerprint=%q manifest=%q summary=%#v", skip, pkg.Fingerprint, pkg.Manifest, pkg.Summary) + } +} diff --git a/internal/build/package_summary.go b/internal/build/package_summary.go new file mode 100644 index 0000000000..76c1becca6 --- /dev/null +++ b/internal/build/package_summary.go @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "sort" + "strings" + + "github.com/xgo-dev/llvm" +) + +// PackageSummary is the immutable, LLVM-free output one package contributes +// to the final link. A backend can emit its object, capture this summary, and +// release its Program and LLVM context before whole-program linking starts. +// +// C archive/shared header declarations still consume LPkg and therefore stay +// on the serial compatibility path. +type PackageSummary struct { + ID string + PkgPath string + Name string + + LinkArgs []string + ArchiveFile string + NeedRuntime bool + NeedPyInit bool + + NeedAbiInit int + MethodByIndex []int + MethodByName []string + GlobalSymbols []string + + FuncInfo []funcInfoRecord + PCLineInfo []pcLineRecord + FuncInfoStubs []string + CSharedExports []string +} + +type packageSummaryMetadata struct { + NeedAbiInit int `yaml:"need_abi_init,omitempty"` + MethodByIndex []int `yaml:"method_by_index,omitempty"` + MethodByName []string `yaml:"method_by_name,omitempty"` + GlobalSymbols []string `yaml:"global_symbols,omitempty"` + + FuncInfo []funcInfoMetadata `yaml:"func_info,omitempty"` + PCLineInfo []pcLineMetadata `yaml:"pcline_info,omitempty"` + FuncInfoStubs []string `yaml:"func_info_stubs,omitempty"` + CSharedExports []string `yaml:"c_shared_exports,omitempty"` +} + +type funcInfoMetadata struct { + Symbol string `yaml:"symbol"` + Name string `yaml:"name,omitempty"` + File string `yaml:"file,omitempty"` + Line uint32 `yaml:"line,omitempty"` + Column uint32 `yaml:"column,omitempty"` +} + +type pcLineMetadata struct { + ID uint64 `yaml:"id"` + Symbol string `yaml:"symbol"` + File string `yaml:"file,omitempty"` + Line uint32 `yaml:"line,omitempty"` + Column uint32 `yaml:"column,omitempty"` +} + +func summarizePackage(pkg *aPackage) *PackageSummary { + if pkg == nil { + return nil + } + summary := &PackageSummary{ + LinkArgs: append([]string(nil), pkg.LinkArgs...), + ArchiveFile: pkg.ArchiveFile, + NeedRuntime: pkg.NeedRt, + NeedPyInit: pkg.NeedPyInit, + } + if pkg.Package != nil { + summary.ID = pkg.ID + summary.PkgPath = pkg.PkgPath + summary.Name = pkg.Name + } + if pkg.LPkg == nil { + return summary + } + + lpkg := pkg.LPkg + if summary.PkgPath == "" { + summary.PkgPath = lpkg.Path() + } + summary.NeedAbiInit = lpkg.NeedAbiInit + for method := range lpkg.MethodByIndex { + summary.MethodByIndex = append(summary.MethodByIndex, method) + } + sort.Ints(summary.MethodByIndex) + for method := range lpkg.MethodByName { + summary.MethodByName = append(summary.MethodByName, method) + } + sort.Strings(summary.MethodByName) + + mod := lpkg.Module() + for global := mod.FirstGlobal(); !global.IsNil(); global = llvm.NextGlobal(global) { + if !global.IsDeclaration() { + summary.GlobalSymbols = append(summary.GlobalSymbols, global.Name()) + } + } + sort.Strings(summary.GlobalSymbols) + summary.FuncInfo = readFuncInfo(mod) + summary.PCLineInfo = readPCLineInfo(mod) + for fn := mod.FirstFunction(); !fn.IsNil(); fn = llvm.NextFunction(fn) { + if fn.IsDeclaration() || fn.BasicBlocksCount() == 0 { + continue + } + if _, ok := strings.CutPrefix(fn.Name(), closureStubPrefix); ok { + summary.FuncInfoStubs = append(summary.FuncInfoStubs, fn.Name()) + } + } + sort.Strings(summary.FuncInfoStubs) + for _, name := range lpkg.ExportFuncs() { + if name != "" { + summary.CSharedExports = append(summary.CSharedExports, name) + } + } + sort.Strings(summary.CSharedExports) + return summary +} + +func (s *PackageSummary) metadata() *packageSummaryMetadata { + if s == nil { + return nil + } + meta := &packageSummaryMetadata{ + NeedAbiInit: s.NeedAbiInit, + MethodByIndex: append([]int(nil), s.MethodByIndex...), + MethodByName: append([]string(nil), s.MethodByName...), + GlobalSymbols: append([]string(nil), s.GlobalSymbols...), + FuncInfoStubs: append([]string(nil), s.FuncInfoStubs...), + CSharedExports: append([]string(nil), s.CSharedExports...), + } + for _, rec := range s.FuncInfo { + meta.FuncInfo = append(meta.FuncInfo, funcInfoMetadata{ + Symbol: rec.symbol, + Name: rec.name, + File: rec.file, + Line: rec.line, + Column: rec.column, + }) + } + for _, rec := range s.PCLineInfo { + meta.PCLineInfo = append(meta.PCLineInfo, pcLineMetadata{ + ID: rec.id, + Symbol: rec.symbol, + File: rec.file, + Line: rec.line, + Column: rec.column, + }) + } + return meta +} + +func summaryFromMetadata(pkg *aPackage, meta *cacheArchiveMetadata) *PackageSummary { + if pkg == nil || pkg.Package == nil || meta == nil || meta.Summary == nil { + return nil + } + summary := &PackageSummary{ + ID: pkg.ID, + PkgPath: pkg.PkgPath, + Name: pkg.Name, + LinkArgs: append([]string(nil), pkg.LinkArgs...), + ArchiveFile: pkg.ArchiveFile, + NeedRuntime: pkg.NeedRt, + NeedPyInit: pkg.NeedPyInit, + NeedAbiInit: meta.Summary.NeedAbiInit, + MethodByIndex: append([]int(nil), meta.Summary.MethodByIndex...), + MethodByName: append([]string(nil), meta.Summary.MethodByName...), + GlobalSymbols: append([]string(nil), meta.Summary.GlobalSymbols...), + FuncInfoStubs: append([]string(nil), meta.Summary.FuncInfoStubs...), + CSharedExports: append([]string(nil), meta.Summary.CSharedExports...), + } + for _, rec := range meta.Summary.FuncInfo { + summary.FuncInfo = append(summary.FuncInfo, funcInfoRecord{ + symbol: rec.Symbol, + name: rec.Name, + file: rec.File, + line: rec.Line, + column: rec.Column, + }) + } + for _, rec := range meta.Summary.PCLineInfo { + summary.PCLineInfo = append(summary.PCLineInfo, pcLineRecord{ + id: rec.ID, + symbol: rec.Symbol, + file: rec.File, + line: rec.Line, + column: rec.Column, + }) + } + return summary +} + +func summariesForPackages(pkgs []Package) []*PackageSummary { + summaries := make([]*PackageSummary, 0, len(pkgs)) + for _, pkg := range pkgs { + if pkg == nil { + continue + } + summary := pkg.Summary + if summary == nil { + summary = summarizePackage(pkg) + } + if summary != nil { + summaries = append(summaries, summary) + } + } + return summaries +} diff --git a/internal/build/package_summary_test.go b/internal/build/package_summary_test.go new file mode 100644 index 0000000000..0e27ad6f56 --- /dev/null +++ b/internal/build/package_summary_test.go @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "go/types" + "reflect" + "testing" + + "github.com/xgo-dev/llvm" + + "github.com/goplus/llgo/internal/packages" + llssa "github.com/goplus/llgo/ssa" +) + +func TestPackageSummaryCapturesLinkerFacts(t *testing.T) { + prog := llssa.NewProgram(nil) + defer prog.Dispose() + lpkg := prog.NewPackage("p", "example.com/p") + lpkg.NeedAbiInit = 3 + lpkg.RecordReflectMethodByIndex("example.com/p.Method", 4) + lpkg.RecordReflectMethodByIndex("example.com/p.Method", 1) + lpkg.RecordReflectMethodByName("example.com/p.MethodByName", "B") + lpkg.RecordReflectMethodByName("example.com/p.MethodByName", "A") + lpkg.SetExport("example.com/p.Export", "Export") + lpkg.EmitFuncInfo("example.com/p.live", "example.com/p.Live", "p.go", 17, 2) + lpkg.EmitPCLineInfo(42, "example.com/p.live", "p.go", 18, 3) + lpkg.NewFunc(closureStubPrefix+"example.com/p.live", llssa.NoArgsNoRet, llssa.InGo).MakeBody(1).Return() + + i32 := lpkg.Module().Context().Int32Type() + defined := llvm.AddGlobal(lpkg.Module(), i32, "example.com/p.defined") + defined.SetInitializer(llvm.ConstInt(i32, 1, false)) + llvm.AddGlobal(lpkg.Module(), i32, "example.com/p.declared") + + pkg := &aPackage{ + Package: &packages.Package{ + ID: "example.com/p", + PkgPath: "example.com/p", + Name: "p", + Types: types.NewPackage("example.com/p", "p"), + }, + LPkg: lpkg, + NeedRt: true, + NeedPyInit: true, + LinkArgs: []string{"-lp"}, + ArchiveFile: "p.a", + } + summary := summarizePackage(pkg) + if got, want := summary.MethodByIndex, []int{1, 4}; !reflect.DeepEqual(got, want) { + t.Fatalf("MethodByIndex = %v, want %v", got, want) + } + if got, want := summary.MethodByName, []string{"A", "B"}; !reflect.DeepEqual(got, want) { + t.Fatalf("MethodByName = %v, want %v", got, want) + } + if got, want := summary.GlobalSymbols, []string{"example.com/p.defined"}; !reflect.DeepEqual(got, want) { + t.Fatalf("GlobalSymbols = %v, want %v", got, want) + } + if got, want := summary.FuncInfoStubs, []string{closureStubPrefix + "example.com/p.live"}; !reflect.DeepEqual(got, want) { + t.Fatalf("FuncInfoStubs = %v, want %v", got, want) + } + if got := collectFuncInfoSummaries([]*PackageSummary{summary}); len(got) != 1 || got[0].symbol != "example.com/p.live" { + t.Fatalf("func info from summary = %+v, want live record", got) + } + if got := linkedPackageGlobals([]*PackageSummary{summary}); len(got) != 1 { + t.Fatalf("globals from summary = %#v, want one defined global", got) + } + + loadedPkg := &aPackage{ + Package: pkg.Package, + LinkArgs: []string{"-lp"}, + ArchiveFile: "p.a", + NeedRt: true, + NeedPyInit: true, + } + loaded := summaryFromMetadata(loadedPkg, &cacheArchiveMetadata{Summary: summary.metadata()}) + if !reflect.DeepEqual(loaded, summary) { + t.Fatalf("cache summary round trip = %#v, want %#v", loaded, summary) + } +} From 942bc2842a953d2ba640503c5928232a9b5f3ba6 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 1 Aug 2026 08:07:20 +0800 Subject: [PATCH 14/14] test: cover package summary boundaries --- internal/build/collect_test.go | 36 ++++++++++++++++++++++++++ internal/build/package_build_test.go | 19 +++++++++----- internal/build/package_summary_test.go | 35 +++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 7 deletions(-) diff --git a/internal/build/collect_test.go b/internal/build/collect_test.go index 2abc1e3c13..7b0716af14 100644 --- a/internal/build/collect_test.go +++ b/internal/build/collect_test.go @@ -171,6 +171,42 @@ func TestDisabledPackageCacheSkipsLoadAndSave(t *testing.T) { } } +func TestTryLoadFromCacheRejectsMissingSummary(t *testing.T) { + t.Setenv(llgoBuildCache, "1") + td := t.TempDir() + oldFunc := cacheRootFunc + cacheRootFunc = func() string { return td } + defer func() { cacheRootFunc = oldFunc }() + + ctx := &context{ + buildConf: &Config{Goos: "linux", Goarch: "amd64"}, + crossCompile: crosscompile.Export{ + LLVMTarget: "x86_64-unknown-linux-gnu", + }, + } + pkg := &aPackage{Package: &packages.Package{ + ID: "example.com/no-summary", + PkgPath: "example.com/no-summary", + Name: "no-summary", + }, Fingerprint: "missing-summary"} + paths := ctx.ensureCacheManager().PackagePaths(ctx.targetTriple(), pkg.PkgPath, pkg.Fingerprint) + if err := ctx.cacheManager.EnsureDir(paths); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(paths.Archive, []byte("archive"), 0o644); err != nil { + t.Fatal(err) + } + manifest := newManifestBuilder() + manifest.env.Goos = "linux" + manifest.pkg.PkgPath = pkg.PkgPath + if err := writeManifest(paths.Manifest, manifest.Build()); err != nil { + t.Fatal(err) + } + if ctx.tryLoadFromCache(pkg) { + t.Fatal("tryLoadFromCache accepted manifest without package summary") + } +} + func TestCollectFingerprintDisablesCycles(t *testing.T) { pkg := &aPackage{Package: &packages.Package{ID: "example.com/cycle", PkgPath: "example.com/cycle"}} ctx := &context{fingerprinting: map[string]bool{pkg.ID: true}} diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go index 9ee12a983a..9a5854dc7e 100644 --- a/internal/build/package_build_test.go +++ b/internal/build/package_build_test.go @@ -240,14 +240,18 @@ func TestPrePackageBuildSkipsExternalLinkOnlyPackage(t *testing.T) { Types: types.NewPackage("example.com/linkonly", "linkonly"), ExportFile: "stale.a", }} - ctx := &context{buildConf: &Config{}, built: make(map[string]none)} + ctx := &context{ + conf: &packages.Config{}, + buildConf: &Config{Goos: "linux", Goarch: "amd64", ForceRebuild: true}, + built: make(map[string]none), + } task := &packageBuildTask{pkg: pkg, kind: cl.PkgLinkExtern, kindParam: "-lexample"} err := prePackageBuild(ctx, task, false) if err != nil { t.Fatal(err) } - if !task.skip || pkg.ExportFile != "" { - t.Fatalf("external link-only pre = skip %v, export %q", task.skip, pkg.ExportFile) + if !task.skip || pkg.ExportFile != "" || pkg.Summary == nil { + t.Fatalf("external link-only pre = skip %v, export %q, summary %#v", task.skip, pkg.ExportFile, pkg.Summary) } if len(pkg.LinkArgs) != 1 || pkg.LinkArgs[0] != "-lexample" { t.Fatalf("external link args = %q, want [-lexample]", pkg.LinkArgs) @@ -280,7 +284,7 @@ func TestBuildSSAPkgsEmptyAndNilEntries(t *testing.T) { buildSSAPkgs(ctx, []ssaBuildEntry{{pkg: pkg}, {pkg: pkg}}) } -func TestPreflightFingerprintsSkippedPackage(t *testing.T) { +func TestPreFingerprintsSkippedPackage(t *testing.T) { pkg := &aPackage{Package: &packages.Package{ ID: "unsafe", PkgPath: "unsafe", @@ -291,11 +295,12 @@ func TestPreflightFingerprintsSkippedPackage(t *testing.T) { buildConf: &Config{Goos: "linux", Goarch: "amd64", ForceRebuild: true}, built: make(map[string]none), } - skip, err := preflightPackageBuild(ctx, newPackageBuildSpec(pkg), false) + task := newPackageBuildTask(pkg) + err := prePackageBuild(ctx, task, false) if err != nil { t.Fatal(err) } - if !skip || pkg.Fingerprint == "" || pkg.Manifest == "" || pkg.Summary == nil { - t.Fatalf("skipped package was not fully prepared: skip=%v fingerprint=%q manifest=%q summary=%#v", skip, pkg.Fingerprint, pkg.Manifest, pkg.Summary) + if !task.skip || pkg.Fingerprint == "" || pkg.Manifest == "" || pkg.Summary == nil { + t.Fatalf("skipped package was not fully prepared: skip=%v fingerprint=%q manifest=%q summary=%#v", task.skip, pkg.Fingerprint, pkg.Manifest, pkg.Summary) } } diff --git a/internal/build/package_summary_test.go b/internal/build/package_summary_test.go index 0e27ad6f56..3f65093777 100644 --- a/internal/build/package_summary_test.go +++ b/internal/build/package_summary_test.go @@ -91,3 +91,38 @@ func TestPackageSummaryCapturesLinkerFacts(t *testing.T) { t.Fatalf("cache summary round trip = %#v, want %#v", loaded, summary) } } + +func TestPackageSummaryEmptyInputs(t *testing.T) { + if got := summarizePackage(nil); got != nil { + t.Fatalf("summarizePackage(nil) = %#v", got) + } + var summary *PackageSummary + if got := summary.metadata(); got != nil { + t.Fatalf("nil summary metadata = %#v", got) + } + if got := summaryFromMetadata(nil, nil); got != nil { + t.Fatalf("summaryFromMetadata(nil, nil) = %#v", got) + } + if got := linkedPackageGlobals(nil); got != nil { + t.Fatalf("linkedPackageGlobals(nil) = %#v", got) + } + if got := linkedPackageGlobals([]*PackageSummary{nil}); len(got) != 0 { + t.Fatalf("linkedPackageGlobals([nil]) = %#v", got) + } + if got := collectFuncInfoSummaries([]*PackageSummary{nil, {FuncInfo: []funcInfoRecord{{}}}}); got != nil { + t.Fatalf("collectFuncInfoSummaries(empty) = %#v", got) + } + if got := collectPCLineInfoSummaries([]*PackageSummary{nil, {PCLineInfo: []pcLineRecord{{}}}}); len(got) != 0 { + t.Fatalf("collectPCLineInfoSummaries(empty) = %#v", got) + } + if got := collectFuncInfoStubRecordsSummaries([]*PackageSummary{nil}, nil); got != nil { + t.Fatalf("collectFuncInfoStubRecordsSummaries(nil) = %#v", got) + } + if got := collectFuncInfoStubRecordsSummaries([]*PackageSummary{nil}, []funcInfoRecord{{symbol: "target"}}); len(got) != 0 { + t.Fatalf("collectFuncInfoStubRecordsSummaries([nil]) = %#v", got) + } + sharedCtx := &context{buildConf: &Config{BuildMode: BuildModeCShared}} + if got := cSharedExportArgsSummaries(sharedCtx, []*PackageSummary{nil}); len(got) != 0 { + t.Fatalf("cSharedExportArgsSummaries([nil]) = %#v", got) + } +}