From 3ca83e5bb46801369fdb920c957a560eed99652c Mon Sep 17 00:00:00 2001 From: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> Date: Wed, 6 May 2026 08:46:05 -0500 Subject: [PATCH] Enhance RPM audit functionality and improve evidence file uploads - Added workspace directory resolution for RPM audit uploads in `packagePromote`. - Updated `UploadPromotionEvidenceFiles` to accept a workspace directory parameter for resolving relative paths to sibling RPM audit JSON files. - Implemented `rpmAuditPathsFromTestResults` to discover and append sibling audit files based on test results. - Introduced new tests for loading and writing RPM audit records, ensuring proper handling of audit data alongside built RPMs. --- internal/cli/cli.go | 7 +- internal/cli/package_promotion.go | 59 +++++++++- internal/cli/release.go | 17 +++ internal/packageexec/audit.go | 172 +++++++++++++++++++++++++++++ internal/packageexec/audit_test.go | 154 ++++++++++++++++++++++++++ internal/packageexec/execute.go | 53 +++++++-- 6 files changed, 447 insertions(+), 15 deletions(-) create mode 100644 internal/packageexec/audit.go create mode 100644 internal/packageexec/audit_test.go diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 915e8f1..d0c9763 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -548,7 +548,12 @@ func packagePromote(c *cli.Context) error { releaseTagsByRuntime[runtimeName] = tag } } - if err := UploadPromotionEvidenceFiles(uploader, releaseTagsByRuntime, testResultsPath); err != nil { + workspaceDir, wdErr := os.Getwd() + if wdErr != nil { + stderr.Warn("failed to resolve working directory for RPM audit upload", "error", wdErr) + workspaceDir = "" + } + if err := UploadPromotionEvidenceFiles(uploader, releaseTagsByRuntime, testResultsPath, workspaceDir); err != nil { stderr.Error("failed to upload promotion evidence files", "error", err) return err } diff --git a/internal/cli/package_promotion.go b/internal/cli/package_promotion.go index ca9aa4f..c959a1b 100644 --- a/internal/cli/package_promotion.go +++ b/internal/cli/package_promotion.go @@ -140,10 +140,18 @@ func BuildPromotionUploader(configPath, token string, manifest PackageManifest) return &runtimeReleaseUploader{clientsByRuntime: clientsByRuntime}, nil } +// UploadPromotionEvidenceFiles uploads pipeline evidence files (manifests, results, +// and sibling RPM audit JSONs) to each runtime's GitHub release. +// +// workspaceDir is used to resolve relative PackagePath values found inside the +// test-results file when looking for sibling *.rpm.audit.json files. Pass the +// process working directory (os.Getwd()) from the caller; an empty string +// disables relative-path resolution. func UploadPromotionEvidenceFiles( uploader promotion.ReleaseAssetUploader, releaseTagsByRuntime map[string]string, testResultsPath string, + workspaceDir string, ) error { if uploader == nil { return nil @@ -152,13 +160,17 @@ func UploadPromotionEvidenceFiles( return nil } - artifactsDir := filepath.Dir(strings.TrimSpace(testResultsPath)) + cleanResultsPath := strings.TrimSpace(testResultsPath) + artifactsDir := filepath.Dir(cleanResultsPath) evidenceFiles := []string{ - strings.TrimSpace(testResultsPath), + cleanResultsPath, filepath.Join(artifactsDir, "package-build-results.json"), filepath.Join(artifactsDir, "package-manifest.built.json"), filepath.Join(artifactsDir, "package-manifest.tested.json"), } + // Append sibling *.rpm.audit.json files discovered from the test results. + evidenceFiles = append(evidenceFiles, rpmAuditPathsFromTestResults(workspaceDir, cleanResultsPath)...) + seen := make(map[string]struct{}) for _, runtimeName := range sortedRuntimeKeys(releaseTagsByRuntime) { releaseTag := releaseTagsByRuntime[runtimeName] @@ -189,6 +201,49 @@ func UploadPromotionEvidenceFiles( return nil } +// rpmAuditPathsFromTestResults reads testResultsPath and returns the absolute +// paths of any sibling *.rpm.audit.json files that exist on disk alongside the +// built RPM packages recorded as passed in the test results. +func rpmAuditPathsFromTestResults(workspaceDir, testResultsPath string) []string { + if strings.TrimSpace(testResultsPath) == "" { + return nil + } + data, err := os.ReadFile(testResultsPath) + if err != nil { + return nil + } + var results promotion.TestResultsFile + if err := json.Unmarshal(data, &results); err != nil { + return nil + } + + var paths []string + seen := make(map[string]struct{}) + for _, r := range results.Results { + if strings.ToLower(strings.TrimSpace(r.Target)) != "rpm" { + continue + } + pkgPath := strings.TrimSpace(r.PackagePath) + if pkgPath == "" { + continue + } + if !filepath.IsAbs(pkgPath) && strings.TrimSpace(workspaceDir) != "" { + pkgPath = filepath.Join(workspaceDir, pkgPath) + } + auditPath := pkgPath + ".audit.json" + if _, ok := seen[auditPath]; ok { + continue + } + seen[auditPath] = struct{}{} + info, statErr := os.Stat(auditPath) + if statErr != nil || info.IsDir() || info.Size() == 0 { + continue + } + paths = append(paths, auditPath) + } + return paths +} + func sortedRuntimeKeys(byRuntime map[string]string) []string { keys := make([]string, 0, len(byRuntime)) for runtimeName := range byRuntime { diff --git a/internal/cli/release.go b/internal/cli/release.go index c3aebf6..d8a7b06 100644 --- a/internal/cli/release.go +++ b/internal/cli/release.go @@ -383,6 +383,12 @@ func (rm *ReleaseManager) collectPackageArtifacts(outputDir, runtimeName, versio } if packageArtifactFilename(filepath.Base(fullPath)) { appendFile(fullPath, info) + if strings.HasSuffix(strings.ToLower(filepath.Base(fullPath)), ".rpm") { + auditPath := fullPath + ".audit.json" + if ainfo, aerr := os.Stat(auditPath); aerr == nil && !ainfo.IsDir() && ainfo.Size() > 0 { + appendFile(auditPath, ainfo) + } + } } } @@ -857,6 +863,17 @@ func getFileInfo(filename, url string, downloadResults []runtime.DownloadResult) }, nil } + // Sibling audit for an RPM (e.g. OSPO-nodejs-….rpm.audit.json) — same platform key as the .rpm binary. + if strings.HasSuffix(strings.ToLower(filename), ".rpm.audit.json") { + baseRPM := strings.TrimSuffix(filename, ".audit.json") + return &fileInfo{ + Version: extractVersionFromFilename(baseRPM), + OS: "linux", + Arch: packageArchFromFilename(baseRPM), + Type: "artifact", + }, nil + } + // Related verification/proof files may append known suffixes to the binary name. // Try resolving back to the original binary filename and map platform via downloadResults. baseFilename := filename diff --git a/internal/packageexec/audit.go b/internal/packageexec/audit.go new file mode 100644 index 0000000..0aacf32 --- /dev/null +++ b/internal/packageexec/audit.go @@ -0,0 +1,172 @@ +package packageexec + +import ( + "encoding/json" + "fmt" + "os" + "strings" + "time" + + "github.com/clean-dependency-project/cdprun/internal/packaging" +) + +// Phase 1 — load download audit (read-only). +// Phase 2 — assemble build + test metadata after container runs. +// Phase 3 — write sibling {rpm}.audit.json next to the .rpm. + +// WriteRPMAuditRecord writes a single JSON file next to the built RPM containing +// inlined download verification (from the upstream tarball's .audit.json), +// build container metadata, and test container output. +func WriteRPMAuditRecord( + workspaceDir string, + runID string, + target Target, + build packaging.BuildResult, + execSpec TargetExecutorSpec, + testStdout string, + testStderr string, + testErr error, +) error { + if strings.ToLower(strings.TrimSpace(target.Target)) != "rpm" { + return fmt.Errorf("WriteRPMAuditRecord: target is not rpm") + } + rpmAbs, err := resolvePath(workspaceDir, build.PackagePath) + if err != nil { + return fmt.Errorf("resolve package path: %w", err) + } + auditPath := rpmAbs + ".audit.json" + + downloadObj, err := loadDownloadAuditObject(workspaceDir, target.InputPath) + if err != nil { + downloadObj = fallbackDownloadObject(target) + } + + testStatus := "success" + overallVer := "success" + if testErr != nil { + testStatus = "failed" + overallVer = "failed" + } + + testOutput, errParse := parseTestOutputJSON(testStdout) + testSection := map[string]interface{}{ + "image": execSpec.Test.Image, + "script": execSpec.Test.Script, + "status": testStatus, + } + if strings.TrimSpace(testStderr) != "" { + testSection["stderr"] = strings.TrimSpace(testStderr) + } + if testErr != nil { + testSection["error"] = testErr.Error() + } + if errParse == nil && testOutput != nil { + testSection["output"] = testOutput + } else if strings.TrimSpace(testStdout) != "" { + testSection["output_raw"] = strings.TrimSpace(testStdout) + } + + record := map[string]interface{}{ + "timestamp": time.Now().UTC().Format(time.RFC3339), + "run_id": runID, + "runtime": target.Runtime, + "version": target.Version, + "package_type": "rpm", + "package_name": build.PackageName, + "package_filename": build.PackageFilename, + "package_path": build.PackagePath, + "package_sha256": build.PackageSHA256, + "install_prefix": target.InstallPrefix, + "arch": build.Arch, + "release": build.Release, + "download": downloadObj, + "build": buildSection(execSpec.Build, build), + "test": testSection, + "verification_type": "package-build-test-rpm", + "verification_status": overallVer, + } + + enc, err := json.MarshalIndent(record, "", " ") + if err != nil { + return fmt.Errorf("marshal rpm audit: %w", err) + } + if err := os.WriteFile(auditPath, enc, 0644); err != nil { + return fmt.Errorf("write rpm audit file: %w", err) + } + return nil +} + +func buildSection(spec ContainerSpec, build packaging.BuildResult) map[string]interface{} { + out := map[string]interface{}{ + "image": spec.Image, + "script": spec.Script, + "status": "success", + } + if build.Duration > 0 { + out["duration_ms"] = build.Duration.Milliseconds() + } + return out +} + +func loadDownloadAuditObject(workspaceDir, inputPath string) (map[string]interface{}, error) { + inputPath = strings.TrimSpace(inputPath) + if inputPath == "" { + return nil, fmt.Errorf("empty input path") + } + full, err := resolvePath(workspaceDir, inputPath) + if err != nil { + return nil, err + } + auditPath := full + ".audit.json" + data, err := os.ReadFile(auditPath) + if err != nil { + return nil, err + } + var m map[string]interface{} + if err := json.Unmarshal(data, &m); err != nil { + return nil, err + } + return m, nil +} + +func fallbackDownloadObject(target Target) map[string]interface{} { + return map[string]interface{}{ + "input_path": target.InputPath, + "input_sha256": target.InputSHA256, + "audit_file_missing": true, + } +} + +func parseTestOutputJSON(stdout string) (map[string]interface{}, error) { + s := strings.TrimSpace(stdout) + if s == "" { + return nil, fmt.Errorf("empty test stdout") + } + var m map[string]interface{} + if err := json.Unmarshal([]byte(s), &m); err != nil { + return nil, err + } + return m, nil +} + +// DownloadAuditPathForInput returns the path to the sibling audit file for an +// input tarball (workspace-relative or absolute), for tests and tooling. +func DownloadAuditPathForInput(workspaceDir, inputPath string) (string, error) { + full, err := resolvePath(workspaceDir, inputPath) + if err != nil { + return "", err + } + return full + ".audit.json", nil +} + +// RPMAuditPath returns the RPM audit file path next to the given RPM path. +func RPMAuditPath(rpmPath string) string { + return rpmPath + ".audit.json" +} + +// IsRPMAuditFilename reports whether name is a sibling audit for an RPM +// (e.g. OSPO-nodejs-22.22.2-1.amzn2023.x86_64.rpm.audit.json). +func IsRPMAuditFilename(name string) bool { + lower := strings.ToLower(strings.TrimSpace(name)) + return strings.HasSuffix(lower, ".rpm.audit.json") +} diff --git a/internal/packageexec/audit_test.go b/internal/packageexec/audit_test.go new file mode 100644 index 0000000..63e5afc --- /dev/null +++ b/internal/packageexec/audit_test.go @@ -0,0 +1,154 @@ +package packageexec + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/clean-dependency-project/cdprun/internal/packaging" +) + +func TestLoadDownloadAuditObject_MissingFallback(t *testing.T) { + tmp := t.TempDir() + target := Target{ + InputPath: "downloads/missing.tar.xz", + InputSHA256: "abc", + } + _, err := loadDownloadAuditObject(tmp, target.InputPath) + if err == nil { + t.Fatalf("expected error for missing file") + } + fb := fallbackDownloadObject(target) + if fb["input_sha256"] != "abc" { + t.Fatalf("fallback sha = %v", fb["input_sha256"]) + } +} + +func TestLoadDownloadAuditObject_ReadsFile(t *testing.T) { + tmp := t.TempDir() + dlDir := filepath.Join(tmp, "downloads") + if err := os.MkdirAll(dlDir, 0755); err != nil { + t.Fatal(err) + } + tarball := filepath.Join(dlDir, "node-v1.0.0-linux-x64.tar.xz") + if err := os.WriteFile(tarball, []byte("x"), 0644); err != nil { + t.Fatal(err) + } + auditPath := tarball + ".audit.json" + payload := map[string]interface{}{ + "checksum_verified": true, + "file_name": "node-v1.0.0-linux-x64.tar.xz", + } + b, _ := json.Marshal(payload) + if err := os.WriteFile(auditPath, b, 0644); err != nil { + t.Fatal(err) + } + + got, err := loadDownloadAuditObject(tmp, "downloads/node-v1.0.0-linux-x64.tar.xz") + if err != nil { + t.Fatalf("loadDownloadAuditObject: %v", err) + } + if got["checksum_verified"] != true { + t.Fatalf("got %#v", got) + } +} + +func TestParseTestOutputJSON(t *testing.T) { + raw := `{"success":true,"passed":2,"failed":0,"tests":[]}` + got, err := parseTestOutputJSON(raw) + if err != nil { + t.Fatal(err) + } + if got["success"] != true { + t.Fatalf("got %#v", got) + } + _, err = parseTestOutputJSON("not json") + if err == nil { + t.Fatal("expected error") + } +} + +func TestWriteRPMAuditRecord(t *testing.T) { + tmp := t.TempDir() + pkgDir := filepath.Join(tmp, "packages") + if err := os.MkdirAll(pkgDir, 0755); err != nil { + t.Fatal(err) + } + rpmPath := filepath.Join(pkgDir, "OSPO-nodejs-1.0.0-1.amzn2023.x86_64.rpm") + if err := os.WriteFile(rpmPath, []byte("fake-rpm"), 0644); err != nil { + t.Fatal(err) + } + + dlDir := filepath.Join(tmp, "downloads") + if err := os.MkdirAll(dlDir, 0755); err != nil { + t.Fatal(err) + } + tarball := filepath.Join(dlDir, "node-v1.0.0-linux-x64.tar.xz") + if err := os.WriteFile(tarball, []byte("x"), 0644); err != nil { + t.Fatal(err) + } + dlAudit := tarball + ".audit.json" + if err := os.WriteFile(dlAudit, []byte(`{"verification_status":"success","checksum_verified":true}`), 0644); err != nil { + t.Fatal(err) + } + + execSpec := TargetExecutorSpec{ + Build: ContainerSpec{Image: "amazonlinux:2023", Shell: "/bin/bash", Script: "build.sh"}, + Test: ContainerSpec{Image: "amazonlinux:2023", Shell: "/bin/bash", Script: "test.sh"}, + } + build := packaging.BuildResult{ + Runtime: "nodejs", + Version: "1.0.0", + PackageType: packaging.PackageTypeRPM, + PackageName: "OSPO-nodejs", + Release: "1", + Arch: "x86_64", + InstallPrefix: "/export/apps/citools/OSPO-nodejs/1.0.0", + PackageFilename: filepath.Base(rpmPath), + PackagePath: "packages/" + filepath.Base(rpmPath), + PackageSHA256: "deadbeef", + Duration: 2 * time.Second, + Input: packaging.InputInfo{ + Path: "downloads/node-v1.0.0-linux-x64.tar.xz", + SHA256: "abc", + }, + } + target := Target{ + Runtime: "nodejs", + Version: "1.0.0", + Target: "rpm", + InputPath: "downloads/node-v1.0.0-linux-x64.tar.xz", + InputSHA256: "abc", + PackageName: "OSPO-nodejs", + InstallPrefix: "/export/apps/citools/OSPO-nodejs/1.0.0", + } + + testOut := `{"success":true,"passed":1,"failed":0}` + if err := WriteRPMAuditRecord(tmp, "run-1", target, build, execSpec, testOut, "", nil); err != nil { + t.Fatal(err) + } + auditData, err := os.ReadFile(rpmPath + ".audit.json") + if err != nil { + t.Fatal(err) + } + var decoded map[string]interface{} + if err := json.Unmarshal(auditData, &decoded); err != nil { + t.Fatal(err) + } + if decoded["verification_status"] != "success" { + t.Fatalf("status = %v", decoded["verification_status"]) + } + dl := decoded["download"].(map[string]interface{}) + if dl["checksum_verified"] != true { + t.Fatalf("download embed = %#v", dl) + } +} + +func TestWriteRPMAuditRecord_WrongTarget(t *testing.T) { + err := WriteRPMAuditRecord(t.TempDir(), "r", Target{Target: "apk"}, packaging.BuildResult{}, TargetExecutorSpec{}, "", "", nil) + if err == nil { + t.Fatal("expected error") + } +} diff --git a/internal/packageexec/execute.go b/internal/packageexec/execute.go index b0a85af..d08e572 100644 --- a/internal/packageexec/execute.go +++ b/internal/packageexec/execute.go @@ -103,7 +103,40 @@ func TestBuiltPackages(ctx context.Context, runner packaging.CommandRunner, opts continue } - err := testOneInContainer(ctx, runner, normalized, buildRecord.Target, buildRecord.Build) + execSpec, execErr := normalized.Executors.Resolve(buildRecord.Target.Runtime, buildRecord.Target.Target) + if execErr != nil { + results.Results = append(results.Results, promotion.TestResult{ + Runtime: buildRecord.Target.Runtime, + Version: buildRecord.Target.Version, + Target: buildRecord.Target.Target, + InputSHA256: buildRecord.Target.InputSHA256, + PackageName: buildRecord.Target.PackageName, + InstallPrefix: buildRecord.Target.InstallPrefix, + Passed: false, + PackageFilename: buildRecord.Build.PackageFilename, + PackagePath: buildRecord.Build.PackagePath, + PackageSHA256: buildRecord.Build.PackageSHA256, + }) + failed++ + continue + } + + testStdout, testStderr, testErr := testOneInContainer(ctx, runner, normalized, buildRecord.Target, buildRecord.Build, execSpec) + if strings.EqualFold(strings.TrimSpace(buildRecord.Target.Target), "rpm") { + if werr := WriteRPMAuditRecord( + normalized.WorkspaceDir, + runID, + buildRecord.Target, + buildRecord.Build, + execSpec, + testStdout, + testStderr, + testErr, + ); werr != nil { + return results, fmt.Errorf("write rpm audit: %w", werr) + } + } + size := int64(0) if absPath, statErr := resolvePath(normalized.WorkspaceDir, buildRecord.Build.PackagePath); statErr == nil { if info, infoErr := os.Stat(absPath); infoErr == nil { @@ -117,14 +150,14 @@ func TestBuiltPackages(ctx context.Context, runner packaging.CommandRunner, opts InputSHA256: buildRecord.Target.InputSHA256, PackageName: buildRecord.Target.PackageName, InstallPrefix: buildRecord.Target.InstallPrefix, - Passed: err == nil, + Passed: testErr == nil, PackageFilename: buildRecord.Build.PackageFilename, PackagePath: buildRecord.Build.PackagePath, PackageSHA256: buildRecord.Build.PackageSHA256, PackageSize: size, } results.Results = append(results.Results, result) - if err != nil { + if testErr != nil { failed++ } } @@ -191,16 +224,12 @@ func buildOneInContainer(ctx context.Context, runner packaging.CommandRunner, op return build, nil } -func testOneInContainer(ctx context.Context, runner packaging.CommandRunner, opts Options, target Target, build packaging.BuildResult) error { +func testOneInContainer(ctx context.Context, runner packaging.CommandRunner, opts Options, target Target, build packaging.BuildResult, execSpec TargetExecutorSpec) (stdout string, stderr string, err error) { packagePath, err := resolvePath(opts.WorkspaceDir, build.PackagePath) if err != nil { - return err + return "", "", err } - execSpec, err := opts.Executors.Resolve(target.Runtime, target.Target) - if err != nil { - return err - } args := []string{ "run", "--rm", "--platform=" + opts.DockerPlatform, "-v", opts.WorkspaceDir + ":/workspace", @@ -211,11 +240,11 @@ func testOneInContainer(ctx context.Context, runner packaging.CommandRunner, opt execSpec.Test.Image, execSpec.Test.Shell, "-lc", execSpec.Test.Script, ) - _, stderr, runErr := runner.Run(ctx, "", "docker", args, nil) + stdout, stderr, runErr := runner.Run(ctx, "", "docker", args, nil) if runErr != nil { - return fmt.Errorf("%s test failed: %w (stderr=%s)", target.Target, runErr, strings.TrimSpace(stderr)) + return stdout, stderr, fmt.Errorf("%s test failed: %w (stderr=%s)", target.Target, runErr, strings.TrimSpace(stderr)) } - return nil + return stdout, stderr, nil } func buildEnvForTarget(opts Options, target Target) []string {