-
Notifications
You must be signed in to change notification settings - Fork 21
feat(overlays): add archive overlay engine and config model #275
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,186 @@ | ||
| // Copyright (c) Microsoft Corporation. | ||
| // Licensed under the MIT License. | ||
|
|
||
| package sources | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "log/slog" | ||
| "os" | ||
| "path/filepath" | ||
|
|
||
| "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" | ||
| "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" | ||
| "github.com/microsoft/azure-linux-dev-tools/internal/utils/archive" | ||
| "github.com/microsoft/azure-linux-dev-tools/internal/utils/rootfs" | ||
| ) | ||
|
|
||
| // processArchive extracts an archive to a temp directory, applies all overlays, | ||
| // and deterministically repacks it with the original compression, atomically | ||
| // replacing the original via a temp file + rename. It returns true when the | ||
| // archive was repacked. In dry-run mode it returns (false, nil) without touching | ||
| // the archive on disk. | ||
| func processArchive( | ||
| dryRunnable opctx.DryRunnable, | ||
| sourcesDirPath string, | ||
| archiveName string, | ||
| overlays []projectconfig.ComponentOverlay, | ||
| ) (repacked bool, err error) { | ||
| archivePath := filepath.Join(sourcesDirPath, archiveName) | ||
|
|
||
| if dryRunnable.DryRun() { | ||
| slog.Info("Dry run; would apply archive overlays", | ||
| "archive", archiveName, "operations", len(overlays)) | ||
|
|
||
| return false, nil | ||
| } | ||
|
|
||
| // The [archive] package operates exclusively through OS primitives ([os.Root], | ||
| // os.*), so extraction must use a genuine on-disk path regardless of the injected | ||
| // FS implementation (which may be in-memory or otherwise non-OS-backed). Keep | ||
| // that directory outside the sources tree: a process crash could otherwise leave | ||
| // it behind for subsequent loose-file overlay globs to match. | ||
| workDir, err := os.MkdirTemp("", "azldev-archive-overlay-") | ||
| if err != nil { | ||
| return false, fmt.Errorf("creating temp directory:\n%w", err) | ||
| } | ||
|
|
||
| defer func() { | ||
| if removeErr := os.RemoveAll(workDir); removeErr != nil { | ||
| slog.Warn("Failed to clean up archive work directory", "error", removeErr) | ||
| } | ||
| }() | ||
|
|
||
| // Extract the archive; compression is inferred from the filename extension. | ||
| // Fail on entry types we can't repack (hardlinks, devices, ...) so they | ||
| // aren't silently dropped from the repacked archive. | ||
| if err := archive.ExtractAuto(archivePath, workDir, archive.WithErrorOnUnsupportedEntry()); err != nil { | ||
| return false, fmt.Errorf("extracting archive:\n%w", err) | ||
| } | ||
|
|
||
| // Determine the root of the extracted content. Most source archives unpack to | ||
| // a single top-level directory (e.g., "pkg-1.0/"), which is used as the root. | ||
| extractRoot, err := resolveExtractRoot(workDir) | ||
| if err != nil { | ||
| return false, fmt.Errorf("resolving extract root:\n%w", err) | ||
| } | ||
|
|
||
| // Confine an OS-backed FS to the extract root so file overlays reuse the same | ||
| // machinery as loose-file overlays. | ||
| extractFS, err := rootfs.New(extractRoot) | ||
| if err != nil { | ||
| return false, fmt.Errorf("confining FS to extract root:\n%w", err) | ||
| } | ||
|
|
||
| defer func() { | ||
| if closeErr := extractFS.Close(); closeErr != nil { | ||
| slog.Warn("Failed to close extract-root FS", "error", closeErr) | ||
| } | ||
| }() | ||
|
|
||
| // Apply each overlay in order. Archive overlays (file-remove / file-search-replace, | ||
| // see [projectconfig.ComponentOverlay.ModifiesArchive]) operate solely on the | ||
| // destination tree, so extractFS is passed as both source and destination FS. | ||
| for _, overlay := range overlays { | ||
| if err := applyNonSpecOverlay(dryRunnable, extractFS, extractFS, overlay); err != nil { | ||
| return false, fmt.Errorf("applying %#q operation:\n%w", overlay.Type, err) | ||
| } | ||
| } | ||
|
|
||
| // Deterministically repack the archive, reusing the original compression, and | ||
| // atomically replace the original (see [repackArchiveAtomic]). | ||
| if err := repackArchiveAtomic(archivePath, archiveName, workDir); err != nil { | ||
| return false, err | ||
| } | ||
|
|
||
| slog.Info("Archive overlay applied", "archive", archiveName) | ||
|
|
||
| return true, nil | ||
| } | ||
|
|
||
| // repackArchiveAtomic deterministically repacks workDir into an archive that | ||
| // replaces archivePath, reusing the compression inferred from archiveName. | ||
| // | ||
| // To avoid corrupting the fetched source archive on a mid-write failure (disk | ||
| // full, permission error, etc.), it repacks to a temp file in the same directory | ||
| // and atomically renames it over the original only on success. Repacking directly | ||
| // over archivePath would truncate it first, leaving the workspace unrecoverable | ||
| // without refetching if the repack then failed. | ||
| func repackArchiveAtomic(archivePath, archiveName, workDir string) (err error) { | ||
| archiveInfo, err := os.Stat(archivePath) | ||
| if err != nil { | ||
| return fmt.Errorf("stating original archive %#q:\n%w", archiveName, err) | ||
| } | ||
|
|
||
| originalPerm := archiveInfo.Mode().Perm() | ||
|
|
||
| // Fall back to the extension only if the archive is empty (nothing to sniff). | ||
| extComp, err := archive.DetectCompression(archiveName) | ||
| if err != nil { | ||
| return fmt.Errorf("detecting compression for %#q:\n%w", archiveName, err) | ||
| } | ||
|
|
||
| comp, err := archive.SniffCompressionFromFile(archivePath, extComp) | ||
| if err != nil { | ||
| return fmt.Errorf("sniffing compression for %#q:\n%w", archiveName, err) | ||
| } | ||
|
|
||
| tmpFile, err := os.CreateTemp(filepath.Dir(archivePath), "."+filepath.Base(archiveName)+".repack-*") | ||
| if err != nil { | ||
| return fmt.Errorf("creating temp archive:\n%w", err) | ||
| } | ||
|
|
||
| tmpPath := tmpFile.Name() | ||
|
|
||
| // Close the handle immediately; CreateDeterministicArchive truncates and | ||
| // reopens this uniquely-created path. | ||
| if closeErr := tmpFile.Close(); closeErr != nil { | ||
| _ = os.Remove(tmpPath) | ||
|
|
||
| return fmt.Errorf("closing temp archive %#q:\n%w", tmpPath, closeErr) | ||
| } | ||
|
|
||
| // Clean up the temp file unless it was successfully renamed over the original. | ||
| repackedOK := false | ||
|
|
||
| defer func() { | ||
| if !repackedOK { | ||
| if removeErr := os.Remove(tmpPath); removeErr != nil && !os.IsNotExist(removeErr) { | ||
| slog.Warn("Failed to clean up temp archive", "path", tmpPath, "error", removeErr) | ||
| } | ||
| } | ||
| }() | ||
|
|
||
| if err := archive.CreateDeterministicArchive(tmpPath, workDir, comp); err != nil { | ||
| return fmt.Errorf("repacking archive:\n%w", err) | ||
| } | ||
|
|
||
| if err := os.Chmod(tmpPath, originalPerm); err != nil { | ||
| return fmt.Errorf("restoring permissions on repacked archive %#q:\n%w", archiveName, err) | ||
| } | ||
|
|
||
| if err := os.Rename(tmpPath, archivePath); err != nil { | ||
| return fmt.Errorf("replacing archive %#q with repacked archive:\n%w", archivePath, err) | ||
| } | ||
|
|
||
| repackedOK = true | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // resolveExtractRoot returns the effective root of an extracted archive. If workDir | ||
| // contains exactly one entry and that entry is a directory (the common case for | ||
| // source archives like "pkg-1.0/"), that subdirectory is returned; otherwise workDir | ||
| // itself is returned. | ||
| func resolveExtractRoot(workDir string) (string, error) { | ||
| entries, err := os.ReadDir(workDir) | ||
| if err != nil { | ||
| return "", fmt.Errorf("reading extracted directory:\n%w", err) | ||
| } | ||
|
|
||
| if len(entries) == 1 && entries[0].IsDir() { | ||
| return filepath.Join(workDir, entries[0].Name()), nil | ||
| } | ||
|
|
||
| return workDir, nil | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.