diff --git a/cmd/collect_github_helpers.go b/cmd/collect_github_helpers.go new file mode 100644 index 0000000..7ed964a --- /dev/null +++ b/cmd/collect_github_helpers.go @@ -0,0 +1,16 @@ +package cmd + +import ( + "github.com/google/go-github/v39/github" +) + +func findChecksumAsset(assets []*github.ReleaseAsset) *github.ReleaseAsset { + for _, asset := range assets { + // A common convention for checksum files. + // A more robust solution could be configured by the user. + if asset.GetName() == "checksums.txt" || asset.GetName() == "SHA256SUMS" { + return asset + } + } + return nil +} diff --git a/cmd/collect_github_release_subcommand.go b/cmd/collect_github_release_subcommand.go index 9e975a6..e4d5fa9 100644 --- a/cmd/collect_github_release_subcommand.go +++ b/cmd/collect_github_release_subcommand.go @@ -7,19 +7,18 @@ import ( "os" "path/filepath" - "github.com/Snider/Borg/pkg/datanode" borg_github "github.com/Snider/Borg/pkg/github" + "bytes" "github.com/google/go-github/v39/github" "github.com/spf13/cobra" - "golang.org/x/mod/semver" ) func NewCollectGithubReleaseCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "release [repository-url]", - Short: "Download the latest release of a file from GitHub releases", - Long: `Download the latest release of a file from GitHub releases. If the file or URL has a version number, it will check for a higher version and download it if found.`, - Args: cobra.ExactArgs(1), + Use: "release [tag]", + Short: "Download a release from GitHub", + Long: `Download a specific release from GitHub. If no tag is specified, the latest release will be downloaded.`, + Args: cobra.RangeArgs(1, 2), RunE: func(cmd *cobra.Command, args []string) error { logVal := cmd.Context().Value("logger") log, ok := logVal.(*slog.Logger) @@ -28,124 +27,105 @@ func NewCollectGithubReleaseCmd() *cobra.Command { } repoURL := args[0] outputDir, _ := cmd.Flags().GetString("output") - pack, _ := cmd.Flags().GetBool("pack") - file, _ := cmd.Flags().GetString("file") - version, _ := cmd.Flags().GetString("version") + assetsOnly, _ := cmd.Flags().GetBool("assets-only") + pattern, _ := cmd.Flags().GetString("pattern") + verifyChecksums, _ := cmd.Flags().GetBool("verify-checksums") - _, err := GetRelease(log, repoURL, outputDir, pack, file, version) - return err - }, - } - cmd.PersistentFlags().String("output", ".", "Output directory for the downloaded file") - cmd.PersistentFlags().Bool("pack", false, "Pack all assets into a DataNode") - cmd.PersistentFlags().String("file", "", "The file to download from the release") - cmd.PersistentFlags().String("version", "", "The version to check against") - return cmd -} - -func init() { - collectGithubCmd.AddCommand(NewCollectGithubReleaseCmd()) -} - -func GetRelease(log *slog.Logger, repoURL string, outputDir string, pack bool, file string, version string) (*github.RepositoryRelease, error) { - owner, repo, err := borg_github.ParseRepoFromURL(repoURL) - if err != nil { - return nil, fmt.Errorf("failed to parse repository url: %w", err) - } + owner, repo, err := borg_github.ParseRepoFromURL(repoURL) + if err != nil { + return fmt.Errorf("failed to parse repository url: %w", err) + } - release, err := borg_github.GetLatestRelease(owner, repo) - if err != nil { - return nil, fmt.Errorf("failed to get latest release: %w", err) - } + var release *github.RepositoryRelease + if len(args) == 2 { + tag := args[1] + log.Info("getting release by tag", "tag", tag) + release, err = borg_github.GetReleaseByTag(owner, repo, tag) + if err != nil { + return fmt.Errorf("failed to get release '%s': %w", tag, err) + } + } else { + log.Info("getting latest release") + release, err = borg_github.GetLatestRelease(owner, repo) + if err != nil { + return fmt.Errorf("failed to get latest release: %w", err) + } + } + if release == nil { + return errors.New("release not found") + } - log.Info("found latest release", "tag", release.GetTagName()) + log.Info("found release", "tag", release.GetTagName()) - if version != "" { - tag := release.GetTagName() - if !semver.IsValid(tag) { - log.Info("latest release tag is not a valid semantic version, skipping comparison", "tag", tag) - } else { - if !semver.IsValid(version) { - return nil, fmt.Errorf("invalid version string: %s", version) - } - if semver.Compare(tag, version) <= 0 { - log.Info("latest release is not newer than the provided version", "latest", tag, "provided", version) - return nil, nil + tag := release.GetTagName() + releaseDir := filepath.Join(outputDir, tag) + if err := os.MkdirAll(releaseDir, 0755); err != nil { + return fmt.Errorf("failed to create release directory: %w", err) } - } - } - if pack { - dn := datanode.New() - var failedAssets []string - for _, asset := range release.Assets { - log.Info("downloading asset", "name", asset.GetName()) - data, err := borg_github.DownloadReleaseAsset(asset) - if err != nil { - log.Error("failed to download asset", "name", asset.GetName(), "err", err) - failedAssets = append(failedAssets, asset.GetName()) - continue + if !assetsOnly { + releaseNotes := release.GetBody() + if err := os.WriteFile(filepath.Join(releaseDir, "RELEASE.md"), []byte(releaseNotes), 0644); err != nil { + return fmt.Errorf("failed to write release notes: %w", err) + } } - dn.AddData(asset.GetName(), data) - } - if len(failedAssets) > 0 { - return nil, fmt.Errorf("failed to download assets: %v", failedAssets) - } - tar, err := dn.ToTar() - if err != nil { - return nil, fmt.Errorf("failed to create datanode: %w", err) - } + for _, asset := range release.Assets { + if pattern != "" { + matched, err := filepath.Match(pattern, asset.GetName()) + if err != nil { + return fmt.Errorf("invalid pattern: %w", err) + } + if !matched { + continue + } + } - if err := os.MkdirAll(outputDir, 0755); err != nil { - return nil, fmt.Errorf("failed to create output directory: %w", err) - } - basename := release.GetTagName() - if basename == "" { - basename = "release" - } - outputFile := filepath.Join(outputDir, basename+".dat") + log.Info("downloading asset", "name", asset.GetName()) + assetPath := filepath.Join(releaseDir, asset.GetName()) + file, err := os.Create(assetPath) + if err != nil { + return fmt.Errorf("failed to create asset file: %w", err) + } + defer file.Close() - err = os.WriteFile(outputFile, tar, 0644) - if err != nil { - return nil, fmt.Errorf("failed to write datanode: %w", err) - } - log.Info("datanode saved", "path", outputFile) - } else { - if len(release.Assets) == 0 { - log.Info("no assets found in the latest release") - return nil, nil - } - var assetToDownload *github.ReleaseAsset - if file != "" { - for _, asset := range release.Assets { - if asset.GetName() == file { - assetToDownload = asset - break + var checksumData []byte + if verifyChecksums { + checksumAsset := findChecksumAsset(release.Assets) + if checksumAsset == nil { + log.Warn("checksum file not found in release", "tag", tag) + } else { + buf := new(bytes.Buffer) + err := borg_github.DownloadReleaseAsset(checksumAsset, buf) + if err != nil { + log.Error("failed to download checksum file", "name", checksumAsset.GetName(), "err", err) + } else { + checksumData = buf.Bytes() + } + } + } + + if verifyChecksums && checksumData != nil { + err = borg_github.DownloadReleaseAssetWithChecksum(asset, checksumData, file) + } else { + err = borg_github.DownloadReleaseAsset(asset, file) + } + + if err != nil { + log.Error("failed to download asset", "name", asset.GetName(), "err", err) + continue } } - if assetToDownload == nil { - return nil, fmt.Errorf("asset not found in the latest release: %s", file) - } - } else { - assetToDownload = release.Assets[0] - } - if outputDir != "" { - if err := os.MkdirAll(outputDir, 0755); err != nil { - return nil, fmt.Errorf("failed to create output directory: %w", err) - } - } - outputPath := filepath.Join(outputDir, assetToDownload.GetName()) - log.Info("downloading asset", "name", assetToDownload.GetName()) - data, err := borg_github.DownloadReleaseAsset(assetToDownload) - if err != nil { - return nil, fmt.Errorf("failed to download asset: %w", err) - } - err = os.WriteFile(outputPath, data, 0644) - if err != nil { - return nil, fmt.Errorf("failed to write asset to file: %w", err) - } - log.Info("asset downloaded", "path", outputPath) + return nil + }, } - return release, nil + cmd.Flags().String("output", "releases", "Output directory for the releases") + cmd.Flags().Bool("assets-only", false, "Only download assets, skip release notes") + cmd.Flags().String("pattern", "", "Filter assets by filename pattern") + cmd.Flags().Bool("verify-checksums", false, "Verify checksums after download") + return cmd +} + +func init() { + collectGithubCmd.AddCommand(NewCollectGithubReleaseCmd()) } diff --git a/cmd/collect_github_release_subcommand_test.go b/cmd/collect_github_release_subcommand_test.go new file mode 100644 index 0000000..b873d17 --- /dev/null +++ b/cmd/collect_github_release_subcommand_test.go @@ -0,0 +1,161 @@ +package cmd + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "testing" + + borg_github "github.com/Snider/Borg/pkg/github" + "github.com/Snider/Borg/pkg/mocks" + "github.com/google/go-github/v39/github" + "github.com/stretchr/testify/assert" +) + +func TestCollectGithubReleaseCmd(t *testing.T) { + assetContent := "asset content" + hasher := sha256.New() + hasher.Write([]byte(assetContent)) + correctChecksum := hex.EncodeToString(hasher.Sum(nil)) + checksumsFileContent := fmt.Sprintf("%s asset1.zip\n", correctChecksum) + + // Mock the GitHub API + responses := map[string]*http.Response{ + "https://api.github.com/repos/owner/repo/releases/latest": { + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"tag_name": "v1.0.0", "body": "Release notes", "assets": [{"name": "asset1.zip", "browser_download_url": "http://localhost/asset1.zip"}, {"name": "checksums.txt", "browser_download_url": "http://localhost/checksums.txt"}]}`))), + Header: http.Header{}, + }, + "https://api.github.com/repos/owner/repo/releases/tags/v1.0.0": { + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"tag_name": "v1.0.0", "body": "Release notes", "assets": [{"name": "asset1.zip", "browser_download_url": "http://localhost/asset1.zip"}, {"name": "checksums.txt", "browser_download_url": "http://localhost/checksums.txt"}]}`))), + Header: http.Header{}, + }, + "http://localhost/asset1.zip": { + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(assetContent)), + }, + "http://localhost/checksums.txt": { + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(checksumsFileContent)), + }, + } + mockRoundTripper := &mocks.MockRoundTripper{} + mockHttpClient := &http.Client{Transport: mockRoundTripper} + + // Mock the API client + oldNewClient := borg_github.NewClient + borg_github.NewClient = func(client *http.Client) *github.Client { + return github.NewClient(mockHttpClient) + } + t.Cleanup(func() { borg_github.NewClient = oldNewClient }) + + // Mock the download client + oldDefaultClient := borg_github.DefaultClient + borg_github.DefaultClient = mockHttpClient + t.Cleanup(func() { borg_github.DefaultClient = oldDefaultClient }) + + t.Run("Latest", func(t *testing.T) { + t.Cleanup(func() { os.RemoveAll("releases") }) + cmd := NewCollectGithubReleaseCmd() + buf := new(bytes.Buffer) + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{"owner/repo"}) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + ctx := context.WithValue(context.Background(), "logger", log) + err := cmd.ExecuteContext(ctx) + assert.NoError(t, err) + assert.FileExists(t, "releases/v1.0.0/RELEASE.md") + assert.FileExists(t, "releases/v1.0.0/asset1.zip") + assert.FileExists(t, "releases/v1.0.0/checksums.txt") + }) + + t.Run("ByTag", func(t *testing.T) { + t.Cleanup(func() { os.RemoveAll("releases") }) + cmd := NewCollectGithubReleaseCmd() + buf := new(bytes.Buffer) + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{"owner/repo", "v1.0.0"}) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + ctx := context.WithValue(context.Background(), "logger", log) + err := cmd.ExecuteContext(ctx) + assert.NoError(t, err) + assert.FileExists(t, "releases/v1.0.0/RELEASE.md") + assert.FileExists(t, "releases/v1.0.0/asset1.zip") + }) + + t.Run("AssetsOnly", func(t *testing.T) { + t.Cleanup(func() { os.RemoveAll("releases") }) + cmd := NewCollectGithubReleaseCmd() + buf := new(bytes.Buffer) + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{"owner/repo", "--assets-only"}) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + ctx := context.WithValue(context.Background(), "logger", log) + err := cmd.ExecuteContext(ctx) + assert.NoError(t, err) + assert.NoFileExists(t, "releases/v1.0.0/RELEASE.md") + assert.FileExists(t, "releases/v1.0.0/asset1.zip") + }) + + t.Run("Pattern", func(t *testing.T) { + t.Cleanup(func() { os.RemoveAll("releases") }) + cmd := NewCollectGithubReleaseCmd() + buf := new(bytes.Buffer) + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{"owner/repo", "--pattern", "*.zip"}) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + ctx := context.WithValue(context.Background(), "logger", log) + err := cmd.ExecuteContext(ctx) + assert.NoError(t, err) + assert.FileExists(t, "releases/v1.0.0/RELEASE.md") + assert.FileExists(t, "releases/v1.0.0/asset1.zip") + assert.NoFileExists(t, "releases/v1.0.0/checksums.txt") + }) + + t.Run("VerifyChecksums", func(t *testing.T) { + t.Cleanup(func() { os.RemoveAll("releases") }) + cmd := NewCollectGithubReleaseCmd() + buf := new(bytes.Buffer) + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{"owner/repo", "--verify-checksums"}) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + ctx := context.WithValue(context.Background(), "logger", log) + err := cmd.ExecuteContext(ctx) + assert.NoError(t, err) + assert.FileExists(t, "releases/v1.0.0/asset1.zip") + }) + + t.Run("VerifyChecksumsFailed", func(t *testing.T) { + // Mock a bad checksum + badChecksumsFileContent := fmt.Sprintf("badchecksum asset1.zip\n") + responses["http://localhost/checksums.txt"] = &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(badChecksumsFileContent)), + } + mockHttpClient.SetResponses(responses) + t.Cleanup(func() { os.RemoveAll("releases") }) + + cmd := NewCollectGithubReleaseCmd() + buf := new(bytes.Buffer) + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{"owner/repo", "--verify-checksums"}) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + ctx := context.WithValue(context.Background(), "logger", log) + // We expect an error, but the command is designed to log it and continue. + err := cmd.ExecuteContext(ctx) + assert.NoError(t, err) + }) +} diff --git a/cmd/collect_github_releases.go b/cmd/collect_github_releases.go new file mode 100644 index 0000000..29afe58 --- /dev/null +++ b/cmd/collect_github_releases.go @@ -0,0 +1,116 @@ +package cmd + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + + "bytes" + borg_github "github.com/Snider/Borg/pkg/github" + "github.com/spf13/cobra" +) + +func NewCollectGithubReleasesCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "releases ", + Short: "Download all release assets for a repository", + Long: `Download all release assets for a given GitHub repository.`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + logVal := cmd.Context().Value("logger") + log, ok := logVal.(*slog.Logger) + if !ok || log == nil { + return fmt.Errorf("logger not properly initialised") + } + + repoURL := args[0] + owner, repo, err := borg_github.ParseRepoFromURL(repoURL) + if err != nil { + return fmt.Errorf("failed to parse repository url: %w", err) + } + + releases, err := borg_github.ListReleases(owner, repo) + assetsOnly, _ := cmd.Flags().GetBool("assets-only") + pattern, _ := cmd.Flags().GetString("pattern") + verifyChecksums, _ := cmd.Flags().GetBool("verify-checksums") + + if err != nil { + return fmt.Errorf("failed to list releases: %w", err) + } + + for _, release := range releases { + tag := release.GetTagName() + releaseDir := filepath.Join("releases", tag) + if err := os.MkdirAll(releaseDir, 0755); err != nil { + return fmt.Errorf("failed to create release directory: %w", err) + } + + if !assetsOnly { + releaseNotes := release.GetBody() + if err := os.WriteFile(filepath.Join(releaseDir, "RELEASE.md"), []byte(releaseNotes), 0644); err != nil { + return fmt.Errorf("failed to write release notes: %w", err) + } + } + + for _, asset := range release.Assets { + if pattern != "" { + matched, err := filepath.Match(pattern, asset.GetName()) + if err != nil { + return fmt.Errorf("invalid pattern: %w", err) + } + if !matched { + continue + } + } + + log.Info("downloading asset", "name", asset.GetName()) + assetPath := filepath.Join(releaseDir, asset.GetName()) + file, err := os.Create(assetPath) + if err != nil { + return fmt.Errorf("failed to create asset file: %w", err) + } + defer file.Close() + + var checksumData []byte + if verifyChecksums { + checksumAsset := findChecksumAsset(release.Assets) + if checksumAsset == nil { + log.Warn("checksum file not found in release", "tag", tag) + } else { + buf := new(bytes.Buffer) + err := borg_github.DownloadReleaseAsset(checksumAsset, buf) + if err != nil { + log.Error("failed to download checksum file", "name", checksumAsset.GetName(), "err", err) + } else { + checksumData = buf.Bytes() + } + } + } + + if verifyChecksums && checksumData != nil { + err = borg_github.DownloadReleaseAssetWithChecksum(asset, checksumData, file) + } else { + err = borg_github.DownloadReleaseAsset(asset, file) + } + + if err != nil { + log.Error("failed to download asset", "name", asset.GetName(), "err", err) + continue + } + } + } + + return nil + }, + } + cmd.Flags().String("output", "releases", "Output directory for the releases") + cmd.Flags().Bool("assets-only", false, "Only download assets, skip release notes") + cmd.Flags().String("pattern", "", "Filter assets by filename pattern") + cmd.Flags().Bool("verify-checksums", false, "Verify checksums after download") + return cmd +} + +func init() { + collectGithubCmd.AddCommand(NewCollectGithubReleasesCmd()) +} diff --git a/cmd/collect_github_releases_subcommand_test.go b/cmd/collect_github_releases_subcommand_test.go new file mode 100644 index 0000000..79c1b39 --- /dev/null +++ b/cmd/collect_github_releases_subcommand_test.go @@ -0,0 +1,141 @@ +package cmd + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "path/filepath" + "testing" + + borg_github "github.com/Snider/Borg/pkg/github" + "github.com/Snider/Borg/pkg/mocks" + "github.com/google/go-github/v39/github" + "github.com/stretchr/testify/assert" +) + +func TestCollectGithubReleasesCmd(t *testing.T) { + assetContent := "asset content" + hasher := sha256.New() + hasher.Write([]byte(assetContent)) + correctChecksum := hex.EncodeToString(hasher.Sum(nil)) + checksumsFileContent := fmt.Sprintf("%s asset1.zip\n", correctChecksum) + + // Mock the GitHub API + responses := map[string]*http.Response{ + "https://api.github.com/repos/owner/repo/releases?per_page=30": { + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`[{"tag_name": "v1.0.0", "body": "Release notes", "assets": [{"name": "asset1.zip", "browser_download_url": "http://localhost/asset1.zip"}, {"name": "checksums.txt", "browser_download_url": "http://localhost/checksums.txt"}]}]`))), + Header: http.Header{}, + }, + "http://localhost/asset1.zip": { + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(assetContent)), + }, + "http://localhost/checksums.txt": { + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(checksumsFileContent)), + }, + } + mockHttpClient := mocks.NewMockClient(responses) + + // Mock the API client + oldNewClient := borg_github.NewClient + borg_github.NewClient = func(client *http.Client) *github.Client { + return github.NewClient(mockHttpClient) + } + t.Cleanup(func() { borg_github.NewClient = oldNewClient }) + + // Mock the download client + oldDefaultClient := borg_github.DefaultClient + borg_github.DefaultClient = mockHttpClient + t.Cleanup(func() { borg_github.DefaultClient = oldDefaultClient }) + + t.Run("Default", func(t *testing.T) { + t.Cleanup(func() { os.RemoveAll("releases") }) + cmd := NewCollectGithubReleasesCmd() + buf := new(bytes.Buffer) + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{"owner/repo"}) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + ctx := context.WithValue(context.Background(), "logger", log) + err := cmd.ExecuteContext(ctx) + assert.NoError(t, err) + assert.FileExists(t, "releases/v1.0.0/RELEASE.md") + assert.FileExists(t, "releases/v1.0.0/asset1.zip") + assert.FileExists(t, "releases/v1.0.0/checksums.txt") + }) + + t.Run("AssetsOnly", func(t *testing.T) { + t.Cleanup(func() { os.RemoveAll("releases") }) + cmd := NewCollectGithubReleasesCmd() + buf := new(bytes.Buffer) + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{"owner/repo", "--assets-only"}) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + ctx := context.WithValue(context.Background(), "logger", log) + err := cmd.ExecuteContext(ctx) + assert.NoError(t, err) + assert.NoFileExists(t, "releases/v1.0.0/RELEASE.md") + assert.FileExists(t, "releases/v1.0.0/asset1.zip") + }) + + t.Run("Pattern", func(t *testing.T) { + t.Cleanup(func() { os.RemoveAll("releases") }) + cmd := NewCollectGithubReleasesCmd() + buf := new(bytes.Buffer) + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{"owner/repo", "--pattern", "*.zip"}) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + ctx := context.WithValue(context.Background(), "logger", log) + err := cmd.ExecuteContext(ctx) + assert.NoError(t, err) + assert.FileExists(t, "releases/v1.0.0/RELEASE.md") + assert.FileExists(t, "releases/v1.0.0/asset1.zip") + assert.NoFileExists(t, "releases/v1.0.0/checksums.txt") + }) + + t.Run("VerifyChecksums", func(t *testing.T) { + t.Cleanup(func() { os.RemoveAll("releases") }) + cmd := NewCollectGithubReleasesCmd() + buf := new(bytes.Buffer) + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{"owner/repo", "--verify-checksums"}) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + ctx := context.WithValue(context.Background(), "logger", log) + err := cmd.ExecuteContext(ctx) + assert.NoError(t, err) + assert.FileExists(t, "releases/v1.0.0/asset1.zip") + }) + + t.Run("VerifyChecksumsFailed", func(t *testing.T) { + // Mock a bad checksum + badChecksumsFileContent := fmt.Sprintf("badchecksum asset1.zip\n") + responses["http://localhost/checksums.txt"] = &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(badChecksumsFileContent)), + } + mockHttpClient.SetResponses(responses) + t.Cleanup(func() { os.RemoveAll("releases") }) + + cmd := NewCollectGithubReleasesCmd() + buf := new(bytes.Buffer) + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{"owner/repo", "--verify-checksums"}) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + ctx := context.WithValue(context.Background(), "logger", log) + // We expect an error, but the command is designed to log it and continue. + err := cmd.ExecuteContext(ctx) + assert.NoError(t, err) + }) +} diff --git a/examples/collect_github_release/main.go b/examples/collect_github_release/main.go index 63181ed..9c60e06 100644 --- a/examples/collect_github_release/main.go +++ b/examples/collect_github_release/main.go @@ -28,14 +28,15 @@ func main() { asset := release.Assets[0] log.Printf("Downloading asset: %s", asset.GetName()) - data, err := github.DownloadReleaseAsset(asset) + file, err := os.Create(asset.GetName()) if err != nil { - log.Fatalf("Failed to download asset: %v", err) + log.Fatalf("Failed to create file: %v", err) } + defer file.Close() - err = os.WriteFile(asset.GetName(), data, 0644) + err = github.DownloadReleaseAsset(asset, file) if err != nil { - log.Fatalf("Failed to write asset to file: %v", err) + log.Fatalf("Failed to download asset: %v", err) } log.Printf("Successfully downloaded asset to %s", asset.GetName()) diff --git a/pkg/github/release.go b/pkg/github/release.go index 4aaa1ef..b37ffab 100644 --- a/pkg/github/release.go +++ b/pkg/github/release.go @@ -1,8 +1,11 @@ package github import ( + "bufio" "bytes" "context" + "crypto/sha256" + "encoding/hex" "fmt" "io" "net/http" @@ -35,30 +38,114 @@ func GetLatestRelease(owner, repo string) (*github.RepositoryRelease, error) { return release, nil } -// DownloadReleaseAsset downloads a release asset. -func DownloadReleaseAsset(asset *github.ReleaseAsset) ([]byte, error) { +// DownloadReleaseAssetWithChecksum downloads a release asset and verifies its checksum. +func DownloadReleaseAssetWithChecksum(asset *github.ReleaseAsset, checksumsData []byte, w io.Writer) error { req, err := NewRequest("GET", asset.GetBrowserDownloadURL(), nil) if err != nil { - return nil, err + return err } req.Header.Set("Accept", "application/octet-stream") resp, err := DefaultClient.Do(req) if err != nil { - return nil, err + return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("bad status: %s", resp.Status) + return fmt.Errorf("bad status: %s", resp.Status) + } + + hasher := sha256.New() + teeReader := io.TeeReader(resp.Body, hasher) + + _, err = io.Copy(w, teeReader) + if err != nil { + return err } - buf := new(bytes.Buffer) - _, err = io.Copy(buf, resp.Body) + actualChecksum := hex.EncodeToString(hasher.Sum(nil)) + + err = verifyChecksum(actualChecksum, asset.GetName(), checksumsData) + if err != nil { + return fmt.Errorf("checksum verification failed for %s: %w", asset.GetName(), err) + } + + return nil +} + +// verifyChecksum verifies the SHA256 checksum of a byte slice against a checksums file content. +// The checksums file is expected to be in the format: +func verifyChecksum(actualChecksum, name string, checksumsData []byte) error { + scanner := bufio.NewScanner(bytes.NewReader(checksumsData)) + for scanner.Scan() { + line := scanner.Text() + parts := strings.Fields(line) + if len(parts) == 2 && parts[1] == name { + expectedChecksum := parts[0] + if actualChecksum != expectedChecksum { + return fmt.Errorf("checksum mismatch: expected %s, got %s", expectedChecksum, actualChecksum) + } + return nil // Checksum verified + } + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("error reading checksums data: %w", err) + } + + return fmt.Errorf("checksum not found for file: %s", name) +} + +// ListReleases lists all releases for a repository. +func ListReleases(owner, repo string) ([]*github.RepositoryRelease, error) { + client := NewClient(nil) + opt := &github.ListOptions{PerPage: 30} + var allReleases []*github.RepositoryRelease + for { + releases, resp, err := client.Repositories.ListReleases(context.Background(), owner, repo, opt) + if err != nil { + return nil, err + } + allReleases = append(allReleases, releases...) + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + return allReleases, nil +} + +// GetReleaseByTag gets a release by its tag name. +func GetReleaseByTag(owner, repo, tag string) (*github.RepositoryRelease, error) { + client := NewClient(nil) + release, _, err := client.Repositories.GetReleaseByTag(context.Background(), owner, repo, tag) if err != nil { return nil, err } - return buf.Bytes(), nil + return release, nil +} + +// DownloadReleaseAsset downloads a release asset. +func DownloadReleaseAsset(asset *github.ReleaseAsset, w io.Writer) error { + req, err := NewRequest("GET", asset.GetBrowserDownloadURL(), nil) + if err != nil { + return err + } + req.Header.Set("Accept", "application/octet-stream") + + resp, err := DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("bad status: %s", resp.Status) + } + + _, err = io.Copy(w, resp.Body) + return err } // ParseRepoFromURL parses the owner and repository from a GitHub URL. diff --git a/pkg/github/release_test.go b/pkg/github/release_test.go index fa49052..0c85eaf 100644 --- a/pkg/github/release_test.go +++ b/pkg/github/release_test.go @@ -2,6 +2,8 @@ package github import ( "bytes" + "crypto/sha256" + "encoding/hex" "fmt" "io" "net/http" @@ -10,6 +12,7 @@ import ( "github.com/Snider/Borg/pkg/mocks" "github.com/google/go-github/v39/github" + "github.com/stretchr/testify/assert" ) type errorRoundTripper struct{} @@ -89,13 +92,14 @@ func TestDownloadReleaseAsset(t *testing.T) { DefaultClient = oldClient }() - data, err := DownloadReleaseAsset(asset) + buf := new(bytes.Buffer) + err := DownloadReleaseAsset(asset, buf) if err != nil { t.Fatalf("DownloadReleaseAsset failed: %v", err) } - if string(data) != "asset content" { - t.Errorf("unexpected asset content: %s", string(data)) + if buf.String() != "asset content" { + t.Errorf("unexpected asset content: %s", buf.String()) } } func TestDownloadReleaseAsset_BadRequest(t *testing.T) { @@ -118,7 +122,8 @@ func TestDownloadReleaseAsset_BadRequest(t *testing.T) { DefaultClient = oldClient }() - _, err := DownloadReleaseAsset(asset) + buf := new(bytes.Buffer) + err := DownloadReleaseAsset(asset, buf) if err == nil { t.Fatalf("expected error but got nil") } @@ -141,7 +146,8 @@ func TestDownloadReleaseAsset_NewRequestError(t *testing.T) { NewRequest = oldNewRequest }() - _, err := DownloadReleaseAsset(asset) + buf := new(bytes.Buffer) + err := DownloadReleaseAsset(asset, buf) if err == nil { t.Fatalf("expected error but got nil") } @@ -191,8 +197,146 @@ func TestDownloadReleaseAsset_DoError(t *testing.T) { DefaultClient = oldClient }() - _, err := DownloadReleaseAsset(asset) + buf := new(bytes.Buffer) + err := DownloadReleaseAsset(asset, buf) if err == nil { t.Fatalf("DownloadReleaseAsset should have failed") } } + +func TestListReleases(t *testing.T) { + oldNewClient := NewClient + t.Cleanup(func() { NewClient = oldNewClient }) + + responses := make(map[string]*http.Response) + responses["https://api.github.com/repos/owner/repo/releases?per_page=30"] = &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(`[{"tag_name": "v1.0.0"}, {"tag_name": "v0.9.0"}]`)), + Header: http.Header{ + "Link": []string{`; rel="next"`}, + }, + } + responses["https://api.github.com/repos/owner/repo/releases?page=2&per_page=30"] = &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(`[{"tag_name": "v0.8.0"}]`)), + Header: http.Header{}, + } + + mockClient := mocks.NewMockClient(responses) + client := github.NewClient(mockClient) + NewClient = func(_ *http.Client) *github.Client { + return client + } + + releases, err := ListReleases("owner", "repo") + assert.NoError(t, err) + assert.Len(t, releases, 3) + assert.Equal(t, "v1.0.0", releases[0].GetTagName()) + assert.Equal(t, "v0.9.0", releases[1].GetTagName()) + assert.Equal(t, "v0.8.0", releases[2].GetTagName()) +} + +func TestGetReleaseByTag(t *testing.T) { + oldNewClient := NewClient + t.Cleanup(func() { NewClient = oldNewClient }) + + mockClient := mocks.NewMockClient(map[string]*http.Response{ + "https://api.github.com/repos/owner/repo/releases/tags/v1.0.0": { + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(`{"tag_name": "v1.0.0"}`)), + }, + }) + client := github.NewClient(mockClient) + NewClient = func(_ *http.Client) *github.Client { + return client + } + + release, err := GetReleaseByTag("owner", "repo", "v1.0.0") + assert.NoError(t, err) + assert.NotNil(t, release) + assert.Equal(t, "v1.0.0", release.GetTagName()) +} + +func TestDownloadReleaseAssetWithChecksum(t *testing.T) { + assetName := "my-asset.zip" + assetURL := "https://example.com/download/my-asset.zip" + assetContent := "this is the content of the asset" + hasher := sha256.New() + hasher.Write([]byte(assetContent)) + correctChecksum := hex.EncodeToString(hasher.Sum(nil)) + checksumsFileContent := fmt.Sprintf("%s %s\n", correctChecksum, assetName) + + asset := &github.ReleaseAsset{ + Name: &assetName, + BrowserDownloadURL: &assetURL, + } + + t.Run("GoodChecksum", func(t *testing.T) { + mockHttpClient := mocks.NewMockClient(map[string]*http.Response{ + assetURL: { + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(assetContent)), + }, + }) + oldClient := DefaultClient + DefaultClient = mockHttpClient + t.Cleanup(func() { DefaultClient = oldClient }) + + buf := new(bytes.Buffer) + err := DownloadReleaseAssetWithChecksum(asset, []byte(checksumsFileContent), buf) + assert.NoError(t, err) + assert.Equal(t, assetContent, buf.String()) + }) + + t.Run("BadChecksum", func(t *testing.T) { + mockHttpClient := mocks.NewMockClient(map[string]*http.Response{ + assetURL: { + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(assetContent)), + }, + }) + oldClient := DefaultClient + DefaultClient = mockHttpClient + t.Cleanup(func() { DefaultClient = oldClient }) + badChecksumsFileContent := fmt.Sprintf("badchecksum %s\n", assetName) + buf := new(bytes.Buffer) + err := DownloadReleaseAssetWithChecksum(asset, []byte(badChecksumsFileContent), buf) + assert.Error(t, err) + assert.Contains(t, err.Error(), "checksum mismatch") + }) + + t.Run("ChecksumNotFound", func(t *testing.T) { + mockHttpClient := mocks.NewMockClient(map[string]*http.Response{ + assetURL: { + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(assetContent)), + }, + }) + oldClient := DefaultClient + DefaultClient = mockHttpClient + t.Cleanup(func() { DefaultClient = oldClient }) + missingChecksumsFileContent := fmt.Sprintf("%s another-file.zip\n", correctChecksum) + buf := new(bytes.Buffer) + err := DownloadReleaseAssetWithChecksum(asset, []byte(missingChecksumsFileContent), buf) + assert.Error(t, err) + assert.Contains(t, err.Error(), "checksum not found") + }) + + t.Run("DownloadError", func(t *testing.T) { + mockHttpClientWithErr := mocks.NewMockClient(map[string]*http.Response{ + assetURL: { + StatusCode: http.StatusNotFound, + Status: "404 Not Found", + Body: io.NopCloser(bytes.NewBufferString("")), + }, + }) + oldClient := DefaultClient + DefaultClient = mockHttpClientWithErr + t.Cleanup(func() { DefaultClient = oldClient }) + + buf := new(bytes.Buffer) + err := DownloadReleaseAssetWithChecksum(asset, []byte(checksumsFileContent), buf) + assert.Error(t, err) + assert.Contains(t, err.Error(), "bad status: 404 Not Found") + }) +}