Merge pull request #83 from sapcc/github-releases

Add support for github-releases

Author: talal

.gitignore

@@ -2,5 +2,6 @@
 /build
 
 # for my own testing
+/.env
 /*.yaml
 !/Makefile.maker.yaml

CHANGELOG.md

@@ -1,5 +1,8 @@
 # v2.9.0 (TBD)
 
+New features:
+- Add support for GitHub releases.
+
 Bugfixes:
 - Fixed transfer of objects from swift source when object name is not a well-formed path. For example, an object name like "a///b" is not wrongly normalized into "a/b" anymore.
 

README.md

@@ -7,6 +7,10 @@
 * [Usage](#usage)
   * [Alternative authentication options](#alternative-authentication-options)
   * [Source specification](#source-specification)
+    * [Yum](#yum)
+    * [Debian](#debian)
+    * [Github Releases](#github-releases)
+    * [Swift](#swift)
   * [File selection](#file-selection)
     * [By name](#by-name)
     * [By age](#by-age)
@@ -238,6 +242,43 @@ jobs:
       object_prefix: ubuntu
 ```
 
+#### Github Releases
+
+If `jobs[].from.url` refers to a GitHub repository, setting `jobs[].from.type` to
+`github-releases` will cause `swift-http-import` to use GitHub's API to discover and
+download GitHub releases for the repository. Repositories hosted on a GitHub Enterprise
+instance are also supported.
+
+Since GitHub's API rate limits the number of requests per IP therefore it is
+**recommended** that you specify a GitHub personal access token in the `jobs[].from.token`
+field. Refer to the [GitHub API docs](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting)
+for more info on its rate limiting. This field is **required** if the repository is hosted on a Github Enterprise instance instead of `github.com`.
+Instead of providing your token as plain text in the config file, you can use the
+`fromEnv` special syntax for the `jobs[].from.token` field. See [Alternative
+authentication options](#alternative-authentication-options) for more details.
+
+If a repository publishes GitHub releases using different tags, e.g. server components at
+`server-x.y.z` and client at `client-x.y.z`, and you only want to get releases whose tag
+matches a specific pattern then you can specify a regex for this in the
+`jobs[].from.tag_name_pattern` field. Only releases whose tag name matches this regex will
+be transferred.
+
+Prerelease and draft releases are not transferred by default. You can override this
+behavior by setting the `jobs[].from.include_prerelease` and `jobs[].from.include_draft`
+field to `true`.
+
+[Link to full example config file](./examples/source-debian.yaml)
+
+```yaml
+jobs:
+  - from:
+      url: https://github.com/sapcc/limesctl
+      type: github-releases
+    to:
+      container: mirror
+      object_prefix: sapcc/limesctl
+```
+
 #### Swift
 
 Alternatively, the source in `jobs[].from` can also be a private Swift container if Swift credentials are specified

examples/source-github-releases.yaml

@@ -0,0 +1,21 @@
+swift:
+  auth_url: https://my.keystone.local:5000/v3
+  user_name: uploader
+  user_domain_name: Default
+  project_name: datastore
+  project_domain_name: Default
+  password: 20g82rzg235oughq
+
+jobs:
+  - from:
+      url: https://github.com/sapcc/limesctl
+      type: github-releases
+      token: ghp_asjdkajsdlbyaksjd2
+      tag_name_pattern: "^v[0-9]+.[0-9]+.[0-9]+$"
+      include_draft: false
+      include_prerelease: false
+    to:
+      container: github
+      object_prefix: sapcc/limesctl
+    match:
+      not_older_than: 12 weeks

go.mod

@@ -4,18 +4,24 @@ go 1.17
 
 require (
 	github.com/cactus/go-statsd-client/v4 v4.0.0
+	github.com/google/go-github/v44 v44.0.0
 	github.com/gophercloud/gophercloud v0.24.0
 	github.com/gophercloud/utils v0.0.0-20220307143606-8e7800759d16
 	github.com/majewsky/schwift v1.1.0
 	github.com/sapcc/go-bits v0.0.0-20220517080651-baba74caa927
 	github.com/ulikunitz/xz v0.5.10
 	golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898
 	golang.org/x/net v0.0.0-20220517181318-183a9ca12b87
+	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
 	gopkg.in/yaml.v2 v2.4.0
 	pault.ag/go/debian v0.12.0
 )
 
 require (
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
 	pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a // indirect
 )

go.sum

@@ -1,6 +1,17 @@
 github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
 github.com/cactus/go-statsd-client/v4 v4.0.0 h1:cjO9CI3GAHtj/Vmmt1/BRq8+hf5MXy/Pr/CM6Dy4dp0=
 github.com/cactus/go-statsd-client/v4 v4.0.0/go.mod h1:m73kwJp6TN0Ja9P6ycdZhWM1MlfxY/95WZ//IptPQ+Y=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-github/v44 v44.0.0 h1:1Lfk2mhM7pTWqwGC6Ft16S3c2LBw8DLcw9TOhYoQ9zE=
+github.com/google/go-github/v44 v44.0.0/go.mod h1:CqZYQRxOcb81M+ufZB7duWNS0lFfas/r7cEAKpLBYww=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/gophercloud/gophercloud v0.15.1-0.20210202035223-633d73521055/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075O6i+LY+pXsKCBsb4=
 github.com/gophercloud/gophercloud v0.17.0/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075O6i+LY+pXsKCBsb4=
 github.com/gophercloud/gophercloud v0.20.0/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075O6i+LY+pXsKCBsb4=
@@ -29,9 +40,12 @@ golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898 h1:SLP7Q4Di66FONjDJbCYrCRrh97focO6sLogHO7/g8F0=
 golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220517181318-183a9ca12b87 h1:cCR+9mKLOGyX4Zx+uBZDXEDAQsvKQ/XbW4vreG5v1jU=
 golang.org/x/net v0.0.0-20220517181318-183a9ca12b87/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -41,8 +55,17 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

pkg/objects/config.go

@@ -199,6 +199,8 @@ func (u *SourceUnmarshaler) UnmarshalYAML(unmarshal func(interface{}) error) err
 			u.Source = &YumSource{}
 		case "debian":
 			u.Source = &DebianSource{}
+		case "github-releases":
+			u.Source = &GithubReleaseSource{}
 		default:
 			return fmt.Errorf("unexpected value: type = %q", probe.Type)
 		}
@@ -219,10 +221,11 @@ type Job struct {
 
 //Compile validates the given JobConfiguration, then creates and prepares a Job from it.
 func (cfg JobConfiguration) Compile(name string, swift SwiftLocation) (job *Job, errors []error) {
-	if cfg.Source.Source == nil {
+	jobSrc := cfg.Source.Source
+	if jobSrc == nil {
 		errors = append(errors, fmt.Errorf("missing value for %s.from", name))
 	} else {
-		errors = append(errors, cfg.Source.Source.Validate(name+".from")...)
+		errors = append(errors, jobSrc.Validate(name+".from")...)
 	}
 	if cfg.Target == nil {
 		errors = append(errors, fmt.Errorf("missing value for %s.to", name))
@@ -242,27 +245,28 @@ func (cfg JobConfiguration) Compile(name string, swift SwiftLocation) (job *Job,
 	}
 
 	if cfg.Match.NotOlderThan != nil {
-		_, isSwiftSource := cfg.Source.Source.(*SwiftLocation)
-		if !isSwiftSource {
-			errors = append(errors, fmt.Errorf("invalid value for %s.match.not_older_than: this option is not supported for source type %T", name, cfg.Source.Source))
+		_, isSwiftSource := jobSrc.(*SwiftLocation)
+		_, isGitHubReleaseSource := jobSrc.(*GithubReleaseSource)
+		if !isSwiftSource && !isGitHubReleaseSource {
+			errors = append(errors, fmt.Errorf("invalid value for %s.match.not_older_than: this option is not supported for source type %T", name, jobSrc))
 		}
 	}
 
 	if cfg.Match.SimplisticComparison != nil {
-		_, isURLSource := cfg.Source.Source.(*URLSource)
-		_, isSwiftSource := cfg.Source.Source.(*SwiftLocation)
+		_, isURLSource := jobSrc.(*URLSource)
+		_, isSwiftSource := jobSrc.(*SwiftLocation)
 		if !isURLSource && !isSwiftSource {
-			errors = append(errors, fmt.Errorf("invalid value for %s.match.simplistic_comparison: this option is not supported for source type %T", name, cfg.Source.Source))
+			errors = append(errors, fmt.Errorf("invalid value for %s.match.simplistic_comparison: this option is not supported for source type %T", name, jobSrc))
 		}
 	}
 
-	_, isYumSource := cfg.Source.Source.(*YumSource)
+	_, isYumSource := jobSrc.(*YumSource)
 	if isYumSource {
-		cfg.Source.Source.(*YumSource).gpgKeyRing = cfg.gpgKeyRing
+		jobSrc.(*YumSource).gpgKeyRing = cfg.gpgKeyRing
 	}
-	_, isDebianSource := cfg.Source.Source.(*DebianSource)
+	_, isDebianSource := jobSrc.(*DebianSource)
 	if isDebianSource {
-		cfg.Source.Source.(*DebianSource).gpgKeyRing = cfg.gpgKeyRing
+		jobSrc.(*DebianSource).gpgKeyRing = cfg.gpgKeyRing
 	}
 
 	if cfg.Segmenting != nil {
@@ -289,7 +293,7 @@ func (cfg JobConfiguration) Compile(name string, swift SwiftLocation) (job *Job,
 	}
 
 	job = &Job{
-		Source:     cfg.Source.Source,
+		Source:     jobSrc,
 		Target:     cfg.Target,
 		Segmenting: cfg.Segmenting,
 		Expiration: cfg.Expiration,
@@ -316,6 +320,10 @@ func (cfg JobConfiguration) Compile(name string, swift SwiftLocation) (job *Job,
 		job.Matcher.NotOlderThan = &cutoff
 	}
 	job.Matcher.SimplisticComparison = cfg.Match.SimplisticComparison
+	_, isGitHubReleaseSource := jobSrc.(*GithubReleaseSource)
+	if isGitHubReleaseSource {
+		jobSrc.(*GithubReleaseSource).notOlderThan = job.Matcher.NotOlderThan
+	}
 
 	//do not try connecting to Swift if credentials are invalid etc.
 	if len(errors) > 0 {

pkg/objects/debian.go

@@ -83,15 +83,12 @@ func (s *DebianSource) Connect(name string) error {
 
 //ListEntries implements the Source interface.
 func (s *DebianSource) ListEntries(directoryPath string) ([]FileSpec, *ListEntriesError) {
-	return nil, &ListEntriesError{
-		Location: s.urlSource.getURLForPath(directoryPath).String(),
-		Message:  "ListEntries is not implemented for DebianSource",
-	}
+	return nil, ErrListEntriesNotSupported
 }
 
 //GetFile implements the Source interface.
-func (s *DebianSource) GetFile(directoryPath string, requestHeaders schwift.ObjectHeaders) (body io.ReadCloser, sourceState FileState, err error) {
-	return s.urlSource.GetFile(directoryPath, requestHeaders)
+func (s *DebianSource) GetFile(path string, requestHeaders schwift.ObjectHeaders) (body io.ReadCloser, sourceState FileState, err error) {
+	return s.urlSource.GetFile(path, requestHeaders)
 }
 
 //ListAllFiles implements the Source interface.

pkg/objects/file.go

@@ -42,9 +42,12 @@ type File struct {
 //downloaded contents and metadata in the remaining fields of the FileSpec to
 //avoid double download.
 type FileSpec struct {
-	Path        string
-	IsDirectory bool
-	//only set for files in Swift sources (otherwise nil)
+	Path string
+	// DownloadPath is set for files whose path for downloading their contents is
+	// different from the Path variable which denotes their actual file path.
+	DownloadPath string
+	IsDirectory  bool
+	//only set for files in Swift and GitHub release sources (otherwise nil)
 	LastModified *time.Time
 	//only set for symlinks (refers to a path below the ObjectPrefix in the same container)
 	SymlinkTargetPath string
@@ -154,7 +157,11 @@ func (f File) PerformTransfer() (TransferResult, int64) {
 		sourceState FileState
 	)
 	if f.Spec.Contents == nil {
-		body, sourceState, err = f.Job.Source.GetFile(f.Spec.Path, requestHeaders)
+		path := f.Spec.Path
+		if f.Spec.DownloadPath != "" {
+			path = f.Spec.DownloadPath
+		}
+		body, sourceState, err = f.Job.Source.GetFile(path, requestHeaders)
 	} else {
 		logg.Debug("using cached contents for %s", f.Spec.Path)
 		body, sourceState, err = f.Spec.toTransferFormat(requestHeaders)