From 07a813ae385c69075b26fd881ff92e2d53db96d8 Mon Sep 17 00:00:00 2001 From: Richard Hartmann Date: Wed, 5 May 2021 19:53:31 +0200 Subject: [PATCH 1/2] Update README.md Signed-off-by: Richard Hartmann --- remote_write/README.md | 1 + remote_write/latest/latest.go | 58 +++++++++++++++++++++++++++ remote_write/targets/grafana_agent.go | 9 ++++- remote_write/targets/otel.go | 9 ++++- remote_write/targets/prometheus.go | 10 +++-- remote_write/targets/telegraf.go | 9 ++++- remote_write/targets/vector.go | 5 ++- remote_write/targets/vmagent.go | 9 ++++- 8 files changed, 98 insertions(+), 12 deletions(-) create mode 100644 remote_write/latest/latest.go diff --git a/remote_write/README.md b/remote_write/README.md index 1c0e3324..6f782444 100644 --- a/remote_write/README.md +++ b/remote_write/README.md @@ -33,6 +33,7 @@ The repo tests the following remote write senders: - [InfluxData's Telegraf](https://github.com/influxdata/telegraf). - The [OpenTelemetry Collector](https://github.com/open-telemetry/opentelemetry-collector). - The [VictoriaMetrics Agent](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmagent), unless you're on [Mac OS X](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1042). +- [Timber.io Vector](https://github.com/timberio/vector). If you want to test a dev version of your sender, simply put your binary in `bin/`. diff --git a/remote_write/latest/latest.go b/remote_write/latest/latest.go new file mode 100644 index 00000000..b4ecaf27 --- /dev/null +++ b/remote_write/latest/latest.go @@ -0,0 +1,58 @@ +package latest + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net/http" + "strings" + "time" +) + +func GetLatestVersion(repo string) string { + url := "https://api.github.com/repos/" + repo + "/releases/latest" + + httpClient := http.Client{ + Timeout: time.Second * 2, // Timeout after 2 seconds + } + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + log.Fatal(err) + } + + req.Header.Add("Accept", "application/vnd.github.v3+json") + + res, getErr := httpClient.Do(req) + if getErr != nil { + log.Fatal(getErr) + } + + if res.Body != nil { + defer res.Body.Close() + } + + body, readErr := ioutil.ReadAll(res.Body) + if readErr != nil { + log.Fatal(readErr) + } + + type PackageInfo struct { + TagName string `json:"tag_name"` + CreatedAt time.Time `json:"created_at"` + PublishedAt time.Time `json:"published_at"` + } + + var packageInfo PackageInfo + + jsonErr := json.Unmarshal(body, &packageInfo) + if jsonErr != nil { + log.Fatal(jsonErr) + } + + version := strings.Trim(packageInfo.TagName, "v") + fmt.Println("repository: " + repo + " - version: " + version) + + return version +} diff --git a/remote_write/targets/grafana_agent.go b/remote_write/targets/grafana_agent.go index 0da2d006..ce12dfcc 100644 --- a/remote_write/targets/grafana_agent.go +++ b/remote_write/targets/grafana_agent.go @@ -3,12 +3,17 @@ package targets import ( "fmt" "os" + + "github.com/prometheus/compliance/remote_write/latest" ) -const grafanaAgentDownloadURL = "https://github.com/grafana/agent/releases/download/v0.13.1/agent-{{.OS}}-{{.Arch}}.zip" +func getGrafanaDownloadURL() string { + version := latest.GetLatestVersion("grafana/agent") + return "https://github.com/grafana/agent/releases/download/" + version + "/agent-{{.OS}}-{{.Arch}}.zip" +} func RunGrafanaAgent(opts TargetOptions) error { - binary, err := downloadBinary(grafanaAgentDownloadURL, "agent-{{.OS}}-{{.Arch}}") + binary, err := downloadBinary(getGrafanaDownloadURL(), "agent-{{.OS}}-{{.Arch}}") if err != nil { return err } diff --git a/remote_write/targets/otel.go b/remote_write/targets/otel.go index 36433b67..1addb75a 100644 --- a/remote_write/targets/otel.go +++ b/remote_write/targets/otel.go @@ -3,12 +3,17 @@ package targets import ( "fmt" "os" + + "github.com/prometheus/compliance/remote_write/latest" ) -const otelDownloadURL = "https://github.com/open-telemetry/opentelemetry-collector/releases/download/v0.26.0/otelcol_{{.OS}}_{{.Arch}}" +func getOtelDownloadURL() string { + version := latest.GetLatestVersion("open-telemetry/opentelemetry-collector") + return "https://github.com/open-telemetry/opentelemetry-collector/releases/download/v" + version + "/otelcol_{{.OS}}_{{.Arch}}" +} func RunOtelCollector(opts TargetOptions) error { - binary, err := downloadBinary(otelDownloadURL, "") + binary, err := downloadBinary(getOtelDownloadURL(), "") if err != nil { return err } diff --git a/remote_write/targets/prometheus.go b/remote_write/targets/prometheus.go index b167bb8b..994bccd7 100644 --- a/remote_write/targets/prometheus.go +++ b/remote_write/targets/prometheus.go @@ -3,12 +3,16 @@ package targets import ( "fmt" "os" -) -const prometheusDownloadURL = "https://github.com/prometheus/prometheus/releases/download/v2.26.0/prometheus-2.26.0.{{.OS}}-{{.Arch}}.tar.gz" + "github.com/prometheus/compliance/remote_write/latest" +) +func getPrometheusDownloadURL() string { + version := latest.GetLatestVersion("prometheus/prometheus") + return "https://github.com/prometheus/prometheus/releases/download/v" + version + "/prometheus-" + version + ".{{.OS}}-{{.Arch}}.tar.gz" +} func RunPrometheus(opts TargetOptions) error { - binary, err := downloadBinary(prometheusDownloadURL, "prometheus") + binary, err := downloadBinary(getPrometheusDownloadURL(), "prometheus") if err != nil { return err } diff --git a/remote_write/targets/telegraf.go b/remote_write/targets/telegraf.go index 66a71c5a..69287d0c 100644 --- a/remote_write/targets/telegraf.go +++ b/remote_write/targets/telegraf.go @@ -3,12 +3,17 @@ package targets import ( "fmt" "os" + + "github.com/prometheus/compliance/remote_write/latest" ) -const telegrafURL = "https://dl.influxdata.com/telegraf/releases/telegraf-1.18.2_{{.OS}}_{{.Arch}}.tar.gz" +func getTelegrafDownloadURL() string { + version := latest.GetLatestVersion("influxdata/telegraf") + return "https://dl.influxdata.com/telegraf/releases/telegraf-" + version + "_{{.OS}}_{{.Arch}}.tar.gz" +} func RunTelegraf(opts TargetOptions) error { - binary, err := downloadBinary(telegrafURL, "telegraf") + binary, err := downloadBinary(getTelegrafDownloadURL(), "telegraf") if err != nil { return err } diff --git a/remote_write/targets/vector.go b/remote_write/targets/vector.go index dac9dc72..d6fc28d5 100644 --- a/remote_write/targets/vector.go +++ b/remote_write/targets/vector.go @@ -4,10 +4,13 @@ import ( "fmt" "os" "runtime" + + "github.com/prometheus/compliance/remote_write/latest" ) + func getVectorDownloadURL() string { - var version string = "0.13.1" + version := latest.GetLatestVersion("timberio/vector") switch runtime.GOOS { case "darwin": return "https://github.com/timberio/vector/releases/download/v" + version + "/vector-" + version + "-x86_64-apple-darwin.tar.gz" diff --git a/remote_write/targets/vmagent.go b/remote_write/targets/vmagent.go index 415af3c0..61ad6bf9 100644 --- a/remote_write/targets/vmagent.go +++ b/remote_write/targets/vmagent.go @@ -3,14 +3,19 @@ package targets import ( "fmt" "os" + + "github.com/prometheus/compliance/remote_write/latest" ) -const vmagentURL = "https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.59.0/vmutils-{{.Arch}}-v1.59.0.tar.gz" +func getVMAgentDownloadURL() string { + version := latest.GetLatestVersion("VictoriaMetrics/VictoriaMetrics") + return "https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v" + version + "/vmutils-{{.Arch}}-v1.58.0.tar.gz" +} func RunVMAgent(opts TargetOptions) error { // NB this won't work on a Mac - need mac builds https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1042! // If you build it yourself and stick it in the bin/ directory, the tests will work. - binary, err := downloadBinary(vmagentURL, "vmagent-prod") + binary, err := downloadBinary(getVMAgentDownloadURL(), "vmagent-prod") if err != nil { return err } From f9371b87ee5a3e0ae94fc3c1c83d974cbd945ca0 Mon Sep 17 00:00:00 2001 From: Richard Hartmann Date: Thu, 13 May 2021 20:59:44 +0200 Subject: [PATCH 2/2] Automate URL building away If you hammer GH API without a login too often, you will get rate limited over time. As local binaries remain on disk, this should not be too much of a concern. Signed-off-by: Richard Hartmann --- remote_write/latest/latest.go | 15 +++++++++++++-- remote_write/targets/grafana_agent.go | 7 +------ remote_write/targets/otel.go | 8 ++------ remote_write/targets/prometheus.go | 6 +----- remote_write/targets/telegraf.go | 9 +++------ remote_write/targets/vector.go | 10 ++++------ remote_write/targets/vmagent.go | 7 +------ 7 files changed, 25 insertions(+), 37 deletions(-) diff --git a/remote_write/latest/latest.go b/remote_write/latest/latest.go index b4ecaf27..5a6a02aa 100644 --- a/remote_write/latest/latest.go +++ b/remote_write/latest/latest.go @@ -2,10 +2,10 @@ package latest import ( "encoding/json" - "fmt" "io/ioutil" "log" "net/http" + "regexp" "strings" "time" ) @@ -52,7 +52,18 @@ func GetLatestVersion(repo string) string { } version := strings.Trim(packageInfo.TagName, "v") - fmt.Println("repository: " + repo + " - version: " + version) +// fmt.Println("repository: " + repo + " - version: " + version) return version } + +func GetDownloadURL(url string) string { + re := regexp.MustCompile("https://github.com/(.+?/.+?)/.*") + repo := re.ReplaceAllString(url, "$1") + version := GetLatestVersion(repo) + + re2 := regexp.MustCompile("(.*)VERSION(.*)") + url = re2.ReplaceAllString(url, "${1}" + version + "${2}") + + return url +} diff --git a/remote_write/targets/grafana_agent.go b/remote_write/targets/grafana_agent.go index ce12dfcc..865c1e5f 100644 --- a/remote_write/targets/grafana_agent.go +++ b/remote_write/targets/grafana_agent.go @@ -7,13 +7,8 @@ import ( "github.com/prometheus/compliance/remote_write/latest" ) -func getGrafanaDownloadURL() string { - version := latest.GetLatestVersion("grafana/agent") - return "https://github.com/grafana/agent/releases/download/" + version + "/agent-{{.OS}}-{{.Arch}}.zip" -} - func RunGrafanaAgent(opts TargetOptions) error { - binary, err := downloadBinary(getGrafanaDownloadURL(), "agent-{{.OS}}-{{.Arch}}") + binary, err := downloadBinary(latest.GetDownloadURL("https://github.com/grafana/agent/releases/download/vVERSION/agent-{{.OS}}-{{.Arch}}.zip"), "agent-{{.OS}}-{{.Arch}}") if err != nil { return err } diff --git a/remote_write/targets/otel.go b/remote_write/targets/otel.go index 1addb75a..c6c5f92a 100644 --- a/remote_write/targets/otel.go +++ b/remote_write/targets/otel.go @@ -7,13 +7,9 @@ import ( "github.com/prometheus/compliance/remote_write/latest" ) -func getOtelDownloadURL() string { - version := latest.GetLatestVersion("open-telemetry/opentelemetry-collector") - return "https://github.com/open-telemetry/opentelemetry-collector/releases/download/v" + version + "/otelcol_{{.OS}}_{{.Arch}}" -} - func RunOtelCollector(opts TargetOptions) error { - binary, err := downloadBinary(getOtelDownloadURL(), "") + binary, err := downloadBinary(latest.GetDownloadURL("https://github.com/open-telemetry/opentelemetry-collector/releases/download/vVERSION/otelcol_{{.OS}}_{{.Arch}}"), "") + if err != nil { return err } diff --git a/remote_write/targets/prometheus.go b/remote_write/targets/prometheus.go index 994bccd7..346a12bf 100644 --- a/remote_write/targets/prometheus.go +++ b/remote_write/targets/prometheus.go @@ -6,13 +6,9 @@ import ( "github.com/prometheus/compliance/remote_write/latest" ) -func getPrometheusDownloadURL() string { - version := latest.GetLatestVersion("prometheus/prometheus") - return "https://github.com/prometheus/prometheus/releases/download/v" + version + "/prometheus-" + version + ".{{.OS}}-{{.Arch}}.tar.gz" -} func RunPrometheus(opts TargetOptions) error { - binary, err := downloadBinary(getPrometheusDownloadURL(), "prometheus") + binary, err := downloadBinary(latest.GetDownloadURL("https://github.com/prometheus/prometheus/releases/download/vVERSION/prometheus-VERSION.{{.OS}}-{{.Arch}}.tar.gz"), "prometheus") if err != nil { return err } diff --git a/remote_write/targets/telegraf.go b/remote_write/targets/telegraf.go index 69287d0c..75a716cb 100644 --- a/remote_write/targets/telegraf.go +++ b/remote_write/targets/telegraf.go @@ -7,13 +7,10 @@ import ( "github.com/prometheus/compliance/remote_write/latest" ) -func getTelegrafDownloadURL() string { - version := latest.GetLatestVersion("influxdata/telegraf") - return "https://dl.influxdata.com/telegraf/releases/telegraf-" + version + "_{{.OS}}_{{.Arch}}.tar.gz" -} - func RunTelegraf(opts TargetOptions) error { - binary, err := downloadBinary(getTelegrafDownloadURL(), "telegraf") + version := "1.18.2" + binary, err := downloadBinary(latest.GetDownloadURL("https://dl.influxdata.com/telegraf/releases/telegraf-" + version + "_{{.OS}}_{{.Arch}}.tar.gz"), "telegraf") + fmt.Println("Telegraf version needs to be updated by hand, for now") if err != nil { return err } diff --git a/remote_write/targets/vector.go b/remote_write/targets/vector.go index d6fc28d5..195f48c4 100644 --- a/remote_write/targets/vector.go +++ b/remote_write/targets/vector.go @@ -8,23 +8,21 @@ import ( "github.com/prometheus/compliance/remote_write/latest" ) - func getVectorDownloadURL() string { - version := latest.GetLatestVersion("timberio/vector") switch runtime.GOOS { case "darwin": - return "https://github.com/timberio/vector/releases/download/v" + version + "/vector-" + version + "-x86_64-apple-darwin.tar.gz" + return "https://github.com/timberio/vector/releases/download/vVERSION/vector-VERSION-x86_64-apple-darwin.tar.gz" case "linux": - return "https://github.com/timberio/vector/releases/download/v" + version + "/vector-" + version + "-x86_64-unknown-linux-gnu.tar.gz" + return "https://github.com/timberio/vector/releases/download/vVERSION/vector-VERSION-x86_64-unknown-linux-gnu.tar.gz" case "windows": - return "https://github.com/timberio/vector/releases/download/v" + version + "/vector-" + version + "-x86_64-pc-windows-msvc.zip" + return "https://github.com/timberio/vector/releases/download/vVERSION/vector-VERSION-x86_64-pc-windows-msvc.zip" default: panic("unsupported OS") } } func RunVector(opts TargetOptions) error { - binary, err := downloadBinary(getVectorDownloadURL(), "vector") + binary, err := downloadBinary(latest.GetDownloadURL(getVectorDownloadURL()), "vector") if err != nil { return err } diff --git a/remote_write/targets/vmagent.go b/remote_write/targets/vmagent.go index 61ad6bf9..a7c517f3 100644 --- a/remote_write/targets/vmagent.go +++ b/remote_write/targets/vmagent.go @@ -7,15 +7,10 @@ import ( "github.com/prometheus/compliance/remote_write/latest" ) -func getVMAgentDownloadURL() string { - version := latest.GetLatestVersion("VictoriaMetrics/VictoriaMetrics") - return "https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v" + version + "/vmutils-{{.Arch}}-v1.58.0.tar.gz" -} - func RunVMAgent(opts TargetOptions) error { // NB this won't work on a Mac - need mac builds https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1042! // If you build it yourself and stick it in the bin/ directory, the tests will work. - binary, err := downloadBinary(getVMAgentDownloadURL(), "vmagent-prod") + binary, err := downloadBinary(latest.GetDownloadURL("https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/vVERSION/vmutils-{{.Arch}}-vVERSION.tar.gz"), "vmagent-prod") if err != nil { return err }