From 48092271eb63efca2470ce0d9c12c05756a62b7e Mon Sep 17 00:00:00 2001 From: Duncan McNamara Date: Mon, 2 Mar 2020 15:01:00 +0100 Subject: [PATCH 1/8] Stats: add per country download counter Add entries in the database to count downloads per country. This is not accessible through the stats endpoint. This is intended for a future metrics integration. --- database/utils.go | 34 ++++++++++++++++++++++++++++++++++ http/http.go | 36 +++++++++++++++++++++++++----------- http/stats.go | 34 ++++++++++++++++++++++++++++++---- 3 files changed, 89 insertions(+), 15 deletions(-) diff --git a/database/utils.go b/database/utils.go index f2203514..26b1ca5f 100644 --- a/database/utils.go +++ b/database/utils.go @@ -41,6 +41,40 @@ func (r *Redis) GetListOfMirrors() (map[int]string, error) { return mirrors, nil } +func (r *Redis) GetListOfCountries() ([]string, error) { + conn, err := r.Connect() + if err != nil { + return nil, err + } + defer conn.Close() + + values, err := redis.Values(conn.Do("SMEMBERS", "COUNTRIES")) + if err != nil { + return nil, err + } + + countries := make([]string, len(values)) + for i, v := range values { + value, okValue := v.([]byte) + if !okValue { + return nil, errors.New("invalid type for countries") + } + countries[i] = string(value) + } + + return countries, nil +} + +func (r *Redis) AddCountry(country string) error { + conn, err := r.Connect() + if err != nil { + return err + } + defer conn.Close() + _, err = conn.Do("SADD", "COUNTRIES", country) + return err +} + type NetReadyError struct { error } diff --git a/http/http.go b/http/http.go index 7b3a71e5..c8fbc558 100644 --- a/http/http.go +++ b/http/http.go @@ -246,7 +246,7 @@ func (h *HTTP) requestDispatcher(w http.ResponseWriter, r *http.Request) { // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: -// +// // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above @@ -256,7 +256,7 @@ func (h *HTTP) requestDispatcher(w http.ResponseWriter, r *http.Request) { // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR @@ -275,9 +275,9 @@ func isZeroTime(t time.Time) bool { } func setLastModified(w http.ResponseWriter, modtime time.Time) { - if !isZeroTime(modtime) { - w.Header().Set("Last-Modified", modtime.UTC().Format(TimeFormat)) - } + if !isZeroTime(modtime) { + w.Header().Set("Last-Modified", modtime.UTC().Format(TimeFormat)) + } } func checkIfModifiedSince(r *http.Request, modtime time.Time) condResult { @@ -347,7 +347,13 @@ func (h *HTTP) mirrorHandler(w http.ResponseWriter, r *http.Request, ctx *Contex return } - remoteIP := network.ExtractRemoteIP(r.Header.Get("X-Forwarded-For")) + var remoteIP string + // This http header should only be used in the context of testing geoip + if r.Header.Get("X-Fake-Ip") != "" { + remoteIP = network.ExtractRemoteIP(r.Header.Get("X-Fake-Ip")) + } else { + remoteIP = network.ExtractRemoteIP(r.Header.Get("X-Forwarded-For")) + } if len(remoteIP) == 0 { remoteIP = network.RemoteIPFromAddr(r.RemoteAddr) } @@ -361,6 +367,14 @@ func (h *HTTP) mirrorHandler(w http.ResponseWriter, r *http.Request, ctx *Contex clientInfo := h.geoip.GetRecord(remoteIP) //TODO return a pointer? + if clientInfo.Country != "" { + err = h.redis.AddCountry(clientInfo.Country) + } + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + mlist, excluded, err := h.engine.Selection(ctx, h.cache, &fileInfo, clientInfo) /* Handle errors */ @@ -435,9 +449,9 @@ func (h *HTTP) mirrorHandler(w http.ResponseWriter, r *http.Request, ctx *Contex if len(mlist) > 0 { timeout := GetConfig().SameDownloadInterval if r.Header.Get("Range") == "" || timeout == 0 { - h.stats.CountDownload(mlist[0], fileInfo) + h.stats.CountDownload(mlist[0], fileInfo, clientInfo) } else { - downloaderID := remoteIP+"/"+r.Header.Get("User-Agent") + downloaderID := remoteIP + "/" + r.Header.Get("User-Agent") hash := sha256.New() hash.Write([]byte(downloaderID)) chk := hex.EncodeToString(hash.Sum(nil)) @@ -445,7 +459,7 @@ func (h *HTTP) mirrorHandler(w http.ResponseWriter, r *http.Request, ctx *Contex rconn := h.redis.Get() defer rconn.Close() - tempKey := "DOWNLOADED_"+chk+"_"+urlPath + tempKey := "DOWNLOADED_" + chk + "_" + urlPath prev := "" if h.redis.IsAtLeastVersion("6.2.0") { @@ -461,10 +475,10 @@ func (h *HTTP) mirrorHandler(w http.ResponseWriter, r *http.Request, ctx *Contex // from counting multiple times a single client // downloading a single file in pieces, such as // torrent clients when files are used as web seeds. - h.stats.CountDownload(mlist[0], fileInfo) + h.stats.CountDownload(mlist[0], fileInfo, clientInfo) } - if ! h.redis.IsAtLeastVersion("6.2.0") { + if !h.redis.IsAtLeastVersion("6.2.0") { // Set the key anyway to reset the timer. rconn.Send("SET", tempKey, 1, "EX", timeout) } diff --git a/http/stats.go b/http/stats.go index 513a946b..a61d79f7 100644 --- a/http/stats.go +++ b/http/stats.go @@ -14,6 +14,7 @@ import ( "github.com/etix/mirrorbits/database" "github.com/etix/mirrorbits/filesystem" "github.com/etix/mirrorbits/mirrors" + "github.com/etix/mirrorbits/network" ) /* @@ -51,6 +52,7 @@ type Stats struct { type countItem struct { mirrorID int filepath string + country string size int64 time time.Time } @@ -75,7 +77,7 @@ func (s *Stats) Terminate() { } // CountDownload is a lightweight method used to count a new download for a specific file and mirror -func (s *Stats) CountDownload(m mirrors.Mirror, fileinfo filesystem.FileInfo) error { +func (s *Stats) CountDownload(m mirrors.Mirror, fileinfo filesystem.FileInfo, clientInfo network.GeoIPRecord) error { if m.Name == "" { return errUnknownMirror } @@ -83,7 +85,7 @@ func (s *Stats) CountDownload(m mirrors.Mirror, fileinfo filesystem.FileInfo) er return errEmptyFileError } - s.countChan <- countItem{m.ID, fileinfo.Path, fileinfo.Size, time.Now().UTC()} + s.countChan <- countItem{m.ID, fileinfo.Path, clientInfo.Country, fileinfo.Size, time.Now().UTC()} return nil } @@ -100,9 +102,15 @@ func (s *Stats) processCountDownload() { return case c := <-s.countChan: date := c.time.Format("2006_01_02|") // Includes separator + mirrorID := strconv.Itoa(c.mirrorID) + if c.country == "" { + c.country = "Unknown" + } s.mapStats["f"+date+c.filepath]++ - s.mapStats["m"+date+strconv.Itoa(c.mirrorID)]++ - s.mapStats["s"+date+strconv.Itoa(c.mirrorID)] += c.size + s.mapStats["m"+date+mirrorID]++ + s.mapStats["s"+date+mirrorID] += c.size + s.mapStats["c"+date+c.country]++ + s.mapStats["S"+date+c.country] += c.size case <-pushTicker.C: s.pushStats() } @@ -169,6 +177,24 @@ func (s *Stats) pushStats() { mkey := fmt.Sprintf("STATS_MIRROR_BYTES_%s", date) + for i := 0; i < 4; i++ { + rconn.Send("HINCRBY", mkey, object, v) + mkey = mkey[:strings.LastIndex(mkey, "_")] + } + } else if typ == "c" { + // Country + + mkey := fmt.Sprintf("STATS_COUNTRY_%s", date) + + for i := 0; i < 4; i++ { + rconn.Send("HINCRBY", mkey, object, v) + mkey = mkey[:strings.LastIndex(mkey, "_")] + } + } else if typ == "S" { + // Country Bytes + + mkey := fmt.Sprintf("STATS_COUNTRY_BYTE_%s", date) + for i := 0; i < 4; i++ { rconn.Send("HINCRBY", mkey, object, v) mkey = mkey[:strings.LastIndex(mkey, "_")] From f081f2fb8d5bef05924621f26cecc860b599ed7d Mon Sep 17 00:00:00 2001 From: Duncan McNamara Date: Mon, 2 Mar 2020 15:42:03 +0100 Subject: [PATCH 2/8] Metrics: add prometheus monitoring metrics route This adds a metrics route to monitoring metrics using prometheus. It exports mirror and files stats, as well as the newly introduced country downloads counter. --- config/config.go | 3 + database/utils.go | 24 +++++++ http/http.go | 10 +++ http/metrics.go | 176 ++++++++++++++++++++++++++++++++++++++++++++++ mirrorbits.conf | 7 ++ 5 files changed, 220 insertions(+) create mode 100644 http/metrics.go diff --git a/config/config.go b/config/config.go index 4650575f..42550967 100644 --- a/config/config.go +++ b/config/config.go @@ -60,6 +60,7 @@ func defaultConfig() Configuration { DisableOnMissingFile: false, RPCListenAddress: "localhost:3390", RPCPassword: "", + MetricsEnabled: false, } } @@ -95,6 +96,8 @@ type Configuration struct { RPCListenAddress string `yaml:"RPCListenAddress"` RPCPassword string `yaml:"RPCPassword"` + + MetricsEnabled bool `yaml:"MetricsEnabled"` } type fallback struct { diff --git a/database/utils.go b/database/utils.go index 26b1ca5f..d090bbea 100644 --- a/database/utils.go +++ b/database/utils.go @@ -41,6 +41,30 @@ func (r *Redis) GetListOfMirrors() (map[int]string, error) { return mirrors, nil } +func (r *Redis) GetListOfFiles() ([]string, error) { + conn, err := r.Connect() + if err != nil { + return nil, err + } + defer conn.Close() + + values, err := redis.Values(conn.Do("SMEMBERS", "FILES")) + if err != nil { + return nil, err + } + + files := make([]string, len(values)) + for i, v := range values { + value, okValue := v.([]byte) + if !okValue { + return nil, errors.New("invalid type for file") + } + files[i] = string(value) + } + + return files, nil +} + func (r *Redis) GetListOfCountries() ([]string, error) { conn, err := r.Connect() if err != nil { diff --git a/http/http.go b/http/http.go index c8fbc558..efafb6ee 100644 --- a/http/http.go +++ b/http/http.go @@ -21,6 +21,7 @@ import ( "time" systemd "github.com/coreos/go-systemd/daemon" + "github.com/etix/mirrorbits/config" . "github.com/etix/mirrorbits/config" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/database" @@ -72,6 +73,7 @@ type HTTP struct { Restarting bool stopped bool stoppedMutex sync.Mutex + metrics *Metrics } // Templates is a struct embedding instances of the precompiled templates @@ -95,6 +97,14 @@ func HTTPServer(redis *database.Redis, cache *mirrors.Cache) *HTTP { h.engine = DefaultEngine{} http.Handle("/", NewGzipHandler(h.requestDispatcher)) + if config.GetConfig().MetricsEnabled { + log.Info("Metrics enabled") + h.metrics = NewMetrics(redis) + http.Handle("/metrics", NewGzipHandler(h.metricsHandler)) + } else { + log.Info("Metrics disabled") + } + // Load the GeoIP databases if err := h.geoip.LoadGeoIP(); err != nil { if gerr, ok := err.(network.GeoIPError); ok { diff --git a/http/metrics.go b/http/metrics.go new file mode 100644 index 00000000..e53e801c --- /dev/null +++ b/http/metrics.go @@ -0,0 +1,176 @@ +package http + +import ( + "fmt" + "net/http" + "sync" + "time" + + "github.com/etix/mirrorbits/database" + "github.com/gomodule/redigo/redis" +) + +type Metrics struct { + metricsResponse string + lock sync.Mutex +} + +type metricsUnit struct { + Day int + Month int + Year int + Total int +} + +// NewMetrics returns a new instance of metrics +func NewMetrics(r *database.Redis) *Metrics { + metrics := Metrics{ + metricsResponse: "", + } + go func() { + for { + metrics.getMetrics(r) + time.Sleep(2 * time.Minute) + } + }() + return &metrics +} + +func statsToPrometheusFormat(metrics metricsUnit, labelName string, labelValue string) string { + var output string + + output += fmt.Sprintf("%s_total{%s=\"%s\"} %d\n", labelName, labelName, labelValue, metrics.Total) + output += fmt.Sprintf("%s_day{%s=\"%s\"} %d\n", labelName, labelName, labelValue, metrics.Day) + output += fmt.Sprintf("%s_month{%s=\"%s\"} %d\n", labelName, labelName, labelValue, metrics.Month) + output += fmt.Sprintf("%s_year{%s=\"%s\"} %d\n\n", labelName, labelName, labelValue, metrics.Year) + + return output +} + +func (m *Metrics) getMetrics(httpRedis *database.Redis) { + rconn := httpRedis.Get() + defer rconn.Close() + + // Get all mirrors ID + mirrorsMap, err := httpRedis.GetListOfMirrors() + if err != nil { + log.Error("Cannot fetch the list of mirrors: " + err.Error()) + return + } + + var mirrorsIDs []int + for id := range mirrorsMap { + // We need a common order to iterate the + // results from Redis. + mirrorsIDs = append(mirrorsIDs, id) + } + + rconn.Send("MULTI") + + today := time.Now() + for _, id := range mirrorsIDs { + rconn.Send("HGET", "STATS_MIRROR", id) + rconn.Send("HGET", "STATS_MIRROR_"+today.Format("2006_01_02"), id) + rconn.Send("HGET", "STATS_MIRROR_"+today.Format("2006_01"), id) + rconn.Send("HGET", "STATS_MIRROR_"+today.Format("2006"), id) + } + + stats, err := redis.Values(rconn.Do("EXEC")) + if err != nil { + log.Error("Cannot fetch stats: " + err.Error()) + return + } + + if len(stats) == 0 { + log.Info("Metrics: no files") + return + } + + var index int64 + var output string + for _, id := range mirrorsIDs { + var mirror metricsUnit + mirror.Total, _ = redis.Int(stats[index], err) + mirror.Day, _ = redis.Int(stats[index+1], err) + mirror.Month, _ = redis.Int(stats[index+2], err) + mirror.Year, _ = redis.Int(stats[index+3], err) + output += statsToPrometheusFormat(mirror, "mirror", mirrorsMap[id]) + index += 4 + } + + // Get all files + var fileList []string + fileList, err = httpRedis.GetListOfFiles() + if err != nil { + log.Error("Cannot fetch list of files: " + err.Error()) + return + } + + rconn.Send("MULTI") + for _, file := range fileList { + rconn.Send("HGET", "STATS_FILE", file) + rconn.Send("HGET", "STATS_FILE_"+today.Format("2006_01_02"), file) + rconn.Send("HGET", "STATS_FILE_"+today.Format("2006_01"), file) + rconn.Send("HGET", "STATS_FILE_"+today.Format("2006"), file) + } + + stats, err = redis.Values(rconn.Do("EXEC")) + if err != nil { + log.Error("Cannot fetch stats: " + err.Error()) + return + } + + index = 0 + for _, name := range fileList { + var metrics metricsUnit + metrics.Total, _ = redis.Int(stats[index], err) + metrics.Day, _ = redis.Int(stats[index+1], err) + metrics.Month, _ = redis.Int(stats[index+2], err) + metrics.Year, _ = redis.Int(stats[index+3], err) + output += statsToPrometheusFormat(metrics, "file", name) + index += 4 + } + + // Get all countries + var countryList []string + countryList, err = httpRedis.GetListOfCountries() + if err != nil { + log.Error("Cannot fetch list of countries: " + err.Error()) + return + } + + rconn.Send("MULTI") + for _, country := range countryList { + rconn.Send("HGET", "STATS_COUNTRY", country) + rconn.Send("HGET", "STATS_COUNTRY_"+today.Format("2006_01_02"), country) + rconn.Send("HGET", "STATS_COUNTRY_"+today.Format("2006_01"), country) + rconn.Send("HGET", "STATS_COUNTRY_"+today.Format("2006"), country) + } + stats, err = redis.Values(rconn.Do("EXEC")) + if err != nil { + log.Error("Cannot fetch stats: " + err.Error()) + return + } + + index = 0 + for _, name := range countryList { + var metrics metricsUnit + metrics.Total, _ = redis.Int(stats[index], err) + metrics.Day, _ = redis.Int(stats[index+1], err) + metrics.Month, _ = redis.Int(stats[index+2], err) + metrics.Year, _ = redis.Int(stats[index+3], err) + output += statsToPrometheusFormat(metrics, "country", name) + index += 4 + } + m.lock.Lock() + m.metricsResponse = output + m.lock.Unlock() + return +} + +func (h *HTTP) metricsHandler(w http.ResponseWriter, r *http.Request) { + h.metrics.lock.Lock() + output := h.metrics.metricsResponse + h.metrics.lock.Unlock() + w.Write([]byte(output)) +} diff --git a/mirrorbits.conf b/mirrorbits.conf index eccd358c..6f9b890f 100644 --- a/mirrorbits.conf +++ b/mirrorbits.conf @@ -134,3 +134,10 @@ # - URL: http://fallback2.mirror/repo/ # CountryCode: us # ContinentCode: na + +################### +##### METRICS ##### +################### + +## Enable the metrics route, default to false +## MetricsEnable: false \ No newline at end of file From ff0098b1a47cd610a0751126c17c5d90b297eb02 Mon Sep 17 00:00:00 2001 From: Duncan McNamara Date: Mon, 2 Mar 2020 15:45:52 +0100 Subject: [PATCH 3/8] Metrics: add detailed metrics for selected files This adds a more detailed metrics category for files served by mirrorbits. These tracked files will detail their downloads per country, but due to the increased number of fields required for this, only files selected through the command line will be tracked. This also adds command line arguments to add, delete, and list the files to monitore closely. --- cli/commands.go | 95 +++++++++++- database/utils.go | 48 ++++++ http/http.go | 12 +- http/metrics.go | 235 ++++++++++++++++++++++++++--- http/stats.go | 20 ++- rpc/rpc.go | 71 +++++++++ rpc/rpc.pb.go | 376 ++++++++++++++++++++++++++++++++++------------ rpc/rpc.proto | 11 ++ 8 files changed, 746 insertions(+), 122 deletions(-) diff --git a/cli/commands.go b/cli/commands.go index e151b78a..3c81ca74 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -106,6 +106,7 @@ func (c *cli) CmdHelp() error { {"geoupdate", "Update geolocation of a mirror"}, {"list", "List all mirrors"}, {"logs", "Print logs of a mirror"}, + {"metrics", "Manage metrics"}, {"refresh", "Refresh the local repository"}, {"reload", "Reload configuration"}, {"remove", "Remove a mirror"}, @@ -115,7 +116,7 @@ func (c *cli) CmdHelp() error { {"upgrade", "Seamless binary upgrade"}, {"version", "Print version information"}, } { - help += fmt.Sprintf(" %-10.10s%s\n", command[0], command[1]) + help += fmt.Sprintf(" %-15.15s%s\n", command[0], command[1]) } fmt.Fprintf(os.Stderr, "%s\n", help) return nil @@ -350,6 +351,98 @@ func (c *cli) CmdAdd(args ...string) error { return nil } +func (c *cli) CmdMetrics(args ...string) error { + cmd := SubCmd("metrics", "COMMAND [OPTIONNAL ARGUMENTS]", "Manage metrics") + _ = cmd.String("add", "", "Add a file to the metrics route") + _ = cmd.String("list", "*", "List files in metrics + Optionnal pattern to filter results") + _ = cmd.String("delete", "", "Delete a file from the metrics route") + + // Can't use cmd.Parse(args) because it doesn't handle -command without + // an argument following which is needed for list + if len(args) < 1 || len(args) > 2 { + cmd.Usage() + return nil + } + if args[0] == "-list" { + pattern := "" + if len(args) == 2 { + pattern = args[1] + } + c.CmdListmetrics(pattern) + } else if args[0] == "-add" && len(args) == 2 { + c.CmdAddmetric(args[1]) + } else if args[0] == "-delete" && len(args) == 2 { + c.CmdDelmetric(args[1]) + } else { + cmd.Usage() + return nil + } + return nil +} + +func (c *cli) CmdAddmetric(file string) error { + client := c.GetRPC() + ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) + defer cancel() + _, err := client.AddMetric(ctx, &rpc.Metric{ + Filename: string(file), + }) + if err != nil { + log.Fatal("Error while adding metric: ", err) + } + + fmt.Printf("File %s successfully added to metrics.\n", file) + return nil +} + +func (c *cli) CmdDelmetric(file string) error { + client := c.GetRPC() + ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) + defer cancel() + _, err := client.DelMetric(ctx, &rpc.Metric{ + Filename: string(file), + }) + if err != nil { + log.Fatal("Error while deleting metric: ", err) + } + + fmt.Printf("File %s successfully deleted from metrics.\n", file) + return nil +} + +func (c *cli) CmdListmetrics(pattern string) error { + filterPattern := "*" + if pattern != "" { + filterPattern = "*" + pattern + "*" + } + + client := c.GetRPC() + ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) + defer cancel() + fileList, err := client.ListMetrics(ctx, &rpc.Metric{ + Filename: string(filterPattern), + }) + if err != nil { + log.Fatal("Error while listing metrics: ", err) + } + + w := new(tabwriter.Writer) + w.Init(os.Stdout, 0, 8, 0, '\t', 0) + if len(fileList.Filename) == 0 { + if pattern != "" { + fmt.Println("There are no tracked files matching your request.") + } else { + fmt.Println("There are no tracked files.") + } + } else { + for _, file := range fileList.Filename { + fmt.Println(file) + } + } + + return nil +} + func (c *cli) CmdRemove(args ...string) error { cmd := SubCmd("remove", "IDENTIFIER", "Remove an existing mirror") force := cmd.Bool("f", false, "Never prompt for confirmation") diff --git a/database/utils.go b/database/utils.go index d090bbea..f790ae86 100644 --- a/database/utils.go +++ b/database/utils.go @@ -65,6 +65,30 @@ func (r *Redis) GetListOfFiles() ([]string, error) { return files, nil } +func (r *Redis) GetListOfTrackedFiles() ([]string, error) { + conn, err := r.Connect() + if err != nil { + return nil, err + } + defer conn.Close() + + values, err := redis.Values(conn.Do("SMEMBERS", "TRACKED_FILES")) + if err != nil { + return nil, err + } + + files := make([]string, len(values)) + for i, v := range values { + value, okValue := v.([]byte) + if !okValue { + return nil, errors.New("invalid type for file") + } + files[i] = string(value) + } + + return files, nil +} + func (r *Redis) GetListOfCountries() ([]string, error) { conn, err := r.Connect() if err != nil { @@ -89,6 +113,30 @@ func (r *Redis) GetListOfCountries() ([]string, error) { return countries, nil } +func (r *Redis) GetListOfTrackFilesFields(file string) ([]string, error) { + conn, err := r.Connect() + if err != nil { + return nil, err + } + defer conn.Close() + + values, err := redis.Values(conn.Do("HKEYS", "STATS_TRACKED_"+file)) + if err != nil { + return nil, err + } + + files := make([]string, len(values)) + for i, v := range values { + value, okValue := v.([]byte) + if !okValue { + return nil, errors.New("invalid type for file") + } + files[i] = string(value) + } + + return files, nil +} + func (r *Redis) AddCountry(country string) error { conn, err := r.Connect() if err != nil { diff --git a/http/http.go b/http/http.go index efafb6ee..acc4415a 100644 --- a/http/http.go +++ b/http/http.go @@ -454,12 +454,20 @@ func (h *HTTP) mirrorHandler(w http.ResponseWriter, r *http.Request, ctx *Contex http.Error(w, err.Error(), status) } + var isTracked = false + if config.GetConfig().MetricsEnabled { + isTracked, err = h.metrics.IsFileTracked(fileInfo.Path) + if err != nil { + log.Error("There was a problem fetching the tracked file list: ", err) + } + } + if !ctx.IsMirrorlist() { logs.LogDownload(resultRenderer.Type(), status, results, err) if len(mlist) > 0 { timeout := GetConfig().SameDownloadInterval if r.Header.Get("Range") == "" || timeout == 0 { - h.stats.CountDownload(mlist[0], fileInfo, clientInfo) + h.stats.CountDownload(mlist[0], fileInfo, clientInfo, isTracked) } else { downloaderID := remoteIP + "/" + r.Header.Get("User-Agent") hash := sha256.New() @@ -485,7 +493,7 @@ func (h *HTTP) mirrorHandler(w http.ResponseWriter, r *http.Request, ctx *Contex // from counting multiple times a single client // downloading a single file in pieces, such as // torrent clients when files are used as web seeds. - h.stats.CountDownload(mlist[0], fileInfo, clientInfo) + h.stats.CountDownload(mlist[0], fileInfo, clientInfo, isTracked) } if !h.redis.IsAtLeastVersion("6.2.0") { diff --git a/http/metrics.go b/http/metrics.go index e53e801c..7e66de59 100644 --- a/http/metrics.go +++ b/http/metrics.go @@ -3,6 +3,10 @@ package http import ( "fmt" "net/http" + "regexp" + "sort" + "strconv" + "strings" "sync" "time" @@ -13,6 +17,7 @@ import ( type Metrics struct { metricsResponse string lock sync.Mutex + trackedFileList []string } type metricsUnit struct { @@ -30,12 +35,27 @@ func NewMetrics(r *database.Redis) *Metrics { go func() { for { metrics.getMetrics(r) + trackedFiles, err := r.GetListOfTrackedFiles() + if err != nil { + log.Error(err.Error) + } else { + metrics.trackedFileList = trackedFiles + } time.Sleep(2 * time.Minute) } }() return &metrics } +func (m *Metrics) IsFileTracked(file string) (bool, error) { + for _, v := range m.trackedFileList { + if v == file { + return true, nil + } + } + return false, nil +} + func statsToPrometheusFormat(metrics metricsUnit, labelName string, labelValue string) string { var output string @@ -47,11 +67,52 @@ func statsToPrometheusFormat(metrics metricsUnit, labelName string, labelValue s return output } +func getSimpleMetrics(rconn redis.Conn, fields []string, name string, prefix string) (string, error) { + rconn.Send("MULTI") + today := time.Now() + for _, field := range fields { + rconn.Send("HGET", prefix, field) + rconn.Send("HGET", prefix+"_"+today.Format("2006_01_02"), field) + rconn.Send("HGET", prefix+"_"+today.Format("2006_01"), field) + rconn.Send("HGET", prefix+"_"+today.Format("2006"), field) + } + + stats, err := redis.Values(rconn.Do("EXEC")) + if err != nil { + return "", err + } + + index := 0 + output := "" + for _, field := range fields { + var metrics metricsUnit + metrics.Total, _ = redis.Int(stats[index], err) + metrics.Day, _ = redis.Int(stats[index+1], err) + metrics.Month, _ = redis.Int(stats[index+2], err) + metrics.Year, _ = redis.Int(stats[index+3], err) + output += statsToPrometheusFormat(metrics, name, field) + index += 4 + } + + return output, nil +} + func (m *Metrics) getMetrics(httpRedis *database.Redis) { rconn := httpRedis.Get() defer rconn.Close() - // Get all mirrors ID + // Get all mirrors + // The output will be similar to: + // mirror_total{mirror="ftp-mirror"} 0 + // mirror_day{mirror="ftp-mirror"} 0 + // mirror_month{mirror="ftp-mirror"} 0 + // mirror_year{mirror="ftp-mirror"} 0 + // + // mirror_total{mirror="rsync-mirror"} 1046 + // mirror_day{mirror="rsync-mirror"} 12 + // mirror_month{mirror="rsync-mirror"} 1046 + // mirror_year{mirror="rsync-mirror"} 1046 + mirrorsMap, err := httpRedis.GetListOfMirrors() if err != nil { log.Error("Cannot fetch the list of mirrors: " + err.Error()) @@ -66,7 +127,6 @@ func (m *Metrics) getMetrics(httpRedis *database.Redis) { } rconn.Send("MULTI") - today := time.Now() for _, id := range mirrorsIDs { rconn.Send("HGET", "STATS_MIRROR", id) @@ -99,6 +159,17 @@ func (m *Metrics) getMetrics(httpRedis *database.Redis) { } // Get all files + // The output will be similar to: + // file_total{file="/file_1.txt"} 538 + // file_day{file="/file_1.txt"} 9 + // file_month{file="/file_1.txt"} 538 + // file_year{file="/file_1.txt"} 538 + // + // file_total{file="/file_2.txt"} 508 + // file_day{file="/file_2.txt"} 3 + // file_month{file="/file_2.txt"} 508 + // file_year{file="/file_2.txt"} 508 + var fileList []string fileList, err = httpRedis.GetListOfFiles() if err != nil { @@ -106,15 +177,7 @@ func (m *Metrics) getMetrics(httpRedis *database.Redis) { return } - rconn.Send("MULTI") - for _, file := range fileList { - rconn.Send("HGET", "STATS_FILE", file) - rconn.Send("HGET", "STATS_FILE_"+today.Format("2006_01_02"), file) - rconn.Send("HGET", "STATS_FILE_"+today.Format("2006_01"), file) - rconn.Send("HGET", "STATS_FILE_"+today.Format("2006"), file) - } - - stats, err = redis.Values(rconn.Do("EXEC")) + fileOutput, err := getSimpleMetrics(rconn, fileList, "file", "STATS_FILE") if err != nil { log.Error("Cannot fetch stats: " + err.Error()) return @@ -130,8 +193,19 @@ func (m *Metrics) getMetrics(httpRedis *database.Redis) { output += statsToPrometheusFormat(metrics, "file", name) index += 4 } + output += fileOutput // Get all countries + // The output will be similar to: + // country_total{country="France"} 501 + // country_day{country="France"} 7 + // country_month{country="France"} 501 + // country_year{country="France"} 501 + // + // country_total{country="United States"} 545 + // country_day{country="United States"} 5 + // country_month{country="United States"} 545 + // country_year{country="United States"} 545 var countryList []string countryList, err = httpRedis.GetListOfCountries() if err != nil { @@ -139,28 +213,122 @@ func (m *Metrics) getMetrics(httpRedis *database.Redis) { return } + countryOutput, err := getSimpleMetrics(rconn, countryList, "country", "STATS_COUNTRY") + if err != nil { + log.Error("Cannot fetch stats: " + err.Error()) + return + } + output += countryOutput + + // Get all tracked files download counts + + // Each STATS_TRACKED_* hash has the following format: + // n) Country_MirrorID + // n+1) Download count for the combination mirror / country + + // So that there is no interruption in the count timeseries, every time + // a new country_mirrorid pair is introduced, it must export a value + // even if there is no database hash for the current day. + // Hence we have to get all fields in the current STATS_TRACKED_* hashes + // and complete with fields that were in previous hashes but set to 0. + // For this, fields in STATS_TRACKED_filename without any date (the total) + // can be used + + // The output will be similar to: + // stats_tracked_{file="/file_1.txt",country="France",mirror="rsync-mirror"} 14 + // stats_tracked_{file="/file_1.txt",country="United States",mirror="rsync-mirror"} 11 + // + // stats_tracked_day{file="/file_1.txt",country="France",mirror="rsync-mirror"} 14 + // stats_tracked_day{file="/file_1.txt",country="United States",mirror="rsync-mirror"} 11 + // + // stats_tracked_month{file="/file_1.txt",country="France",mirror="rsync-mirror"} 14 + // stats_tracked_month{file="/file_1.txt",country="United States",mirror="rsync-mirror"} 11 + // + // stats_tracked_year{file="/file_1.txt",country="France",mirror="rsync-mirror"} 14 + // stats_tracked_year{file="/file_1.txt",country="United States",mirror="rsync-mirror"} 11 + // + // stats_top{file="/file_1.txt",country="France",mirror="rsync-mirror"} 14 + // stats_top{file="/file_1.txt",country="United States",mirror="rsync-mirror"} 11 + + mkeyList := make([]string, 0) + mkeyFileMap := make(map[string]string) + fileFieldsMap := make(map[string][]string) + + // Get tracked files downloads per country and mirror + for _, file := range m.trackedFileList { + mkey := "STATS_TRACKED_" + file + "_" + today.Format("2006_01_02") + // Getting all Country_MirrorID fields for that file + fileFieldsMap[file], err = httpRedis.GetListOfTrackFilesFields(file) + if err != nil { + log.Error("Failed to fetch file fields: ", err.Error()) + return + } + for i := 0; i < 4; i++ { + mkeyList = append(mkeyList, mkey) + mkeyFileMap[mkey] = file + mkey = mkey[:strings.LastIndex(mkey, "_")] + } + } + rconn.Send("MULTI") - for _, country := range countryList { - rconn.Send("HGET", "STATS_COUNTRY", country) - rconn.Send("HGET", "STATS_COUNTRY_"+today.Format("2006_01_02"), country) - rconn.Send("HGET", "STATS_COUNTRY_"+today.Format("2006_01"), country) - rconn.Send("HGET", "STATS_COUNTRY_"+today.Format("2006"), country) + for _, mkey := range mkeyList { + rconn.Send("HGETALL", mkey) } stats, err = redis.Values(rconn.Do("EXEC")) if err != nil { - log.Error("Cannot fetch stats: " + err.Error()) + log.Error("Cannot fetch files per country stats: " + err.Error()) return } index = 0 - for _, name := range countryList { - var metrics metricsUnit - metrics.Total, _ = redis.Int(stats[index], err) - metrics.Day, _ = redis.Int(stats[index+1], err) - metrics.Month, _ = redis.Int(stats[index+2], err) - metrics.Year, _ = redis.Int(stats[index+3], err) - output += statsToPrometheusFormat(metrics, "country", name) - index += 4 + // Get all download count information from the previous redis querries + for _, mkey := range mkeyList { + hashSlice, _ := redis.ByteSlices(stats[index], err) + file := mkeyFileMap[mkey] + fieldList := make([]string, len(fileFieldsMap[file])) + copy(fieldList, fileFieldsMap[file]) + sort.Strings(fieldList) + // This loop gets the download count from the database + for i := 0; i < len(hashSlice); i += 2 { + field, _ := redis.String(hashSlice[i], err) + value, _ := redis.Int(hashSlice[i+1], err) + sep := strings.Index(field, "_") + country := field[:sep] + mirrorID, err := strconv.Atoi(field[sep+1:]) + if err != nil { + log.Error("Failed to convert mirror ID: ", err) + return + } + mirror := mirrorsMap[mirrorID] + durationStr := getDurationFromKey(mkey) + output += fmt.Sprintf("stats_tracked_"+ + "%s{file=\"%s\",country=\"%s\",mirror=\"%s\"} %d\n", + durationStr, mkeyFileMap[mkey], country, mirror, value, + ) + fieldIndex := sort.SearchStrings(fieldList, field) + if fieldIndex < len(fieldList) { + fieldList = append(fieldList[:fieldIndex], fieldList[fieldIndex+1:]...) + } + } + // This loop set the download count to 0 as their is no current entry + // in the database of downloads by these mirrors and countries + for _, field := range fieldList { + sep := strings.Index(field, "_") + country := field[:sep] + mirrorID, err := strconv.Atoi(field[sep+1:]) + if err != nil { + log.Error("Failed to convert mirror ID: ", err) + return + } + mirror := mirrorsMap[mirrorID] + durationStr := getDurationFromKey(mkey) + output += fmt.Sprintf("stats_tracked_"+ + "%s{file=\"%s\",country=\"%s\",mirror=\"%s\"} %d\n", + durationStr, mkeyFileMap[mkey], country, mirror, 0, + ) + } + index++ + output += "\n" } m.lock.Lock() m.metricsResponse = output @@ -174,3 +342,20 @@ func (h *HTTP) metricsHandler(w http.ResponseWriter, r *http.Request) { h.metrics.lock.Unlock() w.Write([]byte(output)) } + +func getDurationFromKey(key string) string { + dayPattern := regexp.MustCompile("([0-9]{4}_[0-9]{2}_[0-9]{2})\\b") + monthPattern := regexp.MustCompile("([0-9]{4}_[0-9]{2})\\b") + yearPattern := regexp.MustCompile("([0-9]{4})\\b") + var durationStr string + if len(dayPattern.FindAllStringIndex(key, -1)) > 0 { + durationStr = "day" + } else if len(monthPattern.FindAllStringIndex(key, -1)) > 0 { + durationStr = "month" + } else if len(yearPattern.FindAllStringIndex(key, -1)) > 0 { + durationStr = "year" + } else { + durationStr = "total" + } + return durationStr +} diff --git a/http/stats.go b/http/stats.go index a61d79f7..b6a913e2 100644 --- a/http/stats.go +++ b/http/stats.go @@ -55,6 +55,7 @@ type countItem struct { country string size int64 time time.Time + tracked bool } // NewStats returns an instance of the stats counter @@ -77,7 +78,7 @@ func (s *Stats) Terminate() { } // CountDownload is a lightweight method used to count a new download for a specific file and mirror -func (s *Stats) CountDownload(m mirrors.Mirror, fileinfo filesystem.FileInfo, clientInfo network.GeoIPRecord) error { +func (s *Stats) CountDownload(m mirrors.Mirror, fileinfo filesystem.FileInfo, clientInfo network.GeoIPRecord, isTracked bool) error { if m.Name == "" { return errUnknownMirror } @@ -85,7 +86,7 @@ func (s *Stats) CountDownload(m mirrors.Mirror, fileinfo filesystem.FileInfo, cl return errEmptyFileError } - s.countChan <- countItem{m.ID, fileinfo.Path, clientInfo.Country, fileinfo.Size, time.Now().UTC()} + s.countChan <- countItem{m.ID, fileinfo.Path, clientInfo.Country, fileinfo.Size, time.Now().UTC(), isTracked} return nil } @@ -111,6 +112,9 @@ func (s *Stats) processCountDownload() { s.mapStats["s"+date+mirrorID] += c.size s.mapStats["c"+date+c.country]++ s.mapStats["S"+date+c.country] += c.size + if c.tracked { + s.mapStats["F"+date+c.filepath+"|"+c.country+"_"+mirrorID]++ + } case <-pushTicker.C: s.pushStats() } @@ -199,6 +203,18 @@ func (s *Stats) pushStats() { rconn.Send("HINCRBY", mkey, object, v) mkey = mkey[:strings.LastIndex(mkey, "_")] } + } else if typ == "F" { + // File downloads per country + + sep := strings.LastIndex(object, "|") + file := object[:sep] + key := object[sep+1:] + mkey := fmt.Sprintf("STATS_TRACKED_%s_%s", file, date) + + for i := 0; i < 4; i++ { + rconn.Send("HINCRBY", mkey, key, v) + mkey = mkey[:strings.LastIndex(mkey, "_")] + } } else { log.Warning("Stats: unknown type", typ) } diff --git a/rpc/rpc.go b/rpc/rpc.go index cb9d7268..5c19d96a 100644 --- a/rpc/rpc.go +++ b/rpc/rpc.go @@ -803,3 +803,74 @@ func (c *CLI) GetMirrorLogs(ctx context.Context, in *GetMirrorLogsRequest) (*Get return &GetMirrorLogsReply{Line: lines}, nil } + +func (c *CLI) AddMetric(ctx context.Context, in *Metric) (*empty.Empty, error) { + conn, err := c.redis.Connect() + if err != nil { + return nil, err + } + defer conn.Close() + + exists, err := redis.Int(conn.Do("SISMEMBER", "FILES", in.Filename)) + if err != nil { + return nil, errors.Wrap(err, "failed to check for file presence") + } else if exists == 0 { + return nil, status.Error(codes.FailedPrecondition, + "file does not exist") + } + + exists, err = redis.Int(conn.Do("SADD", "TRACKED_FILES", in.Filename)) + if err != nil { + return nil, errors.Wrap(err, "failed to add file to metrics") + } else if exists == 0 { + return nil, status.Error(codes.AlreadyExists, + "file already is in metrics") + } + + return &empty.Empty{}, nil +} + +func (c *CLI) DelMetric(ctx context.Context, in *Metric) (*empty.Empty, error) { + conn, err := c.redis.Connect() + if err != nil { + return nil, err + } + defer conn.Close() + + exists, err := redis.Int(conn.Do("SISMEMBER", "TRACKED_FILES", in.Filename)) + if err != nil { + return nil, errors.Wrap(err, "failed to check for file presence") + } else if exists == 0 { + return nil, status.Error(codes.FailedPrecondition, + "file is not part of tracked files") + } + + _, err = conn.Do("SREM", "TRACKED_FILES", in.Filename) + if err != nil { + return nil, errors.Wrap(err, "failed to remove file from tracked files") + } + + return &empty.Empty{}, nil +} + +func (c *CLI) ListMetrics(ctx context.Context, in *Metric) (*MetricsList, error) { + conn, err := c.redis.Connect() + if err != nil { + return nil, err + } + defer conn.Close() + + values, err := redis.Values(conn.Do("SSCAN", "TRACKED_FILES", + "0", "MATCH", in.Filename, "COUNT", 1000)) + if err != nil { + return nil, err + } + + byteValues, err := redis.ByteSlices(values[1], err) + files := make([]string, len(byteValues)) + for i, v := range byteValues { + files[i] = string(v) + } + + return &MetricsList{Filename: files}, nil +} diff --git a/rpc/rpc.pb.go b/rpc/rpc.pb.go index e5e8aadb..88834eac 100644 --- a/rpc/rpc.pb.go +++ b/rpc/rpc.pb.go @@ -1282,6 +1282,84 @@ func (m *GetMirrorLogsReply) GetLine() []string { return nil } +type Metric struct { + Filename string `protobuf:"bytes,1,opt,name=Filename,proto3" json:"Filename,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Metric) Reset() { *m = Metric{} } +func (m *Metric) String() string { return proto.CompactTextString(m) } +func (*Metric) ProtoMessage() {} +func (*Metric) Descriptor() ([]byte, []int) { + return fileDescriptor_77a6da22d6a3feb1, []int{20} +} + +func (m *Metric) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Metric.Unmarshal(m, b) +} +func (m *Metric) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Metric.Marshal(b, m, deterministic) +} +func (m *Metric) XXX_Merge(src proto.Message) { + xxx_messageInfo_Metric.Merge(m, src) +} +func (m *Metric) XXX_Size() int { + return xxx_messageInfo_Metric.Size(m) +} +func (m *Metric) XXX_DiscardUnknown() { + xxx_messageInfo_Metric.DiscardUnknown(m) +} + +var xxx_messageInfo_Metric proto.InternalMessageInfo + +func (m *Metric) GetFilename() string { + if m != nil { + return m.Filename + } + return "" +} + +type MetricsList struct { + Filename []string `protobuf:"bytes,1,rep,name=Filename,proto3" json:"Filename,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *MetricsList) Reset() { *m = MetricsList{} } +func (m *MetricsList) String() string { return proto.CompactTextString(m) } +func (*MetricsList) ProtoMessage() {} +func (*MetricsList) Descriptor() ([]byte, []int) { + return fileDescriptor_77a6da22d6a3feb1, []int{21} +} + +func (m *MetricsList) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_MetricsList.Unmarshal(m, b) +} +func (m *MetricsList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_MetricsList.Marshal(b, m, deterministic) +} +func (m *MetricsList) XXX_Merge(src proto.Message) { + xxx_messageInfo_MetricsList.Merge(m, src) +} +func (m *MetricsList) XXX_Size() int { + return xxx_messageInfo_MetricsList.Size(m) +} +func (m *MetricsList) XXX_DiscardUnknown() { + xxx_messageInfo_MetricsList.DiscardUnknown(m) +} + +var xxx_messageInfo_MetricsList proto.InternalMessageInfo + +func (m *MetricsList) GetFilename() []string { + if m != nil { + return m.Filename + } + return nil +} + func init() { proto.RegisterEnum("ScanMirrorRequest_Method", ScanMirrorRequest_Method_name, ScanMirrorRequest_Method_value) proto.RegisterType((*VersionReply)(nil), "VersionReply") @@ -1305,6 +1383,8 @@ func init() { proto.RegisterType((*StatsMirrorReply)(nil), "StatsMirrorReply") proto.RegisterType((*GetMirrorLogsRequest)(nil), "GetMirrorLogsRequest") proto.RegisterType((*GetMirrorLogsReply)(nil), "GetMirrorLogsReply") + proto.RegisterType((*Metric)(nil), "Metric") + proto.RegisterType((*MetricsList)(nil), "MetricsList") } func init() { @@ -1312,98 +1392,102 @@ func init() { } var fileDescriptor_77a6da22d6a3feb1 = []byte{ - // 1445 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0x4d, 0x73, 0x1b, 0x45, - 0x13, 0xd6, 0x4a, 0xb6, 0x65, 0xb5, 0x64, 0x5b, 0x1e, 0x3b, 0x7e, 0x37, 0x4a, 0xde, 0x44, 0x19, - 0x3e, 0x22, 0x8a, 0x62, 0x43, 0x4c, 0x02, 0xae, 0x10, 0xa0, 0x84, 0xfc, 0x11, 0x83, 0x1c, 0xbb, - 0x56, 0x31, 0x14, 0xdc, 0x36, 0xda, 0x91, 0xbc, 0xc5, 0x6a, 0x47, 0xec, 0x8c, 0x12, 0xab, 0x8a, - 0x9f, 0x41, 0x71, 0xe2, 0x00, 0x3f, 0x80, 0x2a, 0x8e, 0xfc, 0x3c, 0xaa, 0x67, 0x66, 0xa5, 0xd5, - 0xca, 0x1f, 0x90, 0x03, 0xb7, 0xed, 0xa7, 0x7b, 0xa6, 0x7b, 0x7a, 0xba, 0x9f, 0x9e, 0x85, 0x52, - 0x3c, 0xec, 0x3a, 0xc3, 0x98, 0x4b, 0x5e, 0xbb, 0xd5, 0xe7, 0xbc, 0x1f, 0xb2, 0x07, 0x4a, 0x7a, - 0x39, 0xea, 0x3d, 0x60, 0x83, 0xa1, 0x1c, 0x1b, 0xe5, 0xdd, 0xac, 0x52, 0x06, 0x03, 0x26, 0xa4, - 0x37, 0x18, 0x6a, 0x03, 0xfa, 0x9b, 0x05, 0x95, 0x6f, 0x58, 0x2c, 0x02, 0x1e, 0xb9, 0x6c, 0x18, - 0x8e, 0x89, 0x0d, 0x45, 0x23, 0xdb, 0x56, 0xdd, 0x6a, 0x94, 0xdc, 0x44, 0x24, 0x9b, 0xb0, 0xf8, - 0xe5, 0x28, 0x08, 0x7d, 0x3b, 0xaf, 0x70, 0x2d, 0x90, 0xdb, 0x50, 0x3a, 0xe0, 0xc9, 0x8a, 0x82, - 0xd2, 0x4c, 0x01, 0xb2, 0x0a, 0xf9, 0xe3, 0x8e, 0xbd, 0xa0, 0xe0, 0xfc, 0x71, 0x87, 0x10, 0x58, - 0x68, 0xc6, 0xdd, 0x33, 0x7b, 0x51, 0x21, 0xea, 0x9b, 0xdc, 0x01, 0x38, 0xe0, 0x47, 0xde, 0xf9, - 0x49, 0xcc, 0xbb, 0xc2, 0x5e, 0xaa, 0x5b, 0x8d, 0x45, 0x37, 0x85, 0xd0, 0x06, 0x54, 0x8e, 0x3c, - 0xd9, 0x3d, 0x73, 0xd9, 0x8f, 0x23, 0x26, 0x24, 0x46, 0x78, 0xe2, 0x49, 0xc9, 0xe2, 0x49, 0x84, - 0x46, 0xa4, 0xbf, 0x2c, 0xc3, 0xd2, 0x51, 0x10, 0xc7, 0x3c, 0x46, 0xc7, 0x87, 0xbb, 0x4a, 0xbf, - 0xe8, 0xe6, 0x0f, 0x77, 0xd1, 0xf1, 0x73, 0x6f, 0xc0, 0x4c, 0xec, 0xea, 0x1b, 0x37, 0x7a, 0x26, - 0xe5, 0xf0, 0xd4, 0x6d, 0x9b, 0xc0, 0x13, 0x91, 0xd4, 0x60, 0xd9, 0x15, 0xe3, 0xa8, 0x8b, 0x2a, - 0x1d, 0xfc, 0x44, 0x26, 0x5b, 0xb0, 0xb4, 0xaf, 0x17, 0xe9, 0x43, 0x18, 0x89, 0xd4, 0xa1, 0xdc, - 0x19, 0xf2, 0x48, 0xf0, 0x58, 0x39, 0x5a, 0x52, 0xca, 0x34, 0x84, 0x07, 0x35, 0x22, 0xae, 0x2e, - 0x2a, 0x83, 0x14, 0x42, 0xde, 0x85, 0x55, 0x23, 0xb5, 0x79, 0x9f, 0xa3, 0xcd, 0xb2, 0xb2, 0xc9, - 0xa0, 0x98, 0xf2, 0xa6, 0x3f, 0x08, 0x22, 0xe5, 0xa7, 0xa4, 0x53, 0x3e, 0x01, 0xd0, 0x8b, 0x12, - 0xf6, 0x06, 0x5e, 0x10, 0xda, 0xa0, 0xbd, 0x4c, 0x11, 0xd4, 0xb7, 0x46, 0x42, 0xf2, 0xc1, 0xae, - 0x27, 0x3d, 0xbb, 0xac, 0xf5, 0x53, 0x84, 0xbc, 0x0d, 0x2b, 0x2d, 0x1e, 0xc9, 0x20, 0x62, 0x91, - 0x3c, 0x8e, 0xc2, 0xb1, 0x5d, 0xa9, 0x5b, 0x8d, 0x65, 0x77, 0x16, 0xc4, 0xd3, 0xb6, 0xf8, 0x28, - 0x92, 0xf1, 0x58, 0xd9, 0xac, 0x28, 0x9b, 0x34, 0x84, 0x79, 0x6a, 0x76, 0x94, 0x72, 0x55, 0x29, - 0x8d, 0x84, 0x65, 0xd4, 0xe9, 0xf2, 0x98, 0xd9, 0x6b, 0xea, 0x72, 0xb4, 0x80, 0x19, 0x6f, 0x7b, - 0x32, 0x90, 0x23, 0x9f, 0xd9, 0xd5, 0xba, 0xd5, 0xc8, 0xbb, 0x13, 0x19, 0xcf, 0xdb, 0xe6, 0x51, - 0x5f, 0x2b, 0xd7, 0x95, 0x72, 0x0a, 0xcc, 0xc4, 0xdb, 0xe2, 0x3e, 0xb3, 0x89, 0x3a, 0xd2, 0x2c, - 0x48, 0x28, 0x54, 0x4c, 0x70, 0x28, 0x0a, 0x7b, 0x43, 0x19, 0xcd, 0x60, 0x64, 0x1b, 0x36, 0xf7, - 0xce, 0xbb, 0xe1, 0xc8, 0x67, 0xfe, 0x8c, 0xed, 0xa6, 0xb2, 0xbd, 0x50, 0x87, 0xa7, 0x69, 0x8a, - 0x68, 0x34, 0xb0, 0x6f, 0xd4, 0xad, 0xc6, 0x8a, 0xab, 0x05, 0xac, 0xac, 0x16, 0x1f, 0x0c, 0x58, - 0x24, 0xed, 0x2d, 0x5d, 0x59, 0x46, 0x44, 0xcd, 0x5e, 0xe4, 0xbd, 0x0c, 0x99, 0x6f, 0xff, 0x4f, - 0xa5, 0x25, 0x11, 0xb1, 0x62, 0x4f, 0x87, 0xb6, 0xad, 0xc0, 0xfc, 0xe9, 0x10, 0xcf, 0x65, 0x3c, - 0xba, 0xcc, 0x13, 0x3c, 0xb2, 0x6f, 0xea, 0x73, 0xcd, 0x80, 0xe4, 0x09, 0x40, 0x47, 0x7a, 0x92, - 0x75, 0x82, 0xa8, 0xcb, 0xec, 0x5a, 0xdd, 0x6a, 0x94, 0xb7, 0x6b, 0x8e, 0xee, 0x7a, 0x27, 0xe9, - 0x7a, 0xe7, 0x45, 0xd2, 0xf5, 0x6e, 0xca, 0x1a, 0xeb, 0xad, 0x19, 0x86, 0xfc, 0xb5, 0xcb, 0xfc, - 0x20, 0x66, 0x5d, 0x29, 0xec, 0x5b, 0xea, 0x4a, 0x32, 0x28, 0xf9, 0x18, 0xef, 0x46, 0xc8, 0xce, - 0x38, 0xea, 0xda, 0xb7, 0xaf, 0xf5, 0x30, 0xb1, 0x25, 0x5f, 0x01, 0x51, 0xdf, 0xa3, 0x6e, 0x97, - 0x09, 0xd1, 0x1b, 0x85, 0x6a, 0x87, 0xff, 0x5f, 0xbb, 0xc3, 0x05, 0xab, 0xc8, 0x53, 0x28, 0x23, - 0x7a, 0xc4, 0x7d, 0xb4, 0xb3, 0xef, 0x5c, 0xbb, 0x49, 0xda, 0x9c, 0x3e, 0x82, 0x35, 0xcd, 0x0b, - 0xed, 0x40, 0x48, 0xcd, 0x73, 0xf7, 0xa0, 0xa8, 0x21, 0x61, 0x5b, 0xf5, 0x42, 0xa3, 0xbc, 0x5d, - 0x74, 0xb4, 0xec, 0x26, 0x38, 0x75, 0x60, 0x59, 0x7f, 0x1e, 0xee, 0xfe, 0x13, 0x3e, 0xa1, 0x0f, - 0x01, 0x0c, 0x51, 0xa1, 0x83, 0xb7, 0xb2, 0x0e, 0x4a, 0x4e, 0xb2, 0xdb, 0xd4, 0xc5, 0x17, 0xb0, - 0xd1, 0x3a, 0xf3, 0xa2, 0x3e, 0xc3, 0x6b, 0x19, 0x89, 0x84, 0xe2, 0xb2, 0xde, 0x52, 0x55, 0x93, - 0x9f, 0xa9, 0x1a, 0x7a, 0x2f, 0x39, 0xd9, 0xe1, 0xee, 0x25, 0x8b, 0xe9, 0x9f, 0x16, 0xac, 0x36, - 0x7d, 0xdf, 0x9c, 0x4e, 0xc5, 0x96, 0xee, 0x36, 0xeb, 0xaa, 0x6e, 0xcb, 0x67, 0xbb, 0x4d, 0x55, - 0xb6, 0xaa, 0xff, 0x84, 0x33, 0x8d, 0x88, 0xeb, 0x26, 0x2d, 0x67, 0x48, 0x73, 0x0a, 0x90, 0x2a, - 0x14, 0x9a, 0x9d, 0xe7, 0x86, 0x32, 0xf1, 0x13, 0x63, 0xf8, 0xd6, 0x8b, 0xa3, 0x20, 0xea, 0x23, - 0xe9, 0x17, 0x90, 0x63, 0x13, 0x99, 0xde, 0x87, 0xf5, 0xd3, 0xa1, 0xef, 0x49, 0x96, 0x0e, 0x9a, - 0xc0, 0xc2, 0x6e, 0xd0, 0xeb, 0x19, 0xd2, 0x57, 0xdf, 0xb4, 0x0f, 0x9b, 0x07, 0x8c, 0xcf, 0xdb, - 0xde, 0x4d, 0x06, 0x81, 0xb2, 0x4e, 0x5d, 0x6e, 0x32, 0x1f, 0x92, 0xcd, 0xf2, 0xd3, 0xcd, 0x66, - 0x22, 0x2a, 0x64, 0x22, 0xda, 0x06, 0xdb, 0x65, 0xbd, 0x98, 0x09, 0xbc, 0x5d, 0x2e, 0x02, 0xc9, - 0xe3, 0x71, 0x92, 0xf0, 0x2d, 0x58, 0x72, 0xd9, 0x99, 0x27, 0xce, 0x94, 0xb3, 0x65, 0xd7, 0x48, - 0xf4, 0x77, 0x0b, 0xd6, 0x3b, 0x5d, 0x2f, 0x4a, 0x02, 0xbb, 0xf8, 0x6e, 0x91, 0xaf, 0x47, 0x92, - 0xeb, 0x0b, 0x35, 0xd7, 0x9b, 0x42, 0xc8, 0x63, 0x58, 0x3e, 0xc1, 0xf2, 0xee, 0xf2, 0x50, 0xa5, - 0x7c, 0x75, 0xfb, 0xa6, 0x33, 0xb7, 0xab, 0x73, 0xc4, 0xe4, 0x19, 0xf7, 0xdd, 0x89, 0x29, 0x7d, - 0x07, 0x96, 0x34, 0x46, 0x8a, 0x50, 0x68, 0xb6, 0xdb, 0xd5, 0x1c, 0x7e, 0xec, 0xbf, 0x38, 0xa9, - 0x5a, 0xa4, 0x04, 0x8b, 0x6e, 0xe7, 0xbb, 0xe7, 0xad, 0x6a, 0x9e, 0xfe, 0x61, 0xc1, 0x5a, 0x7a, - 0x37, 0xf3, 0x04, 0x48, 0xaa, 0xcd, 0x9a, 0xe5, 0x28, 0x0a, 0x95, 0xfd, 0x20, 0x64, 0xe2, 0x30, - 0xf2, 0xd9, 0xb9, 0x29, 0xc6, 0x82, 0x3b, 0x83, 0xa1, 0xcd, 0xd7, 0x11, 0x7f, 0x1d, 0x25, 0x36, - 0x05, 0x6d, 0x93, 0xc6, 0xd0, 0x83, 0xcb, 0x06, 0xfc, 0x15, 0xf3, 0x55, 0xa5, 0x14, 0xdc, 0x44, - 0xc4, 0x6c, 0xbc, 0xf8, 0xfe, 0xb8, 0xd7, 0x13, 0x4c, 0x1e, 0x09, 0x55, 0x2e, 0x05, 0x37, 0x85, - 0xd0, 0x5f, 0x2d, 0xa8, 0x62, 0xaf, 0x08, 0xf4, 0x79, 0xed, 0x8b, 0x80, 0xec, 0x40, 0x69, 0x17, - 0xf9, 0x4e, 0x7a, 0xb1, 0x54, 0xd1, 0x5e, 0x4d, 0x1a, 0x53, 0x63, 0xf2, 0x08, 0x8a, 0x28, 0xec, - 0x45, 0xfa, 0x04, 0x57, 0xaf, 0x4b, 0x4c, 0xe9, 0x4f, 0xb0, 0x9a, 0x8a, 0x0e, 0x93, 0xf9, 0x21, - 0x2c, 0xf6, 0x30, 0x3d, 0x86, 0x04, 0x6a, 0xce, 0xac, 0xde, 0x51, 0xb9, 0xdb, 0xc3, 0x0e, 0x72, - 0xb5, 0x61, 0x6d, 0x07, 0x60, 0x0a, 0x62, 0xe3, 0xfc, 0xc0, 0xc6, 0xe6, 0x5c, 0xf8, 0x89, 0x23, - 0xe7, 0x95, 0x17, 0x8e, 0x98, 0xc9, 0xbe, 0x16, 0x9e, 0xe4, 0x77, 0x2c, 0xfa, 0xb3, 0x05, 0x44, - 0x6d, 0x7f, 0x75, 0xc5, 0xfd, 0xd7, 0x49, 0x61, 0xe6, 0xca, 0xfe, 0x55, 0x83, 0xe2, 0x13, 0x4c, - 0xc7, 0x2f, 0xcc, 0x41, 0x27, 0xb2, 0x7a, 0x89, 0x8e, 0x25, 0x13, 0xa6, 0xb6, 0xb4, 0x40, 0xf7, - 0x91, 0x0b, 0xa4, 0xe1, 0x79, 0xde, 0x17, 0x57, 0x34, 0xdc, 0x91, 0x77, 0xee, 0x32, 0x31, 0x0a, - 0xcd, 0xde, 0x8b, 0x6e, 0x0a, 0xa1, 0x0d, 0x20, 0x99, 0x7d, 0x0c, 0xfb, 0x84, 0x41, 0xc4, 0xd4, - 0x35, 0x96, 0x5c, 0xf5, 0xbd, 0xfd, 0x57, 0x11, 0x0a, 0xad, 0xf6, 0x21, 0x79, 0x0c, 0x70, 0xc0, - 0x64, 0xf2, 0xe6, 0xdd, 0x9a, 0xcb, 0xc9, 0x1e, 0xbe, 0xc8, 0x6b, 0x2b, 0x4e, 0xfa, 0xa1, 0x4d, - 0x73, 0xe4, 0x53, 0x28, 0x9e, 0x0e, 0xfb, 0xb1, 0xe7, 0xb3, 0x4b, 0xd7, 0x5c, 0x82, 0xd3, 0x1c, - 0x79, 0x82, 0xa4, 0x13, 0x72, 0xcf, 0x7f, 0x83, 0xb5, 0x9f, 0x43, 0x25, 0x3d, 0x75, 0xc8, 0xa6, - 0x73, 0xc1, 0x10, 0xba, 0x62, 0xfd, 0x36, 0x2c, 0xe0, 0x20, 0xbd, 0xd4, 0x73, 0xd5, 0xc9, 0x4c, - 0x5b, 0x9a, 0x23, 0xef, 0x01, 0x98, 0x41, 0x15, 0xf5, 0x38, 0xa9, 0x3a, 0x99, 0xa9, 0x55, 0x4b, - 0x0a, 0x80, 0xe6, 0xc8, 0x7d, 0x7c, 0xdf, 0x9a, 0x79, 0x45, 0x12, 0xbc, 0xb6, 0xe6, 0xcc, 0x0e, - 0x31, 0x9a, 0x23, 0x1f, 0x40, 0x25, 0x4d, 0xfd, 0x53, 0x5b, 0xe2, 0xcc, 0x8d, 0x04, 0x95, 0xb2, - 0x8a, 0xa6, 0x19, 0x63, 0x3e, 0x1f, 0xc4, 0xe5, 0x47, 0x7e, 0x0a, 0x6b, 0x99, 0x41, 0x73, 0xc1, - 0xf2, 0x1b, 0xce, 0x45, 0xc3, 0x88, 0xe6, 0xc8, 0x33, 0x58, 0x9f, 0x9b, 0x1e, 0xe4, 0xa6, 0x73, - 0xd9, 0x44, 0xb9, 0x22, 0x8e, 0x47, 0x00, 0x53, 0xba, 0x26, 0x64, 0x7e, 0x12, 0xd4, 0xaa, 0x4e, - 0x86, 0xcf, 0x69, 0x8e, 0x3c, 0x84, 0xd2, 0x84, 0x76, 0xc8, 0xba, 0x93, 0x25, 0xd0, 0xda, 0x5a, - 0x86, 0x95, 0x68, 0x8e, 0x7c, 0x02, 0xe5, 0x54, 0xd3, 0x92, 0x0d, 0x67, 0x9e, 0x58, 0x6a, 0xeb, - 0x4e, 0xb6, 0xaf, 0x69, 0x8e, 0xec, 0xc0, 0xc2, 0x49, 0x10, 0xf5, 0xdf, 0xa0, 0x2c, 0x3f, 0x83, - 0x95, 0x99, 0xc6, 0x23, 0x98, 0xcf, 0xf9, 0x86, 0xae, 0x6d, 0x38, 0xf3, 0xfd, 0x49, 0x73, 0xe4, - 0x7d, 0x28, 0xab, 0xe7, 0x97, 0x89, 0x78, 0xc5, 0x49, 0xff, 0x35, 0xd6, 0xca, 0xce, 0xf4, 0x6d, - 0x46, 0x73, 0x2f, 0x97, 0x94, 0xf7, 0x8f, 0xfe, 0x0e, 0x00, 0x00, 0xff, 0xff, 0xc5, 0xae, 0x97, - 0x9a, 0x49, 0x0f, 0x00, 0x00, + // 1507 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0x51, 0x73, 0xdb, 0xc4, + 0x13, 0xb7, 0xec, 0x24, 0xb6, 0xd7, 0x4e, 0xe2, 0x5c, 0xd2, 0xfc, 0x55, 0xb7, 0xff, 0xd6, 0x3d, + 0x4a, 0xeb, 0x0e, 0x83, 0x4a, 0x43, 0x0b, 0x99, 0x52, 0x60, 0x8c, 0x9d, 0xa4, 0x01, 0xa7, 0xc9, + 0xc8, 0x0d, 0x0c, 0xbc, 0xa9, 0xd6, 0xd9, 0xd1, 0x20, 0xeb, 0x8c, 0x74, 0x6e, 0xe3, 0x19, 0x3e, + 0x06, 0xc3, 0x13, 0x0f, 0xf0, 0x01, 0x98, 0xe1, 0x23, 0xf0, 0xd1, 0x98, 0xbd, 0x3b, 0xd9, 0xb2, + 0x9c, 0x38, 0xa5, 0x0f, 0xbc, 0x69, 0x7f, 0xbb, 0x77, 0xbb, 0xb7, 0xb7, 0xfb, 0xdb, 0x13, 0x14, + 0xc3, 0x61, 0xd7, 0x1a, 0x86, 0x5c, 0xf0, 0xea, 0x8d, 0x3e, 0xe7, 0x7d, 0x9f, 0x3d, 0x94, 0xd2, + 0xab, 0x51, 0xef, 0x21, 0x1b, 0x0c, 0xc5, 0x58, 0x2b, 0x6f, 0xa7, 0x95, 0xc2, 0x1b, 0xb0, 0x48, + 0x38, 0x83, 0xa1, 0x32, 0xa0, 0xbf, 0x1b, 0x50, 0xfe, 0x96, 0x85, 0x91, 0xc7, 0x03, 0x9b, 0x0d, + 0xfd, 0x31, 0x31, 0x21, 0xaf, 0x65, 0xd3, 0xa8, 0x19, 0xf5, 0xa2, 0x1d, 0x8b, 0x64, 0x0b, 0x96, + 0xbf, 0x1a, 0x79, 0xbe, 0x6b, 0x66, 0x25, 0xae, 0x04, 0x72, 0x13, 0x8a, 0x07, 0x3c, 0x5e, 0x91, + 0x93, 0x9a, 0x29, 0x40, 0xd6, 0x20, 0x7b, 0xdc, 0x31, 0x97, 0x24, 0x9c, 0x3d, 0xee, 0x10, 0x02, + 0x4b, 0x8d, 0xb0, 0x7b, 0x66, 0x2e, 0x4b, 0x44, 0x7e, 0x93, 0x5b, 0x00, 0x07, 0xfc, 0xc8, 0x39, + 0x3f, 0x09, 0x79, 0x37, 0x32, 0x57, 0x6a, 0x46, 0x7d, 0xd9, 0x4e, 0x20, 0xb4, 0x0e, 0xe5, 0x23, + 0x47, 0x74, 0xcf, 0x6c, 0xf6, 0xd3, 0x88, 0x45, 0x02, 0x23, 0x3c, 0x71, 0x84, 0x60, 0xe1, 0x24, + 0x42, 0x2d, 0xd2, 0x5f, 0x0b, 0xb0, 0x72, 0xe4, 0x85, 0x21, 0x0f, 0xd1, 0xf1, 0x61, 0x4b, 0xea, + 0x97, 0xed, 0xec, 0x61, 0x0b, 0x1d, 0xbf, 0x70, 0x06, 0x4c, 0xc7, 0x2e, 0xbf, 0x71, 0xa3, 0xe7, + 0x42, 0x0c, 0x4f, 0xed, 0xb6, 0x0e, 0x3c, 0x16, 0x49, 0x15, 0x0a, 0x76, 0x34, 0x0e, 0xba, 0xa8, + 0x52, 0xc1, 0x4f, 0x64, 0xb2, 0x0d, 0x2b, 0xfb, 0x6a, 0x91, 0x3a, 0x84, 0x96, 0x48, 0x0d, 0x4a, + 0x9d, 0x21, 0x0f, 0x22, 0x1e, 0x4a, 0x47, 0x2b, 0x52, 0x99, 0x84, 0xf0, 0xa0, 0x5a, 0xc4, 0xd5, + 0x79, 0x69, 0x90, 0x40, 0xc8, 0x3d, 0x58, 0xd3, 0x52, 0x9b, 0xf7, 0x39, 0xda, 0x14, 0xa4, 0x4d, + 0x0a, 0xc5, 0x94, 0x37, 0xdc, 0x81, 0x17, 0x48, 0x3f, 0x45, 0x95, 0xf2, 0x09, 0x80, 0x5e, 0xa4, + 0xb0, 0x37, 0x70, 0x3c, 0xdf, 0x04, 0xe5, 0x65, 0x8a, 0xa0, 0xbe, 0x39, 0x8a, 0x04, 0x1f, 0xb4, + 0x1c, 0xe1, 0x98, 0x25, 0xa5, 0x9f, 0x22, 0xe4, 0x2e, 0xac, 0x36, 0x79, 0x20, 0xbc, 0x80, 0x05, + 0xe2, 0x38, 0xf0, 0xc7, 0x66, 0xb9, 0x66, 0xd4, 0x0b, 0xf6, 0x2c, 0x88, 0xa7, 0x6d, 0xf2, 0x51, + 0x20, 0xc2, 0xb1, 0xb4, 0x59, 0x95, 0x36, 0x49, 0x08, 0xf3, 0xd4, 0xe8, 0x48, 0xe5, 0x9a, 0x54, + 0x6a, 0x09, 0xcb, 0xa8, 0xd3, 0xe5, 0x21, 0x33, 0xd7, 0xe5, 0xe5, 0x28, 0x01, 0x33, 0xde, 0x76, + 0x84, 0x27, 0x46, 0x2e, 0x33, 0x2b, 0x35, 0xa3, 0x9e, 0xb5, 0x27, 0x32, 0x9e, 0xb7, 0xcd, 0x83, + 0xbe, 0x52, 0x6e, 0x48, 0xe5, 0x14, 0x98, 0x89, 0xb7, 0xc9, 0x5d, 0x66, 0x12, 0x79, 0xa4, 0x59, + 0x90, 0x50, 0x28, 0xeb, 0xe0, 0x50, 0x8c, 0xcc, 0x4d, 0x69, 0x34, 0x83, 0x91, 0x1d, 0xd8, 0xda, + 0x3b, 0xef, 0xfa, 0x23, 0x97, 0xb9, 0x33, 0xb6, 0x5b, 0xd2, 0xf6, 0x42, 0x1d, 0x9e, 0xa6, 0x11, + 0x05, 0xa3, 0x81, 0x79, 0xad, 0x66, 0xd4, 0x57, 0x6d, 0x25, 0x60, 0x65, 0x35, 0xf9, 0x60, 0xc0, + 0x02, 0x61, 0x6e, 0xab, 0xca, 0xd2, 0x22, 0x6a, 0xf6, 0x02, 0xe7, 0x95, 0xcf, 0x5c, 0xf3, 0x7f, + 0x32, 0x2d, 0xb1, 0x88, 0x15, 0x7b, 0x3a, 0x34, 0x4d, 0x09, 0x66, 0x4f, 0x87, 0x78, 0x2e, 0xed, + 0xd1, 0x66, 0x4e, 0xc4, 0x03, 0xf3, 0xba, 0x3a, 0xd7, 0x0c, 0x48, 0x9e, 0x02, 0x74, 0x84, 0x23, + 0x58, 0xc7, 0x0b, 0xba, 0xcc, 0xac, 0xd6, 0x8c, 0x7a, 0x69, 0xa7, 0x6a, 0xa9, 0xae, 0xb7, 0xe2, + 0xae, 0xb7, 0x5e, 0xc6, 0x5d, 0x6f, 0x27, 0xac, 0xb1, 0xde, 0x1a, 0xbe, 0xcf, 0xdf, 0xd8, 0xcc, + 0xf5, 0x42, 0xd6, 0x15, 0x91, 0x79, 0x43, 0x5e, 0x49, 0x0a, 0x25, 0x9f, 0xe0, 0xdd, 0x44, 0xa2, + 0x33, 0x0e, 0xba, 0xe6, 0xcd, 0x2b, 0x3d, 0x4c, 0x6c, 0xc9, 0xd7, 0x40, 0xe4, 0xf7, 0xa8, 0xdb, + 0x65, 0x51, 0xd4, 0x1b, 0xf9, 0x72, 0x87, 0xff, 0x5f, 0xb9, 0xc3, 0x05, 0xab, 0xc8, 0x33, 0x28, + 0x21, 0x7a, 0xc4, 0x5d, 0xb4, 0x33, 0x6f, 0x5d, 0xb9, 0x49, 0xd2, 0x9c, 0x3e, 0x86, 0x75, 0xc5, + 0x0b, 0x6d, 0x2f, 0x12, 0x8a, 0xe7, 0xee, 0x40, 0x5e, 0x41, 0x91, 0x69, 0xd4, 0x72, 0xf5, 0xd2, + 0x4e, 0xde, 0x52, 0xb2, 0x1d, 0xe3, 0xd4, 0x82, 0x82, 0xfa, 0x3c, 0x6c, 0xbd, 0x0d, 0x9f, 0xd0, + 0x47, 0x00, 0x9a, 0xa8, 0xd0, 0xc1, 0x7b, 0x69, 0x07, 0x45, 0x2b, 0xde, 0x6d, 0xea, 0xe2, 0x4b, + 0xd8, 0x6c, 0x9e, 0x39, 0x41, 0x9f, 0xe1, 0xb5, 0x8c, 0xa2, 0x98, 0xe2, 0xd2, 0xde, 0x12, 0x55, + 0x93, 0x9d, 0xa9, 0x1a, 0x7a, 0x27, 0x3e, 0xd9, 0x61, 0xeb, 0x92, 0xc5, 0xf4, 0x2f, 0x03, 0xd6, + 0x1a, 0xae, 0xab, 0x4f, 0x27, 0x63, 0x4b, 0x76, 0x9b, 0xb1, 0xa8, 0xdb, 0xb2, 0xe9, 0x6e, 0x93, + 0x95, 0x2d, 0xeb, 0x3f, 0xe6, 0x4c, 0x2d, 0xe2, 0xba, 0x49, 0xcb, 0x69, 0xd2, 0x9c, 0x02, 0xa4, + 0x02, 0xb9, 0x46, 0xe7, 0x85, 0xa6, 0x4c, 0xfc, 0xc4, 0x18, 0xbe, 0x73, 0xc2, 0xc0, 0x0b, 0xfa, + 0x48, 0xfa, 0x39, 0xe4, 0xd8, 0x58, 0xa6, 0xf7, 0x61, 0xe3, 0x74, 0xe8, 0x3a, 0x82, 0x25, 0x83, + 0x26, 0xb0, 0xd4, 0xf2, 0x7a, 0x3d, 0x4d, 0xfa, 0xf2, 0x9b, 0xf6, 0x61, 0xeb, 0x80, 0xf1, 0x79, + 0xdb, 0xdb, 0xf1, 0x20, 0x90, 0xd6, 0x89, 0xcb, 0x8d, 0xe7, 0x43, 0xbc, 0x59, 0x76, 0xba, 0xd9, + 0x4c, 0x44, 0xb9, 0x54, 0x44, 0x3b, 0x60, 0xda, 0xac, 0x17, 0xb2, 0x08, 0x6f, 0x97, 0x47, 0x9e, + 0xe0, 0xe1, 0x38, 0x4e, 0xf8, 0x36, 0xac, 0xd8, 0xec, 0xcc, 0x89, 0xce, 0xa4, 0xb3, 0x82, 0xad, + 0x25, 0xfa, 0x87, 0x01, 0x1b, 0x9d, 0xae, 0x13, 0xc4, 0x81, 0x5d, 0x7c, 0xb7, 0xc8, 0xd7, 0x23, + 0xc1, 0xd5, 0x85, 0xea, 0xeb, 0x4d, 0x20, 0xe4, 0x09, 0x14, 0x4e, 0xb0, 0xbc, 0xbb, 0xdc, 0x97, + 0x29, 0x5f, 0xdb, 0xb9, 0x6e, 0xcd, 0xed, 0x6a, 0x1d, 0x31, 0x71, 0xc6, 0x5d, 0x7b, 0x62, 0x4a, + 0xdf, 0x87, 0x15, 0x85, 0x91, 0x3c, 0xe4, 0x1a, 0xed, 0x76, 0x25, 0x83, 0x1f, 0xfb, 0x2f, 0x4f, + 0x2a, 0x06, 0x29, 0xc2, 0xb2, 0xdd, 0xf9, 0xfe, 0x45, 0xb3, 0x92, 0xa5, 0x7f, 0x1a, 0xb0, 0x9e, + 0xdc, 0x4d, 0x3f, 0x01, 0xe2, 0x6a, 0x33, 0x66, 0x39, 0x8a, 0x42, 0x79, 0xdf, 0xf3, 0x59, 0x74, + 0x18, 0xb8, 0xec, 0x5c, 0x17, 0x63, 0xce, 0x9e, 0xc1, 0xd0, 0xe6, 0x9b, 0x80, 0xbf, 0x09, 0x62, + 0x9b, 0x9c, 0xb2, 0x49, 0x62, 0xe8, 0xc1, 0x66, 0x03, 0xfe, 0x9a, 0xb9, 0xb2, 0x52, 0x72, 0x76, + 0x2c, 0x62, 0x36, 0x5e, 0xfe, 0x70, 0xdc, 0xeb, 0x45, 0x4c, 0x1c, 0x45, 0xb2, 0x5c, 0x72, 0x76, + 0x02, 0xa1, 0xbf, 0x19, 0x50, 0xc1, 0x5e, 0x89, 0xd0, 0xe7, 0x95, 0x2f, 0x02, 0xb2, 0x0b, 0xc5, + 0x16, 0xf2, 0x9d, 0x70, 0x42, 0x21, 0xa3, 0x5d, 0x4c, 0x1a, 0x53, 0x63, 0xf2, 0x18, 0xf2, 0x28, + 0xec, 0x05, 0xea, 0x04, 0x8b, 0xd7, 0xc5, 0xa6, 0xf4, 0x67, 0x58, 0x4b, 0x44, 0x87, 0xc9, 0xfc, + 0x08, 0x96, 0x7b, 0x98, 0x1e, 0x4d, 0x02, 0x55, 0x6b, 0x56, 0x6f, 0xc9, 0xdc, 0xed, 0x61, 0x07, + 0xd9, 0xca, 0xb0, 0xba, 0x0b, 0x30, 0x05, 0xb1, 0x71, 0x7e, 0x64, 0x63, 0x7d, 0x2e, 0xfc, 0xc4, + 0x91, 0xf3, 0xda, 0xf1, 0x47, 0x4c, 0x67, 0x5f, 0x09, 0x4f, 0xb3, 0xbb, 0x06, 0xfd, 0xc5, 0x00, + 0x22, 0xb7, 0x5f, 0x5c, 0x71, 0xff, 0x75, 0x52, 0x98, 0xbe, 0xb2, 0x7f, 0xd5, 0xa0, 0xf8, 0x04, + 0x53, 0xf1, 0x47, 0xfa, 0xa0, 0x13, 0x59, 0xbe, 0x44, 0xc7, 0x82, 0x45, 0xba, 0xb6, 0x94, 0x40, + 0xf7, 0x91, 0x0b, 0x84, 0xe6, 0x79, 0xde, 0x8f, 0x16, 0x34, 0xdc, 0x91, 0x73, 0x6e, 0xb3, 0x68, + 0xe4, 0xeb, 0xbd, 0x97, 0xed, 0x04, 0x42, 0xeb, 0x40, 0x52, 0xfb, 0x68, 0xf6, 0xf1, 0xbd, 0x80, + 0xc9, 0x6b, 0x2c, 0xda, 0xf2, 0x9b, 0xde, 0x95, 0x3d, 0x16, 0x7a, 0x5d, 0x8c, 0x16, 0xef, 0x2c, + 0xc0, 0x91, 0xa0, 0xae, 0x6a, 0x22, 0xd3, 0x07, 0x50, 0x52, 0x56, 0x11, 0x4e, 0x9f, 0x94, 0x69, + 0x2e, 0x69, 0xba, 0xf3, 0x77, 0x01, 0x72, 0xcd, 0xf6, 0x21, 0x79, 0x02, 0x70, 0xc0, 0x44, 0xfc, + 0x88, 0xde, 0x9e, 0x4b, 0xf2, 0x1e, 0x3e, 0xf1, 0xab, 0xab, 0x56, 0xf2, 0xe5, 0x4e, 0x33, 0xe4, + 0x33, 0xc8, 0x9f, 0x0e, 0xfb, 0xa1, 0xe3, 0xb2, 0x4b, 0xd7, 0x5c, 0x82, 0xd3, 0x0c, 0x79, 0x8a, + 0x2c, 0xe6, 0x73, 0xc7, 0x7d, 0x87, 0xb5, 0x5f, 0x40, 0x39, 0x39, 0xc6, 0xc8, 0x96, 0x75, 0xc1, + 0x54, 0x5b, 0xb0, 0x7e, 0x07, 0x96, 0x64, 0x6e, 0x2e, 0xf3, 0x5c, 0xb1, 0x52, 0xe3, 0x9b, 0x66, + 0xc8, 0x03, 0x00, 0x3d, 0xf9, 0x82, 0x1e, 0x27, 0x15, 0x2b, 0x35, 0x06, 0xab, 0x71, 0x45, 0xd1, + 0x0c, 0xb9, 0x8f, 0x0f, 0x66, 0x3d, 0x00, 0x49, 0x8c, 0x57, 0xd7, 0xad, 0xd9, 0xa9, 0x48, 0x33, + 0xe4, 0x43, 0x28, 0x27, 0x67, 0xc9, 0xd4, 0x96, 0x58, 0x73, 0x33, 0x46, 0xa6, 0xac, 0xac, 0x78, + 0x4b, 0x9b, 0xcf, 0x07, 0x71, 0xf9, 0x91, 0x9f, 0xc1, 0x7a, 0x6a, 0x72, 0x5d, 0xb0, 0xfc, 0x9a, + 0x75, 0xd1, 0x74, 0xa3, 0x19, 0xf2, 0x1c, 0x36, 0xe6, 0xc6, 0x11, 0xb9, 0x6e, 0x5d, 0x36, 0xa2, + 0x16, 0xc4, 0xf1, 0x18, 0x60, 0xca, 0xff, 0x84, 0xcc, 0x8f, 0x96, 0x6a, 0xc5, 0x4a, 0x0d, 0x08, + 0x9a, 0x21, 0x8f, 0xa0, 0x38, 0xe1, 0x31, 0xb2, 0x61, 0xa5, 0x19, 0xb9, 0xba, 0x9e, 0xa2, 0x39, + 0x9a, 0x21, 0x9f, 0x42, 0x29, 0xc1, 0x02, 0x64, 0xd3, 0x9a, 0x67, 0xaa, 0xea, 0x86, 0x95, 0x26, + 0x0a, 0x9a, 0x21, 0xbb, 0xb0, 0x74, 0xe2, 0x05, 0xfd, 0x77, 0x28, 0xcb, 0xcf, 0x61, 0x75, 0xa6, + 0x93, 0x09, 0xe6, 0x73, 0x9e, 0x21, 0xaa, 0x9b, 0xd6, 0x7c, 0xc3, 0xd3, 0x0c, 0xb1, 0x54, 0xd9, + 0xa8, 0x0e, 0xcf, 0x5b, 0xea, 0x63, 0x81, 0x3b, 0x0b, 0x8a, 0x2d, 0xe6, 0xbf, 0xbd, 0xfd, 0x3d, + 0x28, 0x61, 0x41, 0x6b, 0x72, 0x98, 0xae, 0x28, 0x5b, 0x09, 0xbe, 0xa0, 0x19, 0xf2, 0x01, 0x94, + 0xe4, 0xbb, 0x52, 0x67, 0x6e, 0xd5, 0x4a, 0xfe, 0x0e, 0x57, 0x4b, 0xd6, 0xf4, 0xd1, 0x49, 0x33, + 0xaf, 0x56, 0xa4, 0x9b, 0x8f, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0x6c, 0x40, 0x6d, 0x65, 0x22, + 0x10, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -1434,6 +1518,9 @@ type CLIClient interface { StatsMirror(ctx context.Context, in *StatsMirrorRequest, opts ...grpc.CallOption) (*StatsMirrorReply, error) Ping(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) GetMirrorLogs(ctx context.Context, in *GetMirrorLogsRequest, opts ...grpc.CallOption) (*GetMirrorLogsReply, error) + AddMetric(ctx context.Context, in *Metric, opts ...grpc.CallOption) (*empty.Empty, error) + DelMetric(ctx context.Context, in *Metric, opts ...grpc.CallOption) (*empty.Empty, error) + ListMetrics(ctx context.Context, in *Metric, opts ...grpc.CallOption) (*MetricsList, error) // Tools MatchMirror(ctx context.Context, in *MatchRequest, opts ...grpc.CallOption) (*MatchReply, error) } @@ -1590,6 +1677,33 @@ func (c *cLIClient) GetMirrorLogs(ctx context.Context, in *GetMirrorLogsRequest, return out, nil } +func (c *cLIClient) AddMetric(ctx context.Context, in *Metric, opts ...grpc.CallOption) (*empty.Empty, error) { + out := new(empty.Empty) + err := c.cc.Invoke(ctx, "/CLI/AddMetric", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cLIClient) DelMetric(ctx context.Context, in *Metric, opts ...grpc.CallOption) (*empty.Empty, error) { + out := new(empty.Empty) + err := c.cc.Invoke(ctx, "/CLI/DelMetric", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cLIClient) ListMetrics(ctx context.Context, in *Metric, opts ...grpc.CallOption) (*MetricsList, error) { + out := new(MetricsList) + err := c.cc.Invoke(ctx, "/CLI/ListMetrics", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *cLIClient) MatchMirror(ctx context.Context, in *MatchRequest, opts ...grpc.CallOption) (*MatchReply, error) { out := new(MatchReply) err := c.cc.Invoke(ctx, "/CLI/MatchMirror", in, out, opts...) @@ -1617,6 +1731,9 @@ type CLIServer interface { StatsMirror(context.Context, *StatsMirrorRequest) (*StatsMirrorReply, error) Ping(context.Context, *empty.Empty) (*empty.Empty, error) GetMirrorLogs(context.Context, *GetMirrorLogsRequest) (*GetMirrorLogsReply, error) + AddMetric(context.Context, *Metric) (*empty.Empty, error) + DelMetric(context.Context, *Metric) (*empty.Empty, error) + ListMetrics(context.Context, *Metric) (*MetricsList, error) // Tools MatchMirror(context.Context, *MatchRequest) (*MatchReply, error) } @@ -1673,6 +1790,15 @@ func (*UnimplementedCLIServer) Ping(ctx context.Context, req *empty.Empty) (*emp func (*UnimplementedCLIServer) GetMirrorLogs(ctx context.Context, req *GetMirrorLogsRequest) (*GetMirrorLogsReply, error) { return nil, status.Errorf(codes.Unimplemented, "method GetMirrorLogs not implemented") } +func (*UnimplementedCLIServer) AddMetric(ctx context.Context, req *Metric) (*empty.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method AddMetric not implemented") +} +func (*UnimplementedCLIServer) DelMetric(ctx context.Context, req *Metric) (*empty.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method DelMetric not implemented") +} +func (*UnimplementedCLIServer) ListMetrics(ctx context.Context, req *Metric) (*MetricsList, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListMetrics not implemented") +} func (*UnimplementedCLIServer) MatchMirror(ctx context.Context, req *MatchRequest) (*MatchReply, error) { return nil, status.Errorf(codes.Unimplemented, "method MatchMirror not implemented") } @@ -1969,6 +2095,60 @@ func _CLI_GetMirrorLogs_Handler(srv interface{}, ctx context.Context, dec func(i return interceptor(ctx, in, info, handler) } +func _CLI_AddMetric_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Metric) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CLIServer).AddMetric(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/CLI/AddMetric", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CLIServer).AddMetric(ctx, req.(*Metric)) + } + return interceptor(ctx, in, info, handler) +} + +func _CLI_DelMetric_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Metric) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CLIServer).DelMetric(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/CLI/DelMetric", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CLIServer).DelMetric(ctx, req.(*Metric)) + } + return interceptor(ctx, in, info, handler) +} + +func _CLI_ListMetrics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Metric) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CLIServer).ListMetrics(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/CLI/ListMetrics", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CLIServer).ListMetrics(ctx, req.(*Metric)) + } + return interceptor(ctx, in, info, handler) +} + func _CLI_MatchMirror_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(MatchRequest) if err := dec(in); err != nil { @@ -2055,6 +2235,18 @@ var _CLI_serviceDesc = grpc.ServiceDesc{ MethodName: "GetMirrorLogs", Handler: _CLI_GetMirrorLogs_Handler, }, + { + MethodName: "AddMetric", + Handler: _CLI_AddMetric_Handler, + }, + { + MethodName: "DelMetric", + Handler: _CLI_DelMetric_Handler, + }, + { + MethodName: "ListMetrics", + Handler: _CLI_ListMetrics_Handler, + }, { MethodName: "MatchMirror", Handler: _CLI_MatchMirror_Handler, diff --git a/rpc/rpc.proto b/rpc/rpc.proto index ca75e753..b6e1ed7e 100644 --- a/rpc/rpc.proto +++ b/rpc/rpc.proto @@ -20,6 +20,9 @@ service CLI { rpc StatsMirror (StatsMirrorRequest) returns (StatsMirrorReply) {} rpc Ping (google.protobuf.Empty) returns (google.protobuf.Empty) {} rpc GetMirrorLogs (GetMirrorLogsRequest) returns (GetMirrorLogsReply) {} + rpc AddMetric (Metric) returns (google.protobuf.Empty) {} + rpc DelMetric (Metric) returns (google.protobuf.Empty) {} + rpc ListMetrics (Metric) returns (MetricsList) {} // Tools rpc MatchMirror (MatchRequest) returns (MatchReply) {} @@ -165,3 +168,11 @@ message GetMirrorLogsRequest { message GetMirrorLogsReply { repeated string line = 1; } + +message Metric { + string Filename = 1; +} + +message MetricsList { + repeated string Filename = 1; +} From 44053a4bd36e2ccd7652f54d9115927776e013d5 Mon Sep 17 00:00:00 2001 From: Duncan McNamara Date: Mon, 9 Mar 2020 15:40:55 +0100 Subject: [PATCH 4/8] Metrics: add daily top 10 Add export of current daily download stats for all files that have been downloaded in the current day. With this is also added a retention period for these stats as they can be heavy for the database and should be removed after use. --- config/config.go | 19 ++++++++++++------- http/metrics.go | 38 ++++++++++++++++++++++++++++++++++++++ http/stats.go | 15 ++++++++++++++- mirrorbits.conf | 7 ++++++- 4 files changed, 70 insertions(+), 9 deletions(-) diff --git a/config/config.go b/config/config.go index 42550967..501c21b6 100644 --- a/config/config.go +++ b/config/config.go @@ -55,12 +55,13 @@ func defaultConfig() Configuration { SHA256: true, MD5: false, }, - DisallowRedirects: false, - WeightDistributionRange: 1.5, - DisableOnMissingFile: false, - RPCListenAddress: "localhost:3390", - RPCPassword: "", - MetricsEnabled: false, + DisallowRedirects: false, + WeightDistributionRange: 1.5, + DisableOnMissingFile: false, + RPCListenAddress: "localhost:3390", + RPCPassword: "", + MetricsEnabled: false, + MetricsTopFilesRetention: 0, } } @@ -97,7 +98,8 @@ type Configuration struct { RPCListenAddress string `yaml:"RPCListenAddress"` RPCPassword string `yaml:"RPCPassword"` - MetricsEnabled bool `yaml:"MetricsEnabled"` + MetricsEnabled bool `yaml:"MetricsEnabled"` + MetricsTopFilesRetention int `yaml:"MetricsTopFilesRetention"` } type fallback struct { @@ -170,6 +172,9 @@ func ReloadConfig() error { if c.RepositoryScanInterval < 0 { c.RepositoryScanInterval = 0 } + if c.MetricsTopFilesRetention < 0 { + c.MetricsTopFilesRetention = 0 + } if config != nil && (c.RedisAddress != config.RedisAddress || diff --git a/http/metrics.go b/http/metrics.go index 7e66de59..28655566 100644 --- a/http/metrics.go +++ b/http/metrics.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/etix/mirrorbits/config" "github.com/etix/mirrorbits/database" "github.com/gomodule/redigo/redis" ) @@ -330,6 +331,43 @@ func (m *Metrics) getMetrics(httpRedis *database.Redis) { index++ output += "\n" } + + // Get daily stats for top 10 + retention := config.GetConfig().MetricsTopFilesRetention + for nbDays := 0; nbDays < retention; nbDays++ { + day := today.AddDate(0, 0, nbDays*-1) + rconn.Send("MULTI") + for _, file := range fileList { + mkey := fmt.Sprintf("STATS_TOP_%s_%s", file, + day.Format("2006_01_02")) + rconn.Send("HGETALL", mkey) + } + stats, err = redis.Values(rconn.Do("EXEC")) + if err != nil { + log.Error("Cannot fetch file per country stats: " + err.Error()) + return + } + for index, stat := range stats { + hashSlice, _ := redis.ByteSlices(stat, err) + for i := 0; i < len(hashSlice); i += 2 { + field, _ := redis.String(hashSlice[i], err) + value, _ := redis.Int(hashSlice[i+1], err) + sep := strings.Index(field, "_") + country := field[:sep] + mirrorID, err := strconv.Atoi(field[sep+1:]) + if err != nil { + log.Error("Failed to convert mirror ID: ", err) + return + } + mirror := mirrorsMap[mirrorID] + output += fmt.Sprintf("stats_top"+ + "{file=\"%s\",country=\"%s\",mirror=\"%s\"} %d\n", + fileList[index], country, mirror, value, + ) + } + } + } + m.lock.Lock() m.metricsResponse = output m.lock.Unlock() diff --git a/http/stats.go b/http/stats.go index b6a913e2..cbfcae84 100644 --- a/http/stats.go +++ b/http/stats.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/etix/mirrorbits/config" "github.com/etix/mirrorbits/database" "github.com/etix/mirrorbits/filesystem" "github.com/etix/mirrorbits/mirrors" @@ -139,8 +140,8 @@ func (s *Stats) pushStats() { return } + topFileRetention := config.GetConfig().MetricsTopFilesRetention rconn.Send("MULTI") - for k, v := range s.mapStats { if v == 0 { continue @@ -215,6 +216,18 @@ func (s *Stats) pushStats() { rconn.Send("HINCRBY", mkey, key, v) mkey = mkey[:strings.LastIndex(mkey, "_")] } + if topFileRetention != 0 { + mkey = fmt.Sprintf("STATS_TOP_%s_%s", file, date) + rconn.Send("HINCRBY", mkey, key, v) + t, err := time.Parse("2006_01_02", date) + if err != nil { + log.Error("Failed to parse date: ", err.Error()) + return + } + t = t.AddDate(0, 0, topFileRetention) + topExpireDate := t.Format("2006_01_02") + rconn.Send("EXPIREAT", mkey, topExpireDate) + } } else { log.Warning("Stats: unknown type", typ) } diff --git a/mirrorbits.conf b/mirrorbits.conf index 6f9b890f..cb8cfdbf 100644 --- a/mirrorbits.conf +++ b/mirrorbits.conf @@ -140,4 +140,9 @@ ################### ## Enable the metrics route, default to false -## MetricsEnable: false \ No newline at end of file +## MetricsEnable: false + +## Number of days to keep the daily top 10 metrics in database +## Keep in mind that this will have an impact on the database's RAM usage +## It is not recommended to keep more than a few days of retention. +## MetricsTopFilesRetention: 0 From 09cbf02a38c04b47dea25d94d97b95055197d484 Mon Sep 17 00:00:00 2001 From: Duncan McNamara Date: Thu, 12 Mar 2020 16:02:26 +0100 Subject: [PATCH 5/8] Metrics: Add auto tracked files mode When enabled, files will be automatically added to tracked files. --- cli/commands.go | 40 ++++++ config/config.go | 2 + database/utils.go | 50 ++++++- mirrorbits.conf | 3 + rpc/rpc.go | 59 +++----- rpc/rpc.pb.go | 342 +++++++++++++++++++++++++++++++++------------- rpc/rpc.proto | 7 + scan/scan.go | 5 + 8 files changed, 368 insertions(+), 140 deletions(-) diff --git a/cli/commands.go b/cli/commands.go index 3c81ca74..91ddba2c 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -356,6 +356,9 @@ func (c *cli) CmdMetrics(args ...string) error { _ = cmd.String("add", "", "Add a file to the metrics route") _ = cmd.String("list", "*", "List files in metrics + Optionnal pattern to filter results") _ = cmd.String("delete", "", "Delete a file from the metrics route") + _ = cmd.Bool("auto-enable", true, "Enable automatic addition of new files to tracked files") + _ = cmd.Bool("auto-disable", false, "Disable automatic addition of new files to tracked files") + _ = cmd.Bool("auto-status", true, "Print boolean of automatic addition of new files to tracked files") // Can't use cmd.Parse(args) because it doesn't handle -command without // an argument following which is needed for list @@ -373,6 +376,12 @@ func (c *cli) CmdMetrics(args ...string) error { c.CmdAddmetric(args[1]) } else if args[0] == "-delete" && len(args) == 2 { c.CmdDelmetric(args[1]) + } else if args[0] == "-auto-enable" { + c.CmdEnableauto() + } else if args[0] == "-auto-disable" { + c.CmdDisableauto() + } else if args[0] == "-auto-status" { + c.CmdStatusauto() } else { cmd.Usage() return nil @@ -443,6 +452,37 @@ func (c *cli) CmdListmetrics(pattern string) error { return nil } +func (c *cli) CmdEnableauto() error { + client := c.GetRPC() + ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) + defer cancel() + client.EnableAuto(ctx, &empty.Empty{}) + return nil +} + +func (c *cli) CmdDisableauto() error { + client := c.GetRPC() + ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) + defer cancel() + client.DisableAuto(ctx, &empty.Empty{}) + return nil +} + +func (c *cli) CmdStatusauto() error { + client := c.GetRPC() + ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) + defer cancel() + status, _ := client.GetStatusAuto(ctx, &empty.Empty{}) + + if status.Status { + log.Info("Auto tracked files is enabled") + } else { + log.Info("Auto tracked files is disabled") + } + + return nil +} + func (c *cli) CmdRemove(args ...string) error { cmd := SubCmd("remove", "IDENTIFIER", "Remove an existing mirror") force := cmd.Bool("f", false, "Never prompt for confirmation") diff --git a/config/config.go b/config/config.go index 501c21b6..4abda51a 100644 --- a/config/config.go +++ b/config/config.go @@ -62,6 +62,7 @@ func defaultConfig() Configuration { RPCPassword: "", MetricsEnabled: false, MetricsTopFilesRetention: 0, + MetricsAutoTrackedFiles: false, } } @@ -100,6 +101,7 @@ type Configuration struct { MetricsEnabled bool `yaml:"MetricsEnabled"` MetricsTopFilesRetention int `yaml:"MetricsTopFilesRetention"` + MetricsAutoTrackedFiles bool `yaml:"MetricsAutoTrackedFiles"` } type fallback struct { diff --git a/database/utils.go b/database/utils.go index f790ae86..e0d6ff48 100644 --- a/database/utils.go +++ b/database/utils.go @@ -4,10 +4,12 @@ package database import ( - "errors" "strconv" "github.com/gomodule/redigo/redis" + "github.com/pkg/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func (r *Redis) GetListOfMirrors() (map[int]string, error) { @@ -147,6 +149,52 @@ func (r *Redis) AddCountry(country string) error { return err } +func (r *Redis) AddTrackedFile(file string) error { + conn, err := r.Connect() + if err != nil { + return err + } + defer conn.Close() + + exists, err := redis.Int(conn.Do("SISMEMBER", "FILES", file)) + if err != nil { + return errors.Wrap(err, "failed to check for file presence") + } else if exists == 0 { + return status.Error(codes.FailedPrecondition, + "file does not exist") + } + + exists, err = redis.Int(conn.Do("SADD", "TRACKED_FILES", file)) + if err != nil { + return errors.Wrap(err, "failed to add file to metrics") + } else if exists == 0 { + return status.Error(codes.AlreadyExists, "file already is in metrics") + } + return nil +} + +func (r *Redis) DeleteTrackedFile(file string) error { + conn, err := r.Connect() + if err != nil { + return err + } + defer conn.Close() + + exists, err := redis.Int(conn.Do("SISMEMBER", "TRACKED_FILES", file)) + if err != nil { + return errors.Wrap(err, "failed to check for file presence") + } else if exists == 0 { + return status.Error(codes.FailedPrecondition, + "file is not part of tracked files") + } + + _, err = conn.Do("SREM", "TRACKED_FILES", file) + if err != nil { + return errors.Wrap(err, "failed to remove file from tracked files") + } + return nil +} + type NetReadyError struct { error } diff --git a/mirrorbits.conf b/mirrorbits.conf index cb8cfdbf..8c96dc9c 100644 --- a/mirrorbits.conf +++ b/mirrorbits.conf @@ -146,3 +146,6 @@ ## Keep in mind that this will have an impact on the database's RAM usage ## It is not recommended to keep more than a few days of retention. ## MetricsTopFilesRetention: 0 + +## Enable / Disable automatically adding a new file to tracked files +## MetricsAutoTrackedFiles: false diff --git a/rpc/rpc.go b/rpc/rpc.go index 5c19d96a..4be9ed1c 100644 --- a/rpc/rpc.go +++ b/rpc/rpc.go @@ -805,52 +805,11 @@ func (c *CLI) GetMirrorLogs(ctx context.Context, in *GetMirrorLogsRequest) (*Get } func (c *CLI) AddMetric(ctx context.Context, in *Metric) (*empty.Empty, error) { - conn, err := c.redis.Connect() - if err != nil { - return nil, err - } - defer conn.Close() - - exists, err := redis.Int(conn.Do("SISMEMBER", "FILES", in.Filename)) - if err != nil { - return nil, errors.Wrap(err, "failed to check for file presence") - } else if exists == 0 { - return nil, status.Error(codes.FailedPrecondition, - "file does not exist") - } - - exists, err = redis.Int(conn.Do("SADD", "TRACKED_FILES", in.Filename)) - if err != nil { - return nil, errors.Wrap(err, "failed to add file to metrics") - } else if exists == 0 { - return nil, status.Error(codes.AlreadyExists, - "file already is in metrics") - } - - return &empty.Empty{}, nil + return &empty.Empty{}, c.redis.AddTrackedFile(in.Filename) } func (c *CLI) DelMetric(ctx context.Context, in *Metric) (*empty.Empty, error) { - conn, err := c.redis.Connect() - if err != nil { - return nil, err - } - defer conn.Close() - - exists, err := redis.Int(conn.Do("SISMEMBER", "TRACKED_FILES", in.Filename)) - if err != nil { - return nil, errors.Wrap(err, "failed to check for file presence") - } else if exists == 0 { - return nil, status.Error(codes.FailedPrecondition, - "file is not part of tracked files") - } - - _, err = conn.Do("SREM", "TRACKED_FILES", in.Filename) - if err != nil { - return nil, errors.Wrap(err, "failed to remove file from tracked files") - } - - return &empty.Empty{}, nil + return &empty.Empty{}, c.redis.DeleteTrackedFile(in.Filename) } func (c *CLI) ListMetrics(ctx context.Context, in *Metric) (*MetricsList, error) { @@ -874,3 +833,17 @@ func (c *CLI) ListMetrics(ctx context.Context, in *Metric) (*MetricsList, error) return &MetricsList{Filename: files}, nil } + +func (c *CLI) EnableAuto(context.Context, *empty.Empty) (*empty.Empty, error) { + GetConfig().MetricsAutoTrackedFiles = true + return &empty.Empty{}, nil +} + +func (c *CLI) DisableAuto(context.Context, *empty.Empty) (*empty.Empty, error) { + GetConfig().MetricsAutoTrackedFiles = false + return &empty.Empty{}, nil +} + +func (c *CLI) GetStatusAuto(context.Context, *empty.Empty) (*Status, error) { + return &Status{Status: GetConfig().MetricsAutoTrackedFiles}, nil +} diff --git a/rpc/rpc.pb.go b/rpc/rpc.pb.go index 88834eac..3973d244 100644 --- a/rpc/rpc.pb.go +++ b/rpc/rpc.pb.go @@ -1360,6 +1360,45 @@ func (m *MetricsList) GetFilename() []string { return nil } +type Status struct { + Status bool `protobuf:"varint,1,opt,name=Status,proto3" json:"Status,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Status) Reset() { *m = Status{} } +func (m *Status) String() string { return proto.CompactTextString(m) } +func (*Status) ProtoMessage() {} +func (*Status) Descriptor() ([]byte, []int) { + return fileDescriptor_77a6da22d6a3feb1, []int{22} +} + +func (m *Status) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Status.Unmarshal(m, b) +} +func (m *Status) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Status.Marshal(b, m, deterministic) +} +func (m *Status) XXX_Merge(src proto.Message) { + xxx_messageInfo_Status.Merge(m, src) +} +func (m *Status) XXX_Size() int { + return xxx_messageInfo_Status.Size(m) +} +func (m *Status) XXX_DiscardUnknown() { + xxx_messageInfo_Status.DiscardUnknown(m) +} + +var xxx_messageInfo_Status proto.InternalMessageInfo + +func (m *Status) GetStatus() bool { + if m != nil { + return m.Status + } + return false +} + func init() { proto.RegisterEnum("ScanMirrorRequest_Method", ScanMirrorRequest_Method_name, ScanMirrorRequest_Method_value) proto.RegisterType((*VersionReply)(nil), "VersionReply") @@ -1385,6 +1424,7 @@ func init() { proto.RegisterType((*GetMirrorLogsReply)(nil), "GetMirrorLogsReply") proto.RegisterType((*Metric)(nil), "Metric") proto.RegisterType((*MetricsList)(nil), "MetricsList") + proto.RegisterType((*Status)(nil), "Status") } func init() { @@ -1392,102 +1432,104 @@ func init() { } var fileDescriptor_77a6da22d6a3feb1 = []byte{ - // 1507 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0x51, 0x73, 0xdb, 0xc4, - 0x13, 0xb7, 0xec, 0x24, 0xb6, 0xd7, 0x4e, 0xe2, 0x5c, 0xd2, 0xfc, 0x55, 0xb7, 0xff, 0xd6, 0x3d, - 0x4a, 0xeb, 0x0e, 0x83, 0x4a, 0x43, 0x0b, 0x99, 0x52, 0x60, 0x8c, 0x9d, 0xa4, 0x01, 0xa7, 0xc9, - 0xc8, 0x0d, 0x0c, 0xbc, 0xa9, 0xd6, 0xd9, 0xd1, 0x20, 0xeb, 0x8c, 0x74, 0x6e, 0xe3, 0x19, 0x3e, - 0x06, 0xc3, 0x13, 0x0f, 0xf0, 0x01, 0x98, 0xe1, 0x23, 0xf0, 0xd1, 0x98, 0xbd, 0x3b, 0xd9, 0xb2, - 0x9c, 0x38, 0xa5, 0x0f, 0xbc, 0x69, 0x7f, 0xbb, 0x77, 0xbb, 0xb7, 0xb7, 0xfb, 0xdb, 0x13, 0x14, - 0xc3, 0x61, 0xd7, 0x1a, 0x86, 0x5c, 0xf0, 0xea, 0x8d, 0x3e, 0xe7, 0x7d, 0x9f, 0x3d, 0x94, 0xd2, - 0xab, 0x51, 0xef, 0x21, 0x1b, 0x0c, 0xc5, 0x58, 0x2b, 0x6f, 0xa7, 0x95, 0xc2, 0x1b, 0xb0, 0x48, - 0x38, 0x83, 0xa1, 0x32, 0xa0, 0xbf, 0x1b, 0x50, 0xfe, 0x96, 0x85, 0x91, 0xc7, 0x03, 0x9b, 0x0d, - 0xfd, 0x31, 0x31, 0x21, 0xaf, 0x65, 0xd3, 0xa8, 0x19, 0xf5, 0xa2, 0x1d, 0x8b, 0x64, 0x0b, 0x96, - 0xbf, 0x1a, 0x79, 0xbe, 0x6b, 0x66, 0x25, 0xae, 0x04, 0x72, 0x13, 0x8a, 0x07, 0x3c, 0x5e, 0x91, - 0x93, 0x9a, 0x29, 0x40, 0xd6, 0x20, 0x7b, 0xdc, 0x31, 0x97, 0x24, 0x9c, 0x3d, 0xee, 0x10, 0x02, - 0x4b, 0x8d, 0xb0, 0x7b, 0x66, 0x2e, 0x4b, 0x44, 0x7e, 0x93, 0x5b, 0x00, 0x07, 0xfc, 0xc8, 0x39, - 0x3f, 0x09, 0x79, 0x37, 0x32, 0x57, 0x6a, 0x46, 0x7d, 0xd9, 0x4e, 0x20, 0xb4, 0x0e, 0xe5, 0x23, - 0x47, 0x74, 0xcf, 0x6c, 0xf6, 0xd3, 0x88, 0x45, 0x02, 0x23, 0x3c, 0x71, 0x84, 0x60, 0xe1, 0x24, - 0x42, 0x2d, 0xd2, 0x5f, 0x0b, 0xb0, 0x72, 0xe4, 0x85, 0x21, 0x0f, 0xd1, 0xf1, 0x61, 0x4b, 0xea, - 0x97, 0xed, 0xec, 0x61, 0x0b, 0x1d, 0xbf, 0x70, 0x06, 0x4c, 0xc7, 0x2e, 0xbf, 0x71, 0xa3, 0xe7, - 0x42, 0x0c, 0x4f, 0xed, 0xb6, 0x0e, 0x3c, 0x16, 0x49, 0x15, 0x0a, 0x76, 0x34, 0x0e, 0xba, 0xa8, - 0x52, 0xc1, 0x4f, 0x64, 0xb2, 0x0d, 0x2b, 0xfb, 0x6a, 0x91, 0x3a, 0x84, 0x96, 0x48, 0x0d, 0x4a, - 0x9d, 0x21, 0x0f, 0x22, 0x1e, 0x4a, 0x47, 0x2b, 0x52, 0x99, 0x84, 0xf0, 0xa0, 0x5a, 0xc4, 0xd5, - 0x79, 0x69, 0x90, 0x40, 0xc8, 0x3d, 0x58, 0xd3, 0x52, 0x9b, 0xf7, 0x39, 0xda, 0x14, 0xa4, 0x4d, - 0x0a, 0xc5, 0x94, 0x37, 0xdc, 0x81, 0x17, 0x48, 0x3f, 0x45, 0x95, 0xf2, 0x09, 0x80, 0x5e, 0xa4, - 0xb0, 0x37, 0x70, 0x3c, 0xdf, 0x04, 0xe5, 0x65, 0x8a, 0xa0, 0xbe, 0x39, 0x8a, 0x04, 0x1f, 0xb4, - 0x1c, 0xe1, 0x98, 0x25, 0xa5, 0x9f, 0x22, 0xe4, 0x2e, 0xac, 0x36, 0x79, 0x20, 0xbc, 0x80, 0x05, - 0xe2, 0x38, 0xf0, 0xc7, 0x66, 0xb9, 0x66, 0xd4, 0x0b, 0xf6, 0x2c, 0x88, 0xa7, 0x6d, 0xf2, 0x51, - 0x20, 0xc2, 0xb1, 0xb4, 0x59, 0x95, 0x36, 0x49, 0x08, 0xf3, 0xd4, 0xe8, 0x48, 0xe5, 0x9a, 0x54, - 0x6a, 0x09, 0xcb, 0xa8, 0xd3, 0xe5, 0x21, 0x33, 0xd7, 0xe5, 0xe5, 0x28, 0x01, 0x33, 0xde, 0x76, - 0x84, 0x27, 0x46, 0x2e, 0x33, 0x2b, 0x35, 0xa3, 0x9e, 0xb5, 0x27, 0x32, 0x9e, 0xb7, 0xcd, 0x83, - 0xbe, 0x52, 0x6e, 0x48, 0xe5, 0x14, 0x98, 0x89, 0xb7, 0xc9, 0x5d, 0x66, 0x12, 0x79, 0xa4, 0x59, - 0x90, 0x50, 0x28, 0xeb, 0xe0, 0x50, 0x8c, 0xcc, 0x4d, 0x69, 0x34, 0x83, 0x91, 0x1d, 0xd8, 0xda, - 0x3b, 0xef, 0xfa, 0x23, 0x97, 0xb9, 0x33, 0xb6, 0x5b, 0xd2, 0xf6, 0x42, 0x1d, 0x9e, 0xa6, 0x11, - 0x05, 0xa3, 0x81, 0x79, 0xad, 0x66, 0xd4, 0x57, 0x6d, 0x25, 0x60, 0x65, 0x35, 0xf9, 0x60, 0xc0, - 0x02, 0x61, 0x6e, 0xab, 0xca, 0xd2, 0x22, 0x6a, 0xf6, 0x02, 0xe7, 0x95, 0xcf, 0x5c, 0xf3, 0x7f, - 0x32, 0x2d, 0xb1, 0x88, 0x15, 0x7b, 0x3a, 0x34, 0x4d, 0x09, 0x66, 0x4f, 0x87, 0x78, 0x2e, 0xed, - 0xd1, 0x66, 0x4e, 0xc4, 0x03, 0xf3, 0xba, 0x3a, 0xd7, 0x0c, 0x48, 0x9e, 0x02, 0x74, 0x84, 0x23, - 0x58, 0xc7, 0x0b, 0xba, 0xcc, 0xac, 0xd6, 0x8c, 0x7a, 0x69, 0xa7, 0x6a, 0xa9, 0xae, 0xb7, 0xe2, - 0xae, 0xb7, 0x5e, 0xc6, 0x5d, 0x6f, 0x27, 0xac, 0xb1, 0xde, 0x1a, 0xbe, 0xcf, 0xdf, 0xd8, 0xcc, - 0xf5, 0x42, 0xd6, 0x15, 0x91, 0x79, 0x43, 0x5e, 0x49, 0x0a, 0x25, 0x9f, 0xe0, 0xdd, 0x44, 0xa2, - 0x33, 0x0e, 0xba, 0xe6, 0xcd, 0x2b, 0x3d, 0x4c, 0x6c, 0xc9, 0xd7, 0x40, 0xe4, 0xf7, 0xa8, 0xdb, - 0x65, 0x51, 0xd4, 0x1b, 0xf9, 0x72, 0x87, 0xff, 0x5f, 0xb9, 0xc3, 0x05, 0xab, 0xc8, 0x33, 0x28, - 0x21, 0x7a, 0xc4, 0x5d, 0xb4, 0x33, 0x6f, 0x5d, 0xb9, 0x49, 0xd2, 0x9c, 0x3e, 0x86, 0x75, 0xc5, - 0x0b, 0x6d, 0x2f, 0x12, 0x8a, 0xe7, 0xee, 0x40, 0x5e, 0x41, 0x91, 0x69, 0xd4, 0x72, 0xf5, 0xd2, - 0x4e, 0xde, 0x52, 0xb2, 0x1d, 0xe3, 0xd4, 0x82, 0x82, 0xfa, 0x3c, 0x6c, 0xbd, 0x0d, 0x9f, 0xd0, - 0x47, 0x00, 0x9a, 0xa8, 0xd0, 0xc1, 0x7b, 0x69, 0x07, 0x45, 0x2b, 0xde, 0x6d, 0xea, 0xe2, 0x4b, - 0xd8, 0x6c, 0x9e, 0x39, 0x41, 0x9f, 0xe1, 0xb5, 0x8c, 0xa2, 0x98, 0xe2, 0xd2, 0xde, 0x12, 0x55, - 0x93, 0x9d, 0xa9, 0x1a, 0x7a, 0x27, 0x3e, 0xd9, 0x61, 0xeb, 0x92, 0xc5, 0xf4, 0x2f, 0x03, 0xd6, - 0x1a, 0xae, 0xab, 0x4f, 0x27, 0x63, 0x4b, 0x76, 0x9b, 0xb1, 0xa8, 0xdb, 0xb2, 0xe9, 0x6e, 0x93, - 0x95, 0x2d, 0xeb, 0x3f, 0xe6, 0x4c, 0x2d, 0xe2, 0xba, 0x49, 0xcb, 0x69, 0xd2, 0x9c, 0x02, 0xa4, - 0x02, 0xb9, 0x46, 0xe7, 0x85, 0xa6, 0x4c, 0xfc, 0xc4, 0x18, 0xbe, 0x73, 0xc2, 0xc0, 0x0b, 0xfa, - 0x48, 0xfa, 0x39, 0xe4, 0xd8, 0x58, 0xa6, 0xf7, 0x61, 0xe3, 0x74, 0xe8, 0x3a, 0x82, 0x25, 0x83, - 0x26, 0xb0, 0xd4, 0xf2, 0x7a, 0x3d, 0x4d, 0xfa, 0xf2, 0x9b, 0xf6, 0x61, 0xeb, 0x80, 0xf1, 0x79, - 0xdb, 0xdb, 0xf1, 0x20, 0x90, 0xd6, 0x89, 0xcb, 0x8d, 0xe7, 0x43, 0xbc, 0x59, 0x76, 0xba, 0xd9, - 0x4c, 0x44, 0xb9, 0x54, 0x44, 0x3b, 0x60, 0xda, 0xac, 0x17, 0xb2, 0x08, 0x6f, 0x97, 0x47, 0x9e, - 0xe0, 0xe1, 0x38, 0x4e, 0xf8, 0x36, 0xac, 0xd8, 0xec, 0xcc, 0x89, 0xce, 0xa4, 0xb3, 0x82, 0xad, - 0x25, 0xfa, 0x87, 0x01, 0x1b, 0x9d, 0xae, 0x13, 0xc4, 0x81, 0x5d, 0x7c, 0xb7, 0xc8, 0xd7, 0x23, - 0xc1, 0xd5, 0x85, 0xea, 0xeb, 0x4d, 0x20, 0xe4, 0x09, 0x14, 0x4e, 0xb0, 0xbc, 0xbb, 0xdc, 0x97, - 0x29, 0x5f, 0xdb, 0xb9, 0x6e, 0xcd, 0xed, 0x6a, 0x1d, 0x31, 0x71, 0xc6, 0x5d, 0x7b, 0x62, 0x4a, - 0xdf, 0x87, 0x15, 0x85, 0x91, 0x3c, 0xe4, 0x1a, 0xed, 0x76, 0x25, 0x83, 0x1f, 0xfb, 0x2f, 0x4f, - 0x2a, 0x06, 0x29, 0xc2, 0xb2, 0xdd, 0xf9, 0xfe, 0x45, 0xb3, 0x92, 0xa5, 0x7f, 0x1a, 0xb0, 0x9e, - 0xdc, 0x4d, 0x3f, 0x01, 0xe2, 0x6a, 0x33, 0x66, 0x39, 0x8a, 0x42, 0x79, 0xdf, 0xf3, 0x59, 0x74, - 0x18, 0xb8, 0xec, 0x5c, 0x17, 0x63, 0xce, 0x9e, 0xc1, 0xd0, 0xe6, 0x9b, 0x80, 0xbf, 0x09, 0x62, - 0x9b, 0x9c, 0xb2, 0x49, 0x62, 0xe8, 0xc1, 0x66, 0x03, 0xfe, 0x9a, 0xb9, 0xb2, 0x52, 0x72, 0x76, - 0x2c, 0x62, 0x36, 0x5e, 0xfe, 0x70, 0xdc, 0xeb, 0x45, 0x4c, 0x1c, 0x45, 0xb2, 0x5c, 0x72, 0x76, - 0x02, 0xa1, 0xbf, 0x19, 0x50, 0xc1, 0x5e, 0x89, 0xd0, 0xe7, 0x95, 0x2f, 0x02, 0xb2, 0x0b, 0xc5, - 0x16, 0xf2, 0x9d, 0x70, 0x42, 0x21, 0xa3, 0x5d, 0x4c, 0x1a, 0x53, 0x63, 0xf2, 0x18, 0xf2, 0x28, - 0xec, 0x05, 0xea, 0x04, 0x8b, 0xd7, 0xc5, 0xa6, 0xf4, 0x67, 0x58, 0x4b, 0x44, 0x87, 0xc9, 0xfc, - 0x08, 0x96, 0x7b, 0x98, 0x1e, 0x4d, 0x02, 0x55, 0x6b, 0x56, 0x6f, 0xc9, 0xdc, 0xed, 0x61, 0x07, - 0xd9, 0xca, 0xb0, 0xba, 0x0b, 0x30, 0x05, 0xb1, 0x71, 0x7e, 0x64, 0x63, 0x7d, 0x2e, 0xfc, 0xc4, - 0x91, 0xf3, 0xda, 0xf1, 0x47, 0x4c, 0x67, 0x5f, 0x09, 0x4f, 0xb3, 0xbb, 0x06, 0xfd, 0xc5, 0x00, - 0x22, 0xb7, 0x5f, 0x5c, 0x71, 0xff, 0x75, 0x52, 0x98, 0xbe, 0xb2, 0x7f, 0xd5, 0xa0, 0xf8, 0x04, - 0x53, 0xf1, 0x47, 0xfa, 0xa0, 0x13, 0x59, 0xbe, 0x44, 0xc7, 0x82, 0x45, 0xba, 0xb6, 0x94, 0x40, - 0xf7, 0x91, 0x0b, 0x84, 0xe6, 0x79, 0xde, 0x8f, 0x16, 0x34, 0xdc, 0x91, 0x73, 0x6e, 0xb3, 0x68, - 0xe4, 0xeb, 0xbd, 0x97, 0xed, 0x04, 0x42, 0xeb, 0x40, 0x52, 0xfb, 0x68, 0xf6, 0xf1, 0xbd, 0x80, - 0xc9, 0x6b, 0x2c, 0xda, 0xf2, 0x9b, 0xde, 0x95, 0x3d, 0x16, 0x7a, 0x5d, 0x8c, 0x16, 0xef, 0x2c, - 0xc0, 0x91, 0xa0, 0xae, 0x6a, 0x22, 0xd3, 0x07, 0x50, 0x52, 0x56, 0x11, 0x4e, 0x9f, 0x94, 0x69, - 0x2e, 0x69, 0xba, 0xf3, 0x77, 0x01, 0x72, 0xcd, 0xf6, 0x21, 0x79, 0x02, 0x70, 0xc0, 0x44, 0xfc, - 0x88, 0xde, 0x9e, 0x4b, 0xf2, 0x1e, 0x3e, 0xf1, 0xab, 0xab, 0x56, 0xf2, 0xe5, 0x4e, 0x33, 0xe4, - 0x33, 0xc8, 0x9f, 0x0e, 0xfb, 0xa1, 0xe3, 0xb2, 0x4b, 0xd7, 0x5c, 0x82, 0xd3, 0x0c, 0x79, 0x8a, - 0x2c, 0xe6, 0x73, 0xc7, 0x7d, 0x87, 0xb5, 0x5f, 0x40, 0x39, 0x39, 0xc6, 0xc8, 0x96, 0x75, 0xc1, - 0x54, 0x5b, 0xb0, 0x7e, 0x07, 0x96, 0x64, 0x6e, 0x2e, 0xf3, 0x5c, 0xb1, 0x52, 0xe3, 0x9b, 0x66, - 0xc8, 0x03, 0x00, 0x3d, 0xf9, 0x82, 0x1e, 0x27, 0x15, 0x2b, 0x35, 0x06, 0xab, 0x71, 0x45, 0xd1, - 0x0c, 0xb9, 0x8f, 0x0f, 0x66, 0x3d, 0x00, 0x49, 0x8c, 0x57, 0xd7, 0xad, 0xd9, 0xa9, 0x48, 0x33, - 0xe4, 0x43, 0x28, 0x27, 0x67, 0xc9, 0xd4, 0x96, 0x58, 0x73, 0x33, 0x46, 0xa6, 0xac, 0xac, 0x78, - 0x4b, 0x9b, 0xcf, 0x07, 0x71, 0xf9, 0x91, 0x9f, 0xc1, 0x7a, 0x6a, 0x72, 0x5d, 0xb0, 0xfc, 0x9a, - 0x75, 0xd1, 0x74, 0xa3, 0x19, 0xf2, 0x1c, 0x36, 0xe6, 0xc6, 0x11, 0xb9, 0x6e, 0x5d, 0x36, 0xa2, - 0x16, 0xc4, 0xf1, 0x18, 0x60, 0xca, 0xff, 0x84, 0xcc, 0x8f, 0x96, 0x6a, 0xc5, 0x4a, 0x0d, 0x08, - 0x9a, 0x21, 0x8f, 0xa0, 0x38, 0xe1, 0x31, 0xb2, 0x61, 0xa5, 0x19, 0xb9, 0xba, 0x9e, 0xa2, 0x39, - 0x9a, 0x21, 0x9f, 0x42, 0x29, 0xc1, 0x02, 0x64, 0xd3, 0x9a, 0x67, 0xaa, 0xea, 0x86, 0x95, 0x26, - 0x0a, 0x9a, 0x21, 0xbb, 0xb0, 0x74, 0xe2, 0x05, 0xfd, 0x77, 0x28, 0xcb, 0xcf, 0x61, 0x75, 0xa6, - 0x93, 0x09, 0xe6, 0x73, 0x9e, 0x21, 0xaa, 0x9b, 0xd6, 0x7c, 0xc3, 0xd3, 0x0c, 0xb1, 0x54, 0xd9, - 0xa8, 0x0e, 0xcf, 0x5b, 0xea, 0x63, 0x81, 0x3b, 0x0b, 0x8a, 0x2d, 0xe6, 0xbf, 0xbd, 0xfd, 0x3d, - 0x28, 0x61, 0x41, 0x6b, 0x72, 0x98, 0xae, 0x28, 0x5b, 0x09, 0xbe, 0xa0, 0x19, 0xf2, 0x01, 0x94, - 0xe4, 0xbb, 0x52, 0x67, 0x6e, 0xd5, 0x4a, 0xfe, 0x0e, 0x57, 0x4b, 0xd6, 0xf4, 0xd1, 0x49, 0x33, - 0xaf, 0x56, 0xa4, 0x9b, 0x8f, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0x6c, 0x40, 0x6d, 0x65, 0x22, - 0x10, 0x00, 0x00, + // 1551 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0x5f, 0x6f, 0xdb, 0x46, + 0x12, 0x17, 0x25, 0x5b, 0xb2, 0x46, 0xb2, 0x2d, 0xaf, 0x1d, 0x1f, 0xa3, 0xe4, 0x12, 0x65, 0x2f, + 0x97, 0x28, 0x38, 0x1c, 0x73, 0xd1, 0x25, 0x77, 0x46, 0x2e, 0x77, 0x81, 0x4e, 0xb2, 0x1d, 0xb7, + 0x72, 0x6c, 0x50, 0x71, 0x8b, 0xf6, 0x8d, 0x11, 0x57, 0x32, 0x51, 0x8a, 0xab, 0x92, 0xab, 0xc4, + 0x02, 0xfa, 0x31, 0x8a, 0x3e, 0xf5, 0xa1, 0xfd, 0x00, 0x05, 0xfa, 0xe5, 0xfa, 0x5e, 0xcc, 0xee, + 0x52, 0xa2, 0x28, 0x5b, 0x4e, 0x53, 0xa0, 0x6f, 0x3b, 0xbf, 0x9d, 0xdd, 0x99, 0x9d, 0x3f, 0xbf, + 0x21, 0xa1, 0x18, 0x8e, 0x7a, 0xd6, 0x28, 0xe4, 0x82, 0x57, 0x6f, 0x0d, 0x38, 0x1f, 0xf8, 0xec, + 0xb1, 0x94, 0xde, 0x8e, 0xfb, 0x8f, 0xd9, 0x70, 0x24, 0x26, 0x7a, 0xf3, 0x6e, 0x7a, 0x53, 0x78, + 0x43, 0x16, 0x09, 0x67, 0x38, 0x52, 0x0a, 0xf4, 0x07, 0x03, 0xca, 0x9f, 0xb1, 0x30, 0xf2, 0x78, + 0x60, 0xb3, 0x91, 0x3f, 0x21, 0x26, 0x14, 0xb4, 0x6c, 0x1a, 0x35, 0xa3, 0x5e, 0xb4, 0x63, 0x91, + 0xec, 0xc0, 0xea, 0xff, 0xc7, 0x9e, 0xef, 0x9a, 0x59, 0x89, 0x2b, 0x81, 0xdc, 0x86, 0xe2, 0x21, + 0x8f, 0x4f, 0xe4, 0xe4, 0xce, 0x0c, 0x20, 0x1b, 0x90, 0x3d, 0xe9, 0x9a, 0x2b, 0x12, 0xce, 0x9e, + 0x74, 0x09, 0x81, 0x95, 0x66, 0xd8, 0x3b, 0x37, 0x57, 0x25, 0x22, 0xd7, 0xe4, 0x0e, 0xc0, 0x21, + 0x3f, 0x76, 0x2e, 0x4e, 0x43, 0xde, 0x8b, 0xcc, 0x7c, 0xcd, 0xa8, 0xaf, 0xda, 0x09, 0x84, 0xd6, + 0xa1, 0x7c, 0xec, 0x88, 0xde, 0xb9, 0xcd, 0xbe, 0x1e, 0xb3, 0x48, 0xa0, 0x87, 0xa7, 0x8e, 0x10, + 0x2c, 0x9c, 0x7a, 0xa8, 0x45, 0xfa, 0xdd, 0x1a, 0xe4, 0x8f, 0xbd, 0x30, 0xe4, 0x21, 0x1a, 0x3e, + 0x6a, 0xcb, 0xfd, 0x55, 0x3b, 0x7b, 0xd4, 0x46, 0xc3, 0xaf, 0x9d, 0x21, 0xd3, 0xbe, 0xcb, 0x35, + 0x5e, 0xf4, 0x4a, 0x88, 0xd1, 0x99, 0xdd, 0xd1, 0x8e, 0xc7, 0x22, 0xa9, 0xc2, 0x9a, 0x1d, 0x4d, + 0x82, 0x1e, 0x6e, 0x29, 0xe7, 0xa7, 0x32, 0xd9, 0x85, 0xfc, 0x81, 0x3a, 0xa4, 0x1e, 0xa1, 0x25, + 0x52, 0x83, 0x52, 0x77, 0xc4, 0x83, 0x88, 0x87, 0xd2, 0x50, 0x5e, 0x6e, 0x26, 0x21, 0x7c, 0xa8, + 0x16, 0xf1, 0x74, 0x41, 0x2a, 0x24, 0x10, 0xf2, 0x00, 0x36, 0xb4, 0xd4, 0xe1, 0x03, 0x8e, 0x3a, + 0x6b, 0x52, 0x27, 0x85, 0x62, 0xc8, 0x9b, 0xee, 0xd0, 0x0b, 0xa4, 0x9d, 0xa2, 0x0a, 0xf9, 0x14, + 0x40, 0x2b, 0x52, 0xd8, 0x1f, 0x3a, 0x9e, 0x6f, 0x82, 0xb2, 0x32, 0x43, 0x70, 0xbf, 0x35, 0x8e, + 0x04, 0x1f, 0xb6, 0x1d, 0xe1, 0x98, 0x25, 0xb5, 0x3f, 0x43, 0xc8, 0x7d, 0x58, 0x6f, 0xf1, 0x40, + 0x78, 0x01, 0x0b, 0xc4, 0x49, 0xe0, 0x4f, 0xcc, 0x72, 0xcd, 0xa8, 0xaf, 0xd9, 0xf3, 0x20, 0xbe, + 0xb6, 0xc5, 0xc7, 0x81, 0x08, 0x27, 0x52, 0x67, 0x5d, 0xea, 0x24, 0x21, 0x8c, 0x53, 0xb3, 0x2b, + 0x37, 0x37, 0xe4, 0xa6, 0x96, 0xb0, 0x8c, 0xba, 0x3d, 0x1e, 0x32, 0x73, 0x53, 0x26, 0x47, 0x09, + 0x18, 0xf1, 0x8e, 0x23, 0x3c, 0x31, 0x76, 0x99, 0x59, 0xa9, 0x19, 0xf5, 0xac, 0x3d, 0x95, 0xf1, + 0xbd, 0x1d, 0x1e, 0x0c, 0xd4, 0xe6, 0x96, 0xdc, 0x9c, 0x01, 0x73, 0xfe, 0xb6, 0xb8, 0xcb, 0x4c, + 0x22, 0x9f, 0x34, 0x0f, 0x12, 0x0a, 0x65, 0xed, 0x1c, 0x8a, 0x91, 0xb9, 0x2d, 0x95, 0xe6, 0x30, + 0xd2, 0x80, 0x9d, 0xfd, 0x8b, 0x9e, 0x3f, 0x76, 0x99, 0x3b, 0xa7, 0xbb, 0x23, 0x75, 0x2f, 0xdd, + 0xc3, 0xd7, 0x34, 0xa3, 0x60, 0x3c, 0x34, 0x6f, 0xd4, 0x8c, 0xfa, 0xba, 0xad, 0x04, 0xac, 0xac, + 0x16, 0x1f, 0x0e, 0x59, 0x20, 0xcc, 0x5d, 0x55, 0x59, 0x5a, 0xc4, 0x9d, 0xfd, 0xc0, 0x79, 0xeb, + 0x33, 0xd7, 0xfc, 0x93, 0x0c, 0x4b, 0x2c, 0x62, 0xc5, 0x9e, 0x8d, 0x4c, 0x53, 0x82, 0xd9, 0xb3, + 0x11, 0xbe, 0x4b, 0x5b, 0xb4, 0x99, 0x13, 0xf1, 0xc0, 0xbc, 0xa9, 0xde, 0x35, 0x07, 0x92, 0xe7, + 0x00, 0x5d, 0xe1, 0x08, 0xd6, 0xf5, 0x82, 0x1e, 0x33, 0xab, 0x35, 0xa3, 0x5e, 0x6a, 0x54, 0x2d, + 0xd5, 0xf5, 0x56, 0xdc, 0xf5, 0xd6, 0x9b, 0xb8, 0xeb, 0xed, 0x84, 0x36, 0xd6, 0x5b, 0xd3, 0xf7, + 0xf9, 0x7b, 0x9b, 0xb9, 0x5e, 0xc8, 0x7a, 0x22, 0x32, 0x6f, 0xc9, 0x94, 0xa4, 0x50, 0xf2, 0x2f, + 0xcc, 0x4d, 0x24, 0xba, 0x93, 0xa0, 0x67, 0xde, 0xbe, 0xd6, 0xc2, 0x54, 0x97, 0x7c, 0x02, 0x44, + 0xae, 0xc7, 0xbd, 0x1e, 0x8b, 0xa2, 0xfe, 0xd8, 0x97, 0x37, 0xfc, 0xf9, 0xda, 0x1b, 0x2e, 0x39, + 0x45, 0x5e, 0x40, 0x09, 0xd1, 0x63, 0xee, 0xa2, 0x9e, 0x79, 0xe7, 0xda, 0x4b, 0x92, 0xea, 0xf4, + 0x29, 0x6c, 0x2a, 0x5e, 0xe8, 0x78, 0x91, 0x50, 0x3c, 0x77, 0x0f, 0x0a, 0x0a, 0x8a, 0x4c, 0xa3, + 0x96, 0xab, 0x97, 0x1a, 0x05, 0x4b, 0xc9, 0x76, 0x8c, 0x53, 0x0b, 0xd6, 0xd4, 0xf2, 0xa8, 0xfd, + 0x21, 0x7c, 0x42, 0x9f, 0x00, 0x68, 0xa2, 0x42, 0x03, 0x7f, 0x49, 0x1b, 0x28, 0x5a, 0xf1, 0x6d, + 0x33, 0x13, 0x2f, 0x61, 0xbb, 0x75, 0xee, 0x04, 0x03, 0x86, 0x69, 0x19, 0x47, 0x31, 0xc5, 0xa5, + 0xad, 0x25, 0xaa, 0x26, 0x3b, 0x57, 0x35, 0xf4, 0x5e, 0xfc, 0xb2, 0xa3, 0xf6, 0x15, 0x87, 0xe9, + 0xcf, 0x06, 0x6c, 0x34, 0x5d, 0x57, 0xbf, 0x4e, 0xfa, 0x96, 0xec, 0x36, 0x63, 0x59, 0xb7, 0x65, + 0xd3, 0xdd, 0x26, 0x2b, 0x5b, 0xd6, 0x7f, 0xcc, 0x99, 0x5a, 0xc4, 0x73, 0xd3, 0x96, 0xd3, 0xa4, + 0x39, 0x03, 0x48, 0x05, 0x72, 0xcd, 0xee, 0x6b, 0x4d, 0x99, 0xb8, 0x44, 0x1f, 0x3e, 0x77, 0xc2, + 0xc0, 0x0b, 0x06, 0x48, 0xfa, 0x39, 0xe4, 0xd8, 0x58, 0xa6, 0x0f, 0x61, 0xeb, 0x6c, 0xe4, 0x3a, + 0x82, 0x25, 0x9d, 0x26, 0xb0, 0xd2, 0xf6, 0xfa, 0x7d, 0x4d, 0xfa, 0x72, 0x4d, 0x07, 0xb0, 0x73, + 0xc8, 0xf8, 0xa2, 0xee, 0xdd, 0x78, 0x10, 0x48, 0xed, 0x44, 0x72, 0xe3, 0xf9, 0x10, 0x5f, 0x96, + 0x9d, 0x5d, 0x36, 0xe7, 0x51, 0x2e, 0xe5, 0x51, 0x03, 0x4c, 0x9b, 0xf5, 0x43, 0x16, 0x61, 0x76, + 0x79, 0xe4, 0x09, 0x1e, 0x4e, 0xe2, 0x80, 0xef, 0x42, 0xde, 0x66, 0xe7, 0x4e, 0x74, 0x2e, 0x8d, + 0xad, 0xd9, 0x5a, 0xa2, 0x3f, 0x1a, 0xb0, 0xd5, 0xed, 0x39, 0x41, 0xec, 0xd8, 0xe5, 0xb9, 0x45, + 0xbe, 0x1e, 0x0b, 0xae, 0x12, 0xaa, 0xd3, 0x9b, 0x40, 0xc8, 0x33, 0x58, 0x3b, 0xc5, 0xf2, 0xee, + 0x71, 0x5f, 0x86, 0x7c, 0xa3, 0x71, 0xd3, 0x5a, 0xb8, 0xd5, 0x3a, 0x66, 0xe2, 0x9c, 0xbb, 0xf6, + 0x54, 0x95, 0xfe, 0x15, 0xf2, 0x0a, 0x23, 0x05, 0xc8, 0x35, 0x3b, 0x9d, 0x4a, 0x06, 0x17, 0x07, + 0x6f, 0x4e, 0x2b, 0x06, 0x29, 0xc2, 0xaa, 0xdd, 0xfd, 0xe2, 0x75, 0xab, 0x92, 0xa5, 0x3f, 0x19, + 0xb0, 0x99, 0xbc, 0x4d, 0x7f, 0x02, 0xc4, 0xd5, 0x66, 0xcc, 0x73, 0x14, 0x85, 0xf2, 0x81, 0xe7, + 0xb3, 0xe8, 0x28, 0x70, 0xd9, 0x85, 0x2e, 0xc6, 0x9c, 0x3d, 0x87, 0xa1, 0xce, 0xa7, 0x01, 0x7f, + 0x1f, 0xc4, 0x3a, 0x39, 0xa5, 0x93, 0xc4, 0xd0, 0x82, 0xcd, 0x86, 0xfc, 0x1d, 0x73, 0x65, 0xa5, + 0xe4, 0xec, 0x58, 0xc4, 0x68, 0xbc, 0xf9, 0xf2, 0xa4, 0xdf, 0x8f, 0x98, 0x38, 0x8e, 0x64, 0xb9, + 0xe4, 0xec, 0x04, 0x42, 0xbf, 0x37, 0xa0, 0x82, 0xbd, 0x12, 0xa1, 0xcd, 0x6b, 0xbf, 0x08, 0xc8, + 0x1e, 0x14, 0xdb, 0xc8, 0x77, 0xc2, 0x09, 0x85, 0xf4, 0x76, 0x39, 0x69, 0xcc, 0x94, 0xc9, 0x53, + 0x28, 0xa0, 0xb0, 0x1f, 0xa8, 0x17, 0x2c, 0x3f, 0x17, 0xab, 0xd2, 0x6f, 0x60, 0x23, 0xe1, 0x1d, + 0x06, 0xf3, 0x1f, 0xb0, 0xda, 0xc7, 0xf0, 0x68, 0x12, 0xa8, 0x5a, 0xf3, 0xfb, 0x96, 0x8c, 0xdd, + 0x3e, 0x76, 0x90, 0xad, 0x14, 0xab, 0x7b, 0x00, 0x33, 0x10, 0x1b, 0xe7, 0x2b, 0x36, 0xd1, 0xef, + 0xc2, 0x25, 0x8e, 0x9c, 0x77, 0x8e, 0x3f, 0x66, 0x3a, 0xfa, 0x4a, 0x78, 0x9e, 0xdd, 0x33, 0xe8, + 0xb7, 0x06, 0x10, 0x79, 0xfd, 0xf2, 0x8a, 0xfb, 0xa3, 0x83, 0xc2, 0x74, 0xca, 0x7e, 0x53, 0x83, + 0xe2, 0x27, 0x98, 0xf2, 0x3f, 0xd2, 0x0f, 0x9d, 0xca, 0xf2, 0x4b, 0x74, 0x22, 0x58, 0xa4, 0x6b, + 0x4b, 0x09, 0xf4, 0x00, 0xb9, 0x40, 0x68, 0x9e, 0xe7, 0x83, 0x68, 0x49, 0xc3, 0x1d, 0x3b, 0x17, + 0x36, 0x8b, 0xc6, 0xbe, 0xbe, 0x7b, 0xd5, 0x4e, 0x20, 0xb4, 0x0e, 0x24, 0x75, 0x8f, 0x66, 0x1f, + 0xdf, 0x0b, 0x98, 0x4c, 0x63, 0xd1, 0x96, 0x6b, 0x7a, 0x5f, 0xf6, 0x58, 0xe8, 0xf5, 0xd0, 0x5b, + 0xcc, 0x59, 0x80, 0x23, 0x41, 0xa5, 0x6a, 0x2a, 0xd3, 0x47, 0x50, 0x52, 0x5a, 0x11, 0x4e, 0x9f, + 0x94, 0x6a, 0x6e, 0x4e, 0xb5, 0x06, 0x79, 0x35, 0x08, 0x90, 0x53, 0xd4, 0x2a, 0xe6, 0x14, 0x25, + 0x35, 0x7e, 0x29, 0x42, 0xae, 0xd5, 0x39, 0x22, 0xcf, 0x00, 0x0e, 0x99, 0x88, 0x3f, 0xb3, 0x77, + 0x17, 0xd2, 0xb0, 0x8f, 0x3f, 0x01, 0xd5, 0x75, 0x2b, 0xf9, 0x6d, 0x4f, 0x33, 0xe4, 0x3f, 0x50, + 0x38, 0x1b, 0x0d, 0x42, 0xc7, 0x65, 0x57, 0x9e, 0xb9, 0x02, 0xa7, 0x19, 0xf2, 0x1c, 0x79, 0xce, + 0xe7, 0x8e, 0xfb, 0x11, 0x67, 0xff, 0x07, 0xe5, 0xe4, 0xa0, 0x23, 0x3b, 0xd6, 0x25, 0x73, 0x6f, + 0xc9, 0xf9, 0x06, 0xac, 0xc8, 0xe8, 0x5d, 0x65, 0xb9, 0x62, 0xa5, 0x06, 0x3c, 0xcd, 0x90, 0x47, + 0x00, 0x7a, 0x36, 0x06, 0x7d, 0x4e, 0x2a, 0x56, 0x6a, 0x50, 0x56, 0xe3, 0x9a, 0xa3, 0x19, 0xf2, + 0x10, 0x3f, 0xa9, 0xf5, 0x88, 0x24, 0x31, 0x5e, 0xdd, 0xb4, 0xe6, 0xe7, 0x26, 0xcd, 0x90, 0xbf, + 0x43, 0x39, 0x39, 0x6d, 0x66, 0xba, 0xc4, 0x5a, 0x98, 0x42, 0x32, 0x64, 0x65, 0xc5, 0x6c, 0x5a, + 0x7d, 0xd1, 0x89, 0xab, 0x9f, 0xfc, 0x02, 0x36, 0x53, 0xb3, 0xed, 0x92, 0xe3, 0x37, 0xac, 0xcb, + 0xe6, 0x1f, 0xcd, 0x90, 0x57, 0xb0, 0xb5, 0x30, 0xb0, 0xc8, 0x4d, 0xeb, 0xaa, 0x21, 0xb6, 0xc4, + 0x8f, 0xa7, 0x00, 0xb3, 0x09, 0x41, 0xc8, 0xe2, 0xf0, 0xa9, 0x56, 0xac, 0xd4, 0x08, 0xa1, 0x19, + 0xf2, 0x04, 0x8a, 0x53, 0xa6, 0x23, 0x5b, 0x56, 0x9a, 0xb3, 0xab, 0x9b, 0x29, 0x22, 0xa4, 0x19, + 0xf2, 0x6f, 0x28, 0x25, 0x78, 0x82, 0x6c, 0x5b, 0x8b, 0x5c, 0x56, 0xdd, 0xb2, 0xd2, 0x54, 0x42, + 0x33, 0x64, 0x0f, 0x56, 0x4e, 0xbd, 0x60, 0xf0, 0x11, 0x65, 0xf9, 0x5f, 0x58, 0x9f, 0xeb, 0x75, + 0x82, 0xf1, 0x5c, 0xe4, 0x90, 0xea, 0xb6, 0xb5, 0x48, 0x09, 0x34, 0x43, 0x2c, 0x55, 0x36, 0x8a, + 0x03, 0x0a, 0x96, 0x5a, 0x2c, 0x31, 0x67, 0x41, 0xb1, 0xcd, 0xfc, 0x0f, 0xd7, 0x7f, 0x00, 0x25, + 0x2c, 0x68, 0x4d, 0x1f, 0xb3, 0x13, 0x65, 0x2b, 0xc1, 0x28, 0xb2, 0xbb, 0x40, 0x8d, 0x68, 0xfc, + 0x6e, 0xf8, 0x88, 0x30, 0xbc, 0x84, 0x52, 0xdb, 0x8b, 0x7e, 0xc7, 0x05, 0x0d, 0x19, 0x47, 0xd5, + 0xcc, 0x4b, 0xaf, 0x28, 0x58, 0x4a, 0x89, 0x66, 0xc8, 0xdf, 0xa0, 0x24, 0x3f, 0x97, 0x75, 0xba, + 0xd7, 0xad, 0xe4, 0x5f, 0x7e, 0xb5, 0x64, 0xcd, 0xbe, 0xa5, 0x69, 0xe6, 0x6d, 0x5e, 0xde, 0xf3, + 0xcf, 0x5f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x30, 0xe6, 0x21, 0xd7, 0xf9, 0x10, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -1521,6 +1563,9 @@ type CLIClient interface { AddMetric(ctx context.Context, in *Metric, opts ...grpc.CallOption) (*empty.Empty, error) DelMetric(ctx context.Context, in *Metric, opts ...grpc.CallOption) (*empty.Empty, error) ListMetrics(ctx context.Context, in *Metric, opts ...grpc.CallOption) (*MetricsList, error) + EnableAuto(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) + DisableAuto(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) + GetStatusAuto(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*Status, error) // Tools MatchMirror(ctx context.Context, in *MatchRequest, opts ...grpc.CallOption) (*MatchReply, error) } @@ -1704,6 +1749,33 @@ func (c *cLIClient) ListMetrics(ctx context.Context, in *Metric, opts ...grpc.Ca return out, nil } +func (c *cLIClient) EnableAuto(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) { + out := new(empty.Empty) + err := c.cc.Invoke(ctx, "/CLI/EnableAuto", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cLIClient) DisableAuto(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) { + out := new(empty.Empty) + err := c.cc.Invoke(ctx, "/CLI/DisableAuto", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cLIClient) GetStatusAuto(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*Status, error) { + out := new(Status) + err := c.cc.Invoke(ctx, "/CLI/GetStatusAuto", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *cLIClient) MatchMirror(ctx context.Context, in *MatchRequest, opts ...grpc.CallOption) (*MatchReply, error) { out := new(MatchReply) err := c.cc.Invoke(ctx, "/CLI/MatchMirror", in, out, opts...) @@ -1734,6 +1806,9 @@ type CLIServer interface { AddMetric(context.Context, *Metric) (*empty.Empty, error) DelMetric(context.Context, *Metric) (*empty.Empty, error) ListMetrics(context.Context, *Metric) (*MetricsList, error) + EnableAuto(context.Context, *empty.Empty) (*empty.Empty, error) + DisableAuto(context.Context, *empty.Empty) (*empty.Empty, error) + GetStatusAuto(context.Context, *empty.Empty) (*Status, error) // Tools MatchMirror(context.Context, *MatchRequest) (*MatchReply, error) } @@ -1799,6 +1874,15 @@ func (*UnimplementedCLIServer) DelMetric(ctx context.Context, req *Metric) (*emp func (*UnimplementedCLIServer) ListMetrics(ctx context.Context, req *Metric) (*MetricsList, error) { return nil, status.Errorf(codes.Unimplemented, "method ListMetrics not implemented") } +func (*UnimplementedCLIServer) EnableAuto(ctx context.Context, req *empty.Empty) (*empty.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method EnableAuto not implemented") +} +func (*UnimplementedCLIServer) DisableAuto(ctx context.Context, req *empty.Empty) (*empty.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method DisableAuto not implemented") +} +func (*UnimplementedCLIServer) GetStatusAuto(ctx context.Context, req *empty.Empty) (*Status, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetStatusAuto not implemented") +} func (*UnimplementedCLIServer) MatchMirror(ctx context.Context, req *MatchRequest) (*MatchReply, error) { return nil, status.Errorf(codes.Unimplemented, "method MatchMirror not implemented") } @@ -2149,6 +2233,60 @@ func _CLI_ListMetrics_Handler(srv interface{}, ctx context.Context, dec func(int return interceptor(ctx, in, info, handler) } +func _CLI_EnableAuto_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(empty.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CLIServer).EnableAuto(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/CLI/EnableAuto", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CLIServer).EnableAuto(ctx, req.(*empty.Empty)) + } + return interceptor(ctx, in, info, handler) +} + +func _CLI_DisableAuto_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(empty.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CLIServer).DisableAuto(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/CLI/DisableAuto", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CLIServer).DisableAuto(ctx, req.(*empty.Empty)) + } + return interceptor(ctx, in, info, handler) +} + +func _CLI_GetStatusAuto_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(empty.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CLIServer).GetStatusAuto(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/CLI/GetStatusAuto", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CLIServer).GetStatusAuto(ctx, req.(*empty.Empty)) + } + return interceptor(ctx, in, info, handler) +} + func _CLI_MatchMirror_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(MatchRequest) if err := dec(in); err != nil { @@ -2247,6 +2385,18 @@ var _CLI_serviceDesc = grpc.ServiceDesc{ MethodName: "ListMetrics", Handler: _CLI_ListMetrics_Handler, }, + { + MethodName: "EnableAuto", + Handler: _CLI_EnableAuto_Handler, + }, + { + MethodName: "DisableAuto", + Handler: _CLI_DisableAuto_Handler, + }, + { + MethodName: "GetStatusAuto", + Handler: _CLI_GetStatusAuto_Handler, + }, { MethodName: "MatchMirror", Handler: _CLI_MatchMirror_Handler, diff --git a/rpc/rpc.proto b/rpc/rpc.proto index b6e1ed7e..c4c83cc9 100644 --- a/rpc/rpc.proto +++ b/rpc/rpc.proto @@ -23,6 +23,9 @@ service CLI { rpc AddMetric (Metric) returns (google.protobuf.Empty) {} rpc DelMetric (Metric) returns (google.protobuf.Empty) {} rpc ListMetrics (Metric) returns (MetricsList) {} + rpc EnableAuto (google.protobuf.Empty) returns (google.protobuf.Empty) {} + rpc DisableAuto (google.protobuf.Empty) returns (google.protobuf.Empty) {} + rpc GetStatusAuto (google.protobuf.Empty) returns (Status) {} // Tools rpc MatchMirror (MatchRequest) returns (MatchReply) {} @@ -176,3 +179,7 @@ message Metric { message MetricsList { repeated string Filename = 1; } + +message Status { + bool Status = 1; +} diff --git a/scan/scan.go b/scan/scan.go index 06fb2c05..d0d45734 100644 --- a/scan/scan.go +++ b/scan/scan.go @@ -236,6 +236,11 @@ func (s *scan) ScannerAddFile(f filedata) { ik := fmt.Sprintf("FILEINFO_%d_%s", s.mirrorid, f.path) s.conn.Send("HMSET", ik, "size", f.size, "modTime", f.modTime) + // Add file to tracked files if auto tracked files is enabled + if GetConfig().MetricsAutoTrackedFiles { + s.conn.Send("SADD", "TRACKED_FILES", f.path) + } + // Publish update database.SendPublish(s.conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%d %s", s.mirrorid, f.path)) } From de9c84bae75fce4c29c8cc85b29f54220db75c04 Mon Sep 17 00:00:00 2001 From: Duncan McNamara Date: Wed, 22 May 2024 17:07:50 +0200 Subject: [PATCH 6/8] Reload: reload metrics config --- http/http.go | 8 ++++++++ main.go | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/http/http.go b/http/http.go index acc4415a..ed272adc 100644 --- a/http/http.go +++ b/http/http.go @@ -167,6 +167,14 @@ func (h *HTTP) Reload() { // Reload the GeoIP database h.geoip.LoadGeoIP() + if config.GetConfig().MetricsEnabled && h.metrics == nil { + log.Info("Configuration Reload: Metrics enabled") + h.metrics = NewMetrics(h.redis) + http.Handle("/metrics", NewGzipHandler(h.metricsHandler)) + } else { + log.Info("Configuration Reload: Metrics not enabled") + } + // Reload the templates h.templates.Lock() if t, err := h.LoadTemplates("mirrorlist"); err == nil { diff --git a/main.go b/main.go index cacaa52b..4db90ed7 100644 --- a/main.go +++ b/main.go @@ -104,12 +104,13 @@ func main() { } case syscall.SIGHUP: listenAddress := GetConfig().ListenAddress + metricsEnabled := GetConfig().MetricsEnabled if err := ReloadConfig(); err != nil { log.Warningf("SIGHUP Received: %s\n", err) } else { log.Notice("SIGHUP Received: Reloading configuration...") } - if GetConfig().ListenAddress != listenAddress { + if GetConfig().ListenAddress != listenAddress || GetConfig().MetricsEnabled != metricsEnabled { h.Restarting = true h.Stop(1 * time.Second) } From 10b589ba1441bcb091569eb44c7b3fb1f3861fab Mon Sep 17 00:00:00 2001 From: Duncan McNamara Date: Fri, 29 Mar 2024 15:27:20 +0100 Subject: [PATCH 7/8] Cli: add metrics status --- cli/commands.go | 18 ++++ http/http.go | 1 + rpc/rpc.go | 4 + rpc/rpc.pb.go | 233 ++++++++++++++++++++++++++++-------------------- rpc/rpc.proto | 1 + 5 files changed, 159 insertions(+), 98 deletions(-) diff --git a/cli/commands.go b/cli/commands.go index 91ddba2c..3f53fed4 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -356,6 +356,7 @@ func (c *cli) CmdMetrics(args ...string) error { _ = cmd.String("add", "", "Add a file to the metrics route") _ = cmd.String("list", "*", "List files in metrics + Optionnal pattern to filter results") _ = cmd.String("delete", "", "Delete a file from the metrics route") + _ = cmd.Bool("status", false, "Show if metrics are enabled") _ = cmd.Bool("auto-enable", true, "Enable automatic addition of new files to tracked files") _ = cmd.Bool("auto-disable", false, "Disable automatic addition of new files to tracked files") _ = cmd.Bool("auto-status", true, "Print boolean of automatic addition of new files to tracked files") @@ -382,6 +383,8 @@ func (c *cli) CmdMetrics(args ...string) error { c.CmdDisableauto() } else if args[0] == "-auto-status" { c.CmdStatusauto() + } else if args[0] == "-status" { + c.CmdStatusmetrics() } else { cmd.Usage() return nil @@ -452,6 +455,21 @@ func (c *cli) CmdListmetrics(pattern string) error { return nil } +func (c *cli) CmdStatusmetrics() error { + client := c.GetRPC() + ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) + defer cancel() + status, _ := client.GetStatusMetrics(ctx, &empty.Empty{}) + + if status.Status { + log.Info("Metrics are enabled") + } else { + log.Info("Metrics are disabled") + } + + return nil +} + func (c *cli) CmdEnableauto() error { client := c.GetRPC() ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) diff --git a/http/http.go b/http/http.go index ed272adc..f58ef512 100644 --- a/http/http.go +++ b/http/http.go @@ -414,6 +414,7 @@ func (h *HTTP) mirrorHandler(w http.ResponseWriter, r *http.Request, ctx *Contex sort.Sort(mirrors.ByRank{Mirrors: mlist, ClientInfo: clientInfo}) } else { // No fallback in stock, there's nothing else we can do + log.Error("503, fallback") http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable) return } diff --git a/rpc/rpc.go b/rpc/rpc.go index 4be9ed1c..6746cd2c 100644 --- a/rpc/rpc.go +++ b/rpc/rpc.go @@ -834,6 +834,10 @@ func (c *CLI) ListMetrics(ctx context.Context, in *Metric) (*MetricsList, error) return &MetricsList{Filename: files}, nil } +func (c *CLI) GetStatusMetrics(context.Context, *empty.Empty) (*Status, error) { + return &Status{Status: GetConfig().MetricsEnabled}, nil +} + func (c *CLI) EnableAuto(context.Context, *empty.Empty) (*empty.Empty, error) { GetConfig().MetricsAutoTrackedFiles = true return &empty.Empty{}, nil diff --git a/rpc/rpc.pb.go b/rpc/rpc.pb.go index 3973d244..5ccec31b 100644 --- a/rpc/rpc.pb.go +++ b/rpc/rpc.pb.go @@ -1432,104 +1432,105 @@ func init() { } var fileDescriptor_77a6da22d6a3feb1 = []byte{ - // 1551 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0x5f, 0x6f, 0xdb, 0x46, - 0x12, 0x17, 0x25, 0x5b, 0xb2, 0x46, 0xb2, 0x2d, 0xaf, 0x1d, 0x1f, 0xa3, 0xe4, 0x12, 0x65, 0x2f, - 0x97, 0x28, 0x38, 0x1c, 0x73, 0xd1, 0x25, 0x77, 0x46, 0x2e, 0x77, 0x81, 0x4e, 0xb2, 0x1d, 0xb7, - 0x72, 0x6c, 0x50, 0x71, 0x8b, 0xf6, 0x8d, 0x11, 0x57, 0x32, 0x51, 0x8a, 0xab, 0x92, 0xab, 0xc4, - 0x02, 0xfa, 0x31, 0x8a, 0x3e, 0xf5, 0xa1, 0xfd, 0x00, 0x05, 0xfa, 0xe5, 0xfa, 0x5e, 0xcc, 0xee, - 0x52, 0xa2, 0x28, 0x5b, 0x4e, 0x53, 0xa0, 0x6f, 0x3b, 0xbf, 0x9d, 0xdd, 0x99, 0x9d, 0x3f, 0xbf, - 0x21, 0xa1, 0x18, 0x8e, 0x7a, 0xd6, 0x28, 0xe4, 0x82, 0x57, 0x6f, 0x0d, 0x38, 0x1f, 0xf8, 0xec, - 0xb1, 0x94, 0xde, 0x8e, 0xfb, 0x8f, 0xd9, 0x70, 0x24, 0x26, 0x7a, 0xf3, 0x6e, 0x7a, 0x53, 0x78, - 0x43, 0x16, 0x09, 0x67, 0x38, 0x52, 0x0a, 0xf4, 0x07, 0x03, 0xca, 0x9f, 0xb1, 0x30, 0xf2, 0x78, - 0x60, 0xb3, 0x91, 0x3f, 0x21, 0x26, 0x14, 0xb4, 0x6c, 0x1a, 0x35, 0xa3, 0x5e, 0xb4, 0x63, 0x91, - 0xec, 0xc0, 0xea, 0xff, 0xc7, 0x9e, 0xef, 0x9a, 0x59, 0x89, 0x2b, 0x81, 0xdc, 0x86, 0xe2, 0x21, - 0x8f, 0x4f, 0xe4, 0xe4, 0xce, 0x0c, 0x20, 0x1b, 0x90, 0x3d, 0xe9, 0x9a, 0x2b, 0x12, 0xce, 0x9e, - 0x74, 0x09, 0x81, 0x95, 0x66, 0xd8, 0x3b, 0x37, 0x57, 0x25, 0x22, 0xd7, 0xe4, 0x0e, 0xc0, 0x21, - 0x3f, 0x76, 0x2e, 0x4e, 0x43, 0xde, 0x8b, 0xcc, 0x7c, 0xcd, 0xa8, 0xaf, 0xda, 0x09, 0x84, 0xd6, - 0xa1, 0x7c, 0xec, 0x88, 0xde, 0xb9, 0xcd, 0xbe, 0x1e, 0xb3, 0x48, 0xa0, 0x87, 0xa7, 0x8e, 0x10, - 0x2c, 0x9c, 0x7a, 0xa8, 0x45, 0xfa, 0xdd, 0x1a, 0xe4, 0x8f, 0xbd, 0x30, 0xe4, 0x21, 0x1a, 0x3e, - 0x6a, 0xcb, 0xfd, 0x55, 0x3b, 0x7b, 0xd4, 0x46, 0xc3, 0xaf, 0x9d, 0x21, 0xd3, 0xbe, 0xcb, 0x35, - 0x5e, 0xf4, 0x4a, 0x88, 0xd1, 0x99, 0xdd, 0xd1, 0x8e, 0xc7, 0x22, 0xa9, 0xc2, 0x9a, 0x1d, 0x4d, - 0x82, 0x1e, 0x6e, 0x29, 0xe7, 0xa7, 0x32, 0xd9, 0x85, 0xfc, 0x81, 0x3a, 0xa4, 0x1e, 0xa1, 0x25, - 0x52, 0x83, 0x52, 0x77, 0xc4, 0x83, 0x88, 0x87, 0xd2, 0x50, 0x5e, 0x6e, 0x26, 0x21, 0x7c, 0xa8, - 0x16, 0xf1, 0x74, 0x41, 0x2a, 0x24, 0x10, 0xf2, 0x00, 0x36, 0xb4, 0xd4, 0xe1, 0x03, 0x8e, 0x3a, - 0x6b, 0x52, 0x27, 0x85, 0x62, 0xc8, 0x9b, 0xee, 0xd0, 0x0b, 0xa4, 0x9d, 0xa2, 0x0a, 0xf9, 0x14, - 0x40, 0x2b, 0x52, 0xd8, 0x1f, 0x3a, 0x9e, 0x6f, 0x82, 0xb2, 0x32, 0x43, 0x70, 0xbf, 0x35, 0x8e, - 0x04, 0x1f, 0xb6, 0x1d, 0xe1, 0x98, 0x25, 0xb5, 0x3f, 0x43, 0xc8, 0x7d, 0x58, 0x6f, 0xf1, 0x40, - 0x78, 0x01, 0x0b, 0xc4, 0x49, 0xe0, 0x4f, 0xcc, 0x72, 0xcd, 0xa8, 0xaf, 0xd9, 0xf3, 0x20, 0xbe, - 0xb6, 0xc5, 0xc7, 0x81, 0x08, 0x27, 0x52, 0x67, 0x5d, 0xea, 0x24, 0x21, 0x8c, 0x53, 0xb3, 0x2b, - 0x37, 0x37, 0xe4, 0xa6, 0x96, 0xb0, 0x8c, 0xba, 0x3d, 0x1e, 0x32, 0x73, 0x53, 0x26, 0x47, 0x09, - 0x18, 0xf1, 0x8e, 0x23, 0x3c, 0x31, 0x76, 0x99, 0x59, 0xa9, 0x19, 0xf5, 0xac, 0x3d, 0x95, 0xf1, - 0xbd, 0x1d, 0x1e, 0x0c, 0xd4, 0xe6, 0x96, 0xdc, 0x9c, 0x01, 0x73, 0xfe, 0xb6, 0xb8, 0xcb, 0x4c, - 0x22, 0x9f, 0x34, 0x0f, 0x12, 0x0a, 0x65, 0xed, 0x1c, 0x8a, 0x91, 0xb9, 0x2d, 0x95, 0xe6, 0x30, - 0xd2, 0x80, 0x9d, 0xfd, 0x8b, 0x9e, 0x3f, 0x76, 0x99, 0x3b, 0xa7, 0xbb, 0x23, 0x75, 0x2f, 0xdd, - 0xc3, 0xd7, 0x34, 0xa3, 0x60, 0x3c, 0x34, 0x6f, 0xd4, 0x8c, 0xfa, 0xba, 0xad, 0x04, 0xac, 0xac, - 0x16, 0x1f, 0x0e, 0x59, 0x20, 0xcc, 0x5d, 0x55, 0x59, 0x5a, 0xc4, 0x9d, 0xfd, 0xc0, 0x79, 0xeb, - 0x33, 0xd7, 0xfc, 0x93, 0x0c, 0x4b, 0x2c, 0x62, 0xc5, 0x9e, 0x8d, 0x4c, 0x53, 0x82, 0xd9, 0xb3, - 0x11, 0xbe, 0x4b, 0x5b, 0xb4, 0x99, 0x13, 0xf1, 0xc0, 0xbc, 0xa9, 0xde, 0x35, 0x07, 0x92, 0xe7, - 0x00, 0x5d, 0xe1, 0x08, 0xd6, 0xf5, 0x82, 0x1e, 0x33, 0xab, 0x35, 0xa3, 0x5e, 0x6a, 0x54, 0x2d, - 0xd5, 0xf5, 0x56, 0xdc, 0xf5, 0xd6, 0x9b, 0xb8, 0xeb, 0xed, 0x84, 0x36, 0xd6, 0x5b, 0xd3, 0xf7, - 0xf9, 0x7b, 0x9b, 0xb9, 0x5e, 0xc8, 0x7a, 0x22, 0x32, 0x6f, 0xc9, 0x94, 0xa4, 0x50, 0xf2, 0x2f, - 0xcc, 0x4d, 0x24, 0xba, 0x93, 0xa0, 0x67, 0xde, 0xbe, 0xd6, 0xc2, 0x54, 0x97, 0x7c, 0x02, 0x44, - 0xae, 0xc7, 0xbd, 0x1e, 0x8b, 0xa2, 0xfe, 0xd8, 0x97, 0x37, 0xfc, 0xf9, 0xda, 0x1b, 0x2e, 0x39, - 0x45, 0x5e, 0x40, 0x09, 0xd1, 0x63, 0xee, 0xa2, 0x9e, 0x79, 0xe7, 0xda, 0x4b, 0x92, 0xea, 0xf4, - 0x29, 0x6c, 0x2a, 0x5e, 0xe8, 0x78, 0x91, 0x50, 0x3c, 0x77, 0x0f, 0x0a, 0x0a, 0x8a, 0x4c, 0xa3, - 0x96, 0xab, 0x97, 0x1a, 0x05, 0x4b, 0xc9, 0x76, 0x8c, 0x53, 0x0b, 0xd6, 0xd4, 0xf2, 0xa8, 0xfd, - 0x21, 0x7c, 0x42, 0x9f, 0x00, 0x68, 0xa2, 0x42, 0x03, 0x7f, 0x49, 0x1b, 0x28, 0x5a, 0xf1, 0x6d, - 0x33, 0x13, 0x2f, 0x61, 0xbb, 0x75, 0xee, 0x04, 0x03, 0x86, 0x69, 0x19, 0x47, 0x31, 0xc5, 0xa5, - 0xad, 0x25, 0xaa, 0x26, 0x3b, 0x57, 0x35, 0xf4, 0x5e, 0xfc, 0xb2, 0xa3, 0xf6, 0x15, 0x87, 0xe9, - 0xcf, 0x06, 0x6c, 0x34, 0x5d, 0x57, 0xbf, 0x4e, 0xfa, 0x96, 0xec, 0x36, 0x63, 0x59, 0xb7, 0x65, - 0xd3, 0xdd, 0x26, 0x2b, 0x5b, 0xd6, 0x7f, 0xcc, 0x99, 0x5a, 0xc4, 0x73, 0xd3, 0x96, 0xd3, 0xa4, - 0x39, 0x03, 0x48, 0x05, 0x72, 0xcd, 0xee, 0x6b, 0x4d, 0x99, 0xb8, 0x44, 0x1f, 0x3e, 0x77, 0xc2, - 0xc0, 0x0b, 0x06, 0x48, 0xfa, 0x39, 0xe4, 0xd8, 0x58, 0xa6, 0x0f, 0x61, 0xeb, 0x6c, 0xe4, 0x3a, - 0x82, 0x25, 0x9d, 0x26, 0xb0, 0xd2, 0xf6, 0xfa, 0x7d, 0x4d, 0xfa, 0x72, 0x4d, 0x07, 0xb0, 0x73, - 0xc8, 0xf8, 0xa2, 0xee, 0xdd, 0x78, 0x10, 0x48, 0xed, 0x44, 0x72, 0xe3, 0xf9, 0x10, 0x5f, 0x96, - 0x9d, 0x5d, 0x36, 0xe7, 0x51, 0x2e, 0xe5, 0x51, 0x03, 0x4c, 0x9b, 0xf5, 0x43, 0x16, 0x61, 0x76, - 0x79, 0xe4, 0x09, 0x1e, 0x4e, 0xe2, 0x80, 0xef, 0x42, 0xde, 0x66, 0xe7, 0x4e, 0x74, 0x2e, 0x8d, - 0xad, 0xd9, 0x5a, 0xa2, 0x3f, 0x1a, 0xb0, 0xd5, 0xed, 0x39, 0x41, 0xec, 0xd8, 0xe5, 0xb9, 0x45, - 0xbe, 0x1e, 0x0b, 0xae, 0x12, 0xaa, 0xd3, 0x9b, 0x40, 0xc8, 0x33, 0x58, 0x3b, 0xc5, 0xf2, 0xee, - 0x71, 0x5f, 0x86, 0x7c, 0xa3, 0x71, 0xd3, 0x5a, 0xb8, 0xd5, 0x3a, 0x66, 0xe2, 0x9c, 0xbb, 0xf6, - 0x54, 0x95, 0xfe, 0x15, 0xf2, 0x0a, 0x23, 0x05, 0xc8, 0x35, 0x3b, 0x9d, 0x4a, 0x06, 0x17, 0x07, - 0x6f, 0x4e, 0x2b, 0x06, 0x29, 0xc2, 0xaa, 0xdd, 0xfd, 0xe2, 0x75, 0xab, 0x92, 0xa5, 0x3f, 0x19, - 0xb0, 0x99, 0xbc, 0x4d, 0x7f, 0x02, 0xc4, 0xd5, 0x66, 0xcc, 0x73, 0x14, 0x85, 0xf2, 0x81, 0xe7, - 0xb3, 0xe8, 0x28, 0x70, 0xd9, 0x85, 0x2e, 0xc6, 0x9c, 0x3d, 0x87, 0xa1, 0xce, 0xa7, 0x01, 0x7f, - 0x1f, 0xc4, 0x3a, 0x39, 0xa5, 0x93, 0xc4, 0xd0, 0x82, 0xcd, 0x86, 0xfc, 0x1d, 0x73, 0x65, 0xa5, - 0xe4, 0xec, 0x58, 0xc4, 0x68, 0xbc, 0xf9, 0xf2, 0xa4, 0xdf, 0x8f, 0x98, 0x38, 0x8e, 0x64, 0xb9, - 0xe4, 0xec, 0x04, 0x42, 0xbf, 0x37, 0xa0, 0x82, 0xbd, 0x12, 0xa1, 0xcd, 0x6b, 0xbf, 0x08, 0xc8, - 0x1e, 0x14, 0xdb, 0xc8, 0x77, 0xc2, 0x09, 0x85, 0xf4, 0x76, 0x39, 0x69, 0xcc, 0x94, 0xc9, 0x53, - 0x28, 0xa0, 0xb0, 0x1f, 0xa8, 0x17, 0x2c, 0x3f, 0x17, 0xab, 0xd2, 0x6f, 0x60, 0x23, 0xe1, 0x1d, - 0x06, 0xf3, 0x1f, 0xb0, 0xda, 0xc7, 0xf0, 0x68, 0x12, 0xa8, 0x5a, 0xf3, 0xfb, 0x96, 0x8c, 0xdd, - 0x3e, 0x76, 0x90, 0xad, 0x14, 0xab, 0x7b, 0x00, 0x33, 0x10, 0x1b, 0xe7, 0x2b, 0x36, 0xd1, 0xef, - 0xc2, 0x25, 0x8e, 0x9c, 0x77, 0x8e, 0x3f, 0x66, 0x3a, 0xfa, 0x4a, 0x78, 0x9e, 0xdd, 0x33, 0xe8, - 0xb7, 0x06, 0x10, 0x79, 0xfd, 0xf2, 0x8a, 0xfb, 0xa3, 0x83, 0xc2, 0x74, 0xca, 0x7e, 0x53, 0x83, - 0xe2, 0x27, 0x98, 0xf2, 0x3f, 0xd2, 0x0f, 0x9d, 0xca, 0xf2, 0x4b, 0x74, 0x22, 0x58, 0xa4, 0x6b, - 0x4b, 0x09, 0xf4, 0x00, 0xb9, 0x40, 0x68, 0x9e, 0xe7, 0x83, 0x68, 0x49, 0xc3, 0x1d, 0x3b, 0x17, - 0x36, 0x8b, 0xc6, 0xbe, 0xbe, 0x7b, 0xd5, 0x4e, 0x20, 0xb4, 0x0e, 0x24, 0x75, 0x8f, 0x66, 0x1f, - 0xdf, 0x0b, 0x98, 0x4c, 0x63, 0xd1, 0x96, 0x6b, 0x7a, 0x5f, 0xf6, 0x58, 0xe8, 0xf5, 0xd0, 0x5b, - 0xcc, 0x59, 0x80, 0x23, 0x41, 0xa5, 0x6a, 0x2a, 0xd3, 0x47, 0x50, 0x52, 0x5a, 0x11, 0x4e, 0x9f, - 0x94, 0x6a, 0x6e, 0x4e, 0xb5, 0x06, 0x79, 0x35, 0x08, 0x90, 0x53, 0xd4, 0x2a, 0xe6, 0x14, 0x25, - 0x35, 0x7e, 0x29, 0x42, 0xae, 0xd5, 0x39, 0x22, 0xcf, 0x00, 0x0e, 0x99, 0x88, 0x3f, 0xb3, 0x77, - 0x17, 0xd2, 0xb0, 0x8f, 0x3f, 0x01, 0xd5, 0x75, 0x2b, 0xf9, 0x6d, 0x4f, 0x33, 0xe4, 0x3f, 0x50, - 0x38, 0x1b, 0x0d, 0x42, 0xc7, 0x65, 0x57, 0x9e, 0xb9, 0x02, 0xa7, 0x19, 0xf2, 0x1c, 0x79, 0xce, - 0xe7, 0x8e, 0xfb, 0x11, 0x67, 0xff, 0x07, 0xe5, 0xe4, 0xa0, 0x23, 0x3b, 0xd6, 0x25, 0x73, 0x6f, - 0xc9, 0xf9, 0x06, 0xac, 0xc8, 0xe8, 0x5d, 0x65, 0xb9, 0x62, 0xa5, 0x06, 0x3c, 0xcd, 0x90, 0x47, - 0x00, 0x7a, 0x36, 0x06, 0x7d, 0x4e, 0x2a, 0x56, 0x6a, 0x50, 0x56, 0xe3, 0x9a, 0xa3, 0x19, 0xf2, - 0x10, 0x3f, 0xa9, 0xf5, 0x88, 0x24, 0x31, 0x5e, 0xdd, 0xb4, 0xe6, 0xe7, 0x26, 0xcd, 0x90, 0xbf, - 0x43, 0x39, 0x39, 0x6d, 0x66, 0xba, 0xc4, 0x5a, 0x98, 0x42, 0x32, 0x64, 0x65, 0xc5, 0x6c, 0x5a, - 0x7d, 0xd1, 0x89, 0xab, 0x9f, 0xfc, 0x02, 0x36, 0x53, 0xb3, 0xed, 0x92, 0xe3, 0x37, 0xac, 0xcb, - 0xe6, 0x1f, 0xcd, 0x90, 0x57, 0xb0, 0xb5, 0x30, 0xb0, 0xc8, 0x4d, 0xeb, 0xaa, 0x21, 0xb6, 0xc4, - 0x8f, 0xa7, 0x00, 0xb3, 0x09, 0x41, 0xc8, 0xe2, 0xf0, 0xa9, 0x56, 0xac, 0xd4, 0x08, 0xa1, 0x19, - 0xf2, 0x04, 0x8a, 0x53, 0xa6, 0x23, 0x5b, 0x56, 0x9a, 0xb3, 0xab, 0x9b, 0x29, 0x22, 0xa4, 0x19, - 0xf2, 0x6f, 0x28, 0x25, 0x78, 0x82, 0x6c, 0x5b, 0x8b, 0x5c, 0x56, 0xdd, 0xb2, 0xd2, 0x54, 0x42, - 0x33, 0x64, 0x0f, 0x56, 0x4e, 0xbd, 0x60, 0xf0, 0x11, 0x65, 0xf9, 0x5f, 0x58, 0x9f, 0xeb, 0x75, - 0x82, 0xf1, 0x5c, 0xe4, 0x90, 0xea, 0xb6, 0xb5, 0x48, 0x09, 0x34, 0x43, 0x2c, 0x55, 0x36, 0x8a, - 0x03, 0x0a, 0x96, 0x5a, 0x2c, 0x31, 0x67, 0x41, 0xb1, 0xcd, 0xfc, 0x0f, 0xd7, 0x7f, 0x00, 0x25, - 0x2c, 0x68, 0x4d, 0x1f, 0xb3, 0x13, 0x65, 0x2b, 0xc1, 0x28, 0xb2, 0xbb, 0x40, 0x8d, 0x68, 0xfc, - 0x6e, 0xf8, 0x88, 0x30, 0xbc, 0x84, 0x52, 0xdb, 0x8b, 0x7e, 0xc7, 0x05, 0x0d, 0x19, 0x47, 0xd5, - 0xcc, 0x4b, 0xaf, 0x28, 0x58, 0x4a, 0x89, 0x66, 0xc8, 0xdf, 0xa0, 0x24, 0x3f, 0x97, 0x75, 0xba, - 0xd7, 0xad, 0xe4, 0x5f, 0x7e, 0xb5, 0x64, 0xcd, 0xbe, 0xa5, 0x69, 0xe6, 0x6d, 0x5e, 0xde, 0xf3, - 0xcf, 0x5f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x30, 0xe6, 0x21, 0xd7, 0xf9, 0x10, 0x00, 0x00, + // 1563 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x58, 0x51, 0x73, 0x1a, 0x47, + 0x12, 0x66, 0x41, 0x02, 0xd1, 0x20, 0x09, 0x8d, 0x64, 0xdd, 0x1a, 0xfb, 0x6c, 0x3c, 0xe7, 0xb3, + 0x71, 0x5d, 0xdd, 0xfa, 0xcc, 0xd9, 0x77, 0x2a, 0x9f, 0xef, 0x5c, 0x1c, 0x48, 0xb2, 0x12, 0x64, + 0xa9, 0x16, 0x2b, 0xa9, 0xe4, 0x6d, 0xcd, 0x0e, 0x68, 0x2b, 0xcb, 0x0e, 0xd9, 0x1d, 0x6c, 0x51, + 0x95, 0x9f, 0x91, 0xca, 0x53, 0x1e, 0x92, 0xf7, 0xa4, 0x2a, 0x3f, 0x31, 0xd5, 0x33, 0xb3, 0xb0, + 0x2c, 0x12, 0x72, 0x9c, 0xaa, 0xbc, 0x4d, 0x7f, 0xd3, 0xd3, 0xdd, 0xd3, 0xd3, 0xfd, 0xf5, 0x02, + 0x14, 0xc3, 0x51, 0xcf, 0x1a, 0x85, 0x5c, 0xf0, 0xea, 0xad, 0x01, 0xe7, 0x03, 0x9f, 0x3d, 0x96, + 0xd2, 0xdb, 0x71, 0xff, 0x31, 0x1b, 0x8e, 0xc4, 0x44, 0x6f, 0xde, 0x4d, 0x6f, 0x0a, 0x6f, 0xc8, + 0x22, 0xe1, 0x0c, 0x47, 0x4a, 0x81, 0xfe, 0x60, 0x40, 0xf9, 0x33, 0x16, 0x46, 0x1e, 0x0f, 0x6c, + 0x36, 0xf2, 0x27, 0xc4, 0x84, 0x82, 0x96, 0x4d, 0xa3, 0x66, 0xd4, 0x8b, 0x76, 0x2c, 0x92, 0x1d, + 0x58, 0xfd, 0xff, 0xd8, 0xf3, 0x5d, 0x33, 0x2b, 0x71, 0x25, 0x90, 0xdb, 0x50, 0x3c, 0xe4, 0xf1, + 0x89, 0x9c, 0xdc, 0x99, 0x01, 0x64, 0x03, 0xb2, 0x27, 0x5d, 0x73, 0x45, 0xc2, 0xd9, 0x93, 0x2e, + 0x21, 0xb0, 0xd2, 0x0c, 0x7b, 0xe7, 0xe6, 0xaa, 0x44, 0xe4, 0x9a, 0xdc, 0x01, 0x38, 0xe4, 0xc7, + 0xce, 0xc5, 0x69, 0xc8, 0x7b, 0x91, 0x99, 0xaf, 0x19, 0xf5, 0x55, 0x3b, 0x81, 0xd0, 0x3a, 0x94, + 0x8f, 0x1d, 0xd1, 0x3b, 0xb7, 0xd9, 0xd7, 0x63, 0x16, 0x09, 0x8c, 0xf0, 0xd4, 0x11, 0x82, 0x85, + 0xd3, 0x08, 0xb5, 0x48, 0xbf, 0x5b, 0x83, 0xfc, 0xb1, 0x17, 0x86, 0x3c, 0x44, 0xc7, 0x47, 0x6d, + 0xb9, 0xbf, 0x6a, 0x67, 0x8f, 0xda, 0xe8, 0xf8, 0xb5, 0x33, 0x64, 0x3a, 0x76, 0xb9, 0x46, 0x43, + 0xaf, 0x84, 0x18, 0x9d, 0xd9, 0x1d, 0x1d, 0x78, 0x2c, 0x92, 0x2a, 0xac, 0xd9, 0xd1, 0x24, 0xe8, + 0xe1, 0x96, 0x0a, 0x7e, 0x2a, 0x93, 0x5d, 0xc8, 0x1f, 0xa8, 0x43, 0xea, 0x12, 0x5a, 0x22, 0x35, + 0x28, 0x75, 0x47, 0x3c, 0x88, 0x78, 0x28, 0x1d, 0xe5, 0xe5, 0x66, 0x12, 0xc2, 0x8b, 0x6a, 0x11, + 0x4f, 0x17, 0xa4, 0x42, 0x02, 0x21, 0x0f, 0x60, 0x43, 0x4b, 0x1d, 0x3e, 0xe0, 0xa8, 0xb3, 0x26, + 0x75, 0x52, 0x28, 0xa6, 0xbc, 0xe9, 0x0e, 0xbd, 0x40, 0xfa, 0x29, 0xaa, 0x94, 0x4f, 0x01, 0xf4, + 0x22, 0x85, 0xfd, 0xa1, 0xe3, 0xf9, 0x26, 0x28, 0x2f, 0x33, 0x04, 0xf7, 0x5b, 0xe3, 0x48, 0xf0, + 0x61, 0xdb, 0x11, 0x8e, 0x59, 0x52, 0xfb, 0x33, 0x84, 0xdc, 0x87, 0xf5, 0x16, 0x0f, 0x84, 0x17, + 0xb0, 0x40, 0x9c, 0x04, 0xfe, 0xc4, 0x2c, 0xd7, 0x8c, 0xfa, 0x9a, 0x3d, 0x0f, 0xe2, 0x6d, 0x5b, + 0x7c, 0x1c, 0x88, 0x70, 0x22, 0x75, 0xd6, 0xa5, 0x4e, 0x12, 0xc2, 0x3c, 0x35, 0xbb, 0x72, 0x73, + 0x43, 0x6e, 0x6a, 0x09, 0xcb, 0xa8, 0xdb, 0xe3, 0x21, 0x33, 0x37, 0xe5, 0xe3, 0x28, 0x01, 0x33, + 0xde, 0x71, 0x84, 0x27, 0xc6, 0x2e, 0x33, 0x2b, 0x35, 0xa3, 0x9e, 0xb5, 0xa7, 0x32, 0xde, 0xb7, + 0xc3, 0x83, 0x81, 0xda, 0xdc, 0x92, 0x9b, 0x33, 0x60, 0x2e, 0xde, 0x16, 0x77, 0x99, 0x49, 0xe4, + 0x95, 0xe6, 0x41, 0x42, 0xa1, 0xac, 0x83, 0x43, 0x31, 0x32, 0xb7, 0xa5, 0xd2, 0x1c, 0x46, 0x1a, + 0xb0, 0xb3, 0x7f, 0xd1, 0xf3, 0xc7, 0x2e, 0x73, 0xe7, 0x74, 0x77, 0xa4, 0xee, 0xa5, 0x7b, 0x78, + 0x9b, 0x66, 0x14, 0x8c, 0x87, 0xe6, 0x8d, 0x9a, 0x51, 0x5f, 0xb7, 0x95, 0x80, 0x95, 0xd5, 0xe2, + 0xc3, 0x21, 0x0b, 0x84, 0xb9, 0xab, 0x2a, 0x4b, 0x8b, 0xb8, 0xb3, 0x1f, 0x38, 0x6f, 0x7d, 0xe6, + 0x9a, 0x7f, 0x92, 0x69, 0x89, 0x45, 0xac, 0xd8, 0xb3, 0x91, 0x69, 0x4a, 0x30, 0x7b, 0x36, 0xc2, + 0x7b, 0x69, 0x8f, 0x36, 0x73, 0x22, 0x1e, 0x98, 0x37, 0xd5, 0xbd, 0xe6, 0x40, 0xf2, 0x1c, 0xa0, + 0x2b, 0x1c, 0xc1, 0xba, 0x5e, 0xd0, 0x63, 0x66, 0xb5, 0x66, 0xd4, 0x4b, 0x8d, 0xaa, 0xa5, 0xba, + 0xde, 0x8a, 0xbb, 0xde, 0x7a, 0x13, 0x77, 0xbd, 0x9d, 0xd0, 0xc6, 0x7a, 0x6b, 0xfa, 0x3e, 0x7f, + 0x6f, 0x33, 0xd7, 0x0b, 0x59, 0x4f, 0x44, 0xe6, 0x2d, 0xf9, 0x24, 0x29, 0x94, 0xfc, 0x0b, 0xdf, + 0x26, 0x12, 0xdd, 0x49, 0xd0, 0x33, 0x6f, 0x5f, 0xeb, 0x61, 0xaa, 0x4b, 0x3e, 0x01, 0x22, 0xd7, + 0xe3, 0x5e, 0x8f, 0x45, 0x51, 0x7f, 0xec, 0x4b, 0x0b, 0x7f, 0xbe, 0xd6, 0xc2, 0x25, 0xa7, 0xc8, + 0x0b, 0x28, 0x21, 0x7a, 0xcc, 0x5d, 0xd4, 0x33, 0xef, 0x5c, 0x6b, 0x24, 0xa9, 0x4e, 0x9f, 0xc2, + 0xa6, 0xe2, 0x85, 0x8e, 0x17, 0x09, 0xc5, 0x73, 0xf7, 0xa0, 0xa0, 0xa0, 0xc8, 0x34, 0x6a, 0xb9, + 0x7a, 0xa9, 0x51, 0xb0, 0x94, 0x6c, 0xc7, 0x38, 0xb5, 0x60, 0x4d, 0x2d, 0x8f, 0xda, 0x1f, 0xc2, + 0x27, 0xf4, 0x09, 0x80, 0x26, 0x2a, 0x74, 0xf0, 0x97, 0xb4, 0x83, 0xa2, 0x15, 0x5b, 0x9b, 0xb9, + 0x78, 0x09, 0xdb, 0xad, 0x73, 0x27, 0x18, 0x30, 0x7c, 0x96, 0x71, 0x14, 0x53, 0x5c, 0xda, 0x5b, + 0xa2, 0x6a, 0xb2, 0x73, 0x55, 0x43, 0xef, 0xc5, 0x37, 0x3b, 0x6a, 0x5f, 0x71, 0x98, 0xfe, 0x62, + 0xc0, 0x46, 0xd3, 0x75, 0xf5, 0xed, 0x64, 0x6c, 0xc9, 0x6e, 0x33, 0x96, 0x75, 0x5b, 0x36, 0xdd, + 0x6d, 0xb2, 0xb2, 0x65, 0xfd, 0xc7, 0x9c, 0xa9, 0x45, 0x3c, 0x37, 0x6d, 0x39, 0x4d, 0x9a, 0x33, + 0x80, 0x54, 0x20, 0xd7, 0xec, 0xbe, 0xd6, 0x94, 0x89, 0x4b, 0x8c, 0xe1, 0x73, 0x27, 0x0c, 0xbc, + 0x60, 0x80, 0xa4, 0x9f, 0x43, 0x8e, 0x8d, 0x65, 0xfa, 0x10, 0xb6, 0xce, 0x46, 0xae, 0x23, 0x58, + 0x32, 0x68, 0x02, 0x2b, 0x6d, 0xaf, 0xdf, 0xd7, 0xa4, 0x2f, 0xd7, 0x74, 0x00, 0x3b, 0x87, 0x8c, + 0x2f, 0xea, 0xde, 0x8d, 0x07, 0x81, 0xd4, 0x4e, 0x3c, 0x6e, 0x3c, 0x1f, 0x62, 0x63, 0xd9, 0x99, + 0xb1, 0xb9, 0x88, 0x72, 0xa9, 0x88, 0x1a, 0x60, 0xda, 0xac, 0x1f, 0xb2, 0x08, 0x5f, 0x97, 0x47, + 0x9e, 0xe0, 0xe1, 0x24, 0x4e, 0xf8, 0x2e, 0xe4, 0x6d, 0x76, 0xee, 0x44, 0xe7, 0xd2, 0xd9, 0x9a, + 0xad, 0x25, 0xfa, 0xa3, 0x01, 0x5b, 0xdd, 0x9e, 0x13, 0xc4, 0x81, 0x5d, 0xfe, 0xb6, 0xc8, 0xd7, + 0x63, 0xc1, 0xd5, 0x83, 0xea, 0xe7, 0x4d, 0x20, 0xe4, 0x19, 0xac, 0x9d, 0x62, 0x79, 0xf7, 0xb8, + 0x2f, 0x53, 0xbe, 0xd1, 0xb8, 0x69, 0x2d, 0x58, 0xb5, 0x8e, 0x99, 0x38, 0xe7, 0xae, 0x3d, 0x55, + 0xa5, 0x7f, 0x85, 0xbc, 0xc2, 0x48, 0x01, 0x72, 0xcd, 0x4e, 0xa7, 0x92, 0xc1, 0xc5, 0xc1, 0x9b, + 0xd3, 0x8a, 0x41, 0x8a, 0xb0, 0x6a, 0x77, 0xbf, 0x78, 0xdd, 0xaa, 0x64, 0xe9, 0xcf, 0x06, 0x6c, + 0x26, 0xad, 0xe9, 0x4f, 0x80, 0xb8, 0xda, 0x8c, 0x79, 0x8e, 0xa2, 0x50, 0x3e, 0xf0, 0x7c, 0x16, + 0x1d, 0x05, 0x2e, 0xbb, 0xd0, 0xc5, 0x98, 0xb3, 0xe7, 0x30, 0xd4, 0xf9, 0x34, 0xe0, 0xef, 0x83, + 0x58, 0x27, 0xa7, 0x74, 0x92, 0x18, 0x7a, 0xb0, 0xd9, 0x90, 0xbf, 0x63, 0xae, 0xac, 0x94, 0x9c, + 0x1d, 0x8b, 0x98, 0x8d, 0x37, 0x5f, 0x9e, 0xf4, 0xfb, 0x11, 0x13, 0xc7, 0x91, 0x2c, 0x97, 0x9c, + 0x9d, 0x40, 0xe8, 0xf7, 0x06, 0x54, 0xb0, 0x57, 0x22, 0xf4, 0x79, 0xed, 0x17, 0x01, 0xd9, 0x83, + 0x62, 0x1b, 0xf9, 0x4e, 0x38, 0xa1, 0x90, 0xd1, 0x2e, 0x27, 0x8d, 0x99, 0x32, 0x79, 0x0a, 0x05, + 0x14, 0xf6, 0x03, 0x75, 0x83, 0xe5, 0xe7, 0x62, 0x55, 0xfa, 0x0d, 0x6c, 0x24, 0xa2, 0xc3, 0x64, + 0xfe, 0x03, 0x56, 0xfb, 0x98, 0x1e, 0x4d, 0x02, 0x55, 0x6b, 0x7e, 0xdf, 0x92, 0xb9, 0xdb, 0xc7, + 0x0e, 0xb2, 0x95, 0x62, 0x75, 0x0f, 0x60, 0x06, 0x62, 0xe3, 0x7c, 0xc5, 0x26, 0xfa, 0x5e, 0xb8, + 0xc4, 0x91, 0xf3, 0xce, 0xf1, 0xc7, 0x4c, 0x67, 0x5f, 0x09, 0xcf, 0xb3, 0x7b, 0x06, 0xfd, 0xd6, + 0x00, 0x22, 0xcd, 0x2f, 0xaf, 0xb8, 0x3f, 0x3a, 0x29, 0x4c, 0x3f, 0xd9, 0x6f, 0x6a, 0x50, 0xfc, + 0x04, 0x53, 0xf1, 0x47, 0xfa, 0xa2, 0x53, 0x59, 0x7e, 0x89, 0x4e, 0x04, 0x8b, 0x74, 0x6d, 0x29, + 0x81, 0x1e, 0x20, 0x17, 0x08, 0xcd, 0xf3, 0x7c, 0x10, 0x2d, 0x69, 0xb8, 0x63, 0xe7, 0xc2, 0x66, + 0xd1, 0xd8, 0xd7, 0xb6, 0x57, 0xed, 0x04, 0x42, 0xeb, 0x40, 0x52, 0x76, 0x34, 0xfb, 0xf8, 0x5e, + 0xc0, 0xe4, 0x33, 0x16, 0x6d, 0xb9, 0xa6, 0xf7, 0x65, 0x8f, 0x85, 0x5e, 0x0f, 0xa3, 0xc5, 0x37, + 0x0b, 0x70, 0x24, 0xa8, 0xa7, 0x9a, 0xca, 0xf4, 0x11, 0x94, 0x94, 0x56, 0x84, 0xd3, 0x27, 0xa5, + 0x9a, 0x9b, 0x53, 0xad, 0x41, 0x5e, 0x0d, 0x02, 0xe4, 0x14, 0xb5, 0x8a, 0x39, 0x45, 0x49, 0x8d, + 0x9f, 0x00, 0x72, 0xad, 0xce, 0x11, 0x79, 0x06, 0x70, 0xc8, 0x44, 0xfc, 0x99, 0xbd, 0xbb, 0xf0, + 0x0c, 0xfb, 0xf8, 0x23, 0xa0, 0xba, 0x6e, 0x25, 0xbf, 0xed, 0x69, 0x86, 0xfc, 0x07, 0x0a, 0x67, + 0xa3, 0x41, 0xe8, 0xb8, 0xec, 0xca, 0x33, 0x57, 0xe0, 0x34, 0x43, 0x9e, 0x23, 0xcf, 0xf9, 0xdc, + 0x71, 0x3f, 0xe2, 0xec, 0xff, 0xa0, 0x9c, 0x1c, 0x74, 0x64, 0xc7, 0xba, 0x64, 0xee, 0x2d, 0x39, + 0xdf, 0x80, 0x15, 0x99, 0xbd, 0xab, 0x3c, 0x57, 0xac, 0xd4, 0x80, 0xa7, 0x19, 0xf2, 0x08, 0x40, + 0xcf, 0xc6, 0xa0, 0xcf, 0x49, 0xc5, 0x4a, 0x0d, 0xca, 0x6a, 0x5c, 0x73, 0x34, 0x43, 0x1e, 0xe2, + 0x27, 0xb5, 0x1e, 0x91, 0x24, 0xc6, 0xab, 0x9b, 0xd6, 0xfc, 0xdc, 0xa4, 0x19, 0xf2, 0x77, 0x28, + 0x27, 0xa7, 0xcd, 0x4c, 0x97, 0x58, 0x0b, 0x53, 0x48, 0xa6, 0xac, 0xac, 0x98, 0x4d, 0xab, 0x2f, + 0x06, 0x71, 0xf5, 0x95, 0x5f, 0xc0, 0x66, 0x6a, 0xb6, 0x5d, 0x72, 0xfc, 0x86, 0x75, 0xd9, 0xfc, + 0xa3, 0x19, 0xf2, 0x0a, 0xb6, 0x16, 0x06, 0x16, 0xb9, 0x69, 0x5d, 0x35, 0xc4, 0x96, 0xc4, 0xf1, + 0x14, 0x60, 0x36, 0x21, 0x08, 0x59, 0x1c, 0x3e, 0xd5, 0x8a, 0x95, 0x1a, 0x21, 0x34, 0x43, 0x9e, + 0x40, 0x71, 0xca, 0x74, 0x64, 0xcb, 0x4a, 0x73, 0x76, 0x75, 0x33, 0x45, 0x84, 0x34, 0x43, 0xfe, + 0x0d, 0xa5, 0x04, 0x4f, 0x90, 0x6d, 0x6b, 0x91, 0xcb, 0xaa, 0x5b, 0x56, 0x9a, 0x4a, 0x68, 0x86, + 0xec, 0xc1, 0xca, 0xa9, 0x17, 0x0c, 0x3e, 0xa2, 0x2c, 0xff, 0x0b, 0xeb, 0x73, 0xbd, 0x4e, 0x30, + 0x9f, 0x8b, 0x1c, 0x52, 0xdd, 0xb6, 0x16, 0x29, 0x81, 0x66, 0x88, 0xa5, 0xca, 0x46, 0x71, 0x40, + 0xc1, 0x52, 0x8b, 0x25, 0xee, 0x2c, 0x28, 0xb6, 0x99, 0xff, 0xe1, 0xfa, 0x0f, 0xa0, 0x84, 0x05, + 0xad, 0xe9, 0x63, 0x76, 0xa2, 0x6c, 0x25, 0x18, 0x85, 0x66, 0xc8, 0x33, 0xa8, 0x1c, 0x32, 0xa1, + 0x7a, 0x29, 0x56, 0xbe, 0x2a, 0x19, 0x05, 0x4b, 0xe9, 0xc9, 0xa6, 0x04, 0x35, 0xd9, 0xf1, 0x73, + 0xe3, 0x23, 0xb2, 0xf7, 0x12, 0x4a, 0x6d, 0x2f, 0xfa, 0x1d, 0x06, 0x1a, 0x32, 0xfd, 0x2a, 0x9e, + 0xa5, 0x26, 0x12, 0x41, 0xff, 0x0d, 0x4a, 0xf2, 0x2b, 0x5b, 0x57, 0xc9, 0xba, 0x95, 0xfc, 0x73, + 0xa0, 0x5a, 0xb2, 0x66, 0x9f, 0xe0, 0x34, 0xf3, 0x36, 0x2f, 0xed, 0xfc, 0xf3, 0xd7, 0x00, 0x00, + 0x00, 0xff, 0xff, 0x50, 0xee, 0xcf, 0xe7, 0x30, 0x11, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -1563,6 +1564,7 @@ type CLIClient interface { AddMetric(ctx context.Context, in *Metric, opts ...grpc.CallOption) (*empty.Empty, error) DelMetric(ctx context.Context, in *Metric, opts ...grpc.CallOption) (*empty.Empty, error) ListMetrics(ctx context.Context, in *Metric, opts ...grpc.CallOption) (*MetricsList, error) + GetStatusMetrics(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*Status, error) EnableAuto(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) DisableAuto(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) GetStatusAuto(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*Status, error) @@ -1749,6 +1751,15 @@ func (c *cLIClient) ListMetrics(ctx context.Context, in *Metric, opts ...grpc.Ca return out, nil } +func (c *cLIClient) GetStatusMetrics(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*Status, error) { + out := new(Status) + err := c.cc.Invoke(ctx, "/CLI/GetStatusMetrics", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *cLIClient) EnableAuto(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) { out := new(empty.Empty) err := c.cc.Invoke(ctx, "/CLI/EnableAuto", in, out, opts...) @@ -1806,6 +1817,7 @@ type CLIServer interface { AddMetric(context.Context, *Metric) (*empty.Empty, error) DelMetric(context.Context, *Metric) (*empty.Empty, error) ListMetrics(context.Context, *Metric) (*MetricsList, error) + GetStatusMetrics(context.Context, *empty.Empty) (*Status, error) EnableAuto(context.Context, *empty.Empty) (*empty.Empty, error) DisableAuto(context.Context, *empty.Empty) (*empty.Empty, error) GetStatusAuto(context.Context, *empty.Empty) (*Status, error) @@ -1874,6 +1886,9 @@ func (*UnimplementedCLIServer) DelMetric(ctx context.Context, req *Metric) (*emp func (*UnimplementedCLIServer) ListMetrics(ctx context.Context, req *Metric) (*MetricsList, error) { return nil, status.Errorf(codes.Unimplemented, "method ListMetrics not implemented") } +func (*UnimplementedCLIServer) GetStatusMetrics(ctx context.Context, req *empty.Empty) (*Status, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetStatusMetrics not implemented") +} func (*UnimplementedCLIServer) EnableAuto(ctx context.Context, req *empty.Empty) (*empty.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method EnableAuto not implemented") } @@ -2233,6 +2248,24 @@ func _CLI_ListMetrics_Handler(srv interface{}, ctx context.Context, dec func(int return interceptor(ctx, in, info, handler) } +func _CLI_GetStatusMetrics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(empty.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CLIServer).GetStatusMetrics(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/CLI/GetStatusMetrics", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CLIServer).GetStatusMetrics(ctx, req.(*empty.Empty)) + } + return interceptor(ctx, in, info, handler) +} + func _CLI_EnableAuto_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(empty.Empty) if err := dec(in); err != nil { @@ -2385,6 +2418,10 @@ var _CLI_serviceDesc = grpc.ServiceDesc{ MethodName: "ListMetrics", Handler: _CLI_ListMetrics_Handler, }, + { + MethodName: "GetStatusMetrics", + Handler: _CLI_GetStatusMetrics_Handler, + }, { MethodName: "EnableAuto", Handler: _CLI_EnableAuto_Handler, diff --git a/rpc/rpc.proto b/rpc/rpc.proto index c4c83cc9..169f4c6c 100644 --- a/rpc/rpc.proto +++ b/rpc/rpc.proto @@ -23,6 +23,7 @@ service CLI { rpc AddMetric (Metric) returns (google.protobuf.Empty) {} rpc DelMetric (Metric) returns (google.protobuf.Empty) {} rpc ListMetrics (Metric) returns (MetricsList) {} + rpc GetStatusMetrics (google.protobuf.Empty) returns (Status) {} rpc EnableAuto (google.protobuf.Empty) returns (google.protobuf.Empty) {} rpc DisableAuto (google.protobuf.Empty) returns (google.protobuf.Empty) {} rpc GetStatusAuto (google.protobuf.Empty) returns (Status) {} From 2d201848279ea5ba4882c5984b72f4b9cfc365c1 Mon Sep 17 00:00:00 2001 From: Duncan McNamara Date: Tue, 28 May 2024 18:26:59 +0200 Subject: [PATCH 8/8] Metrics: keep route when disabled This is to return a 503 Service Unavailable error instead of a 404 Not Found, a more helpfull error. --- http/http.go | 9 +-- http/metrics.go | 16 +++-- mirrorbits.conf | 154 ++---------------------------------------------- 3 files changed, 22 insertions(+), 157 deletions(-) diff --git a/http/http.go b/http/http.go index f58ef512..89d1ad3b 100644 --- a/http/http.go +++ b/http/http.go @@ -100,10 +100,11 @@ func HTTPServer(redis *database.Redis, cache *mirrors.Cache) *HTTP { if config.GetConfig().MetricsEnabled { log.Info("Metrics enabled") h.metrics = NewMetrics(redis) - http.Handle("/metrics", NewGzipHandler(h.metricsHandler)) } else { + h.metrics = nil log.Info("Metrics disabled") } + http.Handle("/metrics", NewGzipHandler(h.metricsHandler)) // Load the GeoIP databases if err := h.geoip.LoadGeoIP(); err != nil { @@ -170,9 +171,9 @@ func (h *HTTP) Reload() { if config.GetConfig().MetricsEnabled && h.metrics == nil { log.Info("Configuration Reload: Metrics enabled") h.metrics = NewMetrics(h.redis) - http.Handle("/metrics", NewGzipHandler(h.metricsHandler)) - } else { - log.Info("Configuration Reload: Metrics not enabled") + } else if h.metrics != nil { + h.metrics = nil + log.Info("Configuration Reload: Metrics disabled") } // Reload the templates diff --git a/http/metrics.go b/http/metrics.go index 28655566..2fabd3ad 100644 --- a/http/metrics.go +++ b/http/metrics.go @@ -38,6 +38,7 @@ func NewMetrics(r *database.Redis) *Metrics { metrics.getMetrics(r) trackedFiles, err := r.GetListOfTrackedFiles() if err != nil { + log.Error("WTF") log.Error(err.Error) } else { metrics.trackedFileList = trackedFiles @@ -375,10 +376,17 @@ func (m *Metrics) getMetrics(httpRedis *database.Redis) { } func (h *HTTP) metricsHandler(w http.ResponseWriter, r *http.Request) { - h.metrics.lock.Lock() - output := h.metrics.metricsResponse - h.metrics.lock.Unlock() - w.Write([]byte(output)) + if config.GetConfig().MetricsEnabled { + h.metrics.lock.Lock() + output := h.metrics.metricsResponse + h.metrics.lock.Unlock() + w.Write([]byte(output)) + log.Debug("test") + } else { + log.Errorf("Error: metrics are disabled") + http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable) + } + } func getDurationFromKey(key string) string { diff --git a/mirrorbits.conf b/mirrorbits.conf index 8c96dc9c..0f398340 100644 --- a/mirrorbits.conf +++ b/mirrorbits.conf @@ -1,151 +1,7 @@ # vim: set ft=yaml: -################### -##### GENERAL ##### -################### - -## Path to the local repository -# Repository: /srv/repo - -## Path to the templates (default autodetect) -# Templates: /usr/share/mirrorbits/ - -## A local path or URL containing the JavaScript used by the templates. -## If this is not set (the default), the JavaScript will just be loaded -## from the usual CDNs. See also `contrib/localjs/fetchfiles.sh`. -# LocalJSPath: - -## Path where to store logs (comment to disable) -# LogDir: /var/log/mirrorbits - -## Path to the GeoIP2 mmdb databases -# GeoipDatabasePath: /usr/share/GeoIP/ - -## OutputMode can take on the three values: -## - redirect: HTTP redirect to the destination file on the selected mirror -## - json: return a json document for pre-treatment by an application -## - auto: based on the Accept HTTP header -# OutputMode: auto - -## Enable Gzip compression -# Gzip: false - -## Interval in seconds between which 2 range downloads of a given file -## from a same origin (hashed (IP, user-agent) couple) are considered -## to be the same download. In particular, download statistics are not -## incremented for this file. -# SameDownloadInterval: 600 - -## Host and port to listen on -# ListenAddress: :8080 - -## Host and port to listen for the CLI RPC -# RPCListenAddress: localhost:3390 - -## Password for restricting access to the CLI (optional) -# RPCPassword: - -#################### -##### DATABASE ##### -#################### - -## Redis host and port -# RedisAddress: 10.0.0.1:6379 - -## Redis password (if any) -# RedisPassword: supersecure - -## Redis database ID (if any) -# RedisDB: 0 - -## Redis sentinel name (only if using sentinel) -# RedisSentinelMasterName: mirrorbits - -## List of Redis sentinel hosts (only if using sentinel) -# RedisSentinels: -# - Host: 10.0.0.1:26379 -# - Host: 10.0.0.2:26379 -# - Host: 10.0.0.3:26379 - -################### -##### MIRRORS ##### -################### - -## Relative path to the trace file within the repository (optional). -## The file must contain the number of seconds since epoch and should -## be updated every minute (or so) with a cron on the master repository. -# TraceFileLocation: /trace - -## Interval between two scans of the local repository. -## The repository scan will index new and removed files and collect file -## sizes and checksums. -## This should, more or less, match the frequency where the local repo -## is updated. -# RepositoryScanInterval: 5 - -## Enable or disable specific hashing algorithms -# Hashes: -# SHA256: On -# SHA1: Off -# MD5: Off - -################### -##### MIRRORS ##### -################### - -## Maximum number of concurrent mirror synchronization to do (rsync/ftp) -# ConcurrentSync: 5 - -## Interval in minutes between mirror scan -# ScanInterval: 30 - -## Interval in minutes between mirrors HTTP health checks -# CheckInterval: 1 - -## Allow a mirror to issue an HTTP redirect. -## Setting this to true will disable the mirror if a redirect is detected. -# DisallowRedirects: false - -## Disable a mirror if an active file is missing (HTTP 404) -# DisableOnMissingFile: false - -## Adjust the weight/range of the geographic distribution -# WeightDistributionRange: 1.5 - -## Maximum number of alternative links to return in the HTTP header -# MaxLinkHeaders: 10 - -## Automatically fix timezone offsets. -## Enable this if one or more mirrors are always excluded because their -## last-modification-time mismatch. This option will try to guess the -## offset and adjust the mod time accordingly. -## Affected mirrors will need to be rescanned after enabling this feature. -# FixTimezoneOffsets: false - -## List of mirrors to use as fallback which will be used in case mirrorbits -## is unable to answer a request because the database is unreachable. -## Note: Mirrorbits will redirect to one of these mirrors based on the user -## location but won't be able to know if the mirror has the requested file. -## Therefore only put your most reliable and up-to-date mirrors here. -# Fallbacks: -# - URL: http://fallback1.mirror/repo/ -# CountryCode: fr -# ContinentCode: eu -# - URL: http://fallback2.mirror/repo/ -# CountryCode: us -# ContinentCode: na - -################### -##### METRICS ##### -################### - -## Enable the metrics route, default to false -## MetricsEnable: false - -## Number of days to keep the daily top 10 metrics in database -## Keep in mind that this will have an impact on the database's RAM usage -## It is not recommended to keep more than a few days of retention. -## MetricsTopFilesRetention: 0 - -## Enable / Disable automatically adding a new file to tracked files -## MetricsAutoTrackedFiles: false +Repository: /srv/repo +ListenAddress: :80 +RedisAddress: 172.18.0.1:6379 +LogDir: /var/log/mirrorbits +MetricsEnabled: true