diff --git a/cmd/freno/main.go b/cmd/freno/main.go index 3bf0a7f9..5c7a8e9c 100644 --- a/cmd/freno/main.go +++ b/cmd/freno/main.go @@ -11,6 +11,8 @@ import ( "github.com/github/freno/pkg/http" "github.com/github/freno/pkg/throttle" "github.com/outbrain/golib/log" + "os" + "strconv" ) // AppVersion has to be filled by ldflags: @@ -120,6 +122,16 @@ func httpServe() error { throttler := throttle.NewThrottler() log.Infof("Starting consensus service") log.Infof("- forced leadership: %+v", group.ForceLeadership) + + bypassEnabled, err := strconv.ParseBool(os.Getenv("FRENO_BYPASS_ENABLED")) + if err != nil { + log.Infof("error parsing FRENO_BYPASS_ENABLED: %s", os.Getenv("FRENO_BYPASS_ENABLED")) + bypassEnabled = false + } + if bypassEnabled { + log.Info("- bypass on no hosts is enabled!") + } + throttler.BypassOnNoHostsFound = bypassEnabled consensusServiceProvider, err := group.NewConsensusServiceProvider(throttler) if err != nil { return err diff --git a/pkg/throttle/mysql.go b/pkg/throttle/mysql.go index 2489b34a..532440ba 100644 --- a/pkg/throttle/mysql.go +++ b/pkg/throttle/mysql.go @@ -16,6 +16,7 @@ func aggregateMySQLProbes( ignoreHostsCount int, ignoreDialTcpErrors bool, ignoreHostsThreshold float64, + bypassOnNohosts bool, ) (worstMetric base.MetricResult) { // probes is known not to change. It can be *replaced*, but not changed. // so it's safe to iterate it @@ -46,6 +47,9 @@ func aggregateMySQLProbes( probeValues = append(probeValues, value) } if len(probeValues) == 0 { + if bypassOnNohosts { + return base.NewSimpleMetricResult(0.0) + } return base.NoHostsMetricResult } diff --git a/pkg/throttle/mysql_test.go b/pkg/throttle/mysql_test.go index 2b13f1a3..24c6f5ab 100644 --- a/pkg/throttle/mysql_test.go +++ b/pkg/throttle/mysql_test.go @@ -54,41 +54,48 @@ func TestAggregateMySQLProbesNoErrors(t *testing.T) { probes[clusterKey.Key] = &mysql.Probe{Key: clusterKey.Key} } { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 0, false, 0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 0, false, 0, false) value, err := worstMetric.Get() test.S(t).ExpectNil(err) test.S(t).ExpectEquals(value, 1.7) } { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 1, false, 0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 1, false, 0, false) value, err := worstMetric.Get() test.S(t).ExpectNil(err) test.S(t).ExpectEquals(value, 1.2) } { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 2, false, 0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 2, false, 0, false) value, err := worstMetric.Get() test.S(t).ExpectNil(err) test.S(t).ExpectEquals(value, 1.1) } { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 3, false, 0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 3, false, 0, false) value, err := worstMetric.Get() test.S(t).ExpectNil(err) test.S(t).ExpectEquals(value, 0.6) } { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 4, false, 0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 4, false, 0, false) value, err := worstMetric.Get() test.S(t).ExpectNil(err) test.S(t).ExpectEquals(value, 0.3) } { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 5, false, 0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 5, false, 0, false) value, err := worstMetric.Get() test.S(t).ExpectNil(err) test.S(t).ExpectEquals(value, 0.3) } + { + var emptyProbes mysql.Probes = map[mysql.InstanceKey](*mysql.Probe){} + worstMetric := aggregateMySQLProbes(&emptyProbes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 5, false, 0, true) + value, err := worstMetric.Get() + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(value, 0.0) + } } func TestAggregateMySQLProbesNoErrorsIgnoreHostsThreshold(t *testing.T) { @@ -117,37 +124,37 @@ func TestAggregateMySQLProbesNoErrorsIgnoreHostsThreshold(t *testing.T) { probes[clusterKey.Key] = &mysql.Probe{Key: clusterKey.Key} } { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 0, false, 1.0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 0, false, 1.0, false) value, err := worstMetric.Get() test.S(t).ExpectNil(err) test.S(t).ExpectEquals(value, 1.7) } { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 1, false, 1.0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 1, false, 1.0, false) value, err := worstMetric.Get() test.S(t).ExpectNil(err) test.S(t).ExpectEquals(value, 1.2) } { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 2, false, 1.0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 2, false, 1.0, false) value, err := worstMetric.Get() test.S(t).ExpectNil(err) test.S(t).ExpectEquals(value, 1.1) } { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 3, false, 1.0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 3, false, 1.0, false) value, err := worstMetric.Get() test.S(t).ExpectNil(err) test.S(t).ExpectEquals(value, 0.6) } { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 4, false, 1.0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 4, false, 1.0, false) value, err := worstMetric.Get() test.S(t).ExpectNil(err) test.S(t).ExpectEquals(value, 0.6) } { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 5, false, 1.0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 5, false, 1.0, false) value, err := worstMetric.Get() test.S(t).ExpectNil(err) test.S(t).ExpectEquals(value, 0.6) @@ -180,19 +187,19 @@ func TestAggregateMySQLProbesWithErrors(t *testing.T) { probes[clusterKey.Key] = &mysql.Probe{Key: clusterKey.Key} } { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 0, false, 0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 0, false, 0, false) _, err := worstMetric.Get() test.S(t).ExpectNotNil(err) test.S(t).ExpectEquals(err, base.NoSuchMetricError) } { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 1, false, 0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 1, false, 0, false) value, err := worstMetric.Get() test.S(t).ExpectNil(err) test.S(t).ExpectEquals(value, 1.7) } { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 2, false, 0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 2, false, 0, false) value, err := worstMetric.Get() test.S(t).ExpectNil(err) test.S(t).ExpectEquals(value, 1.2) @@ -200,19 +207,19 @@ func TestAggregateMySQLProbesWithErrors(t *testing.T) { instanceResultsMap[key1cluster] = base.NoSuchMetric { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 0, false, 0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 0, false, 0, false) _, err := worstMetric.Get() test.S(t).ExpectNotNil(err) test.S(t).ExpectEquals(err, base.NoSuchMetricError) } { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 1, false, 0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 1, false, 0, false) _, err := worstMetric.Get() test.S(t).ExpectNotNil(err) test.S(t).ExpectEquals(err, base.NoSuchMetricError) } { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 2, false, 0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 2, false, 0, false) value, err := worstMetric.Get() test.S(t).ExpectNil(err) test.S(t).ExpectEquals(value, 1.7) @@ -245,13 +252,13 @@ func TestAggregateMySQLProbesWithHttpChecks(t *testing.T) { probes[clusterKey.Key] = &mysql.Probe{Key: clusterKey.Key} } { - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 0, false, 0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 0, false, 0, false) _, err := worstMetric.Get() test.S(t).ExpectNil(err) } { clusterInstanceHttpCheckResultMap[mysql.MySQLHttpCheckHashKey(clusterName, &key2)] = http.StatusNotFound - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 0, false, 0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 0, false, 0, false) value, err := worstMetric.Get() test.S(t).ExpectNil(err) test.S(t).ExpectEquals(value, 1.2) @@ -260,7 +267,7 @@ func TestAggregateMySQLProbesWithHttpChecks(t *testing.T) { for hashKey := range clusterInstanceHttpCheckResultMap { clusterInstanceHttpCheckResultMap[hashKey] = http.StatusNotFound } - worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 0, false, 0) + worstMetric := aggregateMySQLProbes(&probes, clusterName, instanceResultsMap, clusterInstanceHttpCheckResultMap, 0, false, 0, false) _, err := worstMetric.Get() test.S(t).ExpectNotNil(err) } diff --git a/pkg/throttle/throttler.go b/pkg/throttle/throttler.go index e0d103c9..58cd006a 100644 --- a/pkg/throttle/throttler.go +++ b/pkg/throttle/throttler.go @@ -78,6 +78,8 @@ type Throttler struct { nonLowPriorityAppRequestsThrottled *cache.Cache httpClient *http.Client + + BypassOnNoHostsFound bool } func NewThrottler() *Throttler { @@ -331,6 +333,10 @@ func (throttler *Throttler) refreshMySQLInventory() error { } } if len(totalHosts) == 0 { + if throttler.BypassOnNoHostsFound { + log.Infof("No hosts for pool: %+v, but bypass is enabled", poolName) + return nil + } return log.Errorf("Unable to get any HAproxy hosts for pool: %+v", poolName) } @@ -441,7 +447,7 @@ func (throttler *Throttler) aggregateMySQLMetrics() error { metricName := fmt.Sprintf("mysql/%s", clusterName) ignoreHostsCount := throttler.mysqlInventory.IgnoreHostsCount[clusterName] ignoreHostsThreshold := throttler.mysqlInventory.IgnoreHostsThreshold[clusterName] - aggregatedMetric := aggregateMySQLProbes(probes, clusterName, throttler.mysqlInventory.InstanceKeyMetrics, throttler.mysqlInventory.ClusterInstanceHttpChecks, ignoreHostsCount, config.Settings().Stores.MySQL.IgnoreDialTcpErrors, ignoreHostsThreshold) + aggregatedMetric := aggregateMySQLProbes(probes, clusterName, throttler.mysqlInventory.InstanceKeyMetrics, throttler.mysqlInventory.ClusterInstanceHttpChecks, ignoreHostsCount, config.Settings().Stores.MySQL.IgnoreDialTcpErrors, ignoreHostsThreshold, throttler.BypassOnNoHostsFound) go throttler.aggregatedMetrics.Set(metricName, aggregatedMetric, cache.DefaultExpiration) if throttler.memcacheClient != nil { go func() {