skip exporting points with no value due to prometheus staleness markers

Author: kwiesmueller

metrics_proto.go

@@ -257,6 +257,10 @@ func (se *statsExporter) protoMetricToTimeSeries(ctx context.Context, mappedRsc
 			mb.recordDroppedTimeseries(1, err)
 			continue
 		}
+		if len(sdPoints) == 0 {
+			// Sending TimeSeries with no points is not allowed, so skip this one
+			continue
+		}
 
 		// Each TimeSeries has labelValues which MUST be correlated
 		// with that from the MetricDescriptor
@@ -355,6 +359,11 @@ func (se *statsExporter) protoTimeSeriesToMonitoringPoints(ts *metricspb.TimeSer
 		if err != nil {
 			return nil, err
 		}
+		if spt == nil {
+			// Skip any points that don't result in a correct value
+			// For example if they are Prometheus staleness markers
+			continue
+		}
 		sptl = append(sptl, spt)
 	}
 	return sptl, nil
@@ -444,6 +453,11 @@ func fromProtoPoint(startTime *timestamppb.Timestamp, pt *metricspb.Point) (*mon
 	if err != nil {
 		return nil, err
 	}
+	// We don't want to send points with no value as those are invalid
+	// Pass the nil upwards instead to skip this point
+	if mptv == nil {
+		return nil, nil
+	}
 
 	endTime := pt.Timestamp
 	interval := &monitoringpb.TimeInterval{

metrics_proto_test.go

@@ -17,11 +17,13 @@ package stackdriver
 import (
 	"context"
 	"fmt"
+	"math"
 	"strings"
 	"testing"
 
 	resourcepb "github.com/census-instrumentation/opencensus-proto/gen-go/resource/v1"
 	"github.com/golang/protobuf/ptypes/timestamp"
+	promvalue "github.com/prometheus/prometheus/model/value"
 	"google.golang.org/api/option"
 	distributionpb "google.golang.org/genproto/googleapis/api/distribution"
 	labelpb "google.golang.org/genproto/googleapis/api/label"
@@ -367,6 +369,35 @@ func TestProtoMetricToCreateTimeSeriesRequest(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "Test staleness marker is skipped",
+			in: &metricspb.Metric{
+				MetricDescriptor: &metricspb.MetricDescriptor{
+					Name:        "with_metric_descriptor_2",
+					Description: "This is a test",
+					Unit:        "By",
+					LabelKeys:   []*metricspb.LabelKey{{Key: "key1"}, {Key: "key2"}, {Key: "key3"}},
+				},
+				Timeseries: []*metricspb.TimeSeries{
+					{
+						StartTimestamp: startTimestamp,
+						LabelValues:    []*metricspb.LabelValue{{}, {}, {HasValue: true, Value: "val3"}},
+						Points: []*metricspb.Point{
+							{
+								Timestamp: endTimestamp,
+								Value: &metricspb.Point_DoubleValue{
+									DoubleValue: math.Float64frombits(promvalue.StaleNaN),
+								},
+							},
+						},
+					},
+				},
+			},
+			statsExporter: &statsExporter{
+				o: Options{ProjectID: "foo", MapResource: DefaultMapResource},
+			},
+			want: nil,
+		},
 	}
 
 	seenResources := make(map[*resourcepb.Resource]*monitoredrespb.MonitoredResource)
@@ -762,6 +793,20 @@ func TestProtoMetricsToMonitoringMetrics_fromProtoPoint(t *testing.T) {
 				},
 			},
 		},
+		{
+			in: &metricspb.Point{
+				Timestamp: endTimestamp,
+				Value:     &metricspb.Point_Int64Value{Int64Value: int64(math.Float64frombits(promvalue.StaleNaN))},
+			},
+			want: nil,
+		},
+		{
+			in: &metricspb.Point{
+				Timestamp: endTimestamp,
+				Value:     &metricspb.Point_DoubleValue{DoubleValue: math.Float64frombits(promvalue.StaleNaN)},
+			},
+			want: nil,
+		},
 	}
 
 	for i, tt := range tests {