First pass at adding queue length

Author: ScottGuymer

azure-devops-client/agentpool.go

@@ -63,25 +63,27 @@ type AgentPoolAgent struct {
 	Status            string
 	Version           string
 	CreatedOn         time.Time
+	AssignedRequest   JobRequest
 
-	AssignedRequest struct {
-		RequestId    int64
-		Demands      []string
-		QueueTime    time.Time
-		AssignTime   time.Time
-		ReceiveTime  time.Time
-		LockedUntil  time.Time
-		ServiceOwner string
-		HostId       string
-		ScopeId      string
-		PlanType     string
-		PlanId       string
-		JobId        string
-		Definition   struct {
-			Id    int64
-			Name  string
-			Links Links `json:"_links"`
-		}
+}
+
+type JobRequest struct {
+	RequestId    int64
+	Demands      []string
+	QueueTime    time.Time
+	AssignTime   *time.Time
+	ReceiveTime  time.Time
+	LockedUntil  time.Time
+	ServiceOwner string
+	HostId       string
+	ScopeId      string
+	PlanType     string
+	PlanId       string
+	JobId        string
+	Definition   struct {
+		Id    int64
+		Name  string
+		Links Links `json:"_links"`
 	}
 }
 
@@ -107,3 +109,32 @@ func (c *AzureDevopsClient) ListAgentPoolAgents(agentPoolId int64) (list AgentPo
 
 	return
 }
+
+
+type AgentPoolJobList struct {
+	Count int `json:"count"`
+	List []JobRequest `json:"value"`
+}
+
+func (c *AzureDevopsClient) ListAgentPoolJobs(agentPoolId int64) (list AgentPoolJobList, error error) {
+	defer c.concurrencyUnlock()
+	c.concurrencyLock()
+
+	url := fmt.Sprintf(
+		"/_apis/distributedtask/pools/%v/jobrequests",
+		fmt.Sprintf("%d", agentPoolId),
+	)
+	response, err := c.rest().R().Get(url)
+	if err := c.checkResponse(response, err); err != nil {
+		error = err
+		return
+	}
+
+	err = json.Unmarshal(response.Body(), &list)
+	if err != nil {
+		error = err
+		return
+	}
+
+	return
+}

collector_structs.go

@@ -71,7 +71,6 @@ func (m *MetricCollectorList) AddBool(labels prometheus.Labels, state bool) {
 	m.list = append(m.list, MetricCollectorRow{labels: labels, value: value})
 }
 
-
 func (m *MetricCollectorList) GaugeSet(gauge *prometheus.GaugeVec) {
 	for _, metric := range m.list {
 		gauge.With(metric.labels).Set(metric.value)

metrics_agentpool.go

@@ -15,6 +15,7 @@ type MetricsCollectorAgentPool struct {
 		agentPoolAgent       *prometheus.GaugeVec
 		agentPoolAgentStatus *prometheus.GaugeVec
 		agentPoolAgentJob    *prometheus.GaugeVec
+		agentPoolQueueLength *prometheus.GaugeVec
 	}
 }
 
@@ -102,6 +103,7 @@ func (m *MetricsCollectorAgentPool) Reset() {
 	m.prometheus.agentPoolAgent.Reset()
 	m.prometheus.agentPoolAgentStatus.Reset()
 	m.prometheus.agentPoolAgentJob.Reset()
+	m.prometheus.agentPoolQueueLength.Reset()
 }
 
 func (m *MetricsCollectorAgentPool) Collect(ctx context.Context, callback chan<- func()) {
@@ -111,6 +113,7 @@ func (m *MetricsCollectorAgentPool) Collect(ctx context.Context, callback chan<-
 
 	for _, agentPoolId := range m.CollectorReference.AgentPoolIdList {
 		m.collectAgentQueues(ctx, callback, agentPoolId)
+		m.collectAgentPoolJobs(ctx, callback, agentPoolId)
 	}
 }
 
@@ -185,7 +188,7 @@ func (m *MetricsCollectorAgentPool) collectAgentQueues(ctx context.Context, call
 				"definitionName":   agentPoolAgent.AssignedRequest.Definition.Name,
 				"scopeID":          agentPoolAgent.AssignedRequest.ScopeId,
 			}
-			agentPoolAgentJobMetric.Add(jobLabels, timeToFloat64(agentPoolAgent.AssignedRequest.AssignTime))
+			agentPoolAgentJobMetric.Add(jobLabels, timeToFloat64(*agentPoolAgent.AssignedRequest.AssignTime))
 		}
 	}
 
@@ -195,3 +198,31 @@ func (m *MetricsCollectorAgentPool) collectAgentQueues(ctx context.Context, call
 		agentPoolAgentJobMetric.GaugeSet(m.prometheus.agentPoolAgentJob)
 	}
 }
+
+func (m *MetricsCollectorAgentPool) collectAgentPoolJobs(ctx context.Context, callback chan<- func(), agentPoolId int64) {
+	list, err := AzureDevopsClient.ListAgentPoolJobs(agentPoolId)
+	if err != nil {
+		Logger.Errorf("agentpool[%v]call[ListAgentJobs]: %v", agentPoolId, err)
+		return
+	}
+
+	agentPoolQueueLength := NewMetricCollectorList()
+
+	notStartedJobCount := 0
+
+	for _, agentPoolJob := range list.List {
+		if agentPoolJob.AssignTime == nil {
+			notStartedJobCount++
+		}
+	}
+
+	infoLabels := prometheus.Labels{
+		"agentPoolID": int64ToString(agentPoolId),
+	}
+
+	agentPoolQueueLength.Add(infoLabels, 1)
+
+	callback <- func() {
+		agentPoolQueueLength.GaugeSet(m.prometheus.agentPoolAgent)
+	}
+}