Merge pull request #24 from SDxKeeper/queue_length

Working version of queue length metric

Author: mblaschke

README.md

@@ -52,7 +52,8 @@ Metrics
 |-------------------------------------------------|---------------|--------------------------------------------------------------------------------------|
 | `azure_devops_stats`                            | live          | General scraper stats                                                                |
 | `azure_devops_agentpool_info`                   | live          | Agent Pool informations                                                              |
-| `azure_devops_agentpool_size`                   | live          | Queue size per agent pool                                                            |
+| `azure_devops_agentpool_size`                   | live          | Number of agents per agent pool                                                      |
+| `azure_devops_agentpool_queue_length`           | live          | Queue length per agent pool                                                          |
 | `azure_devops_agentpool_agent_info`             | live          | Agent information per agent pool                                                     |
 | `azure_devops_agentpool_agent_status`           | live          | Status informations (eg. created date) for each agent in a agent pool                |
 | `azure_devops_agentpool_agent_job`              | live          | Currently running jobs on each agent                                                 |
@@ -155,7 +156,7 @@ Agent pool usage (without PoolMaintenance)
 count by(agentPoolID) (
   azure_devops_agentpool_agent_job{planType!="PoolMaintenance"}
   * on(agentPoolAgentID) group_left(agentPoolID) (azure_devops_agentpool_agent_info)
-) 
+)
 / on (agentPoolID) group_left() (azure_devops_agentpool_size)
 * on (agentPoolID) group_left(agentPoolName) (azure_devops_agentpool_info)
 ```

azure-devops-client/agentpool.go

@@ -63,25 +63,26 @@ type AgentPoolAgent struct {
 	Status            string
 	Version           string
 	CreatedOn         time.Time
+	AssignedRequest   JobRequest
+}
 
-	AssignedRequest struct {
-		RequestId    int64
-		Demands      []string
-		QueueTime    time.Time
-		AssignTime   time.Time
-		ReceiveTime  time.Time
-		LockedUntil  time.Time
-		ServiceOwner string
-		HostId       string
-		ScopeId      string
-		PlanType     string
-		PlanId       string
-		JobId        string
-		Definition   struct {
-			Id    int64
-			Name  string
-			Links Links `json:"_links"`
-		}
+type JobRequest struct {
+	RequestId    int64
+	Demands      []string
+	QueueTime    time.Time
+	AssignTime   *time.Time
+	ReceiveTime  time.Time
+	LockedUntil  time.Time
+	ServiceOwner string
+	HostId       string
+	ScopeId      string
+	PlanType     string
+	PlanId       string
+	JobId        string
+	Definition   struct {
+		Id    int64
+		Name  string
+		Links Links `json:"_links"`
 	}
 }
 
@@ -107,3 +108,31 @@ func (c *AzureDevopsClient) ListAgentPoolAgents(agentPoolId int64) (list AgentPo
 
 	return
 }
+
+type AgentPoolJobList struct {
+	Count int          `json:"count"`
+	List  []JobRequest `json:"value"`
+}
+
+func (c *AzureDevopsClient) ListAgentPoolJobs(agentPoolId int64) (list AgentPoolJobList, error error) {
+	defer c.concurrencyUnlock()
+	c.concurrencyLock()
+
+	url := fmt.Sprintf(
+		"/_apis/distributedtask/pools/%v/jobrequests",
+		fmt.Sprintf("%d", agentPoolId),
+	)
+	response, err := c.rest().R().Get(url)
+	if err := c.checkResponse(response, err); err != nil {
+		error = err
+		return
+	}
+
+	err = json.Unmarshal(response.Body(), &list)
+	if err != nil {
+		error = err
+		return
+	}
+
+	return
+}

collector_structs.go

@@ -71,7 +71,6 @@ func (m *MetricCollectorList) AddBool(labels prometheus.Labels, state bool) {
 	m.list = append(m.list, MetricCollectorRow{labels: labels, value: value})
 }
 
-
 func (m *MetricCollectorList) GaugeSet(gauge *prometheus.GaugeVec) {
 	for _, metric := range m.list {
 		gauge.With(metric.labels).Set(metric.value)

metrics_agentpool.go

@@ -15,6 +15,7 @@ type MetricsCollectorAgentPool struct {
 		agentPoolAgent       *prometheus.GaugeVec
 		agentPoolAgentStatus *prometheus.GaugeVec
 		agentPoolAgentJob    *prometheus.GaugeVec
+		agentPoolQueueLength *prometheus.GaugeVec
 	}
 }
 
@@ -89,11 +90,22 @@ func (m *MetricsCollectorAgentPool) Setup(collector *CollectorAgentPool) {
 		},
 	)
 
+	m.prometheus.agentPoolQueueLength = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "azure_devops_agentpool_queue_length",
+			Help: "Azure DevOps agentpool",
+		},
+		[]string{
+			"agentPoolID",
+		},
+	)
+
 	prometheus.MustRegister(m.prometheus.agentPool)
 	prometheus.MustRegister(m.prometheus.agentPoolSize)
 	prometheus.MustRegister(m.prometheus.agentPoolAgent)
 	prometheus.MustRegister(m.prometheus.agentPoolAgentStatus)
 	prometheus.MustRegister(m.prometheus.agentPoolAgentJob)
+	prometheus.MustRegister(m.prometheus.agentPoolQueueLength)
 }
 
 func (m *MetricsCollectorAgentPool) Reset() {
@@ -102,6 +114,7 @@ func (m *MetricsCollectorAgentPool) Reset() {
 	m.prometheus.agentPoolAgent.Reset()
 	m.prometheus.agentPoolAgentStatus.Reset()
 	m.prometheus.agentPoolAgentJob.Reset()
+	m.prometheus.agentPoolQueueLength.Reset()
 }
 
 func (m *MetricsCollectorAgentPool) Collect(ctx context.Context, callback chan<- func()) {
@@ -111,6 +124,7 @@ func (m *MetricsCollectorAgentPool) Collect(ctx context.Context, callback chan<-
 
 	for _, agentPoolId := range m.CollectorReference.AgentPoolIdList {
 		m.collectAgentQueues(ctx, callback, agentPoolId)
+		m.collectAgentPoolJobs(ctx, callback, agentPoolId)
 	}
 }
 
@@ -185,7 +199,7 @@ func (m *MetricsCollectorAgentPool) collectAgentQueues(ctx context.Context, call
 				"definitionName":   agentPoolAgent.AssignedRequest.Definition.Name,
 				"scopeID":          agentPoolAgent.AssignedRequest.ScopeId,
 			}
-			agentPoolAgentJobMetric.Add(jobLabels, timeToFloat64(agentPoolAgent.AssignedRequest.AssignTime))
+			agentPoolAgentJobMetric.Add(jobLabels, timeToFloat64(*agentPoolAgent.AssignedRequest.AssignTime))
 		}
 	}
 
@@ -195,3 +209,31 @@ func (m *MetricsCollectorAgentPool) collectAgentQueues(ctx context.Context, call
 		agentPoolAgentJobMetric.GaugeSet(m.prometheus.agentPoolAgentJob)
 	}
 }
+
+func (m *MetricsCollectorAgentPool) collectAgentPoolJobs(ctx context.Context, callback chan<- func(), agentPoolId int64) {
+	list, err := AzureDevopsClient.ListAgentPoolJobs(agentPoolId)
+	if err != nil {
+		Logger.Errorf("agentpool[%v]call[ListAgentJobs]: %v", agentPoolId, err)
+		return
+	}
+
+	agentPoolQueueLengthMetric := NewMetricCollectorList()
+
+	notStartedJobCount := 0
+
+	for _, agentPoolJob := range list.List {
+		if agentPoolJob.AssignTime == nil {
+			notStartedJobCount++
+		}
+	}
+
+	infoLabels := prometheus.Labels{
+		"agentPoolID": int64ToString(agentPoolId),
+	}
+
+	agentPoolQueueLengthMetric.Add(infoLabels, float64(notStartedJobCount))
+
+	callback <- func() {
+		agentPoolQueueLengthMetric.GaugeSet(m.prometheus.agentPoolQueueLength)
+	}
+}