Exporting WorkItem query results as metrics (#16)

* Added query-related entities

Gets query path+project id as parameter
Implements collection based on metrics_agentpool.go
Needs to pass projectId to AzureDevopsClient.rest()
Change webdevops/azure-devops-exporter to current repository
Add collectorQueryList and ProjectName
Get Number of Work Items from query

* Getting workitem details

Query each work item and its data for prometheus

* Renamed expected environment variable AZURE_DEVOPS_QUERY to AZURE_DEVOPS_QUERIES

* Parse multiple query entries coming from command line

Parse multiple project whitelist and blacklist entries coming from command line
Uncomment default metrics

* Aligned import statements with rest of the repo

* Fix azure-devops-client package references

* Updated README.md

* Removed updating projects for Query collectors

golint:
Renamed projectId to projectID
Unexported ensureSplitBySpace function

* Added collecting scrape durations of Query collectors

* Added ScrapeTimeQuery defaulting to ScrapeTime (30m)

Removed setting ScrapeTimeLive to ScrapeTime if not set; it's impossible since
 ScrapeTimeLive has a default of 30s.

* Changed long option for Query parameter to 'list.query'

Removed ensureSplitBySpace function since it was not a bug;
 lists should be passed by repeating same option for every item

Author: keremispirli

.gitignore

@@ -1,2 +1,2 @@
 /vendor/
-/azure-devops-exporter
+*.exe

README.md

@@ -31,6 +31,7 @@ Normally no configuration is needed but can be customized using environment vari
 | `AZURE_DEVOPS_FILTER_PROJECT`         | none                                | Whitelist project uuids                                                  |
 | `AZURE_DEVOPS_BLACKLIST_PROJECT`      | none                                | Blacklist project uuids                                                  |
 | `AZURE_DEVOPS_FILTER_AGENTPOOL`       | none                                | Whitelist for agentpool metrics                                          |
+| `AZURE_DEVOPS_QUERIES`                | none                                | Whitelist query and project uuids                                        |
 | `AZURE_DEVOPS_APIVERSION`             | fixed version                       | API version used to query for Azure DevOps                               |
 | `REQUEST_CONCURRENCY`                 | `10`                                | API request concurrency (number of calls at the same time)               |
 | `REQUEST_RETRIES`                     | `3`                                 | API request retries in case of failure                                   |
@@ -71,6 +72,7 @@ Metrics
 | `azure_devops_repository_stats`                 | repository    | Repository stats                                                                     |
 | `azure_devops_repository_commits`               | repository    | Repository commit counter                                                            |
 | `azure_devops_repository_pushes`                | repository    | Repository push counter                                                              |
+| `azure_devops_query_result`                     | live          | Latest results of given queries                                                      |
 | `azure_devops_deployment_info`                  | deployment    | Release deployment informations                                                      |
 | `azure_devops_deployment_status`                | deployment    | Release deployment status informations                                               |
 | `azure_devops_stats_agentpool_builds`           | stats         | Number of buildsper agentpool, project and result (counter)                          |
@@ -106,6 +108,7 @@ Application Options:
       --whitelist.project=                    Filter projects (UUIDs) [$AZURE_DEVOPS_FILTER_PROJECT]
       --blacklist.project=                    Filter projects (UUIDs) [$AZURE_DEVOPS_BLACKLIST_PROJECT]
       --whitelist.agentpool=                  Filter of agent pool (IDs) [$AZURE_DEVOPS_FILTER_AGENTPOOL]
+      --whitelist.queries=                    Pairs of query and project UUIDs in the form: '<queryId>@<projectId>' [%AZURE_DEVOPS_QUERIES%]
       --azuredevops.url=                      Azure DevOps url (empty if hosted by microsoft) [$AZURE_DEVOPS_URL]
       --azuredevops.access-token=             Azure DevOps access token [$AZURE_DEVOPS_ACCESS_TOKEN]
       --azuredevops.organisation=             Azure DevOps organization [$AZURE_DEVOPS_ORGANISATION]
@@ -136,7 +139,7 @@ Next shift
 bottomk(1,
   min by (userName, time) (
     pagerduty_schedule_final_entry{scheduleID="$SCHEDULEID",type="startTime"}
-    * on (userID) group_left(userName) (pagerduty_user_info) 
+    * on (userID) group_left(userName) (pagerduty_user_info)
   ) - time() > 0
 )
 ```

azure-devops-client/query.go

@@ -0,0 +1,44 @@
+package AzureDevopsClient
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+)
+
+type Query struct {
+	Path string `json:"path"`
+}
+
+type WorkItemInfoList struct {
+	List []WorkItemInfo `json:"workItems"`
+}
+
+type WorkItemInfo struct {
+	Id  int    `json:"id"`
+	Url string `json:"url"`
+}
+
+func (c *AzureDevopsClient) QueryWorkItems(queryPath, projectId string) (list WorkItemInfoList, error error) {
+	defer c.concurrencyUnlock()
+	c.concurrencyLock()
+
+	url := fmt.Sprintf(
+		"%v/_apis/wit/wiql/%v?api-version=%v",
+		projectId,
+		queryPath,
+		url.QueryEscape(c.ApiVersion),
+	)
+	response, err := c.rest().R().Get(url)
+	if err := c.checkResponse(response, err); err != nil {
+		error = err
+		return
+	}
+
+	err = json.Unmarshal(response.Body(), &list)
+	if err != nil {
+		error = err
+	}
+
+	return
+}

azure-devops-client/workitem.go

@@ -0,0 +1,37 @@
+package AzureDevopsClient
+
+import (
+	"encoding/json"
+)
+
+type WorkItem struct {
+	Id     int64          `json:"id"`
+	Fields WorkItemFields `json:"fields"`
+}
+
+type WorkItemFields struct {
+	Title        string `json:"System.Title"`
+	Path         string `json:"System.AreaPath"`
+	CreatedDate  string `json:"System.CreatedDate"`
+	AcceptedDate string `json:"Microsoft.VSTS.CodeReview.AcceptedDate"`
+	ResolvedDate string `json:"Microsoft.VSTS.Common.ResolvedDate"`
+	ClosedDate   string `json:"Microsoft.VSTS.Common.ClosedDate"`
+}
+
+func (c *AzureDevopsClient) GetWorkItem(workItemUrl string) (workItem WorkItem, error error) {
+	defer c.concurrencyUnlock()
+	c.concurrencyLock()
+
+	response, err := c.rest().R().Get(workItemUrl)
+	if err := c.checkResponse(response, err); err != nil {
+		error = err
+		return
+	}
+
+	err = json.Unmarshal(response.Body(), &workItem)
+	if err != nil {
+		error = err
+	}
+
+	return
+}

collector_processor_query.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"context"
+)
+
+type CollectorProcessorQueryInterface interface {
+	Setup(collector *CollectorQuery)
+	Reset()
+	Collect(ctx context.Context, callback chan<- func())
+}
+
+type CollectorProcessorQuery struct {
+	CollectorProcessorQueryInterface
+	CollectorReference *CollectorQuery
+}
+
+func NewCollectorQuery(name string, processor CollectorProcessorQueryInterface) *CollectorQuery {
+	collector := CollectorQuery{
+		CollectorBase: CollectorBase{
+			Name: name,
+		},
+		Processor: processor,
+	}
+
+	return &collector
+}

collector_query.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"context"
+	"sync"
+)
+
+type CollectorQuery struct {
+	CollectorBase
+
+	Processor CollectorProcessorQueryInterface
+	QueryList []string
+}
+
+func (c *CollectorQuery) Run() {
+	c.Processor.Setup(c)
+	go func() {
+		for {
+			go func() {
+				c.Collect()
+			}()
+			c.sleepUntilNextCollection()
+		}
+	}()
+}
+
+func (c *CollectorQuery) Collect() {
+	var wg sync.WaitGroup
+	var wgCallback sync.WaitGroup
+
+	ctx := context.Background()
+
+	callbackChannel := make(chan func())
+
+	c.collectionStart()
+
+	wg.Add(1)
+	go func(ctx context.Context, callback chan<- func()) {
+		defer wg.Done()
+		c.Processor.Collect(ctx, callbackChannel)
+	}(ctx, callbackChannel)
+
+	// collect metrics (callbacks) and process them
+	wgCallback.Add(1)
+	go func() {
+		defer wgCallback.Done()
+		var callbackList []func()
+		for callback := range callbackChannel {
+			callbackList = append(callbackList, callback)
+		}
+
+		// reset metric values
+		c.Processor.Reset()
+
+		// process callbacks (set metrics)
+		for _, callback := range callbackList {
+			callback()
+		}
+	}()
+
+	// wait for all funcs
+	wg.Wait()
+	close(callbackChannel)
+	wgCallback.Wait()
+
+	c.collectionFinish()
+}

main.go

@@ -8,6 +8,7 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"strings"
 	"time"
 )
 
@@ -26,6 +27,7 @@ var (
 	collectorGeneralList   map[string]*CollectorGeneral
 	collectorProjectList   map[string]*CollectorProject
 	collectorAgentPoolList map[string]*CollectorAgentPool
+	collectorQueryList     map[string]*CollectorQuery
 )
 
 var opts struct {
@@ -45,6 +47,7 @@ var opts struct {
 	ScrapeTimePullRequest   *time.Duration `long:"scrape.time.pullrequest"      env:"SCRAPE_TIME_PULLREQUEST"        description:"Scrape time for pullrequest metrics  (time.duration)"`
 	ScrapeTimeStats         *time.Duration `long:"scrape.time.stats"            env:"SCRAPE_TIME_STATS"              description:"Scrape time for stats metrics  (time.duration)"`
 	ScrapeTimeResourceUsage *time.Duration `long:"scrape.time.resourceusage"    env:"SCRAPE_TIME_RESOURCEUSAGE"      description:"Scrape time for resourceusage metrics  (time.duration)"`
+	ScrapeTimeQuery         *time.Duration `long:"scrape.time.query"            env:"SCRAPE_TIME_QUERY"              description:"Scrape time for query results  (time.duration)"`
 	ScrapeTimeLive          *time.Duration `long:"scrape.time.live"             env:"SCRAPE_TIME_LIVE"               description:"Scrape time for live metrics (time.duration)"              default:"30s"`
 
 	// summary options
@@ -55,6 +58,9 @@ var opts struct {
 	AzureDevopsBlacklistProjects []string `long:"blacklist.project"    env:"AZURE_DEVOPS_BLACKLIST_PROJECT" env-delim:" "   description:"Filter projects (UUIDs)"`
 	AzureDevopsFilterAgentPoolId []int64  `long:"whitelist.agentpool"  env:"AZURE_DEVOPS_FILTER_AGENTPOOL"  env-delim:" "   description:"Filter of agent pool (IDs)"`
 
+	// query settings
+	QueriesWithProjects []string `long:"list.query"    env:"AZURE_DEVOPS_QUERIES"    env-delim:" "   description:"Pairs of query and project UUIDs in the form: '<queryId>@<projectId>'"`
+
 	// azure settings
 	AzureDevopsUrl          *string `long:"azuredevops.url"                     env:"AZURE_DEVOPS_URL"             description:"Azure DevOps url (empty if hosted by microsoft)"`
 	AzureDevopsAccessToken  string  `long:"azuredevops.access-token"            env:"AZURE_DEVOPS_ACCESS_TOKEN"    description:"Azure DevOps access token" required:"true"`
@@ -95,6 +101,7 @@ func main() {
 	Logger.Infof("set scape interval[Deployment]: %v", scrapeIntervalStatus(opts.ScrapeTimeDeployment))
 	Logger.Infof("set scape interval[Stats]: %v", scrapeIntervalStatus(opts.ScrapeTimeStats))
 	Logger.Infof("set scape interval[ResourceUsage]: %v", scrapeIntervalStatus(opts.ScrapeTimeResourceUsage))
+	Logger.Infof("set scape interval[Queries]: %v", scrapeIntervalStatus(opts.ScrapeTimeQuery))
 	initMetricCollector()
 
 	Logger.Infof("Starting http server on %s", opts.ServerBind)
@@ -118,6 +125,20 @@ func initArgparser() {
 		}
 	}
 
+	// ensure query paths and projects are splitted by '@'
+	if opts.QueriesWithProjects != nil {
+		queryError := false
+		for _, query := range opts.QueriesWithProjects {
+			if strings.Count(query, "@") != 1 {
+				fmt.Println("Query path '", query, "' is malformed; should be '<query UUID>@<project UUID>'")
+				queryError = true
+			}
+		}
+		if queryError {
+			os.Exit(1)
+		}
+	}
+
 	// use default scrape time if null
 	if opts.ScrapeTimeProjects == nil {
 		opts.ScrapeTimeProjects = &opts.ScrapeTime
@@ -151,13 +172,13 @@ func initArgparser() {
 		opts.ScrapeTimeResourceUsage = &opts.ScrapeTime
 	}
 
-	if opts.ScrapeTimeLive == nil {
-		opts.ScrapeTimeLive = &opts.ScrapeTime
-	}
-
 	if opts.StatsSummaryMaxAge == nil {
 		opts.StatsSummaryMaxAge = opts.ScrapeTimeStats
 	}
+
+	if opts.ScrapeTimeQuery == nil {
+		opts.ScrapeTimeQuery = &opts.ScrapeTime
+	}
 }
 
 // Init and build Azure authorzier
@@ -225,6 +246,7 @@ func initMetricCollector() {
 	collectorGeneralList = map[string]*CollectorGeneral{}
 	collectorProjectList = map[string]*CollectorProject{}
 	collectorAgentPoolList = map[string]*CollectorAgentPool{}
+	collectorQueryList = map[string]*CollectorQuery{}
 
 	projectList := getAzureDevOpsProjects()
 
@@ -328,6 +350,16 @@ func initMetricCollector() {
 		Logger.Infof("collector[%s]: disabled", collectorName)
 	}
 
+	collectorName = "Query"
+	if opts.ScrapeTimeQuery.Seconds() > 0 {
+		collectorQueryList[collectorName] = NewCollectorQuery(collectorName, &MetricsCollectorQuery{})
+		collectorQueryList[collectorName].SetAzureProjects(&projectList)
+		collectorQueryList[collectorName].QueryList = opts.QueriesWithProjects
+		collectorQueryList[collectorName].SetScrapeTime(*opts.ScrapeTimeLive)
+	} else {
+		Logger.Infof("collector[%s]: disabled", collectorName)
+	}
+
 	for _, collector := range collectorGeneralList {
 		collector.Run()
 	}
@@ -340,6 +372,10 @@ func initMetricCollector() {
 		collector.Run()
 	}
 
+	for _, collector := range collectorQueryList {
+		collector.Run()
+	}
+
 	// background auto update of projects
 	if opts.ScrapeTimeProjects.Seconds() > 0 {
 		go func() {
@@ -350,6 +386,7 @@ func initMetricCollector() {
 				Logger.Info("daemon: updating project list")
 
 				projectList := getAzureDevOpsProjects()
+				Logger.Infof("daemon: found %v projects", projectList.Count)
 
 				for _, collector := range collectorGeneralList {
 					collector.SetAzureProjects(&projectList)
@@ -363,7 +400,7 @@ func initMetricCollector() {
 					collector.SetAzureProjects(&projectList)
 				}
 
-				Logger.Infof("daemon: found %v projects", projectList.Count)
+				Logger.Info("daemon: skipping Query collectors; they don't use projects")
 				time.Sleep(*opts.ScrapeTimeProjects)
 			}
 		}()

metrics_general.go

@@ -77,6 +77,7 @@ func (m *MetricsCollectorGeneral) collectCollectorStats(ctx context.Context, cal
 			}, *collector.LastScrapeDuration)
 		}
 	}
+
 	for _, collector := range collectorProjectList {
 		if collector.LastScrapeDuration != nil {
 			statsMetrics.AddDuration(prometheus.Labels{
@@ -86,6 +87,15 @@ func (m *MetricsCollectorGeneral) collectCollectorStats(ctx context.Context, cal
 		}
 	}
 
+	for _, collector := range collectorQueryList {
+		if collector.LastScrapeDuration != nil {
+			statsMetrics.AddDuration(prometheus.Labels{
+				"name": collector.Name,
+				"type": "collectorDuration",
+			}, *collector.LastScrapeDuration)
+		}
+	}
+
 	callback <- func() {
 		statsMetrics.GaugeSet(m.prometheus.stats)
 	}