Support querying about counters. Add user agent filter.

Author: jellyterra

README.md

@@ -35,13 +35,17 @@ Example: `analytics-server.yaml`
 
 ```yaml
 listen: ":443"
-db: "redis://default:@localhost/0"
+db: "redis://default:@::1/0"
 
 key: "/etc/letsencrypt/live/symboltics.com/privkey.pem"
 cert: "/etc/letsencrypt/live/symboltics.com/fullchain.pem"
 
 access_token: ""
 
+user_agent_filter:
+  - bot
+  - crawler
+
 header:
   key:
     connecting_ip: "CF-Connecting-IP"

cmd/analytics-server/config.go

@@ -13,6 +13,8 @@ type Config struct {
 
 	AccessToken string `yaml:"access_token"`
 
+	UserAgentFilter []string `yaml:"user_agent_filter"`
+
 	Header struct {
 		Key struct {
 			ConnectingIP string `yaml:"connecting_ip"`

cmd/analytics-server/serve.go

@@ -65,6 +65,13 @@ func Serve(ctx context.Context, config *Config) error {
 		}
 	}()
 
+	go func() {
+		err := db.CountAll(ctx, config.UserAgentFilter)
+		if err != nil {
+			log.Println("Occurred an error while counting targets:", err)
+		}
+	}()
+
 	switch {
 	case config.Key == "" || config.Cert == "":
 		log.Println("Listen on http://" + config.Listen)

server/api_v1.go

@@ -7,6 +7,7 @@ package server
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"github.com/jellyterra/go-httpform"
 	"net/http"
 )
@@ -172,5 +173,86 @@ func V1GetApi(e *ApiEnv) map[string]http.HandlerFunc {
 
 			return 0, nil
 		}),
+
+		"/getAccessCountOfTarget": Wrap(func(w http.ResponseWriter, r *http.Request) (int, error) {
+			ctx := r.Context()
+
+			wrap, err := httpform.WrapFromRequest(r)
+			if err != nil {
+				return http.StatusBadRequest, err
+			}
+
+			var (
+				target = wrap.StringRequired("target")
+			)
+
+			err = wrap.Parse()
+			if err != nil {
+				return http.StatusBadRequest, err
+			}
+
+			c, err := e.Database.GetAccessCountOfTarget(ctx, *target)
+			if err != nil {
+				return http.StatusInternalServerError, err
+			}
+
+			_, _ = w.Write([]byte(c))
+
+			return 0, nil
+		}),
+
+		"/getAddrCountOfTarget": Wrap(func(w http.ResponseWriter, r *http.Request) (int, error) {
+			ctx := r.Context()
+
+			wrap, err := httpform.WrapFromRequest(r)
+			if err != nil {
+				return http.StatusBadRequest, err
+			}
+
+			var (
+				target = wrap.StringRequired("target")
+			)
+
+			err = wrap.Parse()
+			if err != nil {
+				return http.StatusBadRequest, err
+			}
+
+			c, err := e.Database.GetAccessAddrCountOfTarget(ctx, *target)
+			if err != nil {
+				return http.StatusInternalServerError, err
+			}
+
+			_, _ = w.Write([]byte(fmt.Sprint(c)))
+
+			return 0, nil
+		}),
+
+		"/getUuidCountOfTarget": Wrap(func(w http.ResponseWriter, r *http.Request) (int, error) {
+			ctx := r.Context()
+
+			wrap, err := httpform.WrapFromRequest(r)
+			if err != nil {
+				return http.StatusBadRequest, err
+			}
+
+			var (
+				target = wrap.StringRequired("target")
+			)
+
+			err = wrap.Parse()
+			if err != nil {
+				return http.StatusBadRequest, err
+			}
+
+			c, err := e.Database.GetAccessUuidCountOfTarget(ctx, *target)
+			if err != nil {
+				return http.StatusInternalServerError, err
+			}
+
+			_, _ = w.Write([]byte(fmt.Sprint(c)))
+
+			return 0, nil
+		}),
 	}
 }

server/db.go

@@ -8,9 +8,18 @@ import (
 	"context"
 	"github.com/redis/go-redis/v9"
 	"strconv"
+	"strings"
 	"time"
 )
 
+const (
+	kAccessReports_           = "AccessReports:"
+	kAccessReports_Timestamps = "AccessReports:Timestamps"
+	kAccessCounts_Targets     = "AccessCounts:Targets"
+	kAccessAddrs_Target_      = "AccessAddrs:Target:"
+	kAccessUuids_Target_      = "AccessUuids:Target:"
+)
+
 type Database struct {
 	Client *redis.Client
 }
@@ -23,23 +32,85 @@ func (db *Database) AddAccessReport(ctx context.Context, report *AccessReport) e
 		score, _  = strconv.ParseFloat(now.Format("20060102150405.99"), 64)
 	)
 
-	err := db.Client.HSet(ctx, "AccessReports:"+timestamp, *report).Err()
+	err := db.Client.HSet(ctx, kAccessReports_+timestamp, *report).Err()
 	if err != nil {
 		return err
 	}
 
-	err = db.Client.ZAdd(ctx, "AccessReports:Timestamps", redis.Z{Score: score, Member: timestamp}).Err()
+	err = db.Client.ZAdd(ctx, kAccessReports_Timestamps, redis.Z{Score: score, Member: timestamp}).Err()
 	if err != nil {
 		return err
 	}
 
-	return nil
+	return db.Client.HIncrBy(ctx, kAccessCounts_Targets, report.Target, 1).Err()
+}
+
+func (db *Database) AddAccessAddrOfTarget(ctx context.Context, target, addr string) error {
+	return db.Client.SAdd(ctx, kAccessAddrs_Target_+target, addr).Err()
+}
+
+func (db *Database) AddAccessUuidOfTarget(ctx context.Context, target, uuid string) error {
+	return db.Client.SAdd(ctx, kAccessUuids_Target_+target, uuid).Err()
 }
 
 func (db *Database) GetAccessReportsTimestamps(ctx context.Context, start, end string) ([]string, error) {
-	return db.Client.ZRangeByScore(ctx, "AccessReports:Timestamps", &redis.ZRangeBy{Min: start, Max: end}).Result()
+	return db.Client.ZRangeByScore(ctx, kAccessReports_Timestamps, &redis.ZRangeBy{Min: start, Max: end}).Result()
 }
 
 func (db *Database) GetAccessReportByTimestamp(ctx context.Context, timestamp string) (report AccessReport, _ error) {
-	return report, db.Client.HGetAll(ctx, "AccessReports:"+timestamp).Scan(&report)
+	return report, db.Client.HGetAll(ctx, kAccessReports_+timestamp).Scan(&report)
+}
+
+func (db *Database) GetAccessCountOfTarget(ctx context.Context, target string) (string, error) {
+	return db.Client.HGet(ctx, kAccessCounts_Targets, target).Result()
+}
+
+func (db *Database) GetAccessAddrCountOfTarget(ctx context.Context, target string) (int64, error) {
+	return db.Client.SCard(ctx, kAccessAddrs_Target_+target).Result()
+}
+
+func (db *Database) GetAccessUuidCountOfTarget(ctx context.Context, target string) (int64, error) {
+	return db.Client.SCard(ctx, kAccessUuids_Target_+target).Result()
+}
+
+func (db *Database) CountAll(ctx context.Context, userAgentFilter []string) error {
+	timestamps, err := db.GetAccessReportsTimestamps(ctx, "0", time.Now().UTC().Format("20060102150405.99"))
+	if err != nil {
+		return err
+	}
+
+	for _, timestamp := range timestamps {
+		report, err := db.GetAccessReportByTimestamp(ctx, timestamp)
+		if err != nil {
+			return err
+		}
+
+		isFiltered := false
+		for _, substr := range userAgentFilter {
+			if strings.Contains(report.UserAgent, substr) {
+				isFiltered = true
+				break
+			}
+		}
+		if isFiltered {
+			continue
+		}
+
+		err = db.AddAccessAddrOfTarget(ctx, report.Target, report.SourceIP)
+		if err != nil {
+			return err
+		}
+
+		err = db.AddAccessUuidOfTarget(ctx, report.Target, report.UUID)
+		if err != nil {
+			return err
+		}
+
+		err = db.Client.HIncrBy(ctx, kAccessCounts_Targets, report.Target, 1).Err()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
 }