metrics: track and report non-aggregated events

Currently, "usage" reports are sent, which are aggregated.
Add "event" variant, which won't be aggregated.

For compatibility, the "usage" events remain and nothing has
changed in terms of how they parse the command args, for example.
This will ensure continued functioning of anything that relies
on these metrics.

For the "event" variants, the CLI parsing is slightly different
in an attempt to improve data analysis capabilities while still
being respectful of user privacy (i.e. only known values of Docker
CLI commands/flags are ever recorded). Additionally, execution
duration information is now reported with these events.

Signed-off-by: Milas Bowman <milas.bowman@docker.com>

Author: milas

cli/main.go

@@ -260,7 +260,7 @@ func main() {
 		handleError(ctx, err, ctype, currentContext, cc, root, start, duration)
 	}
 	metricsClient.Track(
-		metrics.CmdMeta{
+		metrics.CmdResult{
 			ContextType: ctype,
 			Args:        os.Args[1:],
 			Status:      metrics.SuccessStatus,
@@ -298,7 +298,7 @@ func handleError(
 	// if user canceled request, simply exit without any error message
 	if api.IsErrCanceled(err) || errors.Is(ctx.Err(), context.Canceled) {
 		metricsClient.Track(
-			metrics.CmdMeta{
+			metrics.CmdResult{
 				ContextType: ctype,
 				Args:        os.Args[1:],
 				Status:      metrics.CanceledStatus,
@@ -335,7 +335,7 @@ func exit(ctx string, err error, ctype string, start time.Time, duration time.Du
 	if exit, ok := err.(cli.StatusError); ok {
 		// TODO(milas): shouldn't this use the exit code to determine status?
 		metricsClient.Track(
-			metrics.CmdMeta{
+			metrics.CmdResult{
 				ContextType: ctype,
 				Args:        os.Args[1:],
 				Status:      metrics.SuccessStatus,
@@ -358,7 +358,7 @@ func exit(ctx string, err error, ctype string, start time.Time, duration time.Du
 		exitCode = metrics.CommandSyntaxFailure.ExitCode
 	}
 	metricsClient.Track(
-		metrics.CmdMeta{
+		metrics.CmdResult{
 			ContextType: ctype,
 			Args:        os.Args[1:],
 			Status:      metricsStatus,
@@ -400,7 +400,7 @@ func checkIfUnknownCommandExistInDefaultContext(err error, currentContext string
 
 		if mobycli.IsDefaultContextCommand(dockerCommand) {
 			fmt.Fprintf(os.Stderr, "Command %q not available in current context (%s), you can use the \"default\" context to run this command\n", dockerCommand, currentContext)
-			metricsClient.Track(metrics.CmdMeta{
+			metricsClient.Track(metrics.CmdResult{
 				ContextType: contextType,
 				Args:        os.Args[1:],
 				Status:      metrics.FailureStatus,

cli/metrics/client.go

@@ -26,13 +26,26 @@ import (
 // specified file path.
 const EnvVarDebugMetricsPath = "DOCKER_METRICS_DEBUG_LOG"
 
-type CmdMeta struct {
+// Timeout is the maximum amount of time we'll wait for metrics sending to be
+// acknowledged before giving up.
+const Timeout = 50 * time.Millisecond
+
+// CmdResult provides details about process execution.
+type CmdResult struct {
+	// ContextType is `moby` for Docker or the name of a cloud provider.
 	ContextType string
-	Args        []string
-	Status      string
-	ExitCode    int
-	Start       time.Time
-	Duration    time.Duration
+	// Args minus the process name (argv[0] aka `docker`).
+	Args []string
+	// Status based on exit code as a descriptive value.
+	//
+	// Deprecated: used for usage, events rely exclusively on exit code.
+	Status string
+	// ExitCode is 0 on success; otherwise, failure.
+	ExitCode int
+	// Start time of the process (UTC).
+	Start time.Time
+	// Duration of process execution.
+	Duration time.Duration
 }
 
 type client struct {
@@ -44,8 +57,8 @@ type cliversion struct {
 	f func() string
 }
 
-// Command is a command
-type Command struct {
+// CommandUsage reports a CLI invocation for aggregation.
+type CommandUsage struct {
 	Command string `json:"command"`
 	Context string `json:"context"`
 	Source  string `json:"source"`
@@ -69,9 +82,9 @@ type Client interface {
 	// SendUsage sends the command to Docker Desktop.
 	//
 	// Note that metric collection is best-effort, so any errors are ignored.
-	SendUsage(Command)
+	SendUsage(CommandUsage)
 	// Track creates an event for a command execution and reports it.
-	Track(cmd CmdMeta)
+	Track(CmdResult)
 }
 
 // NewClient returns a new metrics client that will send metrics using the
@@ -107,7 +120,7 @@ func (c *client) WithCliVersionFunc(f func() string) {
 	c.cliversion.f = f
 }
 
-func (c *client) SendUsage(command Command) {
+func (c *client) SendUsage(command CommandUsage) {
 	result := make(chan bool, 1)
 	go func() {
 		c.reporter.Heartbeat(command)
@@ -118,6 +131,6 @@ func (c *client) SendUsage(command Command) {
 	// Posting metrics without Desktop listening returns in less than a ms, and a handful of ms (often <2ms) when Desktop is listening
 	select {
 	case <-result:
-	case <-time.After(50 * time.Millisecond):
+	case <-time.After(Timeout):
 	}
 }

cli/metrics/docker_command.go

@@ -0,0 +1,193 @@
+/*
+   Copyright 2022 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package metrics
+
+import (
+	"strings"
+	"time"
+)
+
+// DockerCLIEvent represents an invocation of `docker` from the the CLI.
+type DockerCLIEvent struct {
+	Command      string    `json:"command,omitempty"`
+	Subcommand   string    `json:"subcommand,omitempty"`
+	Usage        bool      `json:"usage,omitempty"`
+	ExitCode     int32     `json:"exit_code"`
+	StartTime    time.Time `json:"start_time"`
+	DurationSecs float64   `json:"duration_secs,omitempty"`
+}
+
+// NewDockerCLIEvent inspects the command line string and returns a stripped down
+// version suitable for reporting.
+//
+// The parser will only use known values for command/subcommand from a hardcoded
+// built-in set for safety. It also does not attempt to perfectly accurately
+// reflect how arg parsing works in a real program, instead favoring a fairly
+// simple approach that's still reasonably robust.
+//
+// If the command does not map to a known Docker (or first-party plugin)
+// command, `nil` will be returned. Similarly, if no subcommand for the
+// built-in/plugin can be determined, it will be empty.
+func NewDockerCLIEvent(cmd CmdResult) *DockerCLIEvent {
+	if len(cmd.Args) == 0 {
+		return nil
+	}
+
+	cmdPath := findCommand(append([]string{"docker"}, cmd.Args...))
+	if cmdPath == nil {
+		return nil
+	}
+
+	if len(cmdPath) < 2 {
+		// ignore unknown commands; we can't infer anything from them safely
+		// N.B. ONLY compose commands are supported by `cmdHierarchy` currently!
+		return nil
+	}
+
+	// look for a subcommand
+	var subcommand string
+	if len(cmdPath) >= 3 {
+		var subcommandParts []string
+		for _, c := range cmdPath[2:] {
+			subcommandParts = append(subcommandParts, c.name)
+		}
+		subcommand = strings.Join(subcommandParts, "-")
+	}
+
+	var usage bool
+	for _, arg := range cmd.Args {
+		// TODO(milas): also support `docker help build` syntax
+		if arg == "help" {
+			return nil
+		}
+
+		if arg == "--help" || arg == "-h" {
+			usage = true
+		}
+	}
+
+	event := &DockerCLIEvent{
+		Command:      cmdPath[1].name,
+		Subcommand:   subcommand,
+		ExitCode:     int32(cmd.ExitCode),
+		Usage:        usage,
+		StartTime:    cmd.Start,
+		DurationSecs: cmd.Duration.Seconds(),
+	}
+
+	return event
+}
+
+func findCommand(args []string) []*cmdNode {
+	if len(args) == 0 {
+		return nil
+	}
+
+	cmdPath := []*cmdNode{cmdHierarchy}
+	if len(args) == 1 {
+		return cmdPath
+	}
+
+	nodePath := []string{args[0]}
+	for _, v := range args[1:] {
+		v = strings.TrimSpace(v)
+		if v == "" || strings.HasPrefix(v, "-") {
+			continue
+		}
+		candidate := append(nodePath, v)
+		if c := cmdHierarchy.find(candidate); c != nil {
+			cmdPath = append(cmdPath, c)
+			nodePath = candidate
+		}
+	}
+
+	return cmdPath
+}
+
+type cmdNode struct {
+	name     string
+	plugin   bool
+	children []*cmdNode
+}
+
+func (c *cmdNode) find(path []string) *cmdNode {
+	if len(path) == 0 {
+		return nil
+	}
+
+	if c.name != path[0] {
+		return nil
+	}
+
+	if len(path) == 1 {
+		return c
+	}
+
+	remainder := path[1:]
+	for _, child := range c.children {
+		if res := child.find(remainder); res != nil {
+			return res
+		}
+	}
+
+	return nil
+}
+
+var cmdHierarchy = &cmdNode{
+	name: "docker",
+	children: []*cmdNode{
+		{
+			name:   "compose",
+			plugin: true,
+			children: []*cmdNode{
+				{
+					name: "alpha",
+					children: []*cmdNode{
+						{name: "watch"},
+						{name: "dryrun"},
+					},
+				},
+				{name: "build"},
+				{name: "config"},
+				{name: "convert"},
+				{name: "cp"},
+				{name: "create"},
+				{name: "down"},
+				{name: "events"},
+				{name: "exec"},
+				{name: "images"},
+				{name: "kill"},
+				{name: "logs"},
+				{name: "ls"},
+				{name: "pause"},
+				{name: "port"},
+				{name: "ps"},
+				{name: "pull"},
+				{name: "push"},
+				{name: "restart"},
+				{name: "rm"},
+				{name: "run"},
+				{name: "start"},
+				{name: "stop"},
+				{name: "top"},
+				{name: "unpause"},
+				{name: "up"},
+				{name: "version"},
+			},
+		},
+	},
+}