MCP Verifier support to save results on a json file

.github/workflows/build.yml

@@ -20,7 +20,9 @@ jobs:
         go-version: ${{ env.go_version }}
 
     - name: Build
-      run: make build
+      run: |
+        make build
+        make build-woodpecker-mcp-verifier
 
     - name: Test
       run: make test

.github/workflows/pr.yml

@@ -17,7 +17,9 @@ jobs:
         go-version: ${{ env.go_version }}
 
     - name: Build
-      run: make build
+      run: |
+        make build
+        make build-woodpecker-mcp-verifier
 
     - name: Test
       run: make test

.github/workflows/release.yml

@@ -21,7 +21,9 @@ jobs:
         go-version: ${{ env.go_version }}
 
     - name: Build
-      run: make build
+      run: |
+        make build
+        make build-woodpecker-mcp-verifier
 
     - name: Test
       run: make test

.goreleaser.yaml

@@ -13,6 +13,17 @@ builds:
       - "-X github.com/operantai/woodpecker/cmd/woodpecker/cmd.GitCommit={{.Commit}}"
       - "-X github.com/operantai/woodpecker/cmd/woodpecker/cmd.Version={{.Version}}"
       - "-X github.com/operantai/woodpecker/cmd/woodpecker/cmd.BuildDate={{ .Date }}"
+  - env:
+      - CGO_ENABLED=0
+    main: ./cmd/woodpecker-mcp-verifier
+    goos:
+      - linux
+      - windows
+      - darwin
+    ldflags:
+      - "-X github.com/operantai/woodpecker/cmd/woodpecker-mcp-verifier/cmd.GitCommit={{.Commit}}"
+      - "-X github.com/operantai/woodpecker/cmd/woodpecker-mcp-verifier/cmd.Version={{.Version}}"
+      - "-X github.com/operantai/woodpecker/cmd/woodpecker-mcp-verifier/cmd.BuildDate={{ .Date }}"
 
 archives:
   - format: tar.gz

Makefile

@@ -17,6 +17,9 @@ build-woodpecker-ai-verifier: ## Build woodpecker AI verifier container
 build-woodpecker-ai-app: ## Build woodpecker AI app container
 	@docker build -f build/Dockerfile.woodpecker-ai-app .
 
+build-woodpecker-mcp-verifier: ## Builds the woodpecker MCP verifier client tool
+	@go build -o "bin/woodpecker-mcp-verifier" -ldflags $(LD_FLAGS) cmd/woodpecker-mcp-verifier/main.go
+
 # ==================================================================================== #
 # QUALITY
 # ==================================================================================== #

cmd/woodpecker-mcp-verifier/cmd/root.go

@@ -3,6 +3,7 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"strings"
 
 	"github.com/operantai/woodpecker/cmd/woodpecker-mcp-verifier/utils"
@@ -19,10 +20,13 @@ var (
 		Short: "Run a MCP client verifier as a Woodpecker components",
 		Long:  "Run a MCP client verifier as a Woodpecker components",
 	}
-	protocol utils.MCMCPprotocol
-	cmdArgs  []string
+	protocol                        utils.MCMCPprotocol
+	cmdArgs                         []string
+	serverURL, payloadPath, appName string
 )
 
+const experimentType = "woodpecker"
+
 // Execute adds all child commands to the root command and sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
@@ -32,29 +36,39 @@ func Execute() {
 	}
 }
 
-// cleanCmd represents the clean command
 var runCmd = &cobra.Command{
 	Use:   "run",
-	Short: "Run a MCP client verifier as a Woodpecker component",
-	Long:  "Run a MCP client verifier as a Woodpecker component",
-	Run: func(cmd *cobra.Command, args []string) {
-		output.WriteInfo("MCP client verifier starting ...")
-		var serverURL, payloadPath string
+	Short: "Run a MCP client verifier as a Woodpecker experiment",
+	Long:  "Run a MCP client verifier as a Woodpecker experiment",
+	PreRunE: func(cmd *cobra.Command, args []string) error {
+		// Mutually exclusive: either --url or --cmd_args for either
+		// streamable-http or stdio
 		var err error
 
 		if serverURL, err = cmd.Flags().GetString("url"); err != nil {
-			output.WriteFatal("%v", err)
+			return err
+		}
+		if cmdArgs, err = cmd.Flags().GetStringSlice("cmd_args"); err != nil {
+			return err
 		}
+		if serverURL != "" && len(cmdArgs) > 0 {
+			return fmt.Errorf("--url and --cmd_args are mutually exclusive")
+		}
+		return nil
+	},
+	RunE: func(cmd *cobra.Command, args []string) error {
+		output.WriteInfo("MCP client verifier starting ...")
+
 		payloadPath = viper.GetString("payload-path")
 
-		if err := mcpverifier.RunClient(context.Background(), serverURL, protocol, &cmdArgs, payloadPath); err != nil {
-			output.WriteFatal("%v", err)
+		if err := mcpverifier.RunClient(context.Background(), serverURL, protocol, &cmdArgs, payloadPath, experimentType, appName); err != nil {
+			return err
 		}
+		return nil
 	},
 }
 
 func init() {
-	rootCmd.AddCommand(runCmd)
 	// Tells Viper to use this prefix when reading environment variables
 	viper.SetEnvPrefix("woodpecker")
 	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
@@ -64,9 +78,6 @@ func init() {
 	runCmd.Flags().VarP(&protocol, "protocol", "p", "The MCP protocol being used")
 	runCmd.Flags().StringP("payload-path", "t", "/app/payload.json", "The path to the json payload content")
 	runCmd.Flags().StringSliceP("cmd_args", "c", cmdArgs, `If STDIO protocol, a comma separated list of cmd and args. i.e -c "uv,run,server"`)
-	if err := runCmd.MarkFlagRequired("url"); err != nil {
-		output.WriteFatal("%v", err)
-	}
 	if err := runCmd.MarkFlagRequired("protocol"); err != nil {
 		output.WriteFatal("%v", err)
 	}
@@ -75,9 +86,12 @@ func init() {
 	}
 
 	// Sets App name
-	appName := viper.GetString("APP_NAME")
+	appName = viper.GetString("APP_NAME")
 	if appName == "" {
 		output.WriteInfo("Setting WOODPECKER_APP_NAME to woodpecker-mcp-verifier")
-		viper.Set("APP_NAME", "woodpecker-mcp-verifier")
+		appName = "mcp-verifier"
+		viper.Set("APP_NAME", fmt.Sprintf("%s-%s", experimentType, appName))
 	}
+
+	rootCmd.AddCommand(runCmd)
 }

cmd/woodpecker-mcp-verifier/cmd/version.go

@@ -0,0 +1,28 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var (
+	// Build info which gets populated by ldflags at build time
+	Version   string
+	GitCommit string
+	BuildDate string
+)
+
+// versionCmd represents the version command
+var versionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "Output CLI version information",
+	Long:  "Output CLI version information",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Printf("version %s, build %s, built on %s\n", Version, GitCommit, BuildDate)
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(versionCmd)
+}

cmd/woodpecker-mcp-verifier/docs/README.md

@@ -14,7 +14,7 @@ The tool is an MCP client that will perform the following actions:
 In order to run the tool here are some of the input parameters you can pass:
 
 ```bash
-go run cmd/woodpecker-mcp-verifier/main.go run -h
+woodpecker-mcp-verifier -h
 
 Run a MCP client verifier as a Woodpecker component
 

cmd/woodpecker-mcp-verifier/vschema/ai-verifier.go

@@ -37,8 +37,7 @@ func (a *AIFormatter) AnalyzeSchema(inputSchema any) (map[string]any, error) {
 		return nil, err
 	}
 
-	output.WriteInfo("AI response ...")
-	output.WriteJSON(result)
+	output.WriteInfo("Generating AI response ...")
 
 	return result, nil
 }

go.mod

@@ -4,12 +4,12 @@ go 1.24.11
 
 require (
 	github.com/charmbracelet/lipgloss v1.1.0
-	github.com/docker/docker v28.2.2+incompatible
+	github.com/docker/docker v28.3.3+incompatible
 	github.com/docker/go-connections v0.5.0
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/mux v1.8.0
 	github.com/mattn/go-runewidth v0.0.16
-	github.com/modelcontextprotocol/go-sdk v1.2.0
+	github.com/modelcontextprotocol/go-sdk v1.3.1
 	github.com/onsi/ginkgo/v2 v2.27.2
 	github.com/onsi/gomega v1.38.2
 	github.com/spf13/cobra v1.7.0
@@ -56,7 +56,7 @@ require (
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/jsonschema-go v0.3.0 // indirect
+	github.com/google/jsonschema-go v0.4.2 // indirect
 	github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
@@ -80,6 +80,8 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sagikazarmark/locafero v0.11.0 // indirect
+	github.com/segmentio/asm v1.1.3 // indirect
+	github.com/segmentio/encoding v0.5.3 // indirect
 	github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
 	github.com/spf13/cast v1.10.0 // indirect

go.sum

@@ -40,8 +40,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr
 github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
 github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
-github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw=
-github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
+github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
 github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@@ -91,8 +91,8 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/jsonschema-go v0.3.0 h1:6AH2TxVNtk3IlvkkhjrtbUc4S8AvO0Xii0DxIygDg+Q=
-github.com/google/jsonschema-go v0.3.0/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE=
+github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8=
+github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE=
 github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
 github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@@ -141,8 +141,8 @@ github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7z
 github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
 github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
 github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
-github.com/modelcontextprotocol/go-sdk v1.2.0 h1:Y23co09300CEk8iZ/tMxIX1dVmKZkzoSBZOpJwUnc/s=
-github.com/modelcontextprotocol/go-sdk v1.2.0/go.mod h1:6fM3LCm3yV7pAs8isnKLn07oKtB0MP9LHd3DfAcKw10=
+github.com/modelcontextprotocol/go-sdk v1.3.1 h1:TfqtNKOIWN4Z1oqmPAiWDC2Jq7K9OdJaooe0teoXASI=
+github.com/modelcontextprotocol/go-sdk v1.3.1/go.mod h1:DgVX498dMD8UJlseK1S5i1T4tFz2fkBk4xogC3D15nw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -179,6 +179,10 @@ github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWN
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
 github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
+github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc=
+github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg=
+github.com/segmentio/encoding v0.5.3 h1:OjMgICtcSFuNvQCdwqMCv9Tg7lEOXGwm1J5RPQccx6w=
+github.com/segmentio/encoding v0.5.3/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=

internal/mcp-verifier/main.go

@@ -24,13 +24,14 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
+const tmpFileDir = "/tmp/woodpecker"
+
 // RunClient entry point to start the MCP client connection
-func RunClient(ctx context.Context, serverURL string, protocol utils.MCMCPprotocol, cmdArgs *[]string, payloadPath string) error {
-	output.WriteInfo("Connecting to server: %s", serverURL)
+func RunClient(ctx context.Context, serverURL string, protocol utils.MCMCPprotocol, cmdArgs *[]string, payloadPath, experimentType, name string) error {
 	output.WriteInfo("Using protocol: %s", protocol)
 
 	sValidator := vschema.NewVSchema()
-	mcpClient, err := NewMCPClient(WithValidator(sValidator), WithAIFormatter(viper.GetBool("USE_AI_FORMATTER")))
+	mcpClient, err := NewMCPClient(WithValidator(sValidator), WithAIFormatter(viper.GetBool("USE_AI_FORMATTER")), WithExperimentType(experimentType), WithName(name))
 	if err != nil {
 		return err
 	}
@@ -70,6 +71,11 @@ func RunClient(ctx context.Context, serverURL string, protocol utils.MCMCPprotoc
 		}
 
 	}
+	mergedPath, err := mergeTempJSONFilesStreaming(tmpFileDir, experimentType, name)
+	if err != nil {
+		return err
+	}
+	output.WriteInfo("Results saved in: %s", mergedPath)
 	return nil
 }
 
@@ -185,13 +191,30 @@ func (m *mcpClient) ToolCallWithPayload(ctx context.Context, cs IMCPClientSessio
 		if err != nil {
 			return err
 		}
-		resp := map[string]any{
-			"tool":     tool.Name,
-			"response": string(data),
-			"tags":     mPayload.Tags,
+		resp := make(map[string][]ToolResponses)
+
+		resp[tool.Name] = []ToolResponses{{
+			ToolName:   tool.Name,
+			Response:   string(data),
+			Tags:       mPayload.Tags,
+			Parameters: params,
+		},
+		}
+		resultJSON, err := json.Marshal(resp)
+		if err != nil {
+			return fmt.Errorf("failed to marshal experiment results: %w", err)
+		}
+		output.WriteInfo("Saving %s response ...", tool.Name)
+		file, err := createTempFile(m.experimentType, m.name)
+		if err != nil {
+			return fmt.Errorf("unable to create file cache for experiment results %w", err)
 		}
-		output.WriteInfo("Tool response ...")
-		output.WriteJSON(resp)
+
+		_, err = file.Write(resultJSON)
+		if err != nil {
+			return fmt.Errorf("failed to write experiment results: %w", err)
+		}
+
 	}
 	return nil
 }

internal/mcp-verifier/model.go

@@ -60,9 +60,11 @@ type IMCPClient interface {
 }
 
 type mcpClient struct {
-	validator   vschema.IvSchema
-	aiFormatter vschema.IAIFormatter
-	useAi       bool
+	validator      vschema.IvSchema
+	aiFormatter    vschema.IAIFormatter
+	useAi          bool
+	name           string
+	experimentType string
 }
 
 type Option func(*mcpClient)
@@ -79,6 +81,18 @@ func WithAIFormatter(useAI bool) Option {
 	}
 }
 
+func WithName(name string) Option {
+	return func(mc *mcpClient) {
+		mc.name = name
+	}
+}
+
+func WithExperimentType(experimentType string) Option {
+	return func(mc *mcpClient) {
+		mc.experimentType = experimentType
+	}
+}
+
 func NewMCPClient(options ...Option) (IMCPClient, error) {
 	mc := &mcpClient{}
 
@@ -112,3 +126,10 @@ type IMCPClientSession interface {
 	// The params.Arguments can be any value that marshals into a JSON object.
 	CallTool(ctx context.Context, params *mcp.CallToolParams) (*mcp.CallToolResult, error)
 }
+
+type ToolResponses struct {
+	ToolName   string   `json:"toolName"`
+	Response   string   `json:"response"`
+	Tags       []string `json:"tags"`
+	Parameters any      `json:"parameters"`
+}