Support rules with a script node interior type.

pkg/ast/ast.go

@@ -106,12 +106,9 @@ func (b *builderT) descendTree(fn func() error) error {
 }
 
 func Build(data []byte) (*AstT, error) {
-	var (
-		parseTree *parser.TreeT
-		err       error
-	)
 
-	if parseTree, err = parser.Parse(data); err != nil {
+	parseTree, err := parser.Parse(data)
+	if err != nil {
 		log.Error().Any("err", err).Msg("Parser failed")
 		return nil, err
 	}
@@ -157,33 +154,17 @@ func BuildTree(tree *parser.TreeT) (*AstT, error) {
 func (b *builderT) buildTree(parserNode *parser.NodeT, parentMachineAddress *AstNodeAddressT, termIdx *uint32) (*AstNodeT, error) {
 
 	var (
-		machineMatchNode *AstNodeT
-		matchNode        *AstNodeT
-		children         = make([]*AstNodeT, 0)
-		machineAddress   = b.newAstNodeAddress(parserNode.Metadata.RuleHash, parserNode.Metadata.Type.String(), termIdx)
-		err              error
+		machineAddress = b.newAstNodeAddress(parserNode.Metadata.RuleHash, parserNode.Metadata.Type.String(), termIdx)
 	)
 
-	// Build children (either matcher children or nested machines)
-	if parserNode.IsMatcherNode() {
-		if matchNode, err = b.buildMatcherChildren(parserNode, machineAddress, termIdx); err != nil {
-			return nil, err
-		}
-		children = append(children, matchNode)
-	} else if parserNode.IsPromNode() {
-		if matchNode, err = b.buildPromQLNode(parserNode, machineAddress, termIdx); err != nil {
-			return nil, err
-		}
-		children = append(children, matchNode)
-
-	} else {
-		if children, err = b.buildMachineChildren(parserNode, machineAddress); err != nil {
-			return nil, err
-		}
+	children, err := b.buildChildrenNodes(parserNode, machineAddress, termIdx)
+	if err != nil {
+		return nil, err
 	}
 
 	// Build state machine after recursively building children
-	if machineMatchNode, err = b.buildStateMachine(parserNode, parentMachineAddress, machineAddress, children); err != nil {
+	machineMatchNode, err := b.buildStateMachine(parserNode, parentMachineAddress, machineAddress, children)
+	if err != nil {
 		return nil, err
 	}
 
@@ -192,6 +173,35 @@ func (b *builderT) buildTree(parserNode *parser.NodeT, parentMachineAddress *Ast
 	return machineMatchNode, nil
 }
 
+func (b *builderT) buildChildrenNodes(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (children []*AstNodeT, err error) {
+
+	leaf, err := b.buildLeafChild(parserNode, machineAddress, termIdx)
+
+	switch {
+	case err != nil:
+		return nil, err
+	case leaf != nil:
+		return []*AstNodeT{leaf}, nil
+	case parserNode.IsScriptNode():
+		children, err = b.buildScriptChildren(parserNode, machineAddress)
+	default:
+		children, err = b.buildMachineChildren(parserNode, machineAddress)
+	}
+
+	return children, err
+}
+
+func (b *builderT) buildLeafChild(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (leaf *AstNodeT, err error) {
+
+	switch {
+	case parserNode.IsMatcherNode():
+		leaf, err = b.buildMatcherChild(parserNode, machineAddress, termIdx)
+	case parserNode.IsPromNode():
+		leaf, err = b.buildPromQLNode(parserNode, machineAddress, termIdx)
+	}
+	return
+}
+
 func (b *builderT) newAstNodeAddress(ruleHash, name string, termIdx *uint32) *AstNodeAddressT {
 	var address = &AstNodeAddressT{
 		Version:  "v" + strconv.FormatInt(int64(AstVersion), 10),
@@ -220,7 +230,7 @@ func newAstNode(parserNode *parser.NodeT, typ schema.NodeTypeT, scope string, pa
 	}
 }
 
-func (b *builderT) buildMatcherChildren(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (*AstNodeT, error) {
+func (b *builderT) buildMatcherChild(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (*AstNodeT, error) {
 
 	var (
 		matchNode *AstNodeT
@@ -360,7 +370,7 @@ func addNegateOpts(assert *AstNodeT, negateOpts *parser.NegateOptsT) {
 	}
 }
 
-func (b *builderT) buildStateMachine(parserNode *parser.NodeT, parentMachineAddress *AstNodeAddressT, machineAddress *AstNodeAddressT, children []*AstNodeT) (*AstNodeT, error) {
+func (b *builderT) buildStateMachine(parserNode *parser.NodeT, parentMachineAddress, machineAddress *AstNodeAddressT, children []*AstNodeT) (*AstNodeT, error) {
 
 	switch parserNode.Metadata.Type {
 	case schema.NodeTypeSeq, schema.NodeTypeLogSeq:
@@ -371,6 +381,9 @@ func (b *builderT) buildStateMachine(parserNode *parser.NodeT, parentMachineAddr
 			return nil, parserNode.WrapError(ErrInvalidWindow)
 		}
 	case schema.NodeTypeSet, schema.NodeTypeLogSet, schema.NodeTypePromQL:
+	case schema.NodeTypeScript:
+		return b.buildScriptNode(parserNode, parentMachineAddress, machineAddress)
+
 	default:
 		log.Error().
 			Any("address", machineAddress).

pkg/ast/ast_script.go

@@ -0,0 +1,95 @@
+package ast
+
+import (
+	"time"
+
+	"github.com/prequel-dev/prequel-compiler/pkg/parser"
+	"github.com/prequel-dev/prequel-compiler/pkg/schema"
+	"github.com/rs/zerolog/log"
+)
+
+type AstScriptT struct {
+	Code     string
+	Language string
+	Timeout  time.Duration
+}
+
+// Build the child Ast nodes for the script.
+//
+// Script nodes are internal nodes with one input node.
+// The parser node for a script contains a ScriptT struct as its first child, followed by one input node.
+// Build the the child nodes for the script node by building each of the parser node's children;
+// the first child is skipped since it is the script definition, and the remaining child is built as the input to the script node.
+
+func (b *builderT) buildScriptChildren(parserNode *parser.NodeT, machineAddress *AstNodeAddressT) ([]*AstNodeT, error) {
+
+	if len(parserNode.Children) != 2 {
+		log.Error().Int("child_count", len(parserNode.Children)).Msg("Script node must have two children")
+		return nil, parserNode.WrapError(ErrInvalidNodeType)
+	}
+
+	termIdx := uint32(1)
+
+	child := parserNode.Children[1]
+	parserChildNode, ok := child.(*parser.NodeT)
+	if !ok {
+		log.Error().Any("child", child).Msg("Failed to build Script child node")
+		return nil, parserNode.WrapError(ErrInvalidNodeType)
+	}
+
+	leaf, err := b.buildLeafChild(parserChildNode, machineAddress, &termIdx)
+
+	switch {
+	case err != nil:
+		return nil, err
+	case leaf != nil:
+		return []*AstNodeT{leaf}, nil
+	default:
+		node, err := b.buildTree(parserChildNode, machineAddress, &termIdx)
+		if err != nil {
+			return nil, err
+		}
+
+		return []*AstNodeT{node}, nil
+	}
+}
+
+// Validate script definitions and build the script node.
+
+func (b *builderT) buildScriptNode(parserNode *parser.NodeT, parentMachineAddress, machineAddress *AstNodeAddressT) (*AstNodeT, error) {
+
+	// Expects exactly two children, the first should be parser.ScriptT, the following is the script input node.
+
+	if len(parserNode.Children) != 2 {
+		log.Error().Int("child_count", len(parserNode.Children)).Msg("Script node must have exactly two children")
+		return nil, parserNode.WrapError(ErrInvalidNodeType)
+	}
+
+	scriptNode, ok := parserNode.Children[0].(*parser.ScriptT)
+
+	if !ok {
+		log.Error().Any("script", parserNode.Children[0]).Msg("Failed to build Script node")
+		return nil, parserNode.WrapError(ErrMissingScalar)
+	}
+
+	if scriptNode.Code == "" {
+		log.Error().Msg("Script code string is empty")
+		return nil, parserNode.WrapError(ErrMissingScalar)
+	}
+
+	pn := &AstScriptT{
+		Code:     scriptNode.Code,
+		Language: scriptNode.Language,
+	}
+
+	if scriptNode.Timeout != nil {
+		pn.Timeout = *scriptNode.Timeout
+	}
+
+	var (
+		node = newAstNode(parserNode, parserNode.Metadata.Type, schema.ScopeCluster, parentMachineAddress, machineAddress)
+	)
+
+	node.Object = pn
+	return node, nil
+}

pkg/ast/ast_test.go

@@ -33,6 +33,9 @@ func gatherNodeAddresses(node *AstNodeT, out *[]string) {
 	}
 
 	*out = append(*out, node.Metadata.Address.String())
+	for _, child := range node.Children {
+		gatherNodeAddresses(child, out)
+	}
 }
 
 func TestAstSuccess(t *testing.T) {
@@ -73,6 +76,14 @@ func TestAstSuccess(t *testing.T) {
 			rule:              testdata.TestSuccessSimplePromQL,
 			expectedNodeTypes: []string{"machine_set", "promql", "log_set"},
 		},
+		"Success_ChildScript": {
+			rule:              testdata.TestSuccessChildScript,
+			expectedNodeTypes: []string{"machine_seq", "script", "log_seq", "log_set"},
+		},
+		"Success_ChildScriptMultipleInputs": {
+			rule:              testdata.TestSuccessChildScriptMultipleInputs,
+			expectedNodeTypes: []string{"machine_set", "script", "machine_seq", "log_seq", "log_set"},
+		},
 	}
 
 	for name, test := range tests {

pkg/parser/parse.go

@@ -5,6 +5,8 @@ import (
 )
 
 // Note that we prefer lower camel case like Kubernetes
+// Also, have to keep the JSON tags although we are using YAML.
+// The hash function uses JSON serialization, so the JSON tags are required to ensure consistent field names for hashing.
 
 const (
 	docRules   = "rules"
@@ -92,19 +94,6 @@ type ParseNegateOptsT struct {
 	Absolute bool   `yaml:"absolute,omitempty"`
 }
 
-type ParseTermT struct {
-	Field      string            `yaml:"field,omitempty"`
-	StrValue   string            `yaml:"value,omitempty"`
-	JqValue    string            `yaml:"jq,omitempty"`
-	RegexValue string            `yaml:"regex,omitempty"`
-	Count      int               `yaml:"count,omitempty"`
-	Set        *ParseSetT        `yaml:"set,omitempty"`
-	Sequence   *ParseSequenceT   `yaml:"sequence,omitempty"`
-	NegateOpts *ParseNegateOptsT `yaml:",inline,omitempty"`
-	PromQL     *ParsePromQL      `yaml:"promql,omitempty"`
-	Extract    []ParseExtractT   `yaml:"extract,omitempty"`
-}
-
 type ParseSetT struct {
 	Window       string       `yaml:"window,omitempty"`
 	Correlations []string     `yaml:"correlations,omitempty"`
@@ -126,23 +115,55 @@ type ParsePromQL struct {
 	Event    *ParseEventT `yaml:"event,omitempty"`
 }
 
+type ParseScriptT struct {
+	Code     string      `yaml:"code"`
+	Language string      `yaml:"language,omitempty"` // Assumes 'lua' if empty
+	Timeout  string      `yaml:"timeout,omitempty"`  // Uses default if empty; expects duration string
+	Input    *ParseTermT `yaml:"input"`              // Required input
+}
+
+type ParseEventT struct {
+	Source string `yaml:"source"`
+	Origin bool   `yaml:"origin,omitempty" json:"origin,omitempty"`
+}
+
+type ParseTermT struct {
+	Field      string            `yaml:"field,omitempty"`
+	StrValue   string            `yaml:"value,omitempty"`
+	JqValue    string            `yaml:"jq,omitempty"`
+	RegexValue string            `yaml:"regex,omitempty"`
+	Count      int               `yaml:"count,omitempty"`
+	Set        *ParseSetT        `yaml:"set,omitempty"`
+	Sequence   *ParseSequenceT   `yaml:"sequence,omitempty"`
+	NegateOpts *ParseNegateOptsT `yaml:",inline,omitempty"`
+	PromQL     *ParsePromQL      `yaml:"promql,omitempty"`
+	Script     *ParseScriptT     `yaml:"script,omitempty"`
+	Extract    []ParseExtractT   `yaml:"extract,omitempty"`
+}
+
 func (o *ParseTermT) UnmarshalYAML(unmarshal func(any) error) error {
+
+	// Try to unmarshal as a raw string first.
+	// If that fails, unmarshal as a struct.
+	// This allows for a shorthand syntax for simple match terms.
 	var str string
 	if err := unmarshal(&str); err == nil {
 		o.StrValue = str
 		return nil
 	}
+
 	var temp struct {
-		Field       string            `yaml:"field,omitempty"`
-		StrValue    string            `yaml:"value,omitempty"`
-		JqValue     string            `yaml:"jq,omitempty"`
-		RegexValue  string            `yaml:"regex,omitempty"`
-		Count       int               `yaml:"count,omitempty"`
-		Set         *ParseSetT        `yaml:"set,omitempty"`
-		Sequence    *ParseSequenceT   `yaml:"sequence,omitempty"`
-		NegateOpts  *ParseNegateOptsT `yaml:",inline,omitempty"`
-		ParsePromQL *ParsePromQL      `yaml:"promql,omitempty"`
-		Extract     []ParseExtractT   `yaml:"extract,omitempty"`
+		Field       string            `yaml:"field"`
+		StrValue    string            `yaml:"value"`
+		JqValue     string            `yaml:"jq"`
+		RegexValue  string            `yaml:"regex"`
+		Count       int               `yaml:"count"`
+		Set         *ParseSetT        `yaml:"set"`
+		Sequence    *ParseSequenceT   `yaml:"sequence"`
+		NegateOpts  *ParseNegateOptsT `yaml:",inline"`
+		ParsePromQL *ParsePromQL      `yaml:"promql"`
+		Script      *ParseScriptT     `yaml:"script"`
+		Extract     []ParseExtractT   `yaml:"extract"`
 	}
 	if err := unmarshal(&temp); err != nil {
 		return err
@@ -156,22 +177,11 @@ func (o *ParseTermT) UnmarshalYAML(unmarshal func(any) error) error {
 	o.Sequence = temp.Sequence
 	o.NegateOpts = temp.NegateOpts
 	o.PromQL = temp.ParsePromQL
+	o.Script = temp.Script
 	o.Extract = temp.Extract
 	return nil
 }
 
-type ParseEventT struct {
-	Source string `yaml:"source"`
-	Origin bool   `yaml:"origin,omitempty" json:"origin,omitempty"`
-}
-
-type RulesT struct {
-	Rules  []ParseRuleT          `yaml:"rules"`
-	Root   *yaml.Node            `yaml:"-"`
-	TermsT map[string]ParseTermT `yaml:"terms,omitempty"`
-	TermsY map[string]*yaml.Node `yaml:"-"`
-}
-
 func RootNode(data []byte) (*yaml.Node, error) {
 	var root yaml.Node
 	if err := yaml.Unmarshal(data, &root); err != nil {
@@ -180,21 +190,25 @@ func RootNode(data []byte) (*yaml.Node, error) {
 	return &root, nil
 }
 
-func _parse(data []byte) (RulesT, *yaml.Node, error) {
+type RulesT struct {
+	Rules  []ParseRuleT          `yaml:"rules"`
+	Root   *yaml.Node            `yaml:"-"`
+	TermsT map[string]ParseTermT `yaml:"terms,omitempty"`
+	TermsY map[string]*yaml.Node `yaml:"-"`
+}
 
-	var (
-		root  yaml.Node
-		rules RulesT
-		err   error
-	)
+func _parse(data []byte) (*RulesT, *yaml.Node, error) {
 
-	if err = yaml.Unmarshal(data, &root); err != nil {
-		return RulesT{}, nil, err
+	root, err := RootNode(data)
+	if err != nil {
+		return nil, nil, err
 	}
 
+	var rules RulesT
 	if err := root.Decode(&rules); err != nil {
-		return RulesT{}, nil, err
+		return nil, nil, err
+
 	}
 
-	return rules, &root, nil
+	return &rules, root, nil
 }