Support rules with a script node interior type.

pkg/ast/ast.go

@@ -106,12 +106,9 @@ func (b *builderT) descendTree(fn func() error) error {
 }
 
 func Build(data []byte) (*AstT, error) {
-	var (
-		parseTree *parser.TreeT
-		err       error
-	)
 
-	if parseTree, err = parser.Parse(data); err != nil {
+	parseTree, err := parser.Parse(data)
+	if err != nil {
 		log.Error().Any("err", err).Msg("Parser failed")
 		return nil, err
 	}
@@ -157,33 +154,17 @@ func BuildTree(tree *parser.TreeT) (*AstT, error) {
 func (b *builderT) buildTree(parserNode *parser.NodeT, parentMachineAddress *AstNodeAddressT, termIdx *uint32) (*AstNodeT, error) {
 
 	var (
-		machineMatchNode *AstNodeT
-		matchNode        *AstNodeT
-		children         = make([]*AstNodeT, 0)
-		machineAddress   = b.newAstNodeAddress(parserNode.Metadata.RuleHash, parserNode.Metadata.Type.String(), termIdx)
-		err              error
+		machineAddress = b.newAstNodeAddress(parserNode.Metadata.RuleHash, parserNode.Metadata.Type.String(), termIdx)
 	)
 
-	// Build children (either matcher children or nested machines)
-	if parserNode.IsMatcherNode() {
-		if matchNode, err = b.buildMatcherChildren(parserNode, machineAddress, termIdx); err != nil {
-			return nil, err
-		}
-		children = append(children, matchNode)
-	} else if parserNode.IsPromNode() {
-		if matchNode, err = b.buildPromQLNode(parserNode, machineAddress, termIdx); err != nil {
-			return nil, err
-		}
-		children = append(children, matchNode)
-
-	} else {
-		if children, err = b.buildMachineChildren(parserNode, machineAddress); err != nil {
-			return nil, err
-		}
+	children, err := b.buildChildrenNodes(parserNode, machineAddress, termIdx)
+	if err != nil {
+		return nil, err
 	}
 
 	// Build state machine after recursively building children
-	if machineMatchNode, err = b.buildStateMachine(parserNode, parentMachineAddress, machineAddress, children); err != nil {
+	machineMatchNode, err := b.buildStateMachine(parserNode, parentMachineAddress, machineAddress, children)
+	if err != nil {
 		return nil, err
 	}
 
@@ -192,6 +173,35 @@ func (b *builderT) buildTree(parserNode *parser.NodeT, parentMachineAddress *Ast
 	return machineMatchNode, nil
 }
 
+func (b *builderT) buildChildrenNodes(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (children []*AstNodeT, err error) {
+
+	leaf, err := b.buildLeafChild(parserNode, machineAddress, termIdx)
+
+	switch {
+	case err != nil:
+		return nil, err
+	case leaf != nil:
+		return []*AstNodeT{leaf}, nil
+	case parserNode.IsScriptNode():
+		children, err = b.buildScriptChildren(parserNode, machineAddress)
+	default:
+		children, err = b.buildMachineChildren(parserNode, machineAddress)
+	}
+
+	return children, err
+}
+
+func (b *builderT) buildLeafChild(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (leaf *AstNodeT, err error) {
+
+	switch {
+	case parserNode.IsMatcherNode():
+		leaf, err = b.buildMatcherChild(parserNode, machineAddress, termIdx)
+	case parserNode.IsPromNode():
+		leaf, err = b.buildPromQLChild(parserNode, machineAddress, termIdx)
+	}
+	return
+}
+
 func (b *builderT) newAstNodeAddress(ruleHash, name string, termIdx *uint32) *AstNodeAddressT {
 	var address = &AstNodeAddressT{
 		Version:  "v" + strconv.FormatInt(int64(AstVersion), 10),
@@ -220,7 +230,7 @@ func newAstNode(parserNode *parser.NodeT, typ schema.NodeTypeT, scope string, pa
 	}
 }
 
-func (b *builderT) buildMatcherChildren(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (*AstNodeT, error) {
+func (b *builderT) buildMatcherChild(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (*AstNodeT, error) {
 
 	var (
 		matchNode *AstNodeT
@@ -238,9 +248,11 @@ func (b *builderT) buildMatcherChildren(parserNode *parser.NodeT, machineAddress
 		return nil, parserNode.WrapError(ErrInvalidEventType)
 	}
 
-	// Implied that the root node has an origin event
-	b.OriginCnt++
-	parserNode.Metadata.Event.Origin = true
+	// This appears to be a legacy hack to support rules that don't specify origin but have event sources.
+	// We should consider removing this and requiring explicit origin specification in the rules.
+	if b.CurrentDepth == 0 && !parserNode.Metadata.Event.Origin {
+		parserNode.Metadata.Event.Origin = true
+	}
 
 	err = b.descendTree(func() error {
 		if matchNode, err = b.buildMatcherNodes(parserNode, machineAddress, termIdx); err != nil {
@@ -319,10 +331,6 @@ func (b *builderT) buildMachineChildren(parserNode *parser.NodeT, machineAddress
 
 		// If the child has an event/data source, then it is not a state machine. Build it via buildMatcherNodes
 
-		if parserChildNode.Metadata.Event.Origin {
-			b.OriginCnt++
-		}
-
 		if parserChildNode.Metadata.Event.Source == "" {
 			log.Error().
 				Any("address", machineAddress).
@@ -360,7 +368,7 @@ func addNegateOpts(assert *AstNodeT, negateOpts *parser.NegateOptsT) {
 	}
 }
 
-func (b *builderT) buildStateMachine(parserNode *parser.NodeT, parentMachineAddress *AstNodeAddressT, machineAddress *AstNodeAddressT, children []*AstNodeT) (*AstNodeT, error) {
+func (b *builderT) buildStateMachine(parserNode *parser.NodeT, parentMachineAddress, machineAddress *AstNodeAddressT, children []*AstNodeT) (*AstNodeT, error) {
 
 	switch parserNode.Metadata.Type {
 	case schema.NodeTypeSeq, schema.NodeTypeLogSeq:
@@ -371,6 +379,9 @@ func (b *builderT) buildStateMachine(parserNode *parser.NodeT, parentMachineAddr
 			return nil, parserNode.WrapError(ErrInvalidWindow)
 		}
 	case schema.NodeTypeSet, schema.NodeTypeLogSet, schema.NodeTypePromQL:
+	case schema.NodeTypeScript:
+		return b.buildScriptNode(parserNode, parentMachineAddress, machineAddress)
+
 	default:
 		log.Error().
 			Any("address", machineAddress).

pkg/ast/ast_log.go

@@ -152,6 +152,10 @@ func (b *builderT) doBuildLogMatcherNode(parserNode *parser.NodeT, machineAddres
 		Correlations: parserNode.Metadata.Correlations,
 	}
 
+	if parserNode.Metadata.Event.Origin {
+		b.OriginCnt++
+	}
+
 	return matchNode, nil
 }
 

pkg/ast/ast_metrics.go

@@ -15,6 +15,22 @@ type AstPromQL struct {
 	Event    *AstEventT
 }
 
+func (b *builderT) buildPromQLChild(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (*AstNodeT, error) {
+	var child *AstNodeT
+
+	err := b.descendTree(func() error {
+		node, err := b.buildPromQLNode(parserNode, machineAddress, termIdx)
+		if err != nil {
+			return err
+		}
+		child = node
+		return nil
+	})
+
+	return child, err
+
+}
+
 func (b *builderT) buildPromQLNode(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (*AstNodeT, error) {
 
 	// Expects one child of type ParsePromQL
@@ -45,6 +61,9 @@ func (b *builderT) buildPromQLNode(parserNode *parser.NodeT, machineAddress *Ast
 			Source: parserNode.Metadata.Event.Source,
 			Origin: parserNode.Metadata.Event.Origin,
 		}
+		if parserNode.Metadata.Event.Origin {
+			b.OriginCnt++
+		}
 	}
 
 	if promNode.Interval != nil {

pkg/ast/ast_script.go

@@ -0,0 +1,101 @@
+package ast
+
+import (
+	"time"
+
+	"github.com/prequel-dev/prequel-compiler/pkg/parser"
+	"github.com/prequel-dev/prequel-compiler/pkg/schema"
+	"github.com/rs/zerolog/log"
+)
+
+type AstScriptT struct {
+	Code     string
+	Language string
+	Timeout  time.Duration
+}
+
+// Build the child Ast nodes for the script.
+//
+// Script nodes are internal nodes with one input node.
+// The parser node for a script contains a ScriptT struct as its first child, followed by one input node.
+// Build the the child nodes for the script node by building each of the parser node's children;
+// the first child is skipped since it is the script definition, and the remaining child is built as the input to the script node.
+
+func (b *builderT) buildScriptChildren(parserNode *parser.NodeT, machineAddress *AstNodeAddressT) ([]*AstNodeT, error) {
+
+	if len(parserNode.Children) != 2 {
+		log.Error().Int("child_count", len(parserNode.Children)).Msg("Script node must have two children")
+		return nil, parserNode.WrapError(ErrInvalidNodeType)
+	}
+
+	termIdx := uint32(1)
+
+	child := parserNode.Children[1]
+	parserChildNode, ok := child.(*parser.NodeT)
+	if !ok {
+		log.Error().Any("child", child).Msg("Failed to build Script child node")
+		return nil, parserNode.WrapError(ErrInvalidNodeType)
+	}
+
+	leaf, err := b.buildLeafChild(parserChildNode, machineAddress, &termIdx)
+
+	var childList []*AstNodeT
+
+	switch {
+	case err != nil:
+		// fallthrough
+	case leaf != nil:
+		childList = []*AstNodeT{leaf}
+	default:
+		err = b.descendTree(func() error {
+			node, err := b.buildTree(parserChildNode, machineAddress, &termIdx)
+			if err != nil {
+				return err
+			}
+			childList = []*AstNodeT{node}
+			return nil
+		})
+	}
+
+	return childList, err
+}
+
+// Validate script definitions and build the script node.
+
+func (b *builderT) buildScriptNode(parserNode *parser.NodeT, parentMachineAddress, machineAddress *AstNodeAddressT) (*AstNodeT, error) {
+
+	// Expects exactly two children, the first should be parser.ScriptT, the following is the script input node.
+
+	if len(parserNode.Children) != 2 {
+		log.Error().Int("child_count", len(parserNode.Children)).Msg("Script node must have exactly two children")
+		return nil, parserNode.WrapError(ErrInvalidNodeType)
+	}
+
+	scriptNode, ok := parserNode.Children[0].(*parser.ScriptT)
+
+	if !ok {
+		log.Error().Any("script", parserNode.Children[0]).Msg("Failed to build Script node")
+		return nil, parserNode.WrapError(ErrMissingScalar)
+	}
+
+	if scriptNode.Code == "" {
+		log.Error().Msg("Script code string is empty")
+		return nil, parserNode.WrapError(ErrMissingScalar)
+	}
+
+	pn := &AstScriptT{
+		Code:     scriptNode.Code,
+		Language: scriptNode.Language,
+	}
+
+	if scriptNode.Timeout != nil {
+		pn.Timeout = *scriptNode.Timeout
+	}
+
+	var (
+		node = newAstNode(parserNode, parserNode.Metadata.Type, schema.ScopeCluster, parentMachineAddress, machineAddress)
+	)
+
+	node.Object = pn
+	return node, nil
+}

pkg/ast/ast_test.go

@@ -33,6 +33,9 @@ func gatherNodeAddresses(node *AstNodeT, out *[]string) {
 	}
 
 	*out = append(*out, node.Metadata.Address.String())
+	for _, child := range node.Children {
+		gatherNodeAddresses(child, out)
+	}
 }
 
 func TestAstSuccess(t *testing.T) {
@@ -73,6 +76,18 @@ func TestAstSuccess(t *testing.T) {
 			rule:              testdata.TestSuccessSimplePromQL,
 			expectedNodeTypes: []string{"machine_set", "promql", "log_set"},
 		},
+		"Success_ChildScript": {
+			rule:              testdata.TestSuccessChildScript,
+			expectedNodeTypes: []string{"machine_seq", "script", "log_seq", "log_set"},
+		},
+		"Success_ChildScriptMultipleInputs": {
+			rule:              testdata.TestSuccessChildScriptMultipleInputs,
+			expectedNodeTypes: []string{"machine_set", "script", "machine_seq", "log_seq", "log_set"},
+		},
+		"Success_ChildScriptPromQLInput": {
+			rule:              testdata.TestSuccessChildScriptPromQLInput,
+			expectedNodeTypes: []string{"machine_set", "script", "promql"},
+		},
 	}
 
 	for name, test := range tests {
@@ -93,6 +108,10 @@ func TestAstSuccess(t *testing.T) {
 				t.Fatalf("No nodes found in AST")
 			}
 
+			if err = validateTree(ast.Nodes[0]); err != nil {
+				t.Fatalf("Error validating tree: %v", err)
+			}
+
 			var actualNodes []string
 			gatherNodeTypes(ast.Nodes[0], &actualNodes)
 
@@ -262,3 +281,45 @@ func TestFailureExamples(t *testing.T) {
 		}
 	}
 }
+
+// Validate the following invariants on the tree:
+// 1. No duplicate addresses
+// 2. Root node has no parent address
+// 3. Node ids are unique
+// 4. Depth is consistent with distance from root
+
+func validateTree(node *AstNodeT) error {
+	if node == nil {
+		return fmt.Errorf("Root node is nil")
+	}
+
+	if node.Metadata.ParentAddress != nil {
+		return fmt.Errorf("Root node has parent address: %s", node.Metadata.ParentAddress.String())
+	}
+
+	return _validateTree(node, 0, make(map[uint32]struct{})) // start at depth 0 for root
+}
+
+func _validateTree(node *AstNodeT, depth uint32, ids map[uint32]struct{}) error {
+
+	if node == nil {
+		return nil
+	}
+
+	if node.Metadata.Address.Depth != depth {
+		return fmt.Errorf("Node %s has depth %d, expected %d", node.Metadata.Address.String(), node.Metadata.Address.Depth, depth)
+	}
+
+	if _, exists := ids[node.Metadata.Address.NodeId]; exists {
+		return fmt.Errorf("Duplicate node ID %d found", node.Metadata.Address.NodeId)
+	}
+	ids[node.Metadata.Address.NodeId] = struct{}{}
+
+	for _, child := range node.Children {
+		if err := _validateTree(child, depth+1, ids); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}