Add grammar generation infrastructure and NFA priority system

ts/packages/actionGrammar/src/agentGrammarRegistry.ts

@@ -46,9 +46,13 @@ export class AgentGrammar {
      * They are merged as alternatives to enable cache hits on similar requests.
      *
      * @param agrText Grammar rules in .agr format (from grammarGenerator)
+     * @param checkedVariables Optional set of variable names with validation (checked_wildcard paramSpec)
      * @returns Success status and any errors
      */
-    addGeneratedRules(agrText: string): {
+    addGeneratedRules(
+        agrText: string,
+        checkedVariables?: Set<string>,
+    ): {
         success: boolean;
         errors: string[];
         unresolvedEntities?: string[];
@@ -71,6 +75,11 @@ export class AgentGrammar {
             };
         }
 
+        // Add checked variables if provided
+        if (checkedVariables && checkedVariables.size > 0) {
+            newGrammar.checkedVariables = checkedVariables;
+        }
+
         // Validate entity references
         const unresolvedEntities = this.validateEntityReferences(newGrammar);
         if (unresolvedEntities.length > 0) {
@@ -95,6 +104,19 @@ export class AgentGrammar {
             mergedGrammar.entities = Array.from(existingEntities);
         }
 
+        // Merge checked variables
+        if (newGrammar.checkedVariables || this.grammar.checkedVariables) {
+            const existingChecked = new Set(
+                this.grammar.checkedVariables || [],
+            );
+            if (newGrammar.checkedVariables) {
+                for (const varName of newGrammar.checkedVariables) {
+                    existingChecked.add(varName);
+                }
+            }
+            mergedGrammar.checkedVariables = existingChecked;
+        }
+
         // Recompile NFA
         try {
             const newNFA = compileGrammarToNFA(
@@ -239,6 +261,10 @@ export interface AgentMatchResult {
     matched: boolean;
     agentId?: string; // Which agent matched
     captures: Map<string, string | number>;
+    // Priority counts for sorting matches
+    fixedStringPartCount?: number;
+    checkedWildcardCount?: number;
+    uncheckedWildcardCount?: number;
     // Debugging info
     attemptedAgents?: string[];
     tokensConsumed?: number | undefined;
@@ -390,6 +416,9 @@ export class AgentGrammarRegistry {
                     matched: true,
                     agentId: id,
                     captures: result.captures,
+                    fixedStringPartCount: result.fixedStringPartCount,
+                    checkedWildcardCount: result.checkedWildcardCount,
+                    uncheckedWildcardCount: result.uncheckedWildcardCount,
                     attemptedAgents,
                     tokensConsumed: result.tokensConsumed,
                 };

ts/packages/actionGrammar/src/generation/index.ts

@@ -37,6 +37,17 @@ import { ClaudeGrammarGenerator, GrammarAnalysis } from "./grammarGenerator.js";
 import { loadSchemaInfo } from "./schemaReader.js";
 import { GrammarTestCase } from "./testTypes.js";
 
+/**
+ * Convert plural parameter names to singular for grammar variable names
+ * e.g., "artists" -> "artist"
+ */
+function getSingularVariableName(paramName: string): string {
+    if (paramName.endsWith("s") && paramName.length > 1) {
+        return paramName.slice(0, -1);
+    }
+    return paramName;
+}
+
 /**
  * Cache population API for agentServer integration
  */
@@ -66,6 +77,8 @@ export interface CachePopulationResult {
     success: boolean;
     // The generated grammar rule text (if successful)
     generatedRule?: string;
+    // Checked variable names (parameters with checked_wildcard paramSpec)
+    checkedVariables?: Set<string>;
     // Reason for rejection (if not successful)
     rejectionReason?: string;
     // The grammar analysis performed
@@ -117,12 +130,33 @@ export async function populateCache(
             schemaInfo,
         );
 
-        return {
+        // Extract checked variables from the action parameters
+        const checkedVariables = new Set<string>();
+        const actionInfo = schemaInfo.actions.get(testCase.action.actionName);
+        if (actionInfo) {
+            for (const [paramName, paramInfo] of actionInfo.parameters) {
+                if (paramInfo.paramSpec === "checked_wildcard") {
+                    // Handle array parameters (convert plural to singular)
+                    const varName = Array.isArray(
+                        testCase.action.parameters[paramName],
+                    )
+                        ? getSingularVariableName(paramName)
+                        : paramName;
+                    checkedVariables.add(varName);
+                }
+            }
+        }
+
+        const result: CachePopulationResult = {
             success: true,
             generatedRule: grammarRule,
             analysis,
             warnings: [],
         };
+        if (checkedVariables.size > 0) {
+            result.checkedVariables = checkedVariables;
+        }
+        return result;
     } catch (error) {
         return {
             success: false,

ts/packages/actionGrammar/src/grammarMetadata.ts

@@ -0,0 +1,90 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import fs from "fs";
+import {
+    fromJSONParsedActionSchema,
+    ParsedActionSchemaJSON,
+    ActionSchemaTypeDefinition,
+} from "@typeagent/action-schema";
+import { Grammar } from "./grammarTypes.js";
+
+/**
+ * Enrich a Grammar with checked variable metadata from a .pas.json schema file
+ *
+ * Reads the schema file to find parameters with checked_wildcard paramSpec
+ * and adds their variable names to grammar.checkedVariables
+ *
+ * @param grammar The grammar to enrich
+ * @param schemaPath Path to the .pas.json file
+ * @returns The enriched grammar (modifies in place and returns for chaining)
+ */
+export function enrichGrammarWithCheckedVariables(
+    grammar: Grammar,
+    schemaPath: string,
+): Grammar {
+    try {
+        const jsonContent = fs.readFileSync(schemaPath, "utf8");
+        const json: ParsedActionSchemaJSON = JSON.parse(jsonContent);
+        const parsedSchema = fromJSONParsedActionSchema(json);
+
+        const checkedVariables = new Set<string>();
+
+        // Process each action to find parameters with checked_wildcard paramSpec
+        for (const [
+            _actionName,
+            actionDef,
+        ] of parsedSchema.actionSchemas.entries()) {
+            const paramSpecs = (actionDef as ActionSchemaTypeDefinition)
+                .paramSpecs;
+
+            if (!paramSpecs || typeof paramSpecs !== "object") {
+                continue;
+            }
+
+            // Find all parameters with checked_wildcard paramSpec
+            for (const [paramName, paramSpec] of Object.entries(paramSpecs)) {
+                if (paramSpec === "checked_wildcard") {
+                    // Add the parameter name as a checked variable
+                    // Also handle array element specs (e.g., "artists.*" -> "artist")
+                    if (paramName.endsWith(".*")) {
+                        const baseName = paramName.slice(0, -2);
+                        // Convert plural to singular for grammar variable names
+                        const singularName = getSingularVariableName(baseName);
+                        checkedVariables.add(singularName);
+                    } else {
+                        checkedVariables.add(paramName);
+                    }
+                }
+            }
+        }
+
+        // Add to grammar
+        if (checkedVariables.size > 0) {
+            grammar.checkedVariables = checkedVariables;
+        }
+
+        return grammar;
+    } catch (error) {
+        // If schema file doesn't exist or can't be read, return grammar unchanged
+        console.warn(
+            `Could not read schema file ${schemaPath} for checked variable metadata: ${error}`,
+        );
+        return grammar;
+    }
+}
+
+/**
+ * Convert plural parameter names to singular for grammar variable names
+ * e.g., "artists" -> "artist", "devices" -> "device"
+ */
+function getSingularVariableName(plural: string): string {
+    if (plural.endsWith("ies")) {
+        return plural.slice(0, -3) + "y";
+    } else if (plural.endsWith("ses") || plural.endsWith("shes")) {
+        return plural.slice(0, -2);
+    } else if (plural.endsWith("s")) {
+        return plural.slice(0, -1);
+    }
+    return plural;
+}

ts/packages/actionGrammar/src/grammarTypes.ts

@@ -55,6 +55,7 @@ export type GrammarRule = {
 export type Grammar = {
     rules: GrammarRule[];
     entities?: string[] | undefined; // Entity types this grammar depends on (e.g. ["Ordinal", "CalendarDate"])
+    checkedVariables?: Set<string> | undefined; // Variable names with validation (checked_wildcard paramSpec)
 };
 
 /**

ts/packages/actionGrammar/src/index.ts

@@ -35,9 +35,19 @@ export {
 } from "./dynamicGrammarLoader.js";
 
 // NFA system
-export type { NFA, NFAState, NFATransition } from "./nfa.js";
-export { matchNFA, type NFAMatchResult } from "./nfaInterpreter.js";
+export type {
+    NFA,
+    NFAState,
+    NFATransition,
+    AcceptStatePriorityHint,
+} from "./nfa.js";
+export {
+    matchNFA,
+    sortNFAMatches,
+    type NFAMatchResult,
+} from "./nfaInterpreter.js";
 export { compileGrammarToNFA } from "./nfaCompiler.js";
+export { enrichGrammarWithCheckedVariables } from "./grammarMetadata.js";
 
 // Agent Grammar Registry
 export type { AgentMatchResult } from "./agentGrammarRegistry.js";

ts/packages/actionGrammar/src/nfa.ts

@@ -28,11 +28,24 @@ export interface NFATransition {
     // For wildcard transitions: metadata about the variable
     variable?: string | undefined;
     typeName?: string | undefined;
+    checked?: boolean | undefined; // true if wildcard has validation (entity type or checked_wildcard paramSpec)
 
     // Target state
     to: number;
 }
 
+/**
+ * Priority hint for an accepting state
+ * Used when multiple grammar rules share an accepting state (e.g., in DFA construction)
+ * Tracks the best-case priority achievable through this state
+ */
+export interface AcceptStatePriorityHint {
+    // Best achievable counts for any path leading to this state
+    minFixedStringPartCount: number; // Highest fixed string count from any rule
+    maxCheckedWildcardCount: number; // Most checked wildcards from any rule
+    minUncheckedWildcardCount: number; // Fewest unchecked wildcards from any rule
+}
+
 /**
  * An NFA state with outgoing transitions
  */
@@ -43,6 +56,10 @@ export interface NFAState {
     // If true, this is an accepting/final state
     accepting: boolean;
 
+    // Optional: Priority hint for accepting states (used in DFA minimization/merging)
+    // When multiple rules merge into one accepting state, this tracks the best possible priority
+    priorityHint?: AcceptStatePriorityHint | undefined;
+
     // Optional: capture variable value when reaching this state
     capture?:
         | {
@@ -88,12 +105,20 @@ export class NFABuilder {
         tokens?: string[],
         variable?: string,
         typeName?: string,
+        checked?: boolean,
     ): void {
         const state = this.states[from];
         if (!state) {
             throw new Error(`State ${from} does not exist`);
         }
-        state.transitions.push({ type, to, tokens, variable, typeName });
+        state.transitions.push({
+            type,
+            to,
+            tokens,
+            variable,
+            typeName,
+            checked,
+        });
     }
 
     addTokenTransition(from: number, to: number, tokens: string[]): void {
@@ -109,8 +134,17 @@ export class NFABuilder {
         to: number,
         variable: string,
         typeName?: string,
+        checked?: boolean,
     ): void {
-        this.addTransition(from, to, "wildcard", undefined, variable, typeName);
+        this.addTransition(
+            from,
+            to,
+            "wildcard",
+            undefined,
+            variable,
+            typeName,
+            checked,
+        );
     }
 
     build(startState: number, name?: string): NFA {