Sync repos: Release 170.121.2

CONTRIBUTING.md

@@ -117,23 +117,37 @@ Before sending a Pull Request, please do the following:
 
 ### Helpful notes for SQLDOM extensions
 
-1. For changing the DOM classes, modify the XML file (the C# code is generated based on this during the build process) `$(EnlistmentRoot)\Source\SqlDom\SqlScriptDom\Parser\TSql\Ast.xml`. Change Ast.xml to put the class pieces on their appropriate statements.
-    1. The build process is defined in `$(EnlistmentRoot)\Source\SqlDom\SqlScriptDom\SqlScriptDom.props` (Target Name="CreateAST")
-    2. The generated files are dropped in `$(EnlistmentRoot)\obj\<x64|x86>\<Debug|Release>\SqlScriptDom.csproj\`
+1. For changing the DOM classes, modify the XML file (the C# code is generated based on this during the build process) `SqlScriptDom\Parser\TSql\Ast.xml`. Change Ast.xml to put the class pieces on their appropriate statements.
+    1. The build process is defined in `SqlScriptDom\GenerateFiles.props` (Target Name="CreateAST")
+    2. The generated files are dropped in `obj\SqlScriptDom\AnyCPU\<Debug|Release>\<TargetPlatform>\Microsoft.SqlServer.TransactSql.ScriptDom.csproj\`
+
+    Regenerating generated sources (what to run and when)
+    ---------------------------------------------------
+    When you modify `Source\SqlDom\SqlScriptDom\Parser\TSql\Ast.xml` or any `TSql<#>.g` grammar file, the C# parser and DOM sources are produced by MSBuild generation targets (for example `CreateAST`). These targets are invoked automatically during a normal build, so in most cases you can simply run:
+
+    ```powershell
+    dotnet build Source\SqlDom\SqlScriptDom\Microsoft.SqlServer.TransactSql.ScriptDom.csproj -c Debug
+    ```
+
+    If you only want to run generation targets (no compile) or need more detailed generation logs, invoke the MSBuild targets directly:
+
+    ```powershell
+    dotnet msbuild Source\SqlDom\SqlScriptDom\Microsoft.SqlServer.TransactSql.ScriptDom.csproj -t:GLexerParserCompile;GSqlTokenTypesCompile;CreateAST -p:Configuration=Debug
+    ```
+
+    Generated files are written into the `obj` folder for that project (for example `obj\SqlScriptDom\AnyCPU\<Debug|Release>\<TargetPlatform>\Microsoft.SqlServer.TransactSql.ScriptDom.csproj\`). If antlr or related tools are missing, see `Directory.Build.props` for `AntlrLocation` and follow the repo guidance to supply the binaries.
 
 2. For changing the parser, modify the .g file here:
-`$(EnlistmentRoot)\Source\SqlDom\SqlScriptDom\Parser\TSql\TSql<#>.g` where # is the version (ie - 100, 120, 130). This will usually be the latest number if adding new grammar. Change the Tsql(xxx).g file to parse the new syntax.
-    1. The build process is defined in `$(EnlistmentRoot)\Source\SqlDom\SqlScriptDom\SqlScriptDom.props` (Target Name="CreateAST")
-    2. The generated files are dropped in `$(EnlistmentRoot)\obj\x86|x64\Debug|Release\sqlscriptdom.csproj\`
+`SqlScriptDom\Parser\TSql\TSql<#>.g` where # is the version (ie - 100, 120, 130). This will usually be the latest number if adding new grammar. Change the Tsql(xxx).g file to parse the new syntax.
+    1. The build process is defined in `SqlScriptDom\GenerateFiles.props` (Target Name="CreateAST")
+    2. The generated files are dropped in `obj\SqlScriptDom\AnyCPU\<Debug|Release>\<TargetPlatform>\Microsoft.SqlServer.TransactSql.ScriptDom.csproj\`
 
-3. For changing the ScriptGenerator, modify the appropriate file (i.e. Visitor that accepts the modified DOM class) in here: `$(EnlistmentRoot)\Source\SqlDom\SqlScriptDom\ScriptDom\SqlServer\ScriptGenerator`.
-    1. To add a new ScriptGenerator, you need to add the file to `$(EnlistmentRoot)\Source\SqlDom\SqlScriptDom\SqlScriptDom.props`
+3. For changing the ScriptGenerator, modify the appropriate file (i.e. Visitor that accepts the modified DOM class) in here: `SqlScriptDom\ScriptDom\SqlServer\ScriptGenerator`.
     1. Change The visitors SqlScriptGenerator.X to use the new piece from AST.XML
     1. If you're adding syntax that's Azure-only or Standalone-only, implement appropriate Versioning Visitor for your constructs.
 
-4. When adding/removing new files please add/remove an entry to/from `$(EnlistmentRoot)\Source\SqlDom\SqlScriptDom\SqlScriptDom.csproj` 
 
-5. To extend the tests do the following:
+4. To extend the tests do the following:
     1. Baselines# needs to be updated or added with the appropriate .sql file as expected results.
     1. The Only#SyntaxTests.cs needs to be extended or added to specify the appropriate TestScripts script.
     1. Positive tests go in Only#SyntaxTests.cs if adding new grammar.

SqlScriptDom/Parser/TSql/Ast.xml

@@ -271,6 +271,11 @@
                     Summary="The xml data type option."/>
     <Member Name="XmlSchemaCollection" Type="SchemaObjectName" Summary="The xml schema collection. Optional may be null."/>
   </Class>
+  <Class Name="VectorDataTypeReference" Base="DataTypeReference" Summary="Represents vector data types">
+    <InheritedClass Name="DataTypeReference" />
+    <Member Name="Dimension" Type="IntegerLiteral" Summary="The dimension of the vector."/>
+    <Member Name="BaseType" Type="Identifier" Summary="Type of dimension values."/>
+  </Class>
   <Class Name="ScalarFunctionReturnType" Base="FunctionReturnType" Summary="The return type definition for scalar-valued functions">
     <Member Name="DataType" Type="DataTypeReference" Summary="The data type of the return."/>
   </Class>
@@ -2520,6 +2525,10 @@
     <InheritedClass Name="DatabaseOption" />
     <Member Name="OptionState" Type="OptionState" GenerateUpdatePositionInfoCall="false" Summary="Option state."/>
   </Class>
+  <Class Name="OptimizedLockingDatabaseOption" Base="DatabaseOption" Summary="OPTIMIZED_LOCKING option in ALTER DATABASE statement, SET case">
+    <InheritedClass Name="DatabaseOption" />
+    <Member Name="OptionState" Type="OptionState" GenerateUpdatePositionInfoCall="false" Summary="Option state."/>
+  </Class>
   <Class Name="QueryStoreDatabaseOption" Base="DatabaseOption" Summary="Query Store (QDS) option in ALTER DATABASE statement, SET case">
     <InheritedClass Name="DatabaseOption" />
     <Member Name="Clear" Type="bool" Summary="True if Clear QDS option was specified"/>

SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs

@@ -427,6 +427,8 @@ internal static class CodeGenerationSupporter
         internal const string FlushIntervalSecondsAlt = "DATA_FLUSH_INTERVAL_SECONDS";
         internal const string Fn = "FN";
         internal const string Float = "FLOAT";
+        internal const string Float16 = "FLOAT16";
+        internal const string Float32 = "FLOAT32";
         internal const string For = "FOR";
         internal const string ForceFailoverAllowDataLoss = "FORCE_FAILOVER_ALLOW_DATA_LOSS";
         internal const string ForceScan = "FORCESCAN";
@@ -718,6 +720,8 @@ internal static class CodeGenerationSupporter
         internal const string OperatorAudit = "OPERATOR_AUDIT";
         internal const string Optimistic = "OPTIMISTIC";
         internal const string Optimize = "OPTIMIZE";
+        internal const string OptimizedLocking = "OPTIMIZED_LOCKING";
+        internal const string OptimizeForArraySearch = "OPTIMIZE_FOR_ARRAY_SEARCH";
         internal const string OptimizeForSequentialKey = "OPTIMIZE_FOR_SEQUENTIAL_KEY";
         internal const string OptimizerQueue = "OPTIMIZER_QUEUE";
         internal const string Order = "ORDER";

SqlScriptDom/Parser/TSql/DatabaseOptionKind.cs

@@ -98,7 +98,8 @@ public enum DatabaseOptionKind
         Ledger                      = 68,
 
         ManualCutover               = 69,
-        PerformCutover              = 70
+        PerformCutover              = 70,
+        OptimizedLocking            = 71
     }
 
 #pragma warning restore 1591

SqlScriptDom/Parser/TSql/IndexOptionHelper.cs

@@ -46,6 +46,8 @@ private IndexOptionHelper()
             AddOptionMapping(IndexOptionKind.VectorMetric, CodeGenerationSupporter.Metric, SqlVersionFlags.TSql170AndAbove);
             AddOptionMapping(IndexOptionKind.VectorType, CodeGenerationSupporter.Type, SqlVersionFlags.TSql170AndAbove);
 
+            AddOptionMapping(IndexOptionKind.OptimizeForArraySearch, CodeGenerationSupporter.OptimizeForArraySearch, SqlVersionFlags.TSql170AndAbove);
+
         }
 
         internal static readonly IndexOptionHelper Instance = new IndexOptionHelper();

SqlScriptDom/Parser/TSql/IndexOptionKind.cs

@@ -40,6 +40,7 @@ public enum IndexOptionKind
         XmlCompression      = 23,
         VectorMetric        = 24,
         VectorType          = 25,
+        OptimizeForArraySearch = 26,
     }
 
 #pragma warning restore 1591

SqlScriptDom/Parser/TSql/TSql170.g

@@ -3064,17 +3064,14 @@ alterDbModify returns [AlterDatabaseStatement vResult = null]
     ;
 
 alterDbModifyAzureOptions returns [AlterDatabaseSetStatement vResult = FragmentFactory.CreateFragment<AlterDatabaseSetStatement>()]
-{
-    bool hasManualCutover = false;
-}
     :
         azureOptions[vResult, vResult.Options]
         (
             With tManualCutover:Identifier
             {
                 Match(tManualCutover, CodeGenerationSupporter.ManualCutover);
-                hasManualCutover = true;
                 vResult.WithManualCutover = true;
+                UpdateTokenInfo(vResult, tManualCutover);
             }
         )?
     ;
@@ -3247,6 +3244,8 @@ dbOptionStateItem[ref ulong encounteredOptions] returns [DatabaseOption vResult
         vResult = changeTrackingDbOption
     |    {NextTokenMatches(CodeGenerationSupporter.AcceleratedDatabaseRecovery)}?
         vResult = acceleratedDatabaseRecoveryOption
+    |    {NextTokenMatches(CodeGenerationSupporter.OptimizedLocking)}?
+        vResult = optimizedLockingOption
     |    {NextTokenMatches(CodeGenerationSupporter.Containment)}?
         vResult = dbContainmentOption
     |    {NextTokenMatches(CodeGenerationSupporter.Hadr)}?
@@ -3560,6 +3559,30 @@ acceleratedDatabaseRecoveryOption returns [AcceleratedDatabaseRecoveryDatabaseOp
         )
     ;
 
+optimizedLockingOption returns [OptimizedLockingDatabaseOption vResult = FragmentFactory.CreateFragment<OptimizedLockingDatabaseOption>()]
+    : tOptimizedLocking:Identifier
+        {
+            Match(tOptimizedLocking, CodeGenerationSupporter.OptimizedLocking);
+            vResult.OptionKind = DatabaseOptionKind.OptimizedLocking;
+            UpdateTokenInfo(vResult, tOptimizedLocking);
+        }
+        (
+            (EqualsSign tOff:Off
+                {
+                    vResult.OptionState = OptionState.Off;
+                    UpdateTokenInfo(vResult, tOff);
+                }
+            )
+        |
+            (EqualsSign tOn:On
+                {
+                    vResult.OptionState = OptionState.On;
+                    UpdateTokenInfo(vResult, tOn);
+                }
+            )
+        )
+    ;
+
 changeTrackingOnOptions [ChangeTrackingDatabaseOption vParent]
 {
     bool autoCleanupEncountered = false;
@@ -23930,7 +23953,6 @@ dropSecurityPolicyStatement returns [DropSecurityPolicyStatement vResult = Fragm
 createExternalModelStatement returns [CreateExternalModelStatement vResult = FragmentFactory.CreateFragment<CreateExternalModelStatement>()]
 {
     Identifier vName;
-    long encounteredOptions = 0;
 }
     :   tModel:Identifier vName = identifier
         {
@@ -24092,7 +24114,6 @@ externalModelParameters[ExternalModelStatement vParent]
 alterExternalModelStatement returns [AlterExternalModelStatement vResult = FragmentFactory.CreateFragment<AlterExternalModelStatement>()]
 {
     Identifier vName;
-    long encounteredOptions = 0;
 }
     :   tModel:Identifier vName = identifier
         {
@@ -30680,6 +30701,32 @@ xmlDataType [SchemaObjectName vName] returns [XmlDataTypeReference vResult = Fra
         )?
     ;
 
+vectorDataType [SchemaObjectName vName] returns [VectorDataTypeReference vResult = FragmentFactory.CreateFragment<VectorDataTypeReference>()]
+{
+    vResult.Name = vName;
+    vResult.UpdateTokenInfo(vName);
+
+    IntegerLiteral vDimension = null;
+    Identifier vBaseType = null;
+}
+    :
+        (   LeftParenthesis vDimension=integer
+            {
+                vResult.Dimension = vDimension;
+            }
+            (
+                Comma vBaseType=identifier
+                {
+                    vResult.BaseType = vBaseType;
+                }
+            )?
+            tRParen:RightParenthesis
+            {
+                UpdateTokenInfo(vResult,tRParen);
+            }
+        )
+    ;
+
 scalarDataType returns [DataTypeReference vResult = null]
 {
     SchemaObjectName vName;
@@ -30700,6 +30747,9 @@ scalarDataType returns [DataTypeReference vResult = null]
             (
                 {isXmlDataType}?
                 vResult = xmlDataType[vName]
+            |
+                {typeOption == SqlDataTypeOption.Vector}?
+                vResult = vectorDataType[vName]
             |
                 {typeOption != SqlDataTypeOption.None}?
                 vResult = sqlDataTypeWithoutNational[vName, typeOption]
@@ -32103,6 +32153,7 @@ aiGenerateEmbeddingsFunctionCall
     ScalarExpression vInput;
     SchemaObjectName vModelName;
     ScalarExpression vParams = null;
+    ScalarExpression vParamsInner;
 }
     :
         tFunc:Identifier LeftParenthesis
@@ -32114,30 +32165,105 @@ aiGenerateEmbeddingsFunctionCall
         {
             vResult.Input = vInput;
         }
-
-        tUse:Use                         // your reserved keyword
+        tUse:Use
         {
             UpdateTokenInfo(vResult, tUse);
         }
 
         tModel:Identifier
         {
             Match(tModel, CodeGenerationSupporter.Model);
-        }
-
-        vModelName=schemaObjectThreePartName
-        {
-            vResult.ModelName = vModelName;
-        }
-
-        (
-            tParams:Identifier
+            UpdateTokenInfo(vResult, tModel);
+        }
+
+        // --- MODEL NAME: single-part only (strict) ---------------------------------------------
+        // We accept exactly **one identifier token** after "USE MODEL".
+        //
+        // Why:
+        //   - Users may store model names that *visually* contain dots or spaces, e.g. [dbo.MyDefaultModel].
+        //     When bracket-delimited, the lexer emits this as a **single** token (QuotedIdentifier), so it's OK.
+        //   - True multipart names (db.schema.model) must be rejected here, so we do NOT consume any Dot tokens.
+        //
+        // Allowed (single token):
+        //   USE MODEL MyDefaultModel            -- Identifier
+        //   USE MODEL [dbo.MyDefaultModel]      -- QuotedIdentifier (one token; dot lives inside the brackets)
+        //
+        // Rejected (multipart):
+        //   USE MODEL dbo.MyDefaultModel        -- Identifier '.' Identifier  (two tokens + Dot)
+        //   USE MODEL [dbo].[MyDefaultModel]    -- QuotedIdentifier '.' QuotedIdentifier
+        //
+        // Token notes:
+        //   - Identifier         : unquoted identifier; cannot contain spaces or '.'.
+        //   - QuotedIdentifier   : bracket-delimited; may contain spaces and '.' inside the brackets.
+        (
+            vModelId:Identifier
+            {
+                vModelName = this.FragmentFactory.CreateFragment<SchemaObjectName>();
+                vModelName.Identifiers.Add(this.CreateIdentifierFromToken(vModelId));
+                vResult.ModelName = vModelName;
+            }
+          |
+            vModelQId:QuotedIdentifier
             {
-                Match(tParams, CodeGenerationSupporter.Parameters);
+                vModelName = this.FragmentFactory.CreateFragment<SchemaObjectName>();
+                vModelName.Identifiers.Add(this.CreateIdentifierFromToken(vModelQId));
+                vResult.ModelName = vModelName;
             }
-            LeftParenthesis
+        )
+
+        // --- Optional PARAMETERS clause ---------------------------------------------------------
+        // Shape:  [PARAMETERS (<expr>)] | [PARAMETERS <expr-not-string>]
+        // Goals:
+        //   1) Accept a general **expression** as the PARAMETERS value.
+        //   2) Preserve user-written parentheses by constructing a ParenthesisExpression node
+        //      for the "( <expr> )" variant so pretty-printing round-trips exactly.
+        //   3) **Reject** a bare JSON string literal (e.g., PARAMETERS '{...}'); callers must pass
+        //      a parsed JSON expression (e.g., TRY_CONVERT(JSON, N'{}')).
+        // Notes:
+        //   - Some builds tokenize PARAMETERS as a keyword; others as an Identifier. Support both.
+        (
+            (
+                // PARAMETERS as a true keyword token.
+                tParamsKw:Parameters
+                {
+                    UpdateTokenInfo(vResult, tParamsKw);
+                }
+              |
+                // PARAMETERS as an identifier token; enforce its text equals "Parameters".
+                tParams:Identifier
+                {
+                    Match(tParams, CodeGenerationSupporter.Parameters);
+                    UpdateTokenInfo(vResult, tParams);
+                }
+            )
+
+            // ---- Value of PARAMETERS -----------------------------------------------------------
+            (
+                // Variant A: user wrote parentheses around the value: PARAMETERS ( <expr> )
+                // Build a ParenthesisExpression so the printer re-emits parens.
+                tLP:LeftParenthesis
+                    vParamsInner=expression
+                tRP:RightParenthesis
+                {
+                    ParenthesisExpression p = this.FragmentFactory.CreateFragment<ParenthesisExpression>();
+                    p.Expression = vParamsInner;
+                    vParams = p;
+
+                    // Attach LP/RP token info for accurate script generation.
+                    UpdateTokenInfo(p, tLP);
+                    UpdateTokenInfo(p, tRP);
+                }
+              |
+                // Variant B: bare expression without surrounding parentheses.
+                // Guardrail: Disallow a leading string literal so that
+                //   PARAMETERS '{"dimensions":768}'
+                // is a **syntax error**, while
+                //   PARAMETERS TRY_CONVERT(JSON, N'{}')
+                // is allowed.
+                // LA(1) is the next token type; block both ASCII ('...') and Unicode (N'...') strings.
+                { LA(1) != AsciiStringLiteral && LA(1) != UnicodeStringLiteral }?
                 vParams=expression
-            RightParenthesis
+            )
             {
                 vResult.OptionalParameters = vParams;
             }