From 8b0f33b3549e58e63783ad7e8f73ccfc68df6396 Mon Sep 17 00:00:00 2001 From: Steen Larsen Date: Tue, 30 Jun 2026 11:38:57 +0200 Subject: [PATCH 1/7] Make AssocTable cache initial capacity configurable (#54) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AssocTable's lookup cache was created as Dictionary<_,_>(2000). Two AssocTables (action + goto) are constructed on every Tables.Interpret call, so a parser invoked over many small inputs pre-allocated two ~2000-entry dictionaries per parse that never came close to filling — dominating parse-time allocation. Expose ParseSettings.AssocTableCacheInitialCapacity (default 2000, so existing behaviour is unchanged) and use it for the cache's initial capacity. Setting it to 0 grows the cache on demand, allocating only what a parse actually uses. Addresses the allocation half of #54 without changing default behaviour. --- src/FsLexYacc.Runtime/Parsing.fs | 16 ++++++++++++++-- src/FsLexYacc.Runtime/Parsing.fsi | 8 ++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/FsLexYacc.Runtime/Parsing.fs b/src/FsLexYacc.Runtime/Parsing.fs index f66aa7a..b1d2e39 100644 --- a/src/FsLexYacc.Runtime/Parsing.fs +++ b/src/FsLexYacc.Runtime/Parsing.fs @@ -117,7 +117,15 @@ module Flags = let mutable debug = false #endif -module Implementation = +/// Settings that tune the parser runtime. +module ParseSettings = + /// Initial capacity of the per-AssocTable lookup cache allocated on each parse. The default + /// (2000) preserves historical behaviour. Set to 0 to grow the cache on demand, which greatly + /// reduces allocation for parsers invoked over many small inputs (issue #54), at the cost of a + /// few dictionary resizes for very large single parses. Set once at startup, before parsing. + let mutable AssocTableCacheInitialCapacity = 2000 + +module Implementation = // Definitions shared with fsyacc let anyMarker = 0xffff @@ -134,7 +142,11 @@ module Implementation = // Read the tables written by FSYACC. type AssocTable(elemTab:uint16[], offsetTab:uint16[]) = - let cache = Dictionary<_,_>(2000) + // Cache capacity is configurable (issue #54): two AssocTables are constructed per Interpret + // call, so the historical fixed 2000-capacity dominated allocation for parsers run over many + // small inputs. Default is 2000 (unchanged); set ParseSettings.AssocTableCacheInitialCapacity + // to 0 to grow on demand instead. + let cache = Dictionary<_,_>(ParseSettings.AssocTableCacheInitialCapacity) member t.readAssoc (minElemNum,maxElemNum,defaultValueOfAssoc,keyToFind) = // do a binary chop on the table diff --git a/src/FsLexYacc.Runtime/Parsing.fsi b/src/FsLexYacc.Runtime/Parsing.fsi index 9cf7720..cbc12c7 100644 --- a/src/FsLexYacc.Runtime/Parsing.fsi +++ b/src/FsLexYacc.Runtime/Parsing.fsi @@ -118,6 +118,14 @@ exception Accept of obj /// Indicates a parse error has occured and parse recovery is in progress exception RecoverableParseError +/// Settings that tune the parser runtime. +module ParseSettings = + /// Initial capacity of the per-AssocTable lookup cache allocated on each parse. The default + /// (2000) preserves historical behaviour. Set to 0 to grow the cache on demand, which greatly + /// reduces allocation for parsers invoked over many small inputs (issue #54), at the cost of a + /// few dictionary resizes for very large single parses. Set once at startup, before parsing. + val mutable AssocTableCacheInitialCapacity: int + #if __DEBUG module internal Flags = val mutable debug: bool From 7c6fa4794c11b7b54c2c0d2313b4927e39d19f12 Mon Sep 17 00:00:00 2001 From: Steen Larsen Date: Tue, 30 Jun 2026 12:06:26 +0200 Subject: [PATCH 2/7] Add release notes entry for #54 --- RELEASE_NOTES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index f25804a..d145ba2 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,6 @@ #### 11.3.0 - Unreleased * Add Fable support to FsLexYacc.Runtime. +* Make the AssocTable lookup cache initial capacity configurable to avoid pre-allocating 2000 entries per parse #54 #### 11.2.0 - 12 May, 2023 * Add `--open` option for fslex. From 444c880a665509df07942472b1b51f9d07b78b4a Mon Sep 17 00:00:00 2001 From: Steen Larsen Date: Tue, 30 Jun 2026 13:03:38 +0200 Subject: [PATCH 3/7] Bump release notes to 11.4.0 (11.3.0 is already released) --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index d145ba2..cb04c20 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,4 +1,4 @@ -#### 11.3.0 - Unreleased +#### 11.4.0 - Unreleased * Add Fable support to FsLexYacc.Runtime. * Make the AssocTable lookup cache initial capacity configurable to avoid pre-allocating 2000 entries per parse #54 From ba25eb8e4bdbabf3ef7c0f8ede7e114719dddd50 Mon Sep 17 00:00:00 2001 From: Steen Larsen Date: Tue, 30 Jun 2026 14:28:32 +0200 Subject: [PATCH 4/7] Add Interpret overload with explicit AssocTable cache capacity (#54) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a Tables.Interpret overload that takes an explicit initial capacity for the per-parse AssocTable lookup caches, threading it through the interpreter (AssocTable now receives the capacity as a constructor argument). The existing 3-argument Interpret keeps using ParseSettings.AssocTableCacheInitialCapacity, so this is non-breaking — existing generated parsers are unaffected. This is the runtime support for letting fsyacc bake a per-parser capacity into generated code (follows in a separate commit). --- src/FsLexYacc.Runtime/Parsing.fs | 20 +++++++++++--------- src/FsLexYacc.Runtime/Parsing.fsi | 5 +++++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/FsLexYacc.Runtime/Parsing.fs b/src/FsLexYacc.Runtime/Parsing.fs index b1d2e39..9e8216b 100644 --- a/src/FsLexYacc.Runtime/Parsing.fs +++ b/src/FsLexYacc.Runtime/Parsing.fs @@ -141,12 +141,12 @@ module Implementation = //------------------------------------------------------------------------- // Read the tables written by FSYACC. - type AssocTable(elemTab:uint16[], offsetTab:uint16[]) = + type AssocTable(elemTab:uint16[], offsetTab:uint16[], initialCacheCapacity:int) = // Cache capacity is configurable (issue #54): two AssocTables are constructed per Interpret // call, so the historical fixed 2000-capacity dominated allocation for parsers run over many - // small inputs. Default is 2000 (unchanged); set ParseSettings.AssocTableCacheInitialCapacity - // to 0 to grow on demand instead. - let cache = Dictionary<_,_>(ParseSettings.AssocTableCacheInitialCapacity) + // small inputs. The capacity is supplied by the caller (Tables.Interpret), defaulting to + // ParseSettings.AssocTableCacheInitialCapacity (2000) unless an explicit value is passed. + let cache = Dictionary<_,_>(initialCacheCapacity) member t.readAssoc (minElemNum,maxElemNum,defaultValueOfAssoc,keyToFind) = // do a binary chop on the table @@ -215,7 +215,7 @@ module Implementation = new(value,startPos,endPos) = { value=value; startPos=startPos; endPos=endPos } - let interpret (tables: Tables<'tok>) lexer (lexbuf : LexBuffer<_>) initialState = + let interpret (tables: Tables<'tok>) lexer (lexbuf : LexBuffer<_>) initialState (assocCacheInitialCapacity: int) = let localStore = Dictionary() in localStore.["LexBuffer"] <- lexbuf #if __DEBUG @@ -259,8 +259,8 @@ module Implementation = let lhsPos = (Array.zeroCreate 2 : Position[]) let reductions = tables.reductions - let actionTable = AssocTable(tables.actionTableElements, tables.actionTableRowOffsets) - let gotoTable = AssocTable(tables.gotos, tables.sparseGotoTableRowOffsets) + let actionTable = AssocTable(tables.actionTableElements, tables.actionTableRowOffsets, assocCacheInitialCapacity) + let gotoTable = AssocTable(tables.gotos, tables.sparseGotoTableRowOffsets, assocCacheInitialCapacity) let stateToProdIdxsTable = IdxToIdxListTable(tables.stateToProdIdxsTableElements, tables.stateToProdIdxsTableRowOffsets) let parseState = @@ -506,8 +506,10 @@ module Implementation = valueStack.Peep().value type Tables<'tok> with - member tables.Interpret (lexer,lexbuf,startState) = - Implementation.interpret tables lexer lexbuf startState + member tables.Interpret (lexer,lexbuf,startState) = + Implementation.interpret tables lexer lexbuf startState ParseSettings.AssocTableCacheInitialCapacity + member tables.Interpret (lexer,lexbuf,startState,assocTableCacheInitialCapacity) = + Implementation.interpret tables lexer lexbuf startState assocTableCacheInitialCapacity module ParseHelpers = let parse_error (_s:string) = () diff --git a/src/FsLexYacc.Runtime/Parsing.fsi b/src/FsLexYacc.Runtime/Parsing.fsi index cbc12c7..a8975e1 100644 --- a/src/FsLexYacc.Runtime/Parsing.fsi +++ b/src/FsLexYacc.Runtime/Parsing.fsi @@ -113,6 +113,11 @@ type Tables<'tok> = /// Returns an object indicating the final synthesized value for the parse. member Interpret: lexer: (LexBuffer<'char> -> 'tok) * lexbuf: LexBuffer<'char> * startState: int -> obj + /// As Interpret, but with an explicit initial capacity for the per-parse AssocTable lookup caches + /// (issue #54); 0 grows them on demand. Overrides ParseSettings.AssocTableCacheInitialCapacity for + /// this parser. fsyacc emits a call to this overload when --assoc-cache-capacity is passed. + member Interpret: lexer: (LexBuffer<'char> -> 'tok) * lexbuf: LexBuffer<'char> * startState: int * assocTableCacheInitialCapacity: int -> obj + /// Indicates an accept action has occured exception Accept of obj /// Indicates a parse error has occured and parse recovery is in progress From fd08b0e8a06278bfc02fed458f2b4c1cae6f2ea0 Mon Sep 17 00:00:00 2001 From: Steen Larsen Date: Wed, 1 Jul 2026 06:49:32 +0200 Subject: [PATCH 5/7] Add --assoc-cache-capacity CLI option for fsyacc Wire the runtime AssocTable cache-capacity control (issue #54) through to the fsyacc command line, since fslex/fsyacc are most commonly driven via CLI from MSBuild rather than by setting the runtime knob in code. - fsyacc.fs: new `--assoc-cache-capacity ` arg, forwarded into GeneratorState.assocCacheCapacity. - fsyaccdriver.fs: when the capacity is set, emit the 4-arg Interpret overload; otherwise emit the parameterless overload (unchanged default behaviour). - FsYacc.Core.Tests: codegen tests asserting the generated engine line uses the capacity-carrying overload when the flag is given and the default overload when it is not. Sequenced, since fsyacc's spec compiler uses process-global state. Usage from MSBuild: --assoc-cache-capacity 0 --- RELEASE_NOTES.md | 1 + src/FsYacc.Core/fsyaccdriver.fs | 9 ++- src/FsYacc/fsyacc.fs | 7 ++ tests/FsYacc.Core.Tests/AssocCacheCapacity.fs | 65 +++++++++++++++++++ .../FsYacc.Core.Tests.fsproj | 1 + 5 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 tests/FsYacc.Core.Tests/AssocCacheCapacity.fs diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index cb04c20..358c0fc 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,6 +1,7 @@ #### 11.4.0 - Unreleased * Add Fable support to FsLexYacc.Runtime. * Make the AssocTable lookup cache initial capacity configurable to avoid pre-allocating 2000 entries per parse #54 +* Add `--assoc-cache-capacity` option for fsyacc to set the generated parser's AssocTable cache capacity from the command line #54 #### 11.2.0 - 12 May, 2023 * Add `--open` option for fslex. diff --git a/src/FsYacc.Core/fsyaccdriver.fs b/src/FsYacc.Core/fsyaccdriver.fs index de61173..81ba42f 100644 --- a/src/FsYacc.Core/fsyaccdriver.fs +++ b/src/FsYacc.Core/fsyaccdriver.fs @@ -178,6 +178,10 @@ type GeneratorState = map_action_to_int: Action -> int anyMarker: int bufferTypeArgument: string + /// When set, the generated parser passes this initial capacity to the runtime's per-parse + /// AssocTable lookup caches (issue #54); 0 grows them on demand. None emits the default call, + /// which uses ParseSettings.AssocTableCacheInitialCapacity at runtime. + assocCacheCapacity: int option } static member Default = @@ -196,6 +200,7 @@ type GeneratorState = map_action_to_int = actionCoding anyMarker = anyMarker bufferTypeArgument = "'cty" + assocCacheCapacity = None } let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compiledSpec: CompiledSpec) = @@ -649,7 +654,9 @@ let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compile writer.WriteLine " numTerminals = %d;" (Array.length compiledSpec.actionTable.[0]) writer.WriteLine " productionToNonTerminalTable = _fsyacc_productionToNonTerminalTable }" - writer.WriteLine "let engine lexer lexbuf startState = tables.Interpret(lexer, lexbuf, startState)" + match generatorState.assocCacheCapacity with + | Some n -> writer.WriteLine "let engine lexer lexbuf startState = tables.Interpret(lexer, lexbuf, startState, %d)" n + | None -> writer.WriteLine "let engine lexer lexbuf startState = tables.Interpret(lexer, lexbuf, startState)" for id, startState in List.zip spec.StartSymbols compiledSpec.startStates do if not (types.ContainsKey id) then diff --git a/src/FsYacc/fsyacc.fs b/src/FsYacc/fsyacc.fs index 23fde41..909f85a 100644 --- a/src/FsYacc/fsyacc.fs +++ b/src/FsYacc/fsyacc.fs @@ -23,6 +23,7 @@ let mutable inputCodePage = None let mutable lexlib = "FSharp.Text.Lexing" let mutable parslib = "FSharp.Text.Parsing" let mutable bufferTypeArgument = "'cty" +let mutable assocCacheCapacity = None let usage = [ @@ -59,6 +60,11 @@ let usage = "Assume input lexer specification file is encoded with the given codepage." ) ArgInfo("--buffer-type-argument", ArgType.String(fun s -> bufferTypeArgument <- s), "Generic type argument of the LexBuffer type.") + ArgInfo( + "--assoc-cache-capacity", + ArgType.Int(fun i -> assocCacheCapacity <- Some i), + "Initial capacity of the generated parser's per-parse AssocTable lookup caches (0 grows on demand). Omit to use the runtime default (2000). See FsLexYacc issue #54." + ) ] let _ = @@ -118,6 +124,7 @@ let main () = parslib = parslib compat = compat bufferTypeArgument = bufferTypeArgument + assocCacheCapacity = assocCacheCapacity } writeSpecToFile generatorState spec compiledSpec diff --git a/tests/FsYacc.Core.Tests/AssocCacheCapacity.fs b/tests/FsYacc.Core.Tests/AssocCacheCapacity.fs new file mode 100644 index 0000000..cec97af --- /dev/null +++ b/tests/FsYacc.Core.Tests/AssocCacheCapacity.fs @@ -0,0 +1,65 @@ +module FsYacc.Core.Tests.AssocCacheCapacity + +open System.IO +open Expecto +open FsLexYacc.FsYacc.Driver + +// Generates a parser from a minimal grammar and returns the generated source text, +// so we can assert on the `engine` line that invokes the runtime interpreter. +let private generateParser (capacity: int option) = + let grammar = + "%token NUM\n\ + %token EOF\n\ + %start start\n\ + %type start\n\ + %%\n\ + start: NUM EOF { $1 }\n" + let input = Path.GetTempFileName() + let output = Path.GetTempFileName() + File.WriteAllText(input, grammar) + try + let spec = + match readSpecFromFile input None with + | Ok s -> s + | Result.Error(e, line, col) -> failwithf "grammar failed to parse (%d,%d): %s" line col e.Message + use logger = new NullLogger() :> Logger + let compiled = compileSpec spec logger + let state = + { GeneratorState.Default with + input = input + output = Some output + modname = Some "TestParser" + parslib = "FSharp.Text.Parsing" + lexlib = "FSharp.Text.Lexing" + assocCacheCapacity = capacity } + writeSpecToFile state spec compiled + File.ReadAllText output + finally + File.Delete input + if File.Exists output then File.Delete output + +// fsyacc's spec compiler carries process-global mutable state, so the two generation +// runs must not execute concurrently with each other or with the rest of the suite. +[] +let assocCacheCapacityTests = + testSequenced + <| testList "fsyacc --assoc-cache-capacity" [ + test "explicit capacity emits the 4-arg Interpret overload" { + let generated = generateParser (Some 0) + Expect.stringContains + generated + "tables.Interpret(lexer, lexbuf, startState, 0)" + "generated engine should pass the configured capacity to Interpret" + } + + test "no capacity emits the default 3-arg Interpret" { + let generated = generateParser None + Expect.stringContains + generated + "tables.Interpret(lexer, lexbuf, startState)" + "generated engine should use the parameterless Interpret overload" + Expect.isFalse + (generated.Contains "tables.Interpret(lexer, lexbuf, startState,") + "generated engine should not include a capacity argument when the flag is absent" + } + ] diff --git a/tests/FsYacc.Core.Tests/FsYacc.Core.Tests.fsproj b/tests/FsYacc.Core.Tests/FsYacc.Core.Tests.fsproj index 73d221b..58bac53 100644 --- a/tests/FsYacc.Core.Tests/FsYacc.Core.Tests.fsproj +++ b/tests/FsYacc.Core.Tests/FsYacc.Core.Tests.fsproj @@ -8,6 +8,7 @@ + From 4e8399075b8cb5fb4ff833b4db0edefac343f8c8 Mon Sep 17 00:00:00 2001 From: Steen Larsen Date: Wed, 1 Jul 2026 06:56:26 +0200 Subject: [PATCH 6/7] Test that parse results are invariant to AssocTable cache capacity The AssocTable cache initial capacity (#54) is a pure allocation-tuning knob and must never change what a parser produces. Extend the JSON example smoke test to parse each input at the historical default (2000) and at capacities 0, 1, 64 and 100000, asserting identical results and failing with a nonzero exit otherwise. This exercises the runtime interpreter that both the default path and the --assoc-cache-capacity path funnel into (Parser.start reads ParseSettings.AssocTableCacheInitialCapacity), complementing the codegen-only assertions in FsYacc.Core.Tests. --- tests/JsonLexAndYaccExample/Program.fs | 32 +++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/JsonLexAndYaccExample/Program.fs b/tests/JsonLexAndYaccExample/Program.fs index d2aeeff..3320ad9 100644 --- a/tests/JsonLexAndYaccExample/Program.fs +++ b/tests/JsonLexAndYaccExample/Program.fs @@ -1,6 +1,7 @@  module Program open FSharp.Text.Lexing +open FSharp.Text.Parsing open JsonParsing [] @@ -30,7 +31,36 @@ let main argv = complexJson |> parse |> ignore - //test lexing error + // Capacity-invariance check (issue #54): the AssocTable cache initial capacity is a pure + // allocation-tuning knob and must never change parse results. The generated `Parser.start` + // reads ParseSettings.AssocTableCacheInitialCapacity, so setting it here drives the runtime + // interpreter that both the default and the --assoc-cache-capacity code paths funnel into. + // Parse each input at the historical default and at a range of capacities; results must match. + let parseWith capacity json = + ParseSettings.AssocTableCacheInitialCapacity <- capacity + parse json + + let baselineCapacity = 2000 + let capacitiesUnderTest = [ 0; 1; 64; 100000 ] + let invarianceInputs = [ simpleJson; simpleJson2; complexJson ] + + let mismatches = + [ for json in invarianceInputs do + let baseline = parseWith baselineCapacity json + for capacity in capacitiesUnderTest do + if parseWith capacity json <> baseline then + yield capacity ] + + ParseSettings.AssocTableCacheInitialCapacity <- baselineCapacity + + if List.isEmpty mismatches then + printfn "Capacity-invariance OK across capacities %A" (baselineCapacity :: capacitiesUnderTest) + else + eprintfn "Capacity-invariance FAILED: parse results differed at capacities %A" (List.distinct mismatches) + exit 1 + + + //test lexing error try let simpleJson = "{\"f\"\n" + "\n" + ";" let parseResult = simpleJson |> parse From 0ed174faed1c1e83a19e0a2536174217a1f981c5 Mon Sep 17 00:00:00 2001 From: Steen Larsen Date: Wed, 1 Jul 2026 09:08:28 +0200 Subject: [PATCH 7/7] Format with Fantomas --- src/FsLexYacc.Runtime/Parsing.fsi | 3 ++- src/FsYacc.Core/fsyaccdriver.fs | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/FsLexYacc.Runtime/Parsing.fsi b/src/FsLexYacc.Runtime/Parsing.fsi index a8975e1..f50b6ec 100644 --- a/src/FsLexYacc.Runtime/Parsing.fsi +++ b/src/FsLexYacc.Runtime/Parsing.fsi @@ -116,7 +116,8 @@ type Tables<'tok> = /// As Interpret, but with an explicit initial capacity for the per-parse AssocTable lookup caches /// (issue #54); 0 grows them on demand. Overrides ParseSettings.AssocTableCacheInitialCapacity for /// this parser. fsyacc emits a call to this overload when --assoc-cache-capacity is passed. - member Interpret: lexer: (LexBuffer<'char> -> 'tok) * lexbuf: LexBuffer<'char> * startState: int * assocTableCacheInitialCapacity: int -> obj + member Interpret: + lexer: (LexBuffer<'char> -> 'tok) * lexbuf: LexBuffer<'char> * startState: int * assocTableCacheInitialCapacity: int -> obj /// Indicates an accept action has occured exception Accept of obj diff --git a/src/FsYacc.Core/fsyaccdriver.fs b/src/FsYacc.Core/fsyaccdriver.fs index 81ba42f..fe9f869 100644 --- a/src/FsYacc.Core/fsyaccdriver.fs +++ b/src/FsYacc.Core/fsyaccdriver.fs @@ -654,6 +654,7 @@ let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compile writer.WriteLine " numTerminals = %d;" (Array.length compiledSpec.actionTable.[0]) writer.WriteLine " productionToNonTerminalTable = _fsyacc_productionToNonTerminalTable }" + match generatorState.assocCacheCapacity with | Some n -> writer.WriteLine "let engine lexer lexbuf startState = tables.Interpret(lexer, lexbuf, startState, %d)" n | None -> writer.WriteLine "let engine lexer lexbuf startState = tables.Interpret(lexer, lexbuf, startState)"