From e8ae89f5b8f63513e98516181f0871b8865f9b27 Mon Sep 17 00:00:00 2001 From: Marek Kaput Date: Tue, 17 Feb 2026 15:05:49 +0100 Subject: [PATCH 1/7] ci: add scheduled property-test workflow Add a dedicated GitHub Actions workflow for property tests so they no longer run in the regular CI test job. The new workflow runs daily at 00:00 UTC. It automatically creates an issue when property tests fail (including run metadata and log tail). Using a random seed so scheduled executions explore different generated inputs over time. Special consideration: automatic issue creation might be noisy. We need to see how this behaves in time. fix #106 --- .github/workflows/ci.yaml | 3 +- .github/workflows/proptest.yaml | 73 +++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/proptest.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e109b5c..1ae9d11 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -99,7 +99,7 @@ jobs: run: nix develop --command bash -c "mix deps.get" - name: Unit Tests - run: nix develop --command bash -c "mix test" + run: nix develop --command bash -c "mix test --exclude property" lint: runs-on: ubuntu-latest @@ -163,4 +163,3 @@ jobs: - name: Run dialyzer run: nix develop --command bash -c 'mix dialyzer --format github' - diff --git a/.github/workflows/proptest.yaml b/.github/workflows/proptest.yaml new file mode 100644 index 0000000..60c7694 --- /dev/null +++ b/.github/workflows/proptest.yaml @@ -0,0 +1,73 @@ +name: Property Tests +on: + schedule: + - cron: "0 0 * * *" + workflow_dispatch: + +permissions: + contents: read + issues: write + +concurrency: + group: proptest + cancel-in-progress: false + +jobs: + proptest: + runs-on: ubuntu-latest + timeout-minutes: 240 + + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@main + - uses: cachix/cachix-action@v14 + with: + name: elixir-tools + authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}" + + - uses: actions/cache@v4 + with: + path: | + deps + _build + key: ${{ runner.os }}-mix-${{ hashFiles('**/.mise.toml') }}-${{ hashFiles('**/mix.lock') }} + restore-keys: | + ${{ runner.os }}-mix-${{ hashFiles('**/.mise.toml') }}- + + - name: Install Dependencies + run: nix develop --command bash -c "mix deps.get" + + - name: Run property tests + shell: bash + run: | + set -euo pipefail + + nix develop --command bash -c "mix test --only property" 2>&1 | tee property-test.log + + - name: Build failure report + if: ${{ failure() }} + shell: bash + run: | + { + echo "## Property tests failed" + echo "" + echo "- Workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Branch: \`${{ github.ref_name }}\`" + echo "- Commit: \`${{ github.sha }}\`" + echo "- Trigger: \`${{ github.event_name }}\`" + echo "" + echo "### Log tail" + echo '```text' + tail -n 400 property-test.log || true + echo '```' + } > property-test-failure.md + + - name: Open issue on failure + if: ${{ failure() }} + env: + GH_TOKEN: ${{ github.token }} + run: | + gh issue create \ + --repo "${{ github.repository }}" \ + --title "Property tests failed (run #${{ github.run_number }})" \ + --body-file property-test-failure.md From 406c600dde3765cb391b158230b988051226c2a9 Mon Sep 17 00:00:00 2001 From: Marek Kaput Date: Tue, 17 Feb 2026 15:17:38 +0100 Subject: [PATCH 2/7] Remove timeout for proptest job Remove timeout setting for proptest job in workflow --- .github/workflows/proptest.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/proptest.yaml b/.github/workflows/proptest.yaml index 60c7694..5ef15ff 100644 --- a/.github/workflows/proptest.yaml +++ b/.github/workflows/proptest.yaml @@ -15,7 +15,6 @@ concurrency: jobs: proptest: runs-on: ubuntu-latest - timeout-minutes: 240 steps: - uses: actions/checkout@v4 From 11745cf33fe170ae9c22af4dc7fbdf08e84d98da Mon Sep 17 00:00:00 2001 From: Marek Kaput Date: Tue, 17 Feb 2026 16:19:01 +0100 Subject: [PATCH 3/7] test: make property max runs configurable Replace hardcoded property-test max_runs values with a shared option that reads SPITFIRE_PROPERTY_MAX_RUNS. Local/default behavior is unchanged at 1000 runs when the env var is not set. Invalid values now fail fast with a clear ArgumentError. Set SPITFIRE_PROPERTY_MAX_RUNS=250000 in the scheduled proptest workflow so CI runs deeper property checks while regular local runs stay fast. --- .github/workflows/proptest.yaml | 2 + test/property/char_property_test.exs | 101 ++++++++++++++------------- 2 files changed, 54 insertions(+), 49 deletions(-) diff --git a/.github/workflows/proptest.yaml b/.github/workflows/proptest.yaml index 5ef15ff..fc296b2 100644 --- a/.github/workflows/proptest.yaml +++ b/.github/workflows/proptest.yaml @@ -15,6 +15,8 @@ concurrency: jobs: proptest: runs-on: ubuntu-latest + env: + SPITFIRE_PROPERTY_MAX_RUNS: "250000" steps: - uses: actions/checkout@v4 diff --git a/test/property/char_property_test.exs b/test/property/char_property_test.exs index 4315914..0a5c618 100644 --- a/test/property/char_property_test.exs +++ b/test/property/char_property_test.exs @@ -11,6 +11,26 @@ defmodule Spitfire.CharPropertyTest do use ExUnitProperties + @property_max_runs_env_var "SPITFIRE_PROPERTY_MAX_RUNS" + @default_property_max_runs 1000 + + @property_max_runs (case System.get_env(@property_max_runs_env_var) do + nil -> + @default_property_max_runs + + value -> + case Integer.parse(value) do + {runs, ""} when runs > 0 -> + runs + + _ -> + raise ArgumentError, + "expected #{@property_max_runs_env_var} to be a positive integer, got: #{inspect(value)}" + end + end) + + @property_check_opts [max_runs: @property_max_runs, max_shrinking_steps: 50] + setup %{mode: mode} do Process.put(:spitfire_test_mode, mode) :ok @@ -1173,8 +1193,7 @@ defmodule Spitfire.CharPropertyTest do property "grammar trees round-trip through Spitfire in all contexts" do check all( {context, code} <- all_contexts_gen(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1188,8 +1207,7 @@ defmodule Spitfire.CharPropertyTest do property "standalone expressions" do check all( {context, code} <- context_standalone(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1206,8 +1224,7 @@ defmodule Spitfire.CharPropertyTest do context_bitstring(), context_bitstring_positional_then_kw_data() ]), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1220,8 +1237,7 @@ defmodule Spitfire.CharPropertyTest do property "before do block" do check all( {context, code} <- context_before_do(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1232,7 +1248,7 @@ defmodule Spitfire.CharPropertyTest do @tag :property @tag timeout: 120_000 property "after do block" do - check all({context, code} <- context_after_do(), max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- context_after_do(), @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1255,7 +1271,7 @@ defmodule Spitfire.CharPropertyTest do context_fn_multi_arg_with_arrow() ]) - check all({context, code} <- fn_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- fn_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1267,8 +1283,7 @@ defmodule Spitfire.CharPropertyTest do property "inside do block" do check all( {context, code} <- context_inside_do(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1287,7 +1302,7 @@ defmodule Spitfire.CharPropertyTest do context_no_parens_call() ]) - check all({context, code} <- call_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- call_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1306,8 +1321,7 @@ defmodule Spitfire.CharPropertyTest do context_bracket_at_access(), context_bracket_at_access_kw_data_value() ]), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1334,8 +1348,7 @@ defmodule Spitfire.CharPropertyTest do check all( {context, code} <- container_contexts, - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1348,8 +1361,7 @@ defmodule Spitfire.CharPropertyTest do property "inside string interpolation" do check all( {context, code} <- context_interpolation(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1366,7 +1378,7 @@ defmodule Spitfire.CharPropertyTest do context_after_assignment() ]) - check all({context, code} <- op_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- op_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1378,8 +1390,7 @@ defmodule Spitfire.CharPropertyTest do property "inside struct arg" do check all( {context, code} <- context_struct_arg(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1392,8 +1403,7 @@ defmodule Spitfire.CharPropertyTest do property "between do blocks" do check all( {context, code} <- context_between_do_blocks(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1411,7 +1421,7 @@ defmodule Spitfire.CharPropertyTest do context_ternary_third() ]) - check all({context, code} <- ternary_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- ternary_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1430,8 +1440,7 @@ defmodule Spitfire.CharPropertyTest do check all( {context, code} <- map_update_contexts, - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1444,8 +1453,7 @@ defmodule Spitfire.CharPropertyTest do property "after parens call" do check all( {context, code} <- context_after_parens_call(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1458,8 +1466,7 @@ defmodule Spitfire.CharPropertyTest do property "inside no parens call middle" do check all( {context, code} <- context_no_parens_call_middle(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1478,7 +1485,7 @@ defmodule Spitfire.CharPropertyTest do context_after_dot() ]) - check all({context, code} <- dot_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- dot_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1490,8 +1497,7 @@ defmodule Spitfire.CharPropertyTest do property "between operators" do check all( {context, code} <- context_between_operators(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1513,7 +1519,7 @@ defmodule Spitfire.CharPropertyTest do context_after_not() ]) - check all({context, code} <- unary_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- unary_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1529,7 +1535,7 @@ defmodule Spitfire.CharPropertyTest do context_interpolated_keyword() ]) - check all({context, code} <- interp_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- interp_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1541,8 +1547,7 @@ defmodule Spitfire.CharPropertyTest do property "inside charlist interpolation" do check all( {context, code} <- context_charlist_interpolation(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1561,8 +1566,7 @@ defmodule Spitfire.CharPropertyTest do check all( {context, code} <- heredoc_contexts, - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1579,7 +1583,7 @@ defmodule Spitfire.CharPropertyTest do context_sigil_heredoc_interpolation() ]) - check all({context, code} <- sigil_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- sigil_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1589,7 +1593,7 @@ defmodule Spitfire.CharPropertyTest do @tag :property @tag timeout: 120_000 property "inside fn when guard" do - check all({context, code} <- context_fn_when(), max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- context_fn_when(), @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1606,7 +1610,7 @@ defmodule Spitfire.CharPropertyTest do context_def_when() ]) - check all({context, code} <- def_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- def_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1626,7 +1630,7 @@ defmodule Spitfire.CharPropertyTest do context_no_parens_call_kw_value() ]) - check all({context, code} <- kv_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- kv_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1652,7 +1656,7 @@ defmodule Spitfire.CharPropertyTest do context_receive_clause_lhs() ]) - check all({context, code} <- stab_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- stab_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1670,7 +1674,7 @@ defmodule Spitfire.CharPropertyTest do context_nested_no_parens_call() ]) - check all({context, code} <- args_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- args_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1689,7 +1693,7 @@ defmodule Spitfire.CharPropertyTest do context_with_else_body() ]) - check all({context, code} <- comp_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- comp_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1701,8 +1705,7 @@ defmodule Spitfire.CharPropertyTest do property "inside bitstring segment spec" do check all( {context, code} <- context_bitstring_segment_spec(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end From 38c112597cb084a9f525ff73e38cc05b9713147e Mon Sep 17 00:00:00 2001 From: Marek Kaput Date: Tue, 17 Feb 2026 16:22:20 +0100 Subject: [PATCH 4/7] refactor(test): simplify property max-runs parsing Use String.to_integer/1 for SPITFIRE_PROPERTY_MAX_RUNS and remove additional positivity validation to keep the env parsing path minimal. --- test/property/char_property_test.exs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/test/property/char_property_test.exs b/test/property/char_property_test.exs index 0a5c618..7e1ce1f 100644 --- a/test/property/char_property_test.exs +++ b/test/property/char_property_test.exs @@ -19,14 +19,7 @@ defmodule Spitfire.CharPropertyTest do @default_property_max_runs value -> - case Integer.parse(value) do - {runs, ""} when runs > 0 -> - runs - - _ -> - raise ArgumentError, - "expected #{@property_max_runs_env_var} to be a positive integer, got: #{inspect(value)}" - end + String.to_integer(value) end) @property_check_opts [max_runs: @property_max_runs, max_shrinking_steps: 50] From e386c6df7626ae6ea93f88f8939b06227b096b7e Mon Sep 17 00:00:00 2001 From: Marek Kaput Date: Tue, 17 Feb 2026 16:25:08 +0100 Subject: [PATCH 5/7] refactor(test): inline property max-runs env parsing Inline SPITFIRE_PROPERTY_MAX_RUNS and default value in a single module-attribute expression using System.get_env/2 and String.to_integer/1. --- test/property/char_property_test.exs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/test/property/char_property_test.exs b/test/property/char_property_test.exs index 7e1ce1f..147bb7a 100644 --- a/test/property/char_property_test.exs +++ b/test/property/char_property_test.exs @@ -11,16 +11,7 @@ defmodule Spitfire.CharPropertyTest do use ExUnitProperties - @property_max_runs_env_var "SPITFIRE_PROPERTY_MAX_RUNS" - @default_property_max_runs 1000 - - @property_max_runs (case System.get_env(@property_max_runs_env_var) do - nil -> - @default_property_max_runs - - value -> - String.to_integer(value) - end) + @property_max_runs "SPITFIRE_PROPERTY_MAX_RUNS" |> System.get_env("1000") |> String.to_integer() @property_check_opts [max_runs: @property_max_runs, max_shrinking_steps: 50] From b0051ce8619f405b606e9cc5c33d5d27725a547b Mon Sep 17 00:00:00 2001 From: Marek Kaput Date: Tue, 17 Feb 2026 16:26:31 +0100 Subject: [PATCH 6/7] fmt --- test/property/char_property_test.exs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/property/char_property_test.exs b/test/property/char_property_test.exs index 147bb7a..b164ffe 100644 --- a/test/property/char_property_test.exs +++ b/test/property/char_property_test.exs @@ -11,8 +11,7 @@ defmodule Spitfire.CharPropertyTest do use ExUnitProperties - @property_max_runs "SPITFIRE_PROPERTY_MAX_RUNS" |> System.get_env("1000") |> String.to_integer() - + @property_max_runs System.get_env("SPITFIRE_PROPERTY_MAX_RUNS", "1000") |> String.to_integer() @property_check_opts [max_runs: @property_max_runs, max_shrinking_steps: 50] setup %{mode: mode} do From 8c118cef83d1c525e4203515aa9faadde88cf893 Mon Sep 17 00:00:00 2001 From: Marek Kaput Date: Wed, 18 Feb 2026 09:23:55 +0100 Subject: [PATCH 7/7] fmt --- test/property/char_property_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/property/char_property_test.exs b/test/property/char_property_test.exs index b164ffe..28aac9b 100644 --- a/test/property/char_property_test.exs +++ b/test/property/char_property_test.exs @@ -11,7 +11,7 @@ defmodule Spitfire.CharPropertyTest do use ExUnitProperties - @property_max_runs System.get_env("SPITFIRE_PROPERTY_MAX_RUNS", "1000") |> String.to_integer() + @property_max_runs "SPITFIRE_PROPERTY_MAX_RUNS" |> System.get_env("1000") |> String.to_integer() @property_check_opts [max_runs: @property_max_runs, max_shrinking_steps: 50] setup %{mode: mode} do