diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e109b5c..1ae9d11 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -99,7 +99,7 @@ jobs: run: nix develop --command bash -c "mix deps.get" - name: Unit Tests - run: nix develop --command bash -c "mix test" + run: nix develop --command bash -c "mix test --exclude property" lint: runs-on: ubuntu-latest @@ -163,4 +163,3 @@ jobs: - name: Run dialyzer run: nix develop --command bash -c 'mix dialyzer --format github' - diff --git a/.github/workflows/proptest.yaml b/.github/workflows/proptest.yaml new file mode 100644 index 0000000..fc296b2 --- /dev/null +++ b/.github/workflows/proptest.yaml @@ -0,0 +1,74 @@ +name: Property Tests +on: + schedule: + - cron: "0 0 * * *" + workflow_dispatch: + +permissions: + contents: read + issues: write + +concurrency: + group: proptest + cancel-in-progress: false + +jobs: + proptest: + runs-on: ubuntu-latest + env: + SPITFIRE_PROPERTY_MAX_RUNS: "250000" + + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@main + - uses: cachix/cachix-action@v14 + with: + name: elixir-tools + authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}" + + - uses: actions/cache@v4 + with: + path: | + deps + _build + key: ${{ runner.os }}-mix-${{ hashFiles('**/.mise.toml') }}-${{ hashFiles('**/mix.lock') }} + restore-keys: | + ${{ runner.os }}-mix-${{ hashFiles('**/.mise.toml') }}- + + - name: Install Dependencies + run: nix develop --command bash -c "mix deps.get" + + - name: Run property tests + shell: bash + run: | + set -euo pipefail + + nix develop --command bash -c "mix test --only property" 2>&1 | tee property-test.log + + - name: Build failure report + if: ${{ failure() }} + shell: bash + run: | + { + echo "## Property tests failed" + echo "" + echo "- Workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Branch: \`${{ github.ref_name }}\`" + echo "- Commit: \`${{ github.sha }}\`" + echo "- Trigger: \`${{ github.event_name }}\`" + echo "" + echo "### Log tail" + echo '```text' + tail -n 400 property-test.log || true + echo '```' + } > property-test-failure.md + + - name: Open issue on failure + if: ${{ failure() }} + env: + GH_TOKEN: ${{ github.token }} + run: | + gh issue create \ + --repo "${{ github.repository }}" \ + --title "Property tests failed (run #${{ github.run_number }})" \ + --body-file property-test-failure.md diff --git a/test/property/char_property_test.exs b/test/property/char_property_test.exs index 4315914..28aac9b 100644 --- a/test/property/char_property_test.exs +++ b/test/property/char_property_test.exs @@ -11,6 +11,9 @@ defmodule Spitfire.CharPropertyTest do use ExUnitProperties + @property_max_runs "SPITFIRE_PROPERTY_MAX_RUNS" |> System.get_env("1000") |> String.to_integer() + @property_check_opts [max_runs: @property_max_runs, max_shrinking_steps: 50] + setup %{mode: mode} do Process.put(:spitfire_test_mode, mode) :ok @@ -1173,8 +1176,7 @@ defmodule Spitfire.CharPropertyTest do property "grammar trees round-trip through Spitfire in all contexts" do check all( {context, code} <- all_contexts_gen(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1188,8 +1190,7 @@ defmodule Spitfire.CharPropertyTest do property "standalone expressions" do check all( {context, code} <- context_standalone(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1206,8 +1207,7 @@ defmodule Spitfire.CharPropertyTest do context_bitstring(), context_bitstring_positional_then_kw_data() ]), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1220,8 +1220,7 @@ defmodule Spitfire.CharPropertyTest do property "before do block" do check all( {context, code} <- context_before_do(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1232,7 +1231,7 @@ defmodule Spitfire.CharPropertyTest do @tag :property @tag timeout: 120_000 property "after do block" do - check all({context, code} <- context_after_do(), max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- context_after_do(), @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1255,7 +1254,7 @@ defmodule Spitfire.CharPropertyTest do context_fn_multi_arg_with_arrow() ]) - check all({context, code} <- fn_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- fn_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1267,8 +1266,7 @@ defmodule Spitfire.CharPropertyTest do property "inside do block" do check all( {context, code} <- context_inside_do(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1287,7 +1285,7 @@ defmodule Spitfire.CharPropertyTest do context_no_parens_call() ]) - check all({context, code} <- call_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- call_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1306,8 +1304,7 @@ defmodule Spitfire.CharPropertyTest do context_bracket_at_access(), context_bracket_at_access_kw_data_value() ]), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1334,8 +1331,7 @@ defmodule Spitfire.CharPropertyTest do check all( {context, code} <- container_contexts, - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1348,8 +1344,7 @@ defmodule Spitfire.CharPropertyTest do property "inside string interpolation" do check all( {context, code} <- context_interpolation(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1366,7 +1361,7 @@ defmodule Spitfire.CharPropertyTest do context_after_assignment() ]) - check all({context, code} <- op_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- op_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1378,8 +1373,7 @@ defmodule Spitfire.CharPropertyTest do property "inside struct arg" do check all( {context, code} <- context_struct_arg(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1392,8 +1386,7 @@ defmodule Spitfire.CharPropertyTest do property "between do blocks" do check all( {context, code} <- context_between_do_blocks(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1411,7 +1404,7 @@ defmodule Spitfire.CharPropertyTest do context_ternary_third() ]) - check all({context, code} <- ternary_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- ternary_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1430,8 +1423,7 @@ defmodule Spitfire.CharPropertyTest do check all( {context, code} <- map_update_contexts, - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1444,8 +1436,7 @@ defmodule Spitfire.CharPropertyTest do property "after parens call" do check all( {context, code} <- context_after_parens_call(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1458,8 +1449,7 @@ defmodule Spitfire.CharPropertyTest do property "inside no parens call middle" do check all( {context, code} <- context_no_parens_call_middle(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1478,7 +1468,7 @@ defmodule Spitfire.CharPropertyTest do context_after_dot() ]) - check all({context, code} <- dot_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- dot_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1490,8 +1480,7 @@ defmodule Spitfire.CharPropertyTest do property "between operators" do check all( {context, code} <- context_between_operators(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1513,7 +1502,7 @@ defmodule Spitfire.CharPropertyTest do context_after_not() ]) - check all({context, code} <- unary_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- unary_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1529,7 +1518,7 @@ defmodule Spitfire.CharPropertyTest do context_interpolated_keyword() ]) - check all({context, code} <- interp_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- interp_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1541,8 +1530,7 @@ defmodule Spitfire.CharPropertyTest do property "inside charlist interpolation" do check all( {context, code} <- context_charlist_interpolation(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1561,8 +1549,7 @@ defmodule Spitfire.CharPropertyTest do check all( {context, code} <- heredoc_contexts, - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end @@ -1579,7 +1566,7 @@ defmodule Spitfire.CharPropertyTest do context_sigil_heredoc_interpolation() ]) - check all({context, code} <- sigil_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- sigil_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1589,7 +1576,7 @@ defmodule Spitfire.CharPropertyTest do @tag :property @tag timeout: 120_000 property "inside fn when guard" do - check all({context, code} <- context_fn_when(), max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- context_fn_when(), @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1606,7 +1593,7 @@ defmodule Spitfire.CharPropertyTest do context_def_when() ]) - check all({context, code} <- def_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- def_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1626,7 +1613,7 @@ defmodule Spitfire.CharPropertyTest do context_no_parens_call_kw_value() ]) - check all({context, code} <- kv_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- kv_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1652,7 +1639,7 @@ defmodule Spitfire.CharPropertyTest do context_receive_clause_lhs() ]) - check all({context, code} <- stab_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- stab_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1670,7 +1657,7 @@ defmodule Spitfire.CharPropertyTest do context_nested_no_parens_call() ]) - check all({context, code} <- args_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- args_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1689,7 +1676,7 @@ defmodule Spitfire.CharPropertyTest do context_with_else_body() ]) - check all({context, code} <- comp_contexts, max_runs: 1000, max_shrinking_steps: 50) do + check all({context, code} <- comp_contexts, @property_check_opts) do run_comparison(context, code, current_mode()) end end @@ -1701,8 +1688,7 @@ defmodule Spitfire.CharPropertyTest do property "inside bitstring segment spec" do check all( {context, code} <- context_bitstring_segment_spec(), - max_runs: 1000, - max_shrinking_steps: 50 + @property_check_opts ) do run_comparison(context, code, current_mode()) end