diff --git a/src/Test/Fuzz.elm b/src/Test/Fuzz.elm index 433238f1..b20c7f9f 100644 --- a/src/Test/Fuzz.elm +++ b/src/Test/Fuzz.elm @@ -8,6 +8,8 @@ import MicroListExtra as List import MicroMaybeExtra as Maybe import PRNG import Random +import RandomRun exposing (RandomRun) +import Set exposing (Set) import Simplify import Test.Distribution exposing (DistributionReport(..)) import Test.Distribution.Internal exposing (Distribution(..), ExpectedDistribution(..)) @@ -29,26 +31,30 @@ fuzzTest distribution fuzzer untrimmedDesc getExpectation = blankDescriptionFailure else - ElmTestVariant__Labeled desc <| validatedFuzzTest fuzzer getExpectation distribution + ElmTestVariant__Labeled desc <| validatedFuzzTest desc fuzzer getExpectation distribution {-| Knowing that the fuzz test isn't obviously invalid, run the test and package up the results. -} -validatedFuzzTest : Fuzzer a -> (a -> Expectation) -> Distribution a -> Test -validatedFuzzTest fuzzer getExpectation distribution = +validatedFuzzTest : String -> Fuzzer a -> (a -> Expectation) -> Distribution a -> Test +validatedFuzzTest description fuzzer getExpectation distribution = ElmTestVariant__FuzzTest (\seed runs -> let + constants : LoopConstants a + constants = + { fuzzer = fuzzer + , testFn = getExpectation + , initialSeed = seed + , runsNeeded = runs + , distribution = distribution + , skipsAllowed = runs * maxSkippedRunsRatio + , description = description + } + runResult : RunResult runResult = - fuzzLoop - { fuzzer = fuzzer - , testFn = getExpectation - , initialSeed = seed - , runsNeeded = runs - , distribution = distribution - } - (initLoopState seed distribution) + fuzzLoop constants (initLoopState constants) in case runResult.failure of Nothing -> @@ -77,6 +83,8 @@ type alias LoopConstants a = , initialSeed : Random.Seed , runsNeeded : Int , distribution : Distribution a + , skipsAllowed : Int + , description : String } @@ -86,15 +94,17 @@ type alias LoopState = , nextPowerOfTwo : Int , failure : Maybe Failure , currentSeed : Random.Seed + , seenRandomRuns : Set (List Int) + , runsSkipped : Int } -initLoopState : Random.Seed -> Distribution a -> LoopState -initLoopState initialSeed distribution = +initLoopState : LoopConstants a -> LoopState +initLoopState c = let initialDistributionCount : Maybe (Dict (List String) Int) initialDistributionCount = - Test.Distribution.Internal.getDistributionLabels distribution + Test.Distribution.Internal.getDistributionLabels c.distribution |> Maybe.map (\labels -> labels @@ -106,10 +116,29 @@ initLoopState initialSeed distribution = , distributionCount = initialDistributionCount , nextPowerOfTwo = 1 , failure = Nothing - , currentSeed = initialSeed + , currentSeed = c.initialSeed + , seenRandomRuns = Set.empty + , runsSkipped = 0 } +{-| If user specified 100 runs and this ratio is 2, we can only skip 100\*2 = +200 values before stopping. + +Consider `Fuzz.bool`: it only has two possible RandomRuns: + + - [ 0 ] --> False + - [ 1 ] --> True + +We'll likely try those pretty soon. We don't have a good way of figuring out +that's all of them so we'll just skip them for until 200 values have been tried. + +-} +maxSkippedRunsRatio : Int +maxSkippedRunsRatio = + 2 + + {-| Runs fuzz tests repeatedly and returns information about distribution and possible failure. The loop algorithm is roughly: @@ -151,13 +180,13 @@ fuzzLoop c state = Just distributionCount -> DistributionToReport { distributionCount = includeCombinationsInBaseCounts distributionCount - , runsElapsed = state.runsElapsed + , runsElapsed = state.runsElapsed + state.runsSkipped } , failure = Just failure } Nothing -> - if state.runsElapsed < c.runsNeeded then + if state.runsElapsed < c.runsNeeded && state.runsSkipped < c.skipsAllowed then let newState : LoopState newState = @@ -182,7 +211,7 @@ fuzzLoop c state = { distributionReport = DistributionToReport { distributionCount = includeCombinationsInBaseCounts distributionCount - , runsElapsed = state.runsElapsed + , runsElapsed = state.runsElapsed + state.runsSkipped } , failure = Nothing } @@ -210,7 +239,7 @@ fuzzLoop c state = { distributionReport = DistributionCheckSucceeded { distributionCount = distributionCount - , runsElapsed = state.runsElapsed + , runsElapsed = state.runsElapsed + state.runsSkipped } , failure = Nothing } @@ -282,7 +311,7 @@ allSufficientlyCovered c state normalizedDistributionCount = True AtLeast n -> - Test.Distribution.Internal.sufficientlyCovered state.runsElapsed count (n / 100) + Test.Distribution.Internal.sufficientlyCovered (state.runsElapsed + state.runsSkipped) count (n / 100) ) ) ) @@ -323,9 +352,9 @@ findBadZeroRelatedCase c state normalizedDistributionCount = |> Maybe.map (\count -> { label = label - , actualPercentage = toFloat count * 100 / toFloat state.runsElapsed + , actualPercentage = toFloat count * 100 / toFloat (state.runsElapsed + state.runsSkipped) , expectedDistribution = expectedDistribution - , runsElapsed = state.runsElapsed + , runsElapsed = state.runsElapsed + state.runsSkipped , distributionCount = distributionCount } ) @@ -369,14 +398,14 @@ findInsufficientlyCoveredLabel c state normalizedDistributionCount = False AtLeast n -> - Test.Distribution.Internal.insufficientlyCovered state.runsElapsed count (n / 100) + Test.Distribution.Internal.insufficientlyCovered (state.runsElapsed + state.runsSkipped) count (n / 100) ) |> Maybe.map (\( label, count, expectedDistribution ) -> { label = label - , actualPercentage = toFloat count * 100 / toFloat state.runsElapsed + , actualPercentage = toFloat count * 100 / toFloat (state.runsElapsed + state.runsSkipped) , expectedDistribution = expectedDistribution - , runsElapsed = state.runsElapsed + , runsElapsed = state.runsElapsed + state.runsSkipped , distributionCount = distributionCount } ) @@ -475,11 +504,12 @@ runOnce c state = Nothing -> stepSeed state.currentSeed - - ( maybeFailure, newDistributionCounter ) = - case genResult of - Rejected { reason } -> - ( Just + in + case genResult of + Rejected { reason } -> + { state + | failure = + Just { given = Nothing , expectation = Test.Expectation.fail @@ -487,51 +517,76 @@ runOnce c state = , reason = Invalid InvalidFuzzer } } - , state.distributionCount - ) + , currentSeed = nextSeed + , runsElapsed = state.runsElapsed + 1 + } - Generated { prng, value } -> - let - failure : Maybe Failure - failure = - testGeneratedValue - { getExpectation = c.testFn - , fuzzer = c.fuzzer - , randomRun = PRNG.getRun prng - , value = value - , expectation = c.testFn value - } + Generated { prng, value } -> + let + randomRun : RandomRun + randomRun = + PRNG.getRun prng + + randomRunList : List Int + randomRunList = + RandomRun.toList randomRun + + nextDistributionCount : Maybe (Dict (List String) Int) + nextDistributionCount = + Maybe.map2 + (\labels old -> + let + foundLabels : List String + foundLabels = + labels + |> List.filterMap + (\( label, predicate ) -> + if predicate value then + Just label + + else + Nothing + ) + in + Dict.increment foundLabels old + ) + (Test.Distribution.Internal.getDistributionLabels c.distribution) + state.distributionCount + in + if Set.member randomRunList state.seenRandomRuns then + {- We've tried this RandomRun already and know it ends in Pass + (if it ended in Failure we'd have short-circuited). + Let's not do unneeded work! + + We still want to update the distribution counts though, + those don't care about whether the value passes or fails the + test. + -} + { state + | currentSeed = nextSeed + , distributionCount = nextDistributionCount + , runsSkipped = state.runsSkipped + 1 + } - distributionCounter : Maybe (Dict (List String) Int) - distributionCounter = - Maybe.map2 - (\labels old -> - let - foundLabels : List String - foundLabels = - labels - |> List.filterMap - (\( label, predicate ) -> - if predicate value then - Just label - - else - Nothing - ) - in - Dict.increment foundLabels old - ) - (Test.Distribution.Internal.getDistributionLabels c.distribution) - state.distributionCount - in - ( failure, distributionCounter ) - in - { state - | failure = maybeFailure - , distributionCount = newDistributionCounter - , currentSeed = nextSeed - , runsElapsed = state.runsElapsed + 1 - } + else + let + nextFailure : Maybe Failure + nextFailure = + testGeneratedValue + { getExpectation = c.testFn + , fuzzer = c.fuzzer + , randomRun = randomRun + , value = value + , expectation = c.testFn value + } + in + { state + | failure = nextFailure + , distributionCount = nextDistributionCount + , currentSeed = nextSeed + , runsElapsed = state.runsElapsed + 1 + , seenRandomRuns = Set.insert randomRunList state.seenRandomRuns + } includeCombinationsInBaseCounts : Dict (List String) Int -> Dict (List String) Int diff --git a/tests/src/FuzzerTests.elm b/tests/src/FuzzerTests.elm index c81b0cdf..8928b2c1 100644 --- a/tests/src/FuzzerTests.elm +++ b/tests/src/FuzzerTests.elm @@ -1189,6 +1189,24 @@ distributionTests = (Fuzz.intRange 1 20) "Int range boundaries - mandatory" (\n -> Expect.pass) + + {- we don't want failing tests around by default + + , Test.fuzzWith + { runs = 10000 + , distribution = + Test.expectDistribution + [ ( Test.Distribution.atLeast 5.5, "low", \n -> n == 1 ) + , ( Test.Distribution.atLeast 4, "high", \n -> n == 20 ) + , ( Test.Distribution.atLeast 80, "in between", \n -> n > 1 && n < 20 ) + , ( Test.Distribution.zero, "outside", \n -> n < 1 || n > 20 ) + , ( Test.Distribution.moreThanZero, "one", \n -> n == 1 ) + ] + } + (Fuzz.intRange 1 20) + "Failing distribution test" + (\n -> Expect.pass) + -} ] diff --git a/tests/src/Runner/String.elm b/tests/src/Runner/String.elm index 1cd4920a..349a82a6 100644 --- a/tests/src/Runner/String.elm +++ b/tests/src/Runner/String.elm @@ -75,25 +75,19 @@ fromExpectation labels expectation summary = expectation |> Test.Runner.getDistributionReport |> Runner.String.Distribution.report labels - - summaryWithDistribution : Summary - summaryWithDistribution = - case distributionReport of - Nothing -> - summary - - Just distribution -> - { summary - | output = - summary.output - ++ "\n\n" - ++ distribution - ++ "\n" - } in case Test.Runner.getFailureReason expectation of Nothing -> - { summaryWithDistribution | passed = summaryWithDistribution.passed + 1 } + { summary + | passed = summary.passed + 1 + , output = + distributionReport + |> Maybe.map + (\distribution -> + summary.output ++ "\n\n" ++ distribution ++ "\n" + ) + |> Maybe.withDefault summary.output + } Just { given, description, reason } -> let @@ -112,13 +106,20 @@ fromExpectation labels expectation summary = "\n\n" ++ outputLabels labels ++ "\n" + ++ (distributionReport + |> Maybe.map + (\distribution -> + "\n" ++ indentLines distribution ++ "\n\n" + ) + |> Maybe.withDefault "" + ) ++ (prefix ++ indentLines message) ++ "\n" in - { summaryWithDistribution - | output = summaryWithDistribution.output ++ newOutput - , failed = summaryWithDistribution.failed + 1 - , passed = summaryWithDistribution.passed + { summary + | output = summary.output ++ newOutput + , failed = summary.failed + 1 + , passed = summary.passed } diff --git a/tests/src/Runner/String/Distribution.elm b/tests/src/Runner/String/Distribution.elm index e6fa466b..47d82583 100644 --- a/tests/src/Runner/String/Distribution.elm +++ b/tests/src/Runner/String/Distribution.elm @@ -25,8 +25,5 @@ report testBreadcrumbs distributionReport = DistributionCheckSucceeded _ -> Nothing - DistributionCheckFailed _ -> - {- Don't show it here: it's already included in the failure message. - That way the Node runner will show it too. - -} - Nothing + DistributionCheckFailed r -> + Just <| Test.Distribution.distributionReportTable r