Refactor next item rules including adding pointwise utils and switch

tests away from xunit

Author: Frankie Robertson

Project.toml

@@ -52,7 +52,7 @@ LogarithmicNumbers = "1"
 MacroTools = "^0.5.6"
 Measurements = "^2.10.0"
 OrderedCollections = "^1.6"
-PsychometricsBazaarBase = "^0.8.0"
+PsychometricsBazaarBase = "^0.8.1"
 Reexport = "1"
 Setfield = "^1"
 StaticArrays = "1"

src/TerminationConditions.jl

@@ -41,8 +41,7 @@ function (condition::SimpleFunctionTerminationCondition)(responses::TrackedRespo
 end
 
 struct RunForeverTerminationCondition <: TerminationCondition end
-function (condition::RunForeverTerminationCondition)(responses::TrackedResponses,
-        items::AbstractItemBank)
+function (condition::RunForeverTerminationCondition)(::TrackedResponses, ::AbstractItemBank)
     return false
 end
 

src/next_item_rules/NextItemRules.jl

@@ -47,6 +47,7 @@ export MatrixScalarizer, DeterminantScalarizer, TraceScalarizer
 export AbilityCovarianceStateMultiCriterion, StateMultiCriterion, ItemMultiCriterion
 export InformationMatrixCriteria
 export ScalarizedStateCriteron, ScalarizedItemCriteron
+export DRuleItemCriterion, TRuleItemCriterion
 
 # Prelude
 include("./prelude/abstract.jl")
@@ -61,15 +62,18 @@ include("./strategies/exhaustive.jl")
 # Combinators
 include("./combinators/expectation.jl")
 include("./combinators/scalarizers.jl")
+include("./combinators/likelihood.jl")
 
 # Criteria
 include("./criteria/item/information_special.jl")
 include("./criteria/item/information_support.jl")
 include("./criteria/item/information.jl")
 include("./criteria/item/urry.jl")
 include("./criteria/state/ability_variance.jl")
+include("./criteria/pointwise/kl.jl")
 
 # Porcelain
+include("./porcelain/porcelain.jl")
 include("./porcelain/aliases.jl")
 
 end

src/next_item_rules/combinators/likelihood.jl

@@ -0,0 +1,19 @@
+struct LikelihoodWeightedItemCriterion{
+    PointwiseItemCriterionT <: PointwiseItemCriterion,
+    AbilityIntegratorT <: AbilityIntegrator,
+    AbilityEstimatorT <: DistributionAbilityEstimator
+} <: ItemCriterion
+    criterion::PointwiseItemCriterionT
+    integrator::AbilityIntegratorT
+    estimator::AbilityEstimatorT
+end
+
+function compute_criterion(
+        lwic::LikelihoodWeightedItemCriterion,
+        tracked_responses::TrackedResponses,
+        item_idx
+)
+    func = FunctionProduct(
+        pdf(lwic.estimator, tracked_responses), lwic.criterion(tracked_responses, item_idx))
+    lwic.integrator(func, 0, lwic.estimator, tracked_responses)
+end

src/next_item_rules/criteria/pointwise/kl.jl

@@ -0,0 +1,38 @@
+function kl(item_response::ItemResponse, r0, theta)
+    r = resp_vec(item_response, theta)
+    resp = 0.0
+    for (p0, p) in zip(r0, r)
+        resp += p0 * (log(p0) - log(p))
+    end
+    return resp
+end
+
+struct PosteriorExpectedKLInformationItemCriterion{
+    PointEstimatorT <: PointAbilityEstimator,
+    DistributionEstimatorT <: DistributionAbilityEstimator,
+    IntegratorT <: AbilityIntegrator
+} <: PointwiseItemCriterion
+end
+
+function PosteriorExpectedKLInformationItemCriterion(bits...)
+    @requiresome point_estimator = PointAbilityEstimator(bits...)
+    @requiresome distribution_estimator = DistributionAbilityEstimator(bits...)
+    @requiresome integrator = AbilityIntegrator(bits...)
+    PosteriorExpectedKLInformationItemCriterion(
+        point_estimator, distribution_estimator, integrator)
+end
+
+function compute_pointwise_criterion(
+        item_criterion::PosteriorExpectedKLInformationItemCriterion,
+        tracked_responses::TrackedResponses,
+        item_idx)
+    theta_0 = maybe_tracked_ability_estimate(tracked_responses,
+        item_criterion.point_estimator)
+    item_response = ItemResponse(tracked_responses.item_bank, item_idx)
+    r0 = resp_vec(item_response, theta_0)
+    expectation(
+        theta -> kl(item_response, r0, theta),
+        item_criterion.integrator,
+        item_criterion.distribution_estimator,
+        tracked_responses)
+end

src/next_item_rules/criteria/state/ability_variance.jl

@@ -18,7 +18,6 @@ function _get_dist_est_and_integrator(bits...)
     # XXX: Weakness in this initialisation system is showing now
     # This needs ot be explicitly passed dist_est and integrator, but this may
     # be burried within a MeanAbilityEstimator
-    @returnsome find1_instance(AbilityVarianceStateCriterion, bits)
     dist_est = DistributionAbilityEstimator(bits...)
     integrator = AbilityIntegrator(bits...)
     if dist_est !== nothing && integrator !== nothing
@@ -27,12 +26,14 @@ function _get_dist_est_and_integrator(bits...)
     # So let's just handle this case individually for now
     # (Is this going to cause a problem with this being picked over something more appropriate?)
     @requiresome mean_ability_est = MeanAbilityEstimator(bits...)
-    return (dist_est, integrator)
+    return (mean_ability_est.dist_est, mean_ability_est.integrator)
 end
 
 function AbilityVarianceStateCriterion(bits...)
     skip_zero = false
-    @requiresome (dist_est, integrator) = _get_dist_est_and_integrator(bits...)
+    @returnsome find1_instance(AbilityVarianceStateCriterion, bits)
+    @requiresome dist_est_integrator_pair = _get_dist_est_and_integrator(bits...)
+    (dist_est, integrator) = dist_est_integrator_pair
     return AbilityVarianceStateCriterion(dist_est, integrator, skip_zero)
 end
 

src/next_item_rules/porcelain/aliases.jl

@@ -41,12 +41,8 @@ const mirtcat_next_item_aliases = Dict(
     "MEPV" => _mirtcat_helper((bits, ability_estimator) -> ExpectationBasedItemCriterion(
         ability_estimator,
         AbilityVarianceStateCriterion(bits...))),
-    "Drule" => _mirtcat_helper((bits, ability_estimator) -> ScalarizedItemCriteron(
-        InformationMatrixCriteria(ability_estimator),
-        DeterminantScalarizer())),
-    "Trule" => _mirtcat_helper((bits, ability_estimator) -> ScalarizedItemCriteron(
-        InformationMatrixCriteria(ability_estimator),
-        TraceScalarizer()))
+    "Drule" => _mirtcat_helper((bits, ability_estimator) -> DRuleItemCriteron(ability_estimator)),
+    "Trule" => _mirtcat_helper((bits, ability_estimator) -> TRuleItemCriteron(ability_estimator))
 )
 
 # 'MLWI' for maximum likelihood weighted information

src/next_item_rules/porcelain/porcelain.jl

@@ -1 +1,11 @@
+function DRuleItemCriterion(ability_estimator)
+    ScalarizedItemCriteron(
+        InformationMatrixCriteria(ability_estimator),
+        DeterminantScalarizer())
+end
 
+function TRuleItemCriterion(ability_estimator)
+    ScalarizedItemCriteron(
+        InformationMatrixCriteria(ability_estimator),
+        TraceScalarizer())
+end

test/Project.toml

@@ -11,7 +11,6 @@ PsychometricsBazaarBase = "b0d9cada-d963-45e9-a4c6-4746243987f1"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 ResumableFunctions = "c5292f4c-5179-55e1-98c5-05642aab7184"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-XUnit = "3e3c03f2-1a94-11e9-2981-050a4ca824ab"
 
 [compat]
 Aqua = "0.5.5, 0.6.5"

test/ability_estimator_1d.jl

@@ -38,64 +38,67 @@ map_1d = ModeAbilityEstimator(pa_est_1d, optimizer_1d)
 mle_mean_1d = MeanAbilityEstimator(lh_est_1d, integrator_1d)
 mle_mode_1d = ModeAbilityEstimator(lh_est_1d, optimizer_1d)
 
-@testcase "Estimator: single dim MAP" begin
-    @test map_1d(tracked_responses_1d)≈1.0 atol=0.001
-end
+@testset "abilest_1d" begin
+    @testset "Estimator: single dim MAP" begin
+        @test map_1d(tracked_responses_1d)≈1.0 atol=0.001
+    end
 
-@testcase "Estimator: single dim EAP" begin
-    @test eap_1d(tracked_responses_1d)≈1.0 atol=0.001
-end
+    @testset "Estimator: single dim EAP" begin
+        @test eap_1d(tracked_responses_1d)≈1.0 atol=0.001
+    end
 
-@testcase "Estimator: single mle mean" begin
-    @test mle_mean_1d(tracked_responses_1d)≈1.0 atol=0.001
-end
+    @testset "Estimator: single mle mean" begin
+        @test mle_mean_1d(tracked_responses_1d)≈1.0 atol=0.001
+    end
 
-@testcase "Estimator: single mle mode" begin
-    @test mle_mode_1d(tracked_responses_1d)≈1.0 atol=0.001
-end
+    @testset "Estimator: single mle mode" begin
+        @test mle_mode_1d(tracked_responses_1d)≈1.0 atol=0.001
+    end
 
-information_item_criterion = InformationItemCriterion(mle_mean_1d)
+    information_item_criterion = InformationItemCriterion(mle_mean_1d)
 
-@testcase "1 dim neg information smaller closer to current estimate" begin
-    @test (
-        compute_criterion(information_item_criterion, tracked_responses_1d, 5) <
-        compute_criterion(information_item_criterion, tracked_responses_1d, 6)
-    )
-end
+    @testset "1 dim neg information smaller closer to current estimate" begin
+        @test (
+            compute_criterion(information_item_criterion, tracked_responses_1d, 5) <
+            compute_criterion(information_item_criterion, tracked_responses_1d, 6)
+        )
+    end
 
-@testcase "1 dim neg information smaller with igher discrimination" begin
-    @test (
-        compute_criterion(information_item_criterion, tracked_responses_1d, 7) <
-        compute_criterion(information_item_criterion, tracked_responses_1d, 5) <
-        compute_criterion(information_item_criterion, tracked_responses_1d, 8)
-    )
-end
-
-ability_variance_state_criterion = AbilityVarianceStateCriterion(lh_est_1d, integrator_1d)
-ability_variance_item_criterion = ExpectationBasedItemCriterion(
-    mle_mean_1d,
-    ability_variance_state_criterion
-)
+    @testset "1 dim neg information smaller with igher discrimination" begin
+        @test (
+            compute_criterion(information_item_criterion, tracked_responses_1d, 7) <
+            compute_criterion(information_item_criterion, tracked_responses_1d, 5) <
+            compute_criterion(information_item_criterion, tracked_responses_1d, 8)
+        )
+    end
 
-@testcase "postposterior 1 dim variance smaller closer to current estimate" begin
-    @test (
-        compute_criterion(ability_variance_item_criterion, tracked_responses_1d, 5) <
-        compute_criterion(ability_variance_item_criterion, tracked_responses_1d, 6)
+    ability_variance_state_criterion = AbilityVarianceStateCriterion(
+        lh_est_1d, integrator_1d)
+    ability_variance_item_criterion = ExpectationBasedItemCriterion(
+        mle_mean_1d,
+        ability_variance_state_criterion
     )
-end
 
-@testcase "postposterior 1 dim variance smaller with higher discrimination" begin
-    @test (
-        compute_criterion(ability_variance_item_criterion, tracked_responses_1d, 7) <
-        compute_criterion(ability_variance_item_criterion, tracked_responses_1d, 5) <
-        compute_criterion(ability_variance_item_criterion, tracked_responses_1d, 8)
-    )
-end
+    @testset "postposterior 1 dim variance smaller closer to current estimate" begin
+        @test (
+            compute_criterion(ability_variance_item_criterion, tracked_responses_1d, 5) <
+            compute_criterion(ability_variance_item_criterion, tracked_responses_1d, 6)
+        )
+    end
+
+    @testset "postposterior 1 dim variance smaller with higher discrimination" begin
+        @test (
+            compute_criterion(ability_variance_item_criterion, tracked_responses_1d, 7) <
+            compute_criterion(ability_variance_item_criterion, tracked_responses_1d, 5) <
+            compute_criterion(ability_variance_item_criterion, tracked_responses_1d, 8)
+        )
+    end
 
-@testcase "1 dim variance decreases with new responses" begin
-    orig_var = compute_criterion(ability_variance_state_criterion, tracked_responses_1d)
-    next_responses = deepcopy(tracked_responses_1d)
-    add_response!(next_responses, Response(ResponseType(item_bank_1d), 5, 0))
-    new_var = compute_criterion(ability_variance_state_criterion, next_responses)
-    @test new_var < orig_var
+    @testset "1 dim variance decreases with new responses" begin
+        orig_var = compute_criterion(ability_variance_state_criterion, tracked_responses_1d)
+        next_responses = deepcopy(tracked_responses_1d)
+        add_response!(next_responses, Response(ResponseType(item_bank_1d), 5, 0))
+        new_var = compute_criterion(ability_variance_state_criterion, next_responses)
+        @test new_var < orig_var
+    end
 end