Merge #158874

158874: mmaprototype: continue if no candidates left after filtering r=wenyihu6 a=wenyihu6

Previously we started filtering candidates based on their lease disposition.
Asim dd don’t have the full integration yet, so every store stays with an
unknown health status instead of the proper initialized state you get in
TestClusterState. Future PR will address that initialization. This commit
simply prevents the panic by continuing early when filtering on health leaves the
candidate set empty.

Fixes: #158833
Release note: none

Co-authored-by: wenyihu6 <wenyi@cockroachlabs.com>

Author: craig[bot]

pkg/kv/kvserver/allocator/mmaprototype/cluster_state_rebalance_stores.go

@@ -644,6 +644,12 @@ func (re *rebalanceEnv) rebalanceLeasesFromLocalStoreID(
 		}
 
 		candsPL = retainReadyLeaseTargetStoresOnly(ctx, candsPL, re.stores, rangeID)
+		if len(candsPL) == 0 {
+			log.KvDistribution.VEventf(ctx, 2,
+				"result(failed): no candidates to move lease from n%vs%v for r%v after retainReadyLeaseTargetStoresOnly",
+				ss.NodeID, ss.StoreID, rangeID)
+			continue
+		}
 
 		clear(re.scratch.nodes)
 		means := computeMeansForStoreSet(re, candsPL, re.scratch.nodes, re.scratch.stores)

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/heterogeneous_cpu.txt

@@ -46,17 +46,17 @@ failed assertion sample 1
 	max/mean=1.20 tick=4
 	max/mean=1.20 tick=5
 ==========================
-cpu#1: last:  [s1=6399652777, s2=6391952777, s3=7205401827] (stddev=381661598.00, mean=6665669127.00, sum=19997007381)
-cpu#1: thrash_pct: [s1=24%, s2=29%, s3=31%]  (sum=84%)
-cpu_util#1: last:  [s1=0.80, s2=0.80, s3=0.45] (stddev=0.16, mean=0.68, sum=2)
-cpu_util#1: thrash_pct: [s1=6%, s2=7%, s3=4%]  (sum=16%)
-artifacts[mma-count]: c51384f72bf7101d
+cpu#1: last:  [s1=6702050000, s2=6694069444, s3=6603880555] (stddev=44515870.96, mean=6666666666.33, sum=19999999999)
+cpu#1: thrash_pct: [s1=133%, s2=123%, s3=139%]  (sum=395%)
+cpu_util#1: last:  [s1=0.84, s2=0.84, s3=0.41] (stddev=0.20, mean=0.70, sum=2)
+cpu_util#1: thrash_pct: [s1=5%, s2=5%, s3=3%]  (sum=12%)
+artifacts[mma-count]: 1306b93e8b8d4a39
 failed assertion sample 1
   balance stat=cpu_util threshold=(≤1.10) ticks=6
-	max/mean=1.17 tick=0
-	max/mean=1.17 tick=1
-	max/mean=1.17 tick=2
-	max/mean=1.17 tick=3
-	max/mean=1.17 tick=4
-	max/mean=1.17 tick=5
+	max/mean=1.20 tick=0
+	max/mean=1.20 tick=1
+	max/mean=1.20 tick=2
+	max/mean=1.20 tick=3
+	max/mean=1.20 tick=4
+	max/mean=1.20 tick=5
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_cpu.txt

@@ -62,27 +62,27 @@ asserting: |leases(t)/mean_{T}(leases) - 1| ≤ 0.10 ∀ t∈T and each store (T
 
 eval duration=20m samples=1 seed=42 cfgs=(mma-count,mma-only) metrics=(cpu,cpu_util,replicas,leases)
 ----
-cpu#1: last:  [s1=1541887030, s2=1713235417, s3=1580118619, s4=1618283496, s5=1563312037, s6=1657951235, s7=1307364950, s8=1552531971, s9=1561935680, s10=1645749580] (stddev=103163155.55, mean=1574237001.50, sum=15742370015)
-cpu#1: thrash_pct: [s1=25%, s2=29%, s3=18%, s4=29%, s5=28%, s6=16%, s7=19%, s8=27%, s9=30%, s10=18%]  (sum=238%)
-cpu_util#1: last:  [s1=0.19, s2=0.21, s3=0.20, s4=0.20, s5=0.20, s6=0.21, s7=0.16, s8=0.19, s9=0.20, s10=0.21] (stddev=0.01, mean=0.20, sum=2)
-cpu_util#1: thrash_pct: [s1=25%, s2=29%, s3=18%, s4=29%, s5=28%, s6=16%, s7=19%, s8=27%, s9=30%, s10=18%]  (sum=238%)
+cpu#1: last:  [s1=1557575458, s2=1543275402, s3=1430136673, s4=1579385742, s5=1718276619, s6=1707951464, s7=1568550501, s8=1613420899, s9=1572344975, s10=1452633706] (stddev=87757123.10, mean=1574355143.90, sum=15743551439)
+cpu#1: thrash_pct: [s1=38%, s2=47%, s3=49%, s4=71%, s5=24%, s6=42%, s7=59%, s8=62%, s9=24%, s10=28%]  (sum=445%)
+cpu_util#1: last:  [s1=0.19, s2=0.19, s3=0.18, s4=0.20, s5=0.21, s6=0.21, s7=0.20, s8=0.20, s9=0.20, s10=0.18] (stddev=0.01, mean=0.20, sum=2)
+cpu_util#1: thrash_pct: [s1=38%, s2=47%, s3=49%, s4=71%, s5=24%, s6=42%, s7=59%, s8=62%, s9=24%, s10=28%]  (sum=445%)
 leases#1: first: [s1=37, s2=22, s3=14, s4=13, s5=11, s6=11, s7=10, s8=11, s9=10, s10=11] (stddev=8.07, mean=15.00, sum=150)
-leases#1: last:  [s1=10, s2=14, s3=15, s4=16, s5=15, s6=17, s7=16, s8=15, s9=15, s10=17] (stddev=1.90, mean=15.00, sum=150)
-leases#1: thrash_pct: [s1=0%, s2=9%, s3=8%, s4=9%, s5=9%, s6=0%, s7=0%, s8=9%, s9=17%, s10=0%]  (sum=61%)
+leases#1: last:  [s1=11, s2=14, s3=15, s4=15, s5=15, s6=18, s7=15, s8=16, s9=15, s10=16] (stddev=1.67, mean=15.00, sum=150)
+leases#1: thrash_pct: [s1=20%, s2=56%, s3=45%, s4=68%, s5=24%, s6=33%, s7=70%, s8=40%, s9=25%, s10=17%]  (sum=398%)
 replicas#1: first: [s1=80, s2=70, s3=51, s4=42, s5=37, s6=35, s7=34, s8=33, s9=34, s10=34] (stddev=16.02, mean=45.00, sum=450)
-replicas#1: last:  [s1=45, s2=44, s3=45, s4=46, s5=43, s6=45, s7=47, s8=45, s9=45, s10=45] (stddev=1.00, mean=45.00, sum=450)
-replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=5%, s6=0%, s7=0%, s8=0%, s9=6%, s10=0%]  (sum=11%)
-artifacts[mma-count]: 2e50737f44fc4950
+replicas#1: last:  [s1=43, s2=47, s3=45, s4=47, s5=44, s6=44, s7=43, s8=46, s9=45, s10=46] (stddev=1.41, mean=45.00, sum=450)
+replicas#1: thrash_pct: [s1=46%, s2=118%, s3=87%, s4=96%, s5=62%, s6=71%, s7=140%, s8=72%, s9=59%, s10=42%]  (sum=793%)
+artifacts[mma-count]: ffa4c35237bb33cc
 ==========================
-cpu#1: last:  [s1=1706498399, s2=1615687783, s3=1677746697, s4=1624652238, s5=1648924305, s6=1640076380, s7=1135947134, s8=1438619020, s9=1699568316, s10=1553238061] (stddev=164018033.83, mean=1574095833.30, sum=15740958333)
-cpu#1: thrash_pct: [s1=32%, s2=85%, s3=84%, s4=36%, s5=17%, s6=37%, s7=15%, s8=16%, s9=22%, s10=20%]  (sum=364%)
-cpu_util#1: last:  [s1=0.21, s2=0.20, s3=0.21, s4=0.20, s5=0.21, s6=0.21, s7=0.14, s8=0.18, s9=0.21, s10=0.19] (stddev=0.02, mean=0.20, sum=2)
-cpu_util#1: thrash_pct: [s1=32%, s2=85%, s3=84%, s4=36%, s5=17%, s6=37%, s7=15%, s8=16%, s9=22%, s10=20%]  (sum=364%)
+cpu#1: last:  [s1=1619570512, s2=1646824164, s3=1595529169, s4=1620526342, s5=1566499747, s6=1343720904, s7=1548086479, s8=1541998677, s9=1703605806, s10=1551869077] (stddev=90595627.30, mean=1573823087.70, sum=15738230877)
+cpu#1: thrash_pct: [s1=24%, s2=32%, s3=36%, s4=20%, s5=30%, s6=16%, s7=20%, s8=18%, s9=22%, s10=22%]  (sum=240%)
+cpu_util#1: last:  [s1=0.20, s2=0.21, s3=0.20, s4=0.20, s5=0.20, s6=0.17, s7=0.19, s8=0.19, s9=0.21, s10=0.19] (stddev=0.01, mean=0.20, sum=2)
+cpu_util#1: thrash_pct: [s1=24%, s2=32%, s3=36%, s4=20%, s5=30%, s6=16%, s7=20%, s8=18%, s9=22%, s10=22%]  (sum=240%)
 leases#1: first: [s1=37, s2=22, s3=14, s4=13, s5=11, s6=11, s7=10, s8=11, s9=10, s10=11] (stddev=8.07, mean=15.00, sum=150)
-leases#1: last:  [s1=15, s2=9, s3=17, s4=10, s5=23, s6=17, s7=13, s8=16, s9=15, s10=15] (stddev=3.71, mean=15.00, sum=150)
-leases#1: thrash_pct: [s1=0%, s2=41%, s3=66%, s4=37%, s5=0%, s6=39%, s7=0%, s8=0%, s9=0%, s10=0%]  (sum=183%)
+leases#1: last:  [s1=15, s2=15, s3=15, s4=16, s5=15, s6=14, s7=15, s8=15, s9=15, s10=15] (stddev=0.45, mean=15.00, sum=150)
+leases#1: thrash_pct: [s1=0%, s2=9%, s3=15%, s4=0%, s5=9%, s6=0%, s7=0%, s8=0%, s9=0%, s10=0%]  (sum=34%)
 replicas#1: first: [s1=80, s2=70, s3=51, s4=42, s5=37, s6=35, s7=34, s8=33, s9=34, s10=34] (stddev=16.02, mean=45.00, sum=450)
-replicas#1: last:  [s1=80, s2=58, s3=46, s4=45, s5=38, s6=37, s7=35, s8=36, s9=37, s10=38] (stddev=13.42, mean=45.00, sum=450)
-replicas#1: thrash_pct: [s1=0%, s2=0%, s3=5%, s4=0%, s5=0%, s6=5%, s7=0%, s8=0%, s9=0%, s10=0%]  (sum=10%)
-artifacts[mma-only]: 7478cadb8f2e4145
+replicas#1: last:  [s1=58, s2=63, s3=52, s4=45, s5=41, s6=38, s7=39, s8=37, s9=39, s10=38] (stddev=8.90, mean=45.00, sum=450)
+replicas#1: thrash_pct: [s1=0%, s2=5%, s3=9%, s4=0%, s5=5%, s6=0%, s7=0%, s8=0%, s9=0%, s10=0%]  (sum=19%)
+artifacts[mma-only]: 3d537724873044f2
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_cpu_25nodes.txt

@@ -25,31 +25,31 @@ gen_load rate=15000 rw_ratio=0.95 min_block=1 max_block=1 request_cpu_per_access
 
 eval duration=25m samples=1 seed=42 cfgs=(mma-only,mma-count) metrics=(cpu,cpu_util,write_bytes_per_second,replicas,leases)
 ----
-cpu#1: last:  [s1=463047636, s2=464742044, s3=465584574, s4=464309298, s5=465847383, s6=466444044, s7=463910580, s8=466175498, s9=464760769, s10=467967426, s11=468395602, s12=468693867, s13=468326290, s14=319992397, s15=468057733, s16=403289930, s17=467566175, s18=340531137, s19=467968120, s20=468090063, s21=361926413, s22=404937284, s23=469792185, s24=383090767, s25=319789535] (stddev=50329198.14, mean=437329470.00, sum=10933236750)
-cpu#1: thrash_pct: [s1=13%, s2=39%, s3=29%, s4=12%, s5=41%, s6=29%, s7=13%, s8=29%, s9=38%, s10=8%, s11=7%, s12=7%, s13=10%, s14=6%, s15=8%, s16=7%, s17=12%, s18=6%, s19=7%, s20=8%, s21=6%, s22=7%, s23=9%, s24=6%, s25=6%]  (sum=362%)
-cpu_util#1: last:  [s1=0.06, s2=0.06, s3=0.06, s4=0.06, s5=0.06, s6=0.06, s7=0.06, s8=0.06, s9=0.06, s10=0.06, s11=0.06, s12=0.06, s13=0.06, s14=0.04, s15=0.06, s16=0.05, s17=0.06, s18=0.04, s19=0.06, s20=0.06, s21=0.05, s22=0.05, s23=0.06, s24=0.05, s25=0.04] (stddev=0.01, mean=0.05, sum=1)
-cpu_util#1: thrash_pct: [s1=13%, s2=39%, s3=29%, s4=12%, s5=41%, s6=29%, s7=13%, s8=29%, s9=38%, s10=8%, s11=7%, s12=7%, s13=10%, s14=6%, s15=8%, s16=7%, s17=12%, s18=6%, s19=7%, s20=8%, s21=6%, s22=7%, s23=9%, s24=6%, s25=6%]  (sum=362%)
+cpu#1: last:  [s1=465448266, s2=454658033, s3=370200309, s4=466241466, s5=348807435, s6=326294624, s7=465931266, s8=327022268, s9=433135213, s10=468650978, s11=468032284, s12=468703586, s13=319373726, s14=404347235, s15=468052372, s16=467741409, s17=468474449, s18=468209178, s19=468940504, s20=467592518, s21=468116054, s22=468954354, s23=468571821, s24=468690916, s25=468574476] (stddev=52192287.95, mean=437550589.60, sum=10938764740)
+cpu#1: thrash_pct: [s1=11%, s2=9%, s3=8%, s4=10%, s5=8%, s6=8%, s7=11%, s8=8%, s9=9%, s10=8%, s11=11%, s12=7%, s13=5%, s14=6%, s15=9%, s16=7%, s17=14%, s18=9%, s19=7%, s20=10%, s21=7%, s22=7%, s23=10%, s24=7%, s25=7%]  (sum=213%)
+cpu_util#1: last:  [s1=0.06, s2=0.06, s3=0.05, s4=0.06, s5=0.04, s6=0.04, s7=0.06, s8=0.04, s9=0.05, s10=0.06, s11=0.06, s12=0.06, s13=0.04, s14=0.05, s15=0.06, s16=0.06, s17=0.06, s18=0.06, s19=0.06, s20=0.06, s21=0.06, s22=0.06, s23=0.06, s24=0.06, s25=0.06] (stddev=0.01, mean=0.05, sum=1)
+cpu_util#1: thrash_pct: [s1=11%, s2=9%, s3=8%, s4=10%, s5=8%, s6=8%, s7=11%, s8=8%, s9=9%, s10=8%, s11=11%, s12=7%, s13=5%, s14=6%, s15=9%, s16=7%, s17=14%, s18=9%, s19=7%, s20=10%, s21=7%, s22=7%, s23=10%, s24=7%, s25=7%]  (sum=213%)
 leases#1: first: [s1=167, s2=0, s3=0, s4=167, s5=0, s6=0, s7=166, s8=0, s9=0, s10=0, s11=0, s12=0, s13=0, s14=0, s15=0, s16=0, s17=0, s18=0, s19=0, s20=0, s21=0, s22=0, s23=0, s24=0, s25=0] (stddev=54.16, mean=20.00, sum=500)
-leases#1: last:  [s1=21, s2=20, s3=20, s4=21, s5=20, s6=20, s7=21, s8=20, s9=20, s10=22, s11=22, s12=22, s13=22, s14=15, s15=22, s16=19, s17=22, s18=16, s19=22, s20=22, s21=17, s22=19, s23=22, s24=18, s25=15] (stddev=2.19, mean=20.00, sum=500)
-leases#1: thrash_pct: [s1=0%, s2=27%, s3=16%, s4=0%, s5=33%, s6=18%, s7=0%, s8=21%, s9=29%, s10=0%, s11=0%, s12=0%, s13=3%, s14=0%, s15=0%, s16=0%, s17=5%, s18=0%, s19=0%, s20=2%, s21=0%, s22=0%, s23=2%, s24=0%, s25=0%]  (sum=156%)
+leases#1: last:  [s1=22, s2=19, s3=15, s4=22, s5=14, s6=13, s7=22, s8=13, s9=18, s10=22, s11=22, s12=22, s13=15, s14=19, s15=22, s16=22, s17=22, s18=22, s19=22, s20=22, s21=22, s22=22, s23=22, s24=22, s25=22] (stddev=3.21, mean=20.00, sum=500)
+leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%, s10=2%, s11=5%, s12=0%, s13=0%, s14=0%, s15=2%, s16=0%, s17=7%, s18=2%, s19=0%, s20=3%, s21=0%, s22=0%, s23=3%, s24=0%, s25=0%]  (sum=23%)
 replicas#1: first: [s1=167, s2=167, s3=167, s4=167, s5=167, s6=167, s7=166, s8=166, s9=166, s10=0, s11=0, s12=0, s13=0, s14=0, s15=0, s16=0, s17=0, s18=0, s19=0, s20=0, s21=0, s22=0, s23=0, s24=0, s25=0] (stddev=80.00, mean=60.00, sum=1500)
-replicas#1: last:  [s1=84, s2=153, s3=156, s4=86, s5=155, s6=157, s7=85, s8=155, s9=152, s10=22, s11=22, s12=22, s13=22, s14=15, s15=22, s16=19, s17=22, s18=16, s19=22, s20=22, s21=17, s22=19, s23=22, s24=18, s25=15] (stddev=57.14, mean=60.00, sum=1500)
-replicas#1: thrash_pct: [s1=0%, s2=11%, s3=5%, s4=0%, s5=13%, s6=8%, s7=0%, s8=9%, s9=10%, s10=0%, s11=0%, s12=0%, s13=3%, s14=0%, s15=0%, s16=0%, s17=4%, s18=0%, s19=0%, s20=2%, s21=0%, s22=0%, s23=2%, s24=0%, s25=0%]  (sum=67%)
-write_bytes_per_second#1: last:  [s1=125, s2=229, s3=233, s4=128, s5=231, s6=235, s7=126, s8=232, s9=227, s10=32, s11=33, s12=33, s13=32, s14=22, s15=32, s16=28, s17=32, s18=23, s19=32, s20=32, s21=25, s22=28, s23=33, s24=26, s25=22] (stddev=85.61, mean=89.24, sum=2231)
-write_bytes_per_second#1: thrash_pct: [s1=179%, s2=262%, s3=281%, s4=185%, s5=247%, s6=266%, s7=180%, s8=250%, s9=247%, s10=29%, s11=23%, s12=8%, s13=31%, s14=24%, s15=22%, s16=10%, s17=27%, s18=23%, s19=34%, s20=22%, s21=12%, s22=19%, s23=33%, s24=58%, s25=21%]  (sum=2492%)
-artifacts[mma-only]: 99cd3e4d17a5a700
+replicas#1: last:  [s1=22, s2=186, s3=182, s4=22, s5=181, s6=180, s7=22, s8=179, s9=184, s10=22, s11=22, s12=22, s13=15, s14=19, s15=22, s16=22, s17=22, s18=22, s19=22, s20=22, s21=22, s22=22, s23=22, s24=22, s25=22] (stddev=68.58, mean=60.00, sum=1500)
+replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%, s10=2%, s11=4%, s12=0%, s13=0%, s14=0%, s15=2%, s16=0%, s17=6%, s18=2%, s19=0%, s20=3%, s21=0%, s22=0%, s23=3%, s24=0%, s25=0%]  (sum=21%)
+write_bytes_per_second#1: last:  [s1=32, s2=279, s3=273, s4=32, s5=270, s6=269, s7=32, s8=268, s9=275, s10=32, s11=32, s12=33, s13=22, s14=28, s15=32, s16=32, s17=33, s18=32, s19=33, s20=33, s21=33, s22=33, s23=33, s24=33, s25=33] (stddev=102.80, mean=89.48, sum=2237)
+write_bytes_per_second#1: thrash_pct: [s1=33%, s2=219%, s3=213%, s4=36%, s5=227%, s6=237%, s7=20%, s8=228%, s9=248%, s10=22%, s11=34%, s12=43%, s13=18%, s14=34%, s15=18%, s16=19%, s17=43%, s18=30%, s19=27%, s20=20%, s21=28%, s22=20%, s23=24%, s24=30%, s25=30%]  (sum=1902%)
+artifacts[mma-only]: badf16a7721889e3
 ==========================
-cpu#1: last:  [s1=479911909, s2=373580982, s3=417004825, s4=481191026, s5=437968812, s6=418529844, s7=480521667, s8=416565076, s9=375203500, s10=416487421, s11=459073056, s12=417390139, s13=458452447, s14=438198669, s15=438626986, s16=436488614, s17=459362930, s18=437947945, s19=458740016, s20=438620980, s21=437601149, s22=458747503, s23=416674300, s24=436424336, s25=460194196] (stddev=27439196.31, mean=437980333.12, sum=10949508328)
-cpu#1: thrash_pct: [s1=7%, s2=8%, s3=7%, s4=8%, s5=8%, s6=7%, s7=8%, s8=7%, s9=7%, s10=13%, s11=7%, s12=7%, s13=10%, s14=5%, s15=9%, s16=12%, s17=6%, s18=6%, s19=10%, s20=8%, s21=10%, s22=7%, s23=7%, s24=7%, s25=6%]  (sum=199%)
-cpu_util#1: last:  [s1=0.06, s2=0.05, s3=0.05, s4=0.06, s5=0.05, s6=0.05, s7=0.06, s8=0.05, s9=0.05, s10=0.05, s11=0.06, s12=0.05, s13=0.06, s14=0.05, s15=0.05, s16=0.05, s17=0.06, s18=0.05, s19=0.06, s20=0.05, s21=0.05, s22=0.06, s23=0.05, s24=0.05, s25=0.06] (stddev=0.00, mean=0.05, sum=1)
-cpu_util#1: thrash_pct: [s1=7%, s2=8%, s3=7%, s4=8%, s5=8%, s6=7%, s7=8%, s8=7%, s9=7%, s10=13%, s11=7%, s12=7%, s13=10%, s14=5%, s15=9%, s16=12%, s17=6%, s18=6%, s19=10%, s20=8%, s21=10%, s22=7%, s23=7%, s24=7%, s25=6%]  (sum=199%)
+cpu#1: last:  [s1=479582462, s2=416893232, s3=354683378, s4=480755517, s5=375112525, s6=438669079, s7=479287682, s8=374734078, s9=353558405, s10=459115168, s11=459641964, s12=416567087, s13=457864915, s14=479677281, s15=458969381, s16=416838126, s17=439177543, s18=459494023, s19=458443535, s20=458980623, s21=416392513, s22=459704532, s23=479212304, s24=437658822, s25=438217958] (stddev=37993693.80, mean=437969285.32, sum=10949232133)
+cpu#1: thrash_pct: [s1=8%, s2=8%, s3=6%, s4=8%, s5=7%, s6=7%, s7=8%, s8=7%, s9=7%, s10=11%, s11=7%, s12=9%, s13=19%, s14=7%, s15=7%, s16=6%, s17=6%, s18=5%, s19=10%, s20=7%, s21=12%, s22=6%, s23=18%, s24=7%, s25=8%]  (sum=209%)
+cpu_util#1: last:  [s1=0.06, s2=0.05, s3=0.04, s4=0.06, s5=0.05, s6=0.05, s7=0.06, s8=0.05, s9=0.04, s10=0.06, s11=0.06, s12=0.05, s13=0.06, s14=0.06, s15=0.06, s16=0.05, s17=0.05, s18=0.06, s19=0.06, s20=0.06, s21=0.05, s22=0.06, s23=0.06, s24=0.05, s25=0.05] (stddev=0.00, mean=0.05, sum=1)
+cpu_util#1: thrash_pct: [s1=8%, s2=8%, s3=6%, s4=8%, s5=7%, s6=7%, s7=8%, s8=7%, s9=7%, s10=11%, s11=7%, s12=9%, s13=19%, s14=7%, s15=7%, s16=6%, s17=6%, s18=5%, s19=10%, s20=7%, s21=12%, s22=6%, s23=18%, s24=7%, s25=8%]  (sum=209%)
 leases#1: first: [s1=167, s2=0, s3=0, s4=167, s5=0, s6=0, s7=166, s8=0, s9=0, s10=0, s11=0, s12=0, s13=0, s14=0, s15=0, s16=0, s17=0, s18=0, s19=0, s20=0, s21=0, s22=0, s23=0, s24=0, s25=0] (stddev=54.16, mean=20.00, sum=500)
-leases#1: last:  [s1=22, s2=17, s3=19, s4=22, s5=20, s6=19, s7=22, s8=19, s9=17, s10=19, s11=21, s12=19, s13=21, s14=20, s15=20, s16=20, s17=21, s18=20, s19=21, s20=20, s21=20, s22=21, s23=19, s24=20, s25=21] (stddev=1.30, mean=20.00, sum=500)
-leases#1: thrash_pct: [s1=0%, s2=2%, s3=0%, s4=0%, s5=2%, s6=0%, s7=0%, s8=0%, s9=0%, s10=8%, s11=2%, s12=2%, s13=4%, s14=0%, s15=4%, s16=7%, s17=0%, s18=0%, s19=5%, s20=3%, s21=6%, s22=2%, s23=2%, s24=2%, s25=0%]  (sum=50%)
+leases#1: last:  [s1=22, s2=19, s3=16, s4=22, s5=17, s6=20, s7=22, s8=17, s9=16, s10=21, s11=21, s12=19, s13=21, s14=22, s15=21, s16=19, s17=20, s18=21, s19=21, s20=21, s21=19, s22=21, s23=22, s24=20, s25=20] (stddev=1.81, mean=20.00, sum=500)
+leases#1: thrash_pct: [s1=0%, s2=2%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%, s10=6%, s11=2%, s12=3%, s13=12%, s14=2%, s15=2%, s16=0%, s17=0%, s18=0%, s19=5%, s20=2%, s21=7%, s22=2%, s23=12%, s24=2%, s25=3%]  (sum=60%)
 replicas#1: first: [s1=167, s2=167, s3=167, s4=167, s5=167, s6=167, s7=166, s8=166, s9=166, s10=0, s11=0, s12=0, s13=0, s14=0, s15=0, s16=0, s17=0, s18=0, s19=0, s20=0, s21=0, s22=0, s23=0, s24=0, s25=0] (stddev=80.00, mean=60.00, sum=1500)
-replicas#1: last:  [s1=61, s2=60, s3=61, s4=63, s5=58, s6=63, s7=63, s8=60, s9=61, s10=58, s11=58, s12=59, s13=58, s14=61, s15=60, s16=57, s17=62, s18=61, s19=59, s20=60, s21=58, s22=60, s23=59, s24=58, s25=62] (stddev=1.74, mean=60.00, sum=1500)
-replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%, s10=0%, s11=0%, s12=0%, s13=0%, s14=0%, s15=5%, s16=0%, s17=0%, s18=0%, s19=0%, s20=0%, s21=0%, s22=0%, s23=0%, s24=0%, s25=2%]  (sum=7%)
-write_bytes_per_second#1: last:  [s1=91, s2=89, s3=91, s4=94, s5=87, s6=94, s7=94, s8=89, s9=91, s10=87, s11=86, s12=88, s13=87, s14=91, s15=89, s16=85, s17=92, s18=91, s19=88, s20=90, s21=87, s22=90, s23=88, s24=87, s25=92] (stddev=2.50, mean=89.52, sum=2238)
-write_bytes_per_second#1: thrash_pct: [s1=74%, s2=98%, s3=89%, s4=68%, s5=111%, s6=106%, s7=63%, s8=104%, s9=110%, s10=45%, s11=38%, s12=48%, s13=34%, s14=33%, s15=81%, s16=44%, s17=47%, s18=43%, s19=32%, s20=68%, s21=37%, s22=67%, s23=40%, s24=70%, s25=55%]  (sum=1606%)
-artifacts[mma-count]: 783f63043b18076a
+replicas#1: last:  [s1=58, s2=62, s3=61, s4=60, s5=59, s6=60, s7=58, s8=59, s9=61, s10=61, s11=63, s12=59, s13=58, s14=61, s15=61, s16=60, s17=61, s18=61, s19=59, s20=60, s21=58, s22=62, s23=59, s24=60, s25=59] (stddev=1.36, mean=60.00, sum=1500)
+replicas#1: thrash_pct: [s1=0%, s2=10%, s3=4%, s4=0%, s5=2%, s6=2%, s7=0%, s8=4%, s9=6%, s10=2%, s11=2%, s12=11%, s13=2%, s14=0%, s15=4%, s16=0%, s17=5%, s18=0%, s19=0%, s20=0%, s21=0%, s22=2%, s23=2%, s24=4%, s25=0%]  (sum=60%)
+write_bytes_per_second#1: last:  [s1=87, s2=93, s3=91, s4=89, s5=88, s6=90, s7=87, s8=88, s9=91, s10=91, s11=94, s12=88, s13=86, s14=91, s15=91, s16=89, s17=91, s18=91, s19=88, s20=89, s21=86, s22=93, s23=88, s24=89, s25=88] (stddev=2.12, mean=89.48, sum=2237)
+write_bytes_per_second#1: thrash_pct: [s1=49%, s2=97%, s3=115%, s4=73%, s5=103%, s6=112%, s7=54%, s8=100%, s9=107%, s10=49%, s11=39%, s12=47%, s13=45%, s14=37%, s15=57%, s16=53%, s17=47%, s18=51%, s19=37%, s20=37%, s21=33%, s22=53%, s23=46%, s24=51%, s25=35%]  (sum=1528%)
+artifacts[mma-count]: ff789b5ff064514f
 ==========================