From 72e89229ddf24fafe464b28f522c72230f2c8e60 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 10 Mar 2026 19:32:58 +0800 Subject: [PATCH 01/24] add benchmark --- .../benchmark/MapLookupBenchmark.scala | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala new file mode 100644 index 0000000000000..3f9ebbde9b3d3 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.benchmark + +import org.apache.spark.benchmark.Benchmark +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation +import org.apache.spark.sql.functions._ +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types._ + +/** + * Benchmark to measure performance of map lookup operations. + * To run this benchmark: + * {{{ + * 1. without sbt: + * bin/spark-submit --class --jars + * 2. build/sbt "sql/Test/runMain org.apache.spark.sql.execution.benchmark.MapLookupBenchmark" + * 3. generate result: + * SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt \ + * "sql/Test/runMain org.apache.spark.sql.execution.benchmark.MapLookupBenchmark" + * Results will be written to "benchmarks/MapLookupBenchmark-results.txt". + * }}} + */ +object MapLookupBenchmark extends SqlBasedBenchmark { + + override def getSparkSession: SparkSession = { + SparkSession.builder() + .master("local[1]") + .appName("MapLookupBenchmark") + .config("spark.driver.memory", "6g") + .config("spark.executor.memory", "6g") + .getOrCreate() + } + + private def run( + mapSize: Int, + hitRate: Double, + keyType: DataType): Unit = { + val numRows = 10000 + + val benchmark = new Benchmark( + s"MapLookup (size=$mapSize, hit=$hitRate, type=$keyType)", + numRows, + output = output) + + import spark.implicits._ + + // Create a DataFrame with a single column 'm' containing the map, + // and 'k' containing the key to lookup. + // Use `typedLit` to create the map literal directly. + val keys = (0 until mapSize).toArray + val map = keys.zip(keys.map(_.toString)).toMap + val mapCol = typedLit(map) + + // Generate lookup keys + val lookupKeys = (0 until numRows).map { i => + if (i < numRows * hitRate) keys(i % mapSize) else -1 + } + + val lookupDf = lookupKeys.toDF("key").select(mapCol.as("m"), $"key") + + val expr = col("m").getItem(col("key")) + val elementAtExpr = element_at(col("m"), col("key")) + + benchmark.addCase("GetMapValue interpreted") { _ => + withSQLConf( + SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false", + SQLConf.CODEGEN_FACTORY_MODE.key -> "NO_CODEGEN", + SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { + lookupDf.select(expr).write.format("noop").mode("overwrite").save() + } + } + + benchmark.addCase("GetMapValue codegen") { _ => + withSQLConf( + SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true", + SQLConf.CODEGEN_FACTORY_MODE.key -> "CODEGEN_ONLY", + SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { + lookupDf.select(expr).write.format("noop").mode("overwrite").save() + } + } + + benchmark.addCase("element_at interpreted") { _ => + withSQLConf( + SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false", + SQLConf.CODEGEN_FACTORY_MODE.key -> "NO_CODEGEN", + SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { + lookupDf.select(elementAtExpr).write.format("noop").mode("overwrite").save() + } + } + + benchmark.addCase("element_at codegen") { _ => + withSQLConf( + SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true", + SQLConf.CODEGEN_FACTORY_MODE.key -> "CODEGEN_ONLY", + SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { + lookupDf.select(elementAtExpr).write.format("noop").mode("overwrite").save() + } + } + + benchmark.run() + } + + override def runBenchmarkSuite(mainArgs: Array[String]): Unit = { + val sizes = Seq(10, 20, 30, 50, 1000, 1000000) + for (size <- sizes) { + run(size, 1.0, IntegerType) + run(size, 0.5, IntegerType) + run(size, 0.0, IntegerType) + } + } +} From bc49e320b55bc1dbb36e599a62b22dba063dc293 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 10 Mar 2026 19:33:04 +0800 Subject: [PATCH 02/24] fix --- .../expressions/collectionOperations.scala | 4 +- .../expressions/complexTypeExtractors.scala | 218 ++++++++++++++++-- .../CollectionExpressionsSuite.scala | 26 ++- .../expressions/ComplexTypeSuite.scala | 50 ++++ 4 files changed, 278 insertions(+), 20 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index dc3e6dcbd388c..9ff6dc648c72e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -2741,7 +2741,9 @@ case class ElementAt( } } case _: MapType => - (value, ordinal) => getValueEval(value, ordinal, mapKeyType, ordering) + (value, ordinal) => { + getValueEval(value, ordinal, mapKeyType, ordering) + } } override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala index f40077c53311b..d8a8d67112b50 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala @@ -441,7 +441,35 @@ trait GetArrayItemUtil { */ trait GetMapValueUtil extends BinaryExpression with ImplicitCastInputTypes { - // todo: current search is O(n), improve it. + @transient private var lastMap: MapData = _ + @transient private var lastIndex: java.util.HashMap[Any, Int] = _ + + /** + * Minimum map size to use hash-based lookup. + * Below this threshold, the overhead of building a hash index + * exceeds the cost of a linear scan. + * The value 20 is chosen empirically; break-even is around 15-25 + * elements for primitive key types. + */ + private val HASH_LOOKUP_THRESHOLD = 20 + + private def getOrBuildIndex(map: MapData, keyType: DataType): java.util.HashMap[Any, Int] = { + if (lastMap ne map) { + val keys = map.keyArray() + val len = keys.numElements() + val hm = new java.util.HashMap[Any, Int]((len * 1.5).toInt) + var i = 0 + while (i < len) { + val k = keys.get(i, keyType) + if (!hm.containsKey(k)) hm.put(k, i) + i += 1 + } + lastIndex = hm + lastMap = map + } + lastIndex + } + def getValueEval( value: Any, ordinal: Any, @@ -449,6 +477,25 @@ trait GetMapValueUtil extends BinaryExpression with ImplicitCastInputTypes { ordering: Ordering[Any]): Any = { val map = value.asInstanceOf[MapData] val length = map.numElements() + + if (length < HASH_LOOKUP_THRESHOLD || !TypeUtils.typeWithProperEquals(keyType)) { + getValueEvalLinear(map, ordinal, keyType, ordering) + } else { + val idx = getOrBuildIndex(map, keyType).getOrDefault(ordinal, -1) + if (idx == -1 || map.valueArray().isNullAt(idx)) { + null + } else { + map.valueArray().get(idx, dataType) + } + } + } + + private def getValueEvalLinear( + map: MapData, + ordinal: Any, + keyType: DataType, + ordering: Ordering[Any]): Any = { + val length = map.numElements() val keys = map.keyArray() val values = map.valueArray() @@ -473,38 +520,178 @@ trait GetMapValueUtil extends BinaryExpression with ImplicitCastInputTypes { ctx: CodegenContext, ev: ExprCode, mapType: MapType): ExprCode = { + val keyType = mapType.keyType + if (supportsHashLookup(keyType)) { + doGetValueGenCodeWithHashOpt(ctx, ev, mapType) + } else { + doGetValueGenCodeLinear(ctx, ev, mapType) + } + } + + private def supportsHashLookup(keyType: DataType): Boolean = keyType match { + case BooleanType | ByteType | ShortType | IntegerType | LongType | + FloatType | DoubleType | DateType | TimestampType | + TimestampNTZType | _: YearMonthIntervalType | + _: DayTimeIntervalType => true + case st: StringType if st.supportsBinaryEquality => true + case _ => false + } + + private def doGetValueGenCodeLinear( + ctx: CodegenContext, + ev: ExprCode, + mapType: MapType): ExprCode = { + val index = ctx.freshName("index") + val length = ctx.freshName("length") + val keys = ctx.freshName("keys") + val values = ctx.freshName("values") + val keyType = mapType.keyType + + val keyJavaType = CodeGenerator.javaType(keyType) + val loopKey = ctx.freshName("loopKey") + val i = ctx.freshName("i") + + val nullValueCheck = if (mapType.valueContainsNull) { + s""" + |else if ($values.isNullAt($index)) { + | ${ev.isNull} = true; + |} + """.stripMargin + } else { + "" + } + + nullSafeCodeGen(ctx, ev, (eval1, eval2) => { + s""" + |final int $length = $eval1.numElements(); + |final ArrayData $keys = $eval1.keyArray(); + |final ArrayData $values = $eval1.valueArray(); + |int $index = -1; + | + |for (int $i = 0; $i < $length; $i++) { + | $keyJavaType $loopKey = ${CodeGenerator.getValue(keys, keyType, i)}; + | if (${ctx.genEqual(keyType, loopKey, eval2)}) { + | $index = $i; + | break; + | } + |} + | + |if ($index < 0) { + | ${ev.isNull} = true; + |} $nullValueCheck else { + | ${ev.value} = ${CodeGenerator.getValue(values, dataType, index)}; + |} + """.stripMargin + }) + } + + /** + * Generates code for map lookups. + * If the map size is small (less than HASH_LOOKUP_THRESHOLD), it uses a linear scan. + * If the map size is large, it builds a hash index for O(1) lookup. + */ + private def doGetValueGenCodeWithHashOpt( + ctx: CodegenContext, + ev: ExprCode, + mapType: MapType): ExprCode = { val index = ctx.freshName("index") val length = ctx.freshName("length") val keys = ctx.freshName("keys") - val key = ctx.freshName("key") val values = ctx.freshName("values") val keyType = mapType.keyType - val nullCheck = if (mapType.valueContainsNull) { - s" || $values.isNullAt($index)" + + val nullValueCheck = if (mapType.valueContainsNull) { + s""" + |else if ($values.isNullAt($index)) { + | ${ev.isNull} = true; + |} + """.stripMargin } else { "" } val keyJavaType = CodeGenerator.javaType(keyType) + val lastKeyArray = ctx.addMutableState("ArrayData", "lastKeyArray", v => s"$v = null;") + val hashBuckets = ctx.addMutableState("int[]", "hashBuckets", v => s"$v = null;") + val hashMask = ctx.addMutableState("int", "hashMask", v => s"$v = 0;") + + def genHash(v: String): String = keyType match { + case BooleanType => s"($v ? 1 : 0)" + case ByteType | ShortType | IntegerType | DateType | _: YearMonthIntervalType => s"$v" + case LongType | TimestampType | TimestampNTZType | _: DayTimeIntervalType => + s"(int)($v ^ ($v >>> 32))" + case FloatType => s"Float.floatToIntBits($v)" + case DoubleType => + s"(int)(Double.doubleToLongBits($v) ^ (Double.doubleToLongBits($v) >>> 32))" + case _ => s"$v.hashCode()" + } + nullSafeCodeGen(ctx, ev, (eval1, eval2) => { + val i = ctx.freshName("i") + val h = ctx.freshName("h") + val cap = ctx.freshName("cap") + val idx = ctx.freshName("idx") + val candidate = ctx.freshName("candidate") + val loopKey = ctx.freshName("loopKey") + + val buildIndex = + s""" + |int $cap = Math.max(Integer.highestOneBit(Math.max($length * 2 - 1, 1)) << 1, 4); + |if ($hashBuckets == null || $hashBuckets.length < $cap) { + | $hashBuckets = new int[$cap]; + |} + |java.util.Arrays.fill($hashBuckets, 0, $cap, -1); + |$hashMask = $cap - 1; + |for (int $i = 0; $i < $length; $i++) { + | $keyJavaType $loopKey = ${CodeGenerator.getValue(keys, keyType, i)}; + | int $h = (${genHash(loopKey)}) & $hashMask; + | while ($hashBuckets[$h] != -1) { + | $h = ($h + 1) & $hashMask; + | } + | $hashBuckets[$h] = $i; + |} + |$lastKeyArray = $keys; + """.stripMargin + + val lookup = + s""" + |int $h = (${genHash(eval2)}) & $hashMask; + |$index = -1; + |while ($hashBuckets[$h] != -1) { + | int $idx = $hashBuckets[$h]; + | $keyJavaType $candidate = ${CodeGenerator.getValue(keys, keyType, idx)}; + | if (${ctx.genEqual(keyType, candidate, eval2)}) { + | $index = $idx; + | break; + | } + | $h = ($h + 1) & $hashMask; + |} + """.stripMargin + s""" final int $length = $eval1.numElements(); final ArrayData $keys = $eval1.keyArray(); final ArrayData $values = $eval1.valueArray(); + int $index = -1; - int $index = 0; - while ($index < $length) { - final $keyJavaType $key = ${CodeGenerator.getValue(keys, keyType, index)}; - if (${ctx.genEqual(keyType, key, eval2)}) { - break; - } else { - $index++; + if ($length >= $HASH_LOOKUP_THRESHOLD) { + if ($keys != $lastKeyArray) { + $buildIndex + } + $lookup + } else { + for (int $i = 0; $i < $length; $i++) { + $keyJavaType $loopKey = ${CodeGenerator.getValue(keys, keyType, i)}; + if (${ctx.genEqual(keyType, loopKey, eval2)}) { + $index = $i; + break; + } } } - if ($index == $length$nullCheck) { + if ($index < 0) { ${ev.isNull} = true; - } else { + } $nullValueCheck else { ${ev.value} = ${CodeGenerator.getValue(values, dataType, index)}; } """ @@ -547,15 +734,10 @@ case class GetMapValue(child: Expression, key: Expression) /** * `Null` is returned for invalid ordinals. - * - * TODO: We could make nullability more precise in foldable cases (e.g., literal input). - * But, since the key search is O(n), it takes much time to compute nullability. - * If we find efficient key searches, revisit this. */ override def nullable: Boolean = true override def dataType: DataType = child.dataType.asInstanceOf[MapType].valueType - // todo: current search is O(n), improve it. override def nullSafeEval(value: Any, ordinal: Any): Any = { getValueEval(value, ordinal, keyType, ordering) } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala index 1907ec7c23aa6..7c25c4e6c076a 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala @@ -29,7 +29,7 @@ import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch -import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils} +import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, DateTimeTestUtils, DateTimeUtils, GenericArrayData} import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{outstandingZoneIds, LA, UTC} import org.apache.spark.sql.catalyst.util.IntervalUtils._ import org.apache.spark.sql.catalyst.util.TypeUtils.ordinalNumber @@ -1960,6 +1960,30 @@ class CollectionExpressionsSuite checkEvaluation(ElementAt(mb0, Literal(Array[Byte](2, 1), BinaryType)), "2") checkEvaluation(ElementAt(mb0, Literal(Array[Byte](3, 4))), null) + // Int keys + val intMap = Literal.create(Map(1 -> 10, 2 -> 20, 3 -> 30), MapType(IntegerType, IntegerType)) + checkEvaluation(ElementAt(intMap, Literal(1)), 10) + checkEvaluation(ElementAt(intMap, Literal(2)), 20) + checkEvaluation(ElementAt(intMap, Literal(4)), null) + + // Duplicate keys + val keys = new GenericArrayData(Array(1, 2, 1)) + val values = new GenericArrayData(Array(10, 20, 30)) + val dupMapData = new ArrayBasedMapData(keys, values) + val dupMap = Literal.create(dupMapData, MapType(IntegerType, IntegerType)) + checkEvaluation(ElementAt(dupMap, Literal(1)), 10) + checkEvaluation(ElementAt(dupMap, Literal(2)), 20) + + // Null values + val nullValueMap = Literal.create(Map(1 -> null), MapType(IntegerType, StringType)) + checkEvaluation(ElementAt(nullValueMap, Literal(1)), null) + + // NaN keys + val nan = Double.NaN + val doubleMap = Literal.create(Map(1.0 -> 10, nan -> 20), MapType(DoubleType, IntegerType)) + checkEvaluation(ElementAt(doubleMap, Literal(1.0)), 10) + checkEvaluation(ElementAt(doubleMap, Literal(nan)), 20) + // test defaultValueOutOfBound withSQLConf(SQLConf.ANSI_ENABLED.key -> false.toString) { val delimiter = Literal.create(".", StringType) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala index 7baad5ea92a00..b6a48e5476e50 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala @@ -135,6 +135,7 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper { val nullMap = Literal.create(null, typeM) val nullString = Literal.create(null, StringType) + // 1. Basic lookup (String keys) checkEvaluation(GetMapValue(map, Literal("a")), "b") checkEvaluation(GetMapValue(map, nullString), null) checkEvaluation(GetMapValue(nullMap, nullString), null) @@ -143,8 +144,57 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper { val nonNullMap = Literal.create(Map("a" -> 1), MapType(StringType, IntegerType, false)) checkEvaluation(GetMapValue(nonNullMap, Literal("a")), 1) + // 2. Nested map val nestedMap = Literal.create(Map("a" -> Map("b" -> "c")), MapType(StringType, typeM)) checkEvaluation(GetMapValue(nestedMap, Literal("a")), Map("b" -> "c")) + + // 3. Basic lookup (Int keys) + val intMap = Literal.create(Map(1 -> 10, 2 -> 20, 3 -> 30), MapType(IntegerType, IntegerType)) + checkEvaluation(GetMapValue(intMap, Literal(1)), 10) + checkEvaluation(GetMapValue(intMap, Literal(2)), 20) + checkEvaluation(GetMapValue(intMap, Literal(3)), 30) + checkEvaluation(GetMapValue(intMap, Literal(4)), null) + + val emptyMap = Literal.create(Map.empty[Int, Int], MapType(IntegerType, IntegerType)) + checkEvaluation(GetMapValue(emptyMap, Literal(1)), null) + + // 4. Special data + // Duplicate keys: Spark MapType doesn't enforce uniqueness in the underlying + // data structure (ArrayBasedMapData) + // We construct it manually to simulate duplicates. + val keys = new GenericArrayData(Array(1, 2, 1)) + val values = new GenericArrayData(Array(10, 20, 30)) + val dupMapData = new ArrayBasedMapData(keys, values) + val dupMap = Literal.create(dupMapData, MapType(IntegerType, IntegerType)) + // Should return the first match + checkEvaluation(GetMapValue(dupMap, Literal(1)), 10) + checkEvaluation(GetMapValue(dupMap, Literal(2)), 20) + + // Null values + val nullValueMap = Literal.create(Map(1 -> null), MapType(IntegerType, StringType)) + checkEvaluation(GetMapValue(nullValueMap, Literal(1)), null) + + // NaN keys + val nan = Double.NaN + val floatNan = Float.NaN + val doubleMap = Literal.create(Map(1.0 -> 10, nan -> 20), MapType(DoubleType, IntegerType)) + checkEvaluation(GetMapValue(doubleMap, Literal(1.0)), 10) + checkEvaluation(GetMapValue(doubleMap, Literal(nan)), 20) + + val floatMap = Literal.create(Map(1.0f -> 10, floatNan -> 20), MapType(FloatType, IntegerType)) + checkEvaluation(GetMapValue(floatMap, Literal(1.0f)), 10) + checkEvaluation(GetMapValue(floatMap, Literal(floatNan)), 20) + + // 5. Key types + // Long + val longMap = Literal.create(Map(1L -> 10, 2L -> 20), MapType(LongType, IntegerType)) + checkEvaluation(GetMapValue(longMap, Literal(1L)), 10) + checkEvaluation(GetMapValue(longMap, Literal(3L)), null) + + // String + val stringMap = Literal.create(Map("a" -> "A", "b" -> "B"), MapType(StringType, StringType)) + checkEvaluation(GetMapValue(stringMap, Literal("a")), "A") + checkEvaluation(GetMapValue(stringMap, Literal("c")), null) } test("GetStructField") { From c46c9335cfd269b52cd575234beae435b87d780e Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 11 Mar 2026 10:52:47 +0800 Subject: [PATCH 03/24] add times --- .../sql/execution/benchmark/MapLookupBenchmark.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala index 3f9ebbde9b3d3..c8dab794a9720 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala @@ -38,6 +38,7 @@ import org.apache.spark.sql.types._ * }}} */ object MapLookupBenchmark extends SqlBasedBenchmark { + private val NUMBER_OF_ITER = 10 override def getSparkSession: SparkSession = { SparkSession.builder() @@ -57,6 +58,7 @@ object MapLookupBenchmark extends SqlBasedBenchmark { val benchmark = new Benchmark( s"MapLookup (size=$mapSize, hit=$hitRate, type=$keyType)", numRows, + NUMBER_OF_ITER, output = output) import spark.implicits._ @@ -96,7 +98,7 @@ object MapLookupBenchmark extends SqlBasedBenchmark { } } - benchmark.addCase("element_at interpreted") { _ => + benchmark.addCase("ElementAt interpreted") { _ => withSQLConf( SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false", SQLConf.CODEGEN_FACTORY_MODE.key -> "NO_CODEGEN", @@ -105,7 +107,7 @@ object MapLookupBenchmark extends SqlBasedBenchmark { } } - benchmark.addCase("element_at codegen") { _ => + benchmark.addCase("ElementAt codegen") { _ => withSQLConf( SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true", SQLConf.CODEGEN_FACTORY_MODE.key -> "CODEGEN_ONLY", @@ -118,7 +120,7 @@ object MapLookupBenchmark extends SqlBasedBenchmark { } override def runBenchmarkSuite(mainArgs: Array[String]): Unit = { - val sizes = Seq(10, 20, 30, 50, 1000, 1000000) + val sizes = Seq(10, 20, 30, 50, 100, 1000, 1000000) for (size <- sizes) { run(size, 1.0, IntegerType) run(size, 0.5, IntegerType) From f965a0cfc701e3528b43f06e2e7da2b539b840ac Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 11 Mar 2026 11:04:13 +0800 Subject: [PATCH 04/24] revert collectionOperations.scala --- .../spark/sql/catalyst/expressions/collectionOperations.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index 9ff6dc648c72e..dc3e6dcbd388c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -2741,9 +2741,7 @@ case class ElementAt( } } case _: MapType => - (value, ordinal) => { - getValueEval(value, ordinal, mapKeyType, ordering) - } + (value, ordinal) => getValueEval(value, ordinal, mapKeyType, ordering) } override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { From 4a61bc3a225522ecd9ddc588fc6b61c5d4bd81fc Mon Sep 17 00:00:00 2001 From: LuciferYang Date: Wed, 11 Mar 2026 03:27:38 +0000 Subject: [PATCH 05/24] Benchmark results for org.apache.spark.sql.execution.benchmark.MapLookupBenchmark (JDK 21, Scala 2.13, split 1 of 1) --- .../MapLookupBenchmark-jdk21-results.txt | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 sql/core/benchmarks/MapLookupBenchmark-jdk21-results.txt diff --git a/sql/core/benchmarks/MapLookupBenchmark-jdk21-results.txt b/sql/core/benchmarks/MapLookupBenchmark-jdk21-results.txt new file mode 100644 index 0000000000000..c7fde91b665c7 --- /dev/null +++ b/sql/core/benchmarks/MapLookupBenchmark-jdk21-results.txt @@ -0,0 +1,189 @@ +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=10, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 23 33 7 0.4 2348.6 1.0X +GetMapValue codegen 19 26 7 0.5 1886.6 1.2X +ElementAt interpreted 16 21 6 0.6 1629.6 1.4X +ElementAt codegen 15 19 5 0.7 1535.8 1.5X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=10, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 15 20 5 0.7 1470.4 1.0X +GetMapValue codegen 14 18 5 0.7 1415.1 1.0X +ElementAt interpreted 14 16 4 0.7 1365.1 1.1X +ElementAt codegen 14 16 4 0.7 1360.1 1.1X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=10, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 13 15 3 0.8 1296.3 1.0X +GetMapValue codegen 13 14 2 0.8 1275.5 1.0X +ElementAt interpreted 12 16 4 0.8 1244.6 1.0X +ElementAt codegen 12 14 2 0.8 1210.9 1.1X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=20, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 12 15 4 0.8 1225.0 1.0X +GetMapValue codegen 12 13 2 0.9 1153.9 1.1X +ElementAt interpreted 12 14 3 0.9 1172.0 1.0X +ElementAt codegen 11 13 3 0.9 1143.0 1.1X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=20, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 11 14 3 0.9 1126.6 1.0X +GetMapValue codegen 11 13 2 0.9 1113.0 1.0X +ElementAt interpreted 11 13 3 0.9 1132.7 1.0X +ElementAt codegen 11 13 3 0.9 1113.9 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=20, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 11 13 3 0.9 1086.0 1.0X +GetMapValue codegen 11 12 2 0.9 1072.5 1.0X +ElementAt interpreted 11 13 3 0.9 1068.9 1.0X +ElementAt codegen 11 12 2 0.9 1056.7 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=30, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 11 15 4 0.9 1097.5 1.0X +GetMapValue codegen 11 13 2 0.9 1077.5 1.0X +ElementAt interpreted 11 13 3 0.9 1085.0 1.0X +ElementAt codegen 11 12 2 0.9 1065.7 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=30, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 11 12 2 0.9 1057.1 1.0X +GetMapValue codegen 11 12 2 0.9 1059.0 1.0X +ElementAt interpreted 11 12 2 0.9 1058.0 1.0X +ElementAt codegen 10 12 2 1.0 1038.9 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=30, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 10 12 2 1.0 1036.2 1.0X +GetMapValue codegen 10 11 2 1.0 1010.6 1.0X +ElementAt interpreted 10 12 2 1.0 1028.0 1.0X +ElementAt codegen 10 12 2 1.0 1040.3 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=50, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 11 12 2 0.9 1059.2 1.0X +GetMapValue codegen 10 12 2 1.0 1042.8 1.0X +ElementAt interpreted 10 12 2 1.0 1048.8 1.0X +ElementAt codegen 10 12 2 1.0 1024.4 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=50, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 10 11 2 1.0 1031.8 1.0X +GetMapValue codegen 10 11 2 1.0 1013.5 1.0X +ElementAt interpreted 10 12 2 1.0 1034.6 1.0X +ElementAt codegen 10 11 2 1.0 1025.6 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=50, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 10 11 2 1.0 1012.2 1.0X +GetMapValue codegen 10 11 2 1.0 1003.7 1.0X +ElementAt interpreted 10 11 2 1.0 1014.9 1.0X +ElementAt codegen 10 11 2 1.0 1001.0 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=100, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 10 11 2 1.0 1049.6 1.0X +GetMapValue codegen 10 11 2 1.0 1016.8 1.0X +ElementAt interpreted 11 12 2 0.9 1053.8 1.0X +ElementAt codegen 10 12 2 1.0 1027.8 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=100, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 10 11 2 1.0 1026.7 1.0X +GetMapValue codegen 10 11 2 1.0 1007.9 1.0X +ElementAt interpreted 10 11 2 1.0 1026.9 1.0X +ElementAt codegen 10 12 3 1.0 1014.9 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=100, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 10 11 2 1.0 1006.2 1.0X +GetMapValue codegen 10 11 2 1.0 996.3 1.0X +ElementAt interpreted 10 11 2 1.0 1010.6 1.0X +ElementAt codegen 10 11 2 1.0 1002.2 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 11 12 2 0.9 1124.9 1.0X +GetMapValue codegen 11 12 2 0.9 1107.5 1.0X +ElementAt interpreted 11 12 2 0.9 1128.3 1.0X +ElementAt codegen 11 13 2 0.9 1128.3 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 11 12 2 0.9 1106.7 1.0X +GetMapValue codegen 11 13 2 0.9 1110.0 1.0X +ElementAt interpreted 11 12 2 0.9 1103.1 1.0X +ElementAt codegen 11 13 2 0.9 1112.3 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 11 12 2 0.9 1084.4 1.0X +GetMapValue codegen 11 12 2 0.9 1085.7 1.0X +ElementAt interpreted 11 12 2 0.9 1085.7 1.0X +ElementAt codegen 11 12 2 0.9 1077.6 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 1140 1391 203 0.0 114048.5 1.0X +GetMapValue codegen 1653 1744 81 0.0 165283.9 0.7X +ElementAt interpreted 1116 1310 137 0.0 111564.9 1.0X +ElementAt codegen 1554 1679 124 0.0 155392.1 0.7X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 1224 1314 91 0.0 122399.4 1.0X +GetMapValue codegen 1534 1652 119 0.0 153415.9 0.8X +ElementAt interpreted 1116 1361 196 0.0 111570.5 1.1X +ElementAt codegen 1540 1752 141 0.0 153989.8 0.8X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 1108 1344 125 0.0 110803.0 1.0X +GetMapValue codegen 1594 1719 103 0.0 159373.4 0.7X +ElementAt interpreted 1229 1370 106 0.0 122949.0 0.9X +ElementAt codegen 1618 1774 94 0.0 161812.6 0.7X + From bcaba7c1b1fb3335b829c25e64944cc64df44ddd Mon Sep 17 00:00:00 2001 From: LuciferYang Date: Wed, 11 Mar 2026 03:29:13 +0000 Subject: [PATCH 06/24] Benchmark results for org.apache.spark.sql.execution.benchmark.MapLookupBenchmark (JDK 17, Scala 2.13, split 1 of 1) --- .../benchmarks/MapLookupBenchmark-results.txt | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 sql/core/benchmarks/MapLookupBenchmark-results.txt diff --git a/sql/core/benchmarks/MapLookupBenchmark-results.txt b/sql/core/benchmarks/MapLookupBenchmark-results.txt new file mode 100644 index 0000000000000..cc3a43ac8ad2b --- /dev/null +++ b/sql/core/benchmarks/MapLookupBenchmark-results.txt @@ -0,0 +1,189 @@ +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=10, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 31 43 8 0.3 3143.6 1.0X +GetMapValue codegen 29 33 3 0.3 2867.2 1.1X +ElementAt interpreted 26 29 3 0.4 2577.6 1.2X +ElementAt codegen 25 29 3 0.4 2533.6 1.2X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=10, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 24 26 2 0.4 2394.0 1.0X +GetMapValue codegen 24 26 2 0.4 2382.6 1.0X +ElementAt interpreted 23 25 2 0.4 2314.3 1.0X +ElementAt codegen 23 26 3 0.4 2307.6 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=10, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 23 24 2 0.4 2261.5 1.0X +GetMapValue codegen 22 24 2 0.4 2246.5 1.0X +ElementAt interpreted 22 24 2 0.5 2209.2 1.0X +ElementAt codegen 22 24 2 0.5 2216.9 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=20, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 22 24 2 0.5 2189.3 1.0X +GetMapValue codegen 22 23 2 0.5 2167.5 1.0X +ElementAt interpreted 22 24 2 0.5 2153.5 1.0X +ElementAt codegen 21 23 2 0.5 2105.6 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=20, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 21 23 2 0.5 2073.8 1.0X +GetMapValue codegen 21 22 2 0.5 2092.6 1.0X +ElementAt interpreted 21 22 2 0.5 2077.5 1.0X +ElementAt codegen 21 22 3 0.5 2061.6 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=20, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 20 22 2 0.5 1980.1 1.0X +GetMapValue codegen 20 21 2 0.5 1989.1 1.0X +ElementAt interpreted 20 22 2 0.5 2026.5 1.0X +ElementAt codegen 20 22 3 0.5 1999.5 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=30, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 20 21 2 0.5 1990.9 1.0X +GetMapValue codegen 20 21 2 0.5 1987.8 1.0X +ElementAt interpreted 20 22 2 0.5 2023.0 1.0X +ElementAt codegen 20 22 1 0.5 2010.2 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=30, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 20 21 1 0.5 1968.0 1.0X +GetMapValue codegen 20 21 2 0.5 2007.8 1.0X +ElementAt interpreted 20 22 3 0.5 2028.0 1.0X +ElementAt codegen 20 22 4 0.5 1996.1 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=30, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 20 21 2 0.5 1958.4 1.0X +GetMapValue codegen 20 21 2 0.5 1962.2 1.0X +ElementAt interpreted 19 20 1 0.5 1912.2 1.0X +ElementAt codegen 19 21 1 0.5 1941.0 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=50, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 19 20 1 0.5 1949.0 1.0X +GetMapValue codegen 20 22 3 0.5 1999.2 1.0X +ElementAt interpreted 20 22 3 0.5 1950.2 1.0X +ElementAt codegen 20 21 1 0.5 1960.9 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=50, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 20 21 1 0.5 1961.0 1.0X +GetMapValue codegen 20 20 1 0.5 1952.1 1.0X +ElementAt interpreted 20 21 2 0.5 1976.4 1.0X +ElementAt codegen 19 21 1 0.5 1942.2 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=50, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 19 21 3 0.5 1922.4 1.0X +GetMapValue codegen 19 21 1 0.5 1931.6 1.0X +ElementAt interpreted 19 20 1 0.5 1896.8 1.0X +ElementAt codegen 20 21 2 0.5 1986.9 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=100, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 20 21 2 0.5 1962.9 1.0X +GetMapValue codegen 20 21 2 0.5 1954.9 1.0X +ElementAt interpreted 20 21 1 0.5 1981.1 1.0X +ElementAt codegen 20 21 1 0.5 1970.0 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=100, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 19 20 2 0.5 1894.6 1.0X +GetMapValue codegen 19 21 2 0.5 1923.4 1.0X +ElementAt interpreted 19 20 2 0.5 1903.4 1.0X +ElementAt codegen 19 20 1 0.5 1910.0 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=100, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 20 21 3 0.5 1955.7 1.0X +GetMapValue codegen 19 21 3 0.5 1940.5 1.0X +ElementAt interpreted 19 20 1 0.5 1920.4 1.0X +ElementAt codegen 19 20 1 0.5 1949.3 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=1000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 21 22 2 0.5 2075.1 1.0X +GetMapValue codegen 20 21 1 0.5 2037.5 1.0X +ElementAt interpreted 20 22 2 0.5 2015.2 1.0X +ElementAt codegen 21 22 2 0.5 2072.9 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=1000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 20 22 1 0.5 2043.4 1.0X +GetMapValue codegen 20 22 2 0.5 2028.0 1.0X +ElementAt interpreted 21 22 1 0.5 2051.1 1.0X +ElementAt codegen 20 22 1 0.5 2038.6 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=1000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 20 21 1 0.5 2026.8 1.0X +GetMapValue codegen 21 22 2 0.5 2055.1 1.0X +ElementAt interpreted 20 22 2 0.5 2032.8 1.0X +ElementAt codegen 21 22 1 0.5 2055.3 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=1000000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 1869 1994 120 0.0 186905.9 1.0X +GetMapValue codegen 2579 2686 69 0.0 257878.2 0.7X +ElementAt interpreted 1862 1942 83 0.0 186158.8 1.0X +ElementAt codegen 2540 2654 119 0.0 254041.8 0.7X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=1000000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 1796 1886 62 0.0 179574.7 1.0X +GetMapValue codegen 2463 2646 171 0.0 246291.1 0.7X +ElementAt interpreted 1731 1996 140 0.0 173109.4 1.0X +ElementAt codegen 2511 2751 152 0.0 251082.8 0.7X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=1000000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 1739 1981 174 0.0 173911.3 1.0X +GetMapValue codegen 2415 2693 171 0.0 241502.8 0.7X +ElementAt interpreted 1824 1909 74 0.0 182404.5 1.0X +ElementAt codegen 2497 2612 86 0.0 249658.0 0.7X + From 8e5b7952a9d9ed4b034a66aca96cddbcee504466 Mon Sep 17 00:00:00 2001 From: LuciferYang Date: Wed, 11 Mar 2026 03:29:39 +0000 Subject: [PATCH 07/24] Benchmark results for org.apache.spark.sql.execution.benchmark.MapLookupBenchmark (JDK 25, Scala 2.13, split 1 of 1) --- .../MapLookupBenchmark-jdk25-results.txt | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 sql/core/benchmarks/MapLookupBenchmark-jdk25-results.txt diff --git a/sql/core/benchmarks/MapLookupBenchmark-jdk25-results.txt b/sql/core/benchmarks/MapLookupBenchmark-jdk25-results.txt new file mode 100644 index 0000000000000..af495cf62b535 --- /dev/null +++ b/sql/core/benchmarks/MapLookupBenchmark-jdk25-results.txt @@ -0,0 +1,189 @@ +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=10, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 28 40 9 0.4 2847.9 1.0X +GetMapValue codegen 21 29 6 0.5 2097.2 1.4X +ElementAt interpreted 19 23 4 0.5 1903.8 1.5X +ElementAt codegen 18 22 4 0.6 1758.6 1.6X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=10, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 16 20 4 0.6 1586.0 1.0X +GetMapValue codegen 16 20 3 0.6 1594.7 1.0X +ElementAt interpreted 17 21 3 0.6 1664.1 1.0X +ElementAt codegen 15 18 3 0.7 1500.7 1.1X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=10, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 14 17 3 0.7 1395.1 1.0X +GetMapValue codegen 14 17 3 0.7 1386.0 1.0X +ElementAt interpreted 13 16 3 0.7 1334.8 1.0X +ElementAt codegen 13 16 3 0.8 1275.5 1.1X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=20, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 12 16 3 0.8 1244.3 1.0X +GetMapValue codegen 12 16 4 0.8 1216.9 1.0X +ElementAt interpreted 12 14 2 0.9 1174.4 1.1X +ElementAt codegen 12 14 3 0.9 1171.2 1.1X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=20, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 11 14 3 0.9 1146.8 1.0X +GetMapValue codegen 11 14 3 0.9 1122.6 1.0X +ElementAt interpreted 11 13 2 0.9 1119.9 1.0X +ElementAt codegen 11 13 2 0.9 1110.7 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=20, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 11 13 2 0.9 1085.3 1.0X +GetMapValue codegen 11 13 3 0.9 1082.7 1.0X +ElementAt interpreted 11 14 3 0.9 1080.1 1.0X +ElementAt codegen 11 14 3 0.9 1071.7 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=30, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 11 13 3 0.9 1068.6 1.0X +GetMapValue codegen 11 14 2 0.9 1129.2 0.9X +ElementAt interpreted 11 14 2 0.9 1117.1 1.0X +ElementAt codegen 10 13 2 1.0 1038.4 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=30, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 11 13 3 0.9 1052.8 1.0X +GetMapValue codegen 10 12 2 1.0 1045.5 1.0X +ElementAt interpreted 10 12 2 1.0 1025.0 1.0X +ElementAt codegen 10 12 2 1.0 1014.0 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=30, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 10 12 2 1.0 1000.6 1.0X +GetMapValue codegen 10 12 2 1.0 997.3 1.0X +ElementAt interpreted 10 12 2 1.0 1012.2 1.0X +ElementAt codegen 10 12 2 1.0 1005.3 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=50, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 10 13 3 1.0 1004.4 1.0X +GetMapValue codegen 11 13 2 0.9 1114.0 0.9X +ElementAt interpreted 10 13 2 1.0 1023.3 1.0X +ElementAt codegen 10 13 2 1.0 1023.5 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=50, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 10 12 2 1.0 993.6 1.0X +GetMapValue codegen 10 12 2 1.0 998.2 1.0X +ElementAt interpreted 10 12 2 1.0 1011.1 1.0X +ElementAt codegen 10 13 2 1.0 991.0 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=50, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted 10 12 2 1.0 976.4 1.0X +GetMapValue codegen 10 12 2 1.0 988.4 1.0X +ElementAt interpreted 10 12 2 1.0 994.8 1.0X +ElementAt codegen 10 12 2 1.0 1019.9 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=100, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 10 12 2 1.0 1018.1 1.0X +GetMapValue codegen 10 12 2 1.0 1014.4 1.0X +ElementAt interpreted 10 13 2 1.0 1036.3 1.0X +ElementAt codegen 11 13 2 0.9 1096.3 0.9X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=100, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 10 12 2 1.0 991.5 1.0X +GetMapValue codegen 10 12 2 1.0 1011.0 1.0X +ElementAt interpreted 10 12 2 1.0 979.1 1.0X +ElementAt codegen 10 12 2 1.0 1003.2 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=100, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 10 12 2 1.0 984.9 1.0X +GetMapValue codegen 10 13 2 1.0 975.4 1.0X +ElementAt interpreted 11 14 3 0.9 1063.9 0.9X +ElementAt codegen 10 11 1 1.0 996.2 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 11 13 3 0.9 1099.6 1.0X +GetMapValue codegen 11 13 2 0.9 1092.8 1.0X +ElementAt interpreted 11 13 2 0.9 1104.1 1.0X +ElementAt codegen 11 13 2 0.9 1109.0 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 11 12 2 0.9 1077.5 1.0X +GetMapValue codegen 11 13 3 0.9 1128.5 1.0X +ElementAt interpreted 11 14 2 0.9 1086.1 1.0X +ElementAt codegen 12 15 3 0.8 1219.2 0.9X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 11 12 2 0.9 1081.0 1.0X +GetMapValue codegen 11 13 2 0.9 1076.3 1.0X +ElementAt interpreted 11 12 2 0.9 1071.7 1.0X +ElementAt codegen 11 13 2 0.9 1113.1 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 1569 1862 224 0.0 156906.4 1.0X +GetMapValue codegen 2190 2558 318 0.0 218954.4 0.7X +ElementAt interpreted 1662 1881 178 0.0 166221.9 0.9X +ElementAt codegen 2194 2518 308 0.0 219363.6 0.7X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 1232 1762 391 0.0 123159.1 1.0X +GetMapValue codegen 1780 2029 159 0.0 178017.3 0.7X +ElementAt interpreted 1404 1654 233 0.0 140373.1 0.9X +ElementAt codegen 1736 2273 361 0.0 173626.9 0.7X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted 1451 1665 200 0.0 145056.0 1.0X +GetMapValue codegen 1829 2341 408 0.0 182928.5 0.8X +ElementAt interpreted 1407 1868 267 0.0 140654.0 1.0X +ElementAt codegen 1882 2353 316 0.0 188232.9 0.8X + From b5d0d38370e8412cb6ae44a6650bf64bb03b5c7e Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 11 Mar 2026 12:41:51 +0800 Subject: [PATCH 08/24] add more tests --- .../expressions/complexTypeExtractors.scala | 9 +- .../CollectionExpressionsSuite.scala | 123 ++++++++++++++++++ .../expressions/ComplexTypeSuite.scala | 121 +++++++++++++++++ 3 files changed, 248 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala index d8a8d67112b50..e9e74e256d496 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala @@ -445,13 +445,12 @@ trait GetMapValueUtil extends BinaryExpression with ImplicitCastInputTypes { @transient private var lastIndex: java.util.HashMap[Any, Int] = _ /** - * Minimum map size to use hash-based lookup. - * Below this threshold, the overhead of building a hash index + * The threshold is chosen empirically. If the map size is small, the cost of building hash map * exceeds the cost of a linear scan. * The value 20 is chosen empirically; break-even is around 15-25 * elements for primitive key types. */ - private val HASH_LOOKUP_THRESHOLD = 20 + private val hashLookupThreshold = 20 private def getOrBuildIndex(map: MapData, keyType: DataType): java.util.HashMap[Any, Int] = { if (lastMap ne map) { @@ -478,7 +477,7 @@ trait GetMapValueUtil extends BinaryExpression with ImplicitCastInputTypes { val map = value.asInstanceOf[MapData] val length = map.numElements() - if (length < HASH_LOOKUP_THRESHOLD || !TypeUtils.typeWithProperEquals(keyType)) { + if (length < hashLookupThreshold || !TypeUtils.typeWithProperEquals(keyType)) { getValueEvalLinear(map, ordinal, keyType, ordering) } else { val idx = getOrBuildIndex(map, keyType).getOrDefault(ordinal, -1) @@ -674,7 +673,7 @@ trait GetMapValueUtil extends BinaryExpression with ImplicitCastInputTypes { final ArrayData $values = $eval1.valueArray(); int $index = -1; - if ($length >= $HASH_LOOKUP_THRESHOLD) { + if ($length >= $hashLookupThreshold) { if ($keys != $lastKeyArray) { $buildIndex } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala index 7c25c4e6c076a..899ef449eb4db 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala @@ -1983,7 +1983,130 @@ class CollectionExpressionsSuite val doubleMap = Literal.create(Map(1.0 -> 10, nan -> 20), MapType(DoubleType, IntegerType)) checkEvaluation(ElementAt(doubleMap, Literal(1.0)), 10) checkEvaluation(ElementAt(doubleMap, Literal(nan)), 20) + } + + test("SPARK-55959: elementAt - hash lookup") { + // 1. Large map (trigger hash lookup) + // The size must be larger than or equal to hashLookupThreshold (20) to trigger hash lookup. + val size = 30 + val largeKeys = (0 until size).toArray + val largeValues = largeKeys.map(_ * 10) + val largeMapData = new ArrayBasedMapData( + new GenericArrayData(largeKeys), new GenericArrayData(largeValues)) + val largeMap = Literal.create(largeMapData, MapType(IntegerType, IntegerType)) + + // Test hits + checkEvaluation(ElementAt(largeMap, Literal(0)), 0) + checkEvaluation(ElementAt(largeMap, Literal(15)), 150) + checkEvaluation(ElementAt(largeMap, Literal(29)), 290) + + // Test miss + checkEvaluation(ElementAt(largeMap, Literal(30)), null) + checkEvaluation(ElementAt(largeMap, Literal(-1)), null) + + // 2. Boundary check (size = 20, trigger hash lookup) + val size20 = 20 + val keys20 = (0 until size20).toArray + val values20 = keys20.map(_ * 10) + val mapData20 = new ArrayBasedMapData( + new GenericArrayData(keys20), new GenericArrayData(values20)) + val map20 = Literal.create(mapData20, MapType(IntegerType, IntegerType)) + + checkEvaluation(ElementAt(map20, Literal(0)), 0) + checkEvaluation(ElementAt(map20, Literal(19)), 190) + checkEvaluation(ElementAt(map20, Literal(20)), null) + + // 3. Null values + val keysWithNull = (0 until size).toArray + val valuesWithNull = keysWithNull.map { i => + if (i % 2 == 0) null else i * 10 + }.asInstanceOf[Array[Any]] + val mapDataWithNull = new ArrayBasedMapData( + new GenericArrayData(keysWithNull), new GenericArrayData(valuesWithNull)) + val mapWithNull = Literal.create(mapDataWithNull, MapType(IntegerType, IntegerType)) + + checkEvaluation(ElementAt(mapWithNull, Literal(0)), null) + checkEvaluation(ElementAt(mapWithNull, Literal(1)), 10) + + // 4. NaN keys + val sizeNan = 30 + val keysNan = (0 until sizeNan).map(i => if (i == 0) Double.NaN else i.toDouble).toArray + val valuesNan = keysNan.map(k => if (k.isNaN) 999 else k.toInt * 10) + val mapDataNan = new ArrayBasedMapData( + new GenericArrayData(keysNan), new GenericArrayData(valuesNan)) + val mapNan = Literal.create(mapDataNan, MapType(DoubleType, IntegerType)) + + checkEvaluation(ElementAt(mapNan, Literal(Double.NaN)), 999) + checkEvaluation(ElementAt(mapNan, Literal(1.0)), 10) + checkEvaluation(ElementAt(mapNan, Literal(2.0)), 20) + + // 5. Duplicate keys (First win) + val keysDup = (0 until size).flatMap(i => Seq(i, i)).toArray + val valuesDup = (0 until size).flatMap(i => Seq(i * 10, i * 100)).toArray + val mapDataDup = new ArrayBasedMapData( + new GenericArrayData(keysDup), new GenericArrayData(valuesDup)) + val mapDup = Literal.create(mapDataDup, MapType(IntegerType, IntegerType)) + + checkEvaluation(ElementAt(mapDup, Literal(0)), 0) + checkEvaluation(ElementAt(mapDup, Literal(10)), 100) + checkEvaluation(ElementAt(mapDup, Literal(29)), 290) + + // 6. Nested Map Value + val valuesNested = keys20.map { i => + val innerKey = i + val innerValue = i * 10 + new ArrayBasedMapData( + new GenericArrayData(Array(innerKey)), + new GenericArrayData(Array(innerValue))) + }.toArray + val mapDataNested = new ArrayBasedMapData( + new GenericArrayData(keys20), new GenericArrayData(valuesNested)) + val mapNested = Literal.create( + mapDataNested, MapType(IntegerType, MapType(IntegerType, IntegerType))) + + val innerMap0 = new ArrayBasedMapData( + new GenericArrayData(Array(0)), new GenericArrayData(Array(0))) + val innerMap10 = new ArrayBasedMapData( + new GenericArrayData(Array(10)), new GenericArrayData(Array(100))) + + checkEvaluation(ElementAt(mapNested, Literal(0)), innerMap0) + checkEvaluation(ElementAt(mapNested, Literal(10)), innerMap10) + + // 7. Binary Keys + val keysBinary = (0 until size).map(i => Array(i.toByte)).toArray + val valuesBinary = (0 until size).map(_ * 10).toArray + val mapDataBinary = new ArrayBasedMapData( + new GenericArrayData(keysBinary), new GenericArrayData(valuesBinary)) + val mapBinary = Literal.create(mapDataBinary, MapType(BinaryType, IntegerType)) + + checkEvaluation(ElementAt(mapBinary, Literal(Array(0.toByte))), 0) + checkEvaluation(ElementAt(mapBinary, Literal(Array(10.toByte))), 100) + + // 8. Array Keys + val keysArray = (0 until size).map(i => new GenericArrayData(Array(i))).toArray + val valuesArray = (0 until size).map(_ * 10).toArray + val mapDataArray = new ArrayBasedMapData( + new GenericArrayData(keysArray), new GenericArrayData(valuesArray)) + val mapArray = Literal.create( + mapDataArray, MapType(ArrayType(IntegerType), IntegerType)) + + checkEvaluation(ElementAt(mapArray, Literal(Array(0))), 0) + checkEvaluation(ElementAt(mapArray, Literal(Array(10))), 100) + + // 9. Struct Keys + val keysStruct = (0 until size).map(i => create_row(i)).toArray + val valuesStruct = (0 until size).map(_ * 10).toArray + val mapDataStruct = new ArrayBasedMapData( + new GenericArrayData(keysStruct), new GenericArrayData(valuesStruct)) + val structType = new StructType().add("a", "int") + val mapStruct = Literal.create( + mapDataStruct, MapType(structType, IntegerType)) + + checkEvaluation(ElementAt(mapStruct, Literal.create(create_row(0), structType)), 0) + checkEvaluation(ElementAt(mapStruct, Literal.create(create_row(10), structType)), 100) + } + test("defaultValueOutOfBound") { // test defaultValueOutOfBound withSQLConf(SQLConf.ANSI_ENABLED.key -> false.toString) { val delimiter = Literal.create(".", StringType) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala index b6a48e5476e50..3d058146eb485 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala @@ -197,6 +197,127 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(GetMapValue(stringMap, Literal("c")), null) } + test("SPARK-55959: GetMapValue - hash lookup") { + // 1. Large map (trigger hash lookup) + // The size must be larger than or equal to hashLookupThreshold (20) to trigger hash lookup. + val size = 30 + val largeKeys = (0 until size).toArray + val largeValues = largeKeys.map(_ * 10) + val largeMapData = new ArrayBasedMapData( + new GenericArrayData(largeKeys), new GenericArrayData(largeValues)) + val largeMap = Literal.create(largeMapData, MapType(IntegerType, IntegerType)) + + // Test hits + checkEvaluation(GetMapValue(largeMap, Literal(0)), 0) + checkEvaluation(GetMapValue(largeMap, Literal(15)), 150) + checkEvaluation(GetMapValue(largeMap, Literal(29)), 290) + + // Test miss + checkEvaluation(GetMapValue(largeMap, Literal(30)), null) + checkEvaluation(GetMapValue(largeMap, Literal(-1)), null) + + // 2. Boundary check (size = 20, trigger hash lookup) + val size20 = 20 + val keys20 = (0 until size20).toArray + val values20 = keys20.map(_ * 10) + val mapData20 = new ArrayBasedMapData( + new GenericArrayData(keys20), new GenericArrayData(values20)) + val map20 = Literal.create(mapData20, MapType(IntegerType, IntegerType)) + + checkEvaluation(GetMapValue(map20, Literal(0)), 0) + checkEvaluation(GetMapValue(map20, Literal(19)), 190) + checkEvaluation(GetMapValue(map20, Literal(20)), null) + + // 3. Null values + val keysWithNull = (0 until size).toArray + val valuesWithNull = keysWithNull.map { i => + if (i % 2 == 0) null else i * 10 + }.asInstanceOf[Array[Any]] + val mapDataWithNull = new ArrayBasedMapData( + new GenericArrayData(keysWithNull), new GenericArrayData(valuesWithNull)) + val mapWithNull = Literal.create(mapDataWithNull, MapType(IntegerType, IntegerType)) + + checkEvaluation(GetMapValue(mapWithNull, Literal(0)), null) + checkEvaluation(GetMapValue(mapWithNull, Literal(1)), 10) + + // 4. NaN keys + val sizeNan = 30 + val keysNan = (0 until sizeNan).map(i => if (i == 0) Double.NaN else i.toDouble).toArray + val valuesNan = keysNan.map(k => if (k.isNaN) 999 else k.toInt * 10) + val mapDataNan = new ArrayBasedMapData( + new GenericArrayData(keysNan), new GenericArrayData(valuesNan)) + val mapNan = Literal.create(mapDataNan, MapType(DoubleType, IntegerType)) + + checkEvaluation(GetMapValue(mapNan, Literal(Double.NaN)), 999) + checkEvaluation(GetMapValue(mapNan, Literal(1.0)), 10) + checkEvaluation(GetMapValue(mapNan, Literal(2.0)), 20) + + // 5. Duplicate keys (First win) + val keysDup = (0 until size).flatMap(i => Seq(i, i)).toArray + val valuesDup = (0 until size).flatMap(i => Seq(i * 10, i * 100)).toArray + val mapDataDup = new ArrayBasedMapData( + new GenericArrayData(keysDup), new GenericArrayData(valuesDup)) + val mapDup = Literal.create(mapDataDup, MapType(IntegerType, IntegerType)) + + checkEvaluation(GetMapValue(mapDup, Literal(0)), 0) + checkEvaluation(GetMapValue(mapDup, Literal(10)), 100) + checkEvaluation(GetMapValue(mapDup, Literal(29)), 290) + + // 6. Nested Map Value + val valuesNested = keys20.map { i => + val innerKey = i + val innerValue = i * 10 + new ArrayBasedMapData( + new GenericArrayData(Array(innerKey)), + new GenericArrayData(Array(innerValue))) + }.toArray + val mapDataNested = new ArrayBasedMapData( + new GenericArrayData(keys20), new GenericArrayData(valuesNested)) + val mapNested = Literal.create( + mapDataNested, MapType(IntegerType, MapType(IntegerType, IntegerType))) + + val innerMap0 = new ArrayBasedMapData( + new GenericArrayData(Array(0)), new GenericArrayData(Array(0))) + val innerMap10 = new ArrayBasedMapData( + new GenericArrayData(Array(10)), new GenericArrayData(Array(100))) + + checkEvaluation(GetMapValue(mapNested, Literal(0)), innerMap0) + checkEvaluation(GetMapValue(mapNested, Literal(10)), innerMap10) + + // 7. Binary Keys + val keysBinary = (0 until size).map(i => Array(i.toByte)).toArray + val valuesBinary = (0 until size).map(_ * 10).toArray + val mapDataBinary = new ArrayBasedMapData( + new GenericArrayData(keysBinary), new GenericArrayData(valuesBinary)) + val mapBinary = Literal.create(mapDataBinary, MapType(BinaryType, IntegerType)) + + checkEvaluation(GetMapValue(mapBinary, Literal(Array(0.toByte))), 0) + checkEvaluation(GetMapValue(mapBinary, Literal(Array(10.toByte))), 100) + + // 8. Array Keys + val keysArray = (0 until size).map(i => new GenericArrayData(Array(i))).toArray + val valuesArray = (0 until size).map(_ * 10).toArray + val mapDataArray = new ArrayBasedMapData( + new GenericArrayData(keysArray), new GenericArrayData(valuesArray)) + val mapArray = Literal.create( + mapDataArray, MapType(ArrayType(IntegerType), IntegerType)) + + checkEvaluation(GetMapValue(mapArray, Literal(Array(0))), 0) + checkEvaluation(GetMapValue(mapArray, Literal(Array(10))), 100) + + // 9. Struct Keys + val keysStruct = (0 until size).map(i => create_row(i)).toArray + val valuesStruct = (0 until size).map(_ * 10).toArray + val mapDataStruct = new ArrayBasedMapData( + new GenericArrayData(keysStruct), new GenericArrayData(valuesStruct)) + val structType = new StructType().add("a", "int") + val mapStruct = Literal.create( + mapDataStruct, MapType(structType, IntegerType)) + + checkEvaluation(GetMapValue(mapStruct, Literal.create(create_row(0), structType)), 0) + checkEvaluation(GetMapValue(mapStruct, Literal.create(create_row(10), structType)), 100) + } + test("GetStructField") { val typeS = StructType(StructField("a", IntegerType) :: Nil) val struct = Literal.create(create_row(1), typeS) From 8b33dc7a26e9857fcb3516b9b4288953626099e6 Mon Sep 17 00:00:00 2001 From: LuciferYang Date: Wed, 11 Mar 2026 05:46:59 +0000 Subject: [PATCH 09/24] Benchmark results for org.apache.spark.sql.execution.benchmark.MapLookupBenchmark (JDK 17, Scala 2.13, split 1 of 1) --- .../benchmarks/MapLookupBenchmark-results.txt | 210 +++++++++--------- 1 file changed, 105 insertions(+), 105 deletions(-) diff --git a/sql/core/benchmarks/MapLookupBenchmark-results.txt b/sql/core/benchmarks/MapLookupBenchmark-results.txt index cc3a43ac8ad2b..15d02baa87adc 100644 --- a/sql/core/benchmarks/MapLookupBenchmark-results.txt +++ b/sql/core/benchmarks/MapLookupBenchmark-results.txt @@ -1,189 +1,189 @@ OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=10, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 31 43 8 0.3 3143.6 1.0X -GetMapValue codegen 29 33 3 0.3 2867.2 1.1X -ElementAt interpreted 26 29 3 0.4 2577.6 1.2X -ElementAt codegen 25 29 3 0.4 2533.6 1.2X +GetMapValue interpreted 27 37 5 0.4 2738.9 1.0X +GetMapValue codegen 22 27 3 0.4 2233.6 1.2X +ElementAt interpreted 20 23 2 0.5 2025.3 1.4X +ElementAt codegen 20 23 3 0.5 2039.7 1.3X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=10, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 24 26 2 0.4 2394.0 1.0X -GetMapValue codegen 24 26 2 0.4 2382.6 1.0X -ElementAt interpreted 23 25 2 0.4 2314.3 1.0X -ElementAt codegen 23 26 3 0.4 2307.6 1.0X +GetMapValue interpreted 19 21 2 0.5 1870.3 1.0X +GetMapValue codegen 18 20 2 0.6 1781.2 1.1X +ElementAt interpreted 18 20 2 0.6 1793.5 1.0X +ElementAt codegen 18 20 3 0.6 1798.2 1.0X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=10, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 23 24 2 0.4 2261.5 1.0X -GetMapValue codegen 22 24 2 0.4 2246.5 1.0X -ElementAt interpreted 22 24 2 0.5 2209.2 1.0X -ElementAt codegen 22 24 2 0.5 2216.9 1.0X +GetMapValue interpreted 17 19 2 0.6 1693.8 1.0X +GetMapValue codegen 17 19 2 0.6 1725.0 1.0X +ElementAt interpreted 16 18 2 0.6 1629.6 1.0X +ElementAt codegen 17 18 2 0.6 1677.1 1.0X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=20, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 22 24 2 0.5 2189.3 1.0X -GetMapValue codegen 22 23 2 0.5 2167.5 1.0X -ElementAt interpreted 22 24 2 0.5 2153.5 1.0X -ElementAt codegen 21 23 2 0.5 2105.6 1.0X +GetMapValue interpreted 16 18 2 0.6 1629.6 1.0X +GetMapValue codegen 16 18 2 0.6 1635.5 1.0X +ElementAt interpreted 16 18 2 0.6 1616.7 1.0X +ElementAt codegen 15 17 1 0.7 1514.8 1.1X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=20, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 21 23 2 0.5 2073.8 1.0X -GetMapValue codegen 21 22 2 0.5 2092.6 1.0X -ElementAt interpreted 21 22 2 0.5 2077.5 1.0X -ElementAt codegen 21 22 3 0.5 2061.6 1.0X +GetMapValue interpreted 15 17 2 0.7 1495.1 1.0X +GetMapValue codegen 15 16 1 0.7 1491.6 1.0X +ElementAt interpreted 15 16 1 0.7 1537.1 1.0X +ElementAt codegen 15 16 1 0.7 1487.9 1.0X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=20, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 20 22 2 0.5 1980.1 1.0X -GetMapValue codegen 20 21 2 0.5 1989.1 1.0X -ElementAt interpreted 20 22 2 0.5 2026.5 1.0X -ElementAt codegen 20 22 3 0.5 1999.5 1.0X +GetMapValue interpreted 14 16 2 0.7 1437.7 1.0X +GetMapValue codegen 14 16 1 0.7 1417.4 1.0X +ElementAt interpreted 14 16 1 0.7 1415.1 1.0X +ElementAt codegen 14 16 2 0.7 1424.2 1.0X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=30, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 20 21 2 0.5 1990.9 1.0X -GetMapValue codegen 20 21 2 0.5 1987.8 1.0X -ElementAt interpreted 20 22 2 0.5 2023.0 1.0X -ElementAt codegen 20 22 1 0.5 2010.2 1.0X +GetMapValue interpreted 15 16 1 0.7 1456.4 1.0X +GetMapValue codegen 14 16 2 0.7 1413.5 1.0X +ElementAt interpreted 14 16 2 0.7 1425.1 1.0X +ElementAt codegen 14 16 1 0.7 1420.6 1.0X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=30, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 20 21 1 0.5 1968.0 1.0X -GetMapValue codegen 20 21 2 0.5 2007.8 1.0X -ElementAt interpreted 20 22 3 0.5 2028.0 1.0X -ElementAt codegen 20 22 4 0.5 1996.1 1.0X +GetMapValue interpreted 14 16 2 0.7 1400.7 1.0X +GetMapValue codegen 14 16 2 0.7 1396.3 1.0X +ElementAt interpreted 14 16 2 0.7 1405.0 1.0X +ElementAt codegen 14 16 2 0.7 1391.3 1.0X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=30, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 20 21 2 0.5 1958.4 1.0X -GetMapValue codegen 20 21 2 0.5 1962.2 1.0X -ElementAt interpreted 19 20 1 0.5 1912.2 1.0X -ElementAt codegen 19 21 1 0.5 1941.0 1.0X +GetMapValue interpreted 14 16 2 0.7 1417.6 1.0X +GetMapValue codegen 14 15 1 0.7 1414.6 1.0X +ElementAt interpreted 15 15 1 0.7 1457.7 1.0X +ElementAt codegen 14 15 1 0.7 1375.1 1.0X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=50, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 19 20 1 0.5 1949.0 1.0X -GetMapValue codegen 20 22 3 0.5 1999.2 1.0X -ElementAt interpreted 20 22 3 0.5 1950.2 1.0X -ElementAt codegen 20 21 1 0.5 1960.9 1.0X +GetMapValue interpreted 14 15 1 0.7 1372.3 1.0X +GetMapValue codegen 14 15 1 0.7 1366.4 1.0X +ElementAt interpreted 14 15 1 0.7 1392.2 1.0X +ElementAt codegen 14 15 1 0.7 1424.9 1.0X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=50, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 20 21 1 0.5 1961.0 1.0X -GetMapValue codegen 20 20 1 0.5 1952.1 1.0X -ElementAt interpreted 20 21 2 0.5 1976.4 1.0X -ElementAt codegen 19 21 1 0.5 1942.2 1.0X +GetMapValue interpreted 14 15 1 0.7 1393.0 1.0X +GetMapValue codegen 14 15 1 0.7 1358.4 1.0X +ElementAt interpreted 14 15 1 0.7 1368.7 1.0X +ElementAt codegen 14 15 1 0.7 1381.5 1.0X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=50, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 19 21 3 0.5 1922.4 1.0X -GetMapValue codegen 19 21 1 0.5 1931.6 1.0X -ElementAt interpreted 19 20 1 0.5 1896.8 1.0X -ElementAt codegen 20 21 2 0.5 1986.9 1.0X +GetMapValue interpreted 14 15 1 0.7 1415.1 1.0X +GetMapValue codegen 14 15 1 0.7 1366.2 1.0X +ElementAt interpreted 14 15 1 0.7 1383.8 1.0X +ElementAt codegen 14 15 1 0.7 1442.1 1.0X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=100, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 20 21 2 0.5 1962.9 1.0X -GetMapValue codegen 20 21 2 0.5 1954.9 1.0X -ElementAt interpreted 20 21 1 0.5 1981.1 1.0X -ElementAt codegen 20 21 1 0.5 1970.0 1.0X +GetMapValue interpreted 15 15 1 0.7 1456.2 1.0X +GetMapValue codegen 14 15 2 0.7 1446.4 1.0X +ElementAt interpreted 14 15 1 0.7 1380.4 1.1X +ElementAt codegen 14 15 2 0.7 1366.7 1.1X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=100, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 19 20 2 0.5 1894.6 1.0X -GetMapValue codegen 19 21 2 0.5 1923.4 1.0X -ElementAt interpreted 19 20 2 0.5 1903.4 1.0X -ElementAt codegen 19 20 1 0.5 1910.0 1.0X +GetMapValue interpreted 14 15 1 0.7 1447.0 1.0X +GetMapValue codegen 14 15 1 0.7 1436.3 1.0X +ElementAt interpreted 14 15 1 0.7 1396.4 1.0X +ElementAt codegen 14 15 1 0.7 1431.0 1.0X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=100, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 20 21 3 0.5 1955.7 1.0X -GetMapValue codegen 19 21 3 0.5 1940.5 1.0X -ElementAt interpreted 19 20 1 0.5 1920.4 1.0X -ElementAt codegen 19 20 1 0.5 1949.3 1.0X +GetMapValue interpreted 13 15 1 0.7 1335.0 1.0X +GetMapValue codegen 14 15 1 0.7 1424.9 0.9X +ElementAt interpreted 14 15 1 0.7 1436.3 0.9X +ElementAt codegen 14 15 1 0.7 1415.2 0.9X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=1000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 21 22 2 0.5 2075.1 1.0X -GetMapValue codegen 20 21 1 0.5 2037.5 1.0X -ElementAt interpreted 20 22 2 0.5 2015.2 1.0X -ElementAt codegen 21 22 2 0.5 2072.9 1.0X +GetMapValue interpreted 15 16 1 0.7 1529.7 1.0X +GetMapValue codegen 15 16 1 0.7 1533.2 1.0X +ElementAt interpreted 14 16 1 0.7 1439.3 1.1X +ElementAt codegen 15 16 2 0.7 1478.1 1.0X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=1000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 20 22 1 0.5 2043.4 1.0X -GetMapValue codegen 20 22 2 0.5 2028.0 1.0X -ElementAt interpreted 21 22 1 0.5 2051.1 1.0X -ElementAt codegen 20 22 1 0.5 2038.6 1.0X +GetMapValue interpreted 15 16 1 0.7 1519.3 1.0X +GetMapValue codegen 15 16 1 0.7 1520.4 1.0X +ElementAt interpreted 15 16 1 0.7 1473.4 1.0X +ElementAt codegen 14 16 1 0.7 1431.1 1.1X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=1000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 20 21 1 0.5 2026.8 1.0X -GetMapValue codegen 21 22 2 0.5 2055.1 1.0X -ElementAt interpreted 20 22 2 0.5 2032.8 1.0X -ElementAt codegen 21 22 1 0.5 2055.3 1.0X +GetMapValue interpreted 14 15 1 0.7 1410.9 1.0X +GetMapValue codegen 14 16 1 0.7 1432.1 1.0X +ElementAt interpreted 14 15 1 0.7 1413.7 1.0X +ElementAt codegen 14 16 1 0.7 1436.3 1.0X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=1000000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ----------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 1869 1994 120 0.0 186905.9 1.0X -GetMapValue codegen 2579 2686 69 0.0 257878.2 0.7X -ElementAt interpreted 1862 1942 83 0.0 186158.8 1.0X -ElementAt codegen 2540 2654 119 0.0 254041.8 0.7X +GetMapValue interpreted 1077 1302 119 0.0 107704.4 1.0X +GetMapValue codegen 1500 1623 113 0.0 150002.9 0.7X +ElementAt interpreted 1156 1257 96 0.0 115601.2 0.9X +ElementAt codegen 1452 1618 128 0.0 145230.3 0.7X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=1000000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ----------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 1796 1886 62 0.0 179574.7 1.0X -GetMapValue codegen 2463 2646 171 0.0 246291.1 0.7X -ElementAt interpreted 1731 1996 140 0.0 173109.4 1.0X -ElementAt codegen 2511 2751 152 0.0 251082.8 0.7X +GetMapValue interpreted 1071 1347 258 0.0 107077.6 1.0X +GetMapValue codegen 1472 1675 145 0.0 147167.3 0.7X +ElementAt interpreted 1094 1286 106 0.0 109421.5 1.0X +ElementAt codegen 1499 1703 167 0.0 149893.2 0.7X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=1000000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ----------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 1739 1981 174 0.0 173911.3 1.0X -GetMapValue codegen 2415 2693 171 0.0 241502.8 0.7X -ElementAt interpreted 1824 1909 74 0.0 182404.5 1.0X -ElementAt codegen 2497 2612 86 0.0 249658.0 0.7X +GetMapValue interpreted 1050 1244 188 0.0 105042.5 1.0X +GetMapValue codegen 1423 1582 107 0.0 142278.0 0.7X +ElementAt interpreted 1202 1291 56 0.0 120209.1 0.9X +ElementAt codegen 1541 1619 32 0.0 154101.3 0.7X From f7d29cbf7f20c97f127eaef7e3b20f753ae3e059 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 12 Mar 2026 15:22:44 +0800 Subject: [PATCH 10/24] use putIfAbsent --- .../spark/sql/catalyst/expressions/complexTypeExtractors.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala index e9e74e256d496..0c063a95b317e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala @@ -460,7 +460,7 @@ trait GetMapValueUtil extends BinaryExpression with ImplicitCastInputTypes { var i = 0 while (i < len) { val k = keys.get(i, keyType) - if (!hm.containsKey(k)) hm.put(k, i) + hm.putIfAbsent(k, i) i += 1 } lastIndex = hm From e3ac77ea7393f6ff830b3196908fe71c2415b896 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 12 Mar 2026 16:33:01 +0800 Subject: [PATCH 11/24] add config --- .../expressions/complexTypeExtractors.scala | 10 +++++----- .../org/apache/spark/sql/internal/SQLConf.scala | 14 +++++++++++++- .../catalyst/expressions/ComplexTypeSuite.scala | 3 ++- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala index 0c063a95b317e..0c40c30d6c810 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala @@ -445,12 +445,12 @@ trait GetMapValueUtil extends BinaryExpression with ImplicitCastInputTypes { @transient private var lastIndex: java.util.HashMap[Any, Int] = _ /** - * The threshold is chosen empirically. If the map size is small, the cost of building hash map - * exceeds the cost of a linear scan. - * The value 20 is chosen empirically; break-even is around 15-25 - * elements for primitive key types. + * The threshold to determine whether to use hash lookup for map lookup expressions. + * If the map size is small, the cost of building hash map exceeds the cost of a linear scan. + * This is configured by `spark.sql.mapLookupHashThreshold`. */ - private val hashLookupThreshold = 20 + @transient private lazy val hashLookupThreshold = + SQLConf.get.getConf(SQLConf.MAP_LOOKUP_HASH_THRESHOLD) private def getOrBuildIndex(map: MapData, keyType: DataType): java.util.HashMap[Any, Int] = { if (lastMap ne map) { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 84f2f6b90aa7a..545e9c62ac8e3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2494,10 +2494,22 @@ object SQLConf { "otherwise if the size is above the threshold, it'll use broadcast variable. Note that " + "maximum string length allowed in Java is Integer.MAX_VALUE, so anything above it would " + "be meaningless. The default value is set to -1 (disabled by default).") - .version("4.0.0") + .version("4.2.0") .intConf .createWithDefault(-1) + val MAP_LOOKUP_HASH_THRESHOLD = buildConf("spark.sql.mapLookupHashThreshold") + .doc("The threshold to use hash lookup for map lookup expressions. When the map size is " + + "larger than this threshold, we will build a hash map for lookup. Otherwise, we will use " + + "linear scan. This is only effective when the map key type supports hash lookup. " + + "To disable hash lookup, set this to a very large number. " + + "To always use hash lookup, set this to 0. " + + "This configuration affects `map[key]` and `element_at(map, key)`.") + .version("4.2.0") + .intConf + .checkValue(v => v >= 0, "The threshold must be non-negative.") + .createWithDefault(20) + val FILES_MAX_PARTITION_BYTES = buildConf("spark.sql.files.maxPartitionBytes") .doc("The maximum number of bytes to pack into a single partition when reading files. " + "This configuration is effective only when using file-based sources such as Parquet, JSON " + diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala index 3d058146eb485..6952449c900a8 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala @@ -129,7 +129,8 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper { assert(GetArrayItem(stArray4, Literal(1)).nullable) } - test("GetMapValue") { + test("SPARK-55959: GetMapValue - hash lookup") { + def runLookupTests(): Unit = { val typeM = MapType(StringType, StringType) val map = Literal.create(Map("a" -> "b"), typeM) val nullMap = Literal.create(null, typeM) From dcad5e5104627e46f5296a7d64cf86a9464bc066 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 12 Mar 2026 16:51:57 +0800 Subject: [PATCH 12/24] refactor test --- .../apache/spark/sql/internal/SQLConf.scala | 13 +- .../CollectionExpressionsSuite.scala | 330 +++++++----------- .../expressions/ComplexTypeSuite.scala | 284 +++++---------- 3 files changed, 222 insertions(+), 405 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 545e9c62ac8e3..8d9edf9e39462 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2494,21 +2494,16 @@ object SQLConf { "otherwise if the size is above the threshold, it'll use broadcast variable. Note that " + "maximum string length allowed in Java is Integer.MAX_VALUE, so anything above it would " + "be meaningless. The default value is set to -1 (disabled by default).") - .version("4.2.0") + .version("4.0.0") .intConf .createWithDefault(-1) val MAP_LOOKUP_HASH_THRESHOLD = buildConf("spark.sql.mapLookupHashThreshold") - .doc("The threshold to use hash lookup for map lookup expressions. When the map size is " + - "larger than this threshold, we will build a hash map for lookup. Otherwise, we will use " + - "linear scan. This is only effective when the map key type supports hash lookup. " + - "To disable hash lookup, set this to a very large number. " + - "To always use hash lookup, set this to 0. " + - "This configuration affects `map[key]` and `element_at(map, key)`.") + .doc("The threshold to determine whether to use hash lookup for map lookup expressions. " + + "If the map size is small, the cost of building hash map exceeds the cost of a linear scan.") .version("4.2.0") .intConf - .checkValue(v => v >= 0, "The threshold must be non-negative.") - .createWithDefault(20) + .createWithDefault(1000) val FILES_MAX_PARTITION_BYTES = buildConf("spark.sql.files.maxPartitionBytes") .doc("The maximum number of bytes to pack into a single partition when reading files. " + diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala index 899ef449eb4db..997080dd30687 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala @@ -1887,224 +1887,138 @@ class CollectionExpressionsSuite checkEvaluation(ArrayPosition(aa1, aae), 0L) } - test("elementAt") { - val a0 = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType)) - val a1 = Literal.create(Seq[String](null, ""), ArrayType(StringType)) - val a2 = Literal.create(Seq(null), ArrayType(LongType)) - val a3 = Literal.create(null, ArrayType(StringType)) - - intercept[Exception] { - checkEvaluation(ElementAt(a0, Literal(0)), null) - }.getMessage.contains("SQL array indices start at 1") - intercept[Exception] { checkEvaluation(ElementAt(a0, Literal(1.1)), null) } - withSQLConf(SQLConf.ANSI_ENABLED.key -> false.toString) { - checkEvaluation(ElementAt(a0, Literal(4)), null) - checkEvaluation(ElementAt(a0, Literal(-4)), null) - } - - checkEvaluation(ElementAt(a0, Literal(1)), 1) - checkEvaluation(ElementAt(a0, Literal(2)), 2) - checkEvaluation(ElementAt(a0, Literal(3)), 3) - checkEvaluation(ElementAt(a0, Literal(-3)), 1) - checkEvaluation(ElementAt(a0, Literal(-2)), 2) - checkEvaluation(ElementAt(a0, Literal(-1)), 3) - - checkEvaluation(ElementAt(a1, Literal(1)), null) - checkEvaluation(ElementAt(a1, Literal(2)), "") - checkEvaluation(ElementAt(a1, Literal(-2)), null) - checkEvaluation(ElementAt(a1, Literal(-1)), "") - - checkEvaluation(ElementAt(a2, Literal(1)), null) - - checkEvaluation(ElementAt(a3, Literal(1)), null) - - - val m0 = - Literal.create(Map("a" -> "1", "b" -> "2", "c" -> null), MapType(StringType, StringType)) - val m1 = Literal.create(Map[String, String](), MapType(StringType, StringType)) - val m2 = Literal.create(null, MapType(StringType, StringType)) + Seq((Int.MaxValue, "Linear"), (0, "Hash Lookup")).foreach { case (threshold, name) => + test(s"elementAt - $name") { + withSQLConf(SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> threshold.toString) { + val a0 = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType)) + val a1 = Literal.create(Seq[String](null, ""), ArrayType(StringType)) + val a2 = Literal.create(Seq(null), ArrayType(LongType)) + val a3 = Literal.create(null, ArrayType(StringType)) + + intercept[Exception] { + checkEvaluation(ElementAt(a0, Literal(0)), null) + }.getMessage.contains("SQL array indices start at 1") + intercept[Exception] { checkEvaluation(ElementAt(a0, Literal(1.1)), null) } + withSQLConf(SQLConf.ANSI_ENABLED.key -> false.toString) { + checkEvaluation(ElementAt(a0, Literal(4)), null) + checkEvaluation(ElementAt(a0, Literal(-4)), null) + } - assert(ElementAt(m0, Literal(1.0)).checkInputDataTypes() == - DataTypeMismatch( - errorSubClass = "MAP_FUNCTION_DIFF_TYPES", - messageParameters = Map( - "functionName" -> "`element_at`", - "dataType" -> "\"MAP\"", - "leftType" -> "\"MAP\"", - "rightType" -> "\"DOUBLE\"" + checkEvaluation(ElementAt(a0, Literal(1)), 1) + checkEvaluation(ElementAt(a0, Literal(2)), 2) + checkEvaluation(ElementAt(a0, Literal(3)), 3) + checkEvaluation(ElementAt(a0, Literal(-3)), 1) + checkEvaluation(ElementAt(a0, Literal(-2)), 2) + checkEvaluation(ElementAt(a0, Literal(-1)), 3) + + checkEvaluation(ElementAt(a1, Literal(1)), null) + checkEvaluation(ElementAt(a1, Literal(2)), "") + checkEvaluation(ElementAt(a1, Literal(-2)), null) + checkEvaluation(ElementAt(a1, Literal(-1)), "") + + checkEvaluation(ElementAt(a2, Literal(1)), null) + + checkEvaluation(ElementAt(a3, Literal(1)), null) + + + val m0 = + Literal.create(Map("a" -> "1", "b" -> "2", "c" -> null), + MapType(StringType, StringType)) + val m1 = Literal.create(Map[String, String](), MapType(StringType, StringType)) + val m2 = Literal.create(null, MapType(StringType, StringType)) + + assert(ElementAt(m0, Literal(1.0)).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "MAP_FUNCTION_DIFF_TYPES", + messageParameters = Map( + "functionName" -> "`element_at`", + "dataType" -> "\"MAP\"", + "leftType" -> "\"MAP\"", + "rightType" -> "\"DOUBLE\"" + ) + ) ) - ) - ) - withSQLConf(SQLConf.ANSI_ENABLED.key -> false.toString) { - checkEvaluation(ElementAt(m0, Literal("d")), null) - checkEvaluation(ElementAt(m1, Literal("a")), null) - } + withSQLConf(SQLConf.ANSI_ENABLED.key -> false.toString) { + checkEvaluation(ElementAt(m0, Literal("d")), null) + checkEvaluation(ElementAt(m1, Literal("a")), null) + } - checkEvaluation(ElementAt(m0, Literal("a")), "1") - checkEvaluation(ElementAt(m0, Literal("b")), "2") - checkEvaluation(ElementAt(m0, Literal("c")), null) + checkEvaluation(ElementAt(m0, Literal("a")), "1") + checkEvaluation(ElementAt(m0, Literal("b")), "2") + checkEvaluation(ElementAt(m0, Literal("c")), null) - checkEvaluation(ElementAt(m2, Literal("a")), null) + checkEvaluation(ElementAt(m2, Literal("a")), null) - // test binary type as keys - val mb0 = Literal.create( - Map(Array[Byte](1, 2) -> "1", Array[Byte](3, 4) -> null, Array[Byte](2, 1) -> "2"), - MapType(BinaryType, StringType)) - val mb1 = Literal.create(Map[Array[Byte], String](), MapType(BinaryType, StringType)) + // test binary type as keys + val mb0 = Literal.create( + Map(Array[Byte](1, 2) -> "1", Array[Byte](3, 4) -> null, Array[Byte](2, 1) -> "2"), + MapType(BinaryType, StringType)) + val mb1 = Literal.create(Map[Array[Byte], String](), MapType(BinaryType, StringType)) - withSQLConf(SQLConf.ANSI_ENABLED.key -> false.toString) { - checkEvaluation(ElementAt(mb0, Literal(Array[Byte](1, 2, 3))), null) - checkEvaluation(ElementAt(mb1, Literal(Array[Byte](1, 2))), null) + withSQLConf(SQLConf.ANSI_ENABLED.key -> false.toString) { + checkEvaluation(ElementAt(mb0, Literal(Array[Byte](1, 2, 3))), null) + checkEvaluation(ElementAt(mb1, Literal(Array[Byte](1, 2))), null) + } + checkEvaluation(ElementAt(mb0, Literal(Array[Byte](2, 1), BinaryType)), "2") + checkEvaluation(ElementAt(mb0, Literal(Array[Byte](3, 4))), null) + + // Int keys + val intMap = Literal.create(Map(1 -> 10, 2 -> 20, 3 -> 30), + MapType(IntegerType, IntegerType)) + checkEvaluation(ElementAt(intMap, Literal(1)), 10) + checkEvaluation(ElementAt(intMap, Literal(2)), 20) + checkEvaluation(ElementAt(intMap, Literal(4)), null) + + // Duplicate keys + val keys = new GenericArrayData(Array(1, 2, 1)) + val values = new GenericArrayData(Array(10, 20, 30)) + val dupMapData = new ArrayBasedMapData(keys, values) + val dupMap = Literal.create(dupMapData, MapType(IntegerType, IntegerType)) + checkEvaluation(ElementAt(dupMap, Literal(1)), 10) + checkEvaluation(ElementAt(dupMap, Literal(2)), 20) + + // Null values + val nullValueMap = Literal.create(Map(1 -> null), MapType(IntegerType, StringType)) + checkEvaluation(ElementAt(nullValueMap, Literal(1)), null) + + // NaN keys + val nan = Double.NaN + val doubleMap = Literal.create(Map(1.0 -> 10, nan -> 20), + MapType(DoubleType, IntegerType)) + checkEvaluation(ElementAt(doubleMap, Literal(1.0)), 10) + checkEvaluation(ElementAt(doubleMap, Literal(nan)), 20) + + // Nested Map Value + val mapNested = Literal.create( + Map(1 -> Map(10 -> 100), 2 -> Map(20 -> 200)), + MapType(IntegerType, MapType(IntegerType, IntegerType))) + checkEvaluation(ElementAt(mapNested, Literal(1)), Map(10 -> 100)) + checkEvaluation(ElementAt(mapNested, Literal(2)), Map(20 -> 200)) + checkEvaluation(ElementAt(mapNested, Literal(3)), null) + + // Array Keys + val arrayType = ArrayType(IntegerType) + val arrayMap = Literal.create( + Map(Array(1, 2) -> 10, Array(3, 4) -> 20), + MapType(arrayType, IntegerType)) + checkEvaluation(ElementAt(arrayMap, Literal.create(Array(1, 2), arrayType)), 10) + checkEvaluation(ElementAt(arrayMap, Literal.create(Array(3, 4), arrayType)), 20) + checkEvaluation(ElementAt(arrayMap, Literal.create(Array(5, 6), arrayType)), null) + + // Struct Keys + val structType = new StructType().add("a", "int").add("b", "int") + val structMap = Literal.create( + Map(create_row(1, 1) -> 10, create_row(2, 2) -> 20), + MapType(structType, IntegerType)) + checkEvaluation(ElementAt(structMap, Literal.create(create_row(1, 1), structType)), 10) + checkEvaluation(ElementAt(structMap, Literal.create(create_row(2, 2), structType)), 20) + checkEvaluation(ElementAt(structMap, Literal.create(create_row(3, 3), structType)), null) + } } - checkEvaluation(ElementAt(mb0, Literal(Array[Byte](2, 1), BinaryType)), "2") - checkEvaluation(ElementAt(mb0, Literal(Array[Byte](3, 4))), null) - - // Int keys - val intMap = Literal.create(Map(1 -> 10, 2 -> 20, 3 -> 30), MapType(IntegerType, IntegerType)) - checkEvaluation(ElementAt(intMap, Literal(1)), 10) - checkEvaluation(ElementAt(intMap, Literal(2)), 20) - checkEvaluation(ElementAt(intMap, Literal(4)), null) - - // Duplicate keys - val keys = new GenericArrayData(Array(1, 2, 1)) - val values = new GenericArrayData(Array(10, 20, 30)) - val dupMapData = new ArrayBasedMapData(keys, values) - val dupMap = Literal.create(dupMapData, MapType(IntegerType, IntegerType)) - checkEvaluation(ElementAt(dupMap, Literal(1)), 10) - checkEvaluation(ElementAt(dupMap, Literal(2)), 20) - - // Null values - val nullValueMap = Literal.create(Map(1 -> null), MapType(IntegerType, StringType)) - checkEvaluation(ElementAt(nullValueMap, Literal(1)), null) - - // NaN keys - val nan = Double.NaN - val doubleMap = Literal.create(Map(1.0 -> 10, nan -> 20), MapType(DoubleType, IntegerType)) - checkEvaluation(ElementAt(doubleMap, Literal(1.0)), 10) - checkEvaluation(ElementAt(doubleMap, Literal(nan)), 20) } - test("SPARK-55959: elementAt - hash lookup") { - // 1. Large map (trigger hash lookup) - // The size must be larger than or equal to hashLookupThreshold (20) to trigger hash lookup. - val size = 30 - val largeKeys = (0 until size).toArray - val largeValues = largeKeys.map(_ * 10) - val largeMapData = new ArrayBasedMapData( - new GenericArrayData(largeKeys), new GenericArrayData(largeValues)) - val largeMap = Literal.create(largeMapData, MapType(IntegerType, IntegerType)) - - // Test hits - checkEvaluation(ElementAt(largeMap, Literal(0)), 0) - checkEvaluation(ElementAt(largeMap, Literal(15)), 150) - checkEvaluation(ElementAt(largeMap, Literal(29)), 290) - - // Test miss - checkEvaluation(ElementAt(largeMap, Literal(30)), null) - checkEvaluation(ElementAt(largeMap, Literal(-1)), null) - - // 2. Boundary check (size = 20, trigger hash lookup) - val size20 = 20 - val keys20 = (0 until size20).toArray - val values20 = keys20.map(_ * 10) - val mapData20 = new ArrayBasedMapData( - new GenericArrayData(keys20), new GenericArrayData(values20)) - val map20 = Literal.create(mapData20, MapType(IntegerType, IntegerType)) - - checkEvaluation(ElementAt(map20, Literal(0)), 0) - checkEvaluation(ElementAt(map20, Literal(19)), 190) - checkEvaluation(ElementAt(map20, Literal(20)), null) - - // 3. Null values - val keysWithNull = (0 until size).toArray - val valuesWithNull = keysWithNull.map { i => - if (i % 2 == 0) null else i * 10 - }.asInstanceOf[Array[Any]] - val mapDataWithNull = new ArrayBasedMapData( - new GenericArrayData(keysWithNull), new GenericArrayData(valuesWithNull)) - val mapWithNull = Literal.create(mapDataWithNull, MapType(IntegerType, IntegerType)) - - checkEvaluation(ElementAt(mapWithNull, Literal(0)), null) - checkEvaluation(ElementAt(mapWithNull, Literal(1)), 10) - - // 4. NaN keys - val sizeNan = 30 - val keysNan = (0 until sizeNan).map(i => if (i == 0) Double.NaN else i.toDouble).toArray - val valuesNan = keysNan.map(k => if (k.isNaN) 999 else k.toInt * 10) - val mapDataNan = new ArrayBasedMapData( - new GenericArrayData(keysNan), new GenericArrayData(valuesNan)) - val mapNan = Literal.create(mapDataNan, MapType(DoubleType, IntegerType)) - - checkEvaluation(ElementAt(mapNan, Literal(Double.NaN)), 999) - checkEvaluation(ElementAt(mapNan, Literal(1.0)), 10) - checkEvaluation(ElementAt(mapNan, Literal(2.0)), 20) - - // 5. Duplicate keys (First win) - val keysDup = (0 until size).flatMap(i => Seq(i, i)).toArray - val valuesDup = (0 until size).flatMap(i => Seq(i * 10, i * 100)).toArray - val mapDataDup = new ArrayBasedMapData( - new GenericArrayData(keysDup), new GenericArrayData(valuesDup)) - val mapDup = Literal.create(mapDataDup, MapType(IntegerType, IntegerType)) - - checkEvaluation(ElementAt(mapDup, Literal(0)), 0) - checkEvaluation(ElementAt(mapDup, Literal(10)), 100) - checkEvaluation(ElementAt(mapDup, Literal(29)), 290) - - // 6. Nested Map Value - val valuesNested = keys20.map { i => - val innerKey = i - val innerValue = i * 10 - new ArrayBasedMapData( - new GenericArrayData(Array(innerKey)), - new GenericArrayData(Array(innerValue))) - }.toArray - val mapDataNested = new ArrayBasedMapData( - new GenericArrayData(keys20), new GenericArrayData(valuesNested)) - val mapNested = Literal.create( - mapDataNested, MapType(IntegerType, MapType(IntegerType, IntegerType))) - - val innerMap0 = new ArrayBasedMapData( - new GenericArrayData(Array(0)), new GenericArrayData(Array(0))) - val innerMap10 = new ArrayBasedMapData( - new GenericArrayData(Array(10)), new GenericArrayData(Array(100))) - - checkEvaluation(ElementAt(mapNested, Literal(0)), innerMap0) - checkEvaluation(ElementAt(mapNested, Literal(10)), innerMap10) - - // 7. Binary Keys - val keysBinary = (0 until size).map(i => Array(i.toByte)).toArray - val valuesBinary = (0 until size).map(_ * 10).toArray - val mapDataBinary = new ArrayBasedMapData( - new GenericArrayData(keysBinary), new GenericArrayData(valuesBinary)) - val mapBinary = Literal.create(mapDataBinary, MapType(BinaryType, IntegerType)) - - checkEvaluation(ElementAt(mapBinary, Literal(Array(0.toByte))), 0) - checkEvaluation(ElementAt(mapBinary, Literal(Array(10.toByte))), 100) - - // 8. Array Keys - val keysArray = (0 until size).map(i => new GenericArrayData(Array(i))).toArray - val valuesArray = (0 until size).map(_ * 10).toArray - val mapDataArray = new ArrayBasedMapData( - new GenericArrayData(keysArray), new GenericArrayData(valuesArray)) - val mapArray = Literal.create( - mapDataArray, MapType(ArrayType(IntegerType), IntegerType)) - - checkEvaluation(ElementAt(mapArray, Literal(Array(0))), 0) - checkEvaluation(ElementAt(mapArray, Literal(Array(10))), 100) - - // 9. Struct Keys - val keysStruct = (0 until size).map(i => create_row(i)).toArray - val valuesStruct = (0 until size).map(_ * 10).toArray - val mapDataStruct = new ArrayBasedMapData( - new GenericArrayData(keysStruct), new GenericArrayData(valuesStruct)) - val structType = new StructType().add("a", "int") - val mapStruct = Literal.create( - mapDataStruct, MapType(structType, IntegerType)) - - checkEvaluation(ElementAt(mapStruct, Literal.create(create_row(0), structType)), 0) - checkEvaluation(ElementAt(mapStruct, Literal.create(create_row(10), structType)), 100) - } + test("defaultValueOutOfBound") { // test defaultValueOutOfBound diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala index 6952449c900a8..d785dc247ae60 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala @@ -129,194 +129,102 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper { assert(GetArrayItem(stArray4, Literal(1)).nullable) } - test("SPARK-55959: GetMapValue - hash lookup") { - def runLookupTests(): Unit = { - val typeM = MapType(StringType, StringType) - val map = Literal.create(Map("a" -> "b"), typeM) - val nullMap = Literal.create(null, typeM) - val nullString = Literal.create(null, StringType) - - // 1. Basic lookup (String keys) - checkEvaluation(GetMapValue(map, Literal("a")), "b") - checkEvaluation(GetMapValue(map, nullString), null) - checkEvaluation(GetMapValue(nullMap, nullString), null) - checkEvaluation(GetMapValue(map, nullString), null) - - val nonNullMap = Literal.create(Map("a" -> 1), MapType(StringType, IntegerType, false)) - checkEvaluation(GetMapValue(nonNullMap, Literal("a")), 1) - - // 2. Nested map - val nestedMap = Literal.create(Map("a" -> Map("b" -> "c")), MapType(StringType, typeM)) - checkEvaluation(GetMapValue(nestedMap, Literal("a")), Map("b" -> "c")) - - // 3. Basic lookup (Int keys) - val intMap = Literal.create(Map(1 -> 10, 2 -> 20, 3 -> 30), MapType(IntegerType, IntegerType)) - checkEvaluation(GetMapValue(intMap, Literal(1)), 10) - checkEvaluation(GetMapValue(intMap, Literal(2)), 20) - checkEvaluation(GetMapValue(intMap, Literal(3)), 30) - checkEvaluation(GetMapValue(intMap, Literal(4)), null) - - val emptyMap = Literal.create(Map.empty[Int, Int], MapType(IntegerType, IntegerType)) - checkEvaluation(GetMapValue(emptyMap, Literal(1)), null) - - // 4. Special data - // Duplicate keys: Spark MapType doesn't enforce uniqueness in the underlying - // data structure (ArrayBasedMapData) - // We construct it manually to simulate duplicates. - val keys = new GenericArrayData(Array(1, 2, 1)) - val values = new GenericArrayData(Array(10, 20, 30)) - val dupMapData = new ArrayBasedMapData(keys, values) - val dupMap = Literal.create(dupMapData, MapType(IntegerType, IntegerType)) - // Should return the first match - checkEvaluation(GetMapValue(dupMap, Literal(1)), 10) - checkEvaluation(GetMapValue(dupMap, Literal(2)), 20) - - // Null values - val nullValueMap = Literal.create(Map(1 -> null), MapType(IntegerType, StringType)) - checkEvaluation(GetMapValue(nullValueMap, Literal(1)), null) - - // NaN keys - val nan = Double.NaN - val floatNan = Float.NaN - val doubleMap = Literal.create(Map(1.0 -> 10, nan -> 20), MapType(DoubleType, IntegerType)) - checkEvaluation(GetMapValue(doubleMap, Literal(1.0)), 10) - checkEvaluation(GetMapValue(doubleMap, Literal(nan)), 20) - - val floatMap = Literal.create(Map(1.0f -> 10, floatNan -> 20), MapType(FloatType, IntegerType)) - checkEvaluation(GetMapValue(floatMap, Literal(1.0f)), 10) - checkEvaluation(GetMapValue(floatMap, Literal(floatNan)), 20) - - // 5. Key types - // Long - val longMap = Literal.create(Map(1L -> 10, 2L -> 20), MapType(LongType, IntegerType)) - checkEvaluation(GetMapValue(longMap, Literal(1L)), 10) - checkEvaluation(GetMapValue(longMap, Literal(3L)), null) - - // String - val stringMap = Literal.create(Map("a" -> "A", "b" -> "B"), MapType(StringType, StringType)) - checkEvaluation(GetMapValue(stringMap, Literal("a")), "A") - checkEvaluation(GetMapValue(stringMap, Literal("c")), null) - } - - test("SPARK-55959: GetMapValue - hash lookup") { - // 1. Large map (trigger hash lookup) - // The size must be larger than or equal to hashLookupThreshold (20) to trigger hash lookup. - val size = 30 - val largeKeys = (0 until size).toArray - val largeValues = largeKeys.map(_ * 10) - val largeMapData = new ArrayBasedMapData( - new GenericArrayData(largeKeys), new GenericArrayData(largeValues)) - val largeMap = Literal.create(largeMapData, MapType(IntegerType, IntegerType)) - - // Test hits - checkEvaluation(GetMapValue(largeMap, Literal(0)), 0) - checkEvaluation(GetMapValue(largeMap, Literal(15)), 150) - checkEvaluation(GetMapValue(largeMap, Literal(29)), 290) - - // Test miss - checkEvaluation(GetMapValue(largeMap, Literal(30)), null) - checkEvaluation(GetMapValue(largeMap, Literal(-1)), null) - - // 2. Boundary check (size = 20, trigger hash lookup) - val size20 = 20 - val keys20 = (0 until size20).toArray - val values20 = keys20.map(_ * 10) - val mapData20 = new ArrayBasedMapData( - new GenericArrayData(keys20), new GenericArrayData(values20)) - val map20 = Literal.create(mapData20, MapType(IntegerType, IntegerType)) - - checkEvaluation(GetMapValue(map20, Literal(0)), 0) - checkEvaluation(GetMapValue(map20, Literal(19)), 190) - checkEvaluation(GetMapValue(map20, Literal(20)), null) - - // 3. Null values - val keysWithNull = (0 until size).toArray - val valuesWithNull = keysWithNull.map { i => - if (i % 2 == 0) null else i * 10 - }.asInstanceOf[Array[Any]] - val mapDataWithNull = new ArrayBasedMapData( - new GenericArrayData(keysWithNull), new GenericArrayData(valuesWithNull)) - val mapWithNull = Literal.create(mapDataWithNull, MapType(IntegerType, IntegerType)) - - checkEvaluation(GetMapValue(mapWithNull, Literal(0)), null) - checkEvaluation(GetMapValue(mapWithNull, Literal(1)), 10) - - // 4. NaN keys - val sizeNan = 30 - val keysNan = (0 until sizeNan).map(i => if (i == 0) Double.NaN else i.toDouble).toArray - val valuesNan = keysNan.map(k => if (k.isNaN) 999 else k.toInt * 10) - val mapDataNan = new ArrayBasedMapData( - new GenericArrayData(keysNan), new GenericArrayData(valuesNan)) - val mapNan = Literal.create(mapDataNan, MapType(DoubleType, IntegerType)) - - checkEvaluation(GetMapValue(mapNan, Literal(Double.NaN)), 999) - checkEvaluation(GetMapValue(mapNan, Literal(1.0)), 10) - checkEvaluation(GetMapValue(mapNan, Literal(2.0)), 20) - - // 5. Duplicate keys (First win) - val keysDup = (0 until size).flatMap(i => Seq(i, i)).toArray - val valuesDup = (0 until size).flatMap(i => Seq(i * 10, i * 100)).toArray - val mapDataDup = new ArrayBasedMapData( - new GenericArrayData(keysDup), new GenericArrayData(valuesDup)) - val mapDup = Literal.create(mapDataDup, MapType(IntegerType, IntegerType)) - - checkEvaluation(GetMapValue(mapDup, Literal(0)), 0) - checkEvaluation(GetMapValue(mapDup, Literal(10)), 100) - checkEvaluation(GetMapValue(mapDup, Literal(29)), 290) - - // 6. Nested Map Value - val valuesNested = keys20.map { i => - val innerKey = i - val innerValue = i * 10 - new ArrayBasedMapData( - new GenericArrayData(Array(innerKey)), - new GenericArrayData(Array(innerValue))) - }.toArray - val mapDataNested = new ArrayBasedMapData( - new GenericArrayData(keys20), new GenericArrayData(valuesNested)) - val mapNested = Literal.create( - mapDataNested, MapType(IntegerType, MapType(IntegerType, IntegerType))) - - val innerMap0 = new ArrayBasedMapData( - new GenericArrayData(Array(0)), new GenericArrayData(Array(0))) - val innerMap10 = new ArrayBasedMapData( - new GenericArrayData(Array(10)), new GenericArrayData(Array(100))) - - checkEvaluation(GetMapValue(mapNested, Literal(0)), innerMap0) - checkEvaluation(GetMapValue(mapNested, Literal(10)), innerMap10) - - // 7. Binary Keys - val keysBinary = (0 until size).map(i => Array(i.toByte)).toArray - val valuesBinary = (0 until size).map(_ * 10).toArray - val mapDataBinary = new ArrayBasedMapData( - new GenericArrayData(keysBinary), new GenericArrayData(valuesBinary)) - val mapBinary = Literal.create(mapDataBinary, MapType(BinaryType, IntegerType)) - - checkEvaluation(GetMapValue(mapBinary, Literal(Array(0.toByte))), 0) - checkEvaluation(GetMapValue(mapBinary, Literal(Array(10.toByte))), 100) - - // 8. Array Keys - val keysArray = (0 until size).map(i => new GenericArrayData(Array(i))).toArray - val valuesArray = (0 until size).map(_ * 10).toArray - val mapDataArray = new ArrayBasedMapData( - new GenericArrayData(keysArray), new GenericArrayData(valuesArray)) - val mapArray = Literal.create( - mapDataArray, MapType(ArrayType(IntegerType), IntegerType)) - - checkEvaluation(GetMapValue(mapArray, Literal(Array(0))), 0) - checkEvaluation(GetMapValue(mapArray, Literal(Array(10))), 100) - - // 9. Struct Keys - val keysStruct = (0 until size).map(i => create_row(i)).toArray - val valuesStruct = (0 until size).map(_ * 10).toArray - val mapDataStruct = new ArrayBasedMapData( - new GenericArrayData(keysStruct), new GenericArrayData(valuesStruct)) - val structType = new StructType().add("a", "int") - val mapStruct = Literal.create( - mapDataStruct, MapType(structType, IntegerType)) - - checkEvaluation(GetMapValue(mapStruct, Literal.create(create_row(0), structType)), 0) - checkEvaluation(GetMapValue(mapStruct, Literal.create(create_row(10), structType)), 100) + Seq((Int.MaxValue, "Linear"), (0, "Hash Lookup")).foreach { case (threshold, name) => + test(s"GetMapValue - $name") { + withSQLConf(SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> threshold.toString) { + val typeM = MapType(StringType, StringType) + val map = Literal.create(Map("a" -> "b"), typeM) + val nullMap = Literal.create(null, typeM) + val nullString = Literal.create(null, StringType) + + // 1. Basic lookup (String keys) + checkEvaluation(GetMapValue(map, Literal("a")), "b") + checkEvaluation(GetMapValue(map, nullString), null) + checkEvaluation(GetMapValue(nullMap, nullString), null) + checkEvaluation(GetMapValue(map, nullString), null) + + val nonNullMap = Literal.create(Map("a" -> 1), MapType(StringType, IntegerType, false)) + checkEvaluation(GetMapValue(nonNullMap, Literal("a")), 1) + + // 2. Nested map + val nestedMap = Literal.create(Map("a" -> Map("b" -> "c")), MapType(StringType, typeM)) + checkEvaluation(GetMapValue(nestedMap, Literal("a")), Map("b" -> "c")) + + // 3. Basic lookup (Int keys) + val intMap = Literal.create(Map(1 -> 10, 2 -> 20, 3 -> 30), + MapType(IntegerType, IntegerType)) + checkEvaluation(GetMapValue(intMap, Literal(1)), 10) + checkEvaluation(GetMapValue(intMap, Literal(2)), 20) + checkEvaluation(GetMapValue(intMap, Literal(3)), 30) + checkEvaluation(GetMapValue(intMap, Literal(4)), null) + + val emptyMap = Literal.create(Map.empty[Int, Int], MapType(IntegerType, IntegerType)) + checkEvaluation(GetMapValue(emptyMap, Literal(1)), null) + + // 4. Special data + // Duplicate keys: Spark MapType doesn't enforce uniqueness in the underlying + // data structure (ArrayBasedMapData) + // We construct it manually to simulate duplicates. + val keys = new GenericArrayData(Array(1, 2, 1)) + val values = new GenericArrayData(Array(10, 20, 30)) + val dupMapData = new ArrayBasedMapData(keys, values) + val dupMap = Literal.create(dupMapData, MapType(IntegerType, IntegerType)) + // Should return the first match + checkEvaluation(GetMapValue(dupMap, Literal(1)), 10) + checkEvaluation(GetMapValue(dupMap, Literal(2)), 20) + + // Null values + val nullValueMap = Literal.create(Map(1 -> null), MapType(IntegerType, StringType)) + checkEvaluation(GetMapValue(nullValueMap, Literal(1)), null) + + // NaN keys + val nan = Double.NaN + val floatNan = Float.NaN + val doubleMap = Literal.create(Map(1.0 -> 10, nan -> 20), MapType(DoubleType, IntegerType)) + checkEvaluation(GetMapValue(doubleMap, Literal(1.0)), 10) + checkEvaluation(GetMapValue(doubleMap, Literal(nan)), 20) + + val floatMap = Literal.create(Map(1.0f -> 10, floatNan -> 20), + MapType(FloatType, IntegerType)) + checkEvaluation(GetMapValue(floatMap, Literal(1.0f)), 10) + checkEvaluation(GetMapValue(floatMap, Literal(floatNan)), 20) + + // 5. Key types + // Long + val longMap = Literal.create(Map(1L -> 10, 2L -> 20), MapType(LongType, IntegerType)) + checkEvaluation(GetMapValue(longMap, Literal(1L)), 10) + checkEvaluation(GetMapValue(longMap, Literal(3L)), null) + + // String + val stringMap = Literal.create(Map("a" -> "A", "b" -> "B"), MapType(StringType, StringType)) + checkEvaluation(GetMapValue(stringMap, Literal("a")), "A") + checkEvaluation(GetMapValue(stringMap, Literal("c")), null) + + // 6. Binary Keys + val binaryMap = Literal.create(Map(Array(1.toByte) -> 10, Array(2.toByte) -> 20), + MapType(BinaryType, IntegerType)) + checkEvaluation(GetMapValue(binaryMap, Literal(Array(1.toByte))), 10) + checkEvaluation(GetMapValue(binaryMap, Literal(Array(3.toByte))), null) + + // 7. Array Keys + val arrayType = ArrayType(IntegerType) + val arrayMap = Literal.create( + Map(Array(1, 2) -> 10, Array(3, 4) -> 20), + MapType(arrayType, IntegerType)) + checkEvaluation(GetMapValue(arrayMap, Literal.create(Array(1, 2), arrayType)), 10) + checkEvaluation(GetMapValue(arrayMap, Literal.create(Array(3, 4), arrayType)), 20) + checkEvaluation(GetMapValue(arrayMap, Literal.create(Array(5, 6), arrayType)), null) + + // 8. Struct Keys + val structType = new StructType().add("a", "int").add("b", "int") + val structMap = Literal.create( + Map(create_row(1, 1) -> 10, create_row(2, 2) -> 20), + MapType(structType, IntegerType)) + checkEvaluation(GetMapValue(structMap, Literal.create(create_row(1, 1), structType)), 10) + checkEvaluation(GetMapValue(structMap, Literal.create(create_row(2, 2), structType)), 20) + checkEvaluation(GetMapValue(structMap, Literal.create(create_row(3, 3), structType)), null) + } + } } test("GetStructField") { From 4ac347b72bbc5746a8b5fcfc5a5dbdf906153a38 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 12 Mar 2026 19:33:19 +0800 Subject: [PATCH 13/24] refactor benchmark --- .../CollectionExpressionsSuite.scala | 2 +- .../expressions/ComplexTypeSuite.scala | 2 +- .../benchmark/MapLookupBenchmark.scala | 54 +++++++++++++++++-- 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala index 997080dd30687..108b833694337 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala @@ -1887,7 +1887,7 @@ class CollectionExpressionsSuite checkEvaluation(ArrayPosition(aa1, aae), 0L) } - Seq((Int.MaxValue, "Linear"), (0, "Hash Lookup")).foreach { case (threshold, name) => + Seq((Int.MaxValue, "Linear Lookup"), (0, "Hash Lookup")).foreach { case (threshold, name) => test(s"elementAt - $name") { withSQLConf(SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> threshold.toString) { val a0 = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType)) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala index d785dc247ae60..e83e2dcd85db7 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala @@ -129,7 +129,7 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper { assert(GetArrayItem(stArray4, Literal(1)).nullable) } - Seq((Int.MaxValue, "Linear"), (0, "Hash Lookup")).foreach { case (threshold, name) => + Seq((Int.MaxValue, "Linear Lookup"), (0, "Hash Lookup")).foreach { case (threshold, name) => test(s"GetMapValue - $name") { withSQLConf(SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> threshold.toString) { val typeM = MapType(StringType, StringType) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala index c8dab794a9720..9af364c0b8f9d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala @@ -80,37 +80,81 @@ object MapLookupBenchmark extends SqlBasedBenchmark { val expr = col("m").getItem(col("key")) val elementAtExpr = element_at(col("m"), col("key")) - benchmark.addCase("GetMapValue interpreted") { _ => + benchmark.addCase("GetMapValue interpreted - Linear Lookup") { _ => withSQLConf( SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false", SQLConf.CODEGEN_FACTORY_MODE.key -> "NO_CODEGEN", + SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> Int.MaxValue.toString, SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { lookupDf.select(expr).write.format("noop").mode("overwrite").save() } } - benchmark.addCase("GetMapValue codegen") { _ => + benchmark.addCase("GetMapValue interpreted - Hash Lookup") { _ => + withSQLConf( + SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false", + SQLConf.CODEGEN_FACTORY_MODE.key -> "NO_CODEGEN", + SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> 0.toString, + SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { + lookupDf.select(expr).write.format("noop").mode("overwrite").save() + } + } + + benchmark.addCase("GetMapValue codegen - Linear Lookup") { _ => + withSQLConf( + SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true", + SQLConf.CODEGEN_FACTORY_MODE.key -> "CODEGEN_ONLY", + SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> Int.MaxValue.toString, + SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { + lookupDf.select(expr).write.format("noop").mode("overwrite").save() + } + } + + benchmark.addCase("GetMapValue codegen - Hash Lookup") { _ => withSQLConf( SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true", SQLConf.CODEGEN_FACTORY_MODE.key -> "CODEGEN_ONLY", + SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> 0.toString, SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { lookupDf.select(expr).write.format("noop").mode("overwrite").save() } } - benchmark.addCase("ElementAt interpreted") { _ => + benchmark.addCase("ElementAt interpreted - Linear Lookup") { _ => withSQLConf( SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false", SQLConf.CODEGEN_FACTORY_MODE.key -> "NO_CODEGEN", + SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> Int.MaxValue.toString, + SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { + lookupDf.select(elementAtExpr).write.format("noop").mode("overwrite").save() + } + } + + benchmark.addCase("ElementAt interpreted - Hash Lookup") { _ => + withSQLConf( + SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false", + SQLConf.CODEGEN_FACTORY_MODE.key -> "NO_CODEGEN", + SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> 0.toString, + SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { + lookupDf.select(elementAtExpr).write.format("noop").mode("overwrite").save() + } + } + + benchmark.addCase("ElementAt codegen - Linear Lookup") { _ => + withSQLConf( + SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true", + SQLConf.CODEGEN_FACTORY_MODE.key -> "CODEGEN_ONLY", + SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> Int.MaxValue.toString, SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { lookupDf.select(elementAtExpr).write.format("noop").mode("overwrite").save() } } - benchmark.addCase("ElementAt codegen") { _ => + benchmark.addCase("ElementAt codegen - Hash Lookup") { _ => withSQLConf( SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true", SQLConf.CODEGEN_FACTORY_MODE.key -> "CODEGEN_ONLY", + SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> 0.toString, SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { lookupDf.select(elementAtExpr).write.format("noop").mode("overwrite").save() } @@ -120,7 +164,7 @@ object MapLookupBenchmark extends SqlBasedBenchmark { } override def runBenchmarkSuite(mainArgs: Array[String]): Unit = { - val sizes = Seq(10, 20, 30, 50, 100, 1000, 1000000) + val sizes = Seq(1, 10, 100, 1000, 10000, 10000, 1000000) for (size <- sizes) { run(size, 1.0, IntegerType) run(size, 0.5, IntegerType) From 67579071c0f42621784edda1f1dca05c24e801b7 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 12 Mar 2026 19:48:06 +0800 Subject: [PATCH 14/24] fix doc --- .../scala/org/apache/spark/sql/internal/SQLConf.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 8d9edf9e39462..50f017f140545 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2498,11 +2498,15 @@ object SQLConf { .intConf .createWithDefault(-1) - val MAP_LOOKUP_HASH_THRESHOLD = buildConf("spark.sql.mapLookupHashThreshold") - .doc("The threshold to determine whether to use hash lookup for map lookup expressions. " + - "If the map size is small, the cost of building hash map exceeds the cost of a linear scan.") + val MAP_LOOKUP_HASH_THRESHOLD = + buildConf("spark.sql.optimizer.mapLookupHashThreshold") + .internal() + .doc("The minimum number of map entries to attempt hash-based lookup in `element_at` and " + + "the `[]` operator. Below this threshold, linear scan is used. For key types that do not " + + "support hashing (e.g. arrays, structs), linear scan is always used regardless of map size.") .version("4.2.0") .intConf + .checkValue(_ >= 0, "The threshold must be non-negative.") .createWithDefault(1000) val FILES_MAX_PARTITION_BYTES = buildConf("spark.sql.files.maxPartitionBytes") From 091734692853ade8f4a185f1025da0b7dfae6980 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 12 Mar 2026 21:12:40 +0800 Subject: [PATCH 15/24] add more memory --- .../MapLookupBenchmark-jdk21-results.txt | 189 ------------------ .../MapLookupBenchmark-jdk25-results.txt | 189 ------------------ .../benchmarks/MapLookupBenchmark-results.txt | 189 ------------------ .../benchmark/MapLookupBenchmark.scala | 6 +- 4 files changed, 3 insertions(+), 570 deletions(-) delete mode 100644 sql/core/benchmarks/MapLookupBenchmark-jdk21-results.txt delete mode 100644 sql/core/benchmarks/MapLookupBenchmark-jdk25-results.txt delete mode 100644 sql/core/benchmarks/MapLookupBenchmark-results.txt diff --git a/sql/core/benchmarks/MapLookupBenchmark-jdk21-results.txt b/sql/core/benchmarks/MapLookupBenchmark-jdk21-results.txt deleted file mode 100644 index c7fde91b665c7..0000000000000 --- a/sql/core/benchmarks/MapLookupBenchmark-jdk21-results.txt +++ /dev/null @@ -1,189 +0,0 @@ -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=10, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 23 33 7 0.4 2348.6 1.0X -GetMapValue codegen 19 26 7 0.5 1886.6 1.2X -ElementAt interpreted 16 21 6 0.6 1629.6 1.4X -ElementAt codegen 15 19 5 0.7 1535.8 1.5X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=10, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 15 20 5 0.7 1470.4 1.0X -GetMapValue codegen 14 18 5 0.7 1415.1 1.0X -ElementAt interpreted 14 16 4 0.7 1365.1 1.1X -ElementAt codegen 14 16 4 0.7 1360.1 1.1X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=10, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 13 15 3 0.8 1296.3 1.0X -GetMapValue codegen 13 14 2 0.8 1275.5 1.0X -ElementAt interpreted 12 16 4 0.8 1244.6 1.0X -ElementAt codegen 12 14 2 0.8 1210.9 1.1X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=20, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 12 15 4 0.8 1225.0 1.0X -GetMapValue codegen 12 13 2 0.9 1153.9 1.1X -ElementAt interpreted 12 14 3 0.9 1172.0 1.0X -ElementAt codegen 11 13 3 0.9 1143.0 1.1X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=20, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 11 14 3 0.9 1126.6 1.0X -GetMapValue codegen 11 13 2 0.9 1113.0 1.0X -ElementAt interpreted 11 13 3 0.9 1132.7 1.0X -ElementAt codegen 11 13 3 0.9 1113.9 1.0X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=20, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 11 13 3 0.9 1086.0 1.0X -GetMapValue codegen 11 12 2 0.9 1072.5 1.0X -ElementAt interpreted 11 13 3 0.9 1068.9 1.0X -ElementAt codegen 11 12 2 0.9 1056.7 1.0X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=30, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 11 15 4 0.9 1097.5 1.0X -GetMapValue codegen 11 13 2 0.9 1077.5 1.0X -ElementAt interpreted 11 13 3 0.9 1085.0 1.0X -ElementAt codegen 11 12 2 0.9 1065.7 1.0X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=30, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 11 12 2 0.9 1057.1 1.0X -GetMapValue codegen 11 12 2 0.9 1059.0 1.0X -ElementAt interpreted 11 12 2 0.9 1058.0 1.0X -ElementAt codegen 10 12 2 1.0 1038.9 1.0X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=30, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 10 12 2 1.0 1036.2 1.0X -GetMapValue codegen 10 11 2 1.0 1010.6 1.0X -ElementAt interpreted 10 12 2 1.0 1028.0 1.0X -ElementAt codegen 10 12 2 1.0 1040.3 1.0X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=50, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 11 12 2 0.9 1059.2 1.0X -GetMapValue codegen 10 12 2 1.0 1042.8 1.0X -ElementAt interpreted 10 12 2 1.0 1048.8 1.0X -ElementAt codegen 10 12 2 1.0 1024.4 1.0X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=50, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 10 11 2 1.0 1031.8 1.0X -GetMapValue codegen 10 11 2 1.0 1013.5 1.0X -ElementAt interpreted 10 12 2 1.0 1034.6 1.0X -ElementAt codegen 10 11 2 1.0 1025.6 1.0X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=50, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 10 11 2 1.0 1012.2 1.0X -GetMapValue codegen 10 11 2 1.0 1003.7 1.0X -ElementAt interpreted 10 11 2 1.0 1014.9 1.0X -ElementAt codegen 10 11 2 1.0 1001.0 1.0X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=100, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 10 11 2 1.0 1049.6 1.0X -GetMapValue codegen 10 11 2 1.0 1016.8 1.0X -ElementAt interpreted 11 12 2 0.9 1053.8 1.0X -ElementAt codegen 10 12 2 1.0 1027.8 1.0X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=100, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 10 11 2 1.0 1026.7 1.0X -GetMapValue codegen 10 11 2 1.0 1007.9 1.0X -ElementAt interpreted 10 11 2 1.0 1026.9 1.0X -ElementAt codegen 10 12 3 1.0 1014.9 1.0X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=100, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 10 11 2 1.0 1006.2 1.0X -GetMapValue codegen 10 11 2 1.0 996.3 1.0X -ElementAt interpreted 10 11 2 1.0 1010.6 1.0X -ElementAt codegen 10 11 2 1.0 1002.2 1.0X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 11 12 2 0.9 1124.9 1.0X -GetMapValue codegen 11 12 2 0.9 1107.5 1.0X -ElementAt interpreted 11 12 2 0.9 1128.3 1.0X -ElementAt codegen 11 13 2 0.9 1128.3 1.0X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 11 12 2 0.9 1106.7 1.0X -GetMapValue codegen 11 13 2 0.9 1110.0 1.0X -ElementAt interpreted 11 12 2 0.9 1103.1 1.0X -ElementAt codegen 11 13 2 0.9 1112.3 1.0X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 11 12 2 0.9 1084.4 1.0X -GetMapValue codegen 11 12 2 0.9 1085.7 1.0X -ElementAt interpreted 11 12 2 0.9 1085.7 1.0X -ElementAt codegen 11 12 2 0.9 1077.6 1.0X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1000000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 1140 1391 203 0.0 114048.5 1.0X -GetMapValue codegen 1653 1744 81 0.0 165283.9 0.7X -ElementAt interpreted 1116 1310 137 0.0 111564.9 1.0X -ElementAt codegen 1554 1679 124 0.0 155392.1 0.7X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1000000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 1224 1314 91 0.0 122399.4 1.0X -GetMapValue codegen 1534 1652 119 0.0 153415.9 0.8X -ElementAt interpreted 1116 1361 196 0.0 111570.5 1.1X -ElementAt codegen 1540 1752 141 0.0 153989.8 0.8X - -OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1000000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 1108 1344 125 0.0 110803.0 1.0X -GetMapValue codegen 1594 1719 103 0.0 159373.4 0.7X -ElementAt interpreted 1229 1370 106 0.0 122949.0 0.9X -ElementAt codegen 1618 1774 94 0.0 161812.6 0.7X - diff --git a/sql/core/benchmarks/MapLookupBenchmark-jdk25-results.txt b/sql/core/benchmarks/MapLookupBenchmark-jdk25-results.txt deleted file mode 100644 index af495cf62b535..0000000000000 --- a/sql/core/benchmarks/MapLookupBenchmark-jdk25-results.txt +++ /dev/null @@ -1,189 +0,0 @@ -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=10, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 28 40 9 0.4 2847.9 1.0X -GetMapValue codegen 21 29 6 0.5 2097.2 1.4X -ElementAt interpreted 19 23 4 0.5 1903.8 1.5X -ElementAt codegen 18 22 4 0.6 1758.6 1.6X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=10, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 16 20 4 0.6 1586.0 1.0X -GetMapValue codegen 16 20 3 0.6 1594.7 1.0X -ElementAt interpreted 17 21 3 0.6 1664.1 1.0X -ElementAt codegen 15 18 3 0.7 1500.7 1.1X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=10, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 14 17 3 0.7 1395.1 1.0X -GetMapValue codegen 14 17 3 0.7 1386.0 1.0X -ElementAt interpreted 13 16 3 0.7 1334.8 1.0X -ElementAt codegen 13 16 3 0.8 1275.5 1.1X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=20, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 12 16 3 0.8 1244.3 1.0X -GetMapValue codegen 12 16 4 0.8 1216.9 1.0X -ElementAt interpreted 12 14 2 0.9 1174.4 1.1X -ElementAt codegen 12 14 3 0.9 1171.2 1.1X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=20, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 11 14 3 0.9 1146.8 1.0X -GetMapValue codegen 11 14 3 0.9 1122.6 1.0X -ElementAt interpreted 11 13 2 0.9 1119.9 1.0X -ElementAt codegen 11 13 2 0.9 1110.7 1.0X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=20, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 11 13 2 0.9 1085.3 1.0X -GetMapValue codegen 11 13 3 0.9 1082.7 1.0X -ElementAt interpreted 11 14 3 0.9 1080.1 1.0X -ElementAt codegen 11 14 3 0.9 1071.7 1.0X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=30, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 11 13 3 0.9 1068.6 1.0X -GetMapValue codegen 11 14 2 0.9 1129.2 0.9X -ElementAt interpreted 11 14 2 0.9 1117.1 1.0X -ElementAt codegen 10 13 2 1.0 1038.4 1.0X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=30, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 11 13 3 0.9 1052.8 1.0X -GetMapValue codegen 10 12 2 1.0 1045.5 1.0X -ElementAt interpreted 10 12 2 1.0 1025.0 1.0X -ElementAt codegen 10 12 2 1.0 1014.0 1.0X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=30, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 10 12 2 1.0 1000.6 1.0X -GetMapValue codegen 10 12 2 1.0 997.3 1.0X -ElementAt interpreted 10 12 2 1.0 1012.2 1.0X -ElementAt codegen 10 12 2 1.0 1005.3 1.0X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=50, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 10 13 3 1.0 1004.4 1.0X -GetMapValue codegen 11 13 2 0.9 1114.0 0.9X -ElementAt interpreted 10 13 2 1.0 1023.3 1.0X -ElementAt codegen 10 13 2 1.0 1023.5 1.0X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=50, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 10 12 2 1.0 993.6 1.0X -GetMapValue codegen 10 12 2 1.0 998.2 1.0X -ElementAt interpreted 10 12 2 1.0 1011.1 1.0X -ElementAt codegen 10 13 2 1.0 991.0 1.0X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=50, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 10 12 2 1.0 976.4 1.0X -GetMapValue codegen 10 12 2 1.0 988.4 1.0X -ElementAt interpreted 10 12 2 1.0 994.8 1.0X -ElementAt codegen 10 12 2 1.0 1019.9 1.0X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=100, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 10 12 2 1.0 1018.1 1.0X -GetMapValue codegen 10 12 2 1.0 1014.4 1.0X -ElementAt interpreted 10 13 2 1.0 1036.3 1.0X -ElementAt codegen 11 13 2 0.9 1096.3 0.9X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=100, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 10 12 2 1.0 991.5 1.0X -GetMapValue codegen 10 12 2 1.0 1011.0 1.0X -ElementAt interpreted 10 12 2 1.0 979.1 1.0X -ElementAt codegen 10 12 2 1.0 1003.2 1.0X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=100, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 10 12 2 1.0 984.9 1.0X -GetMapValue codegen 10 13 2 1.0 975.4 1.0X -ElementAt interpreted 11 14 3 0.9 1063.9 0.9X -ElementAt codegen 10 11 1 1.0 996.2 1.0X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 11 13 3 0.9 1099.6 1.0X -GetMapValue codegen 11 13 2 0.9 1092.8 1.0X -ElementAt interpreted 11 13 2 0.9 1104.1 1.0X -ElementAt codegen 11 13 2 0.9 1109.0 1.0X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 11 12 2 0.9 1077.5 1.0X -GetMapValue codegen 11 13 3 0.9 1128.5 1.0X -ElementAt interpreted 11 14 2 0.9 1086.1 1.0X -ElementAt codegen 12 15 3 0.8 1219.2 0.9X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 11 12 2 0.9 1081.0 1.0X -GetMapValue codegen 11 13 2 0.9 1076.3 1.0X -ElementAt interpreted 11 12 2 0.9 1071.7 1.0X -ElementAt codegen 11 13 2 0.9 1113.1 1.0X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1000000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 1569 1862 224 0.0 156906.4 1.0X -GetMapValue codegen 2190 2558 318 0.0 218954.4 0.7X -ElementAt interpreted 1662 1881 178 0.0 166221.9 0.9X -ElementAt codegen 2194 2518 308 0.0 219363.6 0.7X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1000000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 1232 1762 391 0.0 123159.1 1.0X -GetMapValue codegen 1780 2029 159 0.0 178017.3 0.7X -ElementAt interpreted 1404 1654 233 0.0 140373.1 0.9X -ElementAt codegen 1736 2273 361 0.0 173626.9 0.7X - -OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1000000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 1451 1665 200 0.0 145056.0 1.0X -GetMapValue codegen 1829 2341 408 0.0 182928.5 0.8X -ElementAt interpreted 1407 1868 267 0.0 140654.0 1.0X -ElementAt codegen 1882 2353 316 0.0 188232.9 0.8X - diff --git a/sql/core/benchmarks/MapLookupBenchmark-results.txt b/sql/core/benchmarks/MapLookupBenchmark-results.txt deleted file mode 100644 index 15d02baa87adc..0000000000000 --- a/sql/core/benchmarks/MapLookupBenchmark-results.txt +++ /dev/null @@ -1,189 +0,0 @@ -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=10, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 27 37 5 0.4 2738.9 1.0X -GetMapValue codegen 22 27 3 0.4 2233.6 1.2X -ElementAt interpreted 20 23 2 0.5 2025.3 1.4X -ElementAt codegen 20 23 3 0.5 2039.7 1.3X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=10, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 19 21 2 0.5 1870.3 1.0X -GetMapValue codegen 18 20 2 0.6 1781.2 1.1X -ElementAt interpreted 18 20 2 0.6 1793.5 1.0X -ElementAt codegen 18 20 3 0.6 1798.2 1.0X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=10, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 17 19 2 0.6 1693.8 1.0X -GetMapValue codegen 17 19 2 0.6 1725.0 1.0X -ElementAt interpreted 16 18 2 0.6 1629.6 1.0X -ElementAt codegen 17 18 2 0.6 1677.1 1.0X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=20, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 16 18 2 0.6 1629.6 1.0X -GetMapValue codegen 16 18 2 0.6 1635.5 1.0X -ElementAt interpreted 16 18 2 0.6 1616.7 1.0X -ElementAt codegen 15 17 1 0.7 1514.8 1.1X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=20, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 15 17 2 0.7 1495.1 1.0X -GetMapValue codegen 15 16 1 0.7 1491.6 1.0X -ElementAt interpreted 15 16 1 0.7 1537.1 1.0X -ElementAt codegen 15 16 1 0.7 1487.9 1.0X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=20, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 14 16 2 0.7 1437.7 1.0X -GetMapValue codegen 14 16 1 0.7 1417.4 1.0X -ElementAt interpreted 14 16 1 0.7 1415.1 1.0X -ElementAt codegen 14 16 2 0.7 1424.2 1.0X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=30, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 15 16 1 0.7 1456.4 1.0X -GetMapValue codegen 14 16 2 0.7 1413.5 1.0X -ElementAt interpreted 14 16 2 0.7 1425.1 1.0X -ElementAt codegen 14 16 1 0.7 1420.6 1.0X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=30, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 14 16 2 0.7 1400.7 1.0X -GetMapValue codegen 14 16 2 0.7 1396.3 1.0X -ElementAt interpreted 14 16 2 0.7 1405.0 1.0X -ElementAt codegen 14 16 2 0.7 1391.3 1.0X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=30, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 14 16 2 0.7 1417.6 1.0X -GetMapValue codegen 14 15 1 0.7 1414.6 1.0X -ElementAt interpreted 15 15 1 0.7 1457.7 1.0X -ElementAt codegen 14 15 1 0.7 1375.1 1.0X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=50, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 14 15 1 0.7 1372.3 1.0X -GetMapValue codegen 14 15 1 0.7 1366.4 1.0X -ElementAt interpreted 14 15 1 0.7 1392.2 1.0X -ElementAt codegen 14 15 1 0.7 1424.9 1.0X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=50, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 14 15 1 0.7 1393.0 1.0X -GetMapValue codegen 14 15 1 0.7 1358.4 1.0X -ElementAt interpreted 14 15 1 0.7 1368.7 1.0X -ElementAt codegen 14 15 1 0.7 1381.5 1.0X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=50, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 14 15 1 0.7 1415.1 1.0X -GetMapValue codegen 14 15 1 0.7 1366.2 1.0X -ElementAt interpreted 14 15 1 0.7 1383.8 1.0X -ElementAt codegen 14 15 1 0.7 1442.1 1.0X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=100, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 15 15 1 0.7 1456.2 1.0X -GetMapValue codegen 14 15 2 0.7 1446.4 1.0X -ElementAt interpreted 14 15 1 0.7 1380.4 1.1X -ElementAt codegen 14 15 2 0.7 1366.7 1.1X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=100, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 14 15 1 0.7 1447.0 1.0X -GetMapValue codegen 14 15 1 0.7 1436.3 1.0X -ElementAt interpreted 14 15 1 0.7 1396.4 1.0X -ElementAt codegen 14 15 1 0.7 1431.0 1.0X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=100, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 13 15 1 0.7 1335.0 1.0X -GetMapValue codegen 14 15 1 0.7 1424.9 0.9X -ElementAt interpreted 14 15 1 0.7 1436.3 0.9X -ElementAt codegen 14 15 1 0.7 1415.2 0.9X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 15 16 1 0.7 1529.7 1.0X -GetMapValue codegen 15 16 1 0.7 1533.2 1.0X -ElementAt interpreted 14 16 1 0.7 1439.3 1.1X -ElementAt codegen 15 16 2 0.7 1478.1 1.0X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 15 16 1 0.7 1519.3 1.0X -GetMapValue codegen 15 16 1 0.7 1520.4 1.0X -ElementAt interpreted 15 16 1 0.7 1473.4 1.0X -ElementAt codegen 14 16 1 0.7 1431.1 1.1X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted 14 15 1 0.7 1410.9 1.0X -GetMapValue codegen 14 16 1 0.7 1432.1 1.0X -ElementAt interpreted 14 15 1 0.7 1413.7 1.0X -ElementAt codegen 14 16 1 0.7 1436.3 1.0X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1000000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 1077 1302 119 0.0 107704.4 1.0X -GetMapValue codegen 1500 1623 113 0.0 150002.9 0.7X -ElementAt interpreted 1156 1257 96 0.0 115601.2 0.9X -ElementAt codegen 1452 1618 128 0.0 145230.3 0.7X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1000000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 1071 1347 258 0.0 107077.6 1.0X -GetMapValue codegen 1472 1675 145 0.0 147167.3 0.7X -ElementAt interpreted 1094 1286 106 0.0 109421.5 1.0X -ElementAt codegen 1499 1703 167 0.0 149893.2 0.7X - -OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1000000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted 1050 1244 188 0.0 105042.5 1.0X -GetMapValue codegen 1423 1582 107 0.0 142278.0 0.7X -ElementAt interpreted 1202 1291 56 0.0 120209.1 0.9X -ElementAt codegen 1541 1619 32 0.0 154101.3 0.7X - diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala index 9af364c0b8f9d..d0ff15fa4adcf 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala @@ -44,8 +44,8 @@ object MapLookupBenchmark extends SqlBasedBenchmark { SparkSession.builder() .master("local[1]") .appName("MapLookupBenchmark") - .config("spark.driver.memory", "6g") - .config("spark.executor.memory", "6g") + .config("spark.driver.memory", "8g") + .config("spark.executor.memory", "8g") .getOrCreate() } @@ -164,7 +164,7 @@ object MapLookupBenchmark extends SqlBasedBenchmark { } override def runBenchmarkSuite(mainArgs: Array[String]): Unit = { - val sizes = Seq(1, 10, 100, 1000, 10000, 10000, 1000000) + val sizes = Seq(1, 10, 100, 1000, 10000, 100000, 1000000) for (size <- sizes) { run(size, 1.0, IntegerType) run(size, 0.5, IntegerType) From 782d7a5ad024fb544f99df2eda3226da4758ed76 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 13 Mar 2026 01:09:57 +0800 Subject: [PATCH 16/24] init --- .../spark/sql/execution/benchmark/MapLookupBenchmark.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala index d0ff15fa4adcf..bd6c83548dc96 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala @@ -44,8 +44,8 @@ object MapLookupBenchmark extends SqlBasedBenchmark { SparkSession.builder() .master("local[1]") .appName("MapLookupBenchmark") - .config("spark.driver.memory", "8g") - .config("spark.executor.memory", "8g") + .config("spark.driver.memory", "6g") + .config("spark.executor.memory", "6g") .getOrCreate() } @@ -164,7 +164,7 @@ object MapLookupBenchmark extends SqlBasedBenchmark { } override def runBenchmarkSuite(mainArgs: Array[String]): Unit = { - val sizes = Seq(1, 10, 100, 1000, 10000, 100000, 1000000) + val sizes = Seq(1, 10, 100, 1000, 10000, 100000) for (size <- sizes) { run(size, 1.0, IntegerType) run(size, 0.5, IntegerType) From 9d21fae443fc04da7bcca087fff622178117dff9 Mon Sep 17 00:00:00 2001 From: LuciferYang Date: Thu, 12 Mar 2026 18:40:05 +0000 Subject: [PATCH 17/24] Benchmark results for org.apache.spark.sql.execution.benchmark.MapLookupBenchmark (JDK 21, Scala 2.13, split 1 of 1) --- .../MapLookupBenchmark-jdk21-results.txt | 234 ++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 sql/core/benchmarks/MapLookupBenchmark-jdk21-results.txt diff --git a/sql/core/benchmarks/MapLookupBenchmark-jdk21-results.txt b/sql/core/benchmarks/MapLookupBenchmark-jdk21-results.txt new file mode 100644 index 0000000000000..4f4400db5cacd --- /dev/null +++ b/sql/core/benchmarks/MapLookupBenchmark-jdk21-results.txt @@ -0,0 +1,234 @@ +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 23 34 7 0.4 2294.8 1.0X +GetMapValue interpreted - Hash Lookup 18 24 5 0.5 1845.4 1.2X +GetMapValue codegen - Linear Lookup 17 21 4 0.6 1652.2 1.4X +GetMapValue codegen - Hash Lookup 16 20 4 0.6 1618.6 1.4X +ElementAt interpreted - Linear Lookup 16 20 4 0.6 1562.5 1.5X +ElementAt interpreted - Hash Lookup 15 17 4 0.7 1463.5 1.6X +ElementAt codegen - Linear Lookup 15 17 3 0.7 1467.1 1.6X +ElementAt codegen - Hash Lookup 14 16 3 0.7 1367.7 1.7X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 14 17 3 0.7 1405.2 1.0X +GetMapValue interpreted - Hash Lookup 14 16 2 0.7 1358.8 1.0X +GetMapValue codegen - Linear Lookup 13 15 3 0.8 1311.0 1.1X +GetMapValue codegen - Hash Lookup 13 15 3 0.8 1274.5 1.1X +ElementAt interpreted - Linear Lookup 12 15 3 0.8 1234.4 1.1X +ElementAt interpreted - Hash Lookup 12 14 3 0.8 1214.1 1.2X +ElementAt codegen - Linear Lookup 12 14 3 0.8 1203.0 1.2X +ElementAt codegen - Hash Lookup 12 14 3 0.9 1176.0 1.2X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 11 14 3 0.9 1142.2 1.0X +GetMapValue interpreted - Hash Lookup 11 13 3 0.9 1132.0 1.0X +GetMapValue codegen - Linear Lookup 11 13 3 0.9 1137.8 1.0X +GetMapValue codegen - Hash Lookup 11 12 2 0.9 1133.6 1.0X +ElementAt interpreted - Linear Lookup 11 13 3 0.9 1122.7 1.0X +ElementAt interpreted - Hash Lookup 11 14 3 0.9 1122.8 1.0X +ElementAt codegen - Linear Lookup 12 14 3 0.8 1178.5 1.0X +ElementAt codegen - Hash Lookup 11 13 3 0.9 1066.7 1.1X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=10, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted - Linear Lookup 11 13 2 0.9 1107.9 1.0X +GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1090.3 1.0X +GetMapValue codegen - Linear Lookup 11 13 3 0.9 1103.6 1.0X +GetMapValue codegen - Hash Lookup 11 13 3 0.9 1083.0 1.0X +ElementAt interpreted - Linear Lookup 11 13 3 0.9 1136.8 1.0X +ElementAt interpreted - Hash Lookup 11 12 2 0.9 1082.7 1.0X +ElementAt codegen - Linear Lookup 11 12 2 0.9 1080.0 1.0X +ElementAt codegen - Hash Lookup 11 12 2 0.9 1071.0 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=10, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted - Linear Lookup 11 12 2 1.0 1052.5 1.0X +GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1067.8 1.0X +GetMapValue codegen - Linear Lookup 11 13 3 0.9 1067.5 1.0X +GetMapValue codegen - Hash Lookup 10 12 2 1.0 1039.0 1.0X +ElementAt interpreted - Linear Lookup 11 11 2 0.9 1059.5 1.0X +ElementAt interpreted - Hash Lookup 11 11 2 0.9 1060.1 1.0X +ElementAt codegen - Linear Lookup 10 11 2 1.0 1049.1 1.0X +ElementAt codegen - Hash Lookup 10 12 3 1.0 1047.9 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=10, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted - Linear Lookup 10 12 2 1.0 1035.3 1.0X +GetMapValue interpreted - Hash Lookup 10 12 2 1.0 1041.9 1.0X +GetMapValue codegen - Linear Lookup 11 12 2 0.9 1084.0 1.0X +GetMapValue codegen - Hash Lookup 10 12 2 1.0 1047.4 1.0X +ElementAt interpreted - Linear Lookup 11 13 2 0.9 1066.0 1.0X +ElementAt interpreted - Hash Lookup 11 12 2 1.0 1052.2 1.0X +ElementAt codegen - Linear Lookup 11 12 2 0.9 1080.6 1.0X +ElementAt codegen - Hash Lookup 10 12 2 1.0 1046.7 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=100, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 11 13 2 0.9 1137.0 1.0X +GetMapValue interpreted - Hash Lookup 11 13 2 0.9 1069.1 1.1X +GetMapValue codegen - Linear Lookup 11 12 2 0.9 1104.0 1.0X +GetMapValue codegen - Hash Lookup 11 12 2 0.9 1096.5 1.0X +ElementAt interpreted - Linear Lookup 11 12 2 0.9 1129.0 1.0X +ElementAt interpreted - Hash Lookup 11 12 2 0.9 1071.3 1.1X +ElementAt codegen - Linear Lookup 11 12 2 0.9 1097.3 1.0X +ElementAt codegen - Hash Lookup 10 12 2 1.0 1035.4 1.1X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=100, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 14 16 2 0.7 1406.4 1.0X +GetMapValue interpreted - Hash Lookup 13 14 2 0.8 1297.4 1.1X +GetMapValue codegen - Linear Lookup 13 14 2 0.7 1337.7 1.1X +GetMapValue codegen - Hash Lookup 10 11 2 1.0 1002.6 1.4X +ElementAt interpreted - Linear Lookup 11 12 2 0.9 1123.6 1.3X +ElementAt interpreted - Hash Lookup 10 11 2 1.0 1024.2 1.4X +ElementAt codegen - Linear Lookup 11 12 2 0.9 1087.2 1.3X +ElementAt codegen - Hash Lookup 10 11 2 1.0 991.7 1.4X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=100, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 11 12 2 0.9 1149.7 1.0X +GetMapValue interpreted - Hash Lookup 10 11 2 1.0 1009.4 1.1X +GetMapValue codegen - Linear Lookup 12 12 2 0.9 1153.7 1.0X +GetMapValue codegen - Hash Lookup 10 11 2 1.0 998.1 1.2X +ElementAt interpreted - Linear Lookup 11 12 2 0.9 1145.4 1.0X +ElementAt interpreted - Hash Lookup 10 11 2 1.0 1012.6 1.1X +ElementAt codegen - Linear Lookup 11 12 2 0.9 1149.1 1.0X +ElementAt codegen - Hash Lookup 10 11 2 1.0 1004.7 1.1X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 18 20 3 0.6 1800.1 1.0X +GetMapValue interpreted - Hash Lookup 12 13 2 0.9 1162.9 1.5X +GetMapValue codegen - Linear Lookup 18 20 2 0.5 1820.2 1.0X +GetMapValue codegen - Hash Lookup 11 12 2 0.9 1136.0 1.6X +ElementAt interpreted - Linear Lookup 18 20 2 0.6 1816.6 1.0X +ElementAt interpreted - Hash Lookup 11 12 2 0.9 1130.9 1.6X +ElementAt codegen - Linear Lookup 18 20 2 0.6 1805.3 1.0X +ElementAt codegen - Hash Lookup 11 12 2 0.9 1133.8 1.6X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 21 23 2 0.5 2123.9 1.0X +GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1101.1 1.9X +GetMapValue codegen - Linear Lookup 21 23 2 0.5 2125.6 1.0X +GetMapValue codegen - Hash Lookup 11 12 2 0.9 1099.0 1.9X +ElementAt interpreted - Linear Lookup 21 23 2 0.5 2124.3 1.0X +ElementAt interpreted - Hash Lookup 11 12 2 0.9 1110.0 1.9X +ElementAt codegen - Linear Lookup 21 23 2 0.5 2108.9 1.0X +ElementAt codegen - Hash Lookup 11 12 2 0.9 1109.9 1.9X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 24 26 2 0.4 2409.5 1.0X +GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1080.2 2.2X +GetMapValue codegen - Linear Lookup 24 26 2 0.4 2404.9 1.0X +GetMapValue codegen - Hash Lookup 11 11 2 0.9 1066.1 2.3X +ElementAt interpreted - Linear Lookup 24 26 2 0.4 2395.0 1.0X +ElementAt interpreted - Hash Lookup 11 12 2 1.0 1050.3 2.3X +ElementAt codegen - Linear Lookup 24 26 2 0.4 2403.8 1.0X +ElementAt codegen - Hash Lookup 11 11 2 0.9 1059.1 2.3X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=10000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +--------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 110 125 9 0.1 10989.7 1.0X +GetMapValue interpreted - Hash Lookup 19 20 1 0.5 1908.0 5.8X +GetMapValue codegen - Linear Lookup 125 127 2 0.1 12505.6 0.9X +GetMapValue codegen - Hash Lookup 21 23 2 0.5 2118.5 5.2X +ElementAt interpreted - Linear Lookup 112 126 10 0.1 11219.8 1.0X +ElementAt interpreted - Hash Lookup 19 21 2 0.5 1923.8 5.7X +ElementAt codegen - Linear Lookup 126 128 2 0.1 12569.5 0.9X +ElementAt codegen - Hash Lookup 21 23 3 0.5 2127.4 5.2X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=10000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +--------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 160 163 2 0.1 16018.0 1.0X +GetMapValue interpreted - Hash Lookup 19 20 2 0.5 1878.3 8.5X +GetMapValue codegen - Linear Lookup 185 193 6 0.1 18535.5 0.9X +GetMapValue codegen - Hash Lookup 21 23 3 0.5 2091.5 7.7X +ElementAt interpreted - Linear Lookup 190 193 2 0.1 18990.0 0.8X +ElementAt interpreted - Hash Lookup 19 21 3 0.5 1877.0 8.5X +ElementAt codegen - Linear Lookup 160 176 13 0.1 16026.5 1.0X +ElementAt codegen - Hash Lookup 21 22 2 0.5 2084.6 7.7X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=10000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +--------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 254 256 2 0.0 25395.0 1.0X +GetMapValue interpreted - Hash Lookup 18 20 2 0.5 1827.6 13.9X +GetMapValue codegen - Linear Lookup 244 247 3 0.0 24386.9 1.0X +GetMapValue codegen - Hash Lookup 21 22 2 0.5 2056.8 12.3X +ElementAt interpreted - Linear Lookup 254 256 2 0.0 25433.3 1.0X +ElementAt interpreted - Hash Lookup 18 20 2 0.5 1833.2 13.9X +ElementAt codegen - Linear Lookup 207 231 21 0.0 20654.0 1.2X +ElementAt codegen - Hash Lookup 21 24 5 0.5 2065.5 12.3X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=100000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +---------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 1025 1132 126 0.0 102479.2 1.0X +GetMapValue interpreted - Hash Lookup 105 112 6 0.1 10452.3 9.8X +GetMapValue codegen - Linear Lookup 1061 1161 102 0.0 106113.4 1.0X +GetMapValue codegen - Hash Lookup 136 141 4 0.1 13608.0 7.5X +ElementAt interpreted - Linear Lookup 1260 1274 9 0.0 125960.6 0.8X +ElementAt interpreted - Hash Lookup 105 116 6 0.1 10508.0 9.8X +ElementAt codegen - Linear Lookup 1055 1234 64 0.0 105538.2 1.0X +ElementAt codegen - Hash Lookup 136 144 4 0.1 13628.5 7.5X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=100000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +---------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 1838 1855 12 0.0 183795.5 1.0X +GetMapValue interpreted - Hash Lookup 104 110 9 0.1 10435.2 17.6X +GetMapValue codegen - Linear Lookup 1777 1793 15 0.0 177695.8 1.0X +GetMapValue codegen - Hash Lookup 134 135 3 0.1 13388.5 13.7X +ElementAt interpreted - Linear Lookup 1845 1862 8 0.0 184467.8 1.0X +ElementAt interpreted - Hash Lookup 105 110 7 0.1 10471.5 17.6X +ElementAt codegen - Linear Lookup 1516 1756 85 0.0 151566.1 1.2X +ElementAt codegen - Hash Lookup 134 140 10 0.1 13449.6 13.7X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=100000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +---------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 2412 2415 3 0.0 241164.8 1.0X +GetMapValue interpreted - Hash Lookup 103 109 12 0.1 10321.0 23.4X +GetMapValue codegen - Linear Lookup 2316 2333 14 0.0 231560.6 1.0X +GetMapValue codegen - Hash Lookup 134 137 3 0.1 13396.9 18.0X +ElementAt interpreted - Linear Lookup 2415 2440 22 0.0 241511.5 1.0X +ElementAt interpreted - Hash Lookup 104 110 14 0.1 10423.0 23.1X +ElementAt codegen - Linear Lookup 2321 2346 20 0.0 232082.3 1.0X +ElementAt codegen - Hash Lookup 135 138 4 0.1 13532.9 17.8X + From c7219678ebc6cfe913a121e89e1f3150d7a41c73 Mon Sep 17 00:00:00 2001 From: LuciferYang Date: Thu, 12 Mar 2026 18:41:10 +0000 Subject: [PATCH 18/24] Benchmark results for org.apache.spark.sql.execution.benchmark.MapLookupBenchmark (JDK 25, Scala 2.13, split 1 of 1) --- .../MapLookupBenchmark-jdk25-results.txt | 234 ++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 sql/core/benchmarks/MapLookupBenchmark-jdk25-results.txt diff --git a/sql/core/benchmarks/MapLookupBenchmark-jdk25-results.txt b/sql/core/benchmarks/MapLookupBenchmark-jdk25-results.txt new file mode 100644 index 0000000000000..10eefa2c38a9f --- /dev/null +++ b/sql/core/benchmarks/MapLookupBenchmark-jdk25-results.txt @@ -0,0 +1,234 @@ +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=1, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 20 29 6 0.5 2019.8 1.0X +GetMapValue interpreted - Hash Lookup 17 21 4 0.6 1740.1 1.2X +GetMapValue codegen - Linear Lookup 17 20 3 0.6 1673.6 1.2X +GetMapValue codegen - Hash Lookup 16 18 3 0.6 1556.9 1.3X +ElementAt interpreted - Linear Lookup 15 18 4 0.7 1475.2 1.4X +ElementAt interpreted - Hash Lookup 14 18 4 0.7 1438.1 1.4X +ElementAt codegen - Linear Lookup 14 17 3 0.7 1427.0 1.4X +ElementAt codegen - Hash Lookup 13 15 2 0.8 1304.8 1.5X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=1, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 13 15 3 0.8 1294.1 1.0X +GetMapValue interpreted - Hash Lookup 12 14 2 0.8 1248.8 1.0X +GetMapValue codegen - Linear Lookup 12 14 2 0.8 1219.6 1.1X +GetMapValue codegen - Hash Lookup 12 15 3 0.8 1196.7 1.1X +ElementAt interpreted - Linear Lookup 12 13 2 0.9 1158.6 1.1X +ElementAt interpreted - Hash Lookup 12 13 2 0.9 1156.3 1.1X +ElementAt codegen - Linear Lookup 12 13 2 0.9 1156.7 1.1X +ElementAt codegen - Hash Lookup 12 13 2 0.9 1172.5 1.1X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=1, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 11 13 2 0.9 1115.8 1.0X +GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1102.2 1.0X +GetMapValue codegen - Linear Lookup 11 12 2 0.9 1082.5 1.0X +GetMapValue codegen - Hash Lookup 11 13 2 0.9 1104.7 1.0X +ElementAt interpreted - Linear Lookup 11 13 3 0.9 1087.7 1.0X +ElementAt interpreted - Hash Lookup 11 13 2 0.9 1093.9 1.0X +ElementAt codegen - Linear Lookup 11 12 2 0.9 1061.0 1.1X +ElementAt codegen - Hash Lookup 11 12 2 0.9 1079.6 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=10, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted - Linear Lookup 11 13 3 0.9 1082.6 1.0X +GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1073.3 1.0X +GetMapValue codegen - Linear Lookup 11 12 1 0.9 1069.2 1.0X +GetMapValue codegen - Hash Lookup 11 12 2 0.9 1059.0 1.0X +ElementAt interpreted - Linear Lookup 11 12 2 0.9 1067.7 1.0X +ElementAt interpreted - Hash Lookup 11 12 2 0.9 1072.4 1.0X +ElementAt codegen - Linear Lookup 11 12 2 0.9 1060.9 1.0X +ElementAt codegen - Hash Lookup 11 12 2 0.9 1066.5 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=10, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted - Linear Lookup 10 11 2 1.0 1035.5 1.0X +GetMapValue interpreted - Hash Lookup 10 12 2 1.0 1044.0 1.0X +GetMapValue codegen - Linear Lookup 10 12 2 1.0 1045.0 1.0X +GetMapValue codegen - Hash Lookup 10 12 2 1.0 1047.9 1.0X +ElementAt interpreted - Linear Lookup 11 11 2 0.9 1060.1 1.0X +ElementAt interpreted - Hash Lookup 10 12 2 1.0 1046.4 1.0X +ElementAt codegen - Linear Lookup 11 12 1 0.9 1054.9 1.0X +ElementAt codegen - Hash Lookup 10 12 2 1.0 1032.8 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=10, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted - Linear Lookup 10 11 2 1.0 1026.6 1.0X +GetMapValue interpreted - Hash Lookup 10 11 2 1.0 1020.7 1.0X +GetMapValue codegen - Linear Lookup 10 11 2 1.0 1036.8 1.0X +GetMapValue codegen - Hash Lookup 10 12 2 1.0 1027.7 1.0X +ElementAt interpreted - Linear Lookup 10 11 2 1.0 1010.0 1.0X +ElementAt interpreted - Hash Lookup 10 12 2 1.0 1030.8 1.0X +ElementAt codegen - Linear Lookup 10 11 1 1.0 1018.8 1.0X +ElementAt codegen - Hash Lookup 10 11 2 1.0 1010.7 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=100, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 11 12 1 0.9 1113.2 1.0X +GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1057.3 1.1X +GetMapValue codegen - Linear Lookup 11 13 2 0.9 1115.7 1.0X +GetMapValue codegen - Hash Lookup 10 11 2 1.0 1011.7 1.1X +ElementAt interpreted - Linear Lookup 11 12 2 0.9 1116.4 1.0X +ElementAt interpreted - Hash Lookup 10 12 2 1.0 1049.7 1.1X +ElementAt codegen - Linear Lookup 11 12 2 0.9 1106.7 1.0X +ElementAt codegen - Hash Lookup 10 12 2 1.0 1023.6 1.1X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=100, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 11 12 1 0.9 1125.4 1.0X +GetMapValue interpreted - Hash Lookup 10 11 2 1.0 1029.9 1.1X +GetMapValue codegen - Linear Lookup 11 13 2 0.9 1127.4 1.0X +GetMapValue codegen - Hash Lookup 10 11 2 1.0 1005.3 1.1X +ElementAt interpreted - Linear Lookup 11 13 2 0.9 1137.4 1.0X +ElementAt interpreted - Hash Lookup 10 11 2 1.0 1027.1 1.1X +ElementAt codegen - Linear Lookup 11 12 1 0.9 1139.5 1.0X +ElementAt codegen - Hash Lookup 10 12 2 1.0 1015.6 1.1X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=100, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 12 13 2 0.9 1164.1 1.0X +GetMapValue interpreted - Hash Lookup 10 11 1 1.0 1015.1 1.1X +GetMapValue codegen - Linear Lookup 12 13 2 0.9 1164.6 1.0X +GetMapValue codegen - Hash Lookup 10 11 2 1.0 992.8 1.2X +ElementAt interpreted - Linear Lookup 11 13 2 0.9 1143.7 1.0X +ElementAt interpreted - Hash Lookup 10 11 2 1.0 1021.5 1.1X +ElementAt codegen - Linear Lookup 12 13 2 0.9 1161.2 1.0X +ElementAt codegen - Hash Lookup 10 11 2 1.0 995.7 1.2X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=1000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 19 20 2 0.5 1861.7 1.0X +GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1126.5 1.7X +GetMapValue codegen - Linear Lookup 18 19 2 0.6 1778.3 1.0X +GetMapValue codegen - Hash Lookup 11 13 2 0.9 1113.8 1.7X +ElementAt interpreted - Linear Lookup 19 21 2 0.5 1877.3 1.0X +ElementAt interpreted - Hash Lookup 11 12 2 0.9 1128.2 1.7X +ElementAt codegen - Linear Lookup 18 20 2 0.6 1786.2 1.0X +ElementAt codegen - Hash Lookup 11 12 2 0.9 1112.4 1.7X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=1000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 22 24 2 0.4 2226.4 1.0X +GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1099.7 2.0X +GetMapValue codegen - Linear Lookup 21 23 2 0.5 2072.1 1.1X +GetMapValue codegen - Hash Lookup 11 12 2 0.9 1094.8 2.0X +ElementAt interpreted - Linear Lookup 22 24 2 0.4 2234.1 1.0X +ElementAt interpreted - Hash Lookup 11 12 2 0.9 1104.6 2.0X +ElementAt codegen - Linear Lookup 21 23 2 0.5 2142.6 1.0X +ElementAt codegen - Hash Lookup 11 12 2 0.9 1097.9 2.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=1000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 26 27 2 0.4 2588.3 1.0X +GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1091.5 2.4X +GetMapValue codegen - Linear Lookup 24 26 2 0.4 2417.9 1.1X +GetMapValue codegen - Hash Lookup 11 12 1 0.9 1088.8 2.4X +ElementAt interpreted - Linear Lookup 26 28 2 0.4 2574.4 1.0X +ElementAt interpreted - Hash Lookup 11 12 2 0.9 1076.1 2.4X +ElementAt codegen - Linear Lookup 24 26 2 0.4 2404.7 1.1X +ElementAt codegen - Hash Lookup 11 12 2 0.9 1080.6 2.4X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=10000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +--------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 92 94 2 0.1 9152.3 1.0X +GetMapValue interpreted - Hash Lookup 19 21 2 0.5 1937.1 4.7X +GetMapValue codegen - Linear Lookup 85 87 2 0.1 8486.8 1.1X +GetMapValue codegen - Hash Lookup 22 23 2 0.5 2166.1 4.2X +ElementAt interpreted - Linear Lookup 91 92 2 0.1 9103.5 1.0X +ElementAt interpreted - Hash Lookup 19 21 2 0.5 1947.9 4.7X +ElementAt codegen - Linear Lookup 85 87 2 0.1 8540.7 1.1X +ElementAt codegen - Hash Lookup 22 24 2 0.5 2170.2 4.2X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=10000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +--------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 127 132 3 0.1 12718.4 1.0X +GetMapValue interpreted - Hash Lookup 19 20 3 0.5 1887.3 6.7X +GetMapValue codegen - Linear Lookup 116 118 2 0.1 11592.6 1.1X +GetMapValue codegen - Hash Lookup 21 23 2 0.5 2111.7 6.0X +ElementAt interpreted - Linear Lookup 128 131 4 0.1 12767.0 1.0X +ElementAt interpreted - Hash Lookup 19 20 2 0.5 1890.6 6.7X +ElementAt codegen - Linear Lookup 115 117 2 0.1 11518.6 1.1X +ElementAt codegen - Hash Lookup 21 23 2 0.5 2132.6 6.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=10000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +--------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 162 170 6 0.1 16219.8 1.0X +GetMapValue interpreted - Hash Lookup 18 20 2 0.5 1832.6 8.9X +GetMapValue codegen - Linear Lookup 146 148 2 0.1 14599.2 1.1X +GetMapValue codegen - Hash Lookup 21 22 2 0.5 2063.8 7.9X +ElementAt interpreted - Linear Lookup 163 168 5 0.1 16282.4 1.0X +ElementAt interpreted - Hash Lookup 18 20 2 0.5 1830.9 8.9X +ElementAt codegen - Linear Lookup 146 149 4 0.1 14576.9 1.1X +ElementAt codegen - Hash Lookup 21 23 2 0.5 2077.6 7.8X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=100000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +---------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 1877 1926 25 0.0 187738.5 1.0X +GetMapValue interpreted - Hash Lookup 108 117 7 0.1 10849.2 17.3X +GetMapValue codegen - Linear Lookup 1926 1938 7 0.0 192584.4 1.0X +GetMapValue codegen - Hash Lookup 140 151 8 0.1 13983.8 13.4X +ElementAt interpreted - Linear Lookup 1807 1866 85 0.0 180696.4 1.0X +ElementAt interpreted - Hash Lookup 111 117 5 0.1 11136.7 16.9X +ElementAt codegen - Linear Lookup 1807 1907 44 0.0 180746.2 1.0X +ElementAt codegen - Hash Lookup 144 154 6 0.1 14384.3 13.1X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=100000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +---------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 2672 2744 107 0.0 267156.0 1.0X +GetMapValue interpreted - Hash Lookup 109 119 7 0.1 10859.6 24.6X +GetMapValue codegen - Linear Lookup 2862 2891 38 0.0 286207.1 0.9X +GetMapValue codegen - Hash Lookup 147 155 6 0.1 14697.1 18.2X +ElementAt interpreted - Linear Lookup 2857 2868 6 0.0 285676.4 0.9X +ElementAt interpreted - Hash Lookup 107 113 4 0.1 10737.2 24.9X +ElementAt codegen - Linear Lookup 2680 2756 113 0.0 268045.1 1.0X +ElementAt codegen - Hash Lookup 139 146 5 0.1 13860.0 19.3X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=100000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +---------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 3547 3803 105 0.0 354705.7 1.0X +GetMapValue interpreted - Hash Lookup 111 119 9 0.1 11100.2 32.0X +GetMapValue codegen - Linear Lookup 3547 3762 111 0.0 354682.6 1.0X +GetMapValue codegen - Hash Lookup 138 144 4 0.1 13825.5 25.7X +ElementAt interpreted - Linear Lookup 3557 3632 117 0.0 355703.5 1.0X +ElementAt interpreted - Hash Lookup 109 112 4 0.1 10888.3 32.6X +ElementAt codegen - Linear Lookup 3783 3789 4 0.0 378265.0 0.9X +ElementAt codegen - Hash Lookup 138 143 5 0.1 13824.6 25.7X + From 3bc2c2a2b4b3c699242fa9feaceaa34abe630a4f Mon Sep 17 00:00:00 2001 From: LuciferYang Date: Thu, 12 Mar 2026 18:44:23 +0000 Subject: [PATCH 19/24] Benchmark results for org.apache.spark.sql.execution.benchmark.MapLookupBenchmark (JDK 17, Scala 2.13, split 1 of 1) --- .../benchmarks/MapLookupBenchmark-results.txt | 234 ++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 sql/core/benchmarks/MapLookupBenchmark-results.txt diff --git a/sql/core/benchmarks/MapLookupBenchmark-results.txt b/sql/core/benchmarks/MapLookupBenchmark-results.txt new file mode 100644 index 0000000000000..42c78283e12c7 --- /dev/null +++ b/sql/core/benchmarks/MapLookupBenchmark-results.txt @@ -0,0 +1,234 @@ +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 29 41 6 0.3 2942.1 1.0X +GetMapValue interpreted - Hash Lookup 24 29 3 0.4 2446.8 1.2X +GetMapValue codegen - Linear Lookup 23 27 3 0.4 2337.8 1.3X +GetMapValue codegen - Hash Lookup 22 25 2 0.4 2225.3 1.3X +ElementAt interpreted - Linear Lookup 21 24 2 0.5 2104.8 1.4X +ElementAt interpreted - Hash Lookup 20 23 3 0.5 2016.2 1.5X +ElementAt codegen - Linear Lookup 20 22 2 0.5 1966.9 1.5X +ElementAt codegen - Hash Lookup 20 22 3 0.5 1970.9 1.5X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 19 21 2 0.5 1930.9 1.0X +GetMapValue interpreted - Hash Lookup 19 20 2 0.5 1865.4 1.0X +GetMapValue codegen - Linear Lookup 18 20 2 0.5 1828.7 1.1X +GetMapValue codegen - Hash Lookup 18 20 2 0.5 1841.3 1.0X +ElementAt interpreted - Linear Lookup 18 20 2 0.5 1825.8 1.1X +ElementAt interpreted - Hash Lookup 18 19 2 0.6 1780.0 1.1X +ElementAt codegen - Linear Lookup 18 20 2 0.6 1754.8 1.1X +ElementAt codegen - Hash Lookup 17 19 2 0.6 1683.7 1.1X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 16 18 2 0.6 1644.3 1.0X +GetMapValue interpreted - Hash Lookup 16 18 2 0.6 1640.6 1.0X +GetMapValue codegen - Linear Lookup 16 18 1 0.6 1611.8 1.0X +GetMapValue codegen - Hash Lookup 17 18 2 0.6 1669.6 1.0X +ElementAt interpreted - Linear Lookup 16 18 2 0.6 1624.9 1.0X +ElementAt interpreted - Hash Lookup 16 18 2 0.6 1648.5 1.0X +ElementAt codegen - Linear Lookup 16 18 2 0.6 1632.9 1.0X +ElementAt codegen - Hash Lookup 17 18 1 0.6 1667.8 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=10, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted - Linear Lookup 16 18 1 0.6 1640.8 1.0X +GetMapValue interpreted - Hash Lookup 17 18 2 0.6 1652.6 1.0X +GetMapValue codegen - Linear Lookup 17 18 1 0.6 1655.0 1.0X +GetMapValue codegen - Hash Lookup 16 18 2 0.6 1600.1 1.0X +ElementAt interpreted - Linear Lookup 16 18 2 0.6 1605.8 1.0X +ElementAt interpreted - Hash Lookup 16 18 2 0.6 1623.1 1.0X +ElementAt codegen - Linear Lookup 16 17 1 0.6 1579.5 1.0X +ElementAt codegen - Hash Lookup 16 18 2 0.6 1620.8 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=10, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted - Linear Lookup 16 17 2 0.6 1561.6 1.0X +GetMapValue interpreted - Hash Lookup 16 17 1 0.6 1553.7 1.0X +GetMapValue codegen - Linear Lookup 16 17 2 0.6 1558.8 1.0X +GetMapValue codegen - Hash Lookup 16 17 2 0.6 1565.9 1.0X +ElementAt interpreted - Linear Lookup 16 17 1 0.6 1585.3 1.0X +ElementAt interpreted - Hash Lookup 17 18 1 0.6 1650.7 0.9X +ElementAt codegen - Linear Lookup 16 17 1 0.6 1566.2 1.0X +ElementAt codegen - Hash Lookup 16 17 1 0.6 1589.7 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=10, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted - Linear Lookup 16 17 1 0.6 1573.9 1.0X +GetMapValue interpreted - Hash Lookup 15 17 1 0.7 1537.7 1.0X +GetMapValue codegen - Linear Lookup 15 16 1 0.7 1521.7 1.0X +GetMapValue codegen - Hash Lookup 15 17 2 0.7 1527.2 1.0X +ElementAt interpreted - Linear Lookup 16 17 1 0.6 1558.9 1.0X +ElementAt interpreted - Hash Lookup 15 17 1 0.7 1536.4 1.0X +ElementAt codegen - Linear Lookup 15 17 1 0.7 1536.8 1.0X +ElementAt codegen - Hash Lookup 15 16 1 0.7 1485.8 1.1X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=100, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 16 17 1 0.6 1616.8 1.0X +GetMapValue interpreted - Hash Lookup 15 17 1 0.6 1547.7 1.0X +GetMapValue codegen - Linear Lookup 16 17 1 0.6 1608.7 1.0X +GetMapValue codegen - Hash Lookup 15 16 1 0.7 1493.3 1.1X +ElementAt interpreted - Linear Lookup 16 18 2 0.6 1634.4 1.0X +ElementAt interpreted - Hash Lookup 16 18 2 0.6 1581.5 1.0X +ElementAt codegen - Linear Lookup 16 18 2 0.6 1625.2 1.0X +ElementAt codegen - Hash Lookup 15 17 1 0.7 1535.6 1.1X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=100, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 17 18 1 0.6 1691.4 1.0X +GetMapValue interpreted - Hash Lookup 15 17 1 0.6 1548.6 1.1X +GetMapValue codegen - Linear Lookup 16 17 1 0.6 1626.4 1.0X +GetMapValue codegen - Hash Lookup 15 17 2 0.7 1508.0 1.1X +ElementAt interpreted - Linear Lookup 16 17 1 0.6 1630.0 1.0X +ElementAt interpreted - Hash Lookup 15 16 1 0.7 1536.8 1.1X +ElementAt codegen - Linear Lookup 17 18 1 0.6 1677.8 1.0X +ElementAt codegen - Hash Lookup 15 17 1 0.6 1544.1 1.1X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=100, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 16 18 1 0.6 1635.0 1.0X +GetMapValue interpreted - Hash Lookup 15 16 1 0.7 1508.2 1.1X +GetMapValue codegen - Linear Lookup 16 17 1 0.6 1612.8 1.0X +GetMapValue codegen - Hash Lookup 15 16 2 0.7 1505.8 1.1X +ElementAt interpreted - Linear Lookup 17 18 1 0.6 1708.3 1.0X +ElementAt interpreted - Hash Lookup 16 17 1 0.6 1579.8 1.0X +ElementAt codegen - Linear Lookup 17 18 1 0.6 1681.9 1.0X +ElementAt codegen - Hash Lookup 16 17 1 0.6 1568.8 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 25 27 1 0.4 2547.4 1.0X +GetMapValue interpreted - Hash Lookup 16 17 1 0.6 1630.9 1.6X +GetMapValue codegen - Linear Lookup 23 24 1 0.4 2281.8 1.1X +GetMapValue codegen - Hash Lookup 16 17 1 0.6 1633.7 1.6X +ElementAt interpreted - Linear Lookup 25 26 1 0.4 2487.0 1.0X +ElementAt interpreted - Hash Lookup 16 17 1 0.6 1621.5 1.6X +ElementAt codegen - Linear Lookup 23 24 3 0.4 2281.2 1.1X +ElementAt codegen - Hash Lookup 16 17 1 0.6 1627.6 1.6X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 27 28 1 0.4 2682.6 1.0X +GetMapValue interpreted - Hash Lookup 16 17 1 0.6 1572.5 1.7X +GetMapValue codegen - Linear Lookup 26 27 1 0.4 2560.7 1.0X +GetMapValue codegen - Hash Lookup 16 17 1 0.6 1596.6 1.7X +ElementAt interpreted - Linear Lookup 27 29 1 0.4 2722.8 1.0X +ElementAt interpreted - Hash Lookup 16 17 1 0.6 1606.6 1.7X +ElementAt codegen - Linear Lookup 26 27 1 0.4 2564.6 1.0X +ElementAt codegen - Hash Lookup 16 17 1 0.6 1626.2 1.6X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 29 31 1 0.3 2929.3 1.0X +GetMapValue interpreted - Hash Lookup 16 17 1 0.6 1585.8 1.8X +GetMapValue codegen - Linear Lookup 29 31 1 0.3 2914.8 1.0X +GetMapValue codegen - Hash Lookup 16 17 1 0.6 1600.4 1.8X +ElementAt interpreted - Linear Lookup 30 31 2 0.3 2959.2 1.0X +ElementAt interpreted - Hash Lookup 16 17 1 0.6 1591.3 1.8X +ElementAt codegen - Linear Lookup 30 32 2 0.3 2978.4 1.0X +ElementAt codegen - Hash Lookup 16 17 1 0.6 1626.4 1.8X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=10000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +--------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 144 148 7 0.1 14443.1 1.0X +GetMapValue interpreted - Hash Lookup 25 26 2 0.4 2473.5 5.8X +GetMapValue codegen - Linear Lookup 123 124 1 0.1 12287.6 1.2X +GetMapValue codegen - Hash Lookup 27 29 2 0.4 2698.3 5.4X +ElementAt interpreted - Linear Lookup 145 146 2 0.1 14491.7 1.0X +ElementAt interpreted - Hash Lookup 25 26 1 0.4 2477.4 5.8X +ElementAt codegen - Linear Lookup 122 124 2 0.1 12203.9 1.2X +ElementAt codegen - Hash Lookup 26 28 2 0.4 2639.9 5.5X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=10000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +--------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 184 186 2 0.1 18378.6 1.0X +GetMapValue interpreted - Hash Lookup 24 26 2 0.4 2431.2 7.6X +GetMapValue codegen - Linear Lookup 174 175 1 0.1 17386.6 1.1X +GetMapValue codegen - Hash Lookup 26 28 2 0.4 2646.1 6.9X +ElementAt interpreted - Linear Lookup 184 187 2 0.1 18412.8 1.0X +ElementAt interpreted - Hash Lookup 24 26 3 0.4 2393.7 7.7X +ElementAt codegen - Linear Lookup 173 175 2 0.1 17266.2 1.1X +ElementAt codegen - Hash Lookup 26 27 2 0.4 2597.9 7.1X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=10000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +--------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 217 221 4 0.0 21659.1 1.0X +GetMapValue interpreted - Hash Lookup 24 25 2 0.4 2399.5 9.0X +GetMapValue codegen - Linear Lookup 221 223 2 0.0 22121.7 1.0X +GetMapValue codegen - Hash Lookup 27 28 2 0.4 2699.8 8.0X +ElementAt interpreted - Linear Lookup 216 223 5 0.0 21607.6 1.0X +ElementAt interpreted - Hash Lookup 24 25 2 0.4 2365.1 9.2X +ElementAt codegen - Linear Lookup 222 224 2 0.0 22179.1 1.0X +ElementAt codegen - Hash Lookup 26 28 2 0.4 2590.9 8.4X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=100000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +---------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 1333 1347 8 0.0 133287.8 1.0X +GetMapValue interpreted - Hash Lookup 112 119 4 0.1 11177.5 11.9X +GetMapValue codegen - Linear Lookup 1114 1122 6 0.0 111444.7 1.2X +GetMapValue codegen - Hash Lookup 142 146 4 0.1 14173.6 9.4X +ElementAt interpreted - Linear Lookup 1341 1351 8 0.0 134058.6 1.0X +ElementAt interpreted - Hash Lookup 113 119 5 0.1 11292.4 11.8X +ElementAt codegen - Linear Lookup 1114 1122 7 0.0 111373.4 1.2X +ElementAt codegen - Hash Lookup 143 152 5 0.1 14268.4 9.3X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=100000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +---------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 1617 1625 7 0.0 161746.2 1.0X +GetMapValue interpreted - Hash Lookup 111 114 3 0.1 11097.3 14.6X +GetMapValue codegen - Linear Lookup 1585 1606 14 0.0 158455.5 1.0X +GetMapValue codegen - Hash Lookup 139 143 5 0.1 13850.9 11.7X +ElementAt interpreted - Linear Lookup 1623 1640 11 0.0 162297.8 1.0X +ElementAt interpreted - Hash Lookup 112 120 7 0.1 11167.1 14.5X +ElementAt codegen - Linear Lookup 1599 1609 7 0.0 159900.2 1.0X +ElementAt codegen - Hash Lookup 141 144 2 0.1 14124.9 11.5X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=100000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +---------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 2039 2066 16 0.0 203937.0 1.0X +GetMapValue interpreted - Hash Lookup 109 111 3 0.1 10883.8 18.7X +GetMapValue codegen - Linear Lookup 2072 2087 13 0.0 207204.3 1.0X +GetMapValue codegen - Hash Lookup 141 144 4 0.1 14083.6 14.5X +ElementAt interpreted - Linear Lookup 2005 2035 21 0.0 200460.7 1.0X +ElementAt interpreted - Hash Lookup 110 116 4 0.1 11043.5 18.5X +ElementAt codegen - Linear Lookup 2091 2216 194 0.0 209069.9 1.0X +ElementAt codegen - Hash Lookup 141 148 10 0.1 14084.8 14.5X + From fa38f0bdad78679e673186abdf5b5e71e8451fbe Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 13 Mar 2026 08:55:12 +0800 Subject: [PATCH 20/24] try add 1M back --- .../benchmark/MapLookupBenchmark.scala | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala index bd6c83548dc96..e9ec876e1f603 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MapLookupBenchmark.scala @@ -38,7 +38,7 @@ import org.apache.spark.sql.types._ * }}} */ object MapLookupBenchmark extends SqlBasedBenchmark { - private val NUMBER_OF_ITER = 10 + private val NUMBER_OF_ITER = 3 override def getSparkSession: SparkSession = { SparkSession.builder() @@ -86,7 +86,7 @@ object MapLookupBenchmark extends SqlBasedBenchmark { SQLConf.CODEGEN_FACTORY_MODE.key -> "NO_CODEGEN", SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> Int.MaxValue.toString, SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { - lookupDf.select(expr).write.format("noop").mode("overwrite").save() + lookupDf.select(expr).noop() } } @@ -96,7 +96,7 @@ object MapLookupBenchmark extends SqlBasedBenchmark { SQLConf.CODEGEN_FACTORY_MODE.key -> "NO_CODEGEN", SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> 0.toString, SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { - lookupDf.select(expr).write.format("noop").mode("overwrite").save() + lookupDf.select(expr).noop() } } @@ -106,7 +106,7 @@ object MapLookupBenchmark extends SqlBasedBenchmark { SQLConf.CODEGEN_FACTORY_MODE.key -> "CODEGEN_ONLY", SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> Int.MaxValue.toString, SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { - lookupDf.select(expr).write.format("noop").mode("overwrite").save() + lookupDf.select(expr).noop() } } @@ -116,7 +116,7 @@ object MapLookupBenchmark extends SqlBasedBenchmark { SQLConf.CODEGEN_FACTORY_MODE.key -> "CODEGEN_ONLY", SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> 0.toString, SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { - lookupDf.select(expr).write.format("noop").mode("overwrite").save() + lookupDf.select(expr).noop() } } @@ -126,7 +126,7 @@ object MapLookupBenchmark extends SqlBasedBenchmark { SQLConf.CODEGEN_FACTORY_MODE.key -> "NO_CODEGEN", SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> Int.MaxValue.toString, SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { - lookupDf.select(elementAtExpr).write.format("noop").mode("overwrite").save() + lookupDf.select(elementAtExpr).noop() } } @@ -136,7 +136,7 @@ object MapLookupBenchmark extends SqlBasedBenchmark { SQLConf.CODEGEN_FACTORY_MODE.key -> "NO_CODEGEN", SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> 0.toString, SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { - lookupDf.select(elementAtExpr).write.format("noop").mode("overwrite").save() + lookupDf.select(elementAtExpr).noop() } } @@ -146,7 +146,7 @@ object MapLookupBenchmark extends SqlBasedBenchmark { SQLConf.CODEGEN_FACTORY_MODE.key -> "CODEGEN_ONLY", SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> Int.MaxValue.toString, SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { - lookupDf.select(elementAtExpr).write.format("noop").mode("overwrite").save() + lookupDf.select(elementAtExpr).noop() } } @@ -156,15 +156,16 @@ object MapLookupBenchmark extends SqlBasedBenchmark { SQLConf.CODEGEN_FACTORY_MODE.key -> "CODEGEN_ONLY", SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> 0.toString, SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConvertToLocalRelation.ruleName) { - lookupDf.select(elementAtExpr).write.format("noop").mode("overwrite").save() + lookupDf.select(elementAtExpr).noop() } } benchmark.run() + System.gc() } override def runBenchmarkSuite(mainArgs: Array[String]): Unit = { - val sizes = Seq(1, 10, 100, 1000, 10000, 100000) + val sizes = Seq(1000000, 100000, 10000, 1000, 100, 10, 1) for (size <- sizes) { run(size, 1.0, IntegerType) run(size, 0.5, IntegerType) From 7c01c08ad09684cdcfa858abd4faf7508f1b7bd8 Mon Sep 17 00:00:00 2001 From: LuciferYang Date: Fri, 13 Mar 2026 01:49:23 +0000 Subject: [PATCH 21/24] Benchmark results for org.apache.spark.sql.execution.benchmark.MapLookupBenchmark (JDK 25, Scala 2.13, split 1 of 1) --- .../MapLookupBenchmark-jdk25-results.txt | 423 ++++++++++-------- 1 file changed, 231 insertions(+), 192 deletions(-) diff --git a/sql/core/benchmarks/MapLookupBenchmark-jdk25-results.txt b/sql/core/benchmarks/MapLookupBenchmark-jdk25-results.txt index 10eefa2c38a9f..e663922616206 100644 --- a/sql/core/benchmarks/MapLookupBenchmark-jdk25-results.txt +++ b/sql/core/benchmarks/MapLookupBenchmark-jdk25-results.txt @@ -1,234 +1,273 @@ OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz -MapLookup (size=1, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted - Linear Lookup 20 29 6 0.5 2019.8 1.0X -GetMapValue interpreted - Hash Lookup 17 21 4 0.6 1740.1 1.2X -GetMapValue codegen - Linear Lookup 17 20 3 0.6 1673.6 1.2X -GetMapValue codegen - Hash Lookup 16 18 3 0.6 1556.9 1.3X -ElementAt interpreted - Linear Lookup 15 18 4 0.7 1475.2 1.4X -ElementAt interpreted - Hash Lookup 14 18 4 0.7 1438.1 1.4X -ElementAt codegen - Linear Lookup 14 17 3 0.7 1427.0 1.4X -ElementAt codegen - Hash Lookup 13 15 2 0.8 1304.8 1.5X +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 29613 30212 849 0.0 2961282.8 1.0X +GetMapValue interpreted - Hash Lookup 1573 1720 127 0.0 157347.0 18.8X +GetMapValue codegen - Linear Lookup 5283 15168 8604 0.0 528254.8 5.6X +GetMapValue codegen - Hash Lookup 1803 2002 188 0.0 180305.1 16.4X +ElementAt interpreted - Linear Lookup 27158 27562 553 0.0 2715764.2 1.1X +ElementAt interpreted - Hash Lookup 1326 1574 307 0.0 132589.2 22.3X +ElementAt codegen - Linear Lookup 4953 15159 8850 0.0 495255.4 6.0X +ElementAt codegen - Hash Lookup 2006 2194 179 0.0 200622.1 14.8X OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz -MapLookup (size=1, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted - Linear Lookup 13 15 3 0.8 1294.1 1.0X -GetMapValue interpreted - Hash Lookup 12 14 2 0.8 1248.8 1.0X -GetMapValue codegen - Linear Lookup 12 14 2 0.8 1219.6 1.1X -GetMapValue codegen - Hash Lookup 12 15 3 0.8 1196.7 1.1X -ElementAt interpreted - Linear Lookup 12 13 2 0.9 1158.6 1.1X -ElementAt interpreted - Hash Lookup 12 13 2 0.9 1156.3 1.1X -ElementAt codegen - Linear Lookup 12 13 2 0.9 1156.7 1.1X -ElementAt codegen - Hash Lookup 12 13 2 0.9 1172.5 1.1X +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 29550 38070 7378 0.0 2955023.1 1.0X +GetMapValue interpreted - Hash Lookup 1581 1853 408 0.0 158062.3 18.7X +GetMapValue codegen - Linear Lookup 29686 30480 693 0.0 2968596.7 1.0X +GetMapValue codegen - Hash Lookup 1522 1748 255 0.0 152202.0 19.4X +ElementAt interpreted - Linear Lookup 10169 31677 18640 0.0 1016905.7 2.9X +ElementAt interpreted - Hash Lookup 1171 1584 366 0.0 117069.3 25.2X +ElementAt codegen - Linear Lookup 6115 11592 9203 0.0 611468.4 4.8X +ElementAt codegen - Hash Lookup 1569 1680 111 0.0 156930.4 18.8X OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz -MapLookup (size=1, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted - Linear Lookup 11 13 2 0.9 1115.8 1.0X -GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1102.2 1.0X -GetMapValue codegen - Linear Lookup 11 12 2 0.9 1082.5 1.0X -GetMapValue codegen - Hash Lookup 11 13 2 0.9 1104.7 1.0X -ElementAt interpreted - Linear Lookup 11 13 3 0.9 1087.7 1.0X -ElementAt interpreted - Hash Lookup 11 13 2 0.9 1093.9 1.0X -ElementAt codegen - Linear Lookup 11 12 2 0.9 1061.0 1.1X -ElementAt codegen - Hash Lookup 11 12 2 0.9 1079.6 1.0X +AMD EPYC 7763 64-Core Processor +MapLookup (size=1000000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 13098 41457 24563 0.0 1309770.8 1.0X +GetMapValue interpreted - Hash Lookup 1455 1841 343 0.0 145480.8 9.0X +GetMapValue codegen - Linear Lookup 41874 43187 1877 0.0 4187354.3 0.3X +GetMapValue codegen - Hash Lookup 2039 2242 205 0.0 203886.9 6.4X +ElementAt interpreted - Linear Lookup 56593 57729 1561 0.0 5659287.7 0.2X +ElementAt interpreted - Hash Lookup 1355 1518 158 0.0 135468.6 9.7X +ElementAt codegen - Linear Lookup 39751 40310 500 0.0 3975123.7 0.3X +ElementAt codegen - Hash Lookup 1691 1923 315 0.0 169138.7 7.7X OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz -MapLookup (size=10, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 11 13 3 0.9 1082.6 1.0X -GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1073.3 1.0X -GetMapValue codegen - Linear Lookup 11 12 1 0.9 1069.2 1.0X -GetMapValue codegen - Hash Lookup 11 12 2 0.9 1059.0 1.0X -ElementAt interpreted - Linear Lookup 11 12 2 0.9 1067.7 1.0X -ElementAt interpreted - Hash Lookup 11 12 2 0.9 1072.4 1.0X -ElementAt codegen - Linear Lookup 11 12 2 0.9 1060.9 1.0X -ElementAt codegen - Hash Lookup 11 12 2 0.9 1066.5 1.0X +AMD EPYC 7763 64-Core Processor +MapLookup (size=100000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +---------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 1246 1251 6 0.0 124601.4 1.0X +GetMapValue interpreted - Hash Lookup 113 122 14 0.1 11310.7 11.0X +GetMapValue codegen - Linear Lookup 1098 1105 10 0.0 109840.9 1.1X +GetMapValue codegen - Hash Lookup 144 155 18 0.1 14351.7 8.7X +ElementAt interpreted - Linear Lookup 1280 1292 11 0.0 128005.9 1.0X +ElementAt interpreted - Hash Lookup 113 121 11 0.1 11256.8 11.1X +ElementAt codegen - Linear Lookup 1097 1099 2 0.0 109718.9 1.1X +ElementAt codegen - Hash Lookup 145 157 18 0.1 14546.0 8.6X OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz -MapLookup (size=10, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 10 11 2 1.0 1035.5 1.0X -GetMapValue interpreted - Hash Lookup 10 12 2 1.0 1044.0 1.0X -GetMapValue codegen - Linear Lookup 10 12 2 1.0 1045.0 1.0X -GetMapValue codegen - Hash Lookup 10 12 2 1.0 1047.9 1.0X -ElementAt interpreted - Linear Lookup 11 11 2 0.9 1060.1 1.0X -ElementAt interpreted - Hash Lookup 10 12 2 1.0 1046.4 1.0X -ElementAt codegen - Linear Lookup 11 12 1 0.9 1054.9 1.0X -ElementAt codegen - Hash Lookup 10 12 2 1.0 1032.8 1.0X +AMD EPYC 7763 64-Core Processor +MapLookup (size=100000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +---------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 1688 1708 21 0.0 168766.8 1.0X +GetMapValue interpreted - Hash Lookup 114 121 11 0.1 11359.8 14.9X +GetMapValue codegen - Linear Lookup 1570 1573 3 0.0 156970.3 1.1X +GetMapValue codegen - Hash Lookup 139 148 14 0.1 13905.3 12.1X +ElementAt interpreted - Linear Lookup 1684 1728 41 0.0 168375.7 1.0X +ElementAt interpreted - Hash Lookup 107 119 11 0.1 10749.4 15.7X +ElementAt codegen - Linear Lookup 1572 1573 1 0.0 157227.0 1.1X +ElementAt codegen - Hash Lookup 139 150 16 0.1 13889.7 12.2X OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz -MapLookup (size=10, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 10 11 2 1.0 1026.6 1.0X -GetMapValue interpreted - Hash Lookup 10 11 2 1.0 1020.7 1.0X -GetMapValue codegen - Linear Lookup 10 11 2 1.0 1036.8 1.0X -GetMapValue codegen - Hash Lookup 10 12 2 1.0 1027.7 1.0X -ElementAt interpreted - Linear Lookup 10 11 2 1.0 1010.0 1.0X -ElementAt interpreted - Hash Lookup 10 12 2 1.0 1030.8 1.0X -ElementAt codegen - Linear Lookup 10 11 1 1.0 1018.8 1.0X -ElementAt codegen - Hash Lookup 10 11 2 1.0 1010.7 1.0X +AMD EPYC 7763 64-Core Processor +MapLookup (size=100000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +---------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 1952 2099 127 0.0 195172.8 1.0X +GetMapValue interpreted - Hash Lookup 106 114 11 0.1 10608.9 18.4X +GetMapValue codegen - Linear Lookup 2039 2045 10 0.0 203905.7 1.0X +GetMapValue codegen - Hash Lookup 139 147 15 0.1 13855.6 14.1X +ElementAt interpreted - Linear Lookup 2133 2164 27 0.0 213265.3 0.9X +ElementAt interpreted - Hash Lookup 108 123 19 0.1 10843.7 18.0X +ElementAt codegen - Linear Lookup 2040 2051 15 0.0 203988.3 1.0X +ElementAt codegen - Hash Lookup 140 148 15 0.1 13975.4 14.0X OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz -MapLookup (size=100, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 11 12 1 0.9 1113.2 1.0X -GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1057.3 1.1X -GetMapValue codegen - Linear Lookup 11 13 2 0.9 1115.7 1.0X -GetMapValue codegen - Hash Lookup 10 11 2 1.0 1011.7 1.1X -ElementAt interpreted - Linear Lookup 11 12 2 0.9 1116.4 1.0X -ElementAt interpreted - Hash Lookup 10 12 2 1.0 1049.7 1.1X -ElementAt codegen - Linear Lookup 11 12 2 0.9 1106.7 1.0X -ElementAt codegen - Hash Lookup 10 12 2 1.0 1023.6 1.1X +AMD EPYC 7763 64-Core Processor +MapLookup (size=10000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +--------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 119 122 2 0.1 11896.7 1.0X +GetMapValue interpreted - Hash Lookup 25 28 4 0.4 2476.6 4.8X +GetMapValue codegen - Linear Lookup 124 126 3 0.1 12395.4 1.0X +GetMapValue codegen - Hash Lookup 26 30 5 0.4 2643.5 4.5X +ElementAt interpreted - Linear Lookup 116 121 3 0.1 11642.2 1.0X +ElementAt interpreted - Hash Lookup 23 27 5 0.4 2331.2 5.1X +ElementAt codegen - Linear Lookup 123 124 1 0.1 12277.5 1.0X +ElementAt codegen - Hash Lookup 25 27 2 0.4 2541.3 4.7X OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz -MapLookup (size=100, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 11 12 1 0.9 1125.4 1.0X -GetMapValue interpreted - Hash Lookup 10 11 2 1.0 1029.9 1.1X -GetMapValue codegen - Linear Lookup 11 13 2 0.9 1127.4 1.0X -GetMapValue codegen - Hash Lookup 10 11 2 1.0 1005.3 1.1X -ElementAt interpreted - Linear Lookup 11 13 2 0.9 1137.4 1.0X -ElementAt interpreted - Hash Lookup 10 11 2 1.0 1027.1 1.1X -ElementAt codegen - Linear Lookup 11 12 1 0.9 1139.5 1.0X -ElementAt codegen - Hash Lookup 10 12 2 1.0 1015.6 1.1X +AMD EPYC 7763 64-Core Processor +MapLookup (size=10000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +--------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 172 175 1 0.1 17236.9 1.0X +GetMapValue interpreted - Hash Lookup 22 24 2 0.5 2192.1 7.9X +GetMapValue codegen - Linear Lookup 169 170 2 0.1 16851.4 1.0X +GetMapValue codegen - Hash Lookup 24 27 5 0.4 2379.3 7.2X +ElementAt interpreted - Linear Lookup 171 175 2 0.1 17115.7 1.0X +ElementAt interpreted - Hash Lookup 22 24 4 0.5 2209.8 7.8X +ElementAt codegen - Linear Lookup 169 170 1 0.1 16880.9 1.0X +ElementAt codegen - Hash Lookup 25 27 4 0.4 2500.4 6.9X OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz -MapLookup (size=100, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 12 13 2 0.9 1164.1 1.0X -GetMapValue interpreted - Hash Lookup 10 11 1 1.0 1015.1 1.1X -GetMapValue codegen - Linear Lookup 12 13 2 0.9 1164.6 1.0X -GetMapValue codegen - Hash Lookup 10 11 2 1.0 992.8 1.2X -ElementAt interpreted - Linear Lookup 11 13 2 0.9 1143.7 1.0X -ElementAt interpreted - Hash Lookup 10 11 2 1.0 1021.5 1.1X -ElementAt codegen - Linear Lookup 12 13 2 0.9 1161.2 1.0X -ElementAt codegen - Hash Lookup 10 11 2 1.0 995.7 1.2X +AMD EPYC 7763 64-Core Processor +MapLookup (size=10000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +--------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 225 230 3 0.0 22512.3 1.0X +GetMapValue interpreted - Hash Lookup 21 24 4 0.5 2143.0 10.5X +GetMapValue codegen - Linear Lookup 215 216 2 0.0 21453.3 1.0X +GetMapValue codegen - Hash Lookup 23 26 4 0.4 2336.9 9.6X +ElementAt interpreted - Linear Lookup 228 230 2 0.0 22799.4 1.0X +ElementAt interpreted - Hash Lookup 21 24 5 0.5 2100.4 10.7X +ElementAt codegen - Linear Lookup 214 215 2 0.0 21386.2 1.1X +ElementAt codegen - Hash Lookup 23 26 5 0.4 2328.5 9.7X OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=1000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 19 20 2 0.5 1861.7 1.0X -GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1126.5 1.7X -GetMapValue codegen - Linear Lookup 18 19 2 0.6 1778.3 1.0X -GetMapValue codegen - Hash Lookup 11 13 2 0.9 1113.8 1.7X -ElementAt interpreted - Linear Lookup 19 21 2 0.5 1877.3 1.0X -ElementAt interpreted - Hash Lookup 11 12 2 0.9 1128.2 1.7X -ElementAt codegen - Linear Lookup 18 20 2 0.6 1786.2 1.0X -ElementAt codegen - Hash Lookup 11 12 2 0.9 1112.4 1.7X +GetMapValue interpreted - Linear Lookup 19 23 4 0.5 1947.5 1.0X +GetMapValue interpreted - Hash Lookup 13 16 3 0.7 1346.7 1.4X +GetMapValue codegen - Linear Lookup 22 25 3 0.5 2211.2 0.9X +GetMapValue codegen - Hash Lookup 13 15 4 0.8 1257.3 1.5X +ElementAt interpreted - Linear Lookup 20 22 2 0.5 1985.4 1.0X +ElementAt interpreted - Hash Lookup 13 15 4 0.8 1297.9 1.5X +ElementAt codegen - Linear Lookup 23 25 3 0.4 2313.0 0.8X +ElementAt codegen - Hash Lookup 13 15 4 0.8 1285.5 1.5X OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=1000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 22 24 2 0.4 2226.4 1.0X -GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1099.7 2.0X -GetMapValue codegen - Linear Lookup 21 23 2 0.5 2072.1 1.1X -GetMapValue codegen - Hash Lookup 11 12 2 0.9 1094.8 2.0X -ElementAt interpreted - Linear Lookup 22 24 2 0.4 2234.1 1.0X -ElementAt interpreted - Hash Lookup 11 12 2 0.9 1104.6 2.0X -ElementAt codegen - Linear Lookup 21 23 2 0.5 2142.6 1.0X -ElementAt codegen - Hash Lookup 11 12 2 0.9 1097.9 2.0X +GetMapValue interpreted - Linear Lookup 23 24 2 0.4 2250.7 1.0X +GetMapValue interpreted - Hash Lookup 12 15 3 0.8 1248.2 1.8X +GetMapValue codegen - Linear Lookup 27 29 3 0.4 2725.2 0.8X +GetMapValue codegen - Hash Lookup 13 14 3 0.8 1254.8 1.8X +ElementAt interpreted - Linear Lookup 22 24 4 0.5 2174.6 1.0X +ElementAt interpreted - Hash Lookup 13 15 2 0.8 1308.6 1.7X +ElementAt codegen - Linear Lookup 28 29 2 0.4 2755.9 0.8X +ElementAt codegen - Hash Lookup 13 14 2 0.8 1254.0 1.8X OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +AMD EPYC 7763 64-Core Processor MapLookup (size=1000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 26 27 2 0.4 2588.3 1.0X -GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1091.5 2.4X -GetMapValue codegen - Linear Lookup 24 26 2 0.4 2417.9 1.1X -GetMapValue codegen - Hash Lookup 11 12 1 0.9 1088.8 2.4X -ElementAt interpreted - Linear Lookup 26 28 2 0.4 2574.4 1.0X -ElementAt interpreted - Hash Lookup 11 12 2 0.9 1076.1 2.4X -ElementAt codegen - Linear Lookup 24 26 2 0.4 2404.7 1.1X -ElementAt codegen - Hash Lookup 11 12 2 0.9 1080.6 2.4X +GetMapValue interpreted - Linear Lookup 24 26 2 0.4 2425.7 1.0X +GetMapValue interpreted - Hash Lookup 13 15 3 0.8 1264.2 1.9X +GetMapValue codegen - Linear Lookup 32 34 3 0.3 3150.3 0.8X +GetMapValue codegen - Hash Lookup 12 14 2 0.8 1246.0 1.9X +ElementAt interpreted - Linear Lookup 24 26 2 0.4 2425.3 1.0X +ElementAt interpreted - Hash Lookup 12 14 2 0.8 1224.1 2.0X +ElementAt codegen - Linear Lookup 31 33 2 0.3 3145.8 0.8X +ElementAt codegen - Hash Lookup 11 13 2 0.9 1143.9 2.1X OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz -MapLookup (size=10000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 92 94 2 0.1 9152.3 1.0X -GetMapValue interpreted - Hash Lookup 19 21 2 0.5 1937.1 4.7X -GetMapValue codegen - Linear Lookup 85 87 2 0.1 8486.8 1.1X -GetMapValue codegen - Hash Lookup 22 23 2 0.5 2166.1 4.2X -ElementAt interpreted - Linear Lookup 91 92 2 0.1 9103.5 1.0X -ElementAt interpreted - Hash Lookup 19 21 2 0.5 1947.9 4.7X -ElementAt codegen - Linear Lookup 85 87 2 0.1 8540.7 1.1X -ElementAt codegen - Hash Lookup 22 24 2 0.5 2170.2 4.2X +AMD EPYC 7763 64-Core Processor +MapLookup (size=100, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 12 13 2 0.9 1170.0 1.0X +GetMapValue interpreted - Hash Lookup 11 13 2 0.9 1116.3 1.0X +GetMapValue codegen - Linear Lookup 12 14 2 0.8 1217.9 1.0X +GetMapValue codegen - Hash Lookup 10 12 3 1.0 1022.6 1.1X +ElementAt interpreted - Linear Lookup 11 13 2 0.9 1145.3 1.0X +ElementAt interpreted - Hash Lookup 10 12 2 1.0 1041.4 1.1X +ElementAt codegen - Linear Lookup 11 13 2 0.9 1142.6 1.0X +ElementAt codegen - Hash Lookup 11 12 2 1.0 1050.4 1.1X OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz -MapLookup (size=10000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 127 132 3 0.1 12718.4 1.0X -GetMapValue interpreted - Hash Lookup 19 20 3 0.5 1887.3 6.7X -GetMapValue codegen - Linear Lookup 116 118 2 0.1 11592.6 1.1X -GetMapValue codegen - Hash Lookup 21 23 2 0.5 2111.7 6.0X -ElementAt interpreted - Linear Lookup 128 131 4 0.1 12767.0 1.0X -ElementAt interpreted - Hash Lookup 19 20 2 0.5 1890.6 6.7X -ElementAt codegen - Linear Lookup 115 117 2 0.1 11518.6 1.1X -ElementAt codegen - Hash Lookup 21 23 2 0.5 2132.6 6.0X +AMD EPYC 7763 64-Core Processor +MapLookup (size=100, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 11 12 2 0.9 1116.4 1.0X +GetMapValue interpreted - Hash Lookup 11 13 2 0.9 1101.3 1.0X +GetMapValue codegen - Linear Lookup 11 12 2 0.9 1118.0 1.0X +GetMapValue codegen - Hash Lookup 10 11 2 1.0 992.0 1.1X +ElementAt interpreted - Linear Lookup 11 13 2 0.9 1143.3 1.0X +ElementAt interpreted - Hash Lookup 10 12 2 1.0 1036.5 1.1X +ElementAt codegen - Linear Lookup 11 12 2 0.9 1110.7 1.0X +ElementAt codegen - Hash Lookup 10 11 2 1.0 981.6 1.1X OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz -MapLookup (size=10000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 162 170 6 0.1 16219.8 1.0X -GetMapValue interpreted - Hash Lookup 18 20 2 0.5 1832.6 8.9X -GetMapValue codegen - Linear Lookup 146 148 2 0.1 14599.2 1.1X -GetMapValue codegen - Hash Lookup 21 22 2 0.5 2063.8 7.9X -ElementAt interpreted - Linear Lookup 163 168 5 0.1 16282.4 1.0X -ElementAt interpreted - Hash Lookup 18 20 2 0.5 1830.9 8.9X -ElementAt codegen - Linear Lookup 146 149 4 0.1 14576.9 1.1X -ElementAt codegen - Hash Lookup 21 23 2 0.5 2077.6 7.8X +AMD EPYC 7763 64-Core Processor +MapLookup (size=100, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 11 12 2 0.9 1133.4 1.0X +GetMapValue interpreted - Hash Lookup 10 11 2 1.0 998.9 1.1X +GetMapValue codegen - Linear Lookup 12 13 2 0.9 1163.8 1.0X +GetMapValue codegen - Hash Lookup 10 11 2 1.0 989.1 1.1X +ElementAt interpreted - Linear Lookup 11 12 2 0.9 1135.1 1.0X +ElementAt interpreted - Hash Lookup 10 11 2 1.0 996.4 1.1X +ElementAt codegen - Linear Lookup 11 13 2 0.9 1140.3 1.0X +ElementAt codegen - Hash Lookup 10 11 2 1.0 970.8 1.2X OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz -MapLookup (size=100000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ----------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 1877 1926 25 0.0 187738.5 1.0X -GetMapValue interpreted - Hash Lookup 108 117 7 0.1 10849.2 17.3X -GetMapValue codegen - Linear Lookup 1926 1938 7 0.0 192584.4 1.0X -GetMapValue codegen - Hash Lookup 140 151 8 0.1 13983.8 13.4X -ElementAt interpreted - Linear Lookup 1807 1866 85 0.0 180696.4 1.0X -ElementAt interpreted - Hash Lookup 111 117 5 0.1 11136.7 16.9X -ElementAt codegen - Linear Lookup 1807 1907 44 0.0 180746.2 1.0X -ElementAt codegen - Hash Lookup 144 154 6 0.1 14384.3 13.1X +AMD EPYC 7763 64-Core Processor +MapLookup (size=10, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted - Linear Lookup 10 11 2 1.0 994.1 1.0X +GetMapValue interpreted - Hash Lookup 10 11 2 1.0 990.1 1.0X +GetMapValue codegen - Linear Lookup 10 11 2 1.0 970.9 1.0X +GetMapValue codegen - Hash Lookup 10 11 2 1.0 958.9 1.0X +ElementAt interpreted - Linear Lookup 10 11 2 1.0 1011.1 1.0X +ElementAt interpreted - Hash Lookup 10 11 2 1.0 996.2 1.0X +ElementAt codegen - Linear Lookup 10 11 2 1.0 959.8 1.0X +ElementAt codegen - Hash Lookup 10 11 2 1.0 958.6 1.0X OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz -MapLookup (size=100000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ----------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 2672 2744 107 0.0 267156.0 1.0X -GetMapValue interpreted - Hash Lookup 109 119 7 0.1 10859.6 24.6X -GetMapValue codegen - Linear Lookup 2862 2891 38 0.0 286207.1 0.9X -GetMapValue codegen - Hash Lookup 147 155 6 0.1 14697.1 18.2X -ElementAt interpreted - Linear Lookup 2857 2868 6 0.0 285676.4 0.9X -ElementAt interpreted - Hash Lookup 107 113 4 0.1 10737.2 24.9X -ElementAt codegen - Linear Lookup 2680 2756 113 0.0 268045.1 1.0X -ElementAt codegen - Hash Lookup 139 146 5 0.1 13860.0 19.3X +AMD EPYC 7763 64-Core Processor +MapLookup (size=10, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted - Linear Lookup 10 11 2 1.0 981.1 1.0X +GetMapValue interpreted - Hash Lookup 10 11 2 1.0 977.1 1.0X +GetMapValue codegen - Linear Lookup 9 11 2 1.1 949.2 1.0X +GetMapValue codegen - Hash Lookup 10 11 2 1.0 959.6 1.0X +ElementAt interpreted - Linear Lookup 10 11 2 1.0 1010.3 1.0X +ElementAt interpreted - Hash Lookup 10 11 2 1.0 1018.2 1.0X +ElementAt codegen - Linear Lookup 10 11 2 1.0 972.2 1.0X +ElementAt codegen - Hash Lookup 10 11 2 1.0 998.7 1.0X OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz -MapLookup (size=100000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ----------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 3547 3803 105 0.0 354705.7 1.0X -GetMapValue interpreted - Hash Lookup 111 119 9 0.1 11100.2 32.0X -GetMapValue codegen - Linear Lookup 3547 3762 111 0.0 354682.6 1.0X -GetMapValue codegen - Hash Lookup 138 144 4 0.1 13825.5 25.7X -ElementAt interpreted - Linear Lookup 3557 3632 117 0.0 355703.5 1.0X -ElementAt interpreted - Hash Lookup 109 112 4 0.1 10888.3 32.6X -ElementAt codegen - Linear Lookup 3783 3789 4 0.0 378265.0 0.9X -ElementAt codegen - Hash Lookup 138 143 5 0.1 13824.6 25.7X +AMD EPYC 7763 64-Core Processor +MapLookup (size=10, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted - Linear Lookup 10 11 2 1.0 1022.4 1.0X +GetMapValue interpreted - Hash Lookup 10 12 2 1.0 992.3 1.0X +GetMapValue codegen - Linear Lookup 10 11 2 1.0 1013.1 1.0X +GetMapValue codegen - Hash Lookup 10 11 2 1.0 1001.9 1.0X +ElementAt interpreted - Linear Lookup 10 12 2 1.0 1045.1 1.0X +ElementAt interpreted - Hash Lookup 10 12 2 1.0 1041.5 1.0X +ElementAt codegen - Linear Lookup 10 11 2 1.0 994.5 1.0X +ElementAt codegen - Hash Lookup 10 12 2 1.0 996.8 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 10 12 2 1.0 1039.9 1.0X +GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1055.8 1.0X +GetMapValue codegen - Linear Lookup 10 11 2 1.0 981.2 1.1X +GetMapValue codegen - Hash Lookup 10 12 2 1.0 1022.1 1.0X +ElementAt interpreted - Linear Lookup 10 11 2 1.0 1040.8 1.0X +ElementAt interpreted - Hash Lookup 10 13 3 1.0 1018.8 1.0X +ElementAt codegen - Linear Lookup 10 11 2 1.0 999.3 1.0X +ElementAt codegen - Hash Lookup 10 11 2 1.0 1022.8 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 10 11 2 1.0 964.3 1.0X +GetMapValue interpreted - Hash Lookup 10 11 2 1.0 974.7 1.0X +GetMapValue codegen - Linear Lookup 10 11 2 1.0 984.7 1.0X +GetMapValue codegen - Hash Lookup 10 11 2 1.0 981.5 1.0X +ElementAt interpreted - Linear Lookup 10 11 2 1.0 1015.1 0.9X +ElementAt interpreted - Hash Lookup 11 12 2 0.9 1057.9 0.9X +ElementAt codegen - Linear Lookup 10 11 2 1.0 991.9 1.0X +ElementAt codegen - Hash Lookup 10 11 2 1.0 995.4 1.0X + +OpenJDK 64-Bit Server VM 25.0.2+10-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 10 12 2 1.0 1018.2 1.0X +GetMapValue interpreted - Hash Lookup 10 11 2 1.0 962.3 1.1X +GetMapValue codegen - Linear Lookup 9 10 2 1.1 919.6 1.1X +GetMapValue codegen - Hash Lookup 9 10 2 1.1 912.6 1.1X +ElementAt interpreted - Linear Lookup 9 11 2 1.1 945.9 1.1X +ElementAt interpreted - Hash Lookup 9 11 2 1.1 944.0 1.1X +ElementAt codegen - Linear Lookup 9 11 2 1.1 923.1 1.1X +ElementAt codegen - Hash Lookup 9 11 2 1.1 931.4 1.1X From 2e7bf82a8816be4fd144e582cb9560d414eaeb10 Mon Sep 17 00:00:00 2001 From: LuciferYang Date: Fri, 13 Mar 2026 01:50:23 +0000 Subject: [PATCH 22/24] Benchmark results for org.apache.spark.sql.execution.benchmark.MapLookupBenchmark (JDK 21, Scala 2.13, split 1 of 1) --- .../MapLookupBenchmark-jdk21-results.txt | 387 ++++++++++-------- 1 file changed, 213 insertions(+), 174 deletions(-) diff --git a/sql/core/benchmarks/MapLookupBenchmark-jdk21-results.txt b/sql/core/benchmarks/MapLookupBenchmark-jdk21-results.txt index 4f4400db5cacd..aacce18afd07a 100644 --- a/sql/core/benchmarks/MapLookupBenchmark-jdk21-results.txt +++ b/sql/core/benchmarks/MapLookupBenchmark-jdk21-results.txt @@ -1,234 +1,273 @@ OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure AMD EPYC 7763 64-Core Processor -MapLookup (size=1, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted - Linear Lookup 23 34 7 0.4 2294.8 1.0X -GetMapValue interpreted - Hash Lookup 18 24 5 0.5 1845.4 1.2X -GetMapValue codegen - Linear Lookup 17 21 4 0.6 1652.2 1.4X -GetMapValue codegen - Hash Lookup 16 20 4 0.6 1618.6 1.4X -ElementAt interpreted - Linear Lookup 16 20 4 0.6 1562.5 1.5X -ElementAt interpreted - Hash Lookup 15 17 4 0.7 1463.5 1.6X -ElementAt codegen - Linear Lookup 15 17 3 0.7 1467.1 1.6X -ElementAt codegen - Hash Lookup 14 16 3 0.7 1367.7 1.7X +MapLookup (size=1000000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 10026 23010 11278 0.0 1002647.5 1.0X +GetMapValue interpreted - Hash Lookup 1198 1356 141 0.0 119820.5 8.4X +GetMapValue codegen - Linear Lookup 12500 17389 4235 0.0 1249953.2 0.8X +GetMapValue codegen - Hash Lookup 1631 1861 200 0.0 163089.3 6.1X +ElementAt interpreted - Linear Lookup 25953 26404 426 0.0 2595299.3 0.4X +ElementAt interpreted - Hash Lookup 1165 1360 299 0.0 116544.6 8.6X +ElementAt codegen - Linear Lookup 17482 18188 617 0.0 1748227.1 0.6X +ElementAt codegen - Hash Lookup 1869 2017 172 0.0 186850.1 5.4X OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure AMD EPYC 7763 64-Core Processor -MapLookup (size=1, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted - Linear Lookup 14 17 3 0.7 1405.2 1.0X -GetMapValue interpreted - Hash Lookup 14 16 2 0.7 1358.8 1.0X -GetMapValue codegen - Linear Lookup 13 15 3 0.8 1311.0 1.1X -GetMapValue codegen - Hash Lookup 13 15 3 0.8 1274.5 1.1X -ElementAt interpreted - Linear Lookup 12 15 3 0.8 1234.4 1.1X -ElementAt interpreted - Hash Lookup 12 14 3 0.8 1214.1 1.2X -ElementAt codegen - Linear Lookup 12 14 3 0.8 1203.0 1.2X -ElementAt codegen - Hash Lookup 12 14 3 0.9 1176.0 1.2X +MapLookup (size=1000000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 42492 42633 242 0.0 4249196.2 1.0X +GetMapValue interpreted - Hash Lookup 1464 1601 154 0.0 146399.6 29.0X +GetMapValue codegen - Linear Lookup 12001 20809 7629 0.0 1200106.8 3.5X +GetMapValue codegen - Hash Lookup 1800 1970 191 0.0 180018.1 23.6X +ElementAt interpreted - Linear Lookup 43724 44931 1073 0.0 4372412.0 1.0X +ElementAt interpreted - Hash Lookup 1171 1503 317 0.0 117088.2 36.3X +ElementAt codegen - Linear Lookup 24145 24366 360 0.0 2414490.0 1.8X +ElementAt codegen - Hash Lookup 1640 1862 197 0.0 163983.0 25.9X OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure AMD EPYC 7763 64-Core Processor -MapLookup (size=1, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted - Linear Lookup 11 14 3 0.9 1142.2 1.0X -GetMapValue interpreted - Hash Lookup 11 13 3 0.9 1132.0 1.0X -GetMapValue codegen - Linear Lookup 11 13 3 0.9 1137.8 1.0X -GetMapValue codegen - Hash Lookup 11 12 2 0.9 1133.6 1.0X -ElementAt interpreted - Linear Lookup 11 13 3 0.9 1122.7 1.0X -ElementAt interpreted - Hash Lookup 11 14 3 0.9 1122.8 1.0X -ElementAt codegen - Linear Lookup 12 14 3 0.8 1178.5 1.0X -ElementAt codegen - Hash Lookup 11 13 3 0.9 1066.7 1.1X +MapLookup (size=1000000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 57909 58999 1062 0.0 5790941.2 1.0X +GetMapValue interpreted - Hash Lookup 1113 1429 296 0.0 111346.2 52.0X +GetMapValue codegen - Linear Lookup 34713 35982 1174 0.0 3471341.5 1.7X +GetMapValue codegen - Hash Lookup 1663 1823 138 0.0 166340.3 34.8X +ElementAt interpreted - Linear Lookup 58047 58688 581 0.0 5804675.6 1.0X +ElementAt interpreted - Hash Lookup 1416 1545 114 0.0 141555.3 40.9X +ElementAt codegen - Linear Lookup 36685 36917 317 0.0 3668505.8 1.6X +ElementAt codegen - Hash Lookup 1834 1970 128 0.0 183373.1 31.6X OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure AMD EPYC 7763 64-Core Processor -MapLookup (size=10, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 11 13 2 0.9 1107.9 1.0X -GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1090.3 1.0X -GetMapValue codegen - Linear Lookup 11 13 3 0.9 1103.6 1.0X -GetMapValue codegen - Hash Lookup 11 13 3 0.9 1083.0 1.0X -ElementAt interpreted - Linear Lookup 11 13 3 0.9 1136.8 1.0X -ElementAt interpreted - Hash Lookup 11 12 2 0.9 1082.7 1.0X -ElementAt codegen - Linear Lookup 11 12 2 0.9 1080.0 1.0X -ElementAt codegen - Hash Lookup 11 12 2 0.9 1071.0 1.0X +MapLookup (size=100000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +---------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 1315 1318 4 0.0 131520.2 1.0X +GetMapValue interpreted - Hash Lookup 110 120 17 0.1 11046.6 11.9X +GetMapValue codegen - Linear Lookup 895 901 7 0.0 89512.7 1.5X +GetMapValue codegen - Hash Lookup 143 151 10 0.1 14283.0 9.2X +ElementAt interpreted - Linear Lookup 1281 1284 4 0.0 128062.2 1.0X +ElementAt interpreted - Hash Lookup 112 120 12 0.1 11174.7 11.8X +ElementAt codegen - Linear Lookup 894 896 1 0.0 89425.2 1.5X +ElementAt codegen - Hash Lookup 141 147 13 0.1 14054.9 9.4X OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure AMD EPYC 7763 64-Core Processor -MapLookup (size=10, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 11 12 2 1.0 1052.5 1.0X -GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1067.8 1.0X -GetMapValue codegen - Linear Lookup 11 13 3 0.9 1067.5 1.0X -GetMapValue codegen - Hash Lookup 10 12 2 1.0 1039.0 1.0X -ElementAt interpreted - Linear Lookup 11 11 2 0.9 1059.5 1.0X -ElementAt interpreted - Hash Lookup 11 11 2 0.9 1060.1 1.0X -ElementAt codegen - Linear Lookup 10 11 2 1.0 1049.1 1.0X -ElementAt codegen - Hash Lookup 10 12 3 1.0 1047.9 1.0X +MapLookup (size=100000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +---------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 1972 2017 40 0.0 197218.8 1.0X +GetMapValue interpreted - Hash Lookup 107 115 11 0.1 10704.7 18.4X +GetMapValue codegen - Linear Lookup 1265 1460 169 0.0 126465.9 1.6X +GetMapValue codegen - Hash Lookup 136 142 9 0.1 13647.2 14.5X +ElementAt interpreted - Linear Lookup 1696 1704 9 0.0 169598.3 1.2X +ElementAt interpreted - Hash Lookup 109 119 13 0.1 10856.4 18.2X +ElementAt codegen - Linear Lookup 1266 1272 10 0.0 126615.6 1.6X +ElementAt codegen - Hash Lookup 138 146 16 0.1 13831.2 14.3X OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure AMD EPYC 7763 64-Core Processor -MapLookup (size=10, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 10 12 2 1.0 1035.3 1.0X -GetMapValue interpreted - Hash Lookup 10 12 2 1.0 1041.9 1.0X -GetMapValue codegen - Linear Lookup 11 12 2 0.9 1084.0 1.0X -GetMapValue codegen - Hash Lookup 10 12 2 1.0 1047.4 1.0X -ElementAt interpreted - Linear Lookup 11 13 2 0.9 1066.0 1.0X -ElementAt interpreted - Hash Lookup 11 12 2 1.0 1052.2 1.0X -ElementAt codegen - Linear Lookup 11 12 2 0.9 1080.6 1.0X -ElementAt codegen - Hash Lookup 10 12 2 1.0 1046.7 1.0X +MapLookup (size=100000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +---------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 2222 2228 11 0.0 222165.8 1.0X +GetMapValue interpreted - Hash Lookup 105 116 14 0.1 10477.0 21.2X +GetMapValue codegen - Linear Lookup 1645 1660 14 0.0 164490.3 1.4X +GetMapValue codegen - Hash Lookup 131 142 16 0.1 13062.0 17.0X +ElementAt interpreted - Linear Lookup 2506 2570 55 0.0 250633.2 0.9X +ElementAt interpreted - Hash Lookup 103 110 11 0.1 10314.8 21.5X +ElementAt codegen - Linear Lookup 1683 1707 21 0.0 168336.7 1.3X +ElementAt codegen - Hash Lookup 132 143 13 0.1 13191.2 16.8X OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure AMD EPYC 7763 64-Core Processor -MapLookup (size=100, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 11 13 2 0.9 1137.0 1.0X -GetMapValue interpreted - Hash Lookup 11 13 2 0.9 1069.1 1.1X -GetMapValue codegen - Linear Lookup 11 12 2 0.9 1104.0 1.0X -GetMapValue codegen - Hash Lookup 11 12 2 0.9 1096.5 1.0X -ElementAt interpreted - Linear Lookup 11 12 2 0.9 1129.0 1.0X -ElementAt interpreted - Hash Lookup 11 12 2 0.9 1071.3 1.1X -ElementAt codegen - Linear Lookup 11 12 2 0.9 1097.3 1.0X -ElementAt codegen - Hash Lookup 10 12 2 1.0 1035.4 1.1X +MapLookup (size=10000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +--------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 136 142 3 0.1 13555.8 1.0X +GetMapValue interpreted - Hash Lookup 25 30 6 0.4 2495.6 5.4X +GetMapValue codegen - Linear Lookup 95 97 1 0.1 9488.8 1.4X +GetMapValue codegen - Hash Lookup 27 31 4 0.4 2680.3 5.1X +ElementAt interpreted - Linear Lookup 120 122 2 0.1 11953.9 1.1X +ElementAt interpreted - Hash Lookup 24 27 5 0.4 2361.7 5.7X +ElementAt codegen - Linear Lookup 95 100 8 0.1 9457.3 1.4X +ElementAt codegen - Hash Lookup 26 28 3 0.4 2577.3 5.3X OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure AMD EPYC 7763 64-Core Processor -MapLookup (size=100, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 14 16 2 0.7 1406.4 1.0X -GetMapValue interpreted - Hash Lookup 13 14 2 0.8 1297.4 1.1X -GetMapValue codegen - Linear Lookup 13 14 2 0.7 1337.7 1.1X -GetMapValue codegen - Hash Lookup 10 11 2 1.0 1002.6 1.4X -ElementAt interpreted - Linear Lookup 11 12 2 0.9 1123.6 1.3X -ElementAt interpreted - Hash Lookup 10 11 2 1.0 1024.2 1.4X -ElementAt codegen - Linear Lookup 11 12 2 0.9 1087.2 1.3X -ElementAt codegen - Hash Lookup 10 11 2 1.0 991.7 1.4X +MapLookup (size=10000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +--------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 180 185 5 0.1 17965.3 1.0X +GetMapValue interpreted - Hash Lookup 22 24 2 0.4 2250.0 8.0X +GetMapValue codegen - Linear Lookup 136 137 1 0.1 13566.0 1.3X +GetMapValue codegen - Hash Lookup 25 28 4 0.4 2521.6 7.1X +ElementAt interpreted - Linear Lookup 174 177 3 0.1 17436.6 1.0X +ElementAt interpreted - Hash Lookup 23 25 4 0.4 2262.5 7.9X +ElementAt codegen - Linear Lookup 134 137 6 0.1 13379.1 1.3X +ElementAt codegen - Hash Lookup 24 28 4 0.4 2449.1 7.3X OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure AMD EPYC 7763 64-Core Processor -MapLookup (size=100, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 11 12 2 0.9 1149.7 1.0X -GetMapValue interpreted - Hash Lookup 10 11 2 1.0 1009.4 1.1X -GetMapValue codegen - Linear Lookup 12 12 2 0.9 1153.7 1.0X -GetMapValue codegen - Hash Lookup 10 11 2 1.0 998.1 1.2X -ElementAt interpreted - Linear Lookup 11 12 2 0.9 1145.4 1.0X -ElementAt interpreted - Hash Lookup 10 11 2 1.0 1012.6 1.1X -ElementAt codegen - Linear Lookup 11 12 2 0.9 1149.1 1.0X -ElementAt codegen - Hash Lookup 10 11 2 1.0 1004.7 1.1X +MapLookup (size=10000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +--------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 233 235 1 0.0 23304.9 1.0X +GetMapValue interpreted - Hash Lookup 22 24 4 0.5 2150.9 10.8X +GetMapValue codegen - Linear Lookup 175 177 1 0.1 17518.0 1.3X +GetMapValue codegen - Hash Lookup 24 26 4 0.4 2410.4 9.7X +ElementAt interpreted - Linear Lookup 229 238 15 0.0 22886.9 1.0X +ElementAt interpreted - Hash Lookup 21 24 4 0.5 2128.1 11.0X +ElementAt codegen - Linear Lookup 175 176 1 0.1 17538.1 1.3X +ElementAt codegen - Hash Lookup 24 27 4 0.4 2398.8 9.7X OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure AMD EPYC 7763 64-Core Processor MapLookup (size=1000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 18 20 3 0.6 1800.1 1.0X -GetMapValue interpreted - Hash Lookup 12 13 2 0.9 1162.9 1.5X -GetMapValue codegen - Linear Lookup 18 20 2 0.5 1820.2 1.0X -GetMapValue codegen - Hash Lookup 11 12 2 0.9 1136.0 1.6X -ElementAt interpreted - Linear Lookup 18 20 2 0.6 1816.6 1.0X -ElementAt interpreted - Hash Lookup 11 12 2 0.9 1130.9 1.6X -ElementAt codegen - Linear Lookup 18 20 2 0.6 1805.3 1.0X -ElementAt codegen - Hash Lookup 11 12 2 0.9 1133.8 1.6X +GetMapValue interpreted - Linear Lookup 21 24 5 0.5 2069.1 1.0X +GetMapValue interpreted - Hash Lookup 14 16 3 0.7 1357.4 1.5X +GetMapValue codegen - Linear Lookup 18 20 3 0.6 1756.8 1.2X +GetMapValue codegen - Hash Lookup 14 16 4 0.7 1389.9 1.5X +ElementAt interpreted - Linear Lookup 21 22 4 0.5 2067.5 1.0X +ElementAt interpreted - Hash Lookup 14 16 3 0.7 1355.4 1.5X +ElementAt codegen - Linear Lookup 17 20 3 0.6 1709.6 1.2X +ElementAt codegen - Hash Lookup 13 16 4 0.8 1317.6 1.6X OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure AMD EPYC 7763 64-Core Processor MapLookup (size=1000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 21 23 2 0.5 2123.9 1.0X -GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1101.1 1.9X -GetMapValue codegen - Linear Lookup 21 23 2 0.5 2125.6 1.0X -GetMapValue codegen - Hash Lookup 11 12 2 0.9 1099.0 1.9X -ElementAt interpreted - Linear Lookup 21 23 2 0.5 2124.3 1.0X -ElementAt interpreted - Hash Lookup 11 12 2 0.9 1110.0 1.9X -ElementAt codegen - Linear Lookup 21 23 2 0.5 2108.9 1.0X -ElementAt codegen - Hash Lookup 11 12 2 0.9 1109.9 1.9X +GetMapValue interpreted - Linear Lookup 22 24 3 0.4 2229.0 1.0X +GetMapValue interpreted - Hash Lookup 13 15 3 0.8 1257.1 1.8X +GetMapValue codegen - Linear Lookup 18 21 4 0.5 1821.8 1.2X +GetMapValue codegen - Hash Lookup 13 16 3 0.7 1336.1 1.7X +ElementAt interpreted - Linear Lookup 23 25 3 0.4 2268.2 1.0X +ElementAt interpreted - Hash Lookup 13 15 3 0.8 1251.0 1.8X +ElementAt codegen - Linear Lookup 19 20 3 0.5 1851.5 1.2X +ElementAt codegen - Hash Lookup 12 14 2 0.8 1208.9 1.8X OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure AMD EPYC 7763 64-Core Processor MapLookup (size=1000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 24 26 2 0.4 2409.5 1.0X -GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1080.2 2.2X -GetMapValue codegen - Linear Lookup 24 26 2 0.4 2404.9 1.0X -GetMapValue codegen - Hash Lookup 11 11 2 0.9 1066.1 2.3X -ElementAt interpreted - Linear Lookup 24 26 2 0.4 2395.0 1.0X -ElementAt interpreted - Hash Lookup 11 12 2 1.0 1050.3 2.3X -ElementAt codegen - Linear Lookup 24 26 2 0.4 2403.8 1.0X -ElementAt codegen - Hash Lookup 11 11 2 0.9 1059.1 2.3X +GetMapValue interpreted - Linear Lookup 24 27 4 0.4 2388.1 1.0X +GetMapValue interpreted - Hash Lookup 12 15 3 0.8 1185.9 2.0X +GetMapValue codegen - Linear Lookup 20 22 3 0.5 1966.4 1.2X +GetMapValue codegen - Hash Lookup 12 14 2 0.8 1209.0 2.0X +ElementAt interpreted - Linear Lookup 24 27 3 0.4 2374.5 1.0X +ElementAt interpreted - Hash Lookup 12 14 2 0.8 1181.1 2.0X +ElementAt codegen - Linear Lookup 20 22 2 0.5 2027.9 1.2X +ElementAt codegen - Hash Lookup 12 14 2 0.8 1186.8 2.0X OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure AMD EPYC 7763 64-Core Processor -MapLookup (size=10000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 110 125 9 0.1 10989.7 1.0X -GetMapValue interpreted - Hash Lookup 19 20 1 0.5 1908.0 5.8X -GetMapValue codegen - Linear Lookup 125 127 2 0.1 12505.6 0.9X -GetMapValue codegen - Hash Lookup 21 23 2 0.5 2118.5 5.2X -ElementAt interpreted - Linear Lookup 112 126 10 0.1 11219.8 1.0X -ElementAt interpreted - Hash Lookup 19 21 2 0.5 1923.8 5.7X -ElementAt codegen - Linear Lookup 126 128 2 0.1 12569.5 0.9X -ElementAt codegen - Hash Lookup 21 23 3 0.5 2127.4 5.2X +MapLookup (size=100, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 12 13 2 0.8 1181.4 1.0X +GetMapValue interpreted - Hash Lookup 11 13 3 0.9 1116.8 1.1X +GetMapValue codegen - Linear Lookup 12 13 2 0.8 1181.9 1.0X +GetMapValue codegen - Hash Lookup 11 13 2 0.9 1102.2 1.1X +ElementAt interpreted - Linear Lookup 13 15 2 0.8 1330.9 0.9X +ElementAt interpreted - Hash Lookup 11 13 2 0.9 1129.0 1.0X +ElementAt codegen - Linear Lookup 11 13 2 0.9 1135.0 1.0X +ElementAt codegen - Hash Lookup 11 13 2 0.9 1122.6 1.1X OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure AMD EPYC 7763 64-Core Processor -MapLookup (size=10000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 160 163 2 0.1 16018.0 1.0X -GetMapValue interpreted - Hash Lookup 19 20 2 0.5 1878.3 8.5X -GetMapValue codegen - Linear Lookup 185 193 6 0.1 18535.5 0.9X -GetMapValue codegen - Hash Lookup 21 23 3 0.5 2091.5 7.7X -ElementAt interpreted - Linear Lookup 190 193 2 0.1 18990.0 0.8X -ElementAt interpreted - Hash Lookup 19 21 3 0.5 1877.0 8.5X -ElementAt codegen - Linear Lookup 160 176 13 0.1 16026.5 1.0X -ElementAt codegen - Hash Lookup 21 22 2 0.5 2084.6 7.7X +MapLookup (size=100, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 12 12 2 0.9 1157.6 1.0X +GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1079.0 1.1X +GetMapValue codegen - Linear Lookup 12 13 2 0.8 1192.9 1.0X +GetMapValue codegen - Hash Lookup 11 13 2 0.9 1115.4 1.0X +ElementAt interpreted - Linear Lookup 13 14 2 0.8 1274.4 0.9X +ElementAt interpreted - Hash Lookup 12 13 2 0.9 1151.2 1.0X +ElementAt codegen - Linear Lookup 11 12 2 0.9 1114.9 1.0X +ElementAt codegen - Hash Lookup 11 13 3 0.9 1063.8 1.1X OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure AMD EPYC 7763 64-Core Processor -MapLookup (size=10000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 254 256 2 0.0 25395.0 1.0X -GetMapValue interpreted - Hash Lookup 18 20 2 0.5 1827.6 13.9X -GetMapValue codegen - Linear Lookup 244 247 3 0.0 24386.9 1.0X -GetMapValue codegen - Hash Lookup 21 22 2 0.5 2056.8 12.3X -ElementAt interpreted - Linear Lookup 254 256 2 0.0 25433.3 1.0X -ElementAt interpreted - Hash Lookup 18 20 2 0.5 1833.2 13.9X -ElementAt codegen - Linear Lookup 207 231 21 0.0 20654.0 1.2X -ElementAt codegen - Hash Lookup 21 24 5 0.5 2065.5 12.3X +MapLookup (size=100, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 12 13 2 0.8 1178.5 1.0X +GetMapValue interpreted - Hash Lookup 11 13 2 0.9 1120.2 1.1X +GetMapValue codegen - Linear Lookup 11 13 2 0.9 1136.8 1.0X +GetMapValue codegen - Hash Lookup 11 12 2 0.9 1090.8 1.1X +ElementAt interpreted - Linear Lookup 12 14 2 0.8 1208.6 1.0X +ElementAt interpreted - Hash Lookup 11 12 2 0.9 1070.0 1.1X +ElementAt codegen - Linear Lookup 12 13 2 0.9 1167.2 1.0X +ElementAt codegen - Hash Lookup 11 12 2 0.9 1077.1 1.1X OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure AMD EPYC 7763 64-Core Processor -MapLookup (size=100000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ----------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 1025 1132 126 0.0 102479.2 1.0X -GetMapValue interpreted - Hash Lookup 105 112 6 0.1 10452.3 9.8X -GetMapValue codegen - Linear Lookup 1061 1161 102 0.0 106113.4 1.0X -GetMapValue codegen - Hash Lookup 136 141 4 0.1 13608.0 7.5X -ElementAt interpreted - Linear Lookup 1260 1274 9 0.0 125960.6 0.8X -ElementAt interpreted - Hash Lookup 105 116 6 0.1 10508.0 9.8X -ElementAt codegen - Linear Lookup 1055 1234 64 0.0 105538.2 1.0X -ElementAt codegen - Hash Lookup 136 144 4 0.1 13628.5 7.5X +MapLookup (size=10, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted - Linear Lookup 11 13 2 0.9 1071.9 1.0X +GetMapValue interpreted - Hash Lookup 11 13 2 0.9 1123.4 1.0X +GetMapValue codegen - Linear Lookup 11 13 2 0.9 1130.3 0.9X +GetMapValue codegen - Hash Lookup 11 12 2 0.9 1102.7 1.0X +ElementAt interpreted - Linear Lookup 11 12 2 0.9 1075.3 1.0X +ElementAt interpreted - Hash Lookup 11 11 2 0.9 1059.3 1.0X +ElementAt codegen - Linear Lookup 10 12 2 1.0 1043.1 1.0X +ElementAt codegen - Hash Lookup 10 11 2 1.0 1037.0 1.0X OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure AMD EPYC 7763 64-Core Processor -MapLookup (size=100000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ----------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 1838 1855 12 0.0 183795.5 1.0X -GetMapValue interpreted - Hash Lookup 104 110 9 0.1 10435.2 17.6X -GetMapValue codegen - Linear Lookup 1777 1793 15 0.0 177695.8 1.0X -GetMapValue codegen - Hash Lookup 134 135 3 0.1 13388.5 13.7X -ElementAt interpreted - Linear Lookup 1845 1862 8 0.0 184467.8 1.0X -ElementAt interpreted - Hash Lookup 105 110 7 0.1 10471.5 17.6X -ElementAt codegen - Linear Lookup 1516 1756 85 0.0 151566.1 1.2X -ElementAt codegen - Hash Lookup 134 140 10 0.1 13449.6 13.7X +MapLookup (size=10, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted - Linear Lookup 11 12 2 0.9 1070.1 1.0X +GetMapValue interpreted - Hash Lookup 11 13 2 0.9 1079.4 1.0X +GetMapValue codegen - Linear Lookup 10 12 2 1.0 1032.1 1.0X +GetMapValue codegen - Hash Lookup 11 13 2 0.9 1111.8 1.0X +ElementAt interpreted - Linear Lookup 11 12 2 0.9 1062.4 1.0X +ElementAt interpreted - Hash Lookup 11 13 2 0.9 1085.6 1.0X +ElementAt codegen - Linear Lookup 11 12 2 0.9 1076.0 1.0X +ElementAt codegen - Hash Lookup 10 12 2 1.0 1043.6 1.0X OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure AMD EPYC 7763 64-Core Processor -MapLookup (size=100000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ----------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 2412 2415 3 0.0 241164.8 1.0X -GetMapValue interpreted - Hash Lookup 103 109 12 0.1 10321.0 23.4X -GetMapValue codegen - Linear Lookup 2316 2333 14 0.0 231560.6 1.0X -GetMapValue codegen - Hash Lookup 134 137 3 0.1 13396.9 18.0X -ElementAt interpreted - Linear Lookup 2415 2440 22 0.0 241511.5 1.0X -ElementAt interpreted - Hash Lookup 104 110 14 0.1 10423.0 23.1X -ElementAt codegen - Linear Lookup 2321 2346 20 0.0 232082.3 1.0X -ElementAt codegen - Hash Lookup 135 138 4 0.1 13532.9 17.8X +MapLookup (size=10, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted - Linear Lookup 10 11 2 1.0 1043.5 1.0X +GetMapValue interpreted - Hash Lookup 10 12 2 1.0 1040.9 1.0X +GetMapValue codegen - Linear Lookup 10 12 2 1.0 1021.2 1.0X +GetMapValue codegen - Hash Lookup 10 12 2 1.0 1048.9 1.0X +ElementAt interpreted - Linear Lookup 11 12 2 0.9 1053.8 1.0X +ElementAt interpreted - Hash Lookup 10 12 2 1.0 1018.0 1.0X +ElementAt codegen - Linear Lookup 10 11 2 1.0 1047.0 1.0X +ElementAt codegen - Hash Lookup 10 12 2 1.0 1026.8 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 11 11 2 0.9 1060.5 1.0X +GetMapValue interpreted - Hash Lookup 11 12 2 0.9 1061.4 1.0X +GetMapValue codegen - Linear Lookup 10 11 2 1.0 1018.5 1.0X +GetMapValue codegen - Hash Lookup 10 12 2 1.0 1045.5 1.0X +ElementAt interpreted - Linear Lookup 10 11 2 1.0 1047.3 1.0X +ElementAt interpreted - Hash Lookup 11 12 2 1.0 1051.0 1.0X +ElementAt codegen - Linear Lookup 10 12 2 1.0 1029.1 1.0X +ElementAt codegen - Hash Lookup 10 11 2 1.0 1013.0 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 10 11 2 1.0 1004.8 1.0X +GetMapValue interpreted - Hash Lookup 10 11 2 1.0 997.9 1.0X +GetMapValue codegen - Linear Lookup 10 11 2 1.0 1002.3 1.0X +GetMapValue codegen - Hash Lookup 10 11 2 1.0 1028.3 1.0X +ElementAt interpreted - Linear Lookup 10 12 2 1.0 1039.1 1.0X +ElementAt interpreted - Hash Lookup 10 11 2 1.0 1008.8 1.0X +ElementAt codegen - Linear Lookup 10 11 2 1.0 984.1 1.0X +ElementAt codegen - Hash Lookup 10 11 2 1.0 982.7 1.0X + +OpenJDK 64-Bit Server VM 21.0.10+7-LTS on Linux 6.14.0-1017-azure +AMD EPYC 7763 64-Core Processor +MapLookup (size=1, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 10 10 1 1.0 984.1 1.0X +GetMapValue interpreted - Hash Lookup 10 11 2 1.0 991.0 1.0X +GetMapValue codegen - Linear Lookup 10 11 2 1.0 984.6 1.0X +GetMapValue codegen - Hash Lookup 10 11 2 1.0 985.1 1.0X +ElementAt interpreted - Linear Lookup 10 11 2 1.0 994.0 1.0X +ElementAt interpreted - Hash Lookup 10 11 2 1.0 988.4 1.0X +ElementAt codegen - Linear Lookup 10 11 2 1.0 969.3 1.0X +ElementAt codegen - Hash Lookup 10 11 2 1.0 979.9 1.0X From 756ccbc26332a4faa47a3e015d2476aef9707244 Mon Sep 17 00:00:00 2001 From: LuciferYang Date: Fri, 13 Mar 2026 02:04:07 +0000 Subject: [PATCH 23/24] Benchmark results for org.apache.spark.sql.execution.benchmark.MapLookupBenchmark (JDK 17, Scala 2.13, split 1 of 1) --- .../benchmarks/MapLookupBenchmark-results.txt | 423 ++++++++++-------- 1 file changed, 231 insertions(+), 192 deletions(-) diff --git a/sql/core/benchmarks/MapLookupBenchmark-results.txt b/sql/core/benchmarks/MapLookupBenchmark-results.txt index 42c78283e12c7..f3c71d5f943e1 100644 --- a/sql/core/benchmarks/MapLookupBenchmark-results.txt +++ b/sql/core/benchmarks/MapLookupBenchmark-results.txt @@ -1,234 +1,273 @@ OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted - Linear Lookup 29 41 6 0.3 2942.1 1.0X -GetMapValue interpreted - Hash Lookup 24 29 3 0.4 2446.8 1.2X -GetMapValue codegen - Linear Lookup 23 27 3 0.4 2337.8 1.3X -GetMapValue codegen - Hash Lookup 22 25 2 0.4 2225.3 1.3X -ElementAt interpreted - Linear Lookup 21 24 2 0.5 2104.8 1.4X -ElementAt interpreted - Hash Lookup 20 23 3 0.5 2016.2 1.5X -ElementAt codegen - Linear Lookup 20 22 2 0.5 1966.9 1.5X -ElementAt codegen - Hash Lookup 20 22 3 0.5 1970.9 1.5X +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=1000000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 22624 36868 12336 0.0 2262398.9 1.0X +GetMapValue interpreted - Hash Lookup 1693 1789 142 0.0 169332.4 13.4X +GetMapValue codegen - Linear Lookup 8798 29525 17951 0.0 879755.0 2.6X +GetMapValue codegen - Hash Lookup 2342 2482 147 0.0 234180.1 9.7X +ElementAt interpreted - Linear Lookup 7806 20753 20310 0.0 780558.3 2.9X +ElementAt interpreted - Hash Lookup 1807 1837 33 0.0 180659.2 12.5X +ElementAt codegen - Linear Lookup 7000 15056 13043 0.0 699956.1 3.2X +ElementAt codegen - Hash Lookup 2523 2619 114 0.0 252261.9 9.0X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted - Linear Lookup 19 21 2 0.5 1930.9 1.0X -GetMapValue interpreted - Hash Lookup 19 20 2 0.5 1865.4 1.0X -GetMapValue codegen - Linear Lookup 18 20 2 0.5 1828.7 1.1X -GetMapValue codegen - Hash Lookup 18 20 2 0.5 1841.3 1.0X -ElementAt interpreted - Linear Lookup 18 20 2 0.5 1825.8 1.1X -ElementAt interpreted - Hash Lookup 18 19 2 0.6 1780.0 1.1X -ElementAt codegen - Linear Lookup 18 20 2 0.6 1754.8 1.1X -ElementAt codegen - Hash Lookup 17 19 2 0.6 1683.7 1.1X +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=1000000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 11143 47997 31917 0.0 1114253.0 1.0X +GetMapValue interpreted - Hash Lookup 1528 1804 268 0.0 152839.8 7.3X +GetMapValue codegen - Linear Lookup 8654 21962 22963 0.0 865436.7 1.3X +GetMapValue codegen - Hash Lookup 2398 2532 117 0.0 239815.2 4.6X +ElementAt interpreted - Linear Lookup 66164 66399 205 0.0 6616373.1 0.2X +ElementAt interpreted - Hash Lookup 1521 1787 231 0.0 152062.6 7.3X +ElementAt codegen - Linear Lookup 8854 35996 23610 0.0 885415.4 1.3X +ElementAt codegen - Hash Lookup 2490 2536 73 0.0 248981.7 4.5X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=1, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------ -GetMapValue interpreted - Linear Lookup 16 18 2 0.6 1644.3 1.0X -GetMapValue interpreted - Hash Lookup 16 18 2 0.6 1640.6 1.0X -GetMapValue codegen - Linear Lookup 16 18 1 0.6 1611.8 1.0X -GetMapValue codegen - Hash Lookup 17 18 2 0.6 1669.6 1.0X -ElementAt interpreted - Linear Lookup 16 18 2 0.6 1624.9 1.0X -ElementAt interpreted - Hash Lookup 16 18 2 0.6 1648.5 1.0X -ElementAt codegen - Linear Lookup 16 18 2 0.6 1632.9 1.0X -ElementAt codegen - Hash Lookup 17 18 1 0.6 1667.8 1.0X +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=1000000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 88708 88898 270 0.0 8870788.8 1.0X +GetMapValue interpreted - Hash Lookup 1415 1557 223 0.0 141535.3 62.7X +GetMapValue codegen - Linear Lookup 11692 38390 27852 0.0 1169181.6 7.6X +GetMapValue codegen - Hash Lookup 2156 2306 170 0.0 215553.5 41.2X +ElementAt interpreted - Linear Lookup 89030 89097 114 0.0 8903010.9 1.0X +ElementAt interpreted - Hash Lookup 1501 1711 250 0.0 150081.7 59.1X +ElementAt codegen - Linear Lookup 67459 68732 1176 0.0 6745912.0 1.3X +ElementAt codegen - Hash Lookup 2479 2525 42 0.0 247904.1 35.8X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=10, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 16 18 1 0.6 1640.8 1.0X -GetMapValue interpreted - Hash Lookup 17 18 2 0.6 1652.6 1.0X -GetMapValue codegen - Linear Lookup 17 18 1 0.6 1655.0 1.0X -GetMapValue codegen - Hash Lookup 16 18 2 0.6 1600.1 1.0X -ElementAt interpreted - Linear Lookup 16 18 2 0.6 1605.8 1.0X -ElementAt interpreted - Hash Lookup 16 18 2 0.6 1623.1 1.0X -ElementAt codegen - Linear Lookup 16 17 1 0.6 1579.5 1.0X -ElementAt codegen - Hash Lookup 16 18 2 0.6 1620.8 1.0X +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=100000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +---------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 1997 2258 227 0.0 199662.0 1.0X +GetMapValue interpreted - Hash Lookup 129 156 15 0.1 12896.4 15.5X +GetMapValue codegen - Linear Lookup 1783 1787 4 0.0 178315.4 1.1X +GetMapValue codegen - Hash Lookup 166 173 5 0.1 16604.0 12.0X +ElementAt interpreted - Linear Lookup 1973 1976 4 0.0 197333.1 1.0X +ElementAt interpreted - Hash Lookup 131 137 8 0.1 13079.7 15.3X +ElementAt codegen - Linear Lookup 1778 1784 5 0.0 177842.6 1.1X +ElementAt codegen - Hash Lookup 168 174 7 0.1 16823.4 11.9X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=10, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 16 17 2 0.6 1561.6 1.0X -GetMapValue interpreted - Hash Lookup 16 17 1 0.6 1553.7 1.0X -GetMapValue codegen - Linear Lookup 16 17 2 0.6 1558.8 1.0X -GetMapValue codegen - Hash Lookup 16 17 2 0.6 1565.9 1.0X -ElementAt interpreted - Linear Lookup 16 17 1 0.6 1585.3 1.0X -ElementAt interpreted - Hash Lookup 17 18 1 0.6 1650.7 0.9X -ElementAt codegen - Linear Lookup 16 17 1 0.6 1566.2 1.0X -ElementAt codegen - Hash Lookup 16 17 1 0.6 1589.7 1.0X +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=100000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +---------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 2652 3109 396 0.0 265207.1 1.0X +GetMapValue interpreted - Hash Lookup 127 133 5 0.1 12725.6 20.8X +GetMapValue codegen - Linear Lookup 2202 2207 7 0.0 220180.4 1.2X +GetMapValue codegen - Hash Lookup 166 175 11 0.1 16576.3 16.0X +ElementAt interpreted - Linear Lookup 3331 3340 14 0.0 333090.9 0.8X +ElementAt interpreted - Hash Lookup 130 135 5 0.1 13035.5 20.3X +ElementAt codegen - Linear Lookup 2206 2211 5 0.0 220605.4 1.2X +ElementAt codegen - Hash Lookup 166 171 5 0.1 16619.1 16.0X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=10, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 16 17 1 0.6 1573.9 1.0X -GetMapValue interpreted - Hash Lookup 15 17 1 0.7 1537.7 1.0X -GetMapValue codegen - Linear Lookup 15 16 1 0.7 1521.7 1.0X -GetMapValue codegen - Hash Lookup 15 17 2 0.7 1527.2 1.0X -ElementAt interpreted - Linear Lookup 16 17 1 0.6 1558.9 1.0X -ElementAt interpreted - Hash Lookup 15 17 1 0.7 1536.4 1.0X -ElementAt codegen - Linear Lookup 15 17 1 0.7 1536.8 1.0X -ElementAt codegen - Hash Lookup 15 16 1 0.7 1485.8 1.1X +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=100000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +---------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 4366 4370 5 0.0 436594.1 1.0X +GetMapValue interpreted - Hash Lookup 119 127 6 0.1 11904.2 36.7X +GetMapValue codegen - Linear Lookup 2880 2886 7 0.0 288000.1 1.5X +GetMapValue codegen - Hash Lookup 153 157 4 0.1 15253.2 28.6X +ElementAt interpreted - Linear Lookup 3417 3419 2 0.0 341703.2 1.3X +ElementAt interpreted - Hash Lookup 120 128 17 0.1 11961.5 36.5X +ElementAt codegen - Linear Lookup 3431 3433 2 0.0 343087.8 1.3X +ElementAt codegen - Hash Lookup 152 159 7 0.1 15183.9 28.8X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=100, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 16 17 1 0.6 1616.8 1.0X -GetMapValue interpreted - Hash Lookup 15 17 1 0.6 1547.7 1.0X -GetMapValue codegen - Linear Lookup 16 17 1 0.6 1608.7 1.0X -GetMapValue codegen - Hash Lookup 15 16 1 0.7 1493.3 1.1X -ElementAt interpreted - Linear Lookup 16 18 2 0.6 1634.4 1.0X -ElementAt interpreted - Hash Lookup 16 18 2 0.6 1581.5 1.0X -ElementAt codegen - Linear Lookup 16 18 2 0.6 1625.2 1.0X -ElementAt codegen - Hash Lookup 15 17 1 0.7 1535.6 1.1X +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=10000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +--------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 161 162 1 0.1 16111.0 1.0X +GetMapValue interpreted - Hash Lookup 32 35 3 0.3 3233.5 5.0X +GetMapValue codegen - Linear Lookup 81 83 2 0.1 8099.3 2.0X +GetMapValue codegen - Hash Lookup 35 38 4 0.3 3530.7 4.6X +ElementAt interpreted - Linear Lookup 160 163 3 0.1 16031.2 1.0X +ElementAt interpreted - Hash Lookup 32 35 3 0.3 3233.5 5.0X +ElementAt codegen - Linear Lookup 81 84 6 0.1 8058.3 2.0X +ElementAt codegen - Hash Lookup 35 37 2 0.3 3460.9 4.7X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=100, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 17 18 1 0.6 1691.4 1.0X -GetMapValue interpreted - Hash Lookup 15 17 1 0.6 1548.6 1.1X -GetMapValue codegen - Linear Lookup 16 17 1 0.6 1626.4 1.0X -GetMapValue codegen - Hash Lookup 15 17 2 0.7 1508.0 1.1X -ElementAt interpreted - Linear Lookup 16 17 1 0.6 1630.0 1.0X -ElementAt interpreted - Hash Lookup 15 16 1 0.7 1536.8 1.1X -ElementAt codegen - Linear Lookup 17 18 1 0.6 1677.8 1.0X -ElementAt codegen - Hash Lookup 15 17 1 0.6 1544.1 1.1X +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=10000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +--------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 211 213 2 0.0 21128.4 1.0X +GetMapValue interpreted - Hash Lookup 31 32 2 0.3 3054.4 6.9X +GetMapValue codegen - Linear Lookup 102 103 1 0.1 10195.1 2.1X +GetMapValue codegen - Hash Lookup 33 35 3 0.3 3300.5 6.4X +ElementAt interpreted - Linear Lookup 211 212 1 0.0 21074.5 1.0X +ElementAt interpreted - Hash Lookup 30 32 2 0.3 3040.3 6.9X +ElementAt codegen - Linear Lookup 102 104 3 0.1 10176.9 2.1X +ElementAt codegen - Hash Lookup 33 35 3 0.3 3267.9 6.5X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=100, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 16 18 1 0.6 1635.0 1.0X -GetMapValue interpreted - Hash Lookup 15 16 1 0.7 1508.2 1.1X -GetMapValue codegen - Linear Lookup 16 17 1 0.6 1612.8 1.0X -GetMapValue codegen - Hash Lookup 15 16 2 0.7 1505.8 1.1X -ElementAt interpreted - Linear Lookup 17 18 1 0.6 1708.3 1.0X -ElementAt interpreted - Hash Lookup 16 17 1 0.6 1579.8 1.0X -ElementAt codegen - Linear Lookup 17 18 1 0.6 1681.9 1.0X -ElementAt codegen - Hash Lookup 16 17 1 0.6 1568.8 1.0X +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=10000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +--------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 262 263 0 0.0 26217.5 1.0X +GetMapValue interpreted - Hash Lookup 30 32 3 0.3 2983.8 8.8X +GetMapValue codegen - Linear Lookup 123 125 3 0.1 12330.7 2.1X +GetMapValue codegen - Hash Lookup 32 34 2 0.3 3193.2 8.2X +ElementAt interpreted - Linear Lookup 262 264 5 0.0 26206.2 1.0X +ElementAt interpreted - Hash Lookup 29 31 2 0.3 2900.4 9.0X +ElementAt codegen - Linear Lookup 123 125 2 0.1 12281.0 2.1X +ElementAt codegen - Hash Lookup 31 33 3 0.3 3146.4 8.3X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz MapLookup (size=1000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 25 27 1 0.4 2547.4 1.0X -GetMapValue interpreted - Hash Lookup 16 17 1 0.6 1630.9 1.6X -GetMapValue codegen - Linear Lookup 23 24 1 0.4 2281.8 1.1X -GetMapValue codegen - Hash Lookup 16 17 1 0.6 1633.7 1.6X -ElementAt interpreted - Linear Lookup 25 26 1 0.4 2487.0 1.0X -ElementAt interpreted - Hash Lookup 16 17 1 0.6 1621.5 1.6X -ElementAt codegen - Linear Lookup 23 24 3 0.4 2281.2 1.1X -ElementAt codegen - Hash Lookup 16 17 1 0.6 1627.6 1.6X +GetMapValue interpreted - Linear Lookup 34 35 2 0.3 3356.9 1.0X +GetMapValue interpreted - Hash Lookup 21 23 2 0.5 2093.5 1.6X +GetMapValue codegen - Linear Lookup 26 27 2 0.4 2568.5 1.3X +GetMapValue codegen - Hash Lookup 21 23 2 0.5 2088.6 1.6X +ElementAt interpreted - Linear Lookup 33 35 1 0.3 3309.3 1.0X +ElementAt interpreted - Hash Lookup 21 22 1 0.5 2094.5 1.6X +ElementAt codegen - Linear Lookup 26 27 2 0.4 2588.7 1.3X +ElementAt codegen - Hash Lookup 21 22 1 0.5 2087.0 1.6X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz MapLookup (size=1000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 27 28 1 0.4 2682.6 1.0X -GetMapValue interpreted - Hash Lookup 16 17 1 0.6 1572.5 1.7X -GetMapValue codegen - Linear Lookup 26 27 1 0.4 2560.7 1.0X -GetMapValue codegen - Hash Lookup 16 17 1 0.6 1596.6 1.7X -ElementAt interpreted - Linear Lookup 27 29 1 0.4 2722.8 1.0X -ElementAt interpreted - Hash Lookup 16 17 1 0.6 1606.6 1.7X -ElementAt codegen - Linear Lookup 26 27 1 0.4 2564.6 1.0X -ElementAt codegen - Hash Lookup 16 17 1 0.6 1626.2 1.6X +GetMapValue interpreted - Linear Lookup 39 41 2 0.3 3894.9 1.0X +GetMapValue interpreted - Hash Lookup 20 22 2 0.5 2016.7 1.9X +GetMapValue codegen - Linear Lookup 28 29 2 0.4 2753.7 1.4X +GetMapValue codegen - Hash Lookup 21 22 1 0.5 2069.5 1.9X +ElementAt interpreted - Linear Lookup 39 40 1 0.3 3890.8 1.0X +ElementAt interpreted - Hash Lookup 20 22 2 0.5 2028.5 1.9X +ElementAt codegen - Linear Lookup 28 29 2 0.4 2777.9 1.4X +ElementAt codegen - Hash Lookup 20 21 1 0.5 2038.8 1.9X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz MapLookup (size=1000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 29 31 1 0.3 2929.3 1.0X -GetMapValue interpreted - Hash Lookup 16 17 1 0.6 1585.8 1.8X -GetMapValue codegen - Linear Lookup 29 31 1 0.3 2914.8 1.0X -GetMapValue codegen - Hash Lookup 16 17 1 0.6 1600.4 1.8X -ElementAt interpreted - Linear Lookup 30 31 2 0.3 2959.2 1.0X -ElementAt interpreted - Hash Lookup 16 17 1 0.6 1591.3 1.8X -ElementAt codegen - Linear Lookup 30 32 2 0.3 2978.4 1.0X -ElementAt codegen - Hash Lookup 16 17 1 0.6 1626.4 1.8X +GetMapValue interpreted - Linear Lookup 44 45 1 0.2 4419.0 1.0X +GetMapValue interpreted - Hash Lookup 20 22 2 0.5 1987.5 2.2X +GetMapValue codegen - Linear Lookup 30 31 1 0.3 2958.1 1.5X +GetMapValue codegen - Hash Lookup 20 22 3 0.5 2019.9 2.2X +ElementAt interpreted - Linear Lookup 44 46 3 0.2 4386.5 1.0X +ElementAt interpreted - Hash Lookup 20 22 2 0.5 1969.9 2.2X +ElementAt codegen - Linear Lookup 30 32 3 0.3 3004.8 1.5X +ElementAt codegen - Hash Lookup 20 22 1 0.5 2032.5 2.2X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=10000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 144 148 7 0.1 14443.1 1.0X -GetMapValue interpreted - Hash Lookup 25 26 2 0.4 2473.5 5.8X -GetMapValue codegen - Linear Lookup 123 124 1 0.1 12287.6 1.2X -GetMapValue codegen - Hash Lookup 27 29 2 0.4 2698.3 5.4X -ElementAt interpreted - Linear Lookup 145 146 2 0.1 14491.7 1.0X -ElementAt interpreted - Hash Lookup 25 26 1 0.4 2477.4 5.8X -ElementAt codegen - Linear Lookup 122 124 2 0.1 12203.9 1.2X -ElementAt codegen - Hash Lookup 26 28 2 0.4 2639.9 5.5X +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=100, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 20 22 2 0.5 2019.1 1.0X +GetMapValue interpreted - Hash Lookup 19 21 2 0.5 1914.3 1.1X +GetMapValue codegen - Linear Lookup 20 22 2 0.5 1971.4 1.0X +GetMapValue codegen - Hash Lookup 19 21 2 0.5 1940.3 1.0X +ElementAt interpreted - Linear Lookup 20 22 2 0.5 2033.9 1.0X +ElementAt interpreted - Hash Lookup 19 20 2 0.5 1906.2 1.1X +ElementAt codegen - Linear Lookup 19 20 1 0.5 1901.1 1.1X +ElementAt codegen - Hash Lookup 19 20 2 0.5 1881.5 1.1X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=10000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 184 186 2 0.1 18378.6 1.0X -GetMapValue interpreted - Hash Lookup 24 26 2 0.4 2431.2 7.6X -GetMapValue codegen - Linear Lookup 174 175 1 0.1 17386.6 1.1X -GetMapValue codegen - Hash Lookup 26 28 2 0.4 2646.1 6.9X -ElementAt interpreted - Linear Lookup 184 187 2 0.1 18412.8 1.0X -ElementAt interpreted - Hash Lookup 24 26 3 0.4 2393.7 7.7X -ElementAt codegen - Linear Lookup 173 175 2 0.1 17266.2 1.1X -ElementAt codegen - Hash Lookup 26 27 2 0.4 2597.9 7.1X +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=100, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 20 22 2 0.5 2038.5 1.0X +GetMapValue interpreted - Hash Lookup 18 20 2 0.5 1849.4 1.1X +GetMapValue codegen - Linear Lookup 19 21 2 0.5 1920.8 1.1X +GetMapValue codegen - Hash Lookup 19 20 1 0.5 1855.1 1.1X +ElementAt interpreted - Linear Lookup 20 22 2 0.5 2040.9 1.0X +ElementAt interpreted - Hash Lookup 19 20 1 0.5 1916.7 1.1X +ElementAt codegen - Linear Lookup 19 21 2 0.5 1917.0 1.1X +ElementAt codegen - Hash Lookup 19 20 1 0.5 1875.5 1.1X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=10000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 217 221 4 0.0 21659.1 1.0X -GetMapValue interpreted - Hash Lookup 24 25 2 0.4 2399.5 9.0X -GetMapValue codegen - Linear Lookup 221 223 2 0.0 22121.7 1.0X -GetMapValue codegen - Hash Lookup 27 28 2 0.4 2699.8 8.0X -ElementAt interpreted - Linear Lookup 216 223 5 0.0 21607.6 1.0X -ElementAt interpreted - Hash Lookup 24 25 2 0.4 2365.1 9.2X -ElementAt codegen - Linear Lookup 222 224 2 0.0 22179.1 1.0X -ElementAt codegen - Hash Lookup 26 28 2 0.4 2590.9 8.4X +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=100, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 21 22 1 0.5 2121.5 1.0X +GetMapValue interpreted - Hash Lookup 18 20 1 0.5 1839.0 1.2X +GetMapValue codegen - Linear Lookup 19 20 1 0.5 1923.0 1.1X +GetMapValue codegen - Hash Lookup 18 20 1 0.5 1840.3 1.2X +ElementAt interpreted - Linear Lookup 21 22 1 0.5 2086.0 1.0X +ElementAt interpreted - Hash Lookup 19 20 1 0.5 1871.2 1.1X +ElementAt codegen - Linear Lookup 19 21 1 0.5 1913.4 1.1X +ElementAt codegen - Hash Lookup 18 19 1 0.5 1840.8 1.2X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=100000, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ----------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 1333 1347 8 0.0 133287.8 1.0X -GetMapValue interpreted - Hash Lookup 112 119 4 0.1 11177.5 11.9X -GetMapValue codegen - Linear Lookup 1114 1122 6 0.0 111444.7 1.2X -GetMapValue codegen - Hash Lookup 142 146 4 0.1 14173.6 9.4X -ElementAt interpreted - Linear Lookup 1341 1351 8 0.0 134058.6 1.0X -ElementAt interpreted - Hash Lookup 113 119 5 0.1 11292.4 11.8X -ElementAt codegen - Linear Lookup 1114 1122 7 0.0 111373.4 1.2X -ElementAt codegen - Hash Lookup 143 152 5 0.1 14268.4 9.3X +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=10, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted - Linear Lookup 19 20 1 0.5 1875.0 1.0X +GetMapValue interpreted - Hash Lookup 19 20 2 0.5 1865.2 1.0X +GetMapValue codegen - Linear Lookup 18 19 1 0.5 1843.4 1.0X +GetMapValue codegen - Hash Lookup 19 21 2 0.5 1891.7 1.0X +ElementAt interpreted - Linear Lookup 19 20 1 0.5 1866.3 1.0X +ElementAt interpreted - Hash Lookup 19 20 1 0.5 1864.8 1.0X +ElementAt codegen - Linear Lookup 18 19 1 0.6 1812.8 1.0X +ElementAt codegen - Hash Lookup 18 19 1 0.5 1847.9 1.0X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=100000, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ----------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 1617 1625 7 0.0 161746.2 1.0X -GetMapValue interpreted - Hash Lookup 111 114 3 0.1 11097.3 14.6X -GetMapValue codegen - Linear Lookup 1585 1606 14 0.0 158455.5 1.0X -GetMapValue codegen - Hash Lookup 139 143 5 0.1 13850.9 11.7X -ElementAt interpreted - Linear Lookup 1623 1640 11 0.0 162297.8 1.0X -ElementAt interpreted - Hash Lookup 112 120 7 0.1 11167.1 14.5X -ElementAt codegen - Linear Lookup 1599 1609 7 0.0 159900.2 1.0X -ElementAt codegen - Hash Lookup 141 144 2 0.1 14124.9 11.5X +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=10, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted - Linear Lookup 18 19 1 0.6 1789.4 1.0X +GetMapValue interpreted - Hash Lookup 18 19 1 0.5 1818.3 1.0X +GetMapValue codegen - Linear Lookup 18 19 1 0.5 1820.4 1.0X +GetMapValue codegen - Hash Lookup 18 19 2 0.6 1812.6 1.0X +ElementAt interpreted - Linear Lookup 18 19 1 0.5 1831.7 1.0X +ElementAt interpreted - Hash Lookup 19 19 1 0.5 1858.2 1.0X +ElementAt codegen - Linear Lookup 18 19 1 0.5 1820.1 1.0X +ElementAt codegen - Hash Lookup 18 19 1 0.5 1822.6 1.0X OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure -AMD EPYC 7763 64-Core Processor -MapLookup (size=100000, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ----------------------------------------------------------------------------------------------------------------------------------- -GetMapValue interpreted - Linear Lookup 2039 2066 16 0.0 203937.0 1.0X -GetMapValue interpreted - Hash Lookup 109 111 3 0.1 10883.8 18.7X -GetMapValue codegen - Linear Lookup 2072 2087 13 0.0 207204.3 1.0X -GetMapValue codegen - Hash Lookup 141 144 4 0.1 14083.6 14.5X -ElementAt interpreted - Linear Lookup 2005 2035 21 0.0 200460.7 1.0X -ElementAt interpreted - Hash Lookup 110 116 4 0.1 11043.5 18.5X -ElementAt codegen - Linear Lookup 2091 2216 194 0.0 209069.9 1.0X -ElementAt codegen - Hash Lookup 141 148 10 0.1 14084.8 14.5X +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=10, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------ +GetMapValue interpreted - Linear Lookup 18 19 1 0.5 1834.2 1.0X +GetMapValue interpreted - Hash Lookup 18 19 1 0.5 1821.2 1.0X +GetMapValue codegen - Linear Lookup 18 19 1 0.5 1838.8 1.0X +GetMapValue codegen - Hash Lookup 18 19 1 0.6 1791.8 1.0X +ElementAt interpreted - Linear Lookup 18 19 1 0.6 1797.1 1.0X +ElementAt interpreted - Hash Lookup 18 19 1 0.6 1793.7 1.0X +ElementAt codegen - Linear Lookup 18 19 1 0.6 1803.3 1.0X +ElementAt codegen - Hash Lookup 18 19 1 0.6 1796.1 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=1, hit=1.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 18 19 1 0.6 1794.5 1.0X +GetMapValue interpreted - Hash Lookup 18 19 1 0.6 1815.4 1.0X +GetMapValue codegen - Linear Lookup 18 19 1 0.6 1791.0 1.0X +GetMapValue codegen - Hash Lookup 18 19 1 0.6 1774.3 1.0X +ElementAt interpreted - Linear Lookup 18 19 1 0.6 1810.9 1.0X +ElementAt interpreted - Hash Lookup 18 19 1 0.5 1822.9 1.0X +ElementAt codegen - Linear Lookup 18 19 1 0.6 1775.3 1.0X +ElementAt codegen - Hash Lookup 18 19 1 0.6 1769.4 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=1, hit=0.5, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 18 19 1 0.6 1793.9 1.0X +GetMapValue interpreted - Hash Lookup 18 19 1 0.6 1793.4 1.0X +GetMapValue codegen - Linear Lookup 18 19 1 0.6 1765.9 1.0X +GetMapValue codegen - Hash Lookup 18 19 1 0.6 1788.2 1.0X +ElementAt interpreted - Linear Lookup 18 19 1 0.6 1772.9 1.0X +ElementAt interpreted - Hash Lookup 18 19 1 0.6 1790.7 1.0X +ElementAt codegen - Linear Lookup 18 19 1 0.6 1778.2 1.0X +ElementAt codegen - Hash Lookup 17 19 1 0.6 1743.9 1.0X + +OpenJDK 64-Bit Server VM 17.0.18+8-LTS on Linux 6.14.0-1017-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +MapLookup (size=1, hit=0.0, type=IntegerType): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +GetMapValue interpreted - Linear Lookup 18 19 2 0.6 1798.7 1.0X +GetMapValue interpreted - Hash Lookup 18 19 1 0.6 1752.1 1.0X +GetMapValue codegen - Linear Lookup 18 18 1 0.6 1755.0 1.0X +GetMapValue codegen - Hash Lookup 17 19 2 0.6 1748.2 1.0X +ElementAt interpreted - Linear Lookup 18 19 1 0.6 1764.6 1.0X +ElementAt interpreted - Hash Lookup 18 19 1 0.6 1756.2 1.0X +ElementAt codegen - Linear Lookup 18 19 1 0.6 1755.2 1.0X +ElementAt codegen - Hash Lookup 17 19 1 0.6 1741.8 1.0X From a6eb0f3b36f9f4cf56678f2fee25acc7783732c3 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 13 Mar 2026 10:45:00 +0800 Subject: [PATCH 24/24] add ConfigBindingPolicy --- .../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index f843d4950e7bb..4a6d67a1ca026 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2508,6 +2508,7 @@ object SQLConf { "the `[]` operator. Below this threshold, linear scan is used. For key types that do not " + "support hashing (e.g. arrays, structs), linear scan is always used regardless of map size.") .version("4.2.0") + .withBindingPolicy(ConfigBindingPolicy.SESSION) .intConf .checkValue(_ >= 0, "The threshold must be non-negative.") .createWithDefault(1000)