diff --git a/api/src/test/java/org/opensearch/sql/api/transpiler/UnifiedQueryTranspilerTest.java b/api/src/test/java/org/opensearch/sql/api/transpiler/UnifiedQueryTranspilerTest.java index ebec4da7ed5..7f821165804 100644 --- a/api/src/test/java/org/opensearch/sql/api/transpiler/UnifiedQueryTranspilerTest.java +++ b/api/src/test/java/org/opensearch/sql/api/transpiler/UnifiedQueryTranspilerTest.java @@ -12,7 +12,7 @@ import org.junit.Before; import org.junit.Test; import org.opensearch.sql.api.UnifiedQueryTestBase; -import org.opensearch.sql.ppl.calcite.OpenSearchSparkSqlDialect; +import org.opensearch.sql.calcite.validate.OpenSearchSparkSqlDialect; public class UnifiedQueryTranspilerTest extends UnifiedQueryTestBase { @@ -43,11 +43,9 @@ public void testToSqlWithCustomDialect() { UnifiedQueryTranspiler customTranspiler = UnifiedQueryTranspiler.builder().dialect(OpenSearchSparkSqlDialect.DEFAULT).build(); String actualSql = customTranspiler.toSql(plan); - String expectedSql = - normalize( - "SELECT *\nFROM `catalog`.`employees`\nWHERE TRY_CAST(`name` AS DOUBLE) = 1.230E2"); + String expectedSql = normalize("SELECT *\nFROM `catalog`.`employees`\nWHERE `name` = 123"); assertEquals( - "Transpiled query using OpenSearchSparkSqlDialect should translate SAFE_CAST to TRY_CAST", + "Numeric types can be implicitly coerced to string with OpenSearchSparkSqlDialect", expectedSql, actualSql); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java index d9cc7251e14..c1ef032e15b 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY; import java.sql.Connection; +import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -17,13 +18,21 @@ import java.util.function.BiFunction; import lombok.Getter; import lombok.Setter; +import org.apache.calcite.config.NullCollation; import org.apache.calcite.rex.RexCorrelVariable; import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.server.CalciteServerStatement; +import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.RelBuilder; import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.calcite.utils.CalciteToolsHelper; +import org.opensearch.sql.calcite.validate.OpenSearchSparkSqlDialect; +import org.opensearch.sql.calcite.validate.PplTypeCoercion; +import org.opensearch.sql.calcite.validate.PplTypeCoercionRule; +import org.opensearch.sql.calcite.validate.PplValidator; +import org.opensearch.sql.calcite.validate.SqlOperatorTableProvider; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.executor.QueryType; import org.opensearch.sql.expression.function.FunctionProperties; @@ -72,6 +81,17 @@ public class CalcitePlanContext { /** Whether we're currently inside a lambda context. */ @Getter @Setter private boolean inLambdaContext = false; + /** + * -- SETTER -- Sets the SQL operator table provider. This must be called during initialization by + * the opensearch module. + * + * @param provider the provider to use for obtaining operator tables + */ + @Setter private static SqlOperatorTableProvider operatorTableProvider; + + /** Cached SqlValidator instance (lazy initialized). */ + private volatile SqlValidator validator; + private CalcitePlanContext(FrameworkConfig config, SysLimit sysLimit, QueryType queryType) { this.config = config; this.sysLimit = sysLimit; @@ -101,6 +121,52 @@ private CalcitePlanContext(CalcitePlanContext parent) { this.inLambdaContext = true; // Mark that we're inside a lambda } + /** + * Gets the SqlValidator instance (singleton per CalcitePlanContext). + * + * @return cached SqlValidator instance + */ + public SqlValidator getValidator() { + if (validator == null) { + synchronized (this) { + // Double-Checked Locking for thread-safety + if (validator == null) { + final CalciteServerStatement statement; + try { + statement = connection.createStatement().unwrap(CalciteServerStatement.class); + } catch (SQLException e) { + throw new RuntimeException(e); + } + if (operatorTableProvider == null) { + throw new IllegalStateException( + "SqlOperatorTableProvider must be set before creating CalcitePlanContext"); + } + SqlValidator.Config validatorConfig = + SqlValidator.Config.DEFAULT + .withTypeCoercionRules(PplTypeCoercionRule.instance()) + .withTypeCoercionFactory(PplTypeCoercion::create) + // Use lenient conformance for PPL compatibility + .withConformance(OpenSearchSparkSqlDialect.DEFAULT.getConformance()) + // Use Spark SQL's NULL collation (NULLs sorted LOW/FIRST) + .withDefaultNullCollation(NullCollation.LOW) + // This ensures that coerced arguments are replaced with cast version in sql + // select list because coercion is performed during select list expansion during + // sql validation. Affects 4356.yml + // See SqlValidatorImpl#validateSelectList and AggConverter#translateAgg + .withIdentifierExpansion(true); + validator = + PplValidator.create( + statement, + config, + operatorTableProvider.getOperatorTable(), + TYPE_FACTORY, + validatorConfig); + } + } + } + return validator; + } + public RexNode resolveJoinCondition( UnresolvedExpression expr, BiFunction transformFunction) { diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index f63b4b1e2e0..32c5fbde61b 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -43,9 +43,12 @@ import java.util.stream.IntStream; import java.util.stream.Stream; import lombok.AllArgsConstructor; +import lombok.NonNull; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.ViewExpanders; +import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.JoinRelType; @@ -55,10 +58,14 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexCorrelVariable; +import org.apache.calcite.rex.RexFieldCollation; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexOver; +import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexVisitorImpl; +import org.apache.calcite.rex.RexWindow; import org.apache.calcite.rex.RexWindowBounds; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.fun.SqlStdOperatorTable; @@ -1740,6 +1747,7 @@ public RelNode visitStreamWindow(StreamWindow node, CalcitePlanContext context) // Default: first get rawExpr List overExpressions = node.getWindowFunctionList().stream().map(w -> rexVisitor.analyze(w, context)).toList(); + overExpressions = embedExistingCollationsIntoOver(overExpressions, context); if (hasGroup) { // only build sequence when there is by condition @@ -1781,6 +1789,84 @@ public RelNode visitStreamWindow(StreamWindow node, CalcitePlanContext context) return context.relBuilder.peek(); } + /** + * Embed existing collation into window function's over clauses. + * + *

Window functions with frame specifications like {@code ROWS n PRECEDING} require ORDER BY to + * determine row order. Without it, results are non-deterministic. + * + *

Without this fix, the initial plan has ORDER BY separate from window functions: + * + * 

+   * LogicalProject(SUM($5) OVER (ROWS 1 PRECEDING))  ← Missing ORDER BY
+   *   LogicalSort(sort0=[$5])
+   *
+ * + *

This causes problems during validation as the order is not bound to the window. With this + * fix, sort collations are embeded into each {@code RexOver} window: + * + * 

+   * LogicalProject(SUM($5) OVER (ORDER BY $5 ROWS 1 PRECEDING))  ← ORDER BY embedded
+   *
+ * + * @param overExpressions Window function expressions (may contain nested {@link RexOver}) + * @param context Plan context for building RexNodes + * @return Expressions with ORDER BY embedded in all window specifications + */ + private List embedExistingCollationsIntoOver( + List overExpressions, CalcitePlanContext context) { + RelCollation existingCollation = context.relBuilder.peek().getTraitSet().getCollation(); + List<@NonNull RelFieldCollation> relCollations = + existingCollation == null ? List.of() : existingCollation.getFieldCollations(); + ImmutableList<@NonNull RexFieldCollation> rexCollations = + relCollations.stream() + .map(f -> relCollationToRexCollation(f, context.relBuilder)) + .collect(ImmutableList.toImmutableList()); + return overExpressions.stream() + .map( + n -> + n.accept( + new RexShuttle() { + @Override + public RexNode visitOver(RexOver over) { + RexWindow window = over.getWindow(); + return context.rexBuilder.makeOver( + over.getType(), + over.getAggOperator(), + over.getOperands(), + window.partitionKeys, + rexCollations, + window.getLowerBound(), + window.getUpperBound(), + window.isRows(), + true, + false, + over.isDistinct(), + over.ignoreNulls()); + } + })) + .toList(); + } + + private static RexFieldCollation relCollationToRexCollation( + RelFieldCollation relCollation, RelBuilder builder) { + RexNode fieldRef = builder.field(relCollation.getFieldIndex()); + + // Convert direction flags to SqlKind set + Set flags = new HashSet<>(); + if (relCollation.direction == RelFieldCollation.Direction.DESCENDING + || relCollation.direction == RelFieldCollation.Direction.STRICTLY_DESCENDING) { + flags.add(SqlKind.DESCENDING); + } + if (relCollation.nullDirection == RelFieldCollation.NullDirection.FIRST) { + flags.add(SqlKind.NULLS_FIRST); + } else if (relCollation.nullDirection == RelFieldCollation.NullDirection.LAST) { + flags.add(SqlKind.NULLS_LAST); + } + + return new RexFieldCollation(fieldRef, flags); + } + private List wrapWindowFunctionsWithGroupNotNull( List overExpressions, RexNode groupNotNull, CalcitePlanContext context) { List wrappedOverExprs = new ArrayList<>(overExpressions.size()); diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index 632b67df9c5..3f132ee08f3 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -480,26 +480,8 @@ public RexNode visitWindowFunction(WindowFunction node, CalcitePlanContext conte (arguments.isEmpty() || arguments.size() == 1) ? Collections.emptyList() : arguments.subList(1, arguments.size()); - List nodes = - PPLFuncImpTable.INSTANCE.validateAggFunctionSignature( - functionName, field, args, context.rexBuilder); - return nodes != null - ? PlanUtils.makeOver( - context, - functionName, - nodes.getFirst(), - nodes.size() <= 1 ? Collections.emptyList() : nodes.subList(1, nodes.size()), - partitions, - List.of(), - node.getWindowFrame()) - : PlanUtils.makeOver( - context, - functionName, - field, - args, - partitions, - List.of(), - node.getWindowFrame()); + return PlanUtils.makeOver( + context, functionName, field, args, partitions, List.of(), node.getWindowFrame()); }) .orElseThrow( () -> diff --git a/core/src/main/java/org/opensearch/sql/calcite/ExtendedRexBuilder.java b/core/src/main/java/org/opensearch/sql/calcite/ExtendedRexBuilder.java index 9b8ac7dfc97..409f79e56ef 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/ExtendedRexBuilder.java +++ b/core/src/main/java/org/opensearch/sql/calcite/ExtendedRexBuilder.java @@ -14,14 +14,19 @@ import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlCallBinding; import org.apache.calcite.sql.SqlIntervalQualifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; +import org.apache.calcite.sql.validate.implicit.TypeCoercionImpl; import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.calcite.type.AbstractExprRelDataType; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; @@ -146,7 +151,7 @@ public RexNode makeCast( // SqlStdOperatorTable.NOT_EQUALS, // ImmutableList.of(exp, makeZeroLiteral(sourceType))); } - } else if (OpenSearchTypeFactory.isUserDefinedType(type)) { + } else if (OpenSearchTypeUtil.isUserDefinedType(type)) { if (RexLiteral.isNullLiteral(exp)) { return super.makeCast(pos, type, exp, matchNullability, safe, format); } @@ -185,4 +190,33 @@ else if ((SqlTypeUtil.isApproximateNumeric(sourceType) || SqlTypeUtil.isDecimal( } return super.makeCast(pos, type, exp, matchNullability, safe, format); } + + /** + * Derives the return type of call to an operator. + * + *

In Calcite, coercion between STRING and NUMERIC operands takes place during converting SQL + * to RelNode. However, as we are building logical plans directly, the coercion is not yet + * implemented at this point. Hence, we duplicate {@link + * TypeCoercionImpl#binaryArithmeticWithStrings} here to infer the correct type, enabling + * operations like {@code "5" / 10}. The actual coercion will be inserted later when performing + * validation on SqlNode. + * + * @see TypeCoercionImpl#binaryArithmeticCoercion(SqlCallBinding) + * @param op the operator being called + * @param exprs actual operands + * @return derived type + */ + @Override + public RelDataType deriveReturnType(SqlOperator op, List exprs) { + if (op.getKind().belongsTo(SqlKind.BINARY_ARITHMETIC) && exprs.size() == 2) { + final RelDataType type1 = exprs.get(0).getType(); + final RelDataType type2 = exprs.get(1).getType(); + if (SqlTypeUtil.isNumeric(type1) && OpenSearchTypeUtil.isCharacter(type2)) { + return type1; + } else if (OpenSearchTypeUtil.isCharacter(type1) && SqlTypeUtil.isNumeric(type2)) { + return type2; + } + } + return super.deriveReturnType(op, exprs); + } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java b/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java index a5cdf0f45f0..34cd63187e4 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java @@ -240,7 +240,7 @@ private void registerCustomizedRules(RelOptPlanner planner) { * return {@link OpenSearchCalcitePreparingStmt} */ @Override - protected CalcitePrepareImpl.CalcitePreparingStmt getPreparingStmt( + public CalcitePrepareImpl.CalcitePreparingStmt getPreparingStmt( CalcitePrepare.Context context, Type elementType, CalciteCatalogReader catalogReader, @@ -369,7 +369,8 @@ public RelNode visit(TableScan scan) { "The 'bins' parameter on timestamp fields requires: (1) pushdown to be enabled" + " (controlled by plugins.calcite.pushdown.enabled, enabled by default), and" + " (2) the timestamp field to be used as an aggregation bucket (e.g., 'stats" - + " count() by @timestamp')."); + + " count() by @timestamp').", + e); } throw Util.throwAsRuntime(e); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java index 17d99fb4fbb..cc06fdf03ca 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java @@ -5,9 +5,6 @@ package org.opensearch.sql.calcite.utils; -import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT.EXPR_DATE; -import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT.EXPR_TIME; -import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP; import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; import static org.opensearch.sql.data.type.ExprCoreType.BINARY; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; @@ -45,12 +42,14 @@ import org.apache.calcite.sql.SqlCollation; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; +import org.checkerframework.checker.nullness.qual.Nullable; import org.opensearch.sql.calcite.type.AbstractExprRelDataType; import org.opensearch.sql.calcite.type.ExprBinaryType; import org.opensearch.sql.calcite.type.ExprDateType; import org.opensearch.sql.calcite.type.ExprIPType; import org.opensearch.sql.calcite.type.ExprTimeStampType; import org.opensearch.sql.calcite.type.ExprTimeType; +import org.opensearch.sql.calcite.validate.PplTypeCoercionRule; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.data.type.ExprCoreType; @@ -259,7 +258,7 @@ public static String getLegacyTypeName(RelDataType relDataType, QueryType queryT /** Converts a Calcite data type to OpenSearch ExprCoreType. */ public static ExprType convertRelDataTypeToExprType(RelDataType type) { - if (isUserDefinedType(type)) { + if (OpenSearchTypeUtil.isUserDefinedType(type)) { AbstractExprRelDataType udt = (AbstractExprRelDataType) type; return udt.getExprType(); } @@ -330,82 +329,66 @@ public Type getJavaClass(RelDataType type) { return super.getJavaClass(type); } - /** - * Whether a given RelDataType is a user-defined type (UDT) - * - * @param type the RelDataType to check - * @return true if the type is a user-defined type, false otherwise - */ - public static boolean isUserDefinedType(RelDataType type) { - return type instanceof AbstractExprRelDataType; - } - - /** - * Checks if the RelDataType represents a numeric type. Supports standard SQL numeric types - * (INTEGER, BIGINT, SMALLINT, TINYINT, FLOAT, DOUBLE, DECIMAL, REAL), OpenSearch UDT numeric - * types, and string types (VARCHAR, CHAR). - * - * @param fieldType the RelDataType to check - * @return true if the type is numeric or string, false otherwise - */ - public static boolean isNumericType(RelDataType fieldType) { - // Check standard SQL numeric types - SqlTypeName sqlType = fieldType.getSqlTypeName(); - if (sqlType == SqlTypeName.INTEGER - || sqlType == SqlTypeName.BIGINT - || sqlType == SqlTypeName.SMALLINT - || sqlType == SqlTypeName.TINYINT - || sqlType == SqlTypeName.FLOAT - || sqlType == SqlTypeName.DOUBLE - || sqlType == SqlTypeName.DECIMAL - || sqlType == SqlTypeName.REAL) { - return true; - } - - // Check string types (VARCHAR, CHAR) - if (sqlType == SqlTypeName.VARCHAR || sqlType == SqlTypeName.CHAR) { - return true; - } - - // Check for OpenSearch UDT numeric types - if (isUserDefinedType(fieldType)) { - AbstractExprRelDataType exprType = (AbstractExprRelDataType) fieldType; - ExprType udtType = exprType.getExprType(); - return ExprCoreType.numberTypes().contains(udtType); - } - - return false; - } - - /** - * Checks if the RelDataType represents a time-based field (timestamp, date, or time). Supports - * both standard SQL time types (including TIMESTAMP, TIMESTAMP_WITH_LOCAL_TIME_ZONE, DATE, TIME, - * and their timezone variants) and OpenSearch UDT time types. - * - * @param fieldType the RelDataType to check - * @return true if the type is time-based, false otherwise - */ - public static boolean isTimeBasedType(RelDataType fieldType) { - // Check standard SQL time types - SqlTypeName sqlType = fieldType.getSqlTypeName(); - if (sqlType == SqlTypeName.TIMESTAMP - || sqlType == SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE - || sqlType == SqlTypeName.DATE - || sqlType == SqlTypeName.TIME - || sqlType == SqlTypeName.TIME_WITH_LOCAL_TIME_ZONE) { - return true; + @Override + public @Nullable RelDataType leastRestrictive(List types) { + // Handle UDTs separately, otherwise the least restrictive type will become VARCHAR + if (types.stream().anyMatch(OpenSearchTypeUtil::isUserDefinedType)) { + int nullCount = 0; + int anyCount = 0; + int nullableCount = 0; + int dateCount = 0; + int timeCount = 0; + int ipCount = 0; + int binaryCount = 0; + int otherCount = 0; + for (RelDataType t : types) { + if (t.isNullable()) { + nullableCount++; + } + if (t.getSqlTypeName() == SqlTypeName.NULL) { + nullCount++; + } else if (t.getSqlTypeName() == SqlTypeName.ANY) { + anyCount++; + } + if (t.getSqlTypeName() == SqlTypeName.OTHER) { + otherCount++; + } + if (OpenSearchTypeUtil.isDate(t)) { + dateCount++; + } else if (OpenSearchTypeUtil.isTime(t)) { + timeCount++; + } else if (OpenSearchTypeUtil.isIp(t)) { + ipCount++; + } else if (OpenSearchTypeUtil.isBinary(t)) { + binaryCount++; + } + } + if (nullCount == 0 && anyCount == 0) { + RelDataType udt; + if (dateCount == types.size()) { + udt = createUDT(ExprUDT.EXPR_DATE, nullableCount > 0); + } else if (timeCount == types.size()) { + udt = createUDT(ExprUDT.EXPR_TIME, nullableCount > 0); + } + // There are cases where UDT IP interleaves with its intermediate SQL type for validation + // OTHER, we check otherCount to patch such cases + else if (ipCount == types.size() || otherCount == types.size()) { + udt = createUDT(ExprUDT.EXPR_IP, nullableCount > 0); + } else if (binaryCount == types.size()) { + udt = createUDT(ExprUDT.EXPR_BINARY, nullableCount > 0); + } else if (binaryCount == 0 && ipCount == 0) { + udt = createUDT(ExprUDT.EXPR_TIMESTAMP, nullableCount > 0); + } else { + udt = createSqlType(SqlTypeName.VARCHAR, nullableCount > 0); + } + return udt; + } } - - // Check for OpenSearch UDT types (EXPR_TIMESTAMP mapped to VARCHAR) - if (isUserDefinedType(fieldType)) { - AbstractExprRelDataType exprType = (AbstractExprRelDataType) fieldType; - ExprType udtType = exprType.getExprType(); - return udtType == ExprCoreType.TIMESTAMP - || udtType == ExprCoreType.DATE - || udtType == ExprCoreType.TIME; + RelDataType type = leastRestrictive(types, PplTypeCoercionRule.assignmentInstance()); + // Convert CHAR(precision) to VARCHAR so that results won't be padded + if (type != null && SqlTypeName.CHAR.equals(type.getSqlTypeName())) { + return createSqlType(SqlTypeName.VARCHAR, type.isNullable()); } - - // Fallback check if type string contains EXPR_TIMESTAMP - return fieldType.toString().contains("EXPR_TIMESTAMP"); + return type; } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeUtil.java b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeUtil.java new file mode 100644 index 00000000000..94c20ca9765 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeUtil.java @@ -0,0 +1,225 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.utils; + +import lombok.experimental.UtilityClass; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.opensearch.sql.calcite.type.AbstractExprRelDataType; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; + +/** + * Utility methods for to derive types, containing special handling logics for user-defined-types. + * + * @see SqlTypeUtil utilities used during SQL validation or type derivation. + */ +@UtilityClass +public class OpenSearchTypeUtil { + /** + * Whether a given RelDataType is a user-defined type (UDT) + * + * @param type the RelDataType to check + * @return true if the type is a user-defined type, false otherwise + */ + public static boolean isUserDefinedType(RelDataType type) { + return type instanceof AbstractExprRelDataType; + } + + /** + * Checks if the RelDataType represents a numeric type. Supports standard SQL numeric types + * (INTEGER, BIGINT, SMALLINT, TINYINT, FLOAT, DOUBLE, DECIMAL, REAL), OpenSearch UDT numeric + * types, and string types (VARCHAR, CHAR). + * + * @param fieldType the RelDataType to check + * @return true if the type is numeric or string, false otherwise + */ + public static boolean isNumericOrCharacter(RelDataType fieldType) { + // Check standard SQL numeric types & string types (VARCHAR, CHAR) + if (SqlTypeUtil.isNumeric(fieldType) || SqlTypeUtil.isCharacter(fieldType)) { + return true; + } + + // Check for OpenSearch UDT numeric types + if (isUserDefinedType(fieldType)) { + AbstractExprRelDataType exprType = (AbstractExprRelDataType) fieldType; + ExprType udtType = exprType.getExprType(); + return ExprCoreType.numberTypes().contains(udtType); + } + + return false; + } + + /** + * Checks if the RelDataType represents a time-based field (timestamp, date, or time). Supports + * both standard SQL time types (including TIMESTAMP, TIMESTAMP_WITH_LOCAL_TIME_ZONE, DATE, TIME, + * and their timezone variants) and OpenSearch UDT time types. + * + * @param fieldType the RelDataType to check + * @return true if the type is time-based, false otherwise + */ + public static boolean isDatetime(RelDataType fieldType) { + // Check standard SQL time types + if (SqlTypeUtil.isDatetime(fieldType)) { + return true; + } + + // Check for OpenSearch UDT types (EXPR_TIMESTAMP mapped to VARCHAR) + if (isUserDefinedType(fieldType)) { + AbstractExprRelDataType exprType = (AbstractExprRelDataType) fieldType; + ExprType udtType = exprType.getExprType(); + return udtType == ExprCoreType.TIMESTAMP + || udtType == ExprCoreType.DATE + || udtType == ExprCoreType.TIME; + } + + return false; + } + + /** + * Checks whether a {@link RelDataType} represents a date type. + * + *

This method returns true for both Calcite's built-in {@link SqlTypeName#DATE} type and + * OpenSearch's user-defined date type {@link OpenSearchTypeFactory.ExprUDT#EXPR_DATE}. + * + * @param type the type to check + * @return true if the type is a date type (built-in or user-defined), false otherwise + */ + public static boolean isDate(RelDataType type) { + if (isUserDefinedType(type)) { + if (((AbstractExprRelDataType) type).getUdt() == OpenSearchTypeFactory.ExprUDT.EXPR_DATE) { + return true; + } + } + return SqlTypeName.DATE.equals(type.getSqlTypeName()); + } + + /** + * Checks whether a {@link RelDataType} represents a timestamp type. + * + *

This method returns true for both Calcite's built-in {@link SqlTypeName#TIMESTAMP} type and + * OpenSearch's user-defined timestamp type {@link OpenSearchTypeFactory.ExprUDT#EXPR_TIMESTAMP}. + * + * @param type the type to check + * @return true if the type is a timestamp type (built-in or user-defined), false otherwise + */ + public static boolean isTimestamp(RelDataType type) { + if (isUserDefinedType(type)) { + if (((AbstractExprRelDataType) type).getUdt() + == OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP) { + return true; + } + } + return SqlTypeName.TIMESTAMP.equals(type.getSqlTypeName()); + } + + /** + * Checks whether a {@link RelDataType} represents a time type. + * + *

This method returns true for both Calcite's built-in {@link SqlTypeName#TIME} type and + * OpenSearch's user-defined time type {@link OpenSearchTypeFactory.ExprUDT#EXPR_TIME}. + * + * @param type the type to check + * @return true if the type is a time type (built-in or user-defined), false otherwise + */ + public static boolean isTime(RelDataType type) { + if (isUserDefinedType(type)) { + if (((AbstractExprRelDataType) type).getUdt() == OpenSearchTypeFactory.ExprUDT.EXPR_TIME) { + return true; + } + } + return SqlTypeName.TIME.equals(type.getSqlTypeName()); + } + + /** + * This method should be used in place for {@link SqlTypeUtil#isCharacter(RelDataType)} because + * user-defined types also have VARCHAR as their SqlTypeName. + */ + public static boolean isCharacter(RelDataType type) { + return !isUserDefinedType(type) && SqlTypeUtil.isCharacter(type); + } + + /** + * Checks whether a {@link RelDataType} represents an IP address type. + * + *

This method returns true only for OpenSearch's user-defined IP type {@link + * OpenSearchTypeFactory.ExprUDT#EXPR_IP}. + * + * @param type the type to check + * @return true if the type is an IP address type, false otherwise + */ + public static boolean isIp(RelDataType type) { + return isIp(type, false); + } + + /** + * Checks whether a {@link RelDataType} represents an IP address type. If {@code acceptOther} is + * set, {@link SqlTypeName#OTHER} is also accepted as an IP type. + * + *

{@link SqlTypeName#OTHER} is "borrowed" to represent IP type during validation because + * SqlTypeName.IP does not exist + * + * @param type the type to check + * @param acceptOther whether to accept OTHER as a valid IP type + * @return true if the type is an IP address type, false otherwise + */ + public static boolean isIp(RelDataType type, boolean acceptOther) { + if (isUserDefinedType(type)) { + return ((AbstractExprRelDataType) type).getUdt() == OpenSearchTypeFactory.ExprUDT.EXPR_IP; + } + if (acceptOther) { + return type.getSqlTypeName() == SqlTypeName.OTHER; + } + return false; + } + + /** + * Checks whether a {@link RelDataType} represents a binary type. + * + *

This method returns true for both Calcite's built-in binary types (BINARY, VARBINARY) and + * OpenSearch's user-defined binary type {@link OpenSearchTypeFactory.ExprUDT#EXPR_BINARY}. + * + * @param type the type to check + * @return true if the type is a binary type (built-in or user-defined), false otherwise + */ + public static boolean isBinary(RelDataType type) { + if (isUserDefinedType(type)) { + return ((AbstractExprRelDataType) type).getUdt() + == OpenSearchTypeFactory.ExprUDT.EXPR_BINARY; + } + return SqlTypeName.BINARY_TYPES.contains(type.getSqlTypeName()); + } + + /** + * Checks whether a {@link RelDataType} represents a scalar type. + * + *

Scalar types include all primitive and atomic types such as numeric types (INTEGER, BIGINT, + * FLOAT, DOUBLE, DECIMAL), string types (VARCHAR, CHAR), boolean, temporal types (DATE, TIME, + * TIMESTAMP), and special scalar types (IP, BINARY, UUID). + * + *

This method returns false for composite types including: + * + *

    + *
  • STRUCT types (structured records with named fields) + *
  • MAP types (key-value pairs) + *
  • ARRAY and MULTISET types (collections) + *
  • ROW types (tuples) + *
+ * + * @param type the type to check; may be null + * @return true if the type is a scalar type, false if it is a composite type or null + */ + public static boolean isScalar(RelDataType type) { + if (type == null) { + return false; + } + return !type.isStruct() + && !SqlTypeUtil.isMap(type) + && !SqlTypeUtil.isCollection(type) + && !SqlTypeUtil.isRow(type); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java index abf37e68392..99555982866 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java @@ -5,10 +5,17 @@ package org.opensearch.sql.calcite.utils; +import java.util.Locale; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.type.CompositeOperandTypeChecker; import org.apache.calcite.sql.type.FamilyOperandTypeChecker; import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.calcite.sql.type.SqlTypeUtil; import org.opensearch.sql.expression.function.UDFOperandMetadata; /** @@ -20,84 +27,32 @@ public class PPLOperandTypes { // This class is not meant to be instantiated. private PPLOperandTypes() {} - /** List of all scalar type signatures (single parameter each) */ - private static final java.util.List> - SCALAR_TYPES = - java.util.List.of( - // Numeric types - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.BYTE), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.SHORT), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.INTEGER), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.LONG), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.FLOAT), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.DOUBLE), - // String type - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.STRING), - // Boolean type - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.BOOLEAN), - // Temporal types - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.DATE), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.TIME), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP), - // Special scalar types - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.IP), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.BINARY)); - - /** Helper method to create scalar types with optional integer parameter */ - private static java.util.List> - createScalarWithOptionalInteger() { - java.util.List> result = - new java.util.ArrayList<>(SCALAR_TYPES); - - // Add scalar + integer combinations - SCALAR_TYPES.forEach( - scalarType -> - result.add( - java.util.List.of( - scalarType.get(0), org.opensearch.sql.data.type.ExprCoreType.INTEGER))); - - return result; - } - public static final UDFOperandMetadata NONE = UDFOperandMetadata.wrap(OperandTypes.family()); - public static final UDFOperandMetadata OPTIONAL_ANY = - UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.ANY).or(OperandTypes.family())); + public static final UDFOperandMetadata OPTIONAL_INTEGER = - UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) OperandTypes.INTEGER.or(OperandTypes.family())); - public static final UDFOperandMetadata STRING = - UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.CHARACTER); - public static final UDFOperandMetadata INTEGER = - UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.INTEGER); - public static final UDFOperandMetadata NUMERIC = - UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.NUMERIC); + UDFOperandMetadata.wrap(OperandTypes.INTEGER.or(OperandTypes.family())); + public static final UDFOperandMetadata STRING = UDFOperandMetadata.wrap(OperandTypes.CHARACTER); + public static final UDFOperandMetadata INTEGER = UDFOperandMetadata.wrap(OperandTypes.INTEGER); + public static final UDFOperandMetadata NUMERIC = UDFOperandMetadata.wrap(OperandTypes.NUMERIC); public static final UDFOperandMetadata NUMERIC_OPTIONAL_STRING = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.NUMERIC.or( - OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.CHARACTER))); + OperandTypes.NUMERIC.or( + OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.CHARACTER))); public static final UDFOperandMetadata ANY_OPTIONAL_INTEGER = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.ANY.or(OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.INTEGER))); - public static final UDFOperandMetadata ANY_OPTIONAL_TIMESTAMP = - UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.ANY.or(OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.TIMESTAMP))); + OperandTypes.ANY.or(OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.INTEGER))); public static final UDFOperandMetadata INTEGER_INTEGER = - UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.INTEGER_INTEGER); + UDFOperandMetadata.wrap(OperandTypes.INTEGER_INTEGER); public static final UDFOperandMetadata STRING_STRING = - UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.CHARACTER_CHARACTER); + UDFOperandMetadata.wrap(OperandTypes.CHARACTER_CHARACTER); public static final UDFOperandMetadata STRING_STRING_STRING = UDFOperandMetadata.wrap( OperandTypes.family( SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER)); public static final UDFOperandMetadata NUMERIC_NUMERIC = - UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.NUMERIC_NUMERIC); + UDFOperandMetadata.wrap(OperandTypes.NUMERIC_NUMERIC); public static final UDFOperandMetadata STRING_INTEGER = UDFOperandMetadata.wrap(OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER)); public static final UDFOperandMetadata STRING_STRING_INTEGER = @@ -107,9 +62,8 @@ private PPLOperandTypes() {} public static final UDFOperandMetadata STRING_OR_STRING_INTEGER = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.CHARACTER) - .or(OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER))); + OperandTypes.family(SqlTypeFamily.CHARACTER) + .or(OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER))); public static final UDFOperandMetadata STRING_STRING_INTEGER_INTEGER = UDFOperandMetadata.wrap( @@ -121,16 +75,18 @@ private PPLOperandTypes() {} public static final UDFOperandMetadata NUMERIC_STRING_OR_STRING_STRING = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - (OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.STRING)) - .or(OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING))); + (OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.STRING)) + .or(OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING))); - public static final UDFOperandMetadata NUMERIC_NUMERIC_OPTIONAL_NUMERIC = + public static final UDFOperandMetadata NUMERIC_NUMERIC_OPTIONAL_NUMERIC_SYMBOL = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.NUMERIC_NUMERIC.or( + OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC, SqlTypeFamily.ANY) + .or( OperandTypes.family( - SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC))); + SqlTypeFamily.NUMERIC, + SqlTypeFamily.NUMERIC, + SqlTypeFamily.NUMERIC, + SqlTypeFamily.ANY))); public static final UDFOperandMetadata NUMERIC_NUMERIC_NUMERIC = UDFOperandMetadata.wrap( OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); @@ -144,61 +100,60 @@ private PPLOperandTypes() {} public static final UDFOperandMetadata WIDTH_BUCKET_OPERAND = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - // 1. Numeric fields: bin age span=10 - OperandTypes.family( - SqlTypeFamily.NUMERIC, + // 1. Numeric fields: bin age span=10 + OperandTypes.family( + SqlTypeFamily.NUMERIC, + SqlTypeFamily.INTEGER, + SqlTypeFamily.NUMERIC, + SqlTypeFamily.NUMERIC) + // 2. Timestamp fields with OpenSearch type system + // Used in: Production + Integration tests (CalciteBinCommandIT) + .or( + OperandTypes.family( + SqlTypeFamily.TIMESTAMP, SqlTypeFamily.INTEGER, - SqlTypeFamily.NUMERIC, - SqlTypeFamily.NUMERIC) - // 2. Timestamp fields with OpenSearch type system - // Used in: Production + Integration tests (CalciteBinCommandIT) - .or( - OperandTypes.family( - SqlTypeFamily.TIMESTAMP, - SqlTypeFamily.INTEGER, - SqlTypeFamily.CHARACTER, // TIMESTAMP - TIMESTAMP = INTERVAL (as STRING) - SqlTypeFamily.TIMESTAMP)) - // 3. Timestamp fields with Calcite SCOTT schema - // Used in: Unit tests (CalcitePPLBinTest) - .or( - OperandTypes.family( - SqlTypeFamily.TIMESTAMP, - SqlTypeFamily.INTEGER, - SqlTypeFamily.TIMESTAMP, // TIMESTAMP - TIMESTAMP = TIMESTAMP - SqlTypeFamily.TIMESTAMP)) - // DATE field with OpenSearch type system - // Used in: Production + Integration tests (CalciteBinCommandIT) - .or( - OperandTypes.family( - SqlTypeFamily.DATE, - SqlTypeFamily.INTEGER, - SqlTypeFamily.CHARACTER, // DATE - DATE = INTERVAL (as STRING) - SqlTypeFamily.DATE)) - // DATE field with Calcite SCOTT schema - // Used in: Unit tests (CalcitePPLBinTest) - .or( - OperandTypes.family( - SqlTypeFamily.DATE, - SqlTypeFamily.INTEGER, - SqlTypeFamily.DATE, // DATE - DATE = DATE - SqlTypeFamily.DATE)) - // TIME field with OpenSearch type system - // Used in: Production + Integration tests (CalciteBinCommandIT) - .or( - OperandTypes.family( - SqlTypeFamily.TIME, - SqlTypeFamily.INTEGER, - SqlTypeFamily.CHARACTER, // TIME - TIME = INTERVAL (as STRING) - SqlTypeFamily.TIME)) - // TIME field with Calcite SCOTT schema - // Used in: Unit tests (CalcitePPLBinTest) - .or( - OperandTypes.family( - SqlTypeFamily.TIME, - SqlTypeFamily.INTEGER, - SqlTypeFamily.TIME, // TIME - TIME = TIME - SqlTypeFamily.TIME))); + SqlTypeFamily.CHARACTER, // TIMESTAMP - TIMESTAMP = INTERVAL (as STRING) + SqlTypeFamily.TIMESTAMP)) + // 3. Timestamp fields with Calcite SCOTT schema + // Used in: Unit tests (CalcitePPLBinTest) + .or( + OperandTypes.family( + SqlTypeFamily.TIMESTAMP, + SqlTypeFamily.INTEGER, + SqlTypeFamily.TIMESTAMP, // TIMESTAMP - TIMESTAMP = TIMESTAMP + SqlTypeFamily.TIMESTAMP)) + // DATE field with OpenSearch type system + // Used in: Production + Integration tests (CalciteBinCommandIT) + .or( + OperandTypes.family( + SqlTypeFamily.DATE, + SqlTypeFamily.INTEGER, + SqlTypeFamily.CHARACTER, // DATE - DATE = INTERVAL (as STRING) + SqlTypeFamily.DATE)) + // DATE field with Calcite SCOTT schema + // Used in: Unit tests (CalcitePPLBinTest) + .or( + OperandTypes.family( + SqlTypeFamily.DATE, + SqlTypeFamily.INTEGER, + SqlTypeFamily.DATE, // DATE - DATE = DATE + SqlTypeFamily.DATE)) + // TIME field with OpenSearch type system + // Used in: Production + Integration tests (CalciteBinCommandIT) + .or( + OperandTypes.family( + SqlTypeFamily.TIME, + SqlTypeFamily.INTEGER, + SqlTypeFamily.CHARACTER, // TIME - TIME = INTERVAL (as STRING) + SqlTypeFamily.TIME)) + // TIME field with Calcite SCOTT schema + // Used in: Unit tests (CalcitePPLBinTest) + .or( + OperandTypes.family( + SqlTypeFamily.TIME, + SqlTypeFamily.INTEGER, + SqlTypeFamily.TIME, // TIME - TIME = TIME + SqlTypeFamily.TIME))); public static final UDFOperandMetadata NUMERIC_NUMERIC_NUMERIC_NUMERIC_NUMERIC = UDFOperandMetadata.wrap( @@ -210,115 +165,130 @@ private PPLOperandTypes() {} SqlTypeFamily.NUMERIC)); public static final UDFOperandMetadata STRING_OR_INTEGER_INTEGER_INTEGER = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family( - SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER) - .or( - OperandTypes.family( - SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER))); + OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER) + .or( + OperandTypes.family( + SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER))); public static final UDFOperandMetadata OPTIONAL_DATE_OR_TIMESTAMP_OR_NUMERIC = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.DATETIME.or(OperandTypes.NUMERIC).or(OperandTypes.family())); + OperandTypes.DATETIME.or(OperandTypes.NUMERIC).or(OperandTypes.family())); public static final UDFOperandMetadata DATETIME_OR_STRING = - UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) OperandTypes.DATETIME.or(OperandTypes.CHARACTER)); + UDFOperandMetadata.wrap(OperandTypes.DATETIME.or(OperandTypes.CHARACTER)); public static final UDFOperandMetadata TIME_OR_TIMESTAMP_OR_STRING = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.CHARACTER.or(OperandTypes.TIME).or(OperandTypes.TIMESTAMP)); + OperandTypes.CHARACTER.or(OperandTypes.TIME).or(OperandTypes.TIMESTAMP)); public static final UDFOperandMetadata DATE_OR_TIMESTAMP_OR_STRING = - UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) OperandTypes.DATE_OR_TIMESTAMP.or(OperandTypes.CHARACTER)); + UDFOperandMetadata.wrap(OperandTypes.DATE_OR_TIMESTAMP.or(OperandTypes.CHARACTER)); public static final UDFOperandMetadata DATETIME_OR_STRING_OR_INTEGER = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.DATETIME.or(OperandTypes.CHARACTER).or(OperandTypes.INTEGER)); + OperandTypes.DATETIME.or(OperandTypes.CHARACTER).or(OperandTypes.INTEGER)); public static final UDFOperandMetadata DATETIME_OPTIONAL_INTEGER = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.DATETIME.or( - OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.INTEGER))); - public static final UDFOperandMetadata ANY_DATETIME_OR_STRING = - UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.ANY) - .or(OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.DATETIME)) - .or(OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.STRING))); + OperandTypes.DATETIME.or( + OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.INTEGER))); public static final UDFOperandMetadata DATETIME_DATETIME = UDFOperandMetadata.wrap(OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.DATETIME)); public static final UDFOperandMetadata DATETIME_OR_STRING_STRING = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.CHARACTER) - .or(OperandTypes.CHARACTER_CHARACTER)); - public static final UDFOperandMetadata DATETIME_OR_STRING_DATETIME_OR_STRING = - UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.CHARACTER_CHARACTER - .or(OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.DATETIME)) - .or(OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.CHARACTER)) - .or(OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.DATETIME))); + OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.CHARACTER) + .or(OperandTypes.CHARACTER_CHARACTER)); public static final UDFOperandMetadata STRING_TIMESTAMP = UDFOperandMetadata.wrap( OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.TIMESTAMP)); public static final UDFOperandMetadata STRING_DATETIME = UDFOperandMetadata.wrap(OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.DATETIME)); public static final UDFOperandMetadata DATETIME_INTERVAL = - UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.DATETIME_INTERVAL); + UDFOperandMetadata.wrap(OperandTypes.DATETIME_INTERVAL); public static final UDFOperandMetadata TIME_TIME = UDFOperandMetadata.wrap(OperandTypes.family(SqlTypeFamily.TIME, SqlTypeFamily.TIME)); public static final UDFOperandMetadata TIMESTAMP_OR_STRING_STRING_STRING = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family( - SqlTypeFamily.TIMESTAMP, SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER) - .or( - OperandTypes.family( - SqlTypeFamily.CHARACTER, - SqlTypeFamily.CHARACTER, - SqlTypeFamily.CHARACTER))); + OperandTypes.family( + SqlTypeFamily.TIMESTAMP, SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER) + .or( + OperandTypes.family( + SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER))); public static final UDFOperandMetadata STRING_INTEGER_DATETIME_OR_STRING = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family( - SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER, SqlTypeFamily.CHARACTER) - .or( - OperandTypes.family( - SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER, SqlTypeFamily.DATETIME))); + OperandTypes.family( + SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER, SqlTypeFamily.CHARACTER) + .or( + OperandTypes.family( + SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER, SqlTypeFamily.DATETIME))); public static final UDFOperandMetadata INTERVAL_DATETIME_DATETIME = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family( - SqlTypeFamily.CHARACTER, SqlTypeFamily.DATETIME, SqlTypeFamily.DATETIME) - .or( - OperandTypes.family( - SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER, SqlTypeFamily.DATETIME)) - .or( - OperandTypes.family( - SqlTypeFamily.CHARACTER, SqlTypeFamily.DATETIME, SqlTypeFamily.CHARACTER)) - .or( - OperandTypes.family( - SqlTypeFamily.CHARACTER, - SqlTypeFamily.CHARACTER, - SqlTypeFamily.CHARACTER))); + OperandTypes.family( + SqlTypeFamily.CHARACTER, SqlTypeFamily.DATETIME, SqlTypeFamily.DATETIME) + .or( + OperandTypes.family( + SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER, SqlTypeFamily.DATETIME)) + .or( + OperandTypes.family( + SqlTypeFamily.CHARACTER, SqlTypeFamily.DATETIME, SqlTypeFamily.CHARACTER)) + .or( + OperandTypes.family( + SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER))); /** * Operand type checker that accepts any scalar type. This includes numeric types, strings, * booleans, datetime types, and special scalar types like IP and BINARY. Excludes complex types * like arrays, structs, and maps. */ - public static final UDFOperandMetadata ANY_SCALAR = UDFOperandMetadata.wrapUDT(SCALAR_TYPES); + public static final UDFOperandMetadata SCALAR = + UDFOperandMetadata.wrap( + new SqlOperandTypeChecker() { + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + if (!getOperandCountRange().isValidCount(callBinding.getOperandCount())) { + return false; + } + return OpenSearchTypeUtil.isScalar(callBinding.getOperandType(0)); + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.of(1); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return String.format(Locale.ROOT, "%s()", opName); + } + }); /** * Operand type checker that accepts any scalar type with an optional integer argument. This is * used for aggregation functions that take a field and an optional limit/size parameter. */ - public static final UDFOperandMetadata ANY_SCALAR_OPTIONAL_INTEGER = - UDFOperandMetadata.wrapUDT(createScalarWithOptionalInteger()); + public static final UDFOperandMetadata SCALAR_OPTIONAL_INTEGER = + UDFOperandMetadata.wrap( + new SqlOperandTypeChecker() { + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + if (!getOperandCountRange().isValidCount(callBinding.getOperandCount())) { + return false; + } + boolean valid = OpenSearchTypeUtil.isScalar(callBinding.getOperandType(0)); + if (callBinding.getOperandCount() == 2) { + valid = valid && SqlTypeUtil.isIntType(callBinding.getOperandType(1)); + } + return valid; + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.between(1, 2); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return String.format( + Locale.ROOT, "%s(), %s(, )", opName, opName); + } + }); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java index d89c36601ef..2418ae7dbb8 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java @@ -10,7 +10,6 @@ import static org.apache.calcite.rex.RexWindowBounds.UNBOUNDED_PRECEDING; import static org.apache.calcite.rex.RexWindowBounds.following; import static org.apache.calcite.rex.RexWindowBounds.preceding; -import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.isTimeBasedType; import com.google.common.collect.ImmutableList; import java.lang.reflect.Method; @@ -668,7 +667,7 @@ private static boolean isNotNullOnRef(RexNode rex) { agg.getGroupSet().stream() .allMatch( group -> - isTimeBasedType( + OpenSearchTypeUtil.isDatetime( agg.getInput().getRowType().getFieldList().get(group).getType())); static boolean isTimeSpan(RexNode rex) { diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/BinnableField.java b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/BinnableField.java index a4e924b631c..d7bfd42d57b 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/BinnableField.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/BinnableField.java @@ -8,7 +8,7 @@ import lombok.Getter; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; -import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.exception.SemanticCheckException; /** Represents a field that supports binning operations. */ @@ -33,8 +33,8 @@ public BinnableField(RexNode fieldExpr, RelDataType fieldType, String fieldName) this.fieldType = fieldType; this.fieldName = fieldName; - this.isTimeBased = OpenSearchTypeFactory.isTimeBasedType(fieldType); - this.isNumeric = OpenSearchTypeFactory.isNumericType(fieldType); + this.isTimeBased = OpenSearchTypeUtil.isDatetime(fieldType); + this.isNumeric = OpenSearchTypeUtil.isNumericOrCharacter(fieldType); // Reject truly unsupported types (e.g., BOOLEAN, ARRAY, MAP) if (!isNumeric && !isTimeBased) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/calcite/OpenSearchSparkSqlDialect.java b/core/src/main/java/org/opensearch/sql/calcite/validate/OpenSearchSparkSqlDialect.java similarity index 56% rename from ppl/src/main/java/org/opensearch/sql/ppl/calcite/OpenSearchSparkSqlDialect.java rename to core/src/main/java/org/opensearch/sql/calcite/validate/OpenSearchSparkSqlDialect.java index 2d044da58e6..cea8534d742 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/calcite/OpenSearchSparkSqlDialect.java +++ b/core/src/main/java/org/opensearch/sql/calcite/validate/OpenSearchSparkSqlDialect.java @@ -3,13 +3,22 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.ppl.calcite; +package org.opensearch.sql.calcite.validate; import com.google.common.collect.ImmutableMap; import java.util.Map; +import lombok.experimental.Delegate; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.SqlAlienSystemTypeNameSpec; import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlWriter; import org.apache.calcite.sql.dialect.SparkSqlDialect; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.validate.SqlConformance; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; /** * Custom Spark SQL dialect that extends Calcite's SparkSqlDialect to handle OpenSearch-specific @@ -51,6 +60,19 @@ public void unparseCall(SqlWriter writer, SqlCall call, int leftPrec, int rightP } } + @Override + public @Nullable SqlNode getCastSpec(RelDataType type) { + // ExprIPType has sql type name OTHER, which can not be handled by spark dialect + if (OpenSearchTypeUtil.isIp(type)) { + return new SqlDataTypeSpec( + // It will use SqlTypeName.OTHER by type.getSqlTypeName() as OTHER is "borrowed" to + // represent IP type (see also: PplTypeCoercionRule.java) + new SqlAlienSystemTypeNameSpec("IP", type.getSqlTypeName(), SqlParserPos.ZERO), + SqlParserPos.ZERO); + } + return super.getCastSpec(type); + } + private void unparseFunction( SqlWriter writer, SqlCall call, @@ -68,4 +90,24 @@ private void unparseFunction( } writer.endList(frame); } + + @Override + public SqlConformance getConformance() { + return new ConformanceDelegate(super.getConformance()); + } + + /** SqlConformance delegator that enables liberal mode for LEFT SEMI/ANTI JOIN support. */ + private static class ConformanceDelegate implements SqlConformance { + @Delegate private final SqlConformance delegate; + + ConformanceDelegate(SqlConformance delegate) { + this.delegate = delegate; + } + + @Override + public boolean isLiberal() { + // This allows SQL feature LEFT ANTI JOIN & LEFT SEMI JOIN + return true; + } + } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/validate/PplConvertletTable.java b/core/src/main/java/org/opensearch/sql/calcite/validate/PplConvertletTable.java new file mode 100644 index 00000000000..b86c519e802 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/validate/PplConvertletTable.java @@ -0,0 +1,75 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.validate; + +import java.util.HashMap; +import java.util.Map; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql2rel.ReflectiveConvertletTable; +import org.apache.calcite.sql2rel.SqlRexConvertlet; +import org.apache.calcite.sql2rel.StandardConvertletTable; +import org.checkerframework.checker.initialization.qual.UnderInitialization; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; + +public class PplConvertletTable extends ReflectiveConvertletTable { + public static PplConvertletTable INSTANCE = new PplConvertletTable(); + private final Map map = new HashMap<>(); + + private PplConvertletTable() { + super(); + registerOperator(SqlStdOperatorTable.EQUALS, ipConvertlet(PPLBuiltinOperators.EQUALS_IP)); + registerOperator( + SqlStdOperatorTable.NOT_EQUALS, ipConvertlet(PPLBuiltinOperators.NOT_EQUALS_IP)); + registerOperator( + SqlStdOperatorTable.GREATER_THAN, ipConvertlet(PPLBuiltinOperators.GREATER_IP)); + registerOperator( + SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, ipConvertlet(PPLBuiltinOperators.GTE_IP)); + registerOperator(SqlStdOperatorTable.LESS_THAN, ipConvertlet(PPLBuiltinOperators.LESS_IP)); + registerOperator( + SqlStdOperatorTable.LESS_THAN_OR_EQUAL, ipConvertlet(PPLBuiltinOperators.LTE_IP)); + // There is no implementation for PPLBuiltinOperators.ATAN. It needs to be replaced to + // SqlStdOperatorTable.ATAN when converted to RelNode + registerOperator( + PPLBuiltinOperators.ATAN, + (cx, call) -> { + ((SqlBasicCall) call).setOperator(SqlStdOperatorTable.ATAN); + return StandardConvertletTable.INSTANCE.convertCall(cx, call); + }); + } + + @Override + public @Nullable SqlRexConvertlet get(SqlCall call) { + SqlRexConvertlet custom = map.get(call.getOperator()); + if (custom != null) return custom; + return StandardConvertletTable.INSTANCE.get(call); + } + + /** Registers a convertlet for a given operator instance. */ + private void registerOperator( + @UnderInitialization PplConvertletTable this, SqlOperator op, SqlRexConvertlet convertlet) { + map.put(op, convertlet); + } + + private SqlRexConvertlet ipConvertlet(SqlFunction substitute) { + return (cx, call) -> { + final RexCall e = (RexCall) StandardConvertletTable.INSTANCE.convertCall(cx, call); + RelDataType type1 = e.getOperands().get(0).getType(); + RelDataType type2 = e.getOperands().get(1).getType(); + if (OpenSearchTypeUtil.isIp(type1) || OpenSearchTypeUtil.isIp(type2)) { + return StandardConvertletTable.INSTANCE.convertFunction(cx, substitute, call); + } + return e; + }; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/validate/PplTypeCoercion.java b/core/src/main/java/org/opensearch/sql/calcite/validate/PplTypeCoercion.java new file mode 100644 index 00000000000..3a5e9c76df2 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/validate/PplTypeCoercion.java @@ -0,0 +1,203 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.validate; + +import static java.util.Objects.requireNonNull; +import static org.opensearch.sql.calcite.validate.ValidationUtils.createUDTWithAttributes; + +import org.apache.calcite.adapter.java.JavaTypeFactory; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeFactoryImpl; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlDynamicParam; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.calcite.sql.type.SqlTypeMappingRule; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.apache.calcite.sql.validate.implicit.TypeCoercion; +import org.apache.calcite.sql.validate.implicit.TypeCoercionImpl; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; + +/** + * Custom type coercion implementation for PPL that extends Calcite's default type coercion with + * additional restrictions. + * + *

This class implements a blacklist approach to prevent certain implicit type conversions that + * are not allowed in PPL semantics. + */ +public class PplTypeCoercion extends TypeCoercionImpl { + + /** + * Creates a custom TypeCoercion instance for PPL. This can be used as a TypeCoercionFactory. + * + * @param typeFactory the type factory + * @param validator the SQL validator + * @return custom PplTypeCoercion instance + */ + public static TypeCoercion create(RelDataTypeFactory typeFactory, SqlValidator validator) { + return new PplTypeCoercion(typeFactory, validator); + } + + public PplTypeCoercion(RelDataTypeFactory typeFactory, SqlValidator validator) { + super(typeFactory, validator); + } + + @Override + public @Nullable RelDataType implicitCast(RelDataType in, SqlTypeFamily expected) { + RelDataType casted = super.implicitCast(in, expected); + if (casted == null) { + // String -> DATETIME is converted to String -> TIMESTAMP + if (OpenSearchTypeUtil.isCharacter(in) && expected == SqlTypeFamily.DATETIME) { + return createUDTWithAttributes(factory, in, OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP); + } + return null; + } + return switch (casted.getSqlTypeName()) { + case SqlTypeName.DATE, SqlTypeName.TIME, SqlTypeName.TIMESTAMP, SqlTypeName.BINARY -> + createUDTWithAttributes(factory, in, casted.getSqlTypeName()); + default -> casted; + }; + } + + /** + * Override super implementation to add special handling for user-defined types (UDTs). Otherwise, + * UDTs will be regarded as character types, invalidating string->datetime casts. + */ + @Override + protected boolean needToCast( + SqlValidatorScope scope, SqlNode node, RelDataType toType, SqlTypeMappingRule mappingRule) { + boolean need = super.needToCast(scope, node, toType, mappingRule); + RelDataType fromType = validator.deriveType(scope, node); + if (OpenSearchTypeUtil.isUserDefinedType(toType) && OpenSearchTypeUtil.isCharacter(fromType)) { + need = true; + } + return need; + } + + @Override + protected boolean dateTimeStringEquality( + SqlCallBinding binding, RelDataType left, RelDataType right) { + if (OpenSearchTypeUtil.isCharacter(left) && OpenSearchTypeUtil.isDatetime(right)) { + // Use user-defined types in place of inbuilt datetime types + RelDataType r = + OpenSearchTypeUtil.isUserDefinedType(right) + ? right + : ValidationUtils.createUDTWithAttributes(factory, right, right.getSqlTypeName()); + return coerceOperandType(binding.getScope(), binding.getCall(), 0, r); + } + if (OpenSearchTypeUtil.isCharacter(right) && OpenSearchTypeUtil.isDatetime(left)) { + RelDataType l = + OpenSearchTypeUtil.isUserDefinedType(left) + ? left + : ValidationUtils.createUDTWithAttributes(factory, left, left.getSqlTypeName()); + return coerceOperandType(binding.getScope(), binding.getCall(), 1, l); + } + return false; + } + + @Override + public @Nullable RelDataType commonTypeForBinaryComparison( + @Nullable RelDataType type1, @Nullable RelDataType type2) { + // Prepend following rules for datetime comparisons: + // - (date, time) -> timestamp + // - (time, timestamp) -> timestamp + // - (ip, string) -> ip + if (type1 != null && type2 != null) { + boolean anyNullable = type1.isNullable() || type2.isNullable(); + if ((SqlTypeUtil.isDate(type1) && OpenSearchTypeUtil.isTime(type2)) + || (OpenSearchTypeUtil.isTime(type1) && SqlTypeUtil.isDate(type2))) { + return factory.createTypeWithNullability( + factory.createSqlType(SqlTypeName.TIMESTAMP), anyNullable); + } + if (OpenSearchTypeUtil.isTime(type1) && SqlTypeUtil.isTimestamp(type2)) { + return factory.createTypeWithNullability(type2, anyNullable); + } + if (SqlTypeUtil.isTimestamp(type1) && OpenSearchTypeUtil.isTime(type2)) { + return factory.createTypeWithNullability(type1, anyNullable); + } + if (OpenSearchTypeUtil.isIp(type1) && OpenSearchTypeUtil.isCharacter(type2)) { + return factory.createTypeWithNullability(type1, anyNullable); + } + if (OpenSearchTypeUtil.isCharacter(type1) && OpenSearchTypeUtil.isIp(type2)) { + return factory.createTypeWithNullability(type2, anyNullable); + } + } + return super.commonTypeForBinaryComparison(type1, type2); + } + + /** + * Cast operand at index {@code index} to target type. we do this base on the fact that validate + * happens before type coercion. + */ + protected boolean coerceOperandType( + @Nullable SqlValidatorScope scope, SqlCall call, int index, RelDataType targetType) { + // Transform the JavaType to SQL type because the SqlDataTypeSpec + // does not support deriving JavaType yet. + if (RelDataTypeFactoryImpl.isJavaType(targetType)) { + targetType = ((JavaTypeFactory) factory).toSql(targetType); + } + + SqlNode operand = call.getOperandList().get(index); + if (operand instanceof SqlDynamicParam) { + // Do not support implicit type coercion for dynamic param. + return false; + } + requireNonNull(scope, "scope"); + RelDataType operandType = validator.deriveType(scope, operand); + if (coerceStringToArray(call, operand, index, operandType, targetType)) { + return true; + } + + // Check it early. + if (!needToCast(scope, operand, targetType, PplTypeCoercionRule.lenientInstance())) { + return false; + } + // Fix up nullable attr. + RelDataType targetType1 = ValidationUtils.syncAttributes(factory, operandType, targetType); + SqlNode desired = castTo(operand, targetType1); + call.setOperand(index, desired); + // SAFE_CAST always results in nullable return type. See + // SqlCastFunction#createTypeWithNullabilityFromExpr + if (SqlKind.SAFE_CAST.equals(desired.getKind())) { + targetType1 = factory.createTypeWithNullability(targetType1, true); + } + updateInferredType(desired, targetType1); + return true; + } + + private static SqlNode castTo(SqlNode node, RelDataType type) { + if (OpenSearchTypeUtil.isDatetime(type) || OpenSearchTypeUtil.isIp(type)) { + ExprType exprType = OpenSearchTypeFactory.convertRelDataTypeToExprType(type); + return switch (exprType) { + case ExprCoreType.DATE -> + PPLBuiltinOperators.DATE.createCall(node.getParserPosition(), node); + case ExprCoreType.TIMESTAMP -> + PPLBuiltinOperators.TIMESTAMP.createCall(node.getParserPosition(), node); + case ExprCoreType.TIME -> + PPLBuiltinOperators.TIME.createCall(node.getParserPosition(), node); + case ExprCoreType.IP -> PPLBuiltinOperators.IP.createCall(node.getParserPosition(), node); + default -> throw new UnsupportedOperationException("Unsupported type: " + exprType); + }; + } + // Use SAFE_CAST instead of CAST to avoid throwing errors when numbers are malformatted + return SqlLibraryOperators.SAFE_CAST.createCall( + node.getParserPosition(), + node, + SqlTypeUtil.convertTypeToSpec(type).withNullable(type.isNullable())); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/validate/PplTypeCoercionRule.java b/core/src/main/java/org/opensearch/sql/calcite/validate/PplTypeCoercionRule.java new file mode 100644 index 00000000000..5fe17dd838c --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/validate/PplTypeCoercionRule.java @@ -0,0 +1,107 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.validate; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import lombok.NonNull; +import org.apache.calcite.sql.type.SqlTypeAssignmentRule; +import org.apache.calcite.sql.type.SqlTypeCoercionRule; +import org.apache.calcite.sql.type.SqlTypeName; + +/** + * Provides PPL-specific type coercion rules that extend Calcite's default type coercion behavior. + * + *

This class defines additional type mapping rules for PPL, particularly for handling custom + * types like IP addresses and number-to-string coercion. These additional rules are merged with + * Calcite's built-in type coercion rules. + * + *

The additional mappings defined include: + * + *

    + *
  • IP can be coerced to/from string types + *
  • VARCHAR can be coerced from numeric types + *
+ * + *

Three variants of type coercion rules are provided: + * + *

    + *
  • {@link #instance()} - Standard type coercion rules + *
  • {@link #lenientInstance()} - More permissive type coercion rules + *
  • {@link #assignmentInstance()} - Rules for type assignment validation + *
+ * + * @see SqlTypeCoercionRule + * @see PplTypeCoercion + */ +public class PplTypeCoercionRule { + /** + * PPL-specific additional type mapping rules + * + *
    + *
  • IP -> IP + *
  • CHARACTER -> IP + *
  • IP -> CHARACTER + *
  • NUMBER -> VARCHAR + *
+ */ + private static final Map> additionalMapping = + Map.of( + SqlTypeName.OTHER, + ImmutableSet.of(SqlTypeName.OTHER, SqlTypeName.VARCHAR, SqlTypeName.CHAR), + SqlTypeName.VARCHAR, + ImmutableSet.builder() + .add(SqlTypeName.OTHER) + .addAll(SqlTypeName.NUMERIC_TYPES) + .build(), + SqlTypeName.CHAR, + ImmutableSet.of(SqlTypeName.OTHER)); + + private static final SqlTypeCoercionRule INSTANCE = + SqlTypeCoercionRule.instance( + mergeMapping(SqlTypeCoercionRule.instance().getTypeMapping(), additionalMapping)); + private static final SqlTypeCoercionRule LENIENT_INSTANCE = + SqlTypeCoercionRule.instance( + mergeMapping(SqlTypeCoercionRule.lenientInstance().getTypeMapping(), additionalMapping)); + private static final SqlTypeCoercionRule ASSIGNMENT_INSTANCE = + SqlTypeCoercionRule.instance( + mergeMapping(SqlTypeAssignmentRule.instance().getTypeMapping(), additionalMapping)); + + public static SqlTypeCoercionRule instance() { + return INSTANCE; + } + + /** Returns an instance that allows more lenient type coercion. */ + public static SqlTypeCoercionRule lenientInstance() { + return LENIENT_INSTANCE; + } + + /** Rules that determine whether a type is assignable from another type. */ + public static SqlTypeCoercionRule assignmentInstance() { + return ASSIGNMENT_INSTANCE; + } + + private static Map> mergeMapping( + Map> base, Map> addition) { + return Stream.concat(base.entrySet().stream(), addition.entrySet().stream()) + .collect( + Collectors.collectingAndThen( + Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue, + (b, a) -> { + Set combined = new HashSet<>(b); + combined.addAll(a); + return ImmutableSet.copyOf(combined); + }), + ImmutableMap::copyOf)); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/validate/PplValidator.java b/core/src/main/java/org/opensearch/sql/calcite/validate/PplValidator.java new file mode 100644 index 00000000000..e231f630be3 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/validate/PplValidator.java @@ -0,0 +1,205 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.validate; + +import static org.opensearch.sql.calcite.validate.ValidationUtils.createUDTWithAttributes; + +import java.util.List; +import java.util.function.Function; +import org.apache.calcite.jdbc.CalcitePrepare; +import org.apache.calcite.jdbc.CalciteSchema; +import org.apache.calcite.prepare.CalciteCatalogReader; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rel.type.RelRecordType; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.server.CalciteServerStatement; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperatorTable; +import org.apache.calcite.sql.type.ArraySqlType; +import org.apache.calcite.sql.type.MapSqlType; +import org.apache.calcite.sql.type.MultisetSqlType; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql.validate.SqlValidatorCatalogReader; +import org.apache.calcite.sql.validate.SqlValidatorImpl; +import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.apache.calcite.tools.FrameworkConfig; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.opensearch.sql.calcite.type.AbstractExprRelDataType; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; +import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; + +/** + * Custom SQL validator for PPL queries. + * + *

This validator extends Calcite's default SqlValidatorImpl to provide PPL-specific validation + * behavior. Currently, it uses the default implementation but can be extended in the future to add + * PPL-specific validation rules. + */ +public class PplValidator extends SqlValidatorImpl { + /** + * Tracks whether the current deriveType call is at the top level (true) or a recursive call + * (false). Top-level calls return user-defined types, while recursive calls return SQL types for + * internal validation. + */ + private boolean top; + + /** + * Creates a SqlValidator configured for PPL validation. + * + * @param statement Calcite server statement + * @param frameworkConfig Framework configuration + * @param operatorTable SQL operator table to use for validation + * @return configured SqlValidator instance + */ + public static PplValidator create( + CalciteServerStatement statement, + FrameworkConfig frameworkConfig, + SqlOperatorTable operatorTable, + RelDataTypeFactory typeFactory, + SqlValidator.Config validatorConfig) { + SchemaPlus defaultSchema = frameworkConfig.getDefaultSchema(); + + final CalcitePrepare.Context prepareContext = statement.createPrepareContext(); + final CalciteSchema schema = + defaultSchema != null ? CalciteSchema.from(defaultSchema) : prepareContext.getRootSchema(); + CalciteCatalogReader catalogReader = + new CalciteCatalogReader( + schema.root(), schema.path(null), typeFactory, prepareContext.config()); + return new PplValidator(operatorTable, catalogReader, typeFactory, validatorConfig); + } + + /** + * Creates a PPL validator. + * + * @param opTab Operator table containing PPL operators + * @param catalogReader Catalog reader for accessing schema information + * @param typeFactory Type factory for creating type information + * @param config Validator configuration + */ + protected PplValidator( + SqlOperatorTable opTab, + SqlValidatorCatalogReader catalogReader, + RelDataTypeFactory typeFactory, + Config config) { + super(opTab, catalogReader, typeFactory, config); + top = true; + } + + /** + * Overrides the deriveType method to map user-defined types (UDTs) to SqlTypes so that they can + * be validated + */ + @Override + public RelDataType deriveType(SqlValidatorScope scope, SqlNode expr) { + // The type has to be sql type during type derivation & validation + boolean original = top; + top = false; + RelDataType type = super.deriveType(scope, expr); + top = original; + if (top) { + return sqlTypeToUserDefinedType(type); + } + return userDefinedTypeToSqlType(type); + } + + @Override + public @Nullable RelDataType getValidatedNodeTypeIfKnown(SqlNode node) { + RelDataType type = super.getValidatedNodeTypeIfKnown(node); + return sqlTypeToUserDefinedType(type); + } + + /** + * Disable nullary call to not confuse with field reference. + * + *

It was originally designed for function calls that have no arguments and require no + * parentheses (for example "CURRENT_USER"). However, PPL does not have such use cases. Besides, + * as nullary calls are resolved before field reference, this will make field references with name + * like USER, LOCALTIME to function calls in an unwanted but subtle way. + * + * @see SqlValidatorImpl.Expander#visit(SqlIdentifier) + */ + @Override + public @Nullable SqlCall makeNullaryCall(SqlIdentifier id) { + return null; + } + + private RelDataType userDefinedTypeToSqlType(RelDataType type) { + return convertType( + type, + t -> { + if (OpenSearchTypeUtil.isUserDefinedType(t)) { + AbstractExprRelDataType exprType = (AbstractExprRelDataType) t; + ExprType udtType = exprType.getExprType(); + OpenSearchTypeFactory typeFactory = (OpenSearchTypeFactory) this.getTypeFactory(); + return switch (udtType) { + case ExprCoreType.TIMESTAMP -> + typeFactory.createSqlType(SqlTypeName.TIMESTAMP, t.isNullable()); + case ExprCoreType.TIME -> typeFactory.createSqlType(SqlTypeName.TIME, t.isNullable()); + case ExprCoreType.DATE -> typeFactory.createSqlType(SqlTypeName.DATE, t.isNullable()); + case ExprCoreType.BINARY -> + typeFactory.createSqlType(SqlTypeName.BINARY, t.isNullable()); + case ExprCoreType.IP -> UserDefinedFunctionUtils.NULLABLE_IP_UDT; + default -> t; + }; + } + return t; + }); + } + + private RelDataType sqlTypeToUserDefinedType(RelDataType type) { + return convertType( + type, + t -> { + OpenSearchTypeFactory typeFactory = (OpenSearchTypeFactory) this.getTypeFactory(); + return switch (t.getSqlTypeName()) { + case TIMESTAMP -> createUDTWithAttributes(typeFactory, t, ExprUDT.EXPR_TIMESTAMP); + case TIME -> createUDTWithAttributes(typeFactory, t, ExprUDT.EXPR_TIME); + case DATE -> createUDTWithAttributes(typeFactory, t, ExprUDT.EXPR_DATE); + case BINARY -> createUDTWithAttributes(typeFactory, t, ExprUDT.EXPR_BINARY); + default -> t; + }; + }); + } + + private RelDataType convertType(RelDataType type, Function convert) { + if (type == null) return null; + + if (type instanceof RelRecordType recordType) { + List subTypes = + recordType.getFieldList().stream().map(RelDataTypeField::getType).map(convert).toList(); + return typeFactory.createTypeWithNullability( + typeFactory.createStructType(subTypes, recordType.getFieldNames()), + recordType.isNullable()); + } + if (type instanceof ArraySqlType arrayType) { + return typeFactory.createTypeWithNullability( + typeFactory.createArrayType(convert.apply(arrayType.getComponentType()), -1), + arrayType.isNullable()); + } + if (type instanceof MapSqlType mapType) { + return typeFactory.createTypeWithNullability( + typeFactory.createMapType( + convert.apply(mapType.getKeyType()), convert.apply(mapType.getValueType())), + mapType.isNullable()); + } + if (type instanceof MultisetSqlType multisetType) { + return typeFactory.createTypeWithNullability( + typeFactory.createMultisetType(convert.apply(multisetType.getComponentType()), -1), + multisetType.isNullable()); + } + + return convert.apply(type); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/validate/SqlOperatorTableProvider.java b/core/src/main/java/org/opensearch/sql/calcite/validate/SqlOperatorTableProvider.java new file mode 100644 index 00000000000..e041c459367 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/validate/SqlOperatorTableProvider.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.validate; + +import org.apache.calcite.sql.SqlOperatorTable; + +/** + * Provider interface for obtaining SqlOperatorTable instances. + * + *

This interface breaks the circular dependency between core and opensearch modules by allowing + * the opensearch module to provide its operator table implementation to the core module through + * dependency injection. + */ +@FunctionalInterface +public interface SqlOperatorTableProvider { + /** + * Gets the SQL operator table to use for validation and query processing. + * + * @return SqlOperatorTable instance + */ + SqlOperatorTable getOperatorTable(); +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/validate/ValidationUtils.java b/core/src/main/java/org/opensearch/sql/calcite/validate/ValidationUtils.java new file mode 100644 index 00000000000..2a90af230ff --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/validate/ValidationUtils.java @@ -0,0 +1,165 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.validate; + +import java.nio.charset.Charset; +import java.util.List; +import lombok.experimental.UtilityClass; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlCollation; +import org.apache.calcite.sql.type.NonNullableAccessors; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; + +@UtilityClass +public class ValidationUtils { + /** + * Sync the nullability, collation, etc. to the target type. Copied from {@link + * org.apache.calcite.sql.validate.implicit.AbstractTypeCoercion} + */ + public static RelDataType syncAttributes( + RelDataTypeFactory factory, RelDataType fromType, RelDataType toType) { + RelDataType syncedType = toType; + if (fromType != null) { + syncedType = factory.createTypeWithNullability(syncedType, fromType.isNullable()); + if (SqlTypeUtil.inCharOrBinaryFamilies(fromType) + && SqlTypeUtil.inCharOrBinaryFamilies(toType)) { + Charset charset = fromType.getCharset(); + if (charset != null && SqlTypeUtil.inCharFamily(syncedType)) { + SqlCollation collation = NonNullableAccessors.getCollation(fromType); + syncedType = factory.createTypeWithCharsetAndCollation(syncedType, charset, collation); + } + } + } + return syncedType; + } + + /** + * Creates a user-defined type with attributes (nullability, charset, collation) copied from + * another type. + * + * @param factory the type factory used to create the UDT + * @param fromType the source type to copy attributes from (nullability, charset, collation) + * @param userDefinedType the user-defined type to create + * @return a new RelDataType representing the UDT with attributes from fromType + */ + public static RelDataType createUDTWithAttributes( + RelDataTypeFactory factory, + RelDataType fromType, + OpenSearchTypeFactory.ExprUDT userDefinedType) { + if (!(factory instanceof OpenSearchTypeFactory typeFactory)) { + throw new IllegalArgumentException("factory must be an instance of OpenSearchTypeFactory"); + } + RelDataType type = typeFactory.createUDT(userDefinedType); + return syncAttributes(typeFactory, fromType, type); + } + + /** + * Creates a user-defined type by mapping a SQL type name to the corresponding UDT, with + * attributes copied from another type. + * + * @param factory the type factory used to create the UDT + * @param fromType the source type to copy attributes from + * @param sqlTypeName the SQL type name to map to a UDT (DATE, TIME, TIMESTAMP, or BINARY) + * @return a new RelDataType representing the UDT with attributes from fromType + * @throws IllegalArgumentException if the sqlTypeName is not supported + */ + public static RelDataType createUDTWithAttributes( + RelDataTypeFactory factory, RelDataType fromType, SqlTypeName sqlTypeName) { + return switch (sqlTypeName) { + case SqlTypeName.DATE -> + createUDTWithAttributes(factory, fromType, OpenSearchTypeFactory.ExprUDT.EXPR_DATE); + case SqlTypeName.TIME -> + createUDTWithAttributes(factory, fromType, OpenSearchTypeFactory.ExprUDT.EXPR_TIME); + case SqlTypeName.TIMESTAMP -> + createUDTWithAttributes(factory, fromType, OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP); + case SqlTypeName.BINARY -> + createUDTWithAttributes(factory, fromType, OpenSearchTypeFactory.ExprUDT.EXPR_BINARY); + default -> throw new IllegalArgumentException("Unsupported type: " + sqlTypeName); + }; + } + + /** + * Special handling for nested window functions that fail validation due to a Calcite bug. + * + *

This method provides a workaround for 2 issues in Calcite v1.41 + * + *

1. where nested window functions within CASE expressions fail validation incorrectly. Only + * {@code CalcitePPLEventstatsIT#testMultipleEventstatsWithNullBucket} should be caught by this + * case. + * + *

Calcite Bug (v1.41): The {@code SqlImplementor.Result#containsOver()} method at + * SqlImplementor.java:L2145 only checks {@code SqlBasicCall} nodes for window functions, missing + * other {@code SqlCall} subclasses like {@code SqlCase}. This causes it to fail at detecting + * window functions inside CASE expressions. + * + *

Impact: When nested window functions exist (e.g., from double eventstats), Calcite's + * {@code RelToSqlConverter} doesn't create the necessary subquery boundary because {@code + * containsOver()} returns false for expressions like: + * + * 

+   * CASE WHEN ... THEN (SUM(age) OVER (...)) END
+   *
+ * + *

This results in invalid SQL with nested aggregations: + * + * 

+   * SUM(CASE WHEN ... THEN (SUM(age) OVER (...)) END) OVER (...)
+   *
+ * + * 2. Projections containing OVER as function operands are not moved down to subqueries. This + * should catch test case {@code CalciteExplainIT.noPushDownForAggOnWindow} + * + *

The check {@code needNewSubquery} at {@link + * org.apache.calcite.rel.rel2sql.SqlImplementor}#L1930 should return true for the following plan + * as it contains window functions nested inside a function call, which should be in an inner + * query if further aggregation is performed on top of it. + * + * 

+   * LogicalProject(age_str=[WIDTH_BUCKET(SAFE_CAST($8), 3, -(MAX(SAFE_CAST($8)) OVER (), MIN(SAFE_CAST($8)) OVER ()), MAX(SAFE_CAST($8)) OVER ())])
+   *   CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])
+   *
+ * + * As a result, it creates a SQL where window functions are in group-by: + * + * 
+   * GROUP BY WIDTH_BUCKET(...MAX(...) OVER ()...)
+   *
+ * + * Ideally, it should have created a SQL like the following for test case noPushDownForAggOnWindow + * + * 
+   *   SELECT COUNT(*) AS `count()`, `age_str`
+   *   FROM (
+   *     SELECT WIDTH_BUCKET(
+   *       SAFE_CAST(`age` AS STRING),
+   *       3,
+   *       (MAX(SAFE_CAST(`age` AS STRING)) OVER (...)) - (MIN(SAFE_CAST(`age` AS STRING)) OVER (...)),
+   *       MAX(SAFE_CAST(`age` AS STRING)) OVER (...)
+   *     ) AS `age_str`
+   *     FROM `OpenSearch`.`opensearch-sql_test_index_account`
+   *   ) subquery
+   *   GROUP BY `age_str`
+   *   ORDER BY 2
+   *
+ * + *

TODO: Remove this workaround when upgrading to a Calcite version that fixes the bugs. + * + * @param e the exception to check + * @return {@code true} if the exception should be tolerated as a known Calcite bug, {@code false} + * otherwise + */ + public static boolean tolerantValidationException(Exception e) { + List acceptableErrorMessages = + List.of( + "Aggregate expressions cannot be nested", + "Windowed aggregate expression is illegal in GROUP BY clause"); + return e.getMessage() != null + && acceptableErrorMessages.stream().anyMatch(e.getMessage()::contains); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/validate/converters/PplRelToSqlNodeConverter.java b/core/src/main/java/org/opensearch/sql/calcite/validate/converters/PplRelToSqlNodeConverter.java new file mode 100644 index 00000000000..11e06ce9ae0 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/validate/converters/PplRelToSqlNodeConverter.java @@ -0,0 +1,172 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.validate.converters; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.Correlate; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.rel2sql.RelToSqlConverter; +import org.apache.calcite.sql.JoinConditionType; +import org.apache.calcite.sql.JoinType; +import org.apache.calcite.sql.SqlDialect; +import org.apache.calcite.sql.SqlHint; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.parser.SqlParserPos; + +/** + * An extension of {@link RelToSqlConverter} to convert a relation algebra tree, translated from a + * PPL query, into a SQL statement. + * + *

This converter is used during the validation phase to convert RelNode back to SqlNode for + * validation and type checking using Calcite's SqlValidator. + */ +public class PplRelToSqlNodeConverter extends RelToSqlConverter { + /** + * Creates a RelToSqlConverter for PPL. + * + * @param dialect the SQL dialect to use for conversion + */ + public PplRelToSqlNodeConverter(SqlDialect dialect) { + super(dialect); + } + + /** Override Correlate visitor to pass on join type */ + @Override + public Result visit(Correlate e) { + Result result = super.visit(e); + SqlNode from = result.asSelect().getFrom(); + if (e.getJoinType() != JoinRelType.INNER && from instanceof SqlJoin join) { + JoinType joinType; + try { + joinType = JoinType.valueOf(e.getJoinType().name()); + } catch (IllegalArgumentException ignored) { + return result; + } + join.setOperand(2, joinType.symbol(POS)); + // INNER, LEFT, RIGHT, FULL, or ASOF join requires a condition + // Use ON TRUE to satisfy SQL syntax because the actual correlation condition logic is inside + // the subquery's WHERE clause + join.setOperand(4, JoinConditionType.ON.symbol(POS)); + join.setOperand(5, SqlLiteral.createBoolean(true, POS)); + } + return result; + } + + /** + * Override to convert ANTI and SEMI joins to Spark SQL's native LEFT ANTI JOIN and LEFT SEMI JOIN + * syntax, instead of using NOT EXISTS / EXISTS subqueries. + * + *

The default implementation in {@link RelToSqlConverter#visitAntiOrSemiJoin} converts + * ANTI/SEMI joins to standard SQL using NOT EXISTS / EXISTS subqueries. However, a subtle bug in + * calcite (as of Calcite 1.41) leads to incorrect results after the conversion: correlation + * variables in the subquery are generated as unqualified identifiers. + * + *

For example: + * + * 

{@code
+   * -- Base implementation generates:
+   * SELECT ... FROM table1 AS t0
+   * WHERE ... AND NOT EXISTS (
+   *   SELECT 1 FROM table2 AS t2
+   *   WHERE name = t2.name    -- 'name' is unqualified!
+   * )
+   * }
+ * + *

The unqualified {@code name} is resolved to the inner scope (t2.name) instead of the outer + * scope (t0.name), resulting in incorrect results. + * + *

The override implementation uses ANTI / SEMI join syntax: + * + * 

{@code
+   * SELECT ... FROM table1 AS t0
+   * LEFT ANTI JOIN table2 AS t2 ON t0.name = t2.name
+   * }
+ */ + @Override + protected Result visitAntiOrSemiJoin(Join e) { + final Result leftResult = visitInput(e, 0).resetAlias(); + final Result rightResult = visitInput(e, 1).resetAlias(); + final Context leftContext = leftResult.qualifiedContext(); + final Context rightContext = rightResult.qualifiedContext(); + + JoinType joinType = + e.getJoinType() == JoinRelType.ANTI ? JoinType.LEFT_ANTI_JOIN : JoinType.LEFT_SEMI_JOIN; + SqlNode sqlCondition = convertConditionToSqlNode(e.getCondition(), leftContext, rightContext); + SqlNode join = + new SqlJoin( + POS, + leftResult.asFrom(), + SqlLiteral.createBoolean(false, POS), + joinType.symbol(POS), // LEFT ANTI JOIN or LEFT SEMI JOIN + rightResult.asFrom(), + JoinConditionType.ON.symbol(POS), + sqlCondition); + + return result(join, leftResult, rightResult); + } + + @Override + public Result visit(Aggregate e) { + Result r = super.visit(e); + if (!e.getHints().isEmpty()) { + List hints = + e.getHints().stream() + .map(relHint -> (SqlNode) toSqlHint(relHint, POS)) + .collect(Collectors.toCollection(ArrayList::new)); + r.asSelect().setHints(SqlNodeList.of(POS, hints)); + } + return r; + } + + /** + * Converts a RelHint to a SqlHint. + * + *

Copied from {@link RelToSqlConverter#toSqlHint(RelHint, SqlParserPos)} (as Calcite 1.41) as + * it is private there + */ + private static SqlHint toSqlHint(RelHint hint, SqlParserPos pos) { + if (hint.kvOptions != null) { + return new SqlHint( + pos, + new SqlIdentifier(hint.hintName, pos), + SqlNodeList.of( + pos, + hint.kvOptions.entrySet().stream() + .flatMap( + e -> + Stream.of( + new SqlIdentifier(e.getKey(), pos), + SqlLiteral.createCharString(e.getValue(), pos))) + .collect(Collectors.toList())), + SqlHint.HintOptionFormat.KV_LIST); + } else if (hint.listOptions != null) { + return new SqlHint( + pos, + new SqlIdentifier(hint.hintName, pos), + SqlNodeList.of( + pos, + hint.listOptions.stream() + .map(e -> SqlLiteral.createCharString(e, pos)) + .collect(Collectors.toList())), + SqlHint.HintOptionFormat.LITERAL_LIST); + } + return new SqlHint( + pos, + new SqlIdentifier(hint.hintName, pos), + SqlNodeList.EMPTY, + SqlHint.HintOptionFormat.EMPTY); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/validate/converters/PplSqlToRelConverter.java b/core/src/main/java/org/opensearch/sql/calcite/validate/converters/PplSqlToRelConverter.java new file mode 100644 index 00000000000..29609d89fcb --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/validate/converters/PplSqlToRelConverter.java @@ -0,0 +1,92 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.validate.converters; + +import java.util.List; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.prepare.Prepare; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.logical.LogicalJoin; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.sql.JoinType; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql2rel.SqlRexConvertletTable; +import org.apache.calcite.sql2rel.SqlToRelConverter; +import org.checkerframework.checker.nullness.qual.Nullable; + +public class PplSqlToRelConverter extends SqlToRelConverter { + public PplSqlToRelConverter( + RelOptTable.ViewExpander viewExpander, + @Nullable SqlValidator validator, + Prepare.CatalogReader catalogReader, + RelOptCluster cluster, + SqlRexConvertletTable convertletTable, + Config config) { + super(viewExpander, validator, catalogReader, cluster, convertletTable, config); + } + + /** + * Override to support Spark SQL's LEFT ANTI JOIN and LEFT SEMI JOIN conversion to RelNode. + * + *

The default implementation in {@link SqlToRelConverter#convertJoinType} does not expect + * LEFT_ANTI_JOIN and LEFT_SEMI_JOIN. This override works around the limitation by first + * temporarily changing LEFT_ANTI_JOIN/LEFT_SEMI_JOIN to LEFT join in the SqlJoin node, then + * calling {@code super.convertFrom()} to perform normal conversion, finally substituting the join + * type in the resulting RelNode to ANTI/SEMI. + * + * @param bb Scope within which to resolve identifiers + * @param from FROM clause of a query. + * @param fieldNames Field aliases, usually come from AS clause, or null + */ + @Override + protected void convertFrom( + Blackboard bb, @Nullable SqlNode from, @Nullable List fieldNames) { + JoinType originalJoinType = null; + if (from instanceof SqlJoin join) { + JoinType joinType = join.getJoinType(); + if (joinType == JoinType.LEFT_SEMI_JOIN || joinType == JoinType.LEFT_ANTI_JOIN) { + join.setOperand(2, JoinType.LEFT.symbol(from.getParserPosition())); + originalJoinType = joinType; + } + } + super.convertFrom(bb, from, fieldNames); + if (originalJoinType != null) { + RelNode root = bb.root(); + if (root != null) { + JoinRelType correctJoinType = + originalJoinType == JoinType.LEFT_SEMI_JOIN ? JoinRelType.SEMI : JoinRelType.ANTI; + RelNode fixedRoot = modifyJoinType(root, correctJoinType); + bb.setRoot(fixedRoot, false); + } + } + } + + private RelNode modifyJoinType(RelNode root, JoinRelType correctJoinType) { + if (root instanceof LogicalProject project) { + RelNode input = project.getInput(); + RelNode fixedInput = modifyJoinType(input, correctJoinType); + if (fixedInput != input) { + return project.copy( + project.getTraitSet(), fixedInput, project.getProjects(), project.getRowType()); + } + } else if (root instanceof LogicalJoin join) { + if (join.getJoinType() == JoinRelType.LEFT) { + return join.copy( + join.getTraitSet(), + join.getCondition(), + join.getLeft(), + join.getRight(), + correctJoinType, + join.isSemiJoinDone()); + } + } + return root; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/validate/shuttles/PplRelToSqlRelShuttle.java b/core/src/main/java/org/opensearch/sql/calcite/validate/shuttles/PplRelToSqlRelShuttle.java new file mode 100644 index 00000000000..1f057fcd725 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/validate/shuttles/PplRelToSqlRelShuttle.java @@ -0,0 +1,83 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.validate.shuttles; + +import java.math.BigDecimal; +import java.util.List; +import org.apache.calcite.avatica.util.TimeUnit; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttleImpl; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.sql.SqlIntervalQualifier; +import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.apache.calcite.sql.type.SqlTypeName; + +/** + * A RelShuttle that recursively visits all RelNodes and their RexNode expressions to fix interval + * literals and float literal before/after SQL conversion. + * + *

This shuttle extends RelShuttleImpl to ensure it visits the entire RelNode tree recursively, + * applying the interval literal fixes at each node. + */ +public class PplRelToSqlRelShuttle extends RelShuttleImpl { + private final RexShuttle rexShuttle; + + public PplRelToSqlRelShuttle(RexBuilder rexBuilder, boolean forward) { + this.rexShuttle = + new RexShuttle() { + /** + * This visitor fixes: 1. float literal: when converting logical plan to sql node, float + * information is missing. All floats will be treated as double. A compulsory cast is + * inserted here to ensure a cast presents in the generated SQL 2. interval literal: we + * create and read the interval literal in a different way that how Calcite originally + * expected it to be. + */ + @Override + public RexNode visitLiteral(RexLiteral literal) { + // 1. Fix float literal + SqlTypeName literalType = literal.getType().getSqlTypeName(); + if (SqlTypeName.REAL.equals(literalType) || SqlTypeName.FLOAT.equals(literalType)) { + return rexBuilder.makeCall( + literal.getType(), SqlLibraryOperators.SAFE_CAST, List.of(literal)); + } + + // 2. Fix interval literal + SqlIntervalQualifier qualifier = literal.getType().getIntervalQualifier(); + if (qualifier == null) { + return literal; + } + BigDecimal value = literal.getValueAs(BigDecimal.class); + if (value == null) { + return literal; + } + TimeUnit unit = qualifier.getUnit(); + // An ad-hoc fix to a Calcite bug in RexLiteral#intervalString -- quarter type does not + // exist in SqlTypeName, rendering it return number of months instead of number of + // quarters. + BigDecimal forwardMultiplier = + TimeUnit.QUARTER.equals(unit) ? BigDecimal.valueOf(1) : unit.multiplier; + + // QUARTER intervals are stored as INTERVAL_MONTH in Calcite's type system + // but the qualifier preserves the actual unit (QUARTER vs MONTH). + // The multiplier for QUARTER is 3 (months), for MONTH is 1. + BigDecimal newValue = + forward + ? value.multiply(forwardMultiplier) + : value.divideToIntegralValue(unit.multiplier); + return rexBuilder.makeIntervalLiteral(newValue, qualifier); + } + }; + } + + @Override + protected RelNode visitChild(RelNode parent, int i, RelNode child) { + RelNode newChild = super.visitChild(parent, i, child); + return newChild.accept(rexShuttle); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/validate/shuttles/SkipRelValidationShuttle.java b/core/src/main/java/org/opensearch/sql/calcite/validate/shuttles/SkipRelValidationShuttle.java new file mode 100644 index 00000000000..90a34f82348 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/validate/shuttles/SkipRelValidationShuttle.java @@ -0,0 +1,175 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.validate.shuttles; + +import java.util.List; +import java.util.function.Predicate; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttleImpl; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rel.logical.LogicalValues; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.sql.SqlKind; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; + +/** + * A RelShuttle that detects if validation should be skipped for certain operations. Currently, it + * detects the following patterns: + * + *

    + *
  • Binning on datetime types, which is only executable after pushdown. + *
  • Aggregates with multiple complex CASE statements, which cause field reference issues during + * the SQL-to-Rel conversion. + *
  • LogicalValues is used to populate empty row values + *
+ * + * Group by multiple CASE statements + * + *

When grouping by multiple CASE expressions, a Calcite 1.41 bug causes field references to + * become invalid during SQL-to-Rel conversion. This affects queries in {@code + * testCaseCanBePushedDownAsCompositeRangeQuery} 2.4 and {@code testCaseCanBePushedDownAsRangeQuery} + * 1.3. E.g. for the following query: + * + * 

{@code
+ * source=opensearch-sql_test_index_bank
+ * | eval age_range = case(age < 30, 'u30', age < 40, 'u40' else 'u100'),
+ *        balance_range = case(balance < 20000, 'medium' else 'high')
+ * | stats avg(balance) as avg_balance by age_range, balance_range
+ * }
+ * + *

During validation, this PPL query is converted to SQL: + * + * 

{@code
+ * SELECT AVG(`balance`) AS `avg_balance`,
+ *        CASE WHEN `age` < 30 THEN 'u30' WHEN `age` < 40 THEN 'u40' ELSE 'u100' END AS `age_range`,
+ *        CASE WHEN `balance` < 20000 THEN 'medium' ELSE 'high' END AS `balance_range`
+ * FROM `OpenSearch`.`opensearch-sql_test_index_bank`
+ * GROUP BY CASE WHEN `age` < 30 THEN 'u30' WHEN `age` < 40 THEN 'u40' ELSE 'u100' END,
+ *          CASE WHEN `balance` < 20000 THEN 'medium' ELSE 'high' END
+ * }
+ * + *

When Calcite converts this SQL back to RelNode, it processes GROUP BY expressions + * sequentially, making field references in the second CASE expression invalid. + * + *

Generate empty row with LogicalValues + * + *

Types in the rows generated with {@code VALUES} will not be preserved, causing validation + * issues when converting SQL back to a logical plan. + * + *

For example, in {@code CalcitePPLAggregationIT.testSumEmpty}, the query {@code + * source=opensearch-sql_test_index_bank_with_null_values | where 1=2 | stats sum(balance)} will be + * converted to the following SQL: + * + * 

{@code
+ * SELECT SUM(CAST(`balance` AS DECIMAL(38, 19))) AS `sum(balance)`
+ * FROM (VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)) AS `t` (`account_number`, `firstname`, `address`, `balance`, `gender`, `age`, `lastname`, `_id`, `_index`, `_score`, `_maxscore`, `_sort`, `_routing`)
+ * WHERE 1 = 0
+ * }
+ * + * When converted back to logical plan, {@code CAST(`balance` AS DECIMAL(38, 19))} will fail because + * the type of balance is lost. + * + *

Note for developers: when validations fail during developing new features, please try + * to solve the root cause instead of adding skipping rules here. Under rare cases when you have to + * skip validation, please document the exact reason. + * + *

WARNING: When a skip pattern is detected, we bypass the entire validation pipeline, + * skipping potentially useful transformation relying on rewriting SQL node + */ +public class SkipRelValidationShuttle extends RelShuttleImpl { + private boolean shouldSkip = false; + private final RexShuttle rexShuttle; + + /** Predicates about patterns of calls that should not be validated. */ + public static final List> SKIP_CALLS; + + /** Predicates about logical aggregates that should not be validated */ + public static final List> SKIP_AGGREGATES; + + /** Predicates about logical values that should not be validated */ + public static final List> SKIP_VALUES; + + static { + // TODO: Make incompatible operations like bin-on-timestamp a validatable UDFs so that they can + // be still be converted to SqlNode and back to RelNode + Predicate binOnTimestamp = + call -> { + if ("WIDTH_BUCKET".equalsIgnoreCase(call.getOperator().getName())) { + if (!call.getOperands().isEmpty()) { + RexNode firstOperand = call.getOperands().get(0); + return OpenSearchTypeUtil.isDatetime(firstOperand.getType()); + } + } + return false; + }; + Predicate groupByMultipleCases = + aggregate -> { + if (aggregate.getGroupCount() > 1 + && aggregate.getInput() instanceof LogicalProject project) { + long nGroupByCase = + project.getProjects().stream().filter(p -> p.isA(SqlKind.CASE)).count(); + return nGroupByCase > 1; + } + return false; + }; + Predicate createEmptyRow = values -> values.getTuples().isEmpty(); + SKIP_CALLS = List.of(binOnTimestamp); + SKIP_AGGREGATES = List.of(groupByMultipleCases); + SKIP_VALUES = List.of(createEmptyRow); + } + + @Override + public RelNode visit(LogicalAggregate aggregate) { + for (Predicate skipAgg : SKIP_AGGREGATES) { + if (skipAgg.test(aggregate)) { + shouldSkip = true; + return aggregate; + } + } + return super.visit(aggregate); + } + + @Override + public RelNode visit(LogicalValues values) { + for (Predicate skipValues : SKIP_VALUES) { + if (skipValues.test(values)) { + shouldSkip = true; + return values; + } + } + return super.visit(values); + } + + public SkipRelValidationShuttle() { + this.rexShuttle = + new RexShuttle() { + @Override + public RexNode visitCall(RexCall call) { + for (Predicate skipCall : SKIP_CALLS) { + if (skipCall.test(call)) { + shouldSkip = true; + return call; + } + } + return super.visitCall(call); + } + }; + } + + /** Returns true if validation should be skipped based on detected conditions. */ + public boolean shouldSkipValidation() { + return shouldSkip; + } + + @Override + protected RelNode visitChild(RelNode parent, int i, RelNode child) { + RelNode newChild = super.visitChild(parent, i, child); + return newChild.accept(rexShuttle); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/validate/shuttles/SqlRewriteShuttle.java b/core/src/main/java/org/opensearch/sql/calcite/validate/shuttles/SqlRewriteShuttle.java new file mode 100644 index 00000000000..d2a2f229147 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/validate/shuttles/SqlRewriteShuttle.java @@ -0,0 +1,69 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.validate.shuttles; + +import java.util.Collections; +import java.util.List; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.fun.SqlCountAggFunction; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.util.SqlShuttle; +import org.opensearch.sql.calcite.OpenSearchSchema; + +public class SqlRewriteShuttle extends SqlShuttle { + @Override + public SqlNode visit(SqlIdentifier id) { + // Remove database qualifier, keeping only table name + if (id.names.size() == 2 && OpenSearchSchema.OPEN_SEARCH_SCHEMA_NAME.equals(id.names.get(0))) { + return new SqlIdentifier(Collections.singletonList(id.names.get(1)), id.getParserPosition()); + } + return super.visit(id); + } + + @Override + public @org.checkerframework.checker.nullness.qual.Nullable SqlNode visit(SqlCall call) { + if (call.getOperator() instanceof SqlCountAggFunction && call.getOperandList().isEmpty()) { + // Convert COUNT() to COUNT(*) so that SqlCall.isCountStar() resolves to True + // This is useful when deriving the return types in SqlCountAggFunction#deriveType + call = + new SqlBasicCall( + SqlStdOperatorTable.COUNT, + List.of(SqlIdentifier.STAR), + call.getParserPosition(), + call.getFunctionQuantifier()); + } else if (call.getKind() == SqlKind.IN || call.getKind() == SqlKind.NOT_IN) { + // Fix for tuple IN / NOT IN queries: Convert SqlNodeList to ROW SqlCall + // + // When RelToSqlConverter converts a tuple expression like (id, name) back to + // SqlNode, it generates a bare SqlNodeList instead of wrapping it in a ROW + // operator. This causes validation to fail because: + // 1. SqlValidator.deriveType() doesn't know how to handle SqlNodeList + // 2. SqlToRelConverter.visit(SqlNodeList) throws UnsupportedOperationException + // + // For example, the query: + // WHERE (id, name) NOT IN (SELECT uid, name FROM ...) + // + // After Rel-to-SQL conversion becomes: + // IN operator with operands: [SqlNodeList[id, name], SqlSelect[...]] + // + // But it should be: + // IN operator with operands: [ROW(id, name), SqlSelect[...]] + // + // This fix wraps the SqlNodeList in a ROW SqlCall before validation, + // ensuring proper type derivation and subsequent SQL-to-Rel conversion. + if (!call.getOperandList().isEmpty() + && call.getOperandList().get(0) instanceof SqlNodeList nodes) { + call.setOperand(0, SqlStdOperatorTable.ROW.createCall(nodes)); + } + } + return super.visit(call); + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index c85849df725..e890c4d0c19 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -6,6 +6,7 @@ package org.opensearch.sql.executor; import java.util.List; +import java.util.Objects; import java.util.Optional; import javax.annotation.Nullable; import lombok.AllArgsConstructor; @@ -17,14 +18,21 @@ import org.apache.calcite.plan.hep.HepPlanner; import org.apache.calcite.plan.hep.HepProgram; import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.prepare.CalciteCatalogReader; import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.logical.LogicalSort; +import org.apache.calcite.rel.rel2sql.RelToSqlConverter; +import org.apache.calcite.rel.rel2sql.SqlImplementor; import org.apache.calcite.rel.rules.FilterMergeRule; +import org.apache.calcite.runtime.CalciteContextException; import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql2rel.SqlToRelConverter; import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.Frameworks; import org.apache.calcite.tools.Programs; @@ -38,10 +46,20 @@ import org.opensearch.sql.calcite.SysLimit; import org.opensearch.sql.calcite.plan.LogicalSystemLimit; import org.opensearch.sql.calcite.plan.LogicalSystemLimit.SystemLimitType; +import org.opensearch.sql.calcite.utils.PPLHintStrategyTable; +import org.opensearch.sql.calcite.validate.OpenSearchSparkSqlDialect; +import org.opensearch.sql.calcite.validate.PplConvertletTable; +import org.opensearch.sql.calcite.validate.ValidationUtils; +import org.opensearch.sql.calcite.validate.converters.PplRelToSqlNodeConverter; +import org.opensearch.sql.calcite.validate.converters.PplSqlToRelConverter; +import org.opensearch.sql.calcite.validate.shuttles.PplRelToSqlRelShuttle; +import org.opensearch.sql.calcite.validate.shuttles.SkipRelValidationShuttle; +import org.opensearch.sql.calcite.validate.shuttles.SqlRewriteShuttle; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.exception.CalciteUnsupportedException; +import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.NonFallbackCalciteException; import org.opensearch.sql.planner.PlanContext; import org.opensearch.sql.planner.Planner; @@ -103,7 +121,8 @@ public void executeWithCalcite( buildFrameworkConfig(), SysLimit.fromSettings(settings), queryType); RelNode relNode = analyze(plan, context); relNode = mergeAdjacentFilters(relNode); - RelNode optimized = optimize(relNode, context); + RelNode validated = validate(relNode, context); + RelNode optimized = optimize(validated, context); RelNode calcitePlan = convertToCalcitePlan(optimized); executionEngine.execute(calcitePlan, context, listener); } catch (Throwable t) { @@ -144,7 +163,8 @@ public void explainWithCalcite( () -> { RelNode relNode = analyze(plan, context); relNode = mergeAdjacentFilters(relNode); - RelNode optimized = optimize(relNode, context); + RelNode validated = validate(relNode, context); + RelNode optimized = optimize(validated, context); RelNode calcitePlan = convertToCalcitePlan(optimized); executionEngine.explain(calcitePlan, format, context, listener); }, @@ -271,6 +291,66 @@ public LogicalPlan analyze(UnresolvedPlan plan, QueryType queryType) { return analyzer.analyze(plan, new AnalysisContext(queryType)); } + /** + * Validates a RelNode by converting it to SqlNode, performing validation, and converting back. + * + *

This process enables Calcite's type validation and implicit casting mechanisms to work on + * PPL queries. + * + * @param relNode the relation node to validate + * @param context the Calcite plan context containing the validator + * @return the validated (and potentially modified) relation node + */ + private RelNode validate(RelNode relNode, CalcitePlanContext context) { + SkipRelValidationShuttle skipShuttle = new SkipRelValidationShuttle(); + relNode.accept(skipShuttle); + if (skipShuttle.shouldSkipValidation()) { + return relNode; + } + // Fix interval literals before conversion to SQL + RelNode sqlRelNode = relNode.accept(new PplRelToSqlRelShuttle(context.rexBuilder, true)); + + // Convert RelNode to SqlNode for validation + RelToSqlConverter rel2sql = new PplRelToSqlNodeConverter(OpenSearchSparkSqlDialect.DEFAULT); + SqlImplementor.Result result = rel2sql.visitRoot(sqlRelNode); + SqlNode root = result.asStatement(); + + // Rewrite SqlNode to remove database qualifiers + SqlNode rewritten = root.accept(new SqlRewriteShuttle()); + SqlValidator validator = context.getValidator(); + try { + validator.validate(Objects.requireNonNull(rewritten)); + } catch (CalciteContextException e) { + if (ValidationUtils.tolerantValidationException(e)) { + return relNode; + } + throw new ExpressionEvaluationException(e.getMessage(), e); + } + + SqlToRelConverter.Config sql2relConfig = + SqlToRelConverter.config() + // Do not remove sort in subqueries so that the orders for queries like `... | sort a + // | fields b` is preserved + .withRemoveSortInSubQuery(false) + // Disable automatic JSON_TYPE_OPERATOR wrapping for nested JSON functions. + // See CALCITE-4989: Calcite wraps nested JSON functions with JSON_TYPE by default + .withAddJsonTypeOperatorEnabled(false) + // Set hint strategy so that hints can be properly propagated. + // See SqlToRelConverter.java#convertSelectImpl + .withHintStrategyTable(PPLHintStrategyTable.getHintStrategyTable()); + SqlToRelConverter sql2rel = + new PplSqlToRelConverter( + context.config.getViewExpander(), + validator, + validator.getCatalogReader().unwrap(CalciteCatalogReader.class), + context.relBuilder.getCluster(), + PplConvertletTable.INSTANCE, + sql2relConfig); + // Convert the validated SqlNode back to RelNode + RelNode validatedRel = sql2rel.convertQuery(rewritten, false, true).project(); + return validatedRel.accept(new PplRelToSqlRelShuttle(context.rexBuilder, false)); + } + /** Translate {@link LogicalPlan} to {@link PhysicalPlan}. */ public PhysicalPlan plan(LogicalPlan plan) { return planner.plan(plan); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CalciteFuncSignature.java b/core/src/main/java/org/opensearch/sql/expression/function/CalciteFuncSignature.java deleted file mode 100644 index a8c4be11102..00000000000 --- a/core/src/main/java/org/opensearch/sql/expression/function/CalciteFuncSignature.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.expression.function; - -import java.util.List; -import org.apache.calcite.rel.type.RelDataType; - -/** Function signature is composed by function name and arguments list. */ -public record CalciteFuncSignature(FunctionName functionName, PPLTypeChecker typeChecker) { - - public boolean match(FunctionName functionName, List argTypes) { - if (!functionName.equals(this.functionName())) return false; - // For complex type checkers (e.g., OperandTypes.COMPARABLE_UNORDERED_COMPARABLE_UNORDERED), - // the typeChecker will be null because only simple family-based type checks are currently - // supported. - if (typeChecker == null) return true; - return typeChecker.checkOperandTypes(argTypes); - } -} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CoercionUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/CoercionUtils.java deleted file mode 100644 index ce78d6dec21..00000000000 --- a/core/src/main/java/org/opensearch/sql/expression/function/CoercionUtils.java +++ /dev/null @@ -1,274 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.expression.function; - -import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN; - -import com.google.common.annotations.VisibleForTesting; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.PriorityQueue; -import java.util.Set; -import java.util.function.BiPredicate; -import java.util.function.BinaryOperator; -import java.util.stream.Collectors; -import javax.annotation.Nullable; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexNode; -import org.apache.commons.lang3.tuple.Pair; -import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; -import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.exception.ExpressionEvaluationException; - -public final class CoercionUtils { - /** - * Casts the arguments to the types specified in the typeChecker. Returns null if no combination - * of parameter types matches the arguments or if casting fails. - * - * @param builder RexBuilder to create casts - * @param typeChecker PPLTypeChecker that provides the parameter types - * @param arguments List of RexNode arguments to be cast - * @return List of cast RexNode arguments or null if casting fails - */ - public static @Nullable List castArguments( - RexBuilder builder, PPLTypeChecker typeChecker, List arguments) { - List> paramTypeCombinations = typeChecker.getParameterTypes(); - - List sourceTypes = - arguments.stream() - .map(node -> OpenSearchTypeFactory.convertRelDataTypeToExprType(node.getType())) - .collect(Collectors.toList()); - // Candidate parameter signatures ordered by decreasing widening distance - PriorityQueue, Integer>> rankedSignatures = - new PriorityQueue<>((left, right) -> Integer.compare(right.getValue(), left.getValue())); - for (List paramTypes : paramTypeCombinations) { - int distance = distance(sourceTypes, paramTypes); - if (distance == TYPE_EQUAL) { - return castArguments(builder, paramTypes, arguments); - } - Optional.of(distance) - .filter(value -> value != IMPOSSIBLE_WIDENING) - .ifPresent(value -> rankedSignatures.add(Pair.of(paramTypes, value))); - } - return Optional.ofNullable(rankedSignatures.peek()) - .map(Pair::getKey) - .map(paramTypes -> castArguments(builder, paramTypes, arguments)) - .orElse(null); - } - - /** - * Widen the arguments to the widest type found among them. If no widest type can be determined, - * returns null. - * - * @param builder RexBuilder to create casts - * @param arguments List of RexNode arguments to be widened - * @return List of widened RexNode arguments or null if no widest type can be determined - */ - public static @Nullable List widenArguments( - RexBuilder builder, List arguments) { - // TODO: Add test on e.g. IP - ExprType widestType = findWidestType(arguments); - if (widestType == null) { - return null; // No widest type found, return null - } - return arguments.stream().map(arg -> cast(builder, widestType, arg)).toList(); - } - - /** - * Casts the arguments to the types specified in paramTypes. Returns null if the number of - * parameters does not match or if casting fails. - */ - private static @Nullable List castArguments( - RexBuilder builder, List paramTypes, List arguments) { - if (paramTypes.size() != arguments.size()) { - return null; // Skip if the number of parameters does not match - } - - List castedArguments = new ArrayList<>(); - for (int i = 0; i < paramTypes.size(); i++) { - ExprType toType = paramTypes.get(i); - RexNode arg = arguments.get(i); - - RexNode castedArg = cast(builder, toType, arg); - - if (castedArg == null) { - return null; - } - castedArguments.add(castedArg); - } - return castedArguments; - } - - private static @Nullable RexNode cast(RexBuilder builder, ExprType targetType, RexNode arg) { - ExprType argType = OpenSearchTypeFactory.convertRelDataTypeToExprType(arg.getType()); - if (!argType.shouldCast(targetType)) { - return arg; - } - if (distance(argType, targetType) != IMPOSSIBLE_WIDENING) { - return builder.makeCast( - OpenSearchTypeFactory.convertExprTypeToRelDataType(targetType), arg, true, true); - } - return resolveCommonType(argType, targetType) - .map( - exprType -> - builder.makeCast( - OpenSearchTypeFactory.convertExprTypeToRelDataType(exprType), arg, true, true)) - .orElse(null); - } - - /** - * Finds the widest type among the given arguments. The widest type is determined by applying the - * widening type rule to each pair of types in the arguments. - * - * @param arguments List of RexNode arguments to find the widest type from - * @return the widest ExprType if found, otherwise null - */ - private static @Nullable ExprType findWidestType(List arguments) { - if (arguments.isEmpty()) { - return null; // No arguments to process - } - ExprType widestType = - OpenSearchTypeFactory.convertRelDataTypeToExprType(arguments.getFirst().getType()); - if (arguments.size() == 1) { - return widestType; - } - - // Iterate pairwise through the arguments and find the widest type - for (int i = 1; i < arguments.size(); i++) { - var type = OpenSearchTypeFactory.convertRelDataTypeToExprType(arguments.get(i).getType()); - try { - final ExprType tempType = widestType; - widestType = resolveCommonType(widestType, type).orElseGet(() -> max(tempType, type)); - } catch (ExpressionEvaluationException e) { - // the two types are not compatible, return null - return null; - } - } - return widestType; - } - - private static boolean areDateAndTime(ExprType type1, ExprType type2) { - return (type1 == ExprCoreType.DATE && type2 == ExprCoreType.TIME) - || (type1 == ExprCoreType.TIME && type2 == ExprCoreType.DATE); - } - - @VisibleForTesting - public static Optional resolveCommonType(ExprType left, ExprType right) { - return COMMON_COERCION_RULES.stream() - .map(rule -> rule.apply(left, right)) - .flatMap(Optional::stream) - .findFirst(); - } - - public static boolean hasString(List rexNodeList) { - return rexNodeList.stream() - .map(RexNode::getType) - .map(OpenSearchTypeFactory::convertRelDataTypeToExprType) - .anyMatch(t -> t == ExprCoreType.STRING); - } - - private static final Set NUMBER_TYPES = ExprCoreType.numberTypes(); - - private static final List COMMON_COERCION_RULES = - List.of( - CoercionRule.of( - (left, right) -> areDateAndTime(left, right), - (left, right) -> ExprCoreType.TIMESTAMP), - CoercionRule.of( - (left, right) -> hasString(left, right) && hasNumber(left, right), - (left, right) -> ExprCoreType.DOUBLE)); - - private static boolean hasString(ExprType left, ExprType right) { - return left == ExprCoreType.STRING || right == ExprCoreType.STRING; - } - - private static boolean hasNumber(ExprType left, ExprType right) { - return NUMBER_TYPES.contains(left) || NUMBER_TYPES.contains(right); - } - - private static boolean hasBoolean(ExprType left, ExprType right) { - return left == ExprCoreType.BOOLEAN || right == ExprCoreType.BOOLEAN; - } - - private record CoercionRule( - BiPredicate predicate, BinaryOperator resolver) { - - Optional apply(ExprType left, ExprType right) { - return predicate.test(left, right) - ? Optional.of(resolver.apply(left, right)) - : Optional.empty(); - } - - static CoercionRule of( - BiPredicate predicate, BinaryOperator resolver) { - return new CoercionRule(predicate, resolver); - } - } - - private static final int IMPOSSIBLE_WIDENING = Integer.MAX_VALUE; - private static final int TYPE_EQUAL = 0; - - private static int distance(ExprType type1, ExprType type2) { - return distance(type1, type2, TYPE_EQUAL); - } - - private static int distance(ExprType type1, ExprType type2, int distance) { - if (type1 == type2) { - return distance; - } else if (type1 == UNKNOWN) { - return IMPOSSIBLE_WIDENING; - } else if (type1 == ExprCoreType.STRING && type2 == ExprCoreType.DOUBLE) { - return 1; - } else { - return type1.getParent().stream() - .map(parentOfType1 -> distance(parentOfType1, type2, distance + 1)) - .reduce(Math::min) - .get(); - } - } - - /** - * The max type among two types. The max is defined as follow if type1 could widen to type2, then - * max is type2, vice versa if type1 couldn't widen to type2 and type2 could't widen to type1, - * then throw {@link ExpressionEvaluationException}. - * - * @param type1 type1 - * @param type2 type2 - * @return the max type among two types. - */ - public static ExprType max(ExprType type1, ExprType type2) { - int type1To2 = distance(type1, type2); - int type2To1 = distance(type2, type1); - - if (type1To2 == Integer.MAX_VALUE && type2To1 == Integer.MAX_VALUE) { - throw new ExpressionEvaluationException( - String.format("no max type of %s and %s ", type1, type2)); - } else { - return type1To2 == Integer.MAX_VALUE ? type1 : type2; - } - } - - public static int distance(List sourceTypes, List targetTypes) { - if (sourceTypes.size() != targetTypes.size()) { - return IMPOSSIBLE_WIDENING; - } - - int totalDistance = 0; - for (int i = 0; i < sourceTypes.size(); i++) { - ExprType source = sourceTypes.get(i); - ExprType target = targetTypes.get(i); - int distance = distance(source, target); - if (distance == IMPOSSIBLE_WIDENING) { - return IMPOSSIBLE_WIDENING; - } else { - totalDistance += distance; - } - } - return totalDistance; - } -} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ArrayFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ArrayFunctionImpl.java index 2e9e53b9ac2..2d02bb84904 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ArrayFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ArrayFunctionImpl.java @@ -22,6 +22,7 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -60,7 +61,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return null; + return UDFOperandMetadata.wrap(OperandTypes.ARRAY_FUNCTION); } public static class ArrayImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ExistsFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ExistsFunctionImpl.java index 4b1c9586773..2b180933d60 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ExistsFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ExistsFunctionImpl.java @@ -14,6 +14,7 @@ import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -36,7 +37,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return null; + return UDFOperandMetadata.wrap(OperandTypes.EXISTS); } public static class ExistsImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/FilterFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/FilterFunctionImpl.java index 953b75303db..dd953bbb9b6 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/FilterFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/FilterFunctionImpl.java @@ -15,6 +15,7 @@ import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -36,7 +37,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return null; + return UDFOperandMetadata.wrap(OperandTypes.MAP_FUNCTION); } public static class FilterImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForallFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForallFunctionImpl.java index 720eee841ab..153a632eb8d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForallFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForallFunctionImpl.java @@ -14,6 +14,7 @@ import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -35,7 +36,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return null; + return UDFOperandMetadata.wrap(OperandTypes.MAP_FUNCTION); } public static class ForallImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/LambdaUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/LambdaUtils.java index 314ac3ad945..5956b32559c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/LambdaUtils.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/LambdaUtils.java @@ -6,17 +6,6 @@ package org.opensearch.sql.expression.function.CollectionUDF; import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexCallBinding; -import org.apache.calcite.rex.RexLambda; -import org.apache.calcite.rex.RexLambdaRef; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; public class LambdaUtils { @@ -37,58 +26,4 @@ public static Object transferLambdaOutputToTargetType(Object candidate, SqlTypeN return candidate; } } - - public static RelDataType inferReturnTypeFromLambda( - RexLambda rexLambda, Map filledTypes, RelDataTypeFactory typeFactory) { - RexCall rexCall = (RexCall) rexLambda.getExpression(); - SqlReturnTypeInference returnInfer = rexCall.getOperator().getReturnTypeInference(); - List lambdaOperands = rexCall.getOperands(); - List filledOperands = new ArrayList<>(); - for (RexNode rexNode : lambdaOperands) { - if (rexNode instanceof RexLambdaRef rexLambdaRef) { - if (rexLambdaRef.getType().getSqlTypeName() == SqlTypeName.ANY) { - filledOperands.add( - new RexLambdaRef( - rexLambdaRef.getIndex(), - rexLambdaRef.getName(), - filledTypes.get(rexLambdaRef.getName()))); - } else { - filledOperands.add(rexNode); - } - } else if (rexNode instanceof RexCall) { - filledOperands.add( - reInferReturnTypeForRexCallInsideLambda((RexCall) rexNode, filledTypes, typeFactory)); - } else { - filledOperands.add(rexNode); - } - } - return returnInfer.inferReturnType( - new RexCallBinding(typeFactory, rexCall.getOperator(), filledOperands, List.of())); - } - - public static RexCall reInferReturnTypeForRexCallInsideLambda( - RexCall rexCall, Map argTypes, RelDataTypeFactory typeFactory) { - List filledOperands = new ArrayList<>(); - List rexCallOperands = rexCall.getOperands(); - for (RexNode rexNode : rexCallOperands) { - if (rexNode instanceof RexLambdaRef rexLambdaRef) { - filledOperands.add( - new RexLambdaRef( - rexLambdaRef.getIndex(), - rexLambdaRef.getName(), - argTypes.get(rexLambdaRef.getName()))); - } else if (rexNode instanceof RexCall) { - filledOperands.add( - reInferReturnTypeForRexCallInsideLambda((RexCall) rexNode, argTypes, typeFactory)); - } else { - filledOperands.add(rexNode); - } - } - RelDataType returnType = - rexCall - .getOperator() - .inferReturnType( - new RexCallBinding(typeFactory, rexCall.getOperator(), filledOperands, List.of())); - return rexCall.clone(returnType, filledOperands); - } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java index 107df5eea4e..ac19f019eb6 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java @@ -18,6 +18,7 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.SqlOperatorBinding; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -50,7 +51,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return null; + return UDFOperandMetadata.wrap(OperandTypes.VARIADIC); } private static RelDataType determineElementType( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapAppendFunctionImpl.java index 4cb0acae612..c7754d1363d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapAppendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapAppendFunctionImpl.java @@ -18,7 +18,9 @@ import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -45,7 +47,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return null; + return UDFOperandMetadata.wrap(OperandTypes.family(SqlTypeFamily.MAP, SqlTypeFamily.MAP)); } public static class MapAppendImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapRemoveFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapRemoveFunctionImpl.java index 1f86fcbe636..a2ba3b51edf 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapRemoveFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapRemoveFunctionImpl.java @@ -16,7 +16,9 @@ import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeFamily; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -42,7 +44,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return null; + return UDFOperandMetadata.wrap(OperandTypes.family(SqlTypeFamily.MAP, SqlTypeFamily.ARRAY)); } public static class MapRemoveImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ReduceFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ReduceFunctionImpl.java index d60a700e816..945e7704c90 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ReduceFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ReduceFunctionImpl.java @@ -5,13 +5,10 @@ package org.opensearch.sql.expression.function.CollectionUDF; -import static org.opensearch.sql.expression.function.CollectionUDF.LambdaUtils.inferReturnTypeFromLambda; import static org.opensearch.sql.expression.function.CollectionUDF.LambdaUtils.transferLambdaOutputToTargetType; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; @@ -19,14 +16,19 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexCallBinding; -import org.apache.calcite.rex.RexLambda; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.sql.type.ArraySqlType; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlLambda; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.apache.calcite.sql.validate.SqlValidator; +import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -43,35 +45,54 @@ public ReduceFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { return sqlOperatorBinding -> { - RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); - RexCallBinding rexCallBinding = (RexCallBinding) sqlOperatorBinding; - List rexNodes = rexCallBinding.operands(); - ArraySqlType listType = (ArraySqlType) rexNodes.get(0).getType(); - RelDataType elementType = listType.getComponentType(); - RelDataType baseType = rexNodes.get(1).getType(); - Map map = new HashMap<>(); - RexLambda mergeLambda = (RexLambda) rexNodes.get(2); - map.put(mergeLambda.getParameters().get(0).getName(), baseType); - map.put(mergeLambda.getParameters().get(1).getName(), elementType); - RelDataType mergedReturnType = - inferReturnTypeFromLambda((RexLambda) rexNodes.get(2), map, typeFactory); - if (mergedReturnType != baseType) { // For different acc, we need to recalculate - map.put(mergeLambda.getParameters().get(0).getName(), mergedReturnType); - mergedReturnType = inferReturnTypeFromLambda((RexLambda) rexNodes.get(2), map, typeFactory); - } - RelDataType finalReturnType; - if (rexNodes.size() > 3) { - finalReturnType = inferReturnTypeFromLambda((RexLambda) rexNodes.get(3), map, typeFactory); - } else { - finalReturnType = mergedReturnType; + if (sqlOperatorBinding instanceof RexCallBinding) { + return sqlOperatorBinding.getOperandType(sqlOperatorBinding.getOperandCount() - 1); + } else if (sqlOperatorBinding instanceof SqlCallBinding callBinding) { + RelDataType elementType = callBinding.getOperandType(0).getComponentType(); + RelDataType baseType = callBinding.getOperandType(1); + SqlLambda reduce1 = callBinding.getCall().operand(2); + SqlNode function1 = reduce1.getExpression(); + SqlValidator validator = callBinding.getValidator(); + // The saved types are ANY because the lambda function is defined as (ANY, ..) -> ANY + // Force it to derive types again by removing existing saved types + validator.removeValidatedNodeType(function1); + if (function1 instanceof SqlCall call) { + List operands = call.getOperandList(); + // The first argument is base (accumulator), while the second is from the array + if (!operands.isEmpty()) validator.setValidatedNodeType(operands.get(0), baseType); + if (operands.size() > 1 && elementType != null) + validator.setValidatedNodeType(operands.get(1), elementType); + } + RelDataType returnType = SqlTypeUtil.deriveType(callBinding, function1); + if (callBinding.getOperandCount() > 3) { + SqlLambda reduce2 = callBinding.getCall().operand(3); + SqlNode function2 = reduce2.getExpression(); + validator.removeValidatedNodeType(function2); + if (function2 instanceof SqlCall call) { + List operands = call.getOperandList(); + if (!operands.isEmpty()) validator.setValidatedNodeType(operands.get(0), returnType); + } + returnType = SqlTypeUtil.deriveType(callBinding, function2); + } + return returnType; } - return finalReturnType; + throw new IllegalStateException( + StringUtils.format( + "sqlOperatorBinding can only be either RexCallBinding or SqlCallBinding, but got %s", + sqlOperatorBinding.getClass())); }; } @Override public UDFOperandMetadata getOperandMetadata() { - return null; + return UDFOperandMetadata.wrap( + OperandTypes.family(SqlTypeFamily.ARRAY, SqlTypeFamily.ANY, SqlTypeFamily.FUNCTION) + .or( + OperandTypes.family( + SqlTypeFamily.ARRAY, + SqlTypeFamily.ANY, + SqlTypeFamily.FUNCTION, + SqlTypeFamily.FUNCTION))); } public static class ReduceImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/TransformFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/TransformFunctionImpl.java index d2184e12e8c..e4a25e68067 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/TransformFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/TransformFunctionImpl.java @@ -6,10 +6,12 @@ package org.opensearch.sql.expression.function.CollectionUDF; import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; +import static org.apache.calcite.util.Static.RESOURCE; import static org.opensearch.sql.expression.function.CollectionUDF.LambdaUtils.transferLambdaOutputToTargetType; import java.util.ArrayList; import java.util.List; +import java.util.stream.IntStream; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; @@ -19,12 +21,19 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexCallBinding; -import org.apache.calcite.rex.RexLambda; -import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlUtil; import org.apache.calcite.sql.type.ArraySqlType; +import org.apache.calcite.sql.type.FamilyOperandTypeChecker; +import org.apache.calcite.sql.type.SqlOperandCountRanges; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlSingleOperandTypeChecker; +import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -43,9 +52,7 @@ public TransformFunctionImpl() { public SqlReturnTypeInference getReturnTypeInference() { return sqlOperatorBinding -> { RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); - RexCallBinding rexCallBinding = (RexCallBinding) sqlOperatorBinding; - List operands = rexCallBinding.operands(); - RelDataType lambdaReturnType = ((RexLambda) operands.get(1)).getExpression().getType(); + RelDataType lambdaReturnType = sqlOperatorBinding.getOperandType(1); return createArrayType( typeFactory, typeFactory.createTypeWithNullability(lambdaReturnType, true), true); }; @@ -53,7 +60,96 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return null; + // Only checks the first two arguments as it allows arbitrary number of arguments to follow them + return UDFOperandMetadata.wrap( + new SqlSingleOperandTypeChecker() { + private static final List families = + List.of(SqlTypeFamily.ARRAY, SqlTypeFamily.FUNCTION); + + /** + * Copied from {@link FamilyOperandTypeChecker#checkSingleOperandType(SqlCallBinding + * callBinding, SqlNode node, int iFormalOperand, boolean throwOnFailure)} + */ + @Override + public boolean checkSingleOperandType( + SqlCallBinding callBinding, + SqlNode operand, + int iFormalOperand, + boolean throwOnFailure) { + // Do not check types after the second operands + if (iFormalOperand > 1) { + return true; + } + SqlTypeFamily family = families.get(iFormalOperand); + switch (family) { + case ANY: + final RelDataType type = SqlTypeUtil.deriveType(callBinding, operand); + SqlTypeName typeName = type.getSqlTypeName(); + + if (typeName == SqlTypeName.CURSOR) { + // We do not allow CURSOR operands, even for ANY + if (throwOnFailure) { + throw callBinding.newValidationSignatureError(); + } + return false; + } + // fall through + case IGNORE: + // no need to check + return true; + default: + break; + } + if (SqlUtil.isNullLiteral(operand, false)) { + if (callBinding.isTypeCoercionEnabled()) { + return true; + } else if (throwOnFailure) { + throw callBinding + .getValidator() + .newValidationError(operand, RESOURCE.nullIllegal()); + } else { + return false; + } + } + RelDataType type = SqlTypeUtil.deriveType(callBinding, operand); + SqlTypeName typeName = type.getSqlTypeName(); + + // Pass type checking for operators if it's of type 'ANY'. + if (typeName.getFamily() == SqlTypeFamily.ANY) { + return true; + } + + if (!family.getTypeNames().contains(typeName)) { + if (throwOnFailure) { + throw callBinding.newValidationSignatureError(); + } + return false; + } + return true; + } + + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + if (!getOperandCountRange().isValidCount(callBinding.getOperandCount())) { + return false; + } + return IntStream.range(0, 2) + .allMatch( + i -> + checkSingleOperandType( + callBinding, callBinding.operand(i), i, throwOnFailure)); + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.from(2); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return ""; + } + }); } public static class TransformImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 2d769194924..18910c17b50 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -23,11 +23,20 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlTypeTransforms; import org.apache.calcite.sql.util.ReflectiveSqlOperatorTable; +import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.util.BuiltInMethod; import org.opensearch.sql.calcite.udf.udaf.FirstAggFunction; import org.opensearch.sql.calcite.udf.udaf.LastAggFunction; @@ -72,7 +81,6 @@ import org.opensearch.sql.expression.function.udf.ToNumberFunction; import org.opensearch.sql.expression.function.udf.ToStringFunction; import org.opensearch.sql.expression.function.udf.condition.EarliestFunction; -import org.opensearch.sql.expression.function.udf.condition.EnhancedCoalesceFunction; import org.opensearch.sql.expression.function.udf.condition.LatestFunction; import org.opensearch.sql.expression.function.udf.datetime.AddSubDateFunction; import org.opensearch.sql.expression.function.udf.datetime.CurrentFunction; @@ -113,66 +121,66 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { Suppliers.memoize(() -> (PPLBuiltinOperators) new PPLBuiltinOperators().init()); // Json Functions - public static final SqlOperator JSON = new JsonFunctionImpl().toUDF("JSON"); - public static final SqlOperator JSON_ARRAY_LENGTH = + public static final SqlFunction JSON = new JsonFunctionImpl().toUDF("JSON"); + public static final SqlFunction JSON_ARRAY_LENGTH = new JsonArrayLengthFunctionImpl().toUDF("JSON_ARRAY_LENGTH"); - public static final SqlOperator JSON_EXTRACT = + public static final SqlFunction JSON_EXTRACT = new JsonExtractFunctionImpl().toUDF("JSON_EXTRACT"); - public static final SqlOperator JSON_EXTRACT_ALL = + public static final SqlFunction JSON_EXTRACT_ALL = new JsonExtractAllFunctionImpl().toUDF("JSON_EXTRACT_ALL"); - public static final SqlOperator JSON_KEYS = new JsonKeysFunctionImpl().toUDF("JSON_KEYS"); - public static final SqlOperator JSON_SET = new JsonSetFunctionImpl().toUDF("JSON_SET"); - public static final SqlOperator JSON_DELETE = new JsonDeleteFunctionImpl().toUDF("JSON_DELETE"); - public static final SqlOperator JSON_APPEND = new JsonAppendFunctionImpl().toUDF("JSON_APPEND"); - public static final SqlOperator JSON_EXTEND = new JsonExtendFunctionImpl().toUDF("JSON_EXTEND"); + public static final SqlFunction JSON_KEYS = new JsonKeysFunctionImpl().toUDF("JSON_KEYS"); + public static final SqlFunction JSON_SET = new JsonSetFunctionImpl().toUDF("JSON_SET"); + public static final SqlFunction JSON_DELETE = new JsonDeleteFunctionImpl().toUDF("JSON_DELETE"); + public static final SqlFunction JSON_APPEND = new JsonAppendFunctionImpl().toUDF("JSON_APPEND"); + public static final SqlFunction JSON_EXTEND = new JsonExtendFunctionImpl().toUDF("JSON_EXTEND"); // Math functions - public static final SqlOperator SPAN = new SpanFunction().toUDF("SPAN"); - public static final SqlOperator E = new EulerFunction().toUDF("E"); - public static final SqlOperator CONV = new ConvFunction().toUDF("CONVERT"); - public static final SqlOperator MOD = new ModFunction().toUDF("MOD"); - public static final SqlOperator DIVIDE = new DivideFunction().toUDF("DIVIDE"); - public static final SqlOperator SHA2 = CryptographicFunction.sha2().toUDF("SHA2"); - public static final SqlOperator CIDRMATCH = new CidrMatchFunction().toUDF("CIDRMATCH"); - public static final SqlOperator SCALAR_MAX = new ScalarMaxFunction().toUDF("SCALAR_MAX"); - public static final SqlOperator SCALAR_MIN = new ScalarMinFunction().toUDF("SCALAR_MIN"); + public static final SqlFunction SPAN = new SpanFunction().toUDF("SPAN"); + public static final SqlFunction E = new EulerFunction().toUDF("E"); + public static final SqlFunction CONV = new ConvFunction().toUDF("CONVERT"); + public static final SqlFunction MOD = new ModFunction().toUDF("MOD"); + public static final SqlFunction DIVIDE = new DivideFunction().toUDF("DIVIDE"); + public static final SqlFunction SHA2 = CryptographicFunction.sha2().toUDF("SHA2"); + public static final SqlFunction CIDRMATCH = new CidrMatchFunction().toUDF("CIDRMATCH"); + public static final SqlFunction SCALAR_MAX = new ScalarMaxFunction().toUDF("SCALAR_MAX"); + public static final SqlFunction SCALAR_MIN = new ScalarMinFunction().toUDF("SCALAR_MIN"); - public static final SqlOperator COSH = + public static final SqlFunction COSH = adaptMathFunctionToUDF( "cosh", ReturnTypes.DOUBLE_FORCE_NULLABLE, NullPolicy.ANY, PPLOperandTypes.NUMERIC) .toUDF("COSH"); - public static final SqlOperator SINH = + public static final SqlFunction SINH = adaptMathFunctionToUDF( "sinh", ReturnTypes.DOUBLE_FORCE_NULLABLE, NullPolicy.ANY, PPLOperandTypes.NUMERIC) .toUDF("SINH"); - public static final SqlOperator RINT = + public static final SqlFunction RINT = adaptMathFunctionToUDF( "rint", ReturnTypes.DOUBLE_FORCE_NULLABLE, NullPolicy.ANY, PPLOperandTypes.NUMERIC) .toUDF("RINT"); - public static final SqlOperator EXPM1 = + public static final SqlFunction EXPM1 = adaptMathFunctionToUDF( "expm1", ReturnTypes.DOUBLE_FORCE_NULLABLE, NullPolicy.ANY, PPLOperandTypes.NUMERIC) .toUDF("EXPM1"); // IP comparing functions - public static final SqlOperator NOT_EQUALS_IP = + public static final SqlFunction NOT_EQUALS_IP = CompareIpFunction.notEquals().toUDF("NOT_EQUALS_IP"); - public static final SqlOperator EQUALS_IP = CompareIpFunction.equals().toUDF("EQUALS_IP"); - public static final SqlOperator GREATER_IP = CompareIpFunction.greater().toUDF("GREATER_IP"); - public static final SqlOperator GTE_IP = CompareIpFunction.greaterOrEquals().toUDF("GTE_IP"); - public static final SqlOperator LESS_IP = CompareIpFunction.less().toUDF("LESS_IP"); - public static final SqlOperator LTE_IP = CompareIpFunction.lessOrEquals().toUDF("LTE_IP"); + public static final SqlFunction EQUALS_IP = CompareIpFunction.equals().toUDF("EQUALS_IP"); + public static final SqlFunction GREATER_IP = CompareIpFunction.greater().toUDF("GREATER_IP"); + public static final SqlFunction GTE_IP = CompareIpFunction.greaterOrEquals().toUDF("GTE_IP"); + public static final SqlFunction LESS_IP = CompareIpFunction.less().toUDF("LESS_IP"); + public static final SqlFunction LTE_IP = CompareIpFunction.lessOrEquals().toUDF("LTE_IP"); // Condition function - public static final SqlOperator EARLIEST = new EarliestFunction().toUDF("EARLIEST"); - public static final SqlOperator LATEST = new LatestFunction().toUDF("LATEST"); + public static final SqlFunction EARLIEST = new EarliestFunction().toUDF("EARLIEST"); + public static final SqlFunction LATEST = new LatestFunction().toUDF("LATEST"); // Datetime function - public static final SqlOperator TIMESTAMP = new TimestampFunction().toUDF("TIMESTAMP"); - public static final SqlOperator DATE = + public static final SqlFunction TIMESTAMP = new TimestampFunction().toUDF("TIMESTAMP"); + public static final SqlFunction DATE = adaptExprMethodToUDF( DateTimeFunctions.class, "exprDate", @@ -180,13 +188,13 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ARG0, PPLOperandTypes.DATE_OR_TIMESTAMP_OR_STRING) .toUDF("DATE"); - public static final SqlOperator YEARWEEK = new YearweekFunction().toUDF("YEARWEEK"); - public static final SqlOperator WEEKDAY = new WeekdayFunction().toUDF("WEEKDAY"); - public static final SqlOperator UNIX_TIMESTAMP = + public static final SqlFunction YEARWEEK = new YearweekFunction().toUDF("YEARWEEK"); + public static final SqlFunction WEEKDAY = new WeekdayFunction().toUDF("WEEKDAY"); + public static final SqlFunction UNIX_TIMESTAMP = new UnixTimestampFunction().toUDF("UNIX_TIMESTAMP"); - public static final SqlOperator STRFTIME = new StrftimeFunction().toUDF("STRFTIME"); - public static final SqlOperator TO_SECONDS = new ToSecondsFunction().toUDF("TO_SECONDS"); - public static final SqlOperator ADDTIME = + public static final SqlFunction STRFTIME = new StrftimeFunction().toUDF("STRFTIME"); + public static final SqlFunction TO_SECONDS = new ToSecondsFunction().toUDF("TO_SECONDS"); + public static final SqlFunction ADDTIME = adaptExprMethodWithPropertiesToUDF( DateTimeFunctions.class, "exprAddTime", @@ -194,7 +202,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.DATETIME_DATETIME) .toUDF("ADDTIME"); - public static final SqlOperator SUBTIME = + public static final SqlFunction SUBTIME = adaptExprMethodWithPropertiesToUDF( DateTimeFunctions.class, "exprSubTime", @@ -202,22 +210,22 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.DATETIME_DATETIME) .toUDF("SUBTIME"); - public static final SqlOperator ADDDATE = new AddSubDateFunction(true).toUDF("ADDDATE"); - public static final SqlOperator SUBDATE = new AddSubDateFunction(false).toUDF("SUBDATE"); - public static final SqlOperator DATE_ADD = new DateAddSubFunction(true).toUDF("DATE_ADD"); - public static final SqlOperator DATE_SUB = new DateAddSubFunction(false).toUDF("DATE_SUB"); - public static final SqlOperator EXTRACT = new ExtractFunction().toUDF("EXTRACT"); - public static final SqlOperator YEAR = new DatePartFunction(TimeUnit.YEAR).toUDF("YEAR"); - public static final SqlOperator QUARTER = new DatePartFunction(TimeUnit.QUARTER).toUDF("QUARTER"); - public static final SqlOperator MONTH = new DatePartFunction(TimeUnit.MONTH).toUDF("MONTH"); - public static final SqlOperator DAY = new DatePartFunction(TimeUnit.DAY).toUDF("DAY"); - public static final SqlOperator DAY_OF_WEEK = + public static final SqlFunction ADDDATE = new AddSubDateFunction(true).toUDF("ADDDATE"); + public static final SqlFunction SUBDATE = new AddSubDateFunction(false).toUDF("SUBDATE"); + public static final SqlFunction DATE_ADD = new DateAddSubFunction(true).toUDF("DATE_ADD"); + public static final SqlFunction DATE_SUB = new DateAddSubFunction(false).toUDF("DATE_SUB"); + public static final SqlFunction EXTRACT = new ExtractFunction().toUDF("EXTRACT"); + public static final SqlFunction YEAR = new DatePartFunction(TimeUnit.YEAR).toUDF("YEAR"); + public static final SqlFunction QUARTER = new DatePartFunction(TimeUnit.QUARTER).toUDF("QUARTER"); + public static final SqlFunction MONTH = new DatePartFunction(TimeUnit.MONTH).toUDF("MONTH"); + public static final SqlFunction DAY = new DatePartFunction(TimeUnit.DAY).toUDF("DAY"); + public static final SqlFunction DAY_OF_WEEK = new DatePartFunction(TimeUnit.DOW).toUDF("DAY_OF_WEEK"); - public static final SqlOperator DAY_OF_YEAR = + public static final SqlFunction DAY_OF_YEAR = new DatePartFunction(TimeUnit.DOY).toUDF("DAY_OF_YEAR"); - public static final SqlOperator HOUR = new DatePartFunction(TimeUnit.HOUR).toUDF("HOUR"); - public static final SqlOperator MINUTE = new DatePartFunction(TimeUnit.MINUTE).toUDF("MINUTE"); - public static final SqlOperator MINUTE_OF_DAY = + public static final SqlFunction HOUR = new DatePartFunction(TimeUnit.HOUR).toUDF("HOUR"); + public static final SqlFunction MINUTE = new DatePartFunction(TimeUnit.MINUTE).toUDF("MINUTE"); + public static final SqlFunction MINUTE_OF_DAY = adaptExprMethodToUDF( DateTimeFunctions.class, "exprMinuteOfDay", @@ -225,22 +233,22 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ARG0, PPLOperandTypes.TIME_OR_TIMESTAMP_OR_STRING) .toUDF("MINUTE_OF_DAY"); - public static final SqlOperator SECOND = new DatePartFunction(TimeUnit.SECOND).toUDF("SECOND"); - public static final SqlOperator MICROSECOND = + public static final SqlFunction SECOND = new DatePartFunction(TimeUnit.SECOND).toUDF("SECOND"); + public static final SqlFunction MICROSECOND = new DatePartFunction(TimeUnit.MICROSECOND).toUDF("MICROSECOND"); - public static final SqlOperator NOW = new CurrentFunction(ExprCoreType.TIMESTAMP).toUDF("NOW"); - public static final SqlOperator CURRENT_TIME = + public static final SqlFunction NOW = new CurrentFunction(ExprCoreType.TIMESTAMP).toUDF("NOW"); + public static final SqlFunction CURRENT_TIME = new CurrentFunction(ExprCoreType.TIME).toUDF("CURRENT_TIME"); - public static final SqlOperator CURRENT_DATE = + public static final SqlFunction CURRENT_DATE = new CurrentFunction(ExprCoreType.DATE).toUDF("CURRENT_DATE"); - public static final SqlOperator DATE_FORMAT = + public static final SqlFunction DATE_FORMAT = new FormatFunction(ExprCoreType.DATE).toUDF("DATE_FORMAT"); - public static final SqlOperator TIME_FORMAT = + public static final SqlFunction TIME_FORMAT = new FormatFunction(ExprCoreType.TIME).toUDF("TIME_FORMAT"); - public static final SqlOperator DAYNAME = new PeriodNameFunction(TimeUnit.DAY).toUDF("DAYNAME"); - public static final SqlOperator MONTHNAME = + public static final SqlFunction DAYNAME = new PeriodNameFunction(TimeUnit.DAY).toUDF("DAYNAME"); + public static final SqlFunction MONTHNAME = new PeriodNameFunction(TimeUnit.MONTH).toUDF("MONTHNAME"); - public static final SqlOperator CONVERT_TZ = + public static final SqlFunction CONVERT_TZ = adaptExprMethodToUDF( DateTimeFunctions.class, "exprConvertTZ", @@ -248,7 +256,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.TIMESTAMP_OR_STRING_STRING_STRING) .toUDF("CONVERT_TZ"); - public static final SqlOperator DATEDIFF = + public static final SqlFunction DATEDIFF = adaptExprMethodWithPropertiesToUDF( DateTimeFunctions.class, "exprDateDiff", @@ -256,10 +264,10 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.DATETIME_DATETIME) .toUDF("DATEDIFF"); - public static final SqlOperator TIMESTAMPDIFF = + public static final SqlFunction TIMESTAMPDIFF = new TimestampDiffFunction().toUDF("TIMESTAMPDIFF"); - public static final SqlOperator LAST_DAY = new LastDayFunction().toUDF("LAST_DAY"); - public static final SqlOperator FROM_DAYS = + public static final SqlFunction LAST_DAY = new LastDayFunction().toUDF("LAST_DAY"); + public static final SqlFunction FROM_DAYS = adaptExprMethodToUDF( DateTimeFunctions.class, "exprFromDays", @@ -267,8 +275,8 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.INTEGER) .toUDF("FROM_DAYS"); - public static final SqlOperator FROM_UNIXTIME = new FromUnixTimeFunction().toUDF("FROM_UNIXTIME"); - public static final SqlOperator GET_FORMAT = + public static final SqlFunction FROM_UNIXTIME = new FromUnixTimeFunction().toUDF("FROM_UNIXTIME"); + public static final SqlFunction GET_FORMAT = adaptExprMethodToUDF( DateTimeFunctions.class, "exprGetFormat", @@ -276,7 +284,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.STRING_STRING) .toUDF("GET_FORMAT"); - public static final SqlOperator MAKEDATE = + public static final SqlFunction MAKEDATE = adaptExprMethodToUDF( DateTimeFunctions.class, "exprMakeDate", @@ -284,7 +292,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.NUMERIC_NUMERIC) .toUDF("MAKEDATE"); - public static final SqlOperator MAKETIME = + public static final SqlFunction MAKETIME = adaptExprMethodToUDF( DateTimeFunctions.class, "exprMakeTime", @@ -292,7 +300,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.NUMERIC_NUMERIC_NUMERIC) .toUDF("MAKETIME"); - public static final SqlOperator PERIOD_DIFF = + public static final SqlFunction PERIOD_DIFF = adaptExprMethodToUDF( DateTimeFunctions.class, "exprPeriodDiff", @@ -300,7 +308,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.INTEGER_INTEGER) .toUDF("PERIOD_DIFF"); - public static final SqlOperator PERIOD_ADD = + public static final SqlFunction PERIOD_ADD = adaptExprMethodToUDF( DateTimeFunctions.class, "exprPeriodAdd", @@ -308,7 +316,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.INTEGER_INTEGER) .toUDF("PERIOD_ADD"); - public static final SqlOperator STR_TO_DATE = + public static final SqlFunction STR_TO_DATE = adaptExprMethodWithPropertiesToUDF( DateTimeFunctions.class, "exprStrToDate", @@ -316,9 +324,9 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.STRING_STRING) .toUDF("STR_TO_DATE"); - public static final SqlOperator SYSDATE = new SysdateFunction().toUDF("SYSDATE"); - public static final SqlOperator SEC_TO_TIME = new SecToTimeFunction().toUDF("SEC_TO_TIME"); - public static final SqlOperator TIME = + public static final SqlFunction SYSDATE = new SysdateFunction().toUDF("SYSDATE"); + public static final SqlFunction SEC_TO_TIME = new SecToTimeFunction().toUDF("SEC_TO_TIME"); + public static final SqlFunction TIME = adaptExprMethodToUDF( DateTimeFunctions.class, "exprTime", @@ -328,9 +336,9 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { .toUDF("TIME"); // IP cast function - public static final SqlOperator IP = + public static final SqlFunction IP = new IPFunction().toUDF(UserDefinedFunctionUtils.IP_FUNCTION_NAME); - public static final SqlOperator TIME_TO_SEC = + public static final SqlFunction TIME_TO_SEC = adaptExprMethodToUDF( DateTimeFunctions.class, "exprTimeToSec", @@ -338,7 +346,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ARG0, PPLOperandTypes.TIME_OR_TIMESTAMP_OR_STRING) .toUDF("TIME_TO_SEC"); - public static final SqlOperator TIMEDIFF = + public static final SqlFunction TIMEDIFF = UserDefinedFunctionUtils.adaptExprMethodToUDF( DateTimeFunctions.class, "exprTimeDiff", @@ -346,8 +354,8 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.TIME_TIME) .toUDF("TIME_DIFF"); - public static final SqlOperator TIMESTAMPADD = new TimestampAddFunction().toUDF("TIMESTAMPADD"); - public static final SqlOperator TO_DAYS = + public static final SqlFunction TIMESTAMPADD = new TimestampAddFunction().toUDF("TIMESTAMPADD"); + public static final SqlFunction TO_DAYS = adaptExprMethodToUDF( DateTimeFunctions.class, "exprToDays", @@ -355,8 +363,8 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ARG0, PPLOperandTypes.DATE_OR_TIMESTAMP_OR_STRING) .toUDF("TO_DAYS"); - public static final SqlOperator DATETIME = new DatetimeFunction().toUDF("DATETIME"); - public static final SqlOperator UTC_DATE = + public static final SqlFunction DATETIME = new DatetimeFunction().toUDF("DATETIME"); + public static final SqlFunction UTC_DATE = adaptExprMethodWithPropertiesToUDF( DateTimeFunctions.class, "exprUtcDate", @@ -364,7 +372,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.NONE, PPLOperandTypes.NONE) .toUDF("UTC_DATE"); - public static final SqlOperator UTC_TIME = + public static final SqlFunction UTC_TIME = adaptExprMethodWithPropertiesToUDF( DateTimeFunctions.class, "exprUtcTime", @@ -372,7 +380,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.NONE, PPLOperandTypes.NONE) .toUDF("UTC_TIME"); - public static final SqlOperator UTC_TIMESTAMP = + public static final SqlFunction UTC_TIMESTAMP = adaptExprMethodWithPropertiesToUDF( DateTimeFunctions.class, "exprUtcTimestamp", @@ -380,61 +388,61 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.NONE, PPLOperandTypes.NONE) .toUDF("UTC_TIMESTAMP"); - public static final SqlOperator WEEK = new WeekFunction().toUDF("WEEK"); - public static final SqlOperator GROK = new ParseFunction().toUDF("GROK"); + public static final SqlFunction WEEK = new WeekFunction().toUDF("WEEK"); + public static final SqlFunction GROK = new ParseFunction().toUDF("GROK"); // TODO: Figure out if there is other option to perform multiple group match in Calcite // For now, keep V2's regexExpression logic to avoid breaking change - public static final SqlOperator PARSE = new ParseFunction().toUDF("PARSE"); - public static final SqlOperator PATTERN_PARSER = + public static final SqlFunction PARSE = new ParseFunction().toUDF("PARSE"); + public static final SqlFunction PATTERN_PARSER = new PatternParserFunctionImpl().toUDF("PATTERN_PARSER"); - public static final SqlOperator FORALL = new ForallFunctionImpl().toUDF("forall"); - public static final SqlOperator EXISTS = new ExistsFunctionImpl().toUDF("exists"); - public static final SqlOperator ARRAY = new ArrayFunctionImpl().toUDF("array"); - public static final SqlOperator MAP_APPEND = new MapAppendFunctionImpl().toUDF("map_append"); - public static final SqlOperator MAP_REMOVE = new MapRemoveFunctionImpl().toUDF("MAP_REMOVE"); - public static final SqlOperator MVAPPEND = new MVAppendFunctionImpl().toUDF("mvappend"); - public static final SqlOperator MVZIP = new MVZipFunctionImpl().toUDF("mvzip"); - public static final SqlOperator MVFIND = new MVFindFunctionImpl().toUDF("mvfind"); - public static final SqlOperator FILTER = new FilterFunctionImpl().toUDF("filter"); - public static final SqlOperator TRANSFORM = new TransformFunctionImpl().toUDF("transform"); - public static final SqlOperator REDUCE = new ReduceFunctionImpl().toUDF("reduce"); + public static final SqlFunction FORALL = new ForallFunctionImpl().toUDF("forall"); + public static final SqlFunction EXISTS = new ExistsFunctionImpl().toUDF("exists"); + public static final SqlFunction ARRAY = new ArrayFunctionImpl().toUDF("array"); + public static final SqlFunction MAP_APPEND = new MapAppendFunctionImpl().toUDF("map_append"); + public static final SqlFunction MAP_REMOVE = new MapRemoveFunctionImpl().toUDF("MAP_REMOVE"); + public static final SqlFunction MVAPPEND = new MVAppendFunctionImpl().toUDF("mvappend"); + public static final SqlFunction MVZIP = new MVZipFunctionImpl().toUDF("mvzip"); + public static final SqlFunction MVFIND = new MVFindFunctionImpl().toUDF("mvfind"); + public static final SqlFunction FILTER = new FilterFunctionImpl().toUDF("filter"); + public static final SqlFunction TRANSFORM = new TransformFunctionImpl().toUDF("transform"); + public static final SqlFunction REDUCE = new ReduceFunctionImpl().toUDF("reduce"); private static final RelevanceQueryFunction RELEVANCE_QUERY_FUNCTION_INSTANCE = new RelevanceQueryFunction(); - public static final SqlOperator MATCH = RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("match"); - public static final SqlOperator MATCH_PHRASE = + public static final SqlFunction MATCH = RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("match"); + public static final SqlFunction MATCH_PHRASE = RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("match_phrase"); - public static final SqlOperator MATCH_BOOL_PREFIX = + public static final SqlFunction MATCH_BOOL_PREFIX = RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("match_bool_prefix"); - public static final SqlOperator MATCH_PHRASE_PREFIX = + public static final SqlFunction MATCH_PHRASE_PREFIX = RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("match_phrase_prefix"); - public static final SqlOperator SIMPLE_QUERY_STRING = + public static final SqlFunction SIMPLE_QUERY_STRING = RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("simple_query_string", false); - public static final SqlOperator QUERY_STRING = + public static final SqlFunction QUERY_STRING = RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("query_string", false); - public static final SqlOperator MULTI_MATCH = + public static final SqlFunction MULTI_MATCH = RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("multi_match", false); - public static final SqlOperator NUMBER_TO_STRING = + public static final SqlFunction NUMBER_TO_STRING = new NumberToStringFunction().toUDF("NUMBER_TO_STRING"); - public static final SqlOperator TONUMBER = new ToNumberFunction().toUDF("TONUMBER"); - public static final SqlOperator TOSTRING = new ToStringFunction().toUDF("TOSTRING"); - public static final SqlOperator WIDTH_BUCKET = + public static final SqlFunction TONUMBER = new ToNumberFunction().toUDF("TONUMBER"); + public static final SqlFunction TOSTRING = new ToStringFunction().toUDF("TOSTRING"); + public static final SqlFunction WIDTH_BUCKET = new org.opensearch.sql.expression.function.udf.binning.WidthBucketFunction() .toUDF("WIDTH_BUCKET"); - public static final SqlOperator SPAN_BUCKET = + public static final SqlFunction SPAN_BUCKET = new org.opensearch.sql.expression.function.udf.binning.SpanBucketFunction() .toUDF("SPAN_BUCKET"); - public static final SqlOperator MINSPAN_BUCKET = + public static final SqlFunction MINSPAN_BUCKET = new org.opensearch.sql.expression.function.udf.binning.MinspanBucketFunction() .toUDF("MINSPAN_BUCKET"); - public static final SqlOperator RANGE_BUCKET = + public static final SqlFunction RANGE_BUCKET = new org.opensearch.sql.expression.function.udf.binning.RangeBucketFunction() .toUDF("RANGE_BUCKET"); - public static final SqlOperator REX_EXTRACT = new RexExtractFunction().toUDF("REX_EXTRACT"); - public static final SqlOperator REX_EXTRACT_MULTI = + public static final SqlFunction REX_EXTRACT = new RexExtractFunction().toUDF("REX_EXTRACT"); + public static final SqlFunction REX_EXTRACT_MULTI = new RexExtractMultiFunction().toUDF("REX_EXTRACT_MULTI"); - public static final SqlOperator REX_OFFSET = new RexOffsetFunction().toUDF("REX_OFFSET"); + public static final SqlFunction REX_OFFSET = new RexOffsetFunction().toUDF("REX_OFFSET"); // Aggregation functions public static final SqlAggFunction AVG_NULLABLE = new NullableSqlAvgAggFunction(SqlKind.AVG); @@ -463,25 +471,47 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { PercentileApproxFunction.class, "percentile_approx", ReturnTypes.ARG0_FORCE_NULLABLE, - PPLOperandTypes.NUMERIC_NUMERIC_OPTIONAL_NUMERIC); + PPLOperandTypes.NUMERIC_NUMERIC_OPTIONAL_NUMERIC_SYMBOL); public static final SqlAggFunction INTERNAL_PATTERN = createUserDefinedAggFunction( LogPatternAggFunction.class, "pattern", ReturnTypes.explicit(UserDefinedFunctionUtils.nullablePatternAggList), - null); + UDFOperandMetadata.wrap( + OperandTypes.VARIADIC)); // operand types of patterns are very flexible public static final SqlAggFunction LIST = createUserDefinedAggFunction( - ListAggFunction.class, "LIST", PPLReturnTypes.STRING_ARRAY, PPLOperandTypes.ANY_SCALAR); + ListAggFunction.class, "LIST", PPLReturnTypes.STRING_ARRAY, PPLOperandTypes.SCALAR); public static final SqlAggFunction VALUES = createUserDefinedAggFunction( ValuesAggFunction.class, "VALUES", PPLReturnTypes.STRING_ARRAY, - PPLOperandTypes.ANY_SCALAR_OPTIONAL_INTEGER); + PPLOperandTypes.SCALAR_OPTIONAL_INTEGER); - public static final SqlOperator ENHANCED_COALESCE = - new EnhancedCoalesceFunction().toUDF("COALESCE"); + public static final SqlFunction ATAN = + new SqlFunction( + "ATAN", + SqlKind.OTHER_FUNCTION, + ReturnTypes.DOUBLE_NULLABLE, + null, + OperandTypes.NUMERIC_OPTIONAL_NUMERIC, + SqlFunctionCategory.NUMERIC) { + @Override + public SqlNode rewriteCall(SqlValidator validator, SqlCall call) { + SqlOperator op = + call.getOperandList().size() == 2 + ? SqlStdOperatorTable.ATAN2 + : SqlStdOperatorTable.ATAN; + ((SqlBasicCall) call).setOperator(op); + return call; + } + }; + // SPARK dialect is not included in lookup table to resolve overrides issues (e.g. reverse + // function won't work if spark is included because there are multiple overrides of reverse, and + // it will choose none of them in the end.) Therefore, SPARK functions used are explicitly + // declared here for lookup. + public static final SqlFunction REGEXP = SqlLibraryOperators.REGEXP; /** * Returns the PPL specific operator table, creating it if necessary. @@ -498,7 +528,7 @@ public static PPLBuiltinOperators instance() { * implementor could be substituted by a single method. */ private static Expression invokeCalciteImplementor( - RexToLixTranslator translator, RexCall call, SqlOperator operator, Expression field) + RexToLixTranslator translator, RexCall call, SqlFunction operator, Expression field) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { RexCallImplementor rexCallImplementor = RexImpTable.INSTANCE.get(operator); Method method = diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 205f3a0f2e1..1dbf2fad18a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -250,19 +250,17 @@ import com.google.common.collect.ImmutableMap; import java.math.BigDecimal; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; -import java.util.StringJoiner; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; import java.util.stream.Stream; -import javax.annotation.Nullable; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexLambda; @@ -273,25 +271,15 @@ import org.apache.calcite.sql.fun.SqlLibraryOperators; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.fun.SqlTrimFunction.Flag; -import org.apache.calcite.sql.type.CompositeOperandTypeChecker; -import org.apache.calcite.sql.type.FamilyOperandTypeChecker; -import org.apache.calcite.sql.type.ImplicitCastOperandTypeChecker; -import org.apache.calcite.sql.type.OperandTypes; -import org.apache.calcite.sql.type.SameOperandTypeChecker; -import org.apache.calcite.sql.type.SqlOperandTypeChecker; -import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction; -import org.apache.calcite.sql.validate.SqlUserDefinedFunction; import org.apache.calcite.tools.RelBuilder; -import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.sql.calcite.CalcitePlanContext; -import org.opensearch.sql.calcite.utils.PPLOperandTypes; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.calcite.utils.PlanUtils; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; -import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.executor.QueryType; import org.opensearch.sql.expression.function.CollectionUDF.MVIndexFunctionImp; @@ -363,43 +351,39 @@ default RexNode resolve(RexBuilder builder, RexNode... args) { * implementations are independent of any specific data storage, should be registered here * internally. */ - private final ImmutableMap>> - functionRegistry; + private final ImmutableMap functionRegistry; /** * The external function registry. Functions whose implementations depend on a specific data * engine should be registered here. This reduces coupling between the core module and particular * storage backends. */ - private final Map>> - externalFunctionRegistry; + private final Map externalFunctionRegistry; /** * The registry for built-in agg functions. Agg Functions defined by the PPL specification, whose * implementations are independent of any specific data storage, should be registered here * internally. */ - private final ImmutableMap> - aggFunctionRegistry; + private final ImmutableMap aggFunctionRegistry; /** * The external agg function registry. Agg Functions whose implementations depend on a specific * data engine should be registered here. This reduces coupling between the core module and * particular storage backends. */ - private final Map> - aggExternalFunctionRegistry; + private final Map aggExternalFunctionRegistry; private PPLFuncImpTable(Builder builder, AggBuilder aggBuilder) { - final ImmutableMap.Builder>> - mapBuilder = ImmutableMap.builder(); - builder.map.forEach((k, v) -> mapBuilder.put(k, List.copyOf(v))); + final ImmutableMap.Builder mapBuilder = + ImmutableMap.builder(); + mapBuilder.putAll(builder.map); this.functionRegistry = ImmutableMap.copyOf(mapBuilder.build()); this.externalFunctionRegistry = new ConcurrentHashMap<>(); - final ImmutableMap.Builder> - aggMapBuilder = ImmutableMap.builder(); - aggBuilder.map.forEach(aggMapBuilder::put); + final ImmutableMap.Builder aggMapBuilder = + ImmutableMap.builder(); + aggMapBuilder.putAll(aggBuilder.map); this.aggFunctionRegistry = ImmutableMap.copyOf(aggMapBuilder.build()); this.aggExternalFunctionRegistry = new ConcurrentHashMap<>(); } @@ -411,20 +395,11 @@ private PPLFuncImpTable(Builder builder, AggBuilder aggBuilder) { * @param operator a SqlOperator representing an externally implemented function */ public void registerExternalOperator(BuiltinFunctionName functionName, SqlOperator operator) { - PPLTypeChecker typeChecker = - wrapSqlOperandTypeChecker( - operator.getOperandTypeChecker(), - functionName.name(), - operator instanceof SqlUserDefinedFunction); - CalciteFuncSignature signature = new CalciteFuncSignature(functionName.getName(), typeChecker); - externalFunctionRegistry.compute( - functionName, - (name, existingList) -> { - List> list = - existingList == null ? new ArrayList<>() : new ArrayList<>(existingList); - list.add(Pair.of(signature, (builder, args) -> builder.makeCall(operator, args))); - return list; - }); + if (externalFunctionRegistry.containsKey(functionName)) { + logger.warn( + String.format(Locale.ROOT, "Function %s is registered multiple times", functionName)); + } + externalFunctionRegistry.put(functionName, (builder, args) -> builder.makeCall(operator, args)); } /** @@ -436,23 +411,16 @@ public void registerExternalOperator(BuiltinFunctionName functionName, SqlOperat */ public void registerExternalAggOperator( BuiltinFunctionName functionName, SqlUserDefinedAggFunction aggFunction) { - PPLTypeChecker typeChecker = - wrapSqlOperandTypeChecker(aggFunction.getOperandTypeChecker(), functionName.name(), true); - CalciteFuncSignature signature = new CalciteFuncSignature(functionName.getName(), typeChecker); + if (aggExternalFunctionRegistry.containsKey(functionName)) { + logger.warn( + String.format( + Locale.ROOT, "Aggregate function %s is registered multiple times", functionName)); + } AggHandler handler = (distinct, field, argList, ctx) -> UserDefinedFunctionUtils.makeAggregateCall( aggFunction, List.of(field), argList, ctx.relBuilder); - aggExternalFunctionRegistry.put(functionName, Pair.of(signature, handler)); - } - - public List validateAggFunctionSignature( - BuiltinFunctionName functionName, - RexNode field, - List argList, - RexBuilder rexBuilder) { - var implementation = getImplementation(functionName); - return validateFunctionArgs(implementation, functionName, field, argList, rexBuilder); + aggExternalFunctionRegistry.put(functionName, handler); } public RelBuilder.AggCall resolveAgg( @@ -462,62 +430,10 @@ public RelBuilder.AggCall resolveAgg( List argList, CalcitePlanContext context) { var implementation = getImplementation(functionName); - - // Validation is done based on original argument types to generate error from user perspective. - List nodes = - validateFunctionArgs(implementation, functionName, field, argList, context.rexBuilder); - - var handler = implementation.getValue(); - return nodes != null - ? handler.apply(distinct, nodes.getFirst(), nodes.subList(1, nodes.size()), context) - : handler.apply(distinct, field, argList, context); - } - - static List validateFunctionArgs( - Pair implementation, - BuiltinFunctionName functionName, - RexNode field, - List argList, - RexBuilder rexBuilder) { - CalciteFuncSignature signature = implementation.getKey(); - - List argTypes = new ArrayList<>(); - if (field != null) { - argTypes.add(field.getType()); - } - - // Currently only PERCENTILE_APPROX, TAKE, EARLIEST, and LATEST have additional arguments. - // Their additional arguments will always come as a map of - List additionalArgTypes = - argList.stream().map(PlanUtils::derefMapCall).map(RexNode::getType).toList(); - argTypes.addAll(additionalArgTypes); - List coercionNodes = null; - if (!signature.match(functionName.getName(), argTypes)) { - List fields = new ArrayList<>(); - fields.add(field); - fields.addAll(argList); - if (CoercionUtils.hasString(fields)) { - coercionNodes = CoercionUtils.castArguments(rexBuilder, signature.typeChecker(), fields); - } - if (coercionNodes == null) { - String errorMessagePattern = - argTypes.size() <= 1 - ? "Aggregation function %s expects field type {%s}, but got %s" - : "Aggregation function %s expects field type and additional arguments {%s}, but" - + " got %s"; - throw new ExpressionEvaluationException( - String.format( - errorMessagePattern, - functionName, - signature.typeChecker().getAllowedSignatures(), - PlanUtils.getActualSignature(argTypes))); - } - } - return coercionNodes; + return implementation.apply(distinct, field, argList, context); } - private Pair getImplementation( - BuiltinFunctionName functionName) { + private AggHandler getImplementation(BuiltinFunctionName functionName) { var implementation = aggExternalFunctionRegistry.get(functionName); if (implementation == null) { implementation = aggFunctionRegistry.get(functionName); @@ -540,13 +456,12 @@ public RexNode resolve( final RexBuilder builder, final BuiltinFunctionName functionName, RexNode... args) { // Check the external function registry first. This allows the data-storage-dependent // function implementations to override the internal ones with the same name. - List> implementList = - externalFunctionRegistry.get(functionName); - // If the function is not part of the external registry, check the internal registry. - if (implementList == null) { - implementList = functionRegistry.get(functionName); - } - if (implementList == null || implementList.isEmpty()) { + // If the function is not part of the external registry, check the internal registry. + FunctionImp implementation = + externalFunctionRegistry.get(functionName) != null + ? externalFunctionRegistry.get(functionName) + : functionRegistry.get(functionName); + if (implementation == null) { throw new IllegalStateException(String.format("Cannot resolve function: %s", functionName)); } @@ -554,39 +469,7 @@ public RexNode resolve( // For example, the REDUCE function requires the second argument to be cast to the // return type of the lambda function. compulsoryCast(builder, functionName, args); - - List argTypes = Arrays.stream(args).map(RexNode::getType).toList(); - try { - for (Map.Entry implement : implementList) { - if (implement.getKey().match(functionName.getName(), argTypes)) { - return implement.getValue().resolve(builder, args); - } - } - - // If no implementation found with exact match, try to cast arguments to match the - // signatures. - RexNode coerced = resolveWithCoercion(builder, functionName, implementList, args); - if (coerced != null) { - return coerced; - } - } catch (Exception e) { - throw new ExpressionEvaluationException( - String.format( - "Cannot resolve function: %s, arguments: %s, caused by: %s", - functionName, PlanUtils.getActualSignature(argTypes), e.getMessage()), - e); - } - StringJoiner allowedSignatures = new StringJoiner(","); - for (var implement : implementList) { - String signature = implement.getKey().typeChecker().getAllowedSignatures(); - if (!signature.isEmpty()) { - allowedSignatures.add(signature); - } - } - throw new ExpressionEvaluationException( - String.format( - "%s function expects {%s}, but got %s", - functionName, allowedSignatures, PlanUtils.getActualSignature(argTypes))); + return implementation.resolve(builder, args); } /** @@ -609,91 +492,25 @@ private void compulsoryCast( } } - private @Nullable RexNode resolveWithCoercion( - final RexBuilder builder, - final BuiltinFunctionName functionName, - List> implementList, - RexNode... args) { - if (BuiltinFunctionName.COMPARATORS.contains(functionName)) { - for (Map.Entry implement : implementList) { - var widenedArgs = CoercionUtils.widenArguments(builder, List.of(args)); - if (widenedArgs != null) { - boolean matchSignature = - implement - .getKey() - .typeChecker() - .checkOperandTypes(widenedArgs.stream().map(RexNode::getType).toList()); - if (matchSignature) { - return implement.getValue().resolve(builder, widenedArgs.toArray(new RexNode[0])); - } - } - } - } else { - for (Map.Entry implement : implementList) { - var signature = implement.getKey(); - var castedArgs = - CoercionUtils.castArguments(builder, signature.typeChecker(), List.of(args)); - if (castedArgs != null) { - // If compatible function is found, replace the original RexNode with cast node - // TODO: check - this is a return-once-found implementation, rest possible combinations - // will be skipped. - // Maybe can be improved to return the best match? E.g. convert to timestamp when date, - // time, and timestamp are all possible. - return implement.getValue().resolve(builder, castedArgs.toArray(new RexNode[0])); - } - } - } - return null; - } - @SuppressWarnings({"UnusedReturnValue", "SameParameterValue"}) private abstract static class AbstractBuilder { /** Maps an operator to an implementation. */ - abstract void register( - BuiltinFunctionName functionName, FunctionImp functionImp, PPLTypeChecker typeChecker); + abstract void register(BuiltinFunctionName functionName, FunctionImp functionImp); /** - * Register one or multiple operators under a single function name. This allows function - * overloading based on operand types. + * Registers an operator for a built-in function name. * - *

When a function is called, the system will try each registered operator in sequence, - * checking if the provided arguments match the operator's type requirements. The first operator - * whose type checker accepts the arguments will be used to execute the function. + *

Each function name can only be registered to one operator. Use {@code + * register(BuiltinFunctionName, FunctionImp)} to dynamically register a built-in function name + * to different operators based on argument count or types if override is desired. * * @param functionName the built-in function name under which to register the operators - * @param operators the operators to associate with this function name, tried in sequence until - * one matches the argument types during resolution + * @param operator the operator to associate with this function name */ - protected void registerOperator(BuiltinFunctionName functionName, SqlOperator... operators) { - for (SqlOperator operator : operators) { - SqlOperandTypeChecker typeChecker; - if (operator instanceof SqlUserDefinedFunction udfOperator) { - typeChecker = extractTypeCheckerFromUDF(udfOperator); - } else { - typeChecker = operator.getOperandTypeChecker(); - } - PPLTypeChecker pplTypeChecker = - wrapSqlOperandTypeChecker( - typeChecker, operator.getName(), operator instanceof SqlUserDefinedFunction); - registerOperator(functionName, operator, pplTypeChecker); - } - } - - /** - * Registers an operator for a built-in function name with a specified {@link PPLTypeChecker}. - * This allows custom type checking logic to be associated with the operator. - * - * @param functionName the built-in function name - * @param operator the SQL operator to register - * @param typeChecker the type checker to use for validating argument types - */ - protected void registerOperator( - BuiltinFunctionName functionName, SqlOperator operator, PPLTypeChecker typeChecker) { + protected void registerOperator(BuiltinFunctionName functionName, SqlOperator operator) { register( - functionName, - (RexBuilder builder, RexNode... args) -> builder.makeCall(operator, args), - typeChecker); + functionName, (RexBuilder builder, RexNode... args) -> builder.makeCall(operator, args)); } protected void registerDivideFunction(BuiltinFunctionName functionName) { @@ -706,39 +523,24 @@ protected void registerDivideFunction(BuiltinFunctionName functionName) { ? PPLBuiltinOperators.DIVIDE : SqlLibraryOperators.SAFE_DIVIDE; return builder.makeCall(operator, left, right); - }, - PPLTypeChecker.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); + }); } void populate() { // register operators for comparison - registerOperator(NOTEQUAL, PPLBuiltinOperators.NOT_EQUALS_IP, SqlStdOperatorTable.NOT_EQUALS); - registerOperator(EQUAL, PPLBuiltinOperators.EQUALS_IP, SqlStdOperatorTable.EQUALS); - registerOperator(GREATER, PPLBuiltinOperators.GREATER_IP, SqlStdOperatorTable.GREATER_THAN); - registerOperator(GTE, PPLBuiltinOperators.GTE_IP, SqlStdOperatorTable.GREATER_THAN_OR_EQUAL); - registerOperator(LESS, PPLBuiltinOperators.LESS_IP, SqlStdOperatorTable.LESS_THAN); - registerOperator(LTE, PPLBuiltinOperators.LTE_IP, SqlStdOperatorTable.LESS_THAN_OR_EQUAL); + registerOperator(NOTEQUAL, SqlStdOperatorTable.NOT_EQUALS); + registerOperator(EQUAL, SqlStdOperatorTable.EQUALS); + registerOperator(GREATER, SqlStdOperatorTable.GREATER_THAN); + registerOperator(GTE, SqlStdOperatorTable.GREATER_THAN_OR_EQUAL); + registerOperator(LESS, SqlStdOperatorTable.LESS_THAN); + registerOperator(LTE, SqlStdOperatorTable.LESS_THAN_OR_EQUAL); // Register std operator registerOperator(AND, SqlStdOperatorTable.AND); registerOperator(OR, SqlStdOperatorTable.OR); registerOperator(NOT, SqlStdOperatorTable.NOT); - - // Register ADDFUNCTION for numeric addition only - registerOperator(ADDFUNCTION, SqlStdOperatorTable.PLUS); - registerOperator( - SUBTRACTFUNCTION, - SqlStdOperatorTable.MINUS, - PPLTypeChecker.wrapFamily((FamilyOperandTypeChecker) OperandTypes.NUMERIC_NUMERIC)); - registerOperator( - SUBTRACT, - SqlStdOperatorTable.MINUS, - PPLTypeChecker.wrapFamily((FamilyOperandTypeChecker) OperandTypes.NUMERIC_NUMERIC)); - // Add DATETIME-DATETIME variant for timestamp binning support - registerOperator( - SUBTRACT, - SqlStdOperatorTable.MINUS, - PPLTypeChecker.family(SqlTypeFamily.DATETIME, SqlTypeFamily.DATETIME)); + registerOperator(SUBTRACTFUNCTION, SqlStdOperatorTable.MINUS); + registerOperator(SUBTRACT, SqlStdOperatorTable.MINUS); registerOperator(MULTIPLY, SqlStdOperatorTable.MULTIPLY); registerOperator(MULTIPLYFUNCTION, SqlStdOperatorTable.MULTIPLY); registerOperator(TRUNCATE, SqlStdOperatorTable.TRUNCATE); @@ -787,46 +589,20 @@ void populate() { } } return builder.makeCall(SqlLibraryOperators.REGEXP_REPLACE_3, args); - }, - wrapSqlOperandTypeChecker( - SqlLibraryOperators.REGEXP_REPLACE_3.getOperandTypeChecker(), REPLACE.name(), false)); + }); registerOperator(UPPER, SqlStdOperatorTable.UPPER); registerOperator(ABS, SqlStdOperatorTable.ABS); registerOperator(ACOS, SqlStdOperatorTable.ACOS); registerOperator(ASIN, SqlStdOperatorTable.ASIN); - registerOperator(ATAN, SqlStdOperatorTable.ATAN); + registerOperator(ATAN, PPLBuiltinOperators.ATAN); registerOperator(ATAN2, SqlStdOperatorTable.ATAN2); - // TODO, workaround to support sequence CompositeOperandTypeChecker. - registerOperator( - CEIL, - SqlStdOperatorTable.CEIL, - PPLTypeChecker.wrapComposite( - (CompositeOperandTypeChecker) - OperandTypes.NUMERIC_OR_INTERVAL.or( - OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.ANY)), - false)); - // TODO, workaround to support sequence CompositeOperandTypeChecker. - registerOperator( - CEILING, - SqlStdOperatorTable.CEIL, - PPLTypeChecker.wrapComposite( - (CompositeOperandTypeChecker) - OperandTypes.NUMERIC_OR_INTERVAL.or( - OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.ANY)), - false)); + registerOperator(CEIL, SqlStdOperatorTable.CEIL); + registerOperator(CEILING, SqlStdOperatorTable.CEIL); registerOperator(COS, SqlStdOperatorTable.COS); registerOperator(COT, SqlStdOperatorTable.COT); registerOperator(DEGREES, SqlStdOperatorTable.DEGREES); registerOperator(EXP, SqlStdOperatorTable.EXP); - // TODO, workaround to support sequence CompositeOperandTypeChecker. - registerOperator( - FLOOR, - SqlStdOperatorTable.FLOOR, - PPLTypeChecker.wrapComposite( - (CompositeOperandTypeChecker) - OperandTypes.NUMERIC_OR_INTERVAL.or( - OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.ANY)), - false)); + registerOperator(FLOOR, SqlStdOperatorTable.FLOOR); registerOperator(LN, SqlStdOperatorTable.LN); registerOperator(LOG10, SqlStdOperatorTable.LOG10); registerOperator(PI, SqlStdOperatorTable.PI); @@ -834,15 +610,7 @@ void populate() { registerOperator(POWER, SqlStdOperatorTable.POWER); registerOperator(RADIANS, SqlStdOperatorTable.RADIANS); registerOperator(RAND, SqlStdOperatorTable.RAND); - // TODO, workaround to support sequence CompositeOperandTypeChecker. - registerOperator( - ROUND, - SqlStdOperatorTable.ROUND, - PPLTypeChecker.wrapComposite( - (CompositeOperandTypeChecker) - OperandTypes.NUMERIC.or( - OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.INTEGER)), - false)); + registerOperator(ROUND, SqlStdOperatorTable.ROUND); registerOperator(SIGN, SqlStdOperatorTable.SIGN); registerOperator(SIGNUM, SqlStdOperatorTable.SIGN); registerOperator(SIN, SqlStdOperatorTable.SIN); @@ -851,17 +619,17 @@ void populate() { registerOperator(IFNULL, SqlStdOperatorTable.COALESCE); registerOperator(EARLIEST, PPLBuiltinOperators.EARLIEST); registerOperator(LATEST, PPLBuiltinOperators.LATEST); - registerOperator(COALESCE, PPLBuiltinOperators.ENHANCED_COALESCE); + registerOperator(COALESCE, SqlStdOperatorTable.COALESCE); // Register library operator - registerOperator(REGEXP, SqlLibraryOperators.REGEXP); + registerOperator(REGEXP, PPLBuiltinOperators.REGEXP); registerOperator(REGEXP_MATCH, SqlLibraryOperators.REGEXP_CONTAINS); registerOperator(CONCAT, SqlLibraryOperators.CONCAT_FUNCTION); registerOperator(CONCAT_WS, SqlLibraryOperators.CONCAT_WS); - registerOperator(CONCAT_WS, SqlLibraryOperators.CONCAT_WS); registerOperator(REVERSE, SqlLibraryOperators.REVERSE); registerOperator(RIGHT, SqlLibraryOperators.RIGHT); registerOperator(LEFT, SqlLibraryOperators.LEFT); + registerOperator(LOG, SqlLibraryOperators.LOG_MYSQL); registerOperator(LOG2, SqlLibraryOperators.LOG2); registerOperator(MD5, SqlLibraryOperators.MD5); registerOperator(SHA1, SqlLibraryOperators.SHA1); @@ -981,21 +749,22 @@ void populate() { registerOperator(INTERNAL_PATTERN_PARSER, PPLBuiltinOperators.PATTERN_PARSER); registerOperator(TONUMBER, PPLBuiltinOperators.TONUMBER); - registerOperator(TOSTRING, PPLBuiltinOperators.TOSTRING); register( TOSTRING, - (FunctionImp1) - (builder, source) -> - builder.makeCast(TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR, true), source), - PPLTypeChecker.family(SqlTypeFamily.ANY)); + (builder, args) -> { + if (args.length == 1) { + return builder.makeCast( + TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR, true), args[0]); + } + return builder.makeCall(PPLBuiltinOperators.TOSTRING, args); + }); // Register MVJOIN to use Calcite's ARRAY_JOIN register( MVJOIN, (FunctionImp2) (builder, array, delimiter) -> - builder.makeCall(SqlLibraryOperators.ARRAY_JOIN, array, delimiter), - PPLTypeChecker.family(SqlTypeFamily.ARRAY, SqlTypeFamily.CHARACTER)); + builder.makeCall(SqlLibraryOperators.ARRAY_JOIN, array, delimiter)); // Register SPLIT with custom logic for empty delimiter // Case 1: Delimiter is not empty string, use SPLIT @@ -1022,20 +791,10 @@ void populate() { // CASE WHEN isEmptyDelimiter THEN splitChars ELSE normalSplit END return builder.makeCall( SqlStdOperatorTable.CASE, isEmptyDelimiter, splitChars, normalSplit); - }, - PPLTypeChecker.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER)); + }); // Register MVINDEX to use Calcite's ITEM/ARRAY_SLICE with index normalization - register( - MVINDEX, - new MVIndexFunctionImp(), - PPLTypeChecker.wrapComposite( - (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.ARRAY, SqlTypeFamily.INTEGER) - .or( - OperandTypes.family( - SqlTypeFamily.ARRAY, SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER)), - false)); + register(MVINDEX, new MVIndexFunctionImp()); registerOperator(ARRAY, PPLBuiltinOperators.ARRAY); registerOperator(MVAPPEND, PPLBuiltinOperators.MVAPPEND); @@ -1061,16 +820,14 @@ void populate() { builder.makeCall( SqlStdOperatorTable.JSON_ARRAY, Stream.concat(Stream.of(builder.makeFlag(NULL_ON_NULL)), Arrays.stream(args)) - .toArray(RexNode[]::new))), - null); + .toArray(RexNode[]::new)))); register( JSON_OBJECT, ((builder, args) -> builder.makeCall( SqlStdOperatorTable.JSON_OBJECT, Stream.concat(Stream.of(builder.makeFlag(NULL_ON_NULL)), Arrays.stream(args)) - .toArray(RexNode[]::new))), - null); + .toArray(RexNode[]::new)))); registerOperator(JSON, PPLBuiltinOperators.JSON); registerOperator(JSON_ARRAY_LENGTH, PPLBuiltinOperators.JSON_ARRAY_LENGTH); registerOperator(JSON_EXTRACT, PPLBuiltinOperators.JSON_EXTRACT); @@ -1082,53 +839,24 @@ void populate() { registerOperator(JSON_EXTEND, PPLBuiltinOperators.JSON_EXTEND); registerOperator(JSON_EXTRACT_ALL, PPLBuiltinOperators.JSON_EXTRACT_ALL); // internal - // Register operators with a different type checker - - // Register ADD (+ symbol) for string concatenation - // Replaced type checker since CONCAT also supports array concatenation - registerOperator( - ADD, - SqlStdOperatorTable.CONCAT, - PPLTypeChecker.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER)); - // Register ADD (+ symbol) for numeric addition - // Replace type checker since PLUS also supports binary addition - registerOperator( - ADD, - SqlStdOperatorTable.PLUS, - PPLTypeChecker.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); - // Replace with a custom CompositeOperandTypeChecker to check both operands as - // SqlStdOperatorTable.ITEM.getOperandTypeChecker() checks only the first - // operand instead - // of all operands. - registerOperator( - INTERNAL_ITEM, - SqlStdOperatorTable.ITEM, - PPLTypeChecker.wrapComposite( - (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.ARRAY, SqlTypeFamily.INTEGER) - .or(OperandTypes.family(SqlTypeFamily.MAP, SqlTypeFamily.ANY)), - false)); - registerOperator( - XOR, - SqlStdOperatorTable.NOT_EQUALS, - PPLTypeChecker.family(SqlTypeFamily.BOOLEAN, SqlTypeFamily.BOOLEAN)); - // SqlStdOperatorTable.CASE.getOperandTypeChecker is null. We manually create a type checker - // for it. The second and third operands are required to be of the same type. If not, it will - // throw an IllegalArgumentException with information Can't find leastRestrictive type - registerOperator( - IF, - SqlStdOperatorTable.CASE, - PPLTypeChecker.family(SqlTypeFamily.BOOLEAN, SqlTypeFamily.ANY, SqlTypeFamily.ANY)); - // Re-define the type checker for is not null, is present, and is null since - // their original type checker ANY isn't compatible with struct types. - registerOperator( - IS_NOT_NULL, - SqlStdOperatorTable.IS_NOT_NULL, - PPLTypeChecker.family(SqlTypeFamily.IGNORE)); - registerOperator( - IS_PRESENT, SqlStdOperatorTable.IS_NOT_NULL, PPLTypeChecker.family(SqlTypeFamily.IGNORE)); - registerOperator( - IS_NULL, SqlStdOperatorTable.IS_NULL, PPLTypeChecker.family(SqlTypeFamily.IGNORE)); + // Register ADD (+ symbol) for string concatenation and numeric addition + // Not creating PPL builtin operator as it will cause confusion during function resolution + FunctionImp add = + (builder, args) -> { + SqlOperator op = + (Stream.of(args).map(RexNode::getType).allMatch(OpenSearchTypeUtil::isCharacter)) + ? SqlStdOperatorTable.CONCAT + : SqlStdOperatorTable.PLUS; + return builder.makeCall(op, args); + }; + register(ADD, add); + register(ADDFUNCTION, add); + registerOperator(INTERNAL_ITEM, SqlStdOperatorTable.ITEM); + registerOperator(XOR, SqlStdOperatorTable.NOT_EQUALS); + registerOperator(IF, SqlStdOperatorTable.CASE); + registerOperator(IS_NOT_NULL, SqlStdOperatorTable.IS_NOT_NULL); + registerOperator(IS_PRESENT, SqlStdOperatorTable.IS_NOT_NULL); + registerOperator(IS_NULL, SqlStdOperatorTable.IS_NULL); // Register implementation. // Note, make the implementation an individual class if too complex. @@ -1140,8 +868,7 @@ void populate() { SqlStdOperatorTable.TRIM, builder.makeFlag(Flag.BOTH), builder.makeLiteral(" "), - arg), - PPLTypeChecker.family(SqlTypeFamily.CHARACTER)); + arg)); register( LTRIM, @@ -1151,8 +878,7 @@ void populate() { SqlStdOperatorTable.TRIM, builder.makeFlag(Flag.LEADING), builder.makeLiteral(" "), - arg), - PPLTypeChecker.family(SqlTypeFamily.CHARACTER)); + arg)); register( RTRIM, (FunctionImp1) @@ -1161,60 +887,19 @@ void populate() { SqlStdOperatorTable.TRIM, builder.makeFlag(Flag.TRAILING), builder.makeLiteral(" "), - arg), - PPLTypeChecker.family(SqlTypeFamily.CHARACTER)); - registerOperator( - ATAN, - SqlStdOperatorTable.ATAN2, - PPLTypeChecker.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); + arg)); register( STRCMP, (FunctionImp2) - (builder, arg1, arg2) -> builder.makeCall(SqlLibraryOperators.STRCMP, arg2, arg1), - PPLTypeChecker.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER)); - // SqlStdOperatorTable.SUBSTRING.getOperandTypeChecker is null. We manually - // create a type - // checker for it. + (builder, arg1, arg2) -> builder.makeCall(SqlLibraryOperators.STRCMP, arg2, arg1)); register( SUBSTRING, (RexBuilder builder, RexNode... args) -> - builder.makeCall(SqlStdOperatorTable.SUBSTRING, args), - PPLTypeChecker.wrapComposite( - (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER) - .or( - OperandTypes.family( - SqlTypeFamily.CHARACTER, - SqlTypeFamily.INTEGER, - SqlTypeFamily.INTEGER)), - false)); + builder.makeCall(SqlStdOperatorTable.SUBSTRING, args)); register( SUBSTR, (RexBuilder builder, RexNode... args) -> - builder.makeCall(SqlStdOperatorTable.SUBSTRING, args), - PPLTypeChecker.wrapComposite( - (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER) - .or( - OperandTypes.family( - SqlTypeFamily.CHARACTER, - SqlTypeFamily.INTEGER, - SqlTypeFamily.INTEGER)), - false)); - register( - LOG, - (FunctionImp2) - (builder, arg1, arg2) -> builder.makeCall(SqlLibraryOperators.LOG, arg2, arg1), - PPLTypeChecker.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); - register( - LOG, - (FunctionImp1) - (builder, arg) -> - builder.makeCall( - SqlLibraryOperators.LOG, - arg, - builder.makeApproxLiteral(BigDecimal.valueOf(Math.E))), - PPLTypeChecker.family(SqlTypeFamily.NUMERIC)); + builder.makeCall(SqlStdOperatorTable.SUBSTRING, args)); // SqlStdOperatorTable.SQRT is declared but not implemented. The call to SQRT in Calcite is // converted to POWER(x, 0.5). register( @@ -1224,14 +909,12 @@ void populate() { builder.makeCall( SqlStdOperatorTable.POWER, arg, - builder.makeApproxLiteral(BigDecimal.valueOf(0.5))), - PPLTypeChecker.family(SqlTypeFamily.NUMERIC)); + builder.makeApproxLiteral(BigDecimal.valueOf(0.5)))); register( TYPEOF, (FunctionImp1) (builder, arg) -> - builder.makeLiteral(getLegacyTypeName(arg.getType(), QueryType.PPL)), - null); + builder.makeLiteral(getLegacyTypeName(arg.getType(), QueryType.PPL))); register( NULLIF, (FunctionImp2) @@ -1240,8 +923,7 @@ void populate() { SqlStdOperatorTable.CASE, builder.makeCall(SqlStdOperatorTable.EQUALS, arg1, arg2), builder.makeNullLiteral(arg1.getType()), - arg1), - PPLTypeChecker.wrapComparable((SameOperandTypeChecker) OperandTypes.SAME_SAME)); + arg1)); register( IS_EMPTY, (FunctionImp1) @@ -1249,8 +931,7 @@ void populate() { builder.makeCall( SqlStdOperatorTable.OR, builder.makeCall(SqlStdOperatorTable.IS_NULL, arg), - builder.makeCall(SqlStdOperatorTable.IS_EMPTY, arg)), - PPLTypeChecker.family(SqlTypeFamily.ANY)); + builder.makeCall(SqlStdOperatorTable.EQUALS, arg, builder.makeLiteral("")))); register( IS_BLANK, (FunctionImp1) @@ -1259,20 +940,19 @@ void populate() { SqlStdOperatorTable.OR, builder.makeCall(SqlStdOperatorTable.IS_NULL, arg), builder.makeCall( - SqlStdOperatorTable.IS_EMPTY, + SqlStdOperatorTable.EQUALS, builder.makeCall( SqlStdOperatorTable.TRIM, builder.makeFlag(Flag.BOTH), builder.makeLiteral(" "), - arg))), - PPLTypeChecker.family(SqlTypeFamily.ANY)); + arg), + builder.makeLiteral("")))); register( ILIKE, (FunctionImp2) (builder, arg1, arg2) -> builder.makeCall( - SqlLibraryOperators.ILIKE, arg1, arg2, builder.makeLiteral("\\")), - PPLTypeChecker.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING)); + SqlLibraryOperators.ILIKE, arg1, arg2, builder.makeLiteral("\\"))); register( LIKE, (FunctionImp3) @@ -1281,44 +961,35 @@ void populate() { ? builder.makeCall( SqlStdOperatorTable.LIKE, arg1, arg2, builder.makeLiteral("\\")) : builder.makeCall( - SqlLibraryOperators.ILIKE, arg1, arg2, builder.makeLiteral("\\")), - PPLTypeChecker.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING, SqlTypeFamily.BOOLEAN)); + SqlLibraryOperators.ILIKE, arg1, arg2, builder.makeLiteral("\\"))); } } private static class Builder extends AbstractBuilder { - private final Map>> map = - new HashMap<>(); + private final Map map = new HashMap<>(); @Override - void register( - BuiltinFunctionName functionName, FunctionImp implement, PPLTypeChecker typeChecker) { - CalciteFuncSignature signature = - new CalciteFuncSignature(functionName.getName(), typeChecker); + void register(BuiltinFunctionName functionName, FunctionImp implement) { if (map.containsKey(functionName)) { - map.get(functionName).add(Pair.of(signature, implement)); - } else { - map.put(functionName, new ArrayList<>(List.of(Pair.of(signature, implement)))); + throw new IllegalStateException( + String.format( + Locale.ROOT, + "Each function can only be registered with one operator: %s", + functionName)); } + map.put(functionName, implement); } } private static class AggBuilder { private static final double MEDIAN_PERCENTILE = 50.0; - private final Map> map = - new HashMap<>(); - - void register( - BuiltinFunctionName functionName, AggHandler aggHandler, PPLTypeChecker typeChecker) { - CalciteFuncSignature signature = - new CalciteFuncSignature(functionName.getName(), typeChecker); - map.put(functionName, Pair.of(signature, aggHandler)); + private final Map map = new HashMap<>(); + + void register(BuiltinFunctionName functionName, AggHandler aggHandler) { + map.put(functionName, aggHandler); } void registerOperator(BuiltinFunctionName functionName, SqlAggFunction aggFunction) { - SqlOperandTypeChecker innerTypeChecker = extractTypeCheckerFromUDF(aggFunction); - PPLTypeChecker typeChecker = - wrapSqlOperandTypeChecker(innerTypeChecker, functionName.name(), true); AggHandler handler = (distinct, field, argList, ctx) -> { List newArgList = @@ -1326,7 +997,7 @@ void registerOperator(BuiltinFunctionName functionName, SqlAggFunction aggFuncti return UserDefinedFunctionUtils.makeAggregateCall( aggFunction, List.of(field), newArgList, ctx.relBuilder); }; - register(functionName, handler, typeChecker); + register(functionName, handler); } void populate() { @@ -1342,11 +1013,7 @@ void populate() { registerOperator(LIST, PPLBuiltinOperators.LIST); registerOperator(VALUES, PPLBuiltinOperators.VALUES); - register( - AVG, - (distinct, field, argList, ctx) -> ctx.relBuilder.avg(distinct, null, field), - wrapSqlOperandTypeChecker( - SqlStdOperatorTable.AVG.getOperandTypeChecker(), AVG.name(), false)); + register(AVG, (distinct, field, argList, ctx) -> ctx.relBuilder.avg(distinct, null, field)); register( COUNT, @@ -1358,8 +1025,7 @@ void populate() { // count(field) should count non-null values of the field return ctx.relBuilder.count(distinct, null, field); } - }, - wrapSqlOperandTypeChecker(PPLOperandTypes.OPTIONAL_ANY, COUNT.name(), false)); + }); register( PERCENTILE_APPROX, @@ -1372,11 +1038,7 @@ void populate() { newArgList.add(ctx.rexBuilder.makeFlag(field.getType().getSqlTypeName())); return UserDefinedFunctionUtils.makeAggregateCall( PPLBuiltinOperators.PERCENTILE_APPROX, List.of(field), newArgList, ctx.relBuilder); - }, - wrapSqlOperandTypeChecker( - extractTypeCheckerFromUDF(PPLBuiltinOperators.PERCENTILE_APPROX), - PERCENTILE_APPROX.name(), - false)); + }); register( MEDIAN, @@ -1399,9 +1061,7 @@ void populate() { List.of(field), medianArgList, ctx.relBuilder); - }, - wrapSqlOperandTypeChecker( - PPLOperandTypes.NUMERIC.getInnerTypeChecker(), MEDIAN.name(), false)); + }); register( EARLIEST, @@ -1409,9 +1069,7 @@ void populate() { List args = resolveTimeField(argList, ctx); return UserDefinedFunctionUtils.makeAggregateCall( SqlStdOperatorTable.ARG_MIN, List.of(field), args, ctx.relBuilder); - }, - wrapSqlOperandTypeChecker( - PPLOperandTypes.ANY_OPTIONAL_TIMESTAMP, EARLIEST.name(), false)); + }); register( LATEST, @@ -1419,9 +1077,7 @@ void populate() { List args = resolveTimeField(argList, ctx); return UserDefinedFunctionUtils.makeAggregateCall( SqlStdOperatorTable.ARG_MAX, List.of(field), args, ctx.relBuilder); - }, - wrapSqlOperandTypeChecker( - PPLOperandTypes.ANY_OPTIONAL_TIMESTAMP, EARLIEST.name(), false)); + }); // Register FIRST function - uses document order register( @@ -1429,9 +1085,7 @@ void populate() { (distinct, field, argList, ctx) -> { // Use our custom FirstAggFunction for document order aggregation return ctx.relBuilder.aggregateCall(PPLBuiltinOperators.FIRST, field); - }, - wrapSqlOperandTypeChecker( - PPLBuiltinOperators.FIRST.getOperandTypeChecker(), FIRST.name(), false)); + }); // Register LAST function - uses document order register( @@ -1439,9 +1093,7 @@ void populate() { (distinct, field, argList, ctx) -> { // Use our custom LastAggFunction for document order aggregation return ctx.relBuilder.aggregateCall(PPLBuiltinOperators.LAST, field); - }, - wrapSqlOperandTypeChecker( - PPLBuiltinOperators.LAST.getOperandTypeChecker(), LAST.name(), false)); + }); } } @@ -1459,79 +1111,4 @@ static List resolveTimeField(List argList, CalcitePlanContext return argList.stream().map(PlanUtils::derefMapCall).collect(Collectors.toList()); } } - - /** - * Wraps a {@link SqlOperandTypeChecker} into a {@link PPLTypeChecker} for use in function - * signature validation. - * - * @param typeChecker the original SQL operand type checker - * @param functionName the name of the function for error reporting - * @param isUserDefinedFunction true if the function is user-defined, false otherwise - * @return a {@link PPLTypeChecker} that delegates to the provided {@code typeChecker} - */ - private static PPLTypeChecker wrapSqlOperandTypeChecker( - SqlOperandTypeChecker typeChecker, String functionName, boolean isUserDefinedFunction) { - PPLTypeChecker pplTypeChecker; - if (typeChecker instanceof ImplicitCastOperandTypeChecker implicitCastTypeChecker) { - pplTypeChecker = PPLTypeChecker.wrapFamily(implicitCastTypeChecker); - } else if (typeChecker instanceof CompositeOperandTypeChecker compositeTypeChecker) { - // UDFs implement their own composite type checkers, which always use OR logic for - // argument - // types. Verifying the composition type would require accessing a protected field in - // CompositeOperandTypeChecker. If access to this field is not allowed, type checking will - // be skipped, so we avoid checking the composition type here. - - // If compositeTypeChecker contains operand checkers other than family type checkers or - // other than OR compositions, the function with be registered with a null type checker, - // which means the function will not be type checked. - try { - pplTypeChecker = PPLTypeChecker.wrapComposite(compositeTypeChecker, !isUserDefinedFunction); - } catch (IllegalArgumentException | UnsupportedOperationException e) { - logger.debug( - String.format( - "Failed to create composite type checker for operator: %s. Will skip its type" - + " checking", - functionName), - e); - pplTypeChecker = null; - } - } else if (typeChecker instanceof SameOperandTypeChecker comparableTypeChecker) { - // Comparison operators like EQUAL, GREATER_THAN, LESS_THAN, etc. - // SameOperandTypeCheckers like COALESCE, IFNULL, etc. - pplTypeChecker = PPLTypeChecker.wrapComparable(comparableTypeChecker); - } else if (typeChecker instanceof UDFOperandMetadata.UDTOperandMetadata udtOperandMetadata) { - pplTypeChecker = PPLTypeChecker.wrapUDT(udtOperandMetadata.allowedParamTypes()); - } else if (typeChecker != null) { - pplTypeChecker = PPLTypeChecker.wrapDefault(typeChecker); - } else { - logger.info( - "Cannot create type checker for function: {}. Will skip its type checking", functionName); - pplTypeChecker = null; - } - return pplTypeChecker; - } - - /** - * Extracts the underlying {@link SqlOperandTypeChecker} from a {@link SqlOperator}. - * - *

For user-defined functions (UDFs) and user-defined aggregate functions (UDAFs), the {@link - * SqlOperandTypeChecker} is typically wrapped in a {@link UDFOperandMetadata}, which contains the - * actual type checker used for operand validation. Most of these wrapped type checkers are - * defined in {@link org.opensearch.sql.calcite.utils.PPLOperandTypes}. This method retrieves the - * inner type checker from {@link UDFOperandMetadata} if present. - * - *

For Calcite's built-in operators, its type checker is returned directly. - * - * @param operator the {@link SqlOperator}, which may be a Calcite built-in operator, a - * user-defined function, or a user-defined aggregation function - * @return the underlying {@link SqlOperandTypeChecker} instance, or {@code null} if not available - */ - private static SqlOperandTypeChecker extractTypeCheckerFromUDF(SqlOperator operator) { - SqlOperandTypeChecker typeChecker = operator.getOperandTypeChecker(); - if (typeChecker instanceof UDFOperandMetadata) { - UDFOperandMetadata udfOperandMetadata = (UDFOperandMetadata) typeChecker; - return udfOperandMetadata.getInnerTypeChecker(); - } - return typeChecker; - } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java deleted file mode 100644 index 521764ba7bb..00000000000 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java +++ /dev/null @@ -1,701 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.expression.function; - -import com.google.common.collect.Lists; -import java.lang.reflect.Field; -import java.lang.reflect.InaccessibleObjectException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import lombok.RequiredArgsConstructor; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.sql.SqlIntervalQualifier; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.sql.type.CompositeOperandTypeChecker; -import org.apache.calcite.sql.type.FamilyOperandTypeChecker; -import org.apache.calcite.sql.type.ImplicitCastOperandTypeChecker; -import org.apache.calcite.sql.type.SameOperandTypeChecker; -import org.apache.calcite.sql.type.SqlOperandTypeChecker; -import org.apache.calcite.sql.type.SqlTypeFamily; -import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.calcite.sql.type.SqlTypeUtil; -import org.apache.calcite.util.Pair; -import org.opensearch.sql.calcite.type.ExprIPType; -import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; -import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; -import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; - -/** - * A custom type checker interface for PPL (Piped Processing Language) functions. - * - *

Provides operand type validation based on specified type families, similar to Calcite's {@link - * SqlOperandTypeChecker}, but adapted for PPL function requirements. This abstraction is necessary - * because {@code SqlOperandTypeChecker::checkOperandTypes(SqlCallBinding, boolean)} cannot be - * directly used for type checking at the logical plan level. - */ -public interface PPLTypeChecker { - /** - * Validates the operand types. - * - * @param types the list of operand types to validate - * @return true if the operand types are valid, false otherwise - */ - boolean checkOperandTypes(List types); - - /** - * Get a string representation of the allowed signatures. The format is like {@code - * [STRING,STRING]|[INTEGER,INTEGER]}. - * - * @return a string representation of the allowed signatures - */ - String getAllowedSignatures(); - - /** - * Get a list of all possible parameter type combinations for the function. - * - *

This method is used to generate the allowed signatures for the function based on the - * parameter types. - * - * @return a list of lists, where each inner list represents an allowed parameter type combination - */ - List> getParameterTypes(); - - private static boolean validateOperands( - List funcTypeFamilies, List operandTypes) { - // If the number of actual operands does not match expectation, return false - if (funcTypeFamilies.size() != operandTypes.size()) { - return false; - } - for (int i = 0; i < operandTypes.size(); i++) { - SqlTypeName paramType = - UserDefinedFunctionUtils.convertRelDataTypeToSqlTypeName(operandTypes.get(i)); - SqlTypeFamily funcTypeFamily = funcTypeFamilies.get(i); - if (paramType.getFamily() == SqlTypeFamily.IGNORE || funcTypeFamily == SqlTypeFamily.IGNORE) { - continue; - } - if (!funcTypeFamily.getTypeNames().contains(paramType)) { - return false; - } - } - return true; - } - - /** - * A custom {@code PPLTypeChecker} that validates operand types against a list of {@link - * SqlTypeFamily}. Instances can be created using {@link #family(SqlTypeFamily...)}. - */ - class PPLFamilyTypeChecker implements PPLTypeChecker { - private final List families; - - public PPLFamilyTypeChecker(SqlTypeFamily... families) { - this.families = List.of(families); - } - - @Override - public boolean checkOperandTypes(List types) { - if (families.size() != types.size()) return false; - return validateOperands(families, types); - } - - @Override - public String getAllowedSignatures() { - return PPLTypeChecker.getFamilySignature(families); - } - - @Override - public List> getParameterTypes() { - return PPLTypeChecker.getExprSignatures(families); - } - - @Override - public String toString() { - return String.format("PPLFamilyTypeChecker[families=%s]", getAllowedSignatures()); - } - } - - /** - * A {@code PPLTypeChecker} implementation that wraps a Calcite {@link - * ImplicitCastOperandTypeChecker}. - * - *

This checker delegates operand count and type validation to the wrapped Calcite type - * checker, allowing PPL functions to leverage Calcite's implicit casting and type family logic - * for operand validation. - */ - class PPLFamilyTypeCheckerWrapper implements PPLTypeChecker { - protected final ImplicitCastOperandTypeChecker innerTypeChecker; - - public PPLFamilyTypeCheckerWrapper(ImplicitCastOperandTypeChecker typeChecker) { - this.innerTypeChecker = typeChecker; - } - - @Override - public boolean checkOperandTypes(List types) { - if (innerTypeChecker instanceof SqlOperandTypeChecker sqlOperandTypeChecker - && !sqlOperandTypeChecker.getOperandCountRange().isValidCount(types.size())) return false; - List families = - IntStream.range(0, types.size()) - .mapToObj(innerTypeChecker::getOperandSqlTypeFamily) - .collect(Collectors.toList()); - return validateOperands(families, types); - } - - @Override - public String getAllowedSignatures() { - if (innerTypeChecker instanceof FamilyOperandTypeChecker familyOperandTypeChecker) { - var allowedExprSignatures = getExprSignatures(familyOperandTypeChecker); - return PPLTypeChecker.formatExprSignatures(allowedExprSignatures); - } else { - return ""; - } - } - - @Override - public List> getParameterTypes() { - if (innerTypeChecker instanceof FamilyOperandTypeChecker familyOperandTypeChecker) { - return getExprSignatures(familyOperandTypeChecker); - } else { - // If the inner type checker is not a FamilyOperandTypeChecker, we cannot provide - // parameter types. - return Collections.emptyList(); - } - } - } - - /** - * A {@code PPLTypeChecker} implementation that wraps a Calcite {@link - * CompositeOperandTypeChecker}. - * - *

This checker allows for the composition of multiple operand type checkers, enabling flexible - * validation of operand types in PPL functions. - * - *

The implementation currently supports only OR compositions of {@link - * ImplicitCastOperandTypeChecker}. - */ - class PPLCompositeTypeChecker implements PPLTypeChecker { - - private final List allowedRules; - - public PPLCompositeTypeChecker(CompositeOperandTypeChecker typeChecker) { - allowedRules = typeChecker.getRules(); - } - - private static boolean validateWithFamilyTypeChecker( - SqlOperandTypeChecker checker, List types) { - if (!checker.getOperandCountRange().isValidCount(types.size())) { - return false; - } - if (checker instanceof ImplicitCastOperandTypeChecker implicitCastOperandTypeChecker) { - List families = - IntStream.range(0, types.size()) - .mapToObj(implicitCastOperandTypeChecker::getOperandSqlTypeFamily) - .toList(); - return validateOperands(families, types); - } - throw new IllegalArgumentException( - "Currently only compositions of ImplicitCastOperandTypeChecker are supported"); - } - - @Override - public boolean checkOperandTypes(List types) { - boolean operandCountValid = - allowedRules.stream() - .anyMatch(rule -> rule.getOperandCountRange().isValidCount(types.size())); - if (!operandCountValid) { - return false; - } - return allowedRules.stream().anyMatch(rule -> validateWithFamilyTypeChecker(rule, types)); - } - - @Override - public String getAllowedSignatures() { - StringBuilder builder = new StringBuilder(); - for (SqlOperandTypeChecker rule : allowedRules) { - if (rule instanceof FamilyOperandTypeChecker familyOperandTypeChecker) { - if (!builder.isEmpty()) { - builder.append("|"); - } - builder.append(PPLTypeChecker.getFamilySignatures(familyOperandTypeChecker)); - } else { - throw new IllegalArgumentException( - "Currently only compositions of FamilyOperandTypeChecker are supported"); - } - } - return builder.toString(); - } - - @Override - public List> getParameterTypes() { - List> parameterTypes = new ArrayList<>(); - for (SqlOperandTypeChecker rule : allowedRules) { - if (rule instanceof FamilyOperandTypeChecker familyOperandTypeChecker) { - parameterTypes.addAll(getExprSignatures(familyOperandTypeChecker)); - } else { - throw new IllegalArgumentException( - "Currently only compositions of FamilyOperandTypeChecker are supported"); - } - } - return parameterTypes; - } - } - - @RequiredArgsConstructor - class PPLComparableTypeChecker implements PPLTypeChecker { - private final SameOperandTypeChecker innerTypeChecker; - - @Override - public boolean checkOperandTypes(List types) { - if (!innerTypeChecker.getOperandCountRange().isValidCount(types.size())) { - return false; - } - // Check comparability of consecutive operands - for (int i = 0; i < types.size() - 1; i++) { - // TODO: Binary, Array UDT? - // DATETIME, NUMERIC, BOOLEAN will be regarded as comparable - // with strings in isComparable - RelDataType type_l = types.get(i); - RelDataType type_r = types.get(i + 1); - // Rule out IP types from built-in comparable functions - if (type_l instanceof ExprIPType || type_r instanceof ExprIPType) { - return false; - } - if (!isComparable(type_l, type_r)) { - return false; - } - } - return true; - } - - /** - * Modified from {@link SqlTypeUtil#isComparable(RelDataType, RelDataType)} to - * - * @param type1 first type - * @param type2 second type - * @return true if the two types are comparable, false otherwise - */ - private static boolean isComparable(RelDataType type1, RelDataType type2) { - if (type1.isStruct() != type2.isStruct()) { - return false; - } - - if (type1.isStruct()) { - int n = type1.getFieldCount(); - if (n != type2.getFieldCount()) { - return false; - } - for (Pair pair : - Pair.zip(type1.getFieldList(), type2.getFieldList())) { - if (!isComparable(pair.left.getType(), pair.right.getType())) { - return false; - } - } - return true; - } - - // Numeric types are comparable without the need to cast - if (SqlTypeUtil.isNumeric(type1) && SqlTypeUtil.isNumeric(type2)) { - return true; - } - - ExprType exprType1 = OpenSearchTypeFactory.convertRelDataTypeToExprType(type1); - ExprType exprType2 = OpenSearchTypeFactory.convertRelDataTypeToExprType(type2); - - if (!exprType1.shouldCast(exprType2)) { - return true; - } - - // If one of the arguments is of type 'ANY', return true. - return type1.getFamily() == SqlTypeFamily.ANY || type2.getFamily() == SqlTypeFamily.ANY; - } - - @Override - public String getAllowedSignatures() { - int min = innerTypeChecker.getOperandCountRange().getMin(); - int max = innerTypeChecker.getOperandCountRange().getMax(); - final String typeName = "COMPARABLE_TYPE"; - if (min == -1 || max == -1) { - // If the range is unbounded, we cannot provide a specific signature - return String.format("[%s...]", typeName); - } else { - // Generate a signature based on the min and max operand counts - List signatures = new ArrayList<>(); - // avoid enumerating too many signatures - final int MAX_ARGS = 10; - max = Math.min(MAX_ARGS, max); - for (int i = min; i <= max; i++) { - signatures.add("[" + String.join(",", Collections.nCopies(i, typeName)) + "]"); - } - return String.join(",", signatures); - } - } - - @Override - public List> getParameterTypes() { - // Should not be used - return List.of(List.of(ExprCoreType.UNKNOWN, ExprCoreType.UNKNOWN)); - } - } - - class PPLDefaultTypeChecker implements PPLTypeChecker { - private final SqlOperandTypeChecker internal; - - public PPLDefaultTypeChecker(SqlOperandTypeChecker typeChecker) { - internal = typeChecker; - } - - @Override - public boolean checkOperandTypes(List types) { - // Basic operand count validation - if (!internal.getOperandCountRange().isValidCount(types.size())) { - return false; - } - - // If the internal checker is a FamilyOperandTypeChecker, use type family validation - if (internal instanceof FamilyOperandTypeChecker familyChecker) { - List families = - IntStream.range(0, types.size()) - .mapToObj(familyChecker::getOperandSqlTypeFamily) - .collect(Collectors.toList()); - return validateOperands(families, types); - } - - // For other types of checkers, we can only validate operand count - // This is a fallback - we assume the types are valid if count is correct - return true; - } - - @Override - public String getAllowedSignatures() { - if (internal instanceof FamilyOperandTypeChecker familyChecker) { - return getFamilySignatures(familyChecker); - } else { - // Generate a generic signature based on operand count range - int min = internal.getOperandCountRange().getMin(); - int max = internal.getOperandCountRange().getMax(); - - if (min == -1 || max == -1) { - return "[ANY...]"; - } else if (min == max) { - return "[" + String.join(",", Collections.nCopies(min, "ANY")) + "]"; - } else { - List signatures = new ArrayList<>(); - final int MAX_ARGS = 10; - max = Math.min(MAX_ARGS, max); - for (int i = min; i <= max; i++) { - signatures.add("[" + String.join(",", Collections.nCopies(i, "ANY")) + "]"); - } - return String.join("|", signatures); - } - } - } - - @Override - public List> getParameterTypes() { - if (internal instanceof FamilyOperandTypeChecker familyChecker) { - return getExprSignatures(familyChecker); - } else { - // For unknown type checkers, return UNKNOWN types - int min = internal.getOperandCountRange().getMin(); - int max = internal.getOperandCountRange().getMax(); - - if (min == -1 || max == -1) { - // Variable arguments - return a single signature with UNKNOWN - return List.of(List.of(ExprCoreType.UNKNOWN)); - } else { - List> parameterTypes = new ArrayList<>(); - final int MAX_ARGS = 10; - max = Math.min(MAX_ARGS, max); - for (int i = min; i <= max; i++) { - parameterTypes.add(Collections.nCopies(i, ExprCoreType.UNKNOWN)); - } - return parameterTypes; - } - } - } - } - - /** - * Creates a {@link PPLFamilyTypeChecker} with a fixed operand count, validating that each operand - * belongs to its corresponding {@link SqlTypeFamily}. - * - *

The number of provided {@code families} determines the required number of operands. Each - * operand is checked against the type family at the same position in the array. - * - * @param families the expected {@link SqlTypeFamily} for each operand, in order - * @return a {@link PPLFamilyTypeChecker} that enforces the specified type families for operands - */ - static PPLFamilyTypeChecker family(SqlTypeFamily... families) { - return new PPLFamilyTypeChecker(families); - } - - /** - * Wraps a Calcite {@link ImplicitCastOperandTypeChecker} (usually a {@link - * FamilyOperandTypeChecker}) into a custom PPLTypeChecker of type {@link - * PPLFamilyTypeCheckerWrapper}. - * - *

The allow operand count may be fixed or variable, depending on the wrapped type checker. - * - * @param typeChecker the Calcite type checker to wrap - * @return a PPLTypeChecker that uses the wrapped type checker - */ - static PPLFamilyTypeCheckerWrapper wrapFamily(ImplicitCastOperandTypeChecker typeChecker) { - return new PPLFamilyTypeCheckerWrapper(typeChecker); - } - - /** - * Wraps a Calcite {@link CompositeOperandTypeChecker} into a custom {@link - * PPLCompositeTypeChecker}. - * - *

This method requires that all rules within the provided {@code CompositeOperandTypeChecker} - * are instances of {@link ImplicitCastOperandTypeChecker}. If any rule does not meet this - * requirement, an {@link IllegalArgumentException} is thrown. - * - *

Additionally, if {@code checkCompositionType} is true, the method checks if the composition - * type of the provided {@code CompositeOperandTypeChecker} is OR via reflection. If it is not, an - * {@link IllegalArgumentException} is thrown. If the reflective access to the composition field - * of CompositeOperandTypeChecker fails, an {@link UnsupportedOperationException} is thrown. - * - * @param typeChecker the Calcite {@link CompositeOperandTypeChecker} to wrap - * @param checkCompositionType if true, checks if the composition type is OR. - * @return a {@link PPLCompositeTypeChecker} that delegates type checking to the wrapped rules - * @throws IllegalArgumentException if any rule is not an {@link ImplicitCastOperandTypeChecker} - */ - static PPLCompositeTypeChecker wrapComposite( - CompositeOperandTypeChecker typeChecker, boolean checkCompositionType) - throws IllegalArgumentException, UnsupportedOperationException { - if (checkCompositionType) { - try { - if (!isCompositionOr(typeChecker)) { - throw new IllegalArgumentException( - "Currently only support CompositeOperandTypeChecker with a OR composition"); - } - } catch (ReflectiveOperationException | InaccessibleObjectException | SecurityException e) { - throw new UnsupportedOperationException( - String.format("Failed to check composition type of %s", typeChecker), e); - } - } - - for (SqlOperandTypeChecker rule : typeChecker.getRules()) { - if (!(rule instanceof ImplicitCastOperandTypeChecker)) { - throw new IllegalArgumentException( - "Currently only compositions of ImplicitCastOperandTypeChecker are supported, found:" - + rule.getClass().getName()); - } - } - return new PPLCompositeTypeChecker(typeChecker); - } - - static PPLComparableTypeChecker wrapComparable(SameOperandTypeChecker typeChecker) { - return new PPLComparableTypeChecker(typeChecker); - } - - /** - * Creates a {@link PPLDefaultTypeChecker} that wraps any {@link SqlOperandTypeChecker} and - * provides basic type checking functionality when specialized PPL type checkers cannot be used. - * - *

This is a fallback wrapper that provides basic operand count validation and attempts to - * extract type family information when possible. It should be used when other specialized PPL - * type checkers (like {@link PPLFamilyTypeChecker}, {@link PPLCompositeTypeChecker}, etc.) are - * not applicable. - * - * @param typeChecker the Calcite type checker to wrap - * @return a {@link PPLDefaultTypeChecker} that provides basic type checking functionality - */ - static PPLDefaultTypeChecker wrapDefault(SqlOperandTypeChecker typeChecker) { - return new PPLDefaultTypeChecker(typeChecker); - } - - /** - * Create a {@link PPLTypeChecker} from a list of allowed signatures consisted of {@link - * ExprType}. This is useful to validate arguments against user-defined types (UDT) that does not - * match any Calcite {@link SqlTypeFamily}. - * - * @param allowedSignatures a list of allowed signatures, where each signature is a list of {@link - * ExprType} representing the expected types of the function arguments. - * @return a {@link PPLTypeChecker} that checks if the operand types match any of the allowed - * signatures - */ - static PPLTypeChecker wrapUDT(List> allowedSignatures) { - return new PPLTypeChecker() { - @Override - public boolean checkOperandTypes(List types) { - List argExprTypes = - types.stream().map(OpenSearchTypeFactory::convertRelDataTypeToExprType).toList(); - for (var allowedSignature : allowedSignatures) { - if (allowedSignature.size() != types.size()) { - continue; // Skip signatures that do not match the operand count - } - // Check if the argument types match the allowed signature - if (IntStream.range(0, allowedSignature.size()) - .allMatch(i -> allowedSignature.get(i).equals(argExprTypes.get(i)))) { - return true; - } - } - return false; - } - - @Override - public String getAllowedSignatures() { - return PPLTypeChecker.formatExprSignatures(allowedSignatures); - } - - @Override - public List> getParameterTypes() { - return allowedSignatures; - } - }; - } - - // Util Functions - /** - * Generates a list of allowed function signatures based on the provided {@link - * FamilyOperandTypeChecker}. The signatures are generated by iterating through the operand count - * range and collecting the corresponding type families. - * - *

If the operand count range is large, the method will limit the maximum number of signatures - * to 10 to avoid excessive enumeration. - * - * @param typeChecker the {@link FamilyOperandTypeChecker} to use for generating signatures - * @return a list of allowed function signatures - */ - private static String getFamilySignatures(FamilyOperandTypeChecker typeChecker) { - var allowedExprSignatures = getExprSignatures(typeChecker); - return formatExprSignatures(allowedExprSignatures); - } - - private static List> getExprSignatures(FamilyOperandTypeChecker typeChecker) { - var operandCountRange = typeChecker.getOperandCountRange(); - int min = operandCountRange.getMin(); - int max = operandCountRange.getMax(); - List families = new ArrayList<>(); - for (int i = 0; i < min; i++) { - families.add(typeChecker.getOperandSqlTypeFamily(i)); - } - List> allowedSignatures = new ArrayList<>(getExprSignatures(families)); - - // Avoid enumerating signatures for infinite args - final int MAX_ARGS = 10; - max = Math.min(max, MAX_ARGS); - for (int i = min; i < max; i++) { - families.add(typeChecker.getOperandSqlTypeFamily(i)); - allowedSignatures.addAll(getExprSignatures(families)); - } - return allowedSignatures; - } - - /** - * Converts a {@link SqlTypeFamily} to a list of {@link ExprType}. This method is used to display - * the allowed signatures for functions based on their type families. - * - * @param family the {@link SqlTypeFamily} to convert - * @return a list of {@link ExprType} corresponding to the concrete types of the family - */ - private static List getExprTypes(SqlTypeFamily family) { - List concreteTypes = - switch (family) { - case DATETIME -> - List.of( - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.TIMESTAMP), - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.DATE), - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.TIME)); - case NUMERIC -> - List.of( - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.INTEGER), - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.DOUBLE)); - // Integer is mapped to BIGINT in family.getDefaultConcreteType - case INTEGER -> - List.of(OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.INTEGER)); - case ANY, IGNORE -> - List.of(OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.ANY)); - case DATETIME_INTERVAL -> - SqlTypeName.INTERVAL_TYPES.stream() - .map( - type -> - OpenSearchTypeFactory.TYPE_FACTORY.createSqlIntervalType( - new SqlIntervalQualifier( - type.getStartUnit(), type.getEndUnit(), SqlParserPos.ZERO))) - .collect(Collectors.toList()); - default -> { - RelDataType type = family.getDefaultConcreteType(OpenSearchTypeFactory.TYPE_FACTORY); - if (type == null) { - yield List.of(OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.OTHER)); - } - yield List.of(type); - } - }; - return concreteTypes.stream() - .map(OpenSearchTypeFactory::convertRelDataTypeToExprType) - .distinct() - .collect(Collectors.toList()); - } - - /** - * Generates a list of all possible {@link ExprType} signatures based on the provided {@link - * SqlTypeFamily} list. - * - * @param families the list of {@link SqlTypeFamily} to generate signatures for - * @return a list of lists, where each inner list contains {@link ExprType} signatures - */ - private static List> getExprSignatures(List families) { - List> exprTypes = - families.stream().map(PPLTypeChecker::getExprTypes).collect(Collectors.toList()); - - // Do a cartesian product of all ExprTypes in the family - return Lists.cartesianProduct(exprTypes); - } - - /** - * Generates a string representation of the function signature based on the provided type - * families. The format is a list of type families enclosed in square brackets, e.g.: "[INTEGER, - * STRING]". - * - * @param families the list of type families to include in the signature - * @return a string representation of the function signature - */ - private static String getFamilySignature(List families) { - List> signatures = getExprSignatures(families); - // Convert each signature to a string representation and then concatenate them - return formatExprSignatures(signatures); - } - - /** - * Checks if the provided {@link CompositeOperandTypeChecker} is of type OR composition. - * - *

This method uses reflection to access the protected "composition" field of the - * CompositeOperandTypeChecker class. - * - * @param typeChecker the CompositeOperandTypeChecker to check - * @return true if the composition is OR, false otherwise - */ - private static boolean isCompositionOr(CompositeOperandTypeChecker typeChecker) - throws NoSuchFieldException, - IllegalAccessException, - InaccessibleObjectException, - SecurityException { - Field compositionField = CompositeOperandTypeChecker.class.getDeclaredField("composition"); - compositionField.setAccessible(true); - CompositeOperandTypeChecker.Composition composition = - (CompositeOperandTypeChecker.Composition) compositionField.get(typeChecker); - return composition == CompositeOperandTypeChecker.Composition.OR; - } - - private static String formatExprSignatures(List> signatures) { - return signatures.stream() - .map( - types -> - "[" - + types.stream() - // Display ExprCoreType.UNDEFINED as "ANY" for better interpretability - .map(t -> t == ExprCoreType.UNDEFINED ? "ANY" : t.toString()) - .collect(Collectors.joining(",")) - + "]") - .collect(Collectors.joining("|")); - } -} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/UDFOperandMetadata.java b/core/src/main/java/org/opensearch/sql/expression/function/UDFOperandMetadata.java index dc4761b26e7..7b131383b95 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/UDFOperandMetadata.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/UDFOperandMetadata.java @@ -5,19 +5,15 @@ package org.opensearch.sql.expression.function; -import java.util.Collections; import java.util.List; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.sql.SqlCallBinding; import org.apache.calcite.sql.SqlOperandCountRange; import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.type.CompositeOperandTypeChecker; -import org.apache.calcite.sql.type.FamilyOperandTypeChecker; import org.apache.calcite.sql.type.SqlOperandMetadata; import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.validate.SqlUserDefinedFunction; -import org.opensearch.sql.data.type.ExprType; /** * This class is created for the compatibility with {@link SqlUserDefinedFunction} constructors when @@ -27,7 +23,7 @@ public interface UDFOperandMetadata extends SqlOperandMetadata { SqlOperandTypeChecker getInnerTypeChecker(); - static UDFOperandMetadata wrap(FamilyOperandTypeChecker typeChecker) { + static UDFOperandMetadata wrap(SqlOperandTypeChecker typeChecker) { return new UDFOperandMetadata() { @Override public SqlOperandTypeChecker getInnerTypeChecker() { @@ -36,57 +32,16 @@ public SqlOperandTypeChecker getInnerTypeChecker() { @Override public List paramTypes(RelDataTypeFactory typeFactory) { - // This function is not used in the current context, so we return an empty list. - return Collections.emptyList(); + // This function is not used in the current context + throw new UnsupportedOperationException( + "paramTypes of UDFOperandMetadata is not implemented and should not be called"); } @Override public List paramNames() { - // This function is not used in the current context, so we return an empty list. - return Collections.emptyList(); - } - - @Override - public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { - return typeChecker.checkOperandTypesWithoutTypeCoercion(callBinding, throwOnFailure); - } - - @Override - public SqlOperandCountRange getOperandCountRange() { - return typeChecker.getOperandCountRange(); - } - - @Override - public String getAllowedSignatures(SqlOperator op, String opName) { - return typeChecker.getAllowedSignatures(op, opName); - } - }; - } - - static UDFOperandMetadata wrap(CompositeOperandTypeChecker typeChecker) { - for (SqlOperandTypeChecker rule : typeChecker.getRules()) { - if (!(rule instanceof FamilyOperandTypeChecker)) { - throw new IllegalArgumentException( - "Currently only compositions of ImplicitCastOperandTypeChecker are supported"); - } - } - - return new UDFOperandMetadata() { - @Override - public SqlOperandTypeChecker getInnerTypeChecker() { - return typeChecker; - } - - @Override - public List paramTypes(RelDataTypeFactory typeFactory) { - // This function is not used in the current context, so we return an empty list. - return Collections.emptyList(); - } - - @Override - public List paramNames() { - // This function is not used in the current context, so we return an empty list. - return Collections.emptyList(); + // This function is not used in the current context + throw new UnsupportedOperationException( + "paramNames of UDFOperandMetadata is not implemented and should not be called"); } @Override @@ -105,40 +60,4 @@ public String getAllowedSignatures(SqlOperator op, String opName) { } }; } - - static UDFOperandMetadata wrapUDT(List> allowSignatures) { - return new UDTOperandMetadata(allowSignatures); - } - - record UDTOperandMetadata(List> allowedParamTypes) implements UDFOperandMetadata { - @Override - public SqlOperandTypeChecker getInnerTypeChecker() { - return this; - } - - @Override - public List paramTypes(RelDataTypeFactory typeFactory) { - return List.of(); - } - - @Override - public List paramNames() { - return List.of(); - } - - @Override - public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { - return false; - } - - @Override - public SqlOperandCountRange getOperandCountRange() { - return null; - } - - @Override - public String getAllowedSignatures(SqlOperator op, String opName) { - return ""; - } - } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/UserDefinedFunctionBuilder.java b/core/src/main/java/org/opensearch/sql/expression/function/UserDefinedFunctionBuilder.java index f52c6b69f0e..2915886bfa7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/UserDefinedFunctionBuilder.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/UserDefinedFunctionBuilder.java @@ -9,6 +9,7 @@ import org.apache.calcite.schema.ImplementableFunction; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperandCountRange; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.InferTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; @@ -32,6 +33,10 @@ public interface UserDefinedFunctionBuilder { UDFOperandMetadata getOperandMetadata(); + default SqlKind getKind() { + return SqlKind.OTHER_FUNCTION; + } + default SqlUserDefinedFunction toUDF(String functionName) { return toUDF(functionName, true); } @@ -50,7 +55,7 @@ default SqlUserDefinedFunction toUDF(String functionName, boolean isDeterministi new SqlIdentifier(Collections.singletonList(functionName), null, SqlParserPos.ZERO, null); return new SqlUserDefinedFunction( udfLtrimIdentifier, - SqlKind.OTHER_FUNCTION, + getKind(), getReturnTypeInference(), InferTypes.ANY_NULLABLE, getOperandMetadata(), @@ -66,6 +71,11 @@ public SqlIdentifier getSqlIdentifier() { // check the code SqlUtil.unparseFunctionSyntax() return null; } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return getOperandMetadata().getOperandCountRange(); + } }; } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java index dd76a002e06..ecbbf62a10d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java @@ -22,6 +22,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -38,7 +39,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return null; + return UDFOperandMetadata.wrap(OperandTypes.ONE_OR_MORE); } public static class JsonAppendImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java index b3a884a4f17..12686bf8027 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java @@ -20,6 +20,7 @@ import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -36,7 +37,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return null; + return UDFOperandMetadata.wrap(OperandTypes.ONE_OR_MORE); } public static class JsonDeleteImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java index dd91f1d95bd..1bb72ad1c7e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java @@ -22,6 +22,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -38,7 +39,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return null; + return UDFOperandMetadata.wrap(OperandTypes.ONE_OR_MORE); } public static class JsonExtendImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractAllFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractAllFunctionImpl.java index 1f91c87bb77..ca12c38818e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractAllFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractAllFunctionImpl.java @@ -28,7 +28,6 @@ import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -57,7 +56,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return UDFOperandMetadata.wrap(OperandTypes.family(SqlTypeFamily.STRING)); + return UDFOperandMetadata.wrap(OperandTypes.CHARACTER); } public static class JsonExtractAllImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java index 76853706f64..b3e4ac10a3f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java @@ -25,6 +25,7 @@ import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.SqlJsonValueEmptyOrErrorBehavior; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -41,7 +42,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return null; + return UDFOperandMetadata.wrap(OperandTypes.VARIADIC); } public static class JsonExtractImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java index 0379aeecb72..0521a5f542d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java @@ -15,6 +15,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -36,7 +37,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return null; + return UDFOperandMetadata.wrap(OperandTypes.ANY); } public static class JsonImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java index 40214ca7556..4a85ce5ce40 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java @@ -17,6 +17,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -33,7 +34,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return null; + return UDFOperandMetadata.wrap(OperandTypes.ANY); } public static class JsonKeysImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java index 27346b478e4..0193152911f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java @@ -5,6 +5,7 @@ package org.opensearch.sql.expression.function.jsonUDF; +import static org.apache.calcite.util.Static.RESOURCE; import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; @@ -18,10 +19,18 @@ import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.fun.SqlJsonModifyFunction; +import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -37,7 +46,38 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return null; + return UDFOperandMetadata.wrap( + new SqlOperandTypeChecker() { + /** + * Copied from {@link SqlJsonModifyFunction#checkOperandTypes(SqlCallBinding, boolean)} + * (Calcite 1.41) + */ + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + final int count = callBinding.getOperandCount(); + for (int i = 1; i < count; i += 2) { + RelDataType nameType = callBinding.getOperandType(i); + if (!OpenSearchTypeUtil.isCharacter(nameType)) { + if (throwOnFailure) { + throw callBinding.newError(RESOURCE.expectedCharacter()); + } + return false; + } + } + return true; + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.from(3); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return "(json_string: STRING, path1: STRING, value1: ANY, path2: STRING, value2: ANY" + + " ...)"; + } + }); } public static class JsonSetImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/RelevanceQueryFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/RelevanceQueryFunction.java index d8e53704804..bc51b9729f2 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/RelevanceQueryFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/RelevanceQueryFunction.java @@ -5,18 +5,16 @@ package org.opensearch.sql.expression.function.udf; -import com.google.common.collect.ImmutableList; import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.rex.RexCall; -import org.apache.calcite.sql.type.CompositeOperandTypeChecker; import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlOperandCountRanges; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.apache.calcite.sql.type.SqlTypeFamily; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -41,53 +39,9 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { return UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family( - ImmutableList.of( - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP), - i -> i > 0 && i < 14) // Parameters 3-14 are optional - .or( - OperandTypes.family( - ImmutableList.of( - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP), - i -> i > 0 && i < 25))); // Parameters 3-25 are optional + OperandTypes.repeat( + SqlOperandCountRanges.between(1, 25), + OperandTypes.MAP)); // Parameters 2-25 are optional } public static class RelevanceQueryImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/SpanFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/SpanFunction.java index f28f12e30b9..394bd93a3a8 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/SpanFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/SpanFunction.java @@ -18,10 +18,10 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; -import org.apache.calcite.sql.type.CompositeOperandTypeChecker; import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.util.BuiltInMethod; import org.opensearch.sql.calcite.type.ExprSqlType; @@ -59,15 +59,13 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { return UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family( - SqlTypeFamily.CHARACTER, SqlTypeFamily.NUMERIC, SqlTypeFamily.CHARACTER) - .or( - OperandTypes.family( - SqlTypeFamily.DATETIME, SqlTypeFamily.NUMERIC, SqlTypeFamily.CHARACTER)) - .or( - OperandTypes.family( - SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC, SqlTypeFamily.ANY))); + OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.NUMERIC, SqlTypeFamily.CHARACTER) + .or( + OperandTypes.family( + SqlTypeFamily.DATETIME, SqlTypeFamily.NUMERIC, SqlTypeFamily.CHARACTER)) + .or( + OperandTypes.family( + SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC, SqlTypeFamily.ANY))); } public static class SpanImplementor implements NotNullImplementor { @@ -86,7 +84,7 @@ public Expression implement( if (SqlTypeUtil.isDecimal(intervalType)) { interval = Expressions.call(interval, "doubleValue"); } - if (SqlTypeUtil.isNull(unitType)) { + if (SqlTypeUtil.isNull(unitType) || SqlTypeName.ANY.equals(unitType.getSqlTypeName())) { return switch (call.getType().getSqlTypeName()) { case BIGINT, INTEGER, SMALLINT, TINYINT -> Expressions.multiply(Expressions.divide(field, interval), interval); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/WidthBucketFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/WidthBucketFunction.java index 08daf9c314b..529719157e2 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/WidthBucketFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/WidthBucketFunction.java @@ -16,8 +16,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; -import org.opensearch.sql.calcite.type.ExprSqlType; -import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.binning.BinConstants; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -51,19 +50,13 @@ public SqlReturnTypeInference getReturnTypeInference() { return (opBinding) -> { RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); RelDataType arg0Type = opBinding.getOperandType(0); - return dateRelatedType(arg0Type) + return OpenSearchTypeUtil.isDatetime(arg0Type) ? arg0Type : typeFactory.createTypeWithNullability( typeFactory.createSqlType(SqlTypeName.VARCHAR, 2000), true); }; } - public static boolean dateRelatedType(RelDataType type) { - return type instanceof ExprSqlType exprSqlType - && List.of(ExprUDT.EXPR_DATE, ExprUDT.EXPR_TIME, ExprUDT.EXPR_TIMESTAMP) - .contains(exprSqlType.getUdt()); - } - @Override public UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.WIDTH_BUCKET_OPERAND; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/EnhancedCoalesceFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/EnhancedCoalesceFunction.java deleted file mode 100644 index c6ff1a64478..00000000000 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/EnhancedCoalesceFunction.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.expression.function.udf.condition; - -import java.util.List; -import java.util.function.Supplier; -import org.apache.calcite.adapter.enumerable.NotNullImplementor; -import org.apache.calcite.adapter.enumerable.NullPolicy; -import org.apache.calcite.linq4j.tree.Expression; -import org.apache.calcite.linq4j.tree.Expressions; -import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.apache.calcite.sql.type.SqlTypeName; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.model.ExprValueUtils; -import org.opensearch.sql.expression.function.ImplementorUDF; -import org.opensearch.sql.expression.function.UDFOperandMetadata; - -public class EnhancedCoalesceFunction extends ImplementorUDF { - - public EnhancedCoalesceFunction() { - super(createImplementor(), NullPolicy.NONE); - } - - private static NotNullImplementor createImplementor() { - return (translator, call, translatedOperands) -> { - List exprValues = - translatedOperands.stream() - .map( - operand -> - (Expression) - Expressions.call( - ExprValueUtils.class, - "fromObjectValue", - Expressions.convert_(Expressions.box(operand), Object.class))) - .toList(); - - Expression returnTypeName = Expressions.constant(call.getType().getSqlTypeName().toString()); - - Expression result = - Expressions.call( - EnhancedCoalesceFunction.class, - "enhancedCoalesceWithType", - Expressions.newArrayInit(ExprValue.class, exprValues), - returnTypeName); - - return Expressions.call(result, "valueForCalcite"); - }; - } - - public static ExprValue enhancedCoalesceWithType(ExprValue[] args, String returnTypeName) { - for (ExprValue arg : args) { - if (arg != null && !arg.isNull() && !arg.isMissing()) { - return coerceToType(arg, returnTypeName); - } - } - return ExprValueUtils.nullValue(); - } - - private static ExprValue coerceToType(ExprValue value, String typeName) { - return switch (typeName) { - case "INTEGER" -> tryConvert(() -> ExprValueUtils.integerValue(value.integerValue()), value); - case "BIGINT" -> tryConvert(() -> ExprValueUtils.longValue(value.longValue()), value); - case "SMALLINT", "TINYINT" -> - tryConvert(() -> ExprValueUtils.integerValue(value.integerValue()), value); - case "DOUBLE" -> tryConvert(() -> ExprValueUtils.doubleValue(value.doubleValue()), value); - case "FLOAT", "REAL" -> - tryConvert(() -> ExprValueUtils.floatValue(value.floatValue()), value); - case "BOOLEAN" -> tryConvert(() -> ExprValueUtils.booleanValue(value.booleanValue()), value); - case "VARCHAR", "CHAR" -> - tryConvert(() -> ExprValueUtils.stringValue(String.valueOf(value.value())), value); - case "DATE" -> tryConvert(() -> ExprValueUtils.dateValue(value.dateValue()), value); - case "TIME" -> tryConvert(() -> ExprValueUtils.timeValue(value.timeValue()), value); - case "TIMESTAMP" -> - tryConvert(() -> ExprValueUtils.timestampValue(value.timestampValue()), value); - case "DECIMAL" -> tryConvert(() -> ExprValueUtils.doubleValue(value.doubleValue()), value); - default -> value; - }; - } - - private static ExprValue tryConvert(Supplier converter, ExprValue fallbackValue) { - try { - return converter.get(); - } catch (Exception e) { - return ExprValueUtils.stringValue(String.valueOf(fallbackValue.value())); - } - } - - @Override - public SqlReturnTypeInference getReturnTypeInference() { - return opBinding -> { - var operandTypes = opBinding.collectOperandTypes(); - - // Let Calcite determine the least restrictive common type - var commonType = opBinding.getTypeFactory().leastRestrictive(operandTypes); - return commonType != null - ? commonType - : opBinding.getTypeFactory().createSqlType(SqlTypeName.VARCHAR); - }; - } - - @Override - public UDFOperandMetadata getOperandMetadata() { - return null; - } -} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CidrMatchFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CidrMatchFunction.java index c3a4fe4efe6..ac9f1335936 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CidrMatchFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CidrMatchFunction.java @@ -11,13 +11,20 @@ import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; -import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; import org.opensearch.sql.expression.ip.IPFunctions; @@ -47,10 +54,29 @@ public UDFOperandMetadata getOperandMetadata() { // EXPR_IP is mapped to SqlTypeFamily.OTHER in // UserDefinedFunctionUtils.convertRelDataTypeToSqlTypeName // We use a specific type checker to serve - return UDFOperandMetadata.wrapUDT( - List.of( - List.of(ExprCoreType.IP, ExprCoreType.STRING), - List.of(ExprCoreType.STRING, ExprCoreType.STRING))); + return UDFOperandMetadata.wrap( + OperandTypes.CHARACTER_CHARACTER.or( + new SqlOperandTypeChecker() { + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + if (!getOperandCountRange().isValidCount(callBinding.getOperandCount())) { + return false; + } + List types = callBinding.collectOperandTypes(); + return OpenSearchTypeUtil.isIp(types.get(0), true) + && OpenSearchTypeUtil.isCharacter(types.get(1)); + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.of(2); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return "CIDRMATCH(, )"; + } + })); } public static class CidrMatchImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java index ce200323f60..13ec0016bc0 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java @@ -15,18 +15,22 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.SqlCallBinding; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperandCountRange; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSyntax; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.InferTypes; import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.validate.SqlUserDefinedFunction; import org.checkerframework.checker.nullness.qual.Nullable; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.data.model.ExprIpValue; -import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.PPLBuiltinOperators; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -120,7 +124,27 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return UDFOperandMetadata.wrapUDT(List.of(List.of(ExprCoreType.IP, ExprCoreType.IP))); + return UDFOperandMetadata.wrap( + new SqlOperandTypeChecker() { + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + if (!getOperandCountRange().isValidCount(callBinding.getOperandCount())) { + return false; + } + return OpenSearchTypeUtil.isIp(callBinding.getOperandType(0), true) + && OpenSearchTypeUtil.isIp(callBinding.getOperandType(1), true); + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.of(2); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return String.format(Locale.ROOT, "%s(, )", opName); + } + }); } public static class CompareImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java index baf6b8a37e1..a3684c26f72 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java @@ -11,9 +11,16 @@ import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.type.ExprCoreType; @@ -40,8 +47,28 @@ public IPFunction() { @Override public UDFOperandMetadata getOperandMetadata() { - return UDFOperandMetadata.wrapUDT( - List.of(List.of(ExprCoreType.IP), List.of(ExprCoreType.STRING))); + return UDFOperandMetadata.wrap( + new SqlOperandTypeChecker() { + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + RelDataType type = callBinding.getOperandType(0); + boolean valid = OpenSearchTypeUtil.isIp(type) || OpenSearchTypeUtil.isCharacter(type); + if (!valid && throwOnFailure) { + throw callBinding.newValidationSignatureError(); + } + return valid; + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.of(1); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return "IP(), IP()"; + } + }); } @Override diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/DivideFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/DivideFunction.java index 1767a2fc69b..b570acc429c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/DivideFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/DivideFunction.java @@ -15,6 +15,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; @@ -41,6 +42,11 @@ public DivideFunction() { super(new DivideImplementor(), NullPolicy.ANY); } + @Override + public SqlKind getKind() { + return SqlKind.DIVIDE; + } + @Override public SqlReturnTypeInference getReturnTypeInference() { return ReturnTypes.QUOTIENT_FORCE_NULLABLE; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ModFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ModFunction.java index 7a8f8e75f92..5de9399adf2 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ModFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ModFunction.java @@ -14,6 +14,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeFamily; @@ -36,6 +37,11 @@ public SqlReturnTypeInference getReturnTypeInference() { return ReturnTypes.LEAST_RESTRICTIVE.andThen(SqlTypeTransforms.FORCE_NULLABLE); } + @Override + public SqlKind getKind() { + return SqlKind.MOD; + } + @Override public UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.NUMERIC_NUMERIC; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ScalarMaxFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ScalarMaxFunction.java index 9b4b0b48c73..90810f85bc5 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ScalarMaxFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ScalarMaxFunction.java @@ -14,6 +14,7 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.data.utils.MixedTypeComparator; @@ -37,7 +38,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return null; + return UDFOperandMetadata.wrap(OperandTypes.VARIADIC); } public static class MaxImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ScalarMinFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ScalarMinFunction.java index 441257a422e..f31f533187f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ScalarMinFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ScalarMinFunction.java @@ -14,6 +14,7 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.data.utils.MixedTypeComparator; @@ -37,7 +38,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return null; + return UDFOperandMetadata.wrap(OperandTypes.VARIADIC); } public static class MinImplementor implements NotNullImplementor { diff --git a/core/src/test/java/org/opensearch/sql/expression/function/AggFunctionTestBase.java b/core/src/test/java/org/opensearch/sql/expression/function/AggFunctionTestBase.java index d20841a2cee..8215c99b695 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/AggFunctionTestBase.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/AggFunctionTestBase.java @@ -16,22 +16,19 @@ public abstract class AggFunctionTestBase { @SuppressWarnings("unchecked") - protected Map> - getAggFunctionRegistry() { + protected Map getAggFunctionRegistry() { try { PPLFuncImpTable funcTable = PPLFuncImpTable.INSTANCE; Field field = PPLFuncImpTable.class.getDeclaredField("aggFunctionRegistry"); field.setAccessible(true); - return (Map>) - field.get(funcTable); + return (Map) field.get(funcTable); } catch (Exception e) { throw new RuntimeException("Failed to access aggFunctionRegistry", e); } } protected void assertFunctionIsRegistered(BuiltinFunctionName functionName) { - Map> registry = - getAggFunctionRegistry(); + Map registry = getAggFunctionRegistry(); assertTrue( registry.containsKey(functionName), functionName.getName().getFunctionName() @@ -48,36 +45,23 @@ protected void assertFunctionsAreRegistered(BuiltinFunctionName... functionNames } protected void assertFunctionHandlerTypes(BuiltinFunctionName... functionNames) { - Map> registry = - getAggFunctionRegistry(); + Map registry = getAggFunctionRegistry(); for (BuiltinFunctionName functionName : functionNames) { - org.apache.commons.lang3.tuple.Pair registryEntry = registry.get(functionName); - assertNotNull( - registryEntry, functionName.getName().getFunctionName() + " should be registered"); - - // Extract the AggHandler from the pair - AggHandler handler = registryEntry.getRight(); - + AggHandler handler = registry.get(functionName); assertNotNull( handler, functionName.getName().getFunctionName() + " handler should not be null"); - assertTrue( - handler instanceof AggHandler, - functionName.getName().getFunctionName() - + " handler should implement AggHandler interface"); } } protected void assertRegistryMinimumSize(int expectedMinimumSize) { - Map> registry = - getAggFunctionRegistry(); + Map registry = getAggFunctionRegistry(); assertTrue( registry.size() >= expectedMinimumSize, "Registry should contain at least " + expectedMinimumSize + " aggregate functions"); } protected void assertKnownFunctionsPresent(Set knownFunctions) { - Map> registry = - getAggFunctionRegistry(); + Map registry = getAggFunctionRegistry(); long foundFunctions = registry.keySet().stream().filter(knownFunctions::contains).count(); assertTrue( diff --git a/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java b/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java deleted file mode 100644 index 30d827f1ecc..00000000000 --- a/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.expression.function; - -import static org.junit.jupiter.api.Assertions.*; -import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; -import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; -import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; - -import java.util.List; -import java.util.stream.Stream; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexNode; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; -import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; - -class CoercionUtilsTest { - - private static final RexBuilder REX_BUILDER = new RexBuilder(OpenSearchTypeFactory.TYPE_FACTORY); - - private static RexNode nullLiteral(ExprCoreType type) { - return REX_BUILDER.makeNullLiteral(OpenSearchTypeFactory.convertExprTypeToRelDataType(type)); - } - - private static Stream commonWidestTypeArguments() { - return Stream.of( - Arguments.of(STRING, INTEGER, DOUBLE), - Arguments.of(INTEGER, STRING, DOUBLE), - Arguments.of(STRING, DOUBLE, DOUBLE), - Arguments.of(INTEGER, BOOLEAN, null)); - } - - @ParameterizedTest - @MethodSource("commonWidestTypeArguments") - public void findCommonWidestType( - ExprCoreType left, ExprCoreType right, ExprCoreType expectedCommonType) { - assertEquals( - expectedCommonType, CoercionUtils.resolveCommonType(left, right).orElseGet(() -> null)); - } - - @Test - void castArgumentsReturnsExactMatchWhenAvailable() { - PPLTypeChecker typeChecker = new StubTypeChecker(List.of(List.of(INTEGER), List.of(DOUBLE))); - List arguments = List.of(nullLiteral(INTEGER)); - - List result = CoercionUtils.castArguments(REX_BUILDER, typeChecker, arguments); - - assertNotNull(result); - assertEquals(1, result.size()); - assertEquals( - INTEGER, OpenSearchTypeFactory.convertRelDataTypeToExprType(result.getFirst().getType())); - } - - @Test - void castArgumentsFallsBackToWidestCandidate() { - PPLTypeChecker typeChecker = - new StubTypeChecker(List.of(List.of(ExprCoreType.LONG), List.of(DOUBLE))); - List arguments = List.of(nullLiteral(STRING)); - - List result = CoercionUtils.castArguments(REX_BUILDER, typeChecker, arguments); - - assertNotNull(result); - assertEquals( - DOUBLE, OpenSearchTypeFactory.convertRelDataTypeToExprType(result.getFirst().getType())); - } - - @Test - void castArgumentsReturnsNullWhenNoCompatibleSignatureExists() { - PPLTypeChecker typeChecker = new StubTypeChecker(List.of(List.of(ExprCoreType.GEO_POINT))); - List arguments = List.of(nullLiteral(INTEGER)); - - assertNull(CoercionUtils.castArguments(REX_BUILDER, typeChecker, arguments)); - } - - private static class StubTypeChecker implements PPLTypeChecker { - private final List> signatures; - - private StubTypeChecker(List> signatures) { - this.signatures = signatures; - } - - @Override - public boolean checkOperandTypes(List types) { - return false; - } - - @Override - public String getAllowedSignatures() { - return ""; - } - - @Override - public List> getParameterTypes() { - return signatures; - } - } -} diff --git a/docs/user/ppl/cmd/bin.md b/docs/user/ppl/cmd/bin.md index 7f8ef389bd1..e9c87fce821 100644 --- a/docs/user/ppl/cmd/bin.md +++ b/docs/user/ppl/cmd/bin.md @@ -450,9 +450,11 @@ fetched rows / total rows = 3/3 +-----------+----------------+ ``` -## Example 20: Binning with string fields - -```ppl +## Example 20: Binning with string fields + + + +```ppl ignore source=accounts | eval age_str = CAST(age AS STRING) | bin age_str bins=3 diff --git a/docs/user/ppl/cmd/patterns.md b/docs/user/ppl/cmd/patterns.md index 7b9cb718891..5508fa2e1a9 100644 --- a/docs/user/ppl/cmd/patterns.md +++ b/docs/user/ppl/cmd/patterns.md @@ -214,8 +214,8 @@ fetched rows / total rows = 4/4 ## Brain Example 3: Return log patterns aggregation result This example shows how to get aggregated results from a raw log field using the brain algorithm. - -```ppl + +```ppl ignore source=apache | patterns message method=brain mode=aggregation variable_count_threshold=2 | fields patterns_field, pattern_count, sample_logs diff --git a/docs/user/ppl/functions/collection.md b/docs/user/ppl/functions/collection.md index c37f8390ddf..8a141776521 100644 --- a/docs/user/ppl/functions/collection.md +++ b/docs/user/ppl/functions/collection.md @@ -162,8 +162,8 @@ Usage: `transform(array, function)` transform the element of array one by one us Argument type: array:ARRAY, function:LAMBDA Return type: ARRAY Example - -```ppl + +```ppl ignore source=people | eval array = array(1, -2, 3), result = transform(array, x -> x + 2) | fields result @@ -180,8 +180,8 @@ fetched rows / total rows = 1/1 | [3,0,5] | +---------+ ``` - -```ppl + +```ppl ignore source=people | eval array = array(1, -2, 3), result = transform(array, (x, i) -> x + i) | fields result @@ -814,8 +814,8 @@ Usage: mvmap(array, expression) iterates over each element of a multivalue array Argument type: array: ARRAY, expression: EXPRESSION Return type: ARRAY Example - -```ppl + +```ppl ignore source=people | eval array = array(1, 2, 3), result = mvmap(array, array * 10) | fields result @@ -832,8 +832,8 @@ fetched rows / total rows = 1/1 | [10,20,30] | +------------+ ``` - -```ppl + +```ppl ignore source=people | eval array = array(1, 2, 3), result = mvmap(array, array + 5) | fields result @@ -854,8 +854,8 @@ fetched rows / total rows = 1/1 Note: For nested expressions like ``mvmap(mvindex(arr, 1, 3), arr * 2)``, the field name (``arr``) is extracted from the first argument and must match the field referenced in the expression. The expression can also reference other single-value fields: - -```ppl + +```ppl ignore source=people | eval array = array(1, 2, 3), multiplier = 10, result = mvmap(array, array * multiplier) | fields result diff --git a/docs/user/ppl/functions/conversion.md b/docs/user/ppl/functions/conversion.md index 9e3b1d1ed7b..4a258b9dbe6 100644 --- a/docs/user/ppl/functions/conversion.md +++ b/docs/user/ppl/functions/conversion.md @@ -120,15 +120,15 @@ Expected output: ```text fetched rows / total rows = 1/1 -+--------+----------+------+-------+--------+ -| divide | multiply | add | minus | concat | -|--------+----------+------+-------+--------| -| 0.5 | 50.0 | 15.0 | -5.0 | 55 | -+--------+----------+------+-------+--------+ ++--------+----------+-----+-------+--------+ +| divide | multiply | add | minus | concat | +|--------+----------+-----+-------+--------| +| 0.5 | 50 | 15 | -5 | 55 | ++--------+----------+-----+-------+--------+ ``` Use string in comparison operator example - + ```ppl source=people | eval e="1000"==1000, en="1000"!=1000, ed="1000"==1000.0, edn="1000"!=1000.0, l="1000">999, ld="1000">999.9, i="malformed"==1000 diff --git a/docs/user/ppl/interfaces/endpoint.md b/docs/user/ppl/interfaces/endpoint.md index e1e9cf705bf..dd5fb8beddd 100644 --- a/docs/user/ppl/interfaces/endpoint.md +++ b/docs/user/ppl/interfaces/endpoint.md @@ -77,7 +77,7 @@ Expected output: ```json { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(name=[$0], country=[$1], state=[$2], month=[$3], year=[$4], age=[$5])\n LogicalFilter(condition=[>($5, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, state_country]])\n", + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(name=[$0], country=[$1], state=[$2], month=[$3], year=[$4], age=[$5])\n LogicalFilter(condition=[>($5, SAFE_CAST(30:BIGINT))])\n CalciteLogicalIndexScan(table=[[OpenSearch, state_country]])\n", "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, state_country]], PushDownContext=[[PROJECT->[name, country, state, month, year, age], FILTER->>($5, 30), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"name\",\"country\",\"state\",\"month\",\"year\",\"age\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } @@ -119,7 +119,7 @@ Expected output: ```json { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(name=[$0], country=[$1], state=[$2], month=[$3], year=[$4], age=[$5])\n LogicalFilter(condition=[>($5, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, state_country]])\n", + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(name=[$0], country=[$1], state=[$2], month=[$3], year=[$4], age=[$5])\n LogicalFilter(condition=[>($5, SAFE_CAST(30:BIGINT))])\n CalciteLogicalIndexScan(table=[[OpenSearch, state_country]])\n", "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, state_country]], PushDownContext=[[PROJECT->[name, country, state, month, year, age], FILTER->>($5, 30), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"name\",\"country\",\"state\",\"month\",\"year\",\"age\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n", "extended": "public org.apache.calcite.linq4j.Enumerable bind(final org.apache.calcite.DataContext root) {\n final org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan v1stashed = (org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan) root.get(\"v1stashed\");\n return v1stashed.scan();\n}\n\n\npublic Class getElementType() {\n return java.lang.Object[].class;\n}\n\n\n" } @@ -146,7 +146,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(name=[$0], country=[$1], state=[$2], month=[$3], year=[$4], age=[$5]) - LogicalFilter(condition=[>($5, 30)]) + LogicalFilter(condition=[>($5, SAFE_CAST(30:BIGINT))]) CalciteLogicalIndexScan(table=[[OpenSearch, state_country]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, state_country]], PushDownContext=[[PROJECT->[name, country, state, month, year, age], FILTER->>($5, 30), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["name","country","state","month","year","age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java index 76b4d772e0a..fe4c1de39e9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java @@ -13,6 +13,7 @@ import java.util.Map; import java.util.Set; import org.junit.AfterClass; +import org.junit.Assume; import org.junit.FixMethodOrder; import org.junit.Test; import org.junit.runners.MethodSorters; @@ -64,6 +65,10 @@ protected Set ignored() throws IOException { // because of too much script push down, which will cause ResourceMonitor restriction. ignored.add(30); } + if (isCalciteEnabled()) { + // Ignore q41 as it needs special handling + ignored.add(41); + } return ignored; } @@ -83,4 +88,15 @@ public void test() throws IOException { timing(summary, "q" + i, ppl); } } + + @Test + public void testQ41() throws IOException { + Assume.assumeTrue(isCalciteEnabled()); + logger.info("Running Query 41"); + String ppl = sanitize(loadFromFile("clickbench/queries/q41.ppl")); + String expected = loadExpectedPlan("clickbench/q41.yaml"); + String alternative = loadExpectedPlan("clickbench/q41_alternative.yaml"); + assertYamlEqualsIgnoreId(expected, alternative, explainQueryYaml(ppl)); + timing(summary, "q" + 41, ppl); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteArrayFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteArrayFunctionIT.java index 1b32b76b611..871400b7cd0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteArrayFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteArrayFunctionIT.java @@ -65,9 +65,7 @@ public void testArrayWithMix() { TEST_INDEX_BANK))); verifyErrorMessageContains( - e, - "Cannot resolve function: ARRAY, arguments: [INTEGER,BOOLEAN], caused by: fail to create" - + " array with fixed type"); + e, "Cannot infer return type for array; operand types: [INTEGER, BOOLEAN]"); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index cdea1738eb0..6c407452dae 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -66,9 +66,9 @@ public void supportSearchSargPushDown_multiRange() throws IOException { String query = "source=opensearch-sql_test_index_account | where (age > 20 and age < 28) or (age > 25 and" + " age < 30) or (age >= 1 and age <= 10) or age = 0 | fields age"; - var result = explainQueryToString(query); - String expected = loadExpectedPlan("explain_sarg_filter_push_multi_range.json"); - assertJsonEqualsIgnoreId(expected, result); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_sarg_filter_push_multi_range.yaml"); + assertYamlEqualsIgnoreId(expected, result); } // Only for Calcite @@ -339,10 +339,10 @@ public void testExplainIsEmptyOrOthers() throws IOException { @Test public void testExplainIsNullOrOthers() throws IOException { // pushdown should work - String expected = loadExpectedPlan("explain_isnull_or_others.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_isnull_or_others.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | where isnull(firstname) or gender = 'M'")); } @@ -370,9 +370,9 @@ public void testPartialPushdownFilterWithIsNull() throws IOException { Locale.ROOT, "source=%s | where isnull(address) and name='david'", TEST_INDEX_NESTED_SIMPLE); - var result = explainQueryToString(query); - String expected = loadExpectedPlan("explain_partial_filter_isnull.json"); - assertJsonEqualsIgnoreId(expected, result); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_partial_filter_isnull.yaml"); + assertYamlEqualsIgnoreId(expected, result); } @Test @@ -381,9 +381,9 @@ public void testSkipScriptEncodingOnExtendedFormat() throws IOException { String query = "source=opensearch-sql_test_index_account | where address = '671 Bristol Street' and age -" + " 2 = 30 | fields firstname, age, address"; - var result = explainQueryToString(query, true); - String expected = loadFromFile("expectedOutput/calcite/explain_skip_script_encoding.json"); - assertJsonEqualsIgnoreId(expected, result); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_skip_script_encoding.yaml"); + assertYamlEqualsIgnoreId(expected, result); } // Only for Calcite, as v2 gets unstable serialized string for function @@ -441,42 +441,42 @@ public void testExplainWithTimechartCount() throws IOException { @Test public void testExplainTimechartPerSecond() throws IOException { - var result = explainQueryToString("source=events | timechart span=2m per_second(cpu_usage)"); + var result = explainQueryYaml("source=events | timechart span=2m per_second(cpu_usage)"); assertTrue( result.contains( - "per_second(cpu_usage)=[DIVIDE(*($1, 1000.0E0), TIMESTAMPDIFF('MILLISECOND':VARCHAR," - + " $0, TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]")); - assertTrue(result.contains("per_second(cpu_usage)=[SUM($0)]")); + "per_second(cpu_usage)=[DIVIDE(*($1, 1000.0E0), TIMESTAMPDIFF('MILLISECOND', $0," + + " TIMESTAMPADD('MINUTE', 2, $0)))]")); + assertTrue(result.contains("per_second(cpu_usage)=[SUM($1)]")); } @Test public void testExplainTimechartPerMinute() throws IOException { - var result = explainQueryToString("source=events | timechart span=2m per_minute(cpu_usage)"); + var result = explainQueryYaml("source=events | timechart span=2m per_minute(cpu_usage)"); assertTrue( result.contains( - "per_minute(cpu_usage)=[DIVIDE(*($1, 60000.0E0), TIMESTAMPDIFF('MILLISECOND':VARCHAR," - + " $0, TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]")); - assertTrue(result.contains("per_minute(cpu_usage)=[SUM($0)]")); + "per_minute(cpu_usage)=[DIVIDE(*($1, 60000.0E0), TIMESTAMPDIFF('MILLISECOND', $0," + + " TIMESTAMPADD('MINUTE', 2, $0)))]")); + assertTrue(result.contains("per_minute(cpu_usage)=[SUM($1)]")); } @Test public void testExplainTimechartPerHour() throws IOException { - var result = explainQueryToString("source=events | timechart span=2m per_hour(cpu_usage)"); + var result = explainQueryYaml("source=events | timechart span=2m per_hour(cpu_usage)"); assertTrue( result.contains( - "per_hour(cpu_usage)=[DIVIDE(*($1, 3600000.0E0), TIMESTAMPDIFF('MILLISECOND':VARCHAR," - + " $0, TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]")); - assertTrue(result.contains("per_hour(cpu_usage)=[SUM($0)]")); + "per_hour(cpu_usage)=[DIVIDE(*($1, 3600000.0E0), TIMESTAMPDIFF('MILLISECOND', $0," + + " TIMESTAMPADD('MINUTE', 2, $0)))]")); + assertTrue(result.contains("per_hour(cpu_usage)=[SUM($1)]")); } @Test public void testExplainTimechartPerDay() throws IOException { - var result = explainQueryToString("source=events | timechart span=2m per_day(cpu_usage)"); + var result = explainQueryYaml("source=events | timechart span=2m per_day(cpu_usage)"); assertTrue( result.contains( - "per_day(cpu_usage)=[DIVIDE(*($1, 8.64E7), TIMESTAMPDIFF('MILLISECOND':VARCHAR, $0," - + " TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]")); - assertTrue(result.contains("per_day(cpu_usage)=[SUM($0)]")); + "per_day(cpu_usage)=[DIVIDE(*($1, 8.64E7), TIMESTAMPDIFF('MILLISECOND', $0," + + " TIMESTAMPADD('MINUTE', 2, $0)))]")); + assertTrue(result.contains("per_day(cpu_usage)=[SUM($1)]")); } @Test @@ -504,10 +504,10 @@ public void supportPushDownScriptOnTextField() throws IOException { @Test public void testExplainBinWithBins() throws IOException { - String expected = loadExpectedPlan("explain_bin_bins.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_bin_bins.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString("source=opensearch-sql_test_index_account | bin age bins=3 | head 5")); + explainQueryYaml("source=opensearch-sql_test_index_account | bin age bins=3 | head 5")); } @Test @@ -554,19 +554,18 @@ public void testExplainBinWithSpan() throws IOException { @Test public void testExplainBinWithMinspan() throws IOException { - String expected = loadExpectedPlan("explain_bin_minspan.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_bin_minspan.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( - "source=opensearch-sql_test_index_account | bin age minspan=5 | head 5")); + explainQueryYaml("source=opensearch-sql_test_index_account | bin age minspan=5 | head 5")); } @Test public void testExplainBinWithStartEnd() throws IOException { - String expected = loadExpectedPlan("explain_bin_start_end.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_bin_start_end.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | bin balance start=0 end=100001 | head 5")); } @@ -584,9 +583,9 @@ public void testExplainBinWithAligntime() throws IOException { public void testExplainCountEval() throws IOException { String query = "source=opensearch-sql_test_index_bank | stats count(eval(age > 30)) as mature_count"; - var result = explainQueryToString(query); - String expected = loadExpectedPlan("explain_count_eval_push.json"); - assertJsonEqualsIgnoreId(expected, result); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_count_eval_push.yaml"); + assertYamlEqualsIgnoreId(expected, result); } @Test @@ -594,9 +593,9 @@ public void testExplainCountEvalComplex() throws IOException { String query = "source=opensearch-sql_test_index_bank | stats count(eval(age > 30 and age < 50)) as" + " mature_count"; - var result = explainQueryToString(query); - String expected = loadExpectedPlan("explain_count_eval_complex_push.json"); - assertJsonEqualsIgnoreId(expected, result); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_count_eval_complex_push.yaml"); + assertYamlEqualsIgnoreId(expected, result); } @Test @@ -762,9 +761,8 @@ public void testExplainRegexMatchInEvalWithOutScriptPushdown() throws IOExceptio String.format( "source=%s |eval has_hello = regexp_match(name, 'hello') | fields has_hello", TEST_INDEX_STRINGS); - var result = explainQueryToString(query); - String expected = loadFromFile("expectedOutput/calcite/explain_regexp_match_in_eval.json"); - assertJsonEqualsIgnoreId(expected, result); + var result = explainQueryYaml(query); + assertYamlEqualsIgnoreId(loadExpectedPlan("explain_regexp_match_in_eval.yaml"), result); } // Only for Calcite @@ -1076,10 +1074,10 @@ public void testExplainAppendCommand() throws IOException { @Test public void testExplainAppendPipeCommand() throws IOException { - String expected = loadExpectedPlan("explain_appendpipe_command.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_appendpipe_command.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( Locale.ROOT, "source=%s | appendpipe [ stats count(balance) as cnt by gender ]", @@ -1091,9 +1089,7 @@ public void testMvjoinExplain() throws IOException { String query = "source=opensearch-sql_test_index_account | eval result = mvjoin(array('a', 'b', 'c'), ',')" + " | fields result | head 1"; - var result = explainQueryToString(query); - String expected = loadExpectedPlan("explain_mvjoin.json"); - assertJsonEqualsIgnoreId(expected, result); + assertYamlEqualsIgnoreId(loadExpectedPlan("explain_mvjoin.yaml"), explainQueryYaml(query)); } @Test @@ -1110,50 +1106,43 @@ public void testPreventLimitPushdown() throws IOException { @Test public void testPushdownLimitIntoAggregation() throws IOException { enabledOnlyWhenPushdownIsEnabled(); - String expected = loadExpectedPlan("explain_limit_agg_pushdown.json"); - assertJsonEqualsIgnoreId( - expected, - explainQueryToString("source=opensearch-sql_test_index_account | stats count() by state")); + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_limit_agg_pushdown.yaml"), + explainQueryYaml("source=opensearch-sql_test_index_account | stats count() by state")); - expected = loadExpectedPlan("explain_limit_agg_pushdown2.yaml"); assertYamlEqualsIgnoreId( - expected, + loadExpectedPlan("explain_limit_agg_pushdown2.yaml"), explainQueryYaml( "source=opensearch-sql_test_index_account | stats count() by state | head 100")); - expected = loadExpectedPlan("explain_limit_agg_pushdown3.json"); - assertJsonEqualsIgnoreId( - expected, - explainQueryToString( + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_limit_agg_pushdown3.yaml"), + explainQueryYaml( "source=opensearch-sql_test_index_account | stats count() by state | head 100 | head 10" + " from 10 ")); - expected = loadExpectedPlan("explain_limit_agg_pushdown4.yaml"); assertYamlEqualsIgnoreId( - expected, + loadExpectedPlan("explain_limit_agg_pushdown4.yaml"), explainQueryYaml( "source=opensearch-sql_test_index_account | stats count() by state | sort state | head" + " 100 | head 10 from 10 ")); - expected = loadExpectedPlan("explain_limit_agg_pushdown_bucket_nullable1.yaml"); assertYamlEqualsIgnoreId( - expected, + loadExpectedPlan("explain_limit_agg_pushdown_bucket_nullable1.yaml"), explainQueryYaml( "source=opensearch-sql_test_index_account | stats bucket_nullable=false count() by" + " state | head 100 | head 10 from 10 ")); - expected = loadExpectedPlan("explain_limit_agg_pushdown_bucket_nullable2.yaml"); assertYamlEqualsIgnoreId( - expected, + loadExpectedPlan("explain_limit_agg_pushdown_bucket_nullable2.yaml"), explainQueryYaml( "source=opensearch-sql_test_index_account | stats bucket_nullable=false count() by" + " state | sort state | head 100 | head 10 from 10 ")); - // Don't pushdown the combination of limit and sort - expected = loadExpectedPlan("explain_limit_agg_pushdown5.json"); - assertJsonEqualsIgnoreId( - expected, - explainQueryToString( + // Don't push down the combination of limit and sort + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_limit_agg_pushdown5.yaml"), + explainQueryYaml( "source=opensearch-sql_test_index_account | stats count() by state | sort `count()` |" + " head 100 | head 10 from 10 ")); } @@ -1626,9 +1615,9 @@ public void testStrftimeFunctionExplain() throws IOException { String query = "source=opensearch-sql_test_index_account | eval formatted_date = strftime(1521467703," + " '%Y-%m-%d') | fields formatted_date | head 1"; - var result = explainQueryToString(query); - String expected = loadExpectedPlan("explain_strftime_function.json"); - assertJsonEqualsIgnoreId(expected, result); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_strftime_function.yaml"); + assertYamlEqualsIgnoreId(expected, result); } // Script generation is not stable in v2 @@ -1817,6 +1806,7 @@ public void testExplainChartWithLimits() throws IOException { String.format( "source=%s | chart limit=0 avg(balance) over state by gender", TEST_INDEX_BANK))); + // TODO: Fix -- now it gets away from push-down assertYamlEqualsIgnoreId( loadExpectedPlan("chart_use_other.yaml"), explainQueryYaml( @@ -2206,13 +2196,12 @@ public void testAliasTypeField() throws IOException { @Test public void testRexStandardizationForScript() throws IOException { enabledOnlyWhenPushdownIsEnabled(); - assertJsonEqualsIgnoreId( - loadExpectedPlan("explain_extended_for_standardization.json"), - explainQueryToString( + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_extended_for_standardization.yaml"), + explainQueryYaml( String.format( "source=%s | eval age_range = case(age < 30, 'u30', age >= 30 and age <= 40, 'u40'" + " else 'u100') | stats avg(age) as avg_age by age_range", - TEST_INDEX_BANK), - true)); + TEST_INDEX_BANK))); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java index 393b0a4a501..59536d11bfd 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java @@ -152,10 +152,10 @@ public void testMultisearchWithTimestampInterleaving() throws IOException { verifySchema( result, - schema("@timestamp", null, "string"), + schema("@timestamp", null, "timestamp"), schema("category", null, "string"), schema("value", null, "int"), - schema("timestamp", null, "string")); + schema("timestamp", null, "timestamp")); verifyDataRows( result, diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index 9710d2f4415..7d846e9b6f7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -27,7 +27,6 @@ import org.junit.jupiter.api.Test; import org.opensearch.client.Request; import org.opensearch.sql.common.utils.StringUtils; -import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.ppl.PPLIntegTestCase; public class CalcitePPLAggregationIT extends PPLIntegTestCase { @@ -742,7 +741,7 @@ public void testSpanByImplicitTimestamp() throws IOException { Throwable t = assertThrowsWithReplace( - SemanticCheckException.class, + IllegalArgumentException.class, () -> executeQuery( StringUtils.format( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendCommandIT.java index d01ddfb2a44..6372b818b2b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendCommandIT.java @@ -253,7 +253,7 @@ public void testAppendSchemaMergeWithTimestampUDT() throws IOException { schema("account_number", "bigint"), schema("firstname", "string"), schema("age", "int"), - schema("birthdate", "string")); + schema("birthdate", "timestamp")); verifyDataRows(actual, rows(32, null, 34, "2018-08-11 00:00:00")); } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendcolIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendcolIT.java index 877c10947b8..f634e12a709 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendcolIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendcolIT.java @@ -26,6 +26,9 @@ public void init() throws Exception { @Test public void testAppendCol() throws IOException { + // Although the plans are identical, not pushing down resulting the cnt in the first two rows + // being null + enabledOnlyWhenPushdownIsEnabled(); JSONObject actual = executeQuery( String.format( @@ -55,6 +58,9 @@ public void testAppendCol() throws IOException { @Test public void testAppendColOverride() throws IOException { + // Although the plans are identical, not pushing down resulting the cnt in the first two rows + // being null + enabledOnlyWhenPushdownIsEnabled(); JSONObject actual = executeQuery( String.format( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java index 5f69159fec5..13e66e0232d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java @@ -10,6 +10,7 @@ import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifyErrorMessageContains; +import static org.opensearch.sql.util.MatcherUtils.verifyNumOfRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; @@ -17,7 +18,6 @@ import org.junit.jupiter.api.Test; import org.opensearch.client.Request; import org.opensearch.client.ResponseException; -import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.ppl.PPLIntegTestCase; public class CalcitePPLBasicIT extends PPLIntegTestCase { @@ -455,16 +455,27 @@ public void testBetweenWithDifferentTypes2() throws IOException { } @Test - public void testBetweenWithIncompatibleTypes() { - Throwable e = - assertThrowsWithReplace( - SemanticCheckException.class, - () -> - executeQuery( - String.format( - "source=%s | where age between '35' and 38.5 | fields firstname, age", - TEST_INDEX_BANK))); - verifyErrorMessageContains(e, "BETWEEN expression types are incompatible"); + public void testBetweenWithMixedTypes() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where age between '35' and 38 | fields firstname, age", + TEST_INDEX_BANK)); + verifyDataRows(actual, rows("Hattie", 36), rows("Elinor", 36)); + } + + @Test + public void testBetweenWithIncompatibleTypes() throws IOException { + // Plan: SAFE_CAST(NUMBER_TO_STRING(38.5:DECIMAL(3, 1))). The least restrictive type between + // int, decimal, and varchar is resolved to varchar. between '35' and '38.5' is then optimized + // to empty rows + JSONObject actual = + executeQuery( + String.format( + "source=%s | where age between '35' and 38.5 | fields firstname, age", + TEST_INDEX_BANK)); + verifySchema(actual, schema("firstname", "string"), schema("age", "int")); + verifyNumOfRows(actual, 0); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java index 674a7d96f8d..c594eaad287 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.util.MatcherUtils.assertJsonEquals; +import static org.opensearch.sql.util.MatcherUtils.assertYamlEqualsIgnoreId; import java.io.IOException; import org.junit.jupiter.api.Test; @@ -33,13 +34,9 @@ public void init() throws Exception { @Test public void testExplainCommand() throws IOException { - var result = explainQueryToString("source=test | where age = 20 | fields name, age"); - String expected = - !isPushdownDisabled() - ? loadFromFile("expectedOutput/calcite/explain_filter_w_pushdown.json") - : loadFromFile("expectedOutput/calcite/explain_filter_wo_pushdown.json"); - - assertJsonEquals(expected, result); + var result = explainQueryYaml("source=test | where age = 20 | fields name, age"); + String expected = loadExpectedPlan("explain_filter.yaml"); + assertYamlEqualsIgnoreId(expected, result); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPatternsIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPatternsIT.java index 46df914e611..bf34b8eb6c2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPatternsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPatternsIT.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableMap; import java.io.IOException; import org.json.JSONObject; +import org.junit.Ignore; import org.junit.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; @@ -287,6 +288,7 @@ public void testBrainLabelMode_ShowNumberedToken() throws IOException { ImmutableList.of("-6620182933895093708")))); } + @Ignore("To be fixed in https://github.com/opensearch-project/sql/issues/4968") @Test public void testBrainAggregationMode_NotShowNumberedToken() throws IOException { JSONObject result = @@ -337,6 +339,7 @@ public void testBrainAggregationMode_NotShowNumberedToken() throws IOException { "PacketResponder failed for blk_-1547954353065580372"))); } + @Ignore("To be fixed in https://github.com/opensearch-project/sql/issues/4968") @Test public void testBrainAggregationMode_ShowNumberedToken() throws IOException { JSONObject result = @@ -420,6 +423,7 @@ public void testBrainAggregationMode_ShowNumberedToken() throws IOException { "PacketResponder failed for blk_-1547954353065580372"))); } + @Ignore("To be fixed in https://github.com/opensearch-project/sql/issues/4968") @Test public void testBrainAggregationModeWithGroupByClause_ShowNumberedToken() throws IOException { JSONObject result = diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java index 582ce47000d..c2ebaebb04a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java @@ -5,6 +5,13 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; import org.opensearch.sql.ppl.WhereCommandIT; public class CalciteWhereCommandIT extends WhereCommandIT { @@ -14,9 +21,14 @@ public void init() throws Exception { enableCalcite(); } - @Override - protected String getIncompatibleTypeErrMsg() { - return "In expression types are incompatible: fields type LONG, values type [INTEGER, INTEGER," - + " STRING]"; + @Test + public void testInWithMixedType() throws IOException { + // Mixed type coercion only work with Calcite enabled + JSONObject result = + executeQuery( + String.format( + "source=%s | where balance in (4180, 5686, '6077') | fields firstname", + TEST_INDEX_ACCOUNT)); + verifyDataRows(result, rows("Hattie"), rows("Dale"), rows("Hughes")); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java index cbfe3c84464..a9fef879f36 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java @@ -19,6 +19,7 @@ import java.time.LocalTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +import java.util.Locale; import java.util.TimeZone; import org.json.JSONObject; import org.junit.After; @@ -26,7 +27,6 @@ import org.junit.Ignore; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; -import org.opensearch.sql.common.utils.StringUtils; @SuppressWarnings("unchecked") public class DateTimeFunctionIT extends PPLIntegTestCase { @@ -1139,9 +1139,12 @@ public void testToDays() throws IOException { private void week(String date, int mode, int expectedResult) throws IOException { JSONObject result = executeQuery( - StringUtils.format( + String.format( + Locale.ROOT, "source=%s | eval f = week(date('%s'), %d) | fields f", - TEST_INDEX_DATE, date, mode)); + TEST_INDEX_DATE, + date, + mode)); verifySchema(result, schema("f", null, "int")); verifySome(result.getJSONArray("datarows"), rows(expectedResult)); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java index 62eadd7ef5e..cce7d41ce4f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java @@ -101,9 +101,9 @@ public void testFilterByCompareStringTimePushDownExplain() throws IOException { @Test public void testFilterByCompareIPCoercion() throws IOException { // Should automatically cast the string literal to IP and pushdown it as a range query - assertJsonEqualsIgnoreId( - loadExpectedPlan("explain_filter_compare_ip.json"), - explainQueryToString( + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_filter_compare_ip.yaml"), + explainQueryYaml( String.format( Locale.ROOT, "source=%s | where host > '1.1.1.1' | fields host", @@ -116,9 +116,9 @@ public void testFilterByCompareIpv6Swapped() throws IOException { Assume.assumeTrue(isCalciteEnabled()); // Test swapping ip and string. In v2, this is pushed down as script; // with Calcite, it will still be pushed down as a range query - assertJsonEqualsIgnoreId( - loadExpectedPlan("explain_filter_compare_ipv6_swapped.json"), - explainQueryToString( + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_filter_compare_ipv6_swapped.yaml"), + explainQueryYaml( String.format( Locale.ROOT, "source=%s | where '::ffff:1234' <= host | fields host", @@ -127,12 +127,12 @@ public void testFilterByCompareIpv6Swapped() throws IOException { @Test public void testWeekArgumentCoercion() throws IOException { - String expected = loadExpectedPlan("explain_week_argument_coercion.json"); + String expected = loadExpectedPlan("explain_week_argument_coercion.yaml"); // Week accepts WEEK(timestamp/date/time, [optional int]), it should cast the string // argument to timestamp with Calcite. In v2, it accepts string, so there is no cast. - assertJsonEqualsIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( Locale.ROOT, "source=%s | eval w = week('2024-12-10') | fields w", @@ -160,10 +160,10 @@ public void testCountAggPushDownExplain() throws IOException { @Test public void testSortPushDownExplain() throws IOException { - String expected = loadExpectedPlan("explain_sort_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_sort_push.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| sort age " + "| where age > 30" @@ -180,10 +180,10 @@ public void testSortWithCountPushDownExplain() throws IOException { @Test public void testSortWithDescPushDownExplain() throws IOException { - String expected = loadExpectedPlan("explain_sort_desc_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_sort_desc_push.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | sort age desc, firstname | fields age," + " firstname")); } @@ -216,10 +216,10 @@ public void testSortWithAggregationExplain() throws IOException { public void testMultiSortPushDownExplain() throws IOException { // TODO: Fix the expected output in expectedOutput/ppl/explain_multi_sort_push.json (v2) // balance and gender should take precedence over account_number and firstname - String expected = loadExpectedPlan("explain_multi_sort_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_multi_sort_push.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account " + "| sort account_number, firstname, address, balance " + "| sort - balance, - gender, account_number " @@ -228,10 +228,10 @@ public void testMultiSortPushDownExplain() throws IOException { @Test public void testSortThenAggregatePushDownExplain() throws IOException { - String expected = loadExpectedPlan("explain_sort_then_agg_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_sort_then_agg_push.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| sort balance, age " + "| stats avg(balance) by state")); @@ -239,10 +239,10 @@ public void testSortThenAggregatePushDownExplain() throws IOException { @Test public void testSortWithRenameExplain() throws IOException { - String expected = loadExpectedPlan("explain_sort_rename_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_sort_rename_push.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account " + "| rename firstname as name " + "| eval alias = name " @@ -368,10 +368,10 @@ public void testLimitWithMultipleOffsetPushdownExplain() throws IOException { @Test public void testFillNullPushDownExplain() throws IOException { - String expected = loadExpectedPlan("explain_fillnull_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_fillnull_push.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + " | fillnull with -1 in age,balance | fields age, balance")); } @@ -431,6 +431,7 @@ public void testPatternsSimplePatternMethodWithAggPushDownExplain() throws IOExc + " show_numbered_token=true")); } + @Ignore("Enable after fixing https://github.com/opensearch-project/sql/issues/4968") @Test public void testPatternsBrainMethodWithAggPushDownExplain() throws IOException { // TODO: Correct calcite expected result once pushdown is supported @@ -444,19 +445,19 @@ public void testPatternsBrainMethodWithAggPushDownExplain() throws IOException { @Test public void testStatsBySpan() throws IOException { - String expected = loadExpectedPlan("explain_stats_by_span.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_stats_by_span.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format("source=%s | stats count() by span(age,10)", TEST_INDEX_BANK))); } @Test public void testStatsBySpanNonBucketNullable() throws IOException { - String expected = loadExpectedPlan("explain_stats_by_span_non_bucket_nullable.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_stats_by_span_non_bucket_nullable.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | stats bucket_nullable=false count() by span(age,10)", TEST_INDEX_BANK))); @@ -518,14 +519,9 @@ public void testDedupKeepEmptyFalsePushdown() throws IOException { @Test public void testSingleFieldRelevanceQueryFunctionExplain() throws IOException { enabledOnlyWhenPushdownIsEnabled(); - String expected = - isCalciteEnabled() - ? loadFromFile("expectedOutput/calcite/explain_single_field_relevance_push.json") - : loadFromFile("expectedOutput/ppl/explain_single_field_relevance_push.json"); - - assertJsonEqualsIgnoreId( - expected, - explainQueryToString( + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_single_field_relevance_push.yaml"), + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| where match(email, '*@gmail.com', boost=1.0)")); } @@ -533,14 +529,9 @@ public void testSingleFieldRelevanceQueryFunctionExplain() throws IOException { @Test public void testMultiFieldsRelevanceQueryFunctionExplain() throws IOException { enabledOnlyWhenPushdownIsEnabled(); - String expected = - isCalciteEnabled() - ? loadFromFile("expectedOutput/calcite/explain_multi_fields_relevance_push.json") - : loadFromFile("expectedOutput/ppl/explain_multi_fields_relevance_push.json"); - - assertJsonEqualsIgnoreId( - expected, - explainQueryToString( + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_multi_fields_relevance_push.yaml"), + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| where simple_query_string(['email', name 4.0], 'gmail'," + " default_operator='or', analyzer=english)")); @@ -661,10 +652,10 @@ public void testExplainOnTake() throws IOException { @Test public void testExplainOnPercentile() throws IOException { - String expected = loadExpectedPlan("explain_percentile.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_percentile.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | stats percentile(balance, 50) as p50," + " percentile(balance, 90) as p90")); } @@ -729,29 +720,28 @@ public void testSearchCommandWithChainedTimeModifier() throws IOException { @Test public void testExplainSearchBasicText() throws IOException { // Example 1: Basic text search without field specification - String expected = loadExpectedPlan("explain_search_basic_text.json"); - assertJsonEqualsIgnoreId( - expected, - explainQueryToString(String.format("search source=%s ERROR", TEST_INDEX_OTEL_LOGS))); + String expected = loadExpectedPlan("explain_search_basic_text.yaml"); + assertYamlEqualsIgnoreId( + expected, explainQueryYaml(String.format("search source=%s ERROR", TEST_INDEX_OTEL_LOGS))); } @Test public void testExplainSearchNumericComparison() throws IOException { // Example 2: Numeric field comparison with greater than - String expected = loadExpectedPlan("explain_search_numeric_comparison.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_search_numeric_comparison.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format("search source=%s severityNumber>15", TEST_INDEX_OTEL_LOGS))); } @Test public void testExplainSearchWildcardStar() throws IOException { // Example 3: Wildcard search with asterisk for pattern matching - String expected = loadExpectedPlan("explain_search_wildcard_star.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_search_wildcard_star.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format("search source=%s severityText=ERR*", TEST_INDEX_OTEL_LOGS))); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java index 224ebd1e1f6..ca7e1ee52fa 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java @@ -5,7 +5,6 @@ package org.opensearch.sql.ppl; -import static org.hamcrest.CoreMatchers.containsString; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_TIME; @@ -15,11 +14,8 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; -import java.util.stream.Collectors; -import org.hamcrest.MatcherAssert; import org.json.JSONObject; import org.junit.jupiter.api.Test; -import org.opensearch.sql.data.type.ExprCoreType; public class WhereCommandIT extends PPLIntegTestCase { @@ -242,29 +238,6 @@ public void testWhereWithNotIn() throws IOException { verifyDataRows(result, rows("Amber"), rows("Dale")); } - @Test - public void testInWithIncompatibleType() { - Exception e = - assertThrows( - Exception.class, - () -> { - executeQuery( - String.format( - "source=%s | where balance in (4180, 5686, '6077') | fields firstname", - TEST_INDEX_ACCOUNT)); - }); - MatcherAssert.assertThat(e.getMessage(), containsString(getIncompatibleTypeErrMsg())); - } - - protected String getIncompatibleTypeErrMsg() { - return String.format( - "function expected %s, but got %s", - ExprCoreType.coreTypes().stream() - .map(type -> String.format("[%s,%s]", type.typeName(), type.typeName())) - .collect(Collectors.joining(",", "{", "}")), - "[LONG,STRING]"); - } - @Test public void testFilterScriptPushDown() throws IOException { JSONObject actual = diff --git a/integ-test/src/test/resources/expectedOutput/calcite/access_struct_subfield_with_item.yaml b/integ-test/src/test/resources/expectedOutput/calcite/access_struct_subfield_with_item.yaml index a3726ad6126..e5e80ae1943 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/access_struct_subfield_with_item.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/access_struct_subfield_with_item.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(host=[$0], info=[GEOIP('dummy-datasource':VARCHAR, $0)], info.dummy_sub_field=[ITEM(GEOIP('dummy-datasource':VARCHAR, $0), 'dummy_sub_field')]) + LogicalProject(host=[$0], info=[GEOIP('dummy-datasource', $0)], info.dummy_sub_field=[ITEM(GEOIP('dummy-datasource', $0), 'dummy_sub_field')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) physical: | - EnumerableCalc(expr#0=[{inputs}], expr#1=['dummy-datasource':VARCHAR], expr#2=[GEOIP($t1, $t0)], expr#3=['dummy_sub_field'], expr#4=[ITEM($t2, $t3)], host=[$t0], $f1=[$t2], $f2=[$t4]) + EnumerableCalc(expr#0=[{inputs}], expr#1=['dummy-datasource'], expr#2=[GEOIP($t1, $t0)], expr#3=['dummy_sub_field'], expr#4=[ITEM($t2, $t3)], host=[$t0], $f1=[$t2], $f2=[$t4]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["host"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_date_range_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_date_range_push.yaml index 792b065745d..b1d9aa62c01 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_date_range_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_date_range_push.yaml @@ -2,10 +2,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(avg(value)=[$2], span(@timestamp,1h)=[$1], value_range=[$0]) - LogicalAggregate(group=[{0, 2}], avg(value)=[AVG($1)]) - LogicalProject(value_range=[$10], value=[$2], span(@timestamp,1h)=[SPAN($0, 1, 'h')]) + LogicalAggregate(group=[{0, 1}], avg(value)=[AVG($2)]) + LogicalProject(value_range=[$10], span(@timestamp,1h)=[SPAN($0, 1, 'h')], value=[$2]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9], value_range=[CASE(<($2, 7000), 'small':VARCHAR, 'large':VARCHAR)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(value)=AVG($1)), PROJECT->[avg(value), span(@timestamp,1h), value_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(@timestamp,1h)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1h"}}}]},"aggregations":{"value_range":{"range":{"field":"value","ranges":[{"key":"small","to":7000.0},{"key":"large","from":7000.0}],"keyed":true},"aggregations":{"avg(value)":{"avg":{"field":"value"}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(value)=AVG($2)), PROJECT->[avg(value), span(@timestamp,1h), value_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(@timestamp,1h)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1h"}}}]},"aggregations":{"value_range":{"range":{"field":"value","ranges":[{"key":"small","to":7000.0},{"key":"large","from":7000.0}],"keyed":true},"aggregations":{"avg(value)":{"avg":{"field":"value"}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_range_count_push.yaml index 498786a6aef..c8720e9e0d3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_range_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_range_count_push.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(age)=AVG($1)), PROJECT->[avg(age), age_range]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"u40","from":30.0,"to":40.0},{"key":"u100","from":40.0}],"keyed":true},"aggregations":{"avg(age)":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(age)=AVG($1)), PROJECT->[avg(age), age_range]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"u40","from":30.0,"to":40.0},{"key":"u100","from":40.0}],"keyed":true},"aggregations":{"avg(age)":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_range_metric_complex_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_range_metric_complex_push.yaml index f3d749487c0..6128b6487f6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_range_metric_complex_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_range_metric_complex_push.yaml @@ -3,7 +3,7 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(avg(balance)=[$1], age_range=[$0]) LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) - LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[35..40), [80..+∞)]), '30-40 or >=80':VARCHAR, null:NULL)], balance=[$7]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[35..40), [80..+∞)]), '30-40 or >=80':VARCHAR, null:VARCHAR)], balance=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_range_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_range_range_metric_push.yaml index 5b44ebfdc68..e242005a6de 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_range_range_metric_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_range_range_metric_push.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2)), PROJECT->[avg_balance, age_range, balance_range]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"u40","from":30.0,"to":40.0},{"key":"u100","from":40.0}],"keyed":true},"aggregations":{"balance_range":{"range":{"field":"balance","ranges":[{"key":"medium","to":20000.0},{"key":"high","from":20000.0}],"keyed":true},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#5599:LogicalAggregate.NONE.[](input=RelSubset#5598,group={0, 1},avg_balance=AVG($2)), PROJECT->[avg_balance, age_range, balance_range]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"u40","from":30.0,"to":40.0},{"key":"u100","from":40.0}],"keyed":true},"aggregations":{"balance_range":{"range":{"field":"balance","ranges":[{"key":"medium","to":20000.0},{"key":"high","from":20000.0}],"keyed":true},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/asc_sort_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/asc_sort_timestamp.yaml index 81138f6fe80..65cbc5437aa 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/asc_sort_timestamp.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/asc_sort_timestamp.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) + LogicalSort(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], SORT->[{ @@ -10,4 +10,4 @@ calcite: "order" : "asc", "missing" : "_first" } - }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/asc_sort_timestamp_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/asc_sort_timestamp_can_match_shortcut.yaml index ce84d53f479..0cfe7b757c4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/asc_sort_timestamp_can_match_shortcut.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/asc_sort_timestamp_can_match_shortcut.yaml @@ -1,14 +1,14 @@ calcite: logical: | LogicalSystemLimit(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) - LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) + LogicalSort(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, 'process.name:kernel':VARCHAR))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query', 'process.name:kernel':VARCHAR)), SORT->[{ + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query':VARCHAR, 'process.name:kernel':VARCHAR)), SORT->[{ "@timestamp" : { "order" : "asc", "missing" : "_first" } - }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/asc_sort_timestamp_no_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/asc_sort_timestamp_no_can_match_shortcut.yaml index ce84d53f479..0cfe7b757c4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/asc_sort_timestamp_no_can_match_shortcut.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/asc_sort_timestamp_no_can_match_shortcut.yaml @@ -1,14 +1,14 @@ calcite: logical: | LogicalSystemLimit(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) - LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) + LogicalSort(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, 'process.name:kernel':VARCHAR))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query', 'process.name:kernel':VARCHAR)), SORT->[{ + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query':VARCHAR, 'process.name:kernel':VARCHAR)), SORT->[{ "@timestamp" : { "order" : "asc", "missing" : "_first" } - }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/asc_sort_with_after_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/asc_sort_with_after_timestamp.yaml index 81138f6fe80..65cbc5437aa 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/asc_sort_with_after_timestamp.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/asc_sort_with_after_timestamp.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) + LogicalSort(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], SORT->[{ @@ -10,4 +10,4 @@ calcite: "order" : "asc", "missing" : "_first" } - }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_date_histogram_daily.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_date_histogram_daily.yaml index 1f3949c6ce3..480f21b3900 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_date_histogram_daily.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_date_histogram_daily.yaml @@ -5,7 +5,7 @@ calcite: LogicalProject(count()=[$1], span(`@timestamp`,1d)=[$0]) LogicalAggregate(group=[{0}], count()=[COUNT()]) LogicalProject(span(`@timestamp`,1d)=[SPAN($17, 1, 'd')]) - LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2022-12-30 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-07 12:00:00':VARCHAR)))]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2022-12-30 00:00:00')), <($17, TIMESTAMP('2023-01-07 12:00:00')))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->SEARCH($0, Sarg[['2022-12-30 00:00:00':VARCHAR..'2023-01-07 12:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1d)], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2022-12-30T00:00:00.000Z","to":"2023-01-07T12:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"span(`@timestamp`,1d)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->SEARCH($0, Sarg[['2022-12-30 00:00:00':EXPR_TIMESTAMP VARCHAR..'2023-01-07 12:00:00':EXPR_TIMESTAMP VARCHAR)]:EXPR_TIMESTAMP VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1d)], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2022-12-30T00:00:00.000Z","to":"2023-01-07T12:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"span(`@timestamp`,1d)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms.yaml index 199988f7e3b..3b07ab00677 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms.yaml @@ -5,7 +5,7 @@ calcite: LogicalProject(count()=[$2], process.name=[$0], cloud.region=[$1]) LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) LogicalProject(process.name=[$7], cloud.region=[$14]) - LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-02 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-02 10:00:00':VARCHAR)), IS NOT NULL($7), IS NOT NULL($14))]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-02 00:00:00')), <($17, TIMESTAMP('2023-01-02 10:00:00')), IS NOT NULL($7), IS NOT NULL($14))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->AND(SEARCH($2, Sarg[['2023-01-02 00:00:00':VARCHAR..'2023-01-02 10:00:00':VARCHAR)]:VARCHAR), IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), process.name, cloud.region], SORT->[1 DESC LAST, 2 ASC FIRST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-02T00:00:00.000Z","to":"2023-01-02T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"exists":{"field":"process.name","boost":1.0}},{"exists":{"field":"cloud.region","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":false,"order":"desc"}}},{"cloud.region":{"terms":{"field":"cloud.region","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->AND(SEARCH($2, Sarg[['2023-01-02 00:00:00':EXPR_TIMESTAMP VARCHAR..'2023-01-02 10:00:00':EXPR_TIMESTAMP VARCHAR)]:EXPR_TIMESTAMP VARCHAR), IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), process.name, cloud.region], SORT->[1 DESC LAST, 2 ASC FIRST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-02T00:00:00.000Z","to":"2023-01-02T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"exists":{"field":"process.name","boost":1.0}},{"exists":{"field":"cloud.region","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":false,"order":"desc"}}},{"cloud.region":{"terms":{"field":"cloud.region","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms_keyword.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms_keyword.yaml index 9c34a7ecd56..404209274cf 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms_keyword.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms_keyword.yaml @@ -5,7 +5,7 @@ calcite: LogicalProject(count()=[$3], process.name=[$0], cloud.region=[$1], aws.cloudwatch.log_stream=[$2]) LogicalAggregate(group=[{0, 1, 2}], count()=[COUNT()]) LogicalProject(process.name=[$7], cloud.region=[$14], aws.cloudwatch.log_stream=[$34]) - LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-02 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-02 10:00:00':VARCHAR)), IS NOT NULL($7), IS NOT NULL($14), IS NOT NULL($34))]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-02 00:00:00')), <($17, TIMESTAMP('2023-01-02 10:00:00')), IS NOT NULL($7), IS NOT NULL($14), IS NOT NULL($34))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->AND(SEARCH($2, Sarg[['2023-01-02 00:00:00':VARCHAR..'2023-01-02 10:00:00':VARCHAR)]:VARCHAR), IS NOT NULL($0), IS NOT NULL($1), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 3},count()=COUNT()), PROJECT->[count(), process.name, cloud.region, aws.cloudwatch.log_stream], SORT->[1 DESC LAST, 2 ASC FIRST, 3 ASC FIRST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-02T00:00:00.000Z","to":"2023-01-02T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"exists":{"field":"process.name","boost":1.0}},{"exists":{"field":"cloud.region","boost":1.0}},{"exists":{"field":"aws.cloudwatch.log_stream","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":false,"order":"desc"}}},{"cloud.region":{"terms":{"field":"cloud.region","missing_bucket":false,"order":"asc"}}},{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->AND(SEARCH($2, Sarg[['2023-01-02 00:00:00':EXPR_TIMESTAMP VARCHAR..'2023-01-02 10:00:00':EXPR_TIMESTAMP VARCHAR)]:EXPR_TIMESTAMP VARCHAR), IS NOT NULL($0), IS NOT NULL($1), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 3},count()=COUNT()), PROJECT->[count(), process.name, cloud.region, aws.cloudwatch.log_stream], SORT->[1 DESC LAST, 2 ASC FIRST, 3 ASC FIRST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-02T00:00:00.000Z","to":"2023-01-02T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"exists":{"field":"process.name","boost":1.0}},{"exists":{"field":"cloud.region","boost":1.0}},{"exists":{"field":"aws.cloudwatch.log_stream","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":false,"order":"desc"}}},{"cloud.region":{"terms":{"field":"cloud.region","missing_bucket":false,"order":"asc"}}},{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_hourly_agg.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_hourly_agg.yaml index 2982377d2ca..be886cd1830 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_hourly_agg.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_hourly_agg.yaml @@ -7,4 +7,4 @@ calcite: LogicalFilter(condition=[IS NOT NULL($17)]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1h)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"span(`@timestamp`,1h)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1h"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1h)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"span(`@timestamp`,1h)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1h"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_minute_agg.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_minute_agg.yaml index 41303581efc..bbd73aff701 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_minute_agg.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_minute_agg.yaml @@ -4,7 +4,7 @@ calcite: LogicalProject(count()=[$1], span(`@timestamp`,1m)=[$0]) LogicalAggregate(group=[{0}], count()=[COUNT()]) LogicalProject(span(`@timestamp`,1m)=[SPAN($17, 1, 'm')]) - LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00')), <($17, TIMESTAMP('2023-01-03 00:00:00')))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->SEARCH($0, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1m)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"span(`@timestamp`,1m)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->SEARCH($0, Sarg[['2023-01-01 00:00:00':EXPR_TIMESTAMP VARCHAR..'2023-01-03 00:00:00':EXPR_TIMESTAMP VARCHAR)]:EXPR_TIMESTAMP VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1m)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"span(`@timestamp`,1m)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/dedup_metrics_size_field.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/dedup_metrics_size_field.yaml index 4387e72fd96..1fb92519cac 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/dedup_metrics_size_field.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/dedup_metrics_size_field.yaml @@ -1,13 +1,12 @@ calcite: logical: | LogicalSystemLimit(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(sort0=[$17], dir0=[DESC-nulls-last]) - LogicalProject(agent=[$0], agent.ephemeral_id=[$1], agent.id=[$2], agent.name=[$3], agent.type=[$4], agent.version=[$5], process=[$6], process.name=[$7], log=[$8], log.file=[$9], log.file.path=[$10], message=[$11], tags=[$12], cloud=[$13], cloud.region=[$14], input=[$15], input.type=[$16], @timestamp=[$17], ecs=[$18], ecs.version=[$19], data_stream=[$20], data_stream.dataset=[$21], data_stream.namespace=[$22], data_stream.type=[$23], meta=[$24], meta.file=[$25], host=[$26], metrics=[$27], metrics.size=[$28], metrics.tmin=[$29], aws=[$30], aws.cloudwatch=[$31], aws.cloudwatch.ingestion_time=[$32], aws.cloudwatch.log_group=[$33], aws.cloudwatch.log_stream=[$34], event=[$35], event.dataset=[$36], event.id=[$37], event.ingested=[$38], _id=[$39], _index=[$40], _score=[$41], _maxscore=[$42], _sort=[$43], _routing=[$44]) - LogicalFilter(condition=[<=($45, 1)]) - LogicalProject(agent=[$0], agent.ephemeral_id=[$1], agent.id=[$2], agent.name=[$3], agent.type=[$4], agent.version=[$5], process=[$6], process.name=[$7], log=[$8], log.file=[$9], log.file.path=[$10], message=[$11], tags=[$12], cloud=[$13], cloud.region=[$14], input=[$15], input.type=[$16], @timestamp=[$17], ecs=[$18], ecs.version=[$19], data_stream=[$20], data_stream.dataset=[$21], data_stream.namespace=[$22], data_stream.type=[$23], meta=[$24], meta.file=[$25], host=[$26], metrics=[$27], metrics.size=[$28], metrics.tmin=[$29], aws=[$30], aws.cloudwatch=[$31], aws.cloudwatch.ingestion_time=[$32], aws.cloudwatch.log_group=[$33], aws.cloudwatch.log_stream=[$34], event=[$35], event.dataset=[$36], event.id=[$37], event.ingested=[$38], _id=[$39], _index=[$40], _score=[$41], _maxscore=[$42], _sort=[$43], _routing=[$44], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $28)]) - LogicalFilter(condition=[IS NOT NULL($28)]) - CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + LogicalSort(sort0=[$7], dir0=[DESC-nulls-last]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalFilter(condition=[<=($45, SAFE_CAST(1:BIGINT))]) + LogicalProject(agent=[$0], agent.ephemeral_id=[$1], agent.id=[$2], agent.name=[$3], agent.type=[$4], agent.version=[$5], process=[$6], process.name=[$7], log=[$8], log.file=[$9], log.file.path=[$10], message=[$11], tags=[$12], cloud=[$13], cloud.region=[$14], input=[$15], input.type=[$16], @timestamp=[$17], ecs=[$18], ecs.version=[$19], data_stream=[$20], data_stream.dataset=[$21], data_stream.namespace=[$22], data_stream.type=[$23], meta=[$24], meta.file=[$25], host=[$26], metrics=[$27], metrics.size=[$28], metrics.tmin=[$29], aws=[$30], aws.cloudwatch=[$31], aws.cloudwatch.ingestion_time=[$32], aws.cloudwatch.log_group=[$33], aws.cloudwatch.log_stream=[$34], event=[$35], event.dataset=[$36], event.id=[$37], event.ingested=[$38], _id=[$39], _index=[$40], _score=[$41], _maxscore=[$42], _sort=[$43], _routing=[$44], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $28)]) + LogicalFilter(condition=[IS NOT NULL($28)]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | CalciteEnumerableTopK(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"metrics.size":{"terms":{"field":"metrics.size","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["metrics.size","agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"metrics.size":{"terms":{"field":"metrics.size","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["metrics.size","agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/default.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/default.yaml index 59e68e48769..7e7deefcf4d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/default.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/default.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(fetch=[10]) + LogicalSort(fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/desc_sort_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/desc_sort_timestamp.yaml index 7e14abeeef2..fc9111f1d0c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/desc_sort_timestamp.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/desc_sort_timestamp.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(sort0=[$17], dir0=[DESC-nulls-last], fetch=[10]) + LogicalSort(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], SORT->[{ @@ -10,4 +10,4 @@ calcite: "order" : "desc", "missing" : "_last" } - }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/desc_sort_timestamp_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/desc_sort_timestamp_can_match_shortcut.yaml index 13239b869cc..af65e52184e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/desc_sort_timestamp_can_match_shortcut.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/desc_sort_timestamp_can_match_shortcut.yaml @@ -1,14 +1,14 @@ calcite: logical: | LogicalSystemLimit(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(sort0=[$17], dir0=[DESC-nulls-last], fetch=[10]) - LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) + LogicalSort(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, 'process.name:kernel':VARCHAR))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query', 'process.name:kernel':VARCHAR)), SORT->[{ + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query':VARCHAR, 'process.name:kernel':VARCHAR)), SORT->[{ "@timestamp" : { "order" : "desc", "missing" : "_last" } - }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/desc_sort_timestamp_no_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/desc_sort_timestamp_no_can_match_shortcut.yaml index 13239b869cc..af65e52184e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/desc_sort_timestamp_no_can_match_shortcut.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/desc_sort_timestamp_no_can_match_shortcut.yaml @@ -1,14 +1,14 @@ calcite: logical: | LogicalSystemLimit(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(sort0=[$17], dir0=[DESC-nulls-last], fetch=[10]) - LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) + LogicalSort(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, 'process.name:kernel':VARCHAR))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query', 'process.name:kernel':VARCHAR)), SORT->[{ + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query':VARCHAR, 'process.name:kernel':VARCHAR)), SORT->[{ "@timestamp" : { "order" : "desc", "missing" : "_last" } - }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/desc_sort_with_after_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/desc_sort_with_after_timestamp.yaml index 7e14abeeef2..fc9111f1d0c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/desc_sort_with_after_timestamp.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/desc_sort_with_after_timestamp.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(sort0=[$17], dir0=[DESC-nulls-last], fetch=[10]) + LogicalSort(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], SORT->[{ @@ -10,4 +10,4 @@ calcite: "order" : "desc", "missing" : "_last" } - }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/keyword_in_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/keyword_in_range.yaml index e1f0873de61..4afea60b2c3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/keyword_in_range.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/keyword_in_range.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(fetch=[10]) - LogicalFilter(condition=[AND(query_string(MAP('query', 'process.name:kernel':VARCHAR)), >=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) + LogicalSort(fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalFilter(condition=[AND(query_string(MAP('query':VARCHAR, 'process.name:kernel':VARCHAR)), >=($17, TIMESTAMP('2023-01-01 00:00:00')), <($17, TIMESTAMP('2023-01-03 00:00:00')))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->AND(query_string(MAP('query', 'process.name:kernel':VARCHAR)), SEARCH($7, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR)), LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->AND(query_string(MAP('query':VARCHAR, 'process.name:kernel':VARCHAR)), SEARCH($7, Sarg[['2023-01-01 00:00:00':EXPR_TIMESTAMP VARCHAR..'2023-01-03 00:00:00':EXPR_TIMESTAMP VARCHAR)]:EXPR_TIMESTAMP VARCHAR)), LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/multi_terms_keyword.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/multi_terms_keyword.yaml index b01761d7dee..45337d7db76 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/multi_terms_keyword.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/multi_terms_keyword.yaml @@ -5,7 +5,7 @@ calcite: LogicalProject(count()=[$2], process.name=[$0], cloud.region=[$1]) LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) LogicalProject(process.name=[$7], cloud.region=[$14]) - LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-05 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-05 05:00:00':VARCHAR)), IS NOT NULL($7), IS NOT NULL($14))]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-05 00:00:00')), <($17, TIMESTAMP('2023-01-05 05:00:00')), IS NOT NULL($7), IS NOT NULL($14))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->AND(SEARCH($2, Sarg[['2023-01-05 00:00:00':VARCHAR..'2023-01-05 05:00:00':VARCHAR)]:VARCHAR), IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[count(), process.name, cloud.region], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-05T00:00:00.000Z","to":"2023-01-05T05:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"exists":{"field":"process.name","boost":1.0}},{"exists":{"field":"cloud.region","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"process.name|cloud.region":{"multi_terms":{"terms":[{"field":"process.name"},{"field":"cloud.region"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->AND(SEARCH($2, Sarg[['2023-01-05 00:00:00':EXPR_TIMESTAMP VARCHAR..'2023-01-05 05:00:00':EXPR_TIMESTAMP VARCHAR)]:EXPR_TIMESTAMP VARCHAR), IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[count(), process.name, cloud.region], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-05T00:00:00.000Z","to":"2023-01-05T05:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"exists":{"field":"process.name","boost":1.0}},{"exists":{"field":"cloud.region","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"process.name|cloud.region":{"multi_terms":{"terms":[{"field":"process.name"},{"field":"cloud.region"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/query_string_on_message.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/query_string_on_message.yaml index 31cbb3b8d70..5d258c4ca3d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/query_string_on_message.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/query_string_on_message.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(fetch=[10]) - LogicalFilter(condition=[query_string(MAP('query', '((message:monkey OR message:jackal) OR message:bear)':VARCHAR))]) + LogicalSort(fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, '((message:monkey OR message:jackal) OR message:bear)':VARCHAR))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query', '((message:monkey OR message:jackal) OR message:bear)':VARCHAR)), LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"((message:monkey OR message:jackal) OR message:bear)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query':VARCHAR, '((message:monkey OR message:jackal) OR message:bear)':VARCHAR)), LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"((message:monkey OR message:jackal) OR message:bear)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/query_string_on_message_filtered.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/query_string_on_message_filtered.yaml index e1471d87a4e..c97239b654f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/query_string_on_message_filtered.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/query_string_on_message_filtered.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(fetch=[10]) - LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 10:00:00':VARCHAR)), query_string(MAP('fields', MAP('message':VARCHAR, 1.0E0:DOUBLE)), MAP('query', 'monkey jackal bear':VARCHAR)))]) + LogicalSort(fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-03 00:00:00')), <($17, TIMESTAMP('2023-01-03 10:00:00')), query_string(MAP('fields':VARCHAR, MAP('message':VARCHAR, 1.0E0:DOUBLE)), MAP('query':VARCHAR, 'monkey jackal bear':VARCHAR)))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->AND(SEARCH($7, Sarg[['2023-01-03 00:00:00':VARCHAR..'2023-01-03 10:00:00':VARCHAR)]:VARCHAR), query_string(MAP('fields', MAP('message':VARCHAR, 1.0E0:DOUBLE)), MAP('query', 'monkey jackal bear':VARCHAR))), LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-03T00:00:00.000Z","to":"2023-01-03T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"query_string":{"query":"monkey jackal bear","fields":["message^1.0"],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->AND(SEARCH($7, Sarg[['2023-01-03 00:00:00':EXPR_TIMESTAMP VARCHAR..'2023-01-03 10:00:00':EXPR_TIMESTAMP VARCHAR)]:EXPR_TIMESTAMP VARCHAR), query_string(MAP('fields':VARCHAR, MAP('message':VARCHAR, 1.0E0:DOUBLE)), MAP('query':VARCHAR, 'monkey jackal bear':VARCHAR))), LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-03T00:00:00.000Z","to":"2023-01-03T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"query_string":{"query":"monkey jackal bear","fields":["message^1.0"],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/query_string_on_message_filtered_sorted_num.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/query_string_on_message_filtered_sorted_num.yaml index 27a43886bc4..3c50f908427 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/query_string_on_message_filtered_sorted_num.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/query_string_on_message_filtered_sorted_num.yaml @@ -1,14 +1,14 @@ calcite: logical: | LogicalSystemLimit(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) - LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 10:00:00':VARCHAR)), query_string(MAP('fields', MAP('message':VARCHAR, 1.0E0:DOUBLE)), MAP('query', 'monkey jackal bear':VARCHAR)))]) + LogicalSort(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-03 00:00:00')), <($17, TIMESTAMP('2023-01-03 10:00:00')), query_string(MAP('fields':VARCHAR, MAP('message':VARCHAR, 1.0E0:DOUBLE)), MAP('query':VARCHAR, 'monkey jackal bear':VARCHAR)))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->AND(SEARCH($7, Sarg[['2023-01-03 00:00:00':VARCHAR..'2023-01-03 10:00:00':VARCHAR)]:VARCHAR), query_string(MAP('fields', MAP('message':VARCHAR, 1.0E0:DOUBLE)), MAP('query', 'monkey jackal bear':VARCHAR))), SORT->[{ + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->AND(SEARCH($7, Sarg[['2023-01-03 00:00:00':EXPR_TIMESTAMP VARCHAR..'2023-01-03 10:00:00':EXPR_TIMESTAMP VARCHAR)]:EXPR_TIMESTAMP VARCHAR), query_string(MAP('fields':VARCHAR, MAP('message':VARCHAR, 1.0E0:DOUBLE)), MAP('query':VARCHAR, 'monkey jackal bear':VARCHAR))), SORT->[{ "@timestamp" : { "order" : "asc", "missing" : "_first" } - }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-03T00:00:00.000Z","to":"2023-01-03T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"query_string":{"query":"monkey jackal bear","fields":["message^1.0"],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-03T00:00:00.000Z","to":"2023-01-03T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"query_string":{"query":"monkey jackal bear","fields":["message^1.0"],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/range.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/range.yaml index 56c63c5c406..24b54dd6950 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/range.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/range.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(fetch=[10]) - LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) + LogicalSort(fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00')), <($17, TIMESTAMP('2023-01-03 00:00:00')))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->SEARCH($7, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->SEARCH($7, Sarg[['2023-01-01 00:00:00':EXPR_TIMESTAMP VARCHAR..'2023-01-03 00:00:00':EXPR_TIMESTAMP VARCHAR)]:EXPR_TIMESTAMP VARCHAR), LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_agg_1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_agg_1.yaml index 86c1551609c..141d7d1ce59 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_agg_1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_agg_1.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(count()=[$1], range_bucket=[$0]) LogicalAggregate(group=[{0}], count()=[COUNT()]) - LogicalProject(range_bucket=[CASE(<($28, -10), 'range_1':VARCHAR, SEARCH($28, Sarg[[-10..10)]), 'range_2':VARCHAR, SEARCH($28, Sarg[[10..100)]), 'range_3':VARCHAR, SEARCH($28, Sarg[[100..1000)]), 'range_4':VARCHAR, SEARCH($28, Sarg[[1000..2000)]), 'range_5':VARCHAR, >=($28, 2000), 'range_6':VARCHAR, null:NULL)]) + LogicalProject(range_bucket=[CASE(<($28, -10), 'range_1':VARCHAR, SEARCH($28, Sarg[[-10L:BIGINT..10L:BIGINT)]:BIGINT), 'range_2':VARCHAR, SEARCH($28, Sarg[[10L:BIGINT..100L:BIGINT)]:BIGINT), 'range_3':VARCHAR, SEARCH($28, Sarg[[100L:BIGINT..1000L:BIGINT)]:BIGINT), 'range_4':VARCHAR, SEARCH($28, Sarg[[1000L:BIGINT..2000L:BIGINT)]:BIGINT), 'range_5':VARCHAR, >=($28, 2000), 'range_6':VARCHAR, null:VARCHAR)]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | EnumerableLimit(fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), range_bucket]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"range_bucket":{"range":{"field":"metrics.size","ranges":[{"key":"range_1","to":-10.0},{"key":"range_2","from":-10.0,"to":10.0},{"key":"range_3","from":10.0,"to":100.0},{"key":"range_4","from":100.0,"to":1000.0},{"key":"range_5","from":1000.0,"to":2000.0},{"key":"range_6","from":2000.0}],"keyed":true}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), range_bucket]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"range_bucket":{"range":{"field":"metrics.size","ranges":[{"key":"range_1","to":-10.0},{"key":"range_2","from":-10.0,"to":10.0},{"key":"range_3","from":10.0,"to":100.0},{"key":"range_4","from":100.0,"to":1000.0},{"key":"range_5","from":1000.0,"to":2000.0},{"key":"range_6","from":2000.0}],"keyed":true}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_agg_2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_agg_2.yaml index daae2d2fc97..7d99f48ddad 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_agg_2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_agg_2.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(count()=[$1], range_bucket=[$0]) LogicalAggregate(group=[{0}], count()=[COUNT()]) - LogicalProject(range_bucket=[CASE(<($28, 100), 'range_1':VARCHAR, SEARCH($28, Sarg[[100..1000)]), 'range_2':VARCHAR, SEARCH($28, Sarg[[1000..2000)]), 'range_3':VARCHAR, >=($28, 2000), 'range_4':VARCHAR, null:NULL)]) + LogicalProject(range_bucket=[CASE(<($28, 100), 'range_1':VARCHAR, SEARCH($28, Sarg[[100L:BIGINT..1000L:BIGINT)]:BIGINT), 'range_2':VARCHAR, SEARCH($28, Sarg[[1000L:BIGINT..2000L:BIGINT)]:BIGINT), 'range_3':VARCHAR, >=($28, 2000), 'range_4':VARCHAR, null:VARCHAR)]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | EnumerableLimit(fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), range_bucket]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"range_bucket":{"range":{"field":"metrics.size","ranges":[{"key":"range_1","to":100.0},{"key":"range_2","from":100.0,"to":1000.0},{"key":"range_3","from":1000.0,"to":2000.0},{"key":"range_4","from":2000.0}],"keyed":true}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), range_bucket]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"range_bucket":{"range":{"field":"metrics.size","ranges":[{"key":"range_1","to":100.0},{"key":"range_2","from":100.0,"to":1000.0},{"key":"range_3","from":1000.0,"to":2000.0},{"key":"range_4","from":2000.0}],"keyed":true}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_field_conjunction_big_range_big_term_query.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_field_conjunction_big_range_big_term_query.yaml index ba8b035ab51..214479a2caa 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_field_conjunction_big_range_big_term_query.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_field_conjunction_big_range_big_term_query.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(fetch=[10]) - LogicalFilter(condition=[AND(=($7, 'systemd'), SEARCH($28, Sarg[[1..100]]))]) + LogicalSort(fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalFilter(condition=[AND(=($7, 'systemd'), >=($28, SAFE_CAST(1:BIGINT)), <=($28, SAFE_CAST(100:BIGINT)))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, process.name, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->AND(=($2, 'systemd'), SEARCH($14, Sarg[[1..100]])), LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"term":{"process.name":{"value":"systemd","boost":1.0}}},{"range":{"metrics.size":{"from":1.0,"to":100.0,"include_lower":true,"include_upper":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, process.name, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->AND(=($2, 'systemd'), SEARCH($14, Sarg[[1L:BIGINT..100L:BIGINT]]:BIGINT)), PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"term":{"process.name":{"value":"systemd","boost":1.0}}},{"range":{"metrics.size":{"from":1.0,"to":100.0,"include_lower":true,"include_upper":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_field_conjunction_small_range_big_term_query.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_field_conjunction_small_range_big_term_query.yaml index 69dddd2ef14..1bd6e498fb6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_field_conjunction_small_range_big_term_query.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_field_conjunction_small_range_big_term_query.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(fetch=[10]) - LogicalFilter(condition=[SEARCH($28, Sarg[[20..30]])]) + LogicalSort(fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalFilter(condition=[AND(>=($28, SAFE_CAST(20:BIGINT)), <=($28, SAFE_CAST(30:BIGINT)))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->SEARCH($13, Sarg[[20..30]]), LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"metrics.size":{"from":20.0,"to":30.0,"include_lower":true,"include_upper":true,"boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->SEARCH($13, Sarg[[20L:BIGINT..30L:BIGINT]]:BIGINT), PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"metrics.size":{"from":20.0,"to":30.0,"include_lower":true,"include_upper":true,"boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_field_conjunction_small_range_small_term_query.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_field_conjunction_small_range_small_term_query.yaml index 612e412b307..af09a8f669a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_field_conjunction_small_range_small_term_query.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_field_conjunction_small_range_small_term_query.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(fetch=[10]) - LogicalFilter(condition=[OR(=($34, 'indigodagger'), SEARCH($28, Sarg[[10..20]]))]) + LogicalSort(fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalFilter(condition=[OR(=($34, 'indigodagger'), AND(>=($28, SAFE_CAST(10:BIGINT)), <=($28, SAFE_CAST(20:BIGINT))))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, aws.cloudwatch.log_stream, event], FILTER->OR(=($15, 'indigodagger'), SEARCH($13, Sarg[[10..20]])), LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"should":[{"term":{"aws.cloudwatch.log_stream":{"value":"indigodagger","boost":1.0}}},{"range":{"metrics.size":{"from":10.0,"to":20.0,"include_lower":true,"include_upper":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, aws.cloudwatch.log_stream, event], FILTER->OR(=($15, 'indigodagger'), SEARCH($13, Sarg[[10L:BIGINT..20L:BIGINT]]:BIGINT)), PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"should":[{"term":{"aws.cloudwatch.log_stream":{"value":"indigodagger","boost":1.0}}},{"range":{"metrics.size":{"from":10.0,"to":20.0,"include_lower":true,"include_upper":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_field_disjunction_big_range_small_term_query.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_field_disjunction_big_range_small_term_query.yaml index 24cabf88754..661540803b9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_field_disjunction_big_range_small_term_query.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_field_disjunction_big_range_small_term_query.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(fetch=[10]) - LogicalFilter(condition=[OR(=($34, 'indigodagger'), SEARCH($28, Sarg[[1..100]]))]) + LogicalSort(fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalFilter(condition=[OR(=($34, 'indigodagger'), AND(>=($28, SAFE_CAST(1:BIGINT)), <=($28, SAFE_CAST(100:BIGINT))))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, aws.cloudwatch.log_stream, event], FILTER->OR(=($15, 'indigodagger'), SEARCH($13, Sarg[[1..100]])), LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"should":[{"term":{"aws.cloudwatch.log_stream":{"value":"indigodagger","boost":1.0}}},{"range":{"metrics.size":{"from":1.0,"to":100.0,"include_lower":true,"include_upper":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, aws.cloudwatch.log_stream, event], FILTER->OR(=($15, 'indigodagger'), SEARCH($13, Sarg[[1L:BIGINT..100L:BIGINT]]:BIGINT)), PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"should":[{"term":{"aws.cloudwatch.log_stream":{"value":"indigodagger","boost":1.0}}},{"range":{"metrics.size":{"from":1.0,"to":100.0,"include_lower":true,"include_upper":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_numeric.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_numeric.yaml index cdf19c603a0..91290f4e5cb 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_numeric.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_numeric.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(fetch=[10]) - LogicalFilter(condition=[SEARCH($28, Sarg[[20..200]])]) + LogicalSort(fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalFilter(condition=[AND(>=($28, SAFE_CAST(20:BIGINT)), <=($28, SAFE_CAST(200:BIGINT)))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->SEARCH($13, Sarg[[20..200]]), LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"metrics.size":{"from":20.0,"to":200.0,"include_lower":true,"include_upper":true,"boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->SEARCH($13, Sarg[[20L:BIGINT..200L:BIGINT]]:BIGINT), PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"metrics.size":{"from":20.0,"to":200.0,"include_lower":true,"include_upper":true,"boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_with_asc_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_with_asc_sort.yaml index e0b91168f1f..8250feebe3a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_with_asc_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_with_asc_sort.yaml @@ -1,14 +1,14 @@ calcite: logical: | LogicalSystemLimit(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) - LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <=($17, TIMESTAMP('2023-01-13 00:00:00':VARCHAR)))]) + LogicalSort(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00')), <=($17, TIMESTAMP('2023-01-13 00:00:00')))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->SEARCH($7, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-13 00:00:00':VARCHAR]]:VARCHAR), SORT->[{ + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->SEARCH($7, Sarg[['2023-01-01 00:00:00':EXPR_TIMESTAMP VARCHAR..'2023-01-13 00:00:00':EXPR_TIMESTAMP VARCHAR]]:EXPR_TIMESTAMP VARCHAR), SORT->[{ "@timestamp" : { "order" : "asc", "missing" : "_first" } - }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-13T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-13T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_with_desc_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_with_desc_sort.yaml index 8af1fc7058d..5f55bec1732 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_with_desc_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_with_desc_sort.yaml @@ -1,14 +1,14 @@ calcite: logical: | LogicalSystemLimit(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(sort0=[$17], dir0=[DESC-nulls-last], fetch=[10]) - LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <=($17, TIMESTAMP('2023-01-13 00:00:00':VARCHAR)))]) + LogicalSort(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00')), <=($17, TIMESTAMP('2023-01-13 00:00:00')))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->SEARCH($7, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-13 00:00:00':VARCHAR]]:VARCHAR), SORT->[{ + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->SEARCH($7, Sarg[['2023-01-01 00:00:00':EXPR_TIMESTAMP VARCHAR..'2023-01-13 00:00:00':EXPR_TIMESTAMP VARCHAR]]:EXPR_TIMESTAMP VARCHAR), SORT->[{ "@timestamp" : { "order" : "desc", "missing" : "_last" } - }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-13T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-13T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/scroll.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/scroll.yaml index 59e68e48769..7e7deefcf4d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/scroll.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/scroll.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(fetch=[10]) + LogicalSort(fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_keyword_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_keyword_can_match_shortcut.yaml index 501c35a492a..058e5f6adbd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_keyword_can_match_shortcut.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_keyword_can_match_shortcut.yaml @@ -1,14 +1,15 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(sort0=[$25], dir0=[ASC-nulls-first], fetch=[10]) - LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) - CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + LogicalProject(agent=[$0], process=[$1], log=[$2], message=[$3], tags=[$4], cloud=[$5], input=[$6], @timestamp=[$7], ecs=[$8], data_stream=[$9], meta=[$10], host=[$11], metrics=[$12], aws=[$13], event=[$14]) + LogicalSort(sort0=[$15], dir0=[ASC-nulls-first], fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35], meta.file=[$25]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, 'process.name:kernel':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, meta.file, host, metrics, aws, event], FILTER->query_string(MAP('query', 'process.name:kernel':VARCHAR)), SORT->[{ + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, meta.file, host, metrics, aws, event], FILTER->query_string(MAP('query':VARCHAR, 'process.name:kernel':VARCHAR)), SORT->[{ "meta.file" : { "order" : "asc", "missing" : "_first" } - }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"meta.file":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"meta.file":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_keyword_no_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_keyword_no_can_match_shortcut.yaml index 501c35a492a..058e5f6adbd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_keyword_no_can_match_shortcut.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_keyword_no_can_match_shortcut.yaml @@ -1,14 +1,15 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(sort0=[$25], dir0=[ASC-nulls-first], fetch=[10]) - LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) - CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + LogicalProject(agent=[$0], process=[$1], log=[$2], message=[$3], tags=[$4], cloud=[$5], input=[$6], @timestamp=[$7], ecs=[$8], data_stream=[$9], meta=[$10], host=[$11], metrics=[$12], aws=[$13], event=[$14]) + LogicalSort(sort0=[$15], dir0=[ASC-nulls-first], fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35], meta.file=[$25]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, 'process.name:kernel':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, meta.file, host, metrics, aws, event], FILTER->query_string(MAP('query', 'process.name:kernel':VARCHAR)), SORT->[{ + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, meta.file, host, metrics, aws, event], FILTER->query_string(MAP('query':VARCHAR, 'process.name:kernel':VARCHAR)), SORT->[{ "meta.file" : { "order" : "asc", "missing" : "_first" } - }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"meta.file":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"meta.file":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_numeric_asc.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_numeric_asc.yaml index cbbc5106ec6..d27de8a02e5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_numeric_asc.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_numeric_asc.yaml @@ -1,13 +1,14 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(sort0=[$28], dir0=[ASC-nulls-first], fetch=[10]) - CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + LogicalProject(agent=[$0], process=[$1], log=[$2], message=[$3], tags=[$4], cloud=[$5], input=[$6], @timestamp=[$7], ecs=[$8], data_stream=[$9], meta=[$10], host=[$11], metrics=[$12], aws=[$13], event=[$14]) + LogicalSort(sort0=[$15], dir0=[ASC-nulls-first], fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35], metrics.size=[$28]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], SORT->[{ + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event, metrics.size], SORT->[{ "metrics.size" : { "order" : "asc", "missing" : "_first" } - }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"metrics.size":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"metrics.size":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_numeric_asc_with_match.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_numeric_asc_with_match.yaml index 9aa906cc6ca..868c753d4f4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_numeric_asc_with_match.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_numeric_asc_with_match.yaml @@ -1,14 +1,15 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(sort0=[$28], dir0=[ASC-nulls-first], fetch=[10]) - LogicalFilter(condition=[query_string(MAP('query', 'log.file.path:\/var\/log\/messages\/solarshark':VARCHAR))]) - CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + LogicalProject(agent=[$0], process=[$1], log=[$2], message=[$3], tags=[$4], cloud=[$5], input=[$6], @timestamp=[$7], ecs=[$8], data_stream=[$9], meta=[$10], host=[$11], metrics=[$12], aws=[$13], event=[$14]) + LogicalSort(sort0=[$15], dir0=[ASC-nulls-first], fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35], metrics.size=[$28]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, 'log.file.path:\/var\/log\/messages\/solarshark':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->query_string(MAP('query', 'log.file.path:\/var\/log\/messages\/solarshark':VARCHAR)), SORT->[{ + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->query_string(MAP('query':VARCHAR, 'log.file.path:\/var\/log\/messages\/solarshark':VARCHAR)), SORT->[{ "metrics.size" : { "order" : "asc", "missing" : "_first" } - }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"log.file.path:\\/var\\/log\\/messages\\/solarshark","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"metrics.size":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"log.file.path:\\/var\\/log\\/messages\\/solarshark","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"metrics.size":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_numeric_desc.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_numeric_desc.yaml index 3f059c7519f..b3d70925ed8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_numeric_desc.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_numeric_desc.yaml @@ -1,13 +1,14 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(sort0=[$28], dir0=[DESC-nulls-last], fetch=[10]) - CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + LogicalProject(agent=[$0], process=[$1], log=[$2], message=[$3], tags=[$4], cloud=[$5], input=[$6], @timestamp=[$7], ecs=[$8], data_stream=[$9], meta=[$10], host=[$11], metrics=[$12], aws=[$13], event=[$14]) + LogicalSort(sort0=[$15], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35], metrics.size=[$28]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], SORT->[{ + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event, metrics.size], SORT->[{ "metrics.size" : { "order" : "desc", "missing" : "_last" } - }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"metrics.size":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"metrics.size":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_numeric_desc_with_match.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_numeric_desc_with_match.yaml index b52bb433722..c5de0ae9a80 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_numeric_desc_with_match.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/sort_numeric_desc_with_match.yaml @@ -1,14 +1,15 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(sort0=[$28], dir0=[DESC-nulls-last], fetch=[10]) - LogicalFilter(condition=[query_string(MAP('query', 'log.file.path:\/var\/log\/messages\/solarshark':VARCHAR))]) - CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + LogicalProject(agent=[$0], process=[$1], log=[$2], message=[$3], tags=[$4], cloud=[$5], input=[$6], @timestamp=[$7], ecs=[$8], data_stream=[$9], meta=[$10], host=[$11], metrics=[$12], aws=[$13], event=[$14]) + LogicalSort(sort0=[$15], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35], metrics.size=[$28]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, 'log.file.path:\/var\/log\/messages\/solarshark':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->query_string(MAP('query', 'log.file.path:\/var\/log\/messages\/solarshark':VARCHAR)), SORT->[{ + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->query_string(MAP('query':VARCHAR, 'log.file.path:\/var\/log\/messages\/solarshark':VARCHAR)), SORT->[{ "metrics.size" : { "order" : "desc", "missing" : "_last" } - }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"log.file.path:\\/var\\/log\\/messages\\/solarshark","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"metrics.size":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"log.file.path:\\/var\\/log\\/messages\\/solarshark","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"metrics.size":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/term.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/term.yaml index 21c0d2d0e5d..747674235d3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/term.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/term.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) - LogicalSort(fetch=[10]) + LogicalSort(fetch=[10]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) LogicalFilter(condition=[=($10, '/var/log/messages/birdknight')]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, log.file.path, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->=($3, '/var/log/messages/birdknight'), LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"term":{"log.file.path":{"value":"/var/log/messages/birdknight","boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, log.file.path, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->=($3, '/var/log/messages/birdknight'), PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"term":{"log.file.path":{"value":"/var/log/messages/birdknight","boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/terms_significant_1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/terms_significant_1.yaml index 938181676fb..a0f4f112e26 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/terms_significant_1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/terms_significant_1.yaml @@ -5,7 +5,7 @@ calcite: LogicalProject(count()=[$2], aws.cloudwatch.log_stream=[$0], process.name=[$1]) LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) LogicalProject(aws.cloudwatch.log_stream=[$34], process.name=[$7]) - LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00')), <($17, TIMESTAMP('2023-01-03 00:00:00')))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->SEARCH($1, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},count()=COUNT()), PROJECT->[count(), aws.cloudwatch.log_stream, process.name], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->SEARCH($1, Sarg[['2023-01-01 00:00:00':EXPR_TIMESTAMP VARCHAR..'2023-01-03 00:00:00':EXPR_TIMESTAMP VARCHAR)]:EXPR_TIMESTAMP VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},count()=COUNT()), PROJECT->[count(), aws.cloudwatch.log_stream, process.name], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/terms_significant_2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/terms_significant_2.yaml index da9a207307d..62cce8bd8b7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/terms_significant_2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/terms_significant_2.yaml @@ -5,7 +5,7 @@ calcite: LogicalProject(count()=[$2], process.name=[$0], aws.cloudwatch.log_stream=[$1]) LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) LogicalProject(process.name=[$7], aws.cloudwatch.log_stream=[$34]) - LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00')), <($17, TIMESTAMP('2023-01-03 00:00:00')))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->SEARCH($1, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},count()=COUNT()), PROJECT->[count(), process.name, aws.cloudwatch.log_stream], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->SEARCH($1, Sarg[['2023-01-01 00:00:00':EXPR_TIMESTAMP VARCHAR..'2023-01-03 00:00:00':EXPR_TIMESTAMP VARCHAR)]:EXPR_TIMESTAMP VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},count()=COUNT()), PROJECT->[count(), process.name, aws.cloudwatch.log_stream], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml index 2f539057b66..9e5618c9e06 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml @@ -3,7 +3,7 @@ calcite: LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) + LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'NULL':VARCHAR, <=($5, 10), $1, 'OTHER':VARCHAR)], avg(balance)=[$2]) LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) @@ -22,7 +22,7 @@ calcite: CalciteEnumerableTopK(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL':VARCHAR], expr#7=[10:BIGINT], expr#8=[<=($t4, $t7)], expr#9=['OTHER':VARCHAR], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) @@ -32,4 +32,4 @@ calcite: EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t0)], expr#3=[IS NOT NULL($t2)], age=[$t2], avg(balance)=[$t1], $condition=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(balance)=AVG($1)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(balance)=AVG($1)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml index df5a461ab85..5f19b544102 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml @@ -3,37 +3,37 @@ calcite: LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'nil', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) + LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'nil':VARCHAR, <=($5, 10), $1, 'OTHER':VARCHAR)], avg(balance)=[$2]) LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], $f1=[SPAN($5, 10, null:ANY)], balance=[$3]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($3))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], $f1=[SPAN($5, 10, null:ANY)], balance=[$3]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($3))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) physical: | CalciteEnumerableTopK(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil':VARCHAR], expr#7=[10:BIGINT], expr#8=[<=($t4, $t7)], expr#9=['OTHER':VARCHAR], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:ANY], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], $f1=[$t5], balance=[$t0]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:ANY], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], $f1=[$t5], balance=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml index 9267e6faab1..8316b2ce13d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml @@ -3,29 +3,29 @@ calcite: LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) LogicalAggregate(group=[{0, 1}], max(value)=[MAX($2)]) - LogicalProject(timestamp=[$0], category=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], max(value)=[$2]) + LogicalProject(timestamp=[$0], category=[CASE(IS NULL($1), 'NULL':VARCHAR, <=($5, 10), $1, 'OTHER':VARCHAR)], max(value)=[$2]) LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalProject(timestamp=[$1], category=[$0], max(value)=[$2]) - LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) - LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) + LogicalAggregate(group=[{0, 1}], max(value)=[MAX($2)]) + LogicalProject(category=[$1], timestamp=[SPAN($3, 1, 'w')], value=[$2]) LogicalFilter(condition=[AND(IS NOT NULL($3), IS NOT NULL($2))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) LogicalProject(category=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(category=[$0], max(value)=[$2]) - LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) - LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) + LogicalAggregate(group=[{0, 1}], max(value)=[MAX($2)]) + LogicalProject(category=[$1], $f1=[SPAN($3, 1, 'w')], value=[$2]) LogicalFilter(condition=[AND(IS NOT NULL($3), IS NOT NULL($2))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | CalciteEnumerableTopK(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000]) EnumerableAggregate(group=[{0, 1}], max(value)=[MAX($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], timestamp=[$t0], category=[$t10], max(value)=[$t2]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL':VARCHAR], expr#7=[10:BIGINT], expr#8=[<=($t4, $t7)], expr#9=['OTHER':VARCHAR], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], timestamp=[$t0], category=[$t10], max(value)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[timestamp0, category, max(value)], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"value","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(value)=MAX($2)), PROJECT->[timestamp, category, max(value)], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"value","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"timestamp":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[category, max(value)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"value","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"category","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(value)=MAX($2)), PROJECT->[category, max(value)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"value","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"category","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"timestamp":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml index 8b9f72596b1..f152da7376f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml @@ -3,7 +3,7 @@ calcite: LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - LogicalProject(flags=[$0], severityText=[CASE(IS NULL($1), 'NULL', <=($5, 2), $1, 'max_among_other')], max(severityNumber)=[$2]) + LogicalProject(flags=[$0], severityText=[CASE(IS NULL($1), 'NULL':VARCHAR, <=($5, 2), $1, 'max_among_other':VARCHAR)], max(severityNumber)=[$2]) LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) @@ -20,11 +20,11 @@ calcite: physical: | CalciteEnumerableTopK(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000]) EnumerableAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[2], expr#8=[<=($t4, $t7)], expr#9=['max_among_other'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], flags=[$t0], severityText=[$t10], max(severityNumber)=[$t2]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL':VARCHAR], expr#7=[2:BIGINT], expr#8=[<=($t4, $t7)], expr#9=['max_among_other':VARCHAR], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], flags=[$t0], severityText=[$t10], max(severityNumber)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($2)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityNumber","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($2)), FILTER->IS NOT NULL($1), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), PROJECT->[severityText, max(severityNumber)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityNumber","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"severityText","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($2)), FILTER->IS NOT NULL($1), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), PROJECT->[severityText, max(severityNumber)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityNumber","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"severityText","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_with_integer_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_with_integer_span.yaml index 5a37ec9b92c..c4518124cdb 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_with_integer_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_with_integer_span.yaml @@ -2,10 +2,9 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[ASC]) - LogicalProject(age=[$0], max(balance)=[$1]) - LogicalAggregate(group=[{1}], max(balance)=[MAX($0)]) - LogicalProject(balance=[$7], age0=[SPAN($10, 10, null:NULL)]) - LogicalFilter(condition=[AND(IS NOT NULL($10), IS NOT NULL($7))]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalAggregate(group=[{0}], max(balance)=[MAX($1)]) + LogicalProject(age=[SPAN($10, 10, null:ANY)], balance=[$7]) + LogicalFilter(condition=[AND(IS NOT NULL($10), IS NOT NULL($7))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},max(balance)=MAX($0)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"age","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age0":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":10.0}}}]},"aggregations":{"max(balance)":{"max":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},max(balance)=MAX($1)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"age","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":10.0}}}]},"aggregations":{"max(balance)":{"max":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_with_timestamp_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_with_timestamp_span.yaml index 8768b05a1a0..301d37cc993 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_with_timestamp_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_with_timestamp_span.yaml @@ -2,10 +2,9 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[ASC]) - LogicalProject(@timestamp=[$0], count=[$1]) - LogicalAggregate(group=[{0}], count=[COUNT()]) - LogicalProject(@timestamp0=[SPAN($0, 1, 'd')]) - LogicalFilter(condition=[IS NOT NULL($0)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalAggregate(group=[{0}], count=[COUNT()]) + LogicalProject(@timestamp=[SPAN($0, 1, 'd')]) + LogicalFilter(condition=[IS NOT NULL($0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count=COUNT()), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"@timestamp0":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count=COUNT()), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"@timestamp":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q19.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q19.yaml index c4005b2ea4f..e545a73c7ec 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q19.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q19.yaml @@ -6,7 +6,7 @@ calcite: LogicalAggregate(group=[{0, 1, 2}], count()=[COUNT()]) LogicalProject(UserID=[$84], m=[$111], SearchPhrase=[$63]) LogicalFilter(condition=[AND(IS NOT NULL($84), IS NOT NULL($111), IS NOT NULL($63))]) - LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104], _id=[$105], _index=[$106], _score=[$107], _maxscore=[$108], _sort=[$109], _routing=[$110], m=[EXTRACT('minute':VARCHAR, $17)]) + LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104], _id=[$105], _index=[$106], _score=[$107], _maxscore=[$108], _sort=[$109], _routing=[$110], m=[EXTRACT('minute', $17)]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},count()=COUNT()), SORT_AGG_METRICS->[3 DESC LAST], PROJECT->[count(), UserID, m, SearchPhrase], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"UserID|m|SearchPhrase":{"multi_terms":{"terms":[{"field":"UserID"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCZXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJFWFRSQUNUIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0sCiAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiQklHSU5UIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9LAogICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAiZHluYW1pYyI6IGZhbHNlCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[2,0],"DIGESTS":["minute","EventTime"]}},"value_type":"long"},{"field":"SearchPhrase"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q2.yaml index d80d488a167..da73c50c2bb 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q2.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalAggregate(group=[{}], count()=[COUNT()]) - LogicalFilter(condition=[<>($19, 0)]) + LogicalFilter(condition=[<>(SAFE_CAST($19), 0)]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER-><>($0, 0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"AdvEngineID","boost":1.0}}],"must_not":[{"term":{"AdvEngineID":{"value":0,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[SCRIPT-><>(SAFE_CAST($0), 0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCSnsKICAib3AiOiB7CiAgICAibmFtZSI6ICI8PiIsCiAgICAia2luZCI6ICJOT1RfRVFVQUxTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlNNQUxMSU5UIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["AdvEngineID",0]}},"boost":1.0}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q24.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q24.yaml index 545df273617..0a7339f9928 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q24.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q24.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104]) - LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) + LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) + LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104]) LogicalFilter(condition=[LIKE($26, '%google%', '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | @@ -11,4 +11,4 @@ calcite: "order" : "asc", "missing" : "_first" } - }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"wildcard":{"URL":{"wildcard":"*google*","boost":1.0}}},"_source":{"includes":["EventDate","URLRegionID","HasGCLID","Income","Interests","Robotness","BrowserLanguage","CounterClass","BrowserCountry","OriginalURL","ClientTimeZone","RefererHash","TraficSourceID","HitColor","RefererRegionID","URLCategoryID","LocalEventTime","EventTime","UTMTerm","AdvEngineID","UserAgentMinor","UserAgentMajor","RemoteIP","Sex","JavaEnable","URLHash","URL","ParamOrderID","OpenstatSourceID","HTTPError","SilverlightVersion3","MobilePhoneModel","SilverlightVersion4","SilverlightVersion1","SilverlightVersion2","IsDownload","IsParameter","CLID","FlashMajor","FlashMinor","UTMMedium","WatchID","DontCountHits","CookieEnable","HID","SocialAction","WindowName","ConnectTiming","PageCharset","IsLink","IsArtifical","JavascriptEnable","ClientEventTime","DNSTiming","CodeVersion","ResponseEndTiming","FUniqID","WindowClientHeight","OpenstatServiceName","UTMContent","HistoryLength","IsOldCounter","MobilePhone","SearchPhrase","FlashMinor2","SearchEngineID","IsEvent","UTMSource","RegionID","OpenstatAdID","UTMCampaign","GoodEvent","IsRefresh","ParamCurrency","Params","ResolutionHeight","ClientIP","FromTag","ParamCurrencyID","ResponseStartTiming","ResolutionWidth","SendTiming","RefererCategoryID","OpenstatCampaignID","UserID","WithHash","UserAgent","ParamPrice","ResolutionDepth","IsMobile","Age","SocialSourceNetworkID","OpenerName","OS","IsNotBounce","Referer","NetMinor","Title","NetMajor","IPNetworkID","FetchTiming","SocialNetwork","SocialSourcePage","CounterID","WindowClientWidth"],"excludes":[]},"sort":[{"EventTime":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"wildcard":{"URL":{"wildcard":"*google*","boost":1.0}}},"_source":{"includes":["EventDate","URLRegionID","HasGCLID","Income","Interests","Robotness","BrowserLanguage","CounterClass","BrowserCountry","OriginalURL","ClientTimeZone","RefererHash","TraficSourceID","HitColor","RefererRegionID","URLCategoryID","LocalEventTime","EventTime","UTMTerm","AdvEngineID","UserAgentMinor","UserAgentMajor","RemoteIP","Sex","JavaEnable","URLHash","URL","ParamOrderID","OpenstatSourceID","HTTPError","SilverlightVersion3","MobilePhoneModel","SilverlightVersion4","SilverlightVersion1","SilverlightVersion2","IsDownload","IsParameter","CLID","FlashMajor","FlashMinor","UTMMedium","WatchID","DontCountHits","CookieEnable","HID","SocialAction","WindowName","ConnectTiming","PageCharset","IsLink","IsArtifical","JavascriptEnable","ClientEventTime","DNSTiming","CodeVersion","ResponseEndTiming","FUniqID","WindowClientHeight","OpenstatServiceName","UTMContent","HistoryLength","IsOldCounter","MobilePhone","SearchPhrase","FlashMinor2","SearchEngineID","IsEvent","UTMSource","RegionID","OpenstatAdID","UTMCampaign","GoodEvent","IsRefresh","ParamCurrency","Params","ResolutionHeight","ClientIP","FromTag","ParamCurrencyID","ResponseStartTiming","ResolutionWidth","SendTiming","RefererCategoryID","OpenstatCampaignID","UserID","WithHash","UserAgent","ParamPrice","ResolutionDepth","IsMobile","Age","SocialSourceNetworkID","OpenerName","OS","IsNotBounce","Referer","NetMinor","Title","NetMajor","IPNetworkID","FetchTiming","SocialNetwork","SocialSourcePage","CounterID","WindowClientWidth"],"excludes":[]},"sort":[{"EventTime":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q25.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q25.yaml index 612b8bc06f8..2186b9f531d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q25.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q25.yaml @@ -1,10 +1,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(SearchPhrase=[$63]) - LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) - LogicalFilter(condition=[<>($63, '')]) - CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + LogicalProject(SearchPhrase=[$0]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10]) + LogicalProject(SearchPhrase=[$63], EventTime=[$17]) + LogicalFilter(condition=[<>($63, '')]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventTime, SearchPhrase], FILTER-><>($1, ''), SORT->[{ "EventTime" : { diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q27.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q27.yaml index 1da73eb16c8..84cd57e7049 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q27.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q27.yaml @@ -1,10 +1,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(SearchPhrase=[$63]) - LogicalSort(sort0=[$17], sort1=[$63], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10]) - LogicalFilter(condition=[<>($63, '')]) - CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + LogicalProject(SearchPhrase=[$0]) + LogicalSort(sort0=[$1], sort1=[$0], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10]) + LogicalProject(SearchPhrase=[$63], EventTime=[$17]) + LogicalFilter(condition=[<>($63, '')]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventTime, SearchPhrase], FILTER-><>($1, ''), SORT->[{ "EventTime" : { diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q28.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q28.yaml index 3b02510db8b..b27b04447ea 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q28.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q28.yaml @@ -2,14 +2,15 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[25]) - LogicalFilter(condition=[>($1, 100000)]) - LogicalProject(l=[$1], c=[$2], CounterID=[$0]) - LogicalAggregate(group=[{0}], l=[AVG($1)], c=[COUNT()]) - LogicalProject(CounterID=[$103], $f2=[CHAR_LENGTH($26)]) - LogicalFilter(condition=[AND(<>($26, ''), IS NOT NULL($103))]) - CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + LogicalProject(l=[$0], c=[$1], CounterID=[$2]) + LogicalFilter(condition=[>($1, SAFE_CAST(100000:BIGINT))]) + LogicalProject(l=[$1], c=[$2], CounterID=[$0]) + LogicalAggregate(group=[{0}], l=[AVG($1)], c=[COUNT()]) + LogicalProject(CounterID=[$103], $f1=[CHAR_LENGTH($26)]) + LogicalFilter(condition=[AND(<>($26, ''), IS NOT NULL($103))]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | EnumerableLimit(fetch=[10000]) CalciteEnumerableTopK(sort0=[$0], dir0=[DESC-nulls-last], fetch=[25]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100000], expr#4=[>($t1, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(<>($0, ''), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},l=AVG($1),c=COUNT()), PROJECT->[l, c, CounterID]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"URL","boost":1.0}}],"must_not":[{"term":{"URL":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"CounterID","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"CounterID":{"terms":{"field":"CounterID","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"l":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["URL"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100000:BIGINT], expr#4=[>($t1, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(<>($0, ''), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},l=AVG($1),c=COUNT()), PROJECT->[l, c, CounterID]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"URL","boost":1.0}}],"must_not":[{"term":{"URL":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"CounterID","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"CounterID":{"terms":{"field":"CounterID","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"l":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["URL"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q29.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q29.yaml index c5d5092d7cf..00330165059 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q29.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q29.yaml @@ -2,16 +2,17 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[25]) - LogicalFilter(condition=[>($1, 100000)]) - LogicalProject(l=[$1], c=[$2], min(Referer)=[$3], k=[$0]) - LogicalAggregate(group=[{0}], l=[AVG($2)], c=[COUNT()], min(Referer)=[MIN($1)]) - LogicalProject(k=[$111], Referer=[$95], $f3=[CHAR_LENGTH($95)]) - LogicalFilter(condition=[IS NOT NULL($111)]) - LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104], _id=[$105], _index=[$106], _score=[$107], _maxscore=[$108], _sort=[$109], _routing=[$110], k=[REGEXP_REPLACE($95, '^https?://(?:www\.)?([^/]+)/.*$':VARCHAR, '$1')]) - LogicalFilter(condition=[<>($95, '')]) - CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + LogicalProject(l=[$0], c=[$1], min(Referer)=[$2], k=[$3]) + LogicalFilter(condition=[>($1, SAFE_CAST(100000:BIGINT))]) + LogicalProject(l=[$1], c=[$2], min(Referer)=[$3], k=[$0]) + LogicalAggregate(group=[{0}], l=[AVG($1)], c=[COUNT()], min(Referer)=[MIN($2)]) + LogicalProject(k=[$111], $f1=[CHAR_LENGTH($95)], Referer=[$95]) + LogicalFilter(condition=[IS NOT NULL($111)]) + LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104], _id=[$105], _index=[$106], _score=[$107], _maxscore=[$108], _sort=[$109], _routing=[$110], k=[REGEXP_REPLACE($95, '^https?://(?:www\.)?([^/]+)/.*$', '$1')]) + LogicalFilter(condition=[<>($95, '')]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | EnumerableLimit(fetch=[10000]) CalciteEnumerableTopK(sort0=[$0], dir0=[DESC-nulls-last], fetch=[25]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100000], expr#5=[>($t1, $t4)], proj#0..3=[{exprs}], $condition=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER-><>($0, ''), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},l=AVG($2),c=COUNT(),min(Referer)=MIN($1)), PROJECT->[l, c, min(Referer), k]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"Referer","boost":1.0}}],"must_not":[{"term":{"Referer":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"k":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCG3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["Referer","^https?://(?:www\\.)?([^/]+)/.*$","$1"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"l":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["Referer"]}}}},"min(Referer)":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"Referer"}],"sort":[{"Referer":{"order":"asc"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100000:BIGINT], expr#5=[>($t1, $t4)], proj#0..3=[{exprs}], $condition=[$t5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER-><>($0, ''), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},l=AVG($1),c=COUNT(),min(Referer)=MIN($2)), PROJECT->[l, c, min(Referer), k]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"Referer","boost":1.0}}],"must_not":[{"term":{"Referer":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"k":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCG3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["Referer","^https?://(?:www\\.)?([^/]+)/.*$","$1"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"l":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["Referer"]}}}},"min(Referer)":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"Referer"}],"sort":[{"Referer":{"order":"asc"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml index de1f6d31f0d..3d3623bec0a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalAggregate(group=[{}], sum(ResolutionWidth)=[SUM($0)], sum(ResolutionWidth+1)=[SUM($1)], sum(ResolutionWidth+2)=[SUM($2)], sum(ResolutionWidth+3)=[SUM($3)], sum(ResolutionWidth+4)=[SUM($4)], sum(ResolutionWidth+5)=[SUM($5)], sum(ResolutionWidth+6)=[SUM($6)], sum(ResolutionWidth+7)=[SUM($7)], sum(ResolutionWidth+8)=[SUM($8)], sum(ResolutionWidth+9)=[SUM($9)], sum(ResolutionWidth+10)=[SUM($10)], sum(ResolutionWidth+11)=[SUM($11)], sum(ResolutionWidth+12)=[SUM($12)], sum(ResolutionWidth+13)=[SUM($13)], sum(ResolutionWidth+14)=[SUM($14)], sum(ResolutionWidth+15)=[SUM($15)], sum(ResolutionWidth+16)=[SUM($16)], sum(ResolutionWidth+17)=[SUM($17)], sum(ResolutionWidth+18)=[SUM($18)], sum(ResolutionWidth+19)=[SUM($19)], sum(ResolutionWidth+20)=[SUM($20)], sum(ResolutionWidth+21)=[SUM($21)], sum(ResolutionWidth+22)=[SUM($22)], sum(ResolutionWidth+23)=[SUM($23)], sum(ResolutionWidth+24)=[SUM($24)], sum(ResolutionWidth+25)=[SUM($25)], sum(ResolutionWidth+26)=[SUM($26)], sum(ResolutionWidth+27)=[SUM($27)], sum(ResolutionWidth+28)=[SUM($28)], sum(ResolutionWidth+29)=[SUM($29)], sum(ResolutionWidth+30)=[SUM($30)], sum(ResolutionWidth+31)=[SUM($31)], sum(ResolutionWidth+32)=[SUM($32)], sum(ResolutionWidth+33)=[SUM($33)], sum(ResolutionWidth+34)=[SUM($34)], sum(ResolutionWidth+35)=[SUM($35)], sum(ResolutionWidth+36)=[SUM($36)], sum(ResolutionWidth+37)=[SUM($37)], sum(ResolutionWidth+38)=[SUM($38)], sum(ResolutionWidth+39)=[SUM($39)], sum(ResolutionWidth+40)=[SUM($40)], sum(ResolutionWidth+41)=[SUM($41)], sum(ResolutionWidth+42)=[SUM($42)], sum(ResolutionWidth+43)=[SUM($43)], sum(ResolutionWidth+44)=[SUM($44)], sum(ResolutionWidth+45)=[SUM($45)], sum(ResolutionWidth+46)=[SUM($46)], sum(ResolutionWidth+47)=[SUM($47)], sum(ResolutionWidth+48)=[SUM($48)], sum(ResolutionWidth+49)=[SUM($49)], sum(ResolutionWidth+50)=[SUM($50)], sum(ResolutionWidth+51)=[SUM($51)], sum(ResolutionWidth+52)=[SUM($52)], sum(ResolutionWidth+53)=[SUM($53)], sum(ResolutionWidth+54)=[SUM($54)], sum(ResolutionWidth+55)=[SUM($55)], sum(ResolutionWidth+56)=[SUM($56)], sum(ResolutionWidth+57)=[SUM($57)], sum(ResolutionWidth+58)=[SUM($58)], sum(ResolutionWidth+59)=[SUM($59)], sum(ResolutionWidth+60)=[SUM($60)], sum(ResolutionWidth+61)=[SUM($61)], sum(ResolutionWidth+62)=[SUM($62)], sum(ResolutionWidth+63)=[SUM($63)], sum(ResolutionWidth+64)=[SUM($64)], sum(ResolutionWidth+65)=[SUM($65)], sum(ResolutionWidth+66)=[SUM($66)], sum(ResolutionWidth+67)=[SUM($67)], sum(ResolutionWidth+68)=[SUM($68)], sum(ResolutionWidth+69)=[SUM($69)], sum(ResolutionWidth+70)=[SUM($70)], sum(ResolutionWidth+71)=[SUM($71)], sum(ResolutionWidth+72)=[SUM($72)], sum(ResolutionWidth+73)=[SUM($73)], sum(ResolutionWidth+74)=[SUM($74)], sum(ResolutionWidth+75)=[SUM($75)], sum(ResolutionWidth+76)=[SUM($76)], sum(ResolutionWidth+77)=[SUM($77)], sum(ResolutionWidth+78)=[SUM($78)], sum(ResolutionWidth+79)=[SUM($79)], sum(ResolutionWidth+80)=[SUM($80)], sum(ResolutionWidth+81)=[SUM($81)], sum(ResolutionWidth+82)=[SUM($82)], sum(ResolutionWidth+83)=[SUM($83)], sum(ResolutionWidth+84)=[SUM($84)], sum(ResolutionWidth+85)=[SUM($85)], sum(ResolutionWidth+86)=[SUM($86)], sum(ResolutionWidth+87)=[SUM($87)], sum(ResolutionWidth+88)=[SUM($88)], sum(ResolutionWidth+89)=[SUM($89)]) - LogicalProject(ResolutionWidth=[$80], $f90=[+($80, 1)], $f91=[+($80, 2)], $f92=[+($80, 3)], $f93=[+($80, 4)], $f94=[+($80, 5)], $f95=[+($80, 6)], $f96=[+($80, 7)], $f97=[+($80, 8)], $f98=[+($80, 9)], $f99=[+($80, 10)], $f100=[+($80, 11)], $f101=[+($80, 12)], $f102=[+($80, 13)], $f103=[+($80, 14)], $f104=[+($80, 15)], $f105=[+($80, 16)], $f106=[+($80, 17)], $f107=[+($80, 18)], $f108=[+($80, 19)], $f109=[+($80, 20)], $f110=[+($80, 21)], $f111=[+($80, 22)], $f112=[+($80, 23)], $f113=[+($80, 24)], $f114=[+($80, 25)], $f115=[+($80, 26)], $f116=[+($80, 27)], $f117=[+($80, 28)], $f118=[+($80, 29)], $f119=[+($80, 30)], $f120=[+($80, 31)], $f121=[+($80, 32)], $f122=[+($80, 33)], $f123=[+($80, 34)], $f124=[+($80, 35)], $f125=[+($80, 36)], $f126=[+($80, 37)], $f127=[+($80, 38)], $f128=[+($80, 39)], $f129=[+($80, 40)], $f130=[+($80, 41)], $f131=[+($80, 42)], $f132=[+($80, 43)], $f133=[+($80, 44)], $f134=[+($80, 45)], $f135=[+($80, 46)], $f136=[+($80, 47)], $f137=[+($80, 48)], $f138=[+($80, 49)], $f139=[+($80, 50)], $f140=[+($80, 51)], $f141=[+($80, 52)], $f142=[+($80, 53)], $f143=[+($80, 54)], $f144=[+($80, 55)], $f145=[+($80, 56)], $f146=[+($80, 57)], $f147=[+($80, 58)], $f148=[+($80, 59)], $f149=[+($80, 60)], $f150=[+($80, 61)], $f151=[+($80, 62)], $f152=[+($80, 63)], $f153=[+($80, 64)], $f154=[+($80, 65)], $f155=[+($80, 66)], $f156=[+($80, 67)], $f157=[+($80, 68)], $f158=[+($80, 69)], $f159=[+($80, 70)], $f160=[+($80, 71)], $f161=[+($80, 72)], $f162=[+($80, 73)], $f163=[+($80, 74)], $f164=[+($80, 75)], $f165=[+($80, 76)], $f166=[+($80, 77)], $f167=[+($80, 78)], $f168=[+($80, 79)], $f169=[+($80, 80)], $f170=[+($80, 81)], $f171=[+($80, 82)], $f172=[+($80, 83)], $f173=[+($80, 84)], $f174=[+($80, 85)], $f175=[+($80, 86)], $f176=[+($80, 87)], $f177=[+($80, 88)], $f178=[+($80, 89)]) + LogicalProject(ResolutionWidth=[$80], $f1=[+($80, 1)], $f2=[+($80, 2)], $f3=[+($80, 3)], $f4=[+($80, 4)], $f5=[+($80, 5)], $f6=[+($80, 6)], $f7=[+($80, 7)], $f8=[+($80, 8)], $f9=[+($80, 9)], $f10=[+($80, 10)], $f11=[+($80, 11)], $f12=[+($80, 12)], $f13=[+($80, 13)], $f14=[+($80, 14)], $f15=[+($80, 15)], $f16=[+($80, 16)], $f17=[+($80, 17)], $f18=[+($80, 18)], $f19=[+($80, 19)], $f20=[+($80, 20)], $f21=[+($80, 21)], $f22=[+($80, 22)], $f23=[+($80, 23)], $f24=[+($80, 24)], $f25=[+($80, 25)], $f26=[+($80, 26)], $f27=[+($80, 27)], $f28=[+($80, 28)], $f29=[+($80, 29)], $f30=[+($80, 30)], $f31=[+($80, 31)], $f32=[+($80, 32)], $f33=[+($80, 33)], $f34=[+($80, 34)], $f35=[+($80, 35)], $f36=[+($80, 36)], $f37=[+($80, 37)], $f38=[+($80, 38)], $f39=[+($80, 39)], $f40=[+($80, 40)], $f41=[+($80, 41)], $f42=[+($80, 42)], $f43=[+($80, 43)], $f44=[+($80, 44)], $f45=[+($80, 45)], $f46=[+($80, 46)], $f47=[+($80, 47)], $f48=[+($80, 48)], $f49=[+($80, 49)], $f50=[+($80, 50)], $f51=[+($80, 51)], $f52=[+($80, 52)], $f53=[+($80, 53)], $f54=[+($80, 54)], $f55=[+($80, 55)], $f56=[+($80, 56)], $f57=[+($80, 57)], $f58=[+($80, 58)], $f59=[+($80, 59)], $f60=[+($80, 60)], $f61=[+($80, 61)], $f62=[+($80, 62)], $f63=[+($80, 63)], $f64=[+($80, 64)], $f65=[+($80, 65)], $f66=[+($80, 66)], $f67=[+($80, 67)], $f68=[+($80, 68)], $f69=[+($80, 69)], $f70=[+($80, 70)], $f71=[+($80, 71)], $f72=[+($80, 72)], $f73=[+($80, 73)], $f74=[+($80, 74)], $f75=[+($80, 75)], $f76=[+($80, 76)], $f77=[+($80, 77)], $f78=[+($80, 78)], $f79=[+($80, 79)], $f80=[+($80, 80)], $f81=[+($80, 81)], $f82=[+($80, 82)], $f83=[+($80, 83)], $f84=[+($80, 84)], $f85=[+($80, 85)], $f86=[+($80, 86)], $f87=[+($80, 87)], $f88=[+($80, 88)], $f89=[+($80, 89)]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t1):BIGINT], expr#3=[+($t0, $t2)], expr#4=[2], expr#5=[*($t1, $t4)], expr#6=[+($t0, $t5)], expr#7=[3], expr#8=[*($t1, $t7)], expr#9=[+($t0, $t8)], expr#10=[4], expr#11=[*($t1, $t10)], expr#12=[+($t0, $t11)], expr#13=[5], expr#14=[*($t1, $t13)], expr#15=[+($t0, $t14)], expr#16=[6], expr#17=[*($t1, $t16)], expr#18=[+($t0, $t17)], expr#19=[7], expr#20=[*($t1, $t19)], expr#21=[+($t0, $t20)], expr#22=[8], expr#23=[*($t1, $t22)], expr#24=[+($t0, $t23)], expr#25=[9], expr#26=[*($t1, $t25)], expr#27=[+($t0, $t26)], expr#28=[10], expr#29=[*($t1, $t28)], expr#30=[+($t0, $t29)], expr#31=[11], expr#32=[*($t1, $t31)], expr#33=[+($t0, $t32)], expr#34=[12], expr#35=[*($t1, $t34)], expr#36=[+($t0, $t35)], expr#37=[13], expr#38=[*($t1, $t37)], expr#39=[+($t0, $t38)], expr#40=[14], expr#41=[*($t1, $t40)], expr#42=[+($t0, $t41)], expr#43=[15], expr#44=[*($t1, $t43)], expr#45=[+($t0, $t44)], expr#46=[16], expr#47=[*($t1, $t46)], expr#48=[+($t0, $t47)], expr#49=[17], expr#50=[*($t1, $t49)], expr#51=[+($t0, $t50)], expr#52=[18], expr#53=[*($t1, $t52)], expr#54=[+($t0, $t53)], expr#55=[19], expr#56=[*($t1, $t55)], expr#57=[+($t0, $t56)], expr#58=[20], expr#59=[*($t1, $t58)], expr#60=[+($t0, $t59)], expr#61=[21], expr#62=[*($t1, $t61)], expr#63=[+($t0, $t62)], expr#64=[22], expr#65=[*($t1, $t64)], expr#66=[+($t0, $t65)], expr#67=[23], expr#68=[*($t1, $t67)], expr#69=[+($t0, $t68)], expr#70=[24], expr#71=[*($t1, $t70)], expr#72=[+($t0, $t71)], expr#73=[25], expr#74=[*($t1, $t73)], expr#75=[+($t0, $t74)], expr#76=[26], expr#77=[*($t1, $t76)], expr#78=[+($t0, $t77)], expr#79=[27], expr#80=[*($t1, $t79)], expr#81=[+($t0, $t80)], expr#82=[28], expr#83=[*($t1, $t82)], expr#84=[+($t0, $t83)], expr#85=[29], expr#86=[*($t1, $t85)], expr#87=[+($t0, $t86)], expr#88=[30], expr#89=[*($t1, $t88)], expr#90=[+($t0, $t89)], expr#91=[31], expr#92=[*($t1, $t91)], expr#93=[+($t0, $t92)], expr#94=[32], expr#95=[*($t1, $t94)], expr#96=[+($t0, $t95)], expr#97=[33], expr#98=[*($t1, $t97)], expr#99=[+($t0, $t98)], expr#100=[34], expr#101=[*($t1, $t100)], expr#102=[+($t0, $t101)], expr#103=[35], expr#104=[*($t1, $t103)], expr#105=[+($t0, $t104)], expr#106=[36], expr#107=[*($t1, $t106)], expr#108=[+($t0, $t107)], expr#109=[37], expr#110=[*($t1, $t109)], expr#111=[+($t0, $t110)], expr#112=[38], expr#113=[*($t1, $t112)], expr#114=[+($t0, $t113)], expr#115=[39], expr#116=[*($t1, $t115)], expr#117=[+($t0, $t116)], expr#118=[40], expr#119=[*($t1, $t118)], expr#120=[+($t0, $t119)], expr#121=[41], expr#122=[*($t1, $t121)], expr#123=[+($t0, $t122)], expr#124=[42], expr#125=[*($t1, $t124)], expr#126=[+($t0, $t125)], expr#127=[43], expr#128=[*($t1, $t127)], expr#129=[+($t0, $t128)], expr#130=[44], expr#131=[*($t1, $t130)], expr#132=[+($t0, $t131)], expr#133=[45], expr#134=[*($t1, $t133)], expr#135=[+($t0, $t134)], expr#136=[46], expr#137=[*($t1, $t136)], expr#138=[+($t0, $t137)], expr#139=[47], expr#140=[*($t1, $t139)], expr#141=[+($t0, $t140)], expr#142=[48], expr#143=[*($t1, $t142)], expr#144=[+($t0, $t143)], expr#145=[49], expr#146=[*($t1, $t145)], expr#147=[+($t0, $t146)], expr#148=[50], expr#149=[*($t1, $t148)], expr#150=[+($t0, $t149)], expr#151=[51], expr#152=[*($t1, $t151)], expr#153=[+($t0, $t152)], expr#154=[52], expr#155=[*($t1, $t154)], expr#156=[+($t0, $t155)], expr#157=[53], expr#158=[*($t1, $t157)], expr#159=[+($t0, $t158)], expr#160=[54], expr#161=[*($t1, $t160)], expr#162=[+($t0, $t161)], expr#163=[55], expr#164=[*($t1, $t163)], expr#165=[+($t0, $t164)], expr#166=[56], expr#167=[*($t1, $t166)], expr#168=[+($t0, $t167)], expr#169=[57], expr#170=[*($t1, $t169)], expr#171=[+($t0, $t170)], expr#172=[58], expr#173=[*($t1, $t172)], expr#174=[+($t0, $t173)], expr#175=[59], expr#176=[*($t1, $t175)], expr#177=[+($t0, $t176)], expr#178=[60], expr#179=[*($t1, $t178)], expr#180=[+($t0, $t179)], expr#181=[61], expr#182=[*($t1, $t181)], expr#183=[+($t0, $t182)], expr#184=[62], expr#185=[*($t1, $t184)], expr#186=[+($t0, $t185)], expr#187=[63], expr#188=[*($t1, $t187)], expr#189=[+($t0, $t188)], expr#190=[64], expr#191=[*($t1, $t190)], expr#192=[+($t0, $t191)], expr#193=[65], expr#194=[*($t1, $t193)], expr#195=[+($t0, $t194)], expr#196=[66], expr#197=[*($t1, $t196)], expr#198=[+($t0, $t197)], expr#199=[67], expr#200=[*($t1, $t199)], expr#201=[+($t0, $t200)], expr#202=[68], expr#203=[*($t1, $t202)], expr#204=[+($t0, $t203)], expr#205=[69], expr#206=[*($t1, $t205)], expr#207=[+($t0, $t206)], expr#208=[70], expr#209=[*($t1, $t208)], expr#210=[+($t0, $t209)], expr#211=[71], expr#212=[*($t1, $t211)], expr#213=[+($t0, $t212)], expr#214=[72], expr#215=[*($t1, $t214)], expr#216=[+($t0, $t215)], expr#217=[73], expr#218=[*($t1, $t217)], expr#219=[+($t0, $t218)], expr#220=[74], expr#221=[*($t1, $t220)], expr#222=[+($t0, $t221)], expr#223=[75], expr#224=[*($t1, $t223)], expr#225=[+($t0, $t224)], expr#226=[76], expr#227=[*($t1, $t226)], expr#228=[+($t0, $t227)], expr#229=[77], expr#230=[*($t1, $t229)], expr#231=[+($t0, $t230)], expr#232=[78], expr#233=[*($t1, $t232)], expr#234=[+($t0, $t233)], expr#235=[79], expr#236=[*($t1, $t235)], expr#237=[+($t0, $t236)], expr#238=[80], expr#239=[*($t1, $t238)], expr#240=[+($t0, $t239)], expr#241=[81], expr#242=[*($t1, $t241)], expr#243=[+($t0, $t242)], expr#244=[82], expr#245=[*($t1, $t244)], expr#246=[+($t0, $t245)], expr#247=[83], expr#248=[*($t1, $t247)], expr#249=[+($t0, $t248)], expr#250=[84], expr#251=[*($t1, $t250)], expr#252=[+($t0, $t251)], expr#253=[85], expr#254=[*($t1, $t253)], expr#255=[+($t0, $t254)], expr#256=[86], expr#257=[*($t1, $t256)], expr#258=[+($t0, $t257)], expr#259=[87], expr#260=[*($t1, $t259)], expr#261=[+($t0, $t260)], expr#262=[88], expr#263=[*($t1, $t262)], expr#264=[+($t0, $t263)], expr#265=[89], expr#266=[*($t1, $t265)], expr#267=[+($t0, $t266)], sum(ResolutionWidth)=[$t0], sum(ResolutionWidth+1)=[$t3], sum(ResolutionWidth+2)=[$t6], sum(ResolutionWidth+3)=[$t9], sum(ResolutionWidth+4)=[$t12], sum(ResolutionWidth+5)=[$t15], sum(ResolutionWidth+6)=[$t18], sum(ResolutionWidth+7)=[$t21], sum(ResolutionWidth+8)=[$t24], sum(ResolutionWidth+9)=[$t27], sum(ResolutionWidth+10)=[$t30], sum(ResolutionWidth+11)=[$t33], sum(ResolutionWidth+12)=[$t36], sum(ResolutionWidth+13)=[$t39], sum(ResolutionWidth+14)=[$t42], sum(ResolutionWidth+15)=[$t45], sum(ResolutionWidth+16)=[$t48], sum(ResolutionWidth+17)=[$t51], sum(ResolutionWidth+18)=[$t54], sum(ResolutionWidth+19)=[$t57], sum(ResolutionWidth+20)=[$t60], sum(ResolutionWidth+21)=[$t63], sum(ResolutionWidth+22)=[$t66], sum(ResolutionWidth+23)=[$t69], sum(ResolutionWidth+24)=[$t72], sum(ResolutionWidth+25)=[$t75], sum(ResolutionWidth+26)=[$t78], sum(ResolutionWidth+27)=[$t81], sum(ResolutionWidth+28)=[$t84], sum(ResolutionWidth+29)=[$t87], sum(ResolutionWidth+30)=[$t90], sum(ResolutionWidth+31)=[$t93], sum(ResolutionWidth+32)=[$t96], sum(ResolutionWidth+33)=[$t99], sum(ResolutionWidth+34)=[$t102], sum(ResolutionWidth+35)=[$t105], sum(ResolutionWidth+36)=[$t108], sum(ResolutionWidth+37)=[$t111], sum(ResolutionWidth+38)=[$t114], sum(ResolutionWidth+39)=[$t117], sum(ResolutionWidth+40)=[$t120], sum(ResolutionWidth+41)=[$t123], sum(ResolutionWidth+42)=[$t126], sum(ResolutionWidth+43)=[$t129], sum(ResolutionWidth+44)=[$t132], sum(ResolutionWidth+45)=[$t135], sum(ResolutionWidth+46)=[$t138], sum(ResolutionWidth+47)=[$t141], sum(ResolutionWidth+48)=[$t144], sum(ResolutionWidth+49)=[$t147], sum(ResolutionWidth+50)=[$t150], sum(ResolutionWidth+51)=[$t153], sum(ResolutionWidth+52)=[$t156], sum(ResolutionWidth+53)=[$t159], sum(ResolutionWidth+54)=[$t162], sum(ResolutionWidth+55)=[$t165], sum(ResolutionWidth+56)=[$t168], sum(ResolutionWidth+57)=[$t171], sum(ResolutionWidth+58)=[$t174], sum(ResolutionWidth+59)=[$t177], sum(ResolutionWidth+60)=[$t180], sum(ResolutionWidth+61)=[$t183], sum(ResolutionWidth+62)=[$t186], sum(ResolutionWidth+63)=[$t189], sum(ResolutionWidth+64)=[$t192], sum(ResolutionWidth+65)=[$t195], sum(ResolutionWidth+66)=[$t198], sum(ResolutionWidth+67)=[$t201], sum(ResolutionWidth+68)=[$t204], sum(ResolutionWidth+69)=[$t207], sum(ResolutionWidth+70)=[$t210], sum(ResolutionWidth+71)=[$t213], sum(ResolutionWidth+72)=[$t216], sum(ResolutionWidth+73)=[$t219], sum(ResolutionWidth+74)=[$t222], sum(ResolutionWidth+75)=[$t225], sum(ResolutionWidth+76)=[$t228], sum(ResolutionWidth+77)=[$t231], sum(ResolutionWidth+78)=[$t234], sum(ResolutionWidth+79)=[$t237], sum(ResolutionWidth+80)=[$t240], sum(ResolutionWidth+81)=[$t243], sum(ResolutionWidth+82)=[$t246], sum(ResolutionWidth+83)=[$t249], sum(ResolutionWidth+84)=[$t252], sum(ResolutionWidth+85)=[$t255], sum(ResolutionWidth+86)=[$t258], sum(ResolutionWidth+87)=[$t261], sum(ResolutionWidth+88)=[$t264], sum(ResolutionWidth+89)=[$t267]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q35.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q35.yaml index 439c445af4b..8146a6ebaad 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q35.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q35.yaml @@ -10,4 +10,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], c=[$t1], const=[$t2], URL=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"URL":{"terms":{"field":"URL","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"URL":{"terms":{"field":"URL","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q37.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q37.yaml index 5cf2fb92261..bcd0be6cbe0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q37.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q37.yaml @@ -5,7 +5,7 @@ calcite: LogicalProject(PageViews=[$1], URL=[$0]) LogicalAggregate(group=[{0}], PageViews=[COUNT()]) LogicalProject(URL=[$26]) - LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2013-07-31 00:00:00':VARCHAR)), =($42, 0), =($72, 0), <>($26, ''))]) + LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00')), <=($0, TIMESTAMP('2013-07-31 00:00:00')), =(SAFE_CAST($42), 0), =(SAFE_CAST($72), 0), <>($26, ''))]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(=($4, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-31 00:00:00':VARCHAR]]:VARCHAR), =($2, 0), =($3, 0), <>($1, '')), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},PageViews=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[PageViews, URL], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"DontCountHits":{"value":0,"boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}},{"bool":{"must":[{"exists":{"field":"URL","boost":1.0}}],"must_not":[{"term":{"URL":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"URL":{"terms":{"field":"URL","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[SCRIPT->AND(=($4, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':EXPR_TIMESTAMP VARCHAR..'2013-07-31 00:00:00':EXPR_TIMESTAMP VARCHAR]]:EXPR_TIMESTAMP VARCHAR), =(SAFE_CAST($2), 0), =(SAFE_CAST($3), 0), <>($1, '')), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},PageViews=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[PageViews, URL], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgImtpbmQiOiAiU0FGRV9DQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["DontCountHits",0]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgImtpbmQiOiAiU0FGRV9DQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["IsRefresh",0]}},"boost":1.0}},{"bool":{"must":[{"exists":{"field":"URL","boost":1.0}}],"must_not":[{"term":{"URL":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"URL":{"terms":{"field":"URL","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q38.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q38.yaml index 5ae72b31e00..b468969d454 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q38.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q38.yaml @@ -5,7 +5,7 @@ calcite: LogicalProject(PageViews=[$1], Title=[$0]) LogicalAggregate(group=[{0}], PageViews=[COUNT()]) LogicalProject(Title=[$97]) - LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2013-07-31 00:00:00':VARCHAR)), =($42, 0), =($72, 0), <>($97, ''))]) + LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00')), <=($0, TIMESTAMP('2013-07-31 00:00:00')), =(SAFE_CAST($42), 0), =(SAFE_CAST($72), 0), <>($97, ''))]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(=($4, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-31 00:00:00':VARCHAR]]:VARCHAR), =($1, 0), =($2, 0), <>($3, '')), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={3},PageViews=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[PageViews, Title], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"DontCountHits":{"value":0,"boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}},{"bool":{"must":[{"exists":{"field":"Title","boost":1.0}}],"must_not":[{"term":{"Title":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"Title":{"terms":{"field":"Title","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[SCRIPT->AND(=($4, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':EXPR_TIMESTAMP VARCHAR..'2013-07-31 00:00:00':EXPR_TIMESTAMP VARCHAR]]:EXPR_TIMESTAMP VARCHAR), =(SAFE_CAST($1), 0), =(SAFE_CAST($2), 0), <>($3, '')), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={3},PageViews=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[PageViews, Title], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgImtpbmQiOiAiU0FGRV9DQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["DontCountHits",0]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgImtpbmQiOiAiU0FGRV9DQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["IsRefresh",0]}},"boost":1.0}},{"bool":{"must":[{"exists":{"field":"Title","boost":1.0}}],"must_not":[{"term":{"Title":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"Title":{"terms":{"field":"Title","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q39.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q39.yaml index 6f08007b702..ee737b7b79b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q39.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q39.yaml @@ -5,9 +5,9 @@ calcite: LogicalProject(PageViews=[$1], URL=[$0]) LogicalAggregate(group=[{0}], PageViews=[COUNT()]) LogicalProject(URL=[$26]) - LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2013-07-31 00:00:00':VARCHAR)), =($72, 0), <>($49, 0), =($35, 0), IS NOT NULL($26))]) + LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00')), <=($0, TIMESTAMP('2013-07-31 00:00:00')), =(SAFE_CAST($72), 0), <>(SAFE_CAST($49), 0), =(SAFE_CAST($35), 0), IS NOT NULL($26))]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableLimit(offset=[1000], fetch=[10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(=($5, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-31 00:00:00':VARCHAR]]:VARCHAR), =($4, 0), <>($3, 0), =($2, 0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},PageViews=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[PageViews, URL], LIMIT->[10 from 1000]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}},{"bool":{"must":[{"exists":{"field":"IsLink","boost":1.0}}],"must_not":[{"term":{"IsLink":{"value":0,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"term":{"IsDownload":{"value":0,"boost":1.0}}},{"exists":{"field":"URL","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"URL":{"terms":{"field":"URL","size":1010,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[SCRIPT->AND(=($5, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':EXPR_TIMESTAMP VARCHAR..'2013-07-31 00:00:00':EXPR_TIMESTAMP VARCHAR]]:EXPR_TIMESTAMP VARCHAR), =(SAFE_CAST($4), 0), <>(SAFE_CAST($3), 0), =(SAFE_CAST($2), 0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},PageViews=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[PageViews, URL], LIMIT->[10 from 1000]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgImtpbmQiOiAiU0FGRV9DQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["IsRefresh",0]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCSnsKICAib3AiOiB7CiAgICAibmFtZSI6ICI8PiIsCiAgICAia2luZCI6ICJOT1RfRVFVQUxTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlNNQUxMSU5UIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["IsLink",0]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgImtpbmQiOiAiU0FGRV9DQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["IsDownload",0]}},"boost":1.0}},{"exists":{"field":"URL","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"URL":{"terms":{"field":"URL","size":1010,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q40.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q40.yaml index e9eefc046b2..147ce3361f9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q40.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q40.yaml @@ -6,10 +6,10 @@ calcite: LogicalAggregate(group=[{0, 1, 2, 3, 4}], PageViews=[COUNT()]) LogicalProject(TraficSourceID=[$12], SearchEngineID=[$65], AdvEngineID=[$19], Src=[$111], Dst=[$112]) LogicalFilter(condition=[AND(IS NOT NULL($12), IS NOT NULL($65), IS NOT NULL($19), IS NOT NULL($111), IS NOT NULL($112))]) - LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104], _id=[$105], _index=[$106], _score=[$107], _maxscore=[$108], _sort=[$109], _routing=[$110], Src=[CASE(AND(=($65, 0), =($19, 0)), $95, '':VARCHAR)], Dst=[$26]) - LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2013-07-31 00:00:00':VARCHAR)), =($72, 0))]) + LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104], _id=[$105], _index=[$106], _score=[$107], _maxscore=[$108], _sort=[$109], _routing=[$110], Src=[CASE(AND(=(SAFE_CAST($65), 0), =(SAFE_CAST($19), 0)), $95, '':VARCHAR)], Dst=[$26]) + LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00')), <=($0, TIMESTAMP('2013-07-31 00:00:00')), =(SAFE_CAST($72), 0))]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableLimit(offset=[1000], fetch=[10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(=($7, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-31 00:00:00':VARCHAR]]:VARCHAR), =($5, 0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2, 3, 4},PageViews=COUNT()), SORT_AGG_METRICS->[5 DESC LAST], PROJECT->[PageViews, TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst], LIMIT->[10 from 1000]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"TraficSourceID|AdvEngineID|SearchEngineID|Src|Dst":{"multi_terms":{"terms":[{"field":"TraficSourceID"},{"field":"AdvEngineID"},{"field":"SearchEngineID"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQGAXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiQU5EIiwKICAgICAgICAia2luZCI6ICJBTkQiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgIm9wIjogewogICAgICAgICAgICAibmFtZSI6ICI9IiwKICAgICAgICAgICAgImtpbmQiOiAiRVFVQUxTIiwKICAgICAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgICAgICB9LAogICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgICAgIH0KICAgICAgICAgICAgfSwKICAgICAgICAgICAgewogICAgICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiPSIsCiAgICAgICAgICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAgICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICAgICAgfSwKICAgICAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICAgICAgewogICAgICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMywKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9CiAgICAgICAgICBdCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogNCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDUsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,0,2,0,2],"DIGESTS":["SearchEngineID",0,"AdvEngineID",0,"Referer",""]}}},{"field":"URL"}],"size":1010,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[SCRIPT->AND(=($7, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':EXPR_TIMESTAMP VARCHAR..'2013-07-31 00:00:00':EXPR_TIMESTAMP VARCHAR]]:EXPR_TIMESTAMP VARCHAR), =(SAFE_CAST($5), 0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2, 3, 4},PageViews=COUNT()), SORT_AGG_METRICS->[5 DESC LAST], PROJECT->[PageViews, TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst], LIMIT->[10 from 1000]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgImtpbmQiOiAiU0FGRV9DQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["IsRefresh",0]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"TraficSourceID|AdvEngineID|SearchEngineID|Src|Dst":{"multi_terms":{"terms":[{"field":"TraficSourceID"},{"field":"AdvEngineID"},{"field":"SearchEngineID"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQI0XsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiQU5EIiwKICAgICAgICAia2luZCI6ICJBTkQiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgIm9wIjogewogICAgICAgICAgICAibmFtZSI6ICI9IiwKICAgICAgICAgICAgImtpbmQiOiAiRVFVQUxTIiwKICAgICAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgICAgICB9LAogICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgIm9wIjogewogICAgICAgICAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAgICAgICAgICJraW5kIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICAgICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgXSwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgICAgIH0KICAgICAgICAgICAgfSwKICAgICAgICAgICAgewogICAgICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiPSIsCiAgICAgICAgICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAgICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICAgICAgfSwKICAgICAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICAgICAgewogICAgICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgICAgICJuYW1lIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICAgICAgICAgInN5bnRheCI6ICJTUEVDSUFMIgogICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAgICAgInR5cGUiOiAiU01BTExJTlQiLAogICAgICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMywKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9CiAgICAgICAgICBdCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogNCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDUsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,0,2,0,2],"DIGESTS":["SearchEngineID",0,"AdvEngineID",0,"Referer",""]}}},{"field":"URL"}],"size":1010,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q41.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q41.yaml index dc908d4df76..9378c9fbd4f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q41.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q41.yaml @@ -5,9 +5,9 @@ calcite: LogicalProject(PageViews=[$2], URLHash=[$0], EventDate=[$1]) LogicalAggregate(group=[{0, 1}], PageViews=[COUNT()]) LogicalProject(URLHash=[$25], EventDate=[$0]) - LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2013-07-31 00:00:00':VARCHAR)), =($72, 0), SEARCH($12, Sarg[-1, 6]), =($11, 3594120000172545465), IS NOT NULL($25))]) + LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00')), <=($0, TIMESTAMP('2013-07-31 00:00:00')), =(SAFE_CAST($72), 0), OR(=(SAFE_CAST($12), -1), =(SAFE_CAST($12), 6)), =($11, 3594120000172545465), IS NOT NULL($25))]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableLimit(offset=[100], fetch=[10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(=($5, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-31 00:00:00':VARCHAR]]:VARCHAR), =($4, 0), SEARCH($2, Sarg[-1, 6]), =($1, 3594120000172545465), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 3},PageViews=COUNT()), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[PageViews, URLHash, EventDate], LIMIT->[10 from 100]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}},{"terms":{"TraficSourceID":[-1.0,6.0],"boost":1.0}},{"term":{"RefererHash":{"value":3594120000172545465,"boost":1.0}}},{"exists":{"field":"URLHash","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"EventDate|URLHash":{"multi_terms":{"terms":[{"field":"EventDate","value_type":"long"},{"field":"URLHash"}],"size":110,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[SCRIPT->AND(=($5, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':EXPR_TIMESTAMP VARCHAR..'2013-07-31 00:00:00':EXPR_TIMESTAMP VARCHAR]]:EXPR_TIMESTAMP VARCHAR), =(SAFE_CAST($4), 0), SEARCH(SAFE_CAST($2), Sarg[-1, 6]), =($1, 3594120000172545465), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 3},PageViews=COUNT()), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[PageViews, URLHash, EventDate], LIMIT->[10 from 100]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgImtpbmQiOiAiU0FGRV9DQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["IsRefresh",0]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQGDnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJPUiIsCiAgICAia2luZCI6ICJPUiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIj0iLAogICAgICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAib3AiOiB7CiAgICAgICAgICAgICJuYW1lIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgICAgICJraW5kIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIlNNQUxMSU5UIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICI9IiwKICAgICAgICAia2luZCI6ICJFUVVBTFMiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgIm9wIjogewogICAgICAgICAgICAibmFtZSI6ICJTQUZFX0NBU1QiLAogICAgICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgICAgICB9LAogICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9CiAgICAgICAgICBdLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMywKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,0,2],"DIGESTS":["TraficSourceID",6,"TraficSourceID",-1]}},"boost":1.0}},{"term":{"RefererHash":{"value":3594120000172545465,"boost":1.0}}},{"exists":{"field":"URLHash","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"EventDate|URLHash":{"multi_terms":{"terms":[{"field":"EventDate","value_type":"long"},{"field":"URLHash"}],"size":110,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q41_alternative.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q41_alternative.yaml new file mode 100644 index 00000000000..b6c30ab64c9 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q41_alternative.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], offset=[100], fetch=[10]) + LogicalProject(PageViews=[$2], URLHash=[$0], EventDate=[$1]) + LogicalAggregate(group=[{0, 1}], PageViews=[COUNT()]) + LogicalProject(URLHash=[$25], EventDate=[$0]) + LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00')), <=($0, TIMESTAMP('2013-07-31 00:00:00')), =(SAFE_CAST($72), 0), OR(=(SAFE_CAST($12), -1), =(SAFE_CAST($12), 6)), =($11, 3594120000172545465), IS NOT NULL($25))]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(offset=[100], fetch=[10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[SCRIPT->AND(=($5, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':EXPR_TIMESTAMP VARCHAR..'2013-07-31 00:00:00':EXPR_TIMESTAMP VARCHAR]]:EXPR_TIMESTAMP VARCHAR), =(SAFE_CAST($4), 0), SEARCH(SAFE_CAST($2), Sarg[-1, 6]), =($1, 3594120000172545465), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 3},PageViews=COUNT()), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[PageViews, URLHash, EventDate], LIMIT->[10 from 100]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgImtpbmQiOiAiU0FGRV9DQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["IsRefresh",0]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQGDnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJPUiIsCiAgICAia2luZCI6ICJPUiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIj0iLAogICAgICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAib3AiOiB7CiAgICAgICAgICAgICJuYW1lIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgICAgICJraW5kIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIlNNQUxMSU5UIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICI9IiwKICAgICAgICAia2luZCI6ICJFUVVBTFMiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgIm9wIjogewogICAgICAgICAgICAibmFtZSI6ICJTQUZFX0NBU1QiLAogICAgICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgICAgICB9LAogICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9CiAgICAgICAgICBdLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMywKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,0,2],"DIGESTS":["TraficSourceID",-1,"TraficSourceID",6]}},"boost":1.0}},{"term":{"RefererHash":{"value":3594120000172545465,"boost":1.0}}},{"exists":{"field":"URLHash","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"EventDate|URLHash":{"multi_terms":{"terms":[{"field":"EventDate","value_type":"long"},{"field":"URLHash"}],"size":110,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q42.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q42.yaml index 3b0bf781c00..9a2dee39390 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q42.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q42.yaml @@ -5,9 +5,9 @@ calcite: LogicalProject(PageViews=[$2], WindowClientWidth=[$0], WindowClientHeight=[$1]) LogicalAggregate(group=[{0, 1}], PageViews=[COUNT()]) LogicalProject(WindowClientWidth=[$104], WindowClientHeight=[$57]) - LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2013-07-31 00:00:00':VARCHAR)), =($72, 0), =($42, 0), =($25, 2868770270353813622), IS NOT NULL($104), IS NOT NULL($57))]) + LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00')), <=($0, TIMESTAMP('2013-07-31 00:00:00')), =(SAFE_CAST($72), 0), =(SAFE_CAST($42), 0), =($25, 2868770270353813622), IS NOT NULL($104), IS NOT NULL($57))]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableLimit(offset=[10000], fetch=[10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(=($5, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-31 00:00:00':VARCHAR]]:VARCHAR), =($4, 0), =($2, 0), =($1, 2868770270353813622), IS NOT NULL($6), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={3, 6},PageViews=COUNT()), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[PageViews, WindowClientWidth, WindowClientHeight]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}},{"term":{"DontCountHits":{"value":0,"boost":1.0}}},{"term":{"URLHash":{"value":2868770270353813622,"boost":1.0}}},{"exists":{"field":"WindowClientWidth","boost":1.0}},{"exists":{"field":"WindowClientHeight","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"WindowClientHeight|WindowClientWidth":{"multi_terms":{"terms":[{"field":"WindowClientHeight"},{"field":"WindowClientWidth"}],"size":10000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[SCRIPT->AND(=($5, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':EXPR_TIMESTAMP VARCHAR..'2013-07-31 00:00:00':EXPR_TIMESTAMP VARCHAR]]:EXPR_TIMESTAMP VARCHAR), =(SAFE_CAST($4), 0), =(SAFE_CAST($2), 0), =($1, 2868770270353813622), IS NOT NULL($6), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={3, 6},PageViews=COUNT()), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[PageViews, WindowClientWidth, WindowClientHeight]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgImtpbmQiOiAiU0FGRV9DQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["IsRefresh",0]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgImtpbmQiOiAiU0FGRV9DQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["DontCountHits",0]}},"boost":1.0}},{"term":{"URLHash":{"value":2868770270353813622,"boost":1.0}}},{"exists":{"field":"WindowClientWidth","boost":1.0}},{"exists":{"field":"WindowClientHeight","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"WindowClientHeight|WindowClientWidth":{"multi_terms":{"terms":[{"field":"WindowClientHeight"},{"field":"WindowClientWidth"}],"size":10000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q43.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q43.yaml index 5b2c5816127..68fe91fe259 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q43.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q43.yaml @@ -5,9 +5,9 @@ calcite: LogicalProject(PageViews=[$1], M=[$0]) LogicalAggregate(group=[{0}], PageViews=[COUNT()]) LogicalProject(M=[SPAN($17, 1, 'm')]) - LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2013-07-15 00:00:00':VARCHAR)), =($72, 0), =($42, 0), IS NOT NULL($17))]) + LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00')), <=($0, TIMESTAMP('2013-07-15 00:00:00')), =(SAFE_CAST($72), 0), =(SAFE_CAST($42), 0), IS NOT NULL($17))]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableLimit(offset=[1000], fetch=[10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(=($4, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-15 00:00:00':VARCHAR]]:VARCHAR), =($3, 0), =($2, 0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},PageViews=COUNT()), PROJECT->[PageViews, M], SORT->[1 ASC FIRST], LIMIT->[10 from 1000]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-15T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}},{"term":{"DontCountHits":{"value":0,"boost":1.0}}},{"exists":{"field":"EventTime","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1010,"sources":[{"M":{"date_histogram":{"field":"EventTime","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=1010, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[SCRIPT->AND(=($4, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':EXPR_TIMESTAMP VARCHAR..'2013-07-15 00:00:00':EXPR_TIMESTAMP VARCHAR]]:EXPR_TIMESTAMP VARCHAR), =(SAFE_CAST($3), 0), =(SAFE_CAST($2), 0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},PageViews=COUNT()), PROJECT->[PageViews, M], SORT->[1 ASC FIRST], LIMIT->[10 from 1000]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-15T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgImtpbmQiOiAiU0FGRV9DQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["IsRefresh",0]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgImtpbmQiOiAiU0FGRV9DQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["DontCountHits",0]}},"boost":1.0}},{"exists":{"field":"EventTime","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1010,"sources":[{"M":{"date_histogram":{"field":"EventTime","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=1010, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q8.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q8.yaml index e4ba0272b81..3d576f8a051 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q8.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q8.yaml @@ -5,7 +5,7 @@ calcite: LogicalProject(count()=[$1], AdvEngineID=[$0]) LogicalAggregate(group=[{0}], count()=[COUNT()]) LogicalProject(AdvEngineID=[$19]) - LogicalFilter(condition=[<>($19, 0)]) + LogicalFilter(condition=[<>(SAFE_CAST($19), 0)]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER-><>($0, 0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[count(), AdvEngineID], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"AdvEngineID","boost":1.0}}],"must_not":[{"term":{"AdvEngineID":{"value":0,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"AdvEngineID":{"terms":{"field":"AdvEngineID","size":10000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[SCRIPT-><>(SAFE_CAST($0), 0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[count(), AdvEngineID], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCSnsKICAib3AiOiB7CiAgICAibmFtZSI6ICI8PiIsCiAgICAia2luZCI6ICJOT1RfRVFVQUxTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlNNQUxMSU5UIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["AdvEngineID",0]}},"boost":1.0}},"aggregations":{"AdvEngineID":{"terms":{"field":"AdvEngineID","size":10000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain.yaml new file mode 100644 index 00000000000..e487a622561 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain.yaml @@ -0,0 +1,20 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age2=[$2]) + LogicalFilter(condition=[<=($3, SAFE_CAST(1:BIGINT))]) + LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2)]) + LogicalFilter(condition=[IS NOT NULL($2)]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) + LogicalProject(avg_age=[$2], state=[$0], age2=[+($2, 2)]) + LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) + LogicalProject(state=[$7], city=[$5], age=[$8]) + LogicalFilter(condition=[>($8, SAFE_CAST(30:BIGINT))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], age2=[$t0]) + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1:BIGINT], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3]) + EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], age2=[$t2], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), SORT->[1 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_add_col_totals.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_add_col_totals.yaml index 0a8139b1eaa..f27fc2cc118 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_add_col_totals.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_add_col_totals.yaml @@ -4,15 +4,18 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) LogicalUnion(all=[true]) LogicalSort(fetch=[5]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalProject(account_number=[null:BIGINT], firstname=[null:VARCHAR], address=[null:VARCHAR], balance=[$0], gender=[null:VARCHAR], city=[null:VARCHAR], employer=[null:VARCHAR], state=[null:VARCHAR], age=[$1], email=[null:VARCHAR], lastname=[null:VARCHAR], _id=[null:VARCHAR], _index=[null:VARCHAR], _score=[null:REAL], _maxscore=[null:REAL], _sort=[null:BIGINT], _routing=[null:VARCHAR]) - LogicalAggregate(group=[{}], balance=[SUM($3)], age=[SUM($8)]) - LogicalSort(fetch=[5]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[null:BIGINT], firstname=[null:VARCHAR], address=[null:VARCHAR], balance=[$0], gender=[null:VARCHAR], city=[null:VARCHAR], employer=[null:VARCHAR], state=[null:VARCHAR], age=[$1], email=[null:VARCHAR], lastname=[null:VARCHAR], _id=[null:VARCHAR], _index=[null:VARCHAR], _score=[SAFE_CAST(null:REAL)], _maxscore=[SAFE_CAST(null:REAL)], _sort=[null:BIGINT], _routing=[null:VARCHAR]) + LogicalAggregate(group=[{}], balance=[SUM($0)], age=[SUM($1)]) + LogicalProject(balance=[$3], age=[$8]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableUnion(all=[true]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) EnumerableCalc(expr#0..1=[{inputs}], expr#2=[null:BIGINT], expr#3=[null:VARCHAR], account_number=[$t2], firstname=[$t3], address=[$t3], balance=[$t0], gender=[$t3], city=[$t3], employer=[$t3], state=[$t3], age=[$t1], email=[$t3], lastname=[$t3]) EnumerableAggregate(group=[{}], balance=[SUM($0)], age=[SUM($1)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[balance, age], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["balance","age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[balance, age], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["balance","age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_add_totals.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_add_totals.yaml index 0c8b4ec26a2..76fccaaa6e3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_add_totals.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_add_totals.yaml @@ -3,20 +3,19 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], CustomSum=[$17], all_emp_total=[$18]) LogicalUnion(all=[true]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], CustomSum=[+($3, $8)], all_emp_total=[null:VARCHAR(13)]) - LogicalSort(fetch=[5]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], CustomSum=[+($3, $8)], all_emp_total=[null:VARCHAR]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalProject(account_number=[null:BIGINT], firstname=[null:VARCHAR], address=[null:VARCHAR], balance=[$0], gender=[null:VARCHAR], city=[null:VARCHAR], employer=[null:VARCHAR], state=[null:VARCHAR], age=[$1], email=[null:VARCHAR], lastname=[null:VARCHAR], _id=[null:VARCHAR], _index=[null:VARCHAR], _score=[null:REAL], _maxscore=[null:REAL], _sort=[null:BIGINT], _routing=[null:VARCHAR], CustomSum=[null:BIGINT], all_emp_total=['ColTotal':VARCHAR(13)]) + LogicalProject(account_number=[null:BIGINT], firstname=[null:VARCHAR], address=[null:VARCHAR], balance=[$0], gender=[null:VARCHAR], city=[null:VARCHAR], employer=[null:VARCHAR], state=[null:VARCHAR], age=[$1], email=[null:VARCHAR], lastname=[null:VARCHAR], _id=[null:VARCHAR], _index=[null:VARCHAR], _score=[SAFE_CAST(null:REAL)], _maxscore=[SAFE_CAST(null:REAL)], _sort=[null:BIGINT], _routing=[null:VARCHAR], CustomSum=[null:BIGINT], all_emp_total=['ColTotal']) LogicalAggregate(group=[{}], balance=[SUM($0)], age=[SUM($1)]) - LogicalProject(balance=[$3], age=[$8]) - LogicalSort(fetch=[5]) + LogicalSort(fetch=[5]) + LogicalProject(balance=[$3], age=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableUnion(all=[true]) - EnumerableCalc(expr#0..10=[{inputs}], expr#11=[+($t3, $t8)], expr#12=[null:VARCHAR(13)], proj#0..12=[{exprs}]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) - EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[null:BIGINT], expr#3=[null:VARCHAR], expr#4=['ColTotal':VARCHAR(13)], account_number=[$t2], firstname=[$t3], address=[$t3], balance=[$t0], gender=[$t3], city=[$t3], employer=[$t3], state=[$t3], age=[$t1], email=[$t3], lastname=[$t3], CustomSum=[$t2], all_emp_total=[$t4]) - EnumerableAggregate(group=[{}], balance=[SUM($0)], age=[SUM($1)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[balance, age], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["balance","age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..10=[{inputs}], expr#11=[+($t3, $t8)], expr#12=[null:VARCHAR], proj#0..12=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[null:BIGINT], expr#3=[null:VARCHAR], expr#4=['ColTotal'], account_number=[$t2], firstname=[$t3], address=[$t3], balance=[$t0], gender=[$t3], city=[$t3], employer=[$t3], state=[$t3], age=[$t1], email=[$t3], lastname=[$t3], CustomSum=[$t2], all_emp_total=[$t4]) + EnumerableAggregate(group=[{}], balance=[SUM($0)], age=[SUM($1)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[balance, age], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["balance","age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml index e731c2d3433..9669fcaa594 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(count()=[$1], c1=[$1], gender=[$0]) - LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalAggregate(group=[{0}], c1=[COUNT()]) LogicalProject(gender=[$4]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], c1=[$t1], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c1=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml index 026d83de0e8..67c9f44ebf7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(c1=[$1], c2=[$1], gender=[$0]) - LogicalAggregate(group=[{0}], c1=[COUNT($1)]) + LogicalAggregate(group=[{0}], c2=[COUNT($1)]) LogicalProject(gender=[$4], balance=[$3]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], c1=[$t1], c2=[$t1], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c1=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"c1":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c2=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"c2":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by3.yaml index 4e49ba34da0..0541c3a40f0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by3.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by3.yaml @@ -1,9 +1,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(count(account_number)=[$1], c2=[$2], gender=[$0]) - LogicalAggregate(group=[{0}], count(account_number)=[COUNT($1)], c2=[COUNT($2)]) - LogicalProject(gender=[$4], account_number=[$0], account_number_alias=[$0]) + LogicalProject(count(account_number)=[$1], c2=[$1], gender=[$0]) + LogicalAggregate(group=[{0}], c2=[COUNT($1)]) + LogicalProject(gender=[$4], account_number=[$0]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(account_number)=COUNT($1),c2=COUNT($2)), PROJECT->[count(account_number), c2, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(account_number)":{"value_count":{"field":"account_number"}},"c2":{"value_count":{"field":"account_number"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..1=[{inputs}], count(account_number)=[$t1], c2=[$t1], gender=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c2=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"c2":{"value_count":{"field":"account_number"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by5.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by5.yaml index 40e1333630a..d7bb4d3dfd0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by5.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by5.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(gender=[$4], balance=[$3], account_number=[$0]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(balance)=COUNT($1),count(account_number)=COUNT($2)), PROJECT->[count(balance), count(account_number), gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(balance)":{"value_count":{"field":"balance"}},"count(account_number)":{"value_count":{"field":"account_number"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(balance)=COUNT($1),count(account_number)=COUNT($2)), PROJECT->[count(balance), count(account_number), gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(balance)":{"value_count":{"field":"balance"}},"count(account_number)":{"value_count":{"field":"account_number"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml index f349523ec56..503eeedd753 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml @@ -3,7 +3,7 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(count(b_1)=[$1], c3=[$2], gender=[$0]) LogicalAggregate(group=[{0}], count(b_1)=[COUNT($1)], c3=[COUNT($2)]) - LogicalProject(gender=[$4], b_1=[+($3, 1)], $f3=[POWER($3, 2)]) + LogicalProject(gender=[$4], $f1=[+($3, 1)], $f2=[POWER($3, 2)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(b_1)=COUNT($1),c3=COUNT($2)), PROJECT->[count(b_1), c3, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(b_1)":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}},"c3":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBVHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJQT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",2]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having1.yaml index bc06e988e1f..7afae759d8e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having1.yaml @@ -1,12 +1,13 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalFilter(condition=[>($0, 10)]) - LogicalProject(c=[$1], state=[$0]) - LogicalAggregate(group=[{0}], c=[COUNT()]) - LogicalProject(state=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(c=[$0], state=[$1]) + LogicalFilter(condition=[>($0, SAFE_CAST(10:BIGINT))]) + LogicalProject(c=[$1], state=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()]) + LogicalProject(state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10], expr#3=[>($t0, $t2)], proj#0..1=[{exprs}], $condition=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10:BIGINT], expr#3=[>($t0, $t2)], proj#0..1=[{exprs}], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having2.yaml index 8f1a667a604..1e8d27d50d6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having2.yaml @@ -1,13 +1,14 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalFilter(condition=[>($0, 10)]) - LogicalProject(count()=[$1], state=[$0]) - LogicalAggregate(group=[{0}], count()=[COUNT()]) - LogicalProject(state=[$7]) - LogicalFilter(condition=[IS NOT NULL($7)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(count()=[$0], state=[$1]) + LogicalFilter(condition=[>($0, SAFE_CAST(10:BIGINT))]) + LogicalProject(count()=[$1], state=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(state=[$7]) + LogicalFilter(condition=[IS NOT NULL($7)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10], expr#3=[>($t0, $t2)], proj#0..1=[{exprs}], $condition=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10:BIGINT], expr#3=[>($t0, $t2)], proj#0..1=[{exprs}], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml index e7589d8109d..c93c937da63 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml @@ -1,12 +1,13 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalFilter(condition=[OR(>($3, 1000), >($4, 1))]) - LogicalProject(avg=[$1], cnt=[$2], state=[$0], new_avg=[+($1, 1000)], new_cnt=[+($2, 1)]) - LogicalAggregate(group=[{0}], avg=[AVG($1)], cnt=[COUNT()]) - LogicalProject(state=[$7], balance=[$3]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(avg=[$0], cnt=[$1], state=[$2], new_avg=[$3], new_cnt=[$4]) + LogicalFilter(condition=[OR(>($3, SAFE_CAST(1000)), >($4, SAFE_CAST(1:BIGINT)))]) + LogicalProject(avg=[$1], cnt=[$2], state=[$0], new_avg=[+($1, 1000)], new_cnt=[+($2, 1)]) + LogicalAggregate(group=[{0}], avg=[AVG($1)], cnt=[COUNT()]) + LogicalProject(state=[$7], balance=[$3]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1000], expr#4=[+($t1, $t3)], expr#5=[1], expr#6=[+($t2, $t5)], expr#7=[>($t4, $t3)], expr#8=[>($t6, $t5)], expr#9=[OR($t7, $t8)], avg=[$t1], cnt=[$t2], state=[$t0], new_avg=[$t4], new_cnt=[$t6], $condition=[$t9]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg=AVG($1),cnt=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1000], expr#4=[+($t1, $t3)], expr#5=[1], expr#6=[+($t2, $t5)], expr#7=[1000.0E0:DOUBLE], expr#8=[>($t4, $t7)], expr#9=[1:BIGINT], expr#10=[>($t6, $t9)], expr#11=[OR($t8, $t10)], avg=[$t1], cnt=[$t2], state=[$t0], new_avg=[$t4], new_cnt=[$t6], $condition=[$t11]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg=AVG($1),cnt=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join1.yaml index ea76cdee61e..53c26284f11 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join1.yaml @@ -7,7 +7,7 @@ calcite: LogicalAggregate(group=[{0}], c=[COUNT()]) LogicalProject(state=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[50000]) LogicalProject(c=[$1], state=[$0]) LogicalAggregate(group=[{0}], c=[COUNT()]) LogicalProject(state=[$9]) @@ -16,4 +16,4 @@ calcite: EnumerableLimit(fetch=[10000]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[inner]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state], LIMIT->50000, SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}}]}}}}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state], LIMIT->50000, SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}}]}}}}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join2.yaml index ca7ab3fc668..d1c9274704d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join2.yaml @@ -8,7 +8,7 @@ calcite: LogicalProject(state=[$7]) LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[50000]) LogicalProject(c=[$1], state=[$0]) LogicalAggregate(group=[{0}], c=[COUNT()]) LogicalProject(state=[$9]) @@ -18,4 +18,4 @@ calcite: EnumerableLimit(fetch=[10000]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[inner]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state], LIMIT->50000, SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state], LIMIT->50000, SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join3.yaml index 1326030ea7e..95c3b952489 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join3.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join3.yaml @@ -7,7 +7,7 @@ calcite: LogicalAggregate(group=[{0}], c=[COUNT()]) LogicalProject(state=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[50000]) LogicalProject(c=[$1], state=[$0]) LogicalAggregate(group=[{0}], c=[COUNT()]) LogicalProject(state=[$9]) @@ -16,4 +16,4 @@ calcite: EnumerableLimit(fetch=[10000]) EnumerableHashJoin(condition=[=($1, $2)], joinType=[semi]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state], LIMIT->50000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}}]}}}}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0}), SORT->[0]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0}), SORT->[0]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join4.yaml index 36bf1245a2d..83eb7a68f9a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join4.yaml @@ -8,7 +8,7 @@ calcite: LogicalAggregate(group=[{0}], c=[COUNT()]) LogicalProject(state=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[50000]) LogicalProject(c=[$1], state=[$0]) LogicalAggregate(group=[{0}], c=[COUNT()]) LogicalProject(state=[$7]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure1.yaml index 75389120405..4c23776ebc1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure1.yaml @@ -8,4 +8,4 @@ calcite: LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT_AGG_METRICS->[1 ASC FIRST], PROJECT->[count(), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"asc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT_AGG_METRICS->[1 ASC FIRST], PROJECT->[count(), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"asc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure3.yaml index e60bbe90fdc..610af67120c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure3.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure3.yaml @@ -9,4 +9,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},cnt=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[cnt, span(birthdate,1d)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"span(birthdate,1d)":{"date_histogram":{"field":"birthdate","fixed_interval":"1d","offset":0,"order":[{"_count":"desc"},{"_key":"asc"}],"keyed":false,"min_doc_count":0}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},cnt=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[cnt, span(birthdate,1d)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"span(birthdate,1d)":{"date_histogram":{"field":"birthdate","fixed_interval":"1d","offset":0,"order":[{"_count":"desc"},{"_key":"asc"}],"keyed":false,"min_doc_count":0}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml index 57132615c41..d3ab0d251ff 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last]) LogicalProject(sum(balance)=[$1], span(age,5)=[$0]) - LogicalAggregate(group=[{1}], sum(balance)=[SUM($0)]) - LogicalProject(balance=[$7], span(age,5)=[SPAN($10, 5, null:NULL)]) + LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)]) + LogicalProject(span(age,5)=[SPAN($10, 5, null:ANY)], balance=[$7]) LogicalFilter(condition=[IS NOT NULL($10)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(balance)=SUM($0)), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[sum(balance), span(age,5)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"span(age,5)":{"histogram":{"field":"age","interval":5.0,"offset":0.0,"order":[{"sum(balance)":"desc"},{"_key":"asc"}],"keyed":false,"min_doc_count":0},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1)), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[sum(balance), span(age,5)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"span(age,5)":{"histogram":{"field":"age","interval":5.0,"offset":0.0,"order":[{"sum(balance)":"desc"},{"_key":"asc"}],"keyed":false,"min_doc_count":0},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml index 68cb12a49dd..fe57833ef75 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[ASC-nulls-first]) LogicalProject(c=[$2], s=[$3], span(age,5)=[$1], state=[$0]) - LogicalAggregate(group=[{0, 2}], c=[COUNT()], s=[SUM($1)]) - LogicalProject(state=[$7], balance=[$3], span(age,5)=[SPAN($8, 5, null:NULL)]) + LogicalAggregate(group=[{0, 1}], c=[COUNT()], s=[SUM($2)]) + LogicalProject(state=[$7], span(age,5)=[SPAN($8, 5, null:ANY)], balance=[$3]) LogicalFilter(condition=[AND(IS NOT NULL($8), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),s=SUM($1)), PROJECT->[c, s, span(age,5), state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}},{"span(age,5)":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":5.0}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},c=COUNT(),s=SUM($2)), PROJECT->[c, s, span(age,5), state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}},{"span(age,5)":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":5.0}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_terms.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_terms.yaml index 4caf7759fc6..4b9fe086469 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_terms.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_terms.yaml @@ -8,4 +8,4 @@ calcite: LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), SORT_AGG_METRICS->[2 ASC FIRST], PROJECT->[count(), gender, state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender|state":{"multi_terms":{"terms":[{"field":"gender.keyword"},{"field":"state.keyword"}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"asc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), SORT_AGG_METRICS->[2 ASC FIRST], PROJECT->[count(), gender, state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender|state":{"multi_terms":{"terms":[{"field":"gender.keyword"},{"field":"state.keyword"}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"asc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_terms_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_terms_script.yaml index 13d8350c11f..a159b984054 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_terms_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_terms_script.yaml @@ -9,4 +9,4 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_gender=[LOWER($4)], new_state=[LOWER($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), SORT_AGG_METRICS->[2 ASC FIRST], PROJECT->[count(), new_gender, new_state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"new_gender|new_state":{"multi_terms":{"terms":[{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}}},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}}}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"asc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), SORT_AGG_METRICS->[2 ASC FIRST], PROJECT->[count(), new_gender, new_state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"new_gender|new_state":{"multi_terms":{"terms":[{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}}},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}}}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"asc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_script.yaml index 7e010cba2ad..a5744f9d739 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_script.yaml @@ -9,4 +9,4 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_state=[LOWER($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT_AGG_METRICS->[1 ASC FIRST], PROJECT->[count(), new_state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"asc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT_AGG_METRICS->[1 ASC FIRST], PROJECT->[count(), new_state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"asc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml index c543431c519..92a673991c4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml @@ -3,7 +3,7 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(sum=[$2], len=[$0], gender=[$1]) LogicalAggregate(group=[{0, 1}], sum=[SUM($2)]) - LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f3=[+($7, 100)]) + LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f2=[+($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100], expr#4=[*($t2, $t3)], expr#5=[+($t1, $t4)], expr#6=[CHAR_LENGTH($t0)], sum=[$t5], len=[$t6], gender=[$t0]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml index c17bd10e18a..f54e969e5d4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0]) LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)]) - LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)]) + LogicalProject(gender=[$4], balance=[$7], $f2=[+($7, 100)], $f3=[-($7, 100)], $f4=[*($7, 100)], $f5=[DIVIDE($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCEHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCB3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiRElWSURFIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_appendpipe_command.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_appendpipe_command.json deleted file mode 100644 index 1375ff21c2b..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_appendpipe_command.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], cnt=[$19])\n LogicalUnion(all=[true])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], cnt=[null:BIGINT])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n LogicalProject(account_number=[null:BIGINT], firstname=[null:VARCHAR], address=[null:VARCHAR], birthdate=[null:EXPR_TIMESTAMP VARCHAR], gender=[$0], city=[null:VARCHAR], lastname=[null:VARCHAR], balance=[null:BIGINT], employer=[null:VARCHAR], state=[null:VARCHAR], age=[null:INTEGER], email=[null:VARCHAR], male=[null:BOOLEAN], _id=[null:VARCHAR], _index=[null:VARCHAR], _score=[null:REAL], _maxscore=[null:REAL], _sort=[null:BIGINT], _routing=[null:VARCHAR], cnt=[$1])\n LogicalAggregate(group=[{0}], cnt=[COUNT($1)])\n LogicalProject(gender=[$4], balance=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical":"EnumerableLimit(fetch=[10000])\n EnumerableUnion(all=[true])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[null:BIGINT], proj#0..13=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"birthdate\",\"gender\",\"city\",\"lastname\",\"balance\",\"employer\",\"state\",\"age\",\"email\",\"male\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[null:BIGINT], expr#3=[null:VARCHAR], expr#4=[null:EXPR_TIMESTAMP VARCHAR], expr#5=[null:INTEGER], expr#6=[null:BOOLEAN], account_number=[$t2], firstname=[$t3], address=[$t3], birthdate=[$t4], gender=[$t0], city=[$t3], lastname=[$t3], balance=[$t2], employer=[$t3], state=[$t3], age=[$t5], email=[$t3], male=[$t6], cnt=[$t1])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},cnt=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"cnt\":{\"value_count\":{\"field\":\"balance\"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_appendpipe_command.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_appendpipe_command.yaml new file mode 100644 index 00000000000..95f6cc76fd1 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_appendpipe_command.yaml @@ -0,0 +1,18 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], cnt=[$19]) + LogicalUnion(all=[true]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], cnt=[null:BIGINT]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalProject(account_number=[null:BIGINT], firstname=[null:VARCHAR], address=[null:VARCHAR], birthdate=[null:VARCHAR], gender=[$0], city=[null:VARCHAR], lastname=[null:VARCHAR], balance=[null:BIGINT], employer=[null:VARCHAR], state=[null:VARCHAR], age=[null:INTEGER], email=[null:VARCHAR], male=[null:BOOLEAN], _id=[null:VARCHAR], _index=[null:VARCHAR], _score=[SAFE_CAST(null:REAL)], _maxscore=[SAFE_CAST(null:REAL)], _sort=[null:BIGINT], _routing=[null:VARCHAR], cnt=[$1]) + LogicalAggregate(group=[{0}], cnt=[COUNT($1)]) + LogicalProject(gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableUnion(all=[true]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[null:BIGINT], proj#0..13=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[null:BIGINT], expr#3=[null:VARCHAR], expr#4=[null:INTEGER], expr#5=[null:BOOLEAN], account_number=[$t2], firstname=[$t3], address=[$t3], birthdate=[$t3], gender=[$t0], city=[$t3], lastname=[$t3], balance=[$t2], employer=[$t3], state=[$t3], age=[$t4], email=[$t3], male=[$t5], cnt=[$t1]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},cnt=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"cnt":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_aligntime.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_aligntime.yaml index 9a82afe29ec..8c986b5ccb3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_aligntime.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_aligntime.yaml @@ -1,10 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(category=[$0], value=[$1], timestamp=[$2], @timestamp=[$9]) - LogicalSort(fetch=[5]) - LogicalProject(category=[$1], value=[$2], timestamp=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9], @timestamp=[FROM_UNIXTIME(*(*(FLOOR(DIVIDE(DIVIDE(UNIX_TIMESTAMP($0), 3600), 2)), 2), 3600))]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalSort(fetch=[5]) + LogicalProject(category=[$1], value=[$2], timestamp=[$3], @timestamp=[FROM_UNIXTIME(*(*(FLOOR(DIVIDE(DIVIDE(UNIX_TIMESTAMP($0), 3600), 2)), 2), 3600))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | EnumerableCalc(expr#0..3=[{inputs}], expr#4=[UNIX_TIMESTAMP($t3)], expr#5=[3600], expr#6=[DIVIDE($t4, $t5)], expr#7=[2], expr#8=[DIVIDE($t6, $t7)], expr#9=[FLOOR($t8)], expr#10=[*($t9, $t7)], expr#11=[*($t10, $t5)], expr#12=[FROM_UNIXTIME($t11)], proj#0..2=[{exprs}], $f3=[$t12]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[category, value, timestamp, @timestamp], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["category","value","timestamp","@timestamp"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[category, value, timestamp, @timestamp], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["category","value","timestamp","@timestamp"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_bins.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_bins.json deleted file mode 100644 index ff327963630..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_bins.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[WIDTH_BUCKET($8, 3, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[3], expr#14=[-($t11, $t12)], expr#15=[WIDTH_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_bins.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_bins.yaml new file mode 100644 index 00000000000..1997d0bf50b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_bins.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], age=[WIDTH_BUCKET($8, 3, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[3], expr#14=[-($t11, $t12)], expr#15=[WIDTH_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15]) + EnumerableLimit(fetch=[5]) + EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json deleted file mode 100644 index aaa807ed1db..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.yaml new file mode 100644 index 00000000000..9d94237a610 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15]) + EnumerableLimit(fetch=[5]) + EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_span.yaml index 3a1cee955d3..f9de150ef50 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_span.yaml @@ -1,10 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16]) - LogicalSort(fetch=[5]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[SPAN_BUCKET($8, 10)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], age=[SPAN_BUCKET($8, 10)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..10=[{inputs}], expr#11=[10], expr#12=[SPAN_BUCKET($t10, $t11)], proj#0..9=[{exprs}], $f10=[$t12]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, email, lastname, age], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","email","lastname","age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_start_end.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_start_end.json deleted file mode 100644 index 288a9d728e9..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_start_end.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], gender=[$3], city=[$4], employer=[$5], state=[$6], age=[$7], email=[$8], lastname=[$9], balance=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], balance=[RANGE_BUCKET($3, MIN($3) OVER (), MAX($3) OVER (), 0, 100001)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[0], expr#14=[100001], expr#15=[RANGE_BUCKET($t3, $t11, $t12, $t13, $t14)], proj#0..2=[{exprs}], gender=[$t4], city=[$t5], employer=[$t6], state=[$t7], age=[$t8], email=[$t9], lastname=[$t10], balance=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MIN($3), MAX($3)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_start_end.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_start_end.yaml new file mode 100644 index 00000000000..84da06b6b9c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_start_end.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], balance=[RANGE_BUCKET($3, MIN($3) OVER (), MAX($3) OVER (), 0, 100001)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[0], expr#14=[100001], expr#15=[RANGE_BUCKET($t3, $t11, $t12, $t13, $t14)], proj#0..2=[{exprs}], gender=[$t4], city=[$t5], employer=[$t6], state=[$t7], age=[$t8], email=[$t9], lastname=[$t10], balance=[$t15]) + EnumerableLimit(fetch=[5]) + EnumerableWindow(window#0=[window(aggs [MIN($3), MAX($3)])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multi_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multi_group_key.yaml new file mode 100644 index 00000000000..c8ce52e490c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multi_group_key.yaml @@ -0,0 +1,32 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{0, 1}], max(value)=[MAX($2)]) + LogicalProject(timestamp=[$0], category=[CASE(IS NULL($1), 'NULL':VARCHAR, <=($5, 10), $1, 'OTHER':VARCHAR)], max(value)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(timestamp=[$1], category=[$0], max(value)=[$2]) + LogicalAggregate(group=[{0, 1}], max(value)=[MAX($2)]) + LogicalProject(category=[$1], timestamp=[SPAN($3, 1, 'w')], value=[$2]) + LogicalFilter(condition=[AND(IS NOT NULL($3), IS NOT NULL($2))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalProject(category=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(category=[$0], max(value)=[$2]) + LogicalAggregate(group=[{0, 1}], max(value)=[MAX($2)]) + LogicalProject(category=[$1], $f1=[SPAN($3, 1, 'w')], value=[$2]) + LogicalFilter(condition=[AND(IS NOT NULL($3), IS NOT NULL($2))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableAggregate(group=[{0, 1}], max(value)=[MAX($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL':VARCHAR], expr#7=[10:BIGINT], expr#8=[<=($t4, $t7)], expr#9=['OTHER':VARCHAR], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], timestamp=[$t0], category=[$t10], max(value)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(value)=MAX($2)), PROJECT->[timestamp, category, max(value)], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"value","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"timestamp":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(value)=MAX($2)), PROJECT->[category, max(value)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"value","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"category","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"timestamp":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml new file mode 100644 index 00000000000..c4518124cdb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalAggregate(group=[{0}], max(balance)=[MAX($1)]) + LogicalProject(age=[SPAN($10, 10, null:ANY)], balance=[$7]) + LogicalFilter(condition=[AND(IS NOT NULL($10), IS NOT NULL($7))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},max(balance)=MAX($1)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"age","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":10.0}}}]},"aggregations":{"max(balance)":{"max":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_dedup.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_dedup.yaml new file mode 100644 index 00000000000..2aae73e3bb5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_dedup.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[<=($17, SAFE_CAST(1:BIGINT))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4)]) + LogicalFilter(condition=[IS NOT NULL($4)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["gender","account_number","firstname","address","balance","city","employer","state","age","email","lastname"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj.yaml index b747b30f714..c5d6cdceae1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj.yaml @@ -1,19 +1,20 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[$13], b.account_number=[$14], b.firstname=[$15], b.address=[$16], b.birthdate=[$17], b.gender=[$18], b.city=[$19], b.lastname=[$20], b.balance=[$21], b.employer=[$22], b.state=[$23], b.age=[$24], b.email=[$25], b.male=[$26]) - LogicalJoin(condition=[=($13, $15)], joinType=[inner]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[REX_EXTRACT($6, '(?^[A-Z])', 'initial')]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[$13], b.account_number=[$15], b.firstname=[$16], b.address=[$17], b.birthdate=[$18], b.gender=[$19], b.city=[$20], b.lastname=[$21], b.balance=[$22], b.employer=[$23], b.state=[$24], b.age=[$25], b.email=[$26], b.male=[$27]) + LogicalJoin(condition=[=($14, $16)], joinType=[inner]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[REX_EXTRACT($6, '(?^[A-Z])', 'initial')], $f14=[CAST(REX_EXTRACT($6, '(?^[A-Z])', 'initial')):VARCHAR]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[50000]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableLimit(fetch=[10000]) - EnumerableMergeJoin(condition=[=($13, $15)], joinType=[inner]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=['(?^[A-Z])'], expr#14=['initial'], expr#15=[REX_EXTRACT($t6, $t13, $t14)], proj#0..12=[{exprs}], $f13=[$t15]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[REX_EXTRACT($6, '(?^[A-Z])', 'initial') ASCENDING NULLS_LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQC63sKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRVhfRVhUUkFDVCIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAicHJlY2lzaW9uIjogMjAwMAogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":true,"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["lastname","(?^[A-Z])","initial"]}},"type":"string","order":"asc"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], LIMIT->50000, SORT->[{ + EnumerableCalc(expr#0..27=[{inputs}], proj#0..13=[{exprs}], b.account_number=[$t15], b.firstname=[$t16], b.address=[$t17], b.birthdate=[$t18], b.gender=[$t19], b.city=[$t20], b.lastname=[$t21], b.balance=[$t22], b.employer=[$t23], b.state=[$t24], b.age=[$t25], b.email=[$t26], b.male=[$t27]) + EnumerableLimit(fetch=[10000]) + EnumerableMergeJoin(condition=[=($14, $16)], joinType=[inner]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=['(?^[A-Z])'], expr#14=['initial'], expr#15=[REX_EXTRACT($t6, $t13, $t14)], expr#16=[CAST($t15):VARCHAR], proj#0..12=[{exprs}], $f13=[$t15], $f14=[$t16]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[CAST(REX_EXTRACT($6, '(?^[A-Z])', 'initial')):VARCHAR ASCENDING NULLS_LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQESHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVYX0VYVFJBQ1QiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAyMDAwCiAgICAgIH0sCiAgICAgICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAgICAgImR5bmFtaWMiOiBmYWxzZQogICAgfQogIF0sCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAicHJlY2lzaW9uIjogLTEKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":true,"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["lastname","(?^[A-Z])","initial"]}},"type":"string","order":"asc"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], LIMIT->50000, SORT->[{ "firstname" : { "order" : "asc", "missing" : "_last" diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj_max_option.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj_max_option.yaml new file mode 100644 index 00000000000..d76548486ce --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj_max_option.yaml @@ -0,0 +1,21 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$14], firstname=[$15], address=[$16], birthdate=[$17], gender=[$18], city=[$19], lastname=[$20], balance=[$21], employer=[$22], state=[$23], age=[$24], email=[$25], male=[$26]) + LogicalJoin(condition=[=($13, $20)], joinType=[left]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], lastname=[REX_EXTRACT($6, '(?^[A-Z])', 'lastname')], $f13=[CAST(REX_EXTRACT($6, '(?^[A-Z])', 'lastname')):VARCHAR]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSort(fetch=[50000]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + LogicalFilter(condition=[<=($13, SAFE_CAST(1:BIGINT))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _row_number_join_max_dedup_=[ROW_NUMBER() OVER (PARTITION BY $6)]) + LogicalFilter(condition=[IS NOT NULL($6)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableCalc(expr#0..13=[{inputs}], account_number=[$t1], firstname=[$t2], address=[$t3], birthdate=[$t4], gender=[$t5], city=[$t6], lastname=[$t7], balance=[$t8], employer=[$t9], state=[$t10], age=[$t11], email=[$t12], male=[$t13]) + EnumerableLimit(fetch=[10000]) + EnumerableMergeJoin(condition=[=($0, $7)], joinType=[left]) + EnumerableCalc(expr#0=[{inputs}], expr#1=['(?^[A-Z])'], expr#2=['lastname'], expr#3=[REX_EXTRACT($t0, $t1, $t2)], expr#4=[CAST($t3):VARCHAR], $f0=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[lastname], LIMIT->10000, SORT_EXPR->[CAST(REX_EXTRACT($0, '(?^[A-Z])', 'lastname')):VARCHAR ASCENDING NULLS_LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["lastname"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQESHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVYX0VYVFJBQ1QiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAyMDAwCiAgICAgIH0sCiAgICAgICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAgICAgImR5bmFtaWMiOiBmYWxzZQogICAgfQogIF0sCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAicHJlY2lzaW9uIjogLTEKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":true,"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["lastname","(?^[A-Z])","lastname"]}},"type":"string","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->50000, SORT->[6]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"lastname":{"terms":{"field":"lastname","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["lastname","account_number","firstname","address","birthdate","gender","city","balance","employer","state","age","email","male"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj_w_max_option.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj_w_max_option.yaml index 6a9d9fd09b2..d76548486ce 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj_w_max_option.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj_w_max_option.yaml @@ -1,13 +1,13 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$13], firstname=[$14], address=[$15], birthdate=[$16], gender=[$17], city=[$18], lastname=[$19], balance=[$20], employer=[$21], state=[$22], age=[$23], email=[$24], male=[$25]) - LogicalJoin(condition=[=($12, $19)], joinType=[left]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], lastname=[REX_EXTRACT($6, '(?^[A-Z])', 'lastname')]) + LogicalProject(account_number=[$14], firstname=[$15], address=[$16], birthdate=[$17], gender=[$18], city=[$19], lastname=[$20], balance=[$21], employer=[$22], state=[$23], age=[$24], email=[$25], male=[$26]) + LogicalJoin(condition=[=($13, $20)], joinType=[left]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], lastname=[REX_EXTRACT($6, '(?^[A-Z])', 'lastname')], $f13=[CAST(REX_EXTRACT($6, '(?^[A-Z])', 'lastname')):VARCHAR]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[50000]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) - LogicalFilter(condition=[<=($13, 1)]) + LogicalFilter(condition=[<=($13, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _row_number_join_max_dedup_=[ROW_NUMBER() OVER (PARTITION BY $6)]) LogicalFilter(condition=[IS NOT NULL($6)]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) @@ -16,6 +16,6 @@ calcite: EnumerableCalc(expr#0..13=[{inputs}], account_number=[$t1], firstname=[$t2], address=[$t3], birthdate=[$t4], gender=[$t5], city=[$t6], lastname=[$t7], balance=[$t8], employer=[$t9], state=[$t10], age=[$t11], email=[$t12], male=[$t13]) EnumerableLimit(fetch=[10000]) EnumerableMergeJoin(condition=[=($0, $7)], joinType=[left]) - EnumerableCalc(expr#0=[{inputs}], expr#1=['(?^[A-Z])'], expr#2=['lastname'], expr#3=[REX_EXTRACT($t0, $t1, $t2)], $f0=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[lastname], LIMIT->10000, SORT_EXPR->[REX_EXTRACT($0, '(?^[A-Z])', 'lastname') ASCENDING NULLS_LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["lastname"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQC63sKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRVhfRVhUUkFDVCIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAicHJlY2lzaW9uIjogMjAwMAogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":true,"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["lastname","(?^[A-Z])","lastname"]}},"type":"string","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0=[{inputs}], expr#1=['(?^[A-Z])'], expr#2=['lastname'], expr#3=[REX_EXTRACT($t0, $t1, $t2)], expr#4=[CAST($t3):VARCHAR], $f0=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[lastname], LIMIT->10000, SORT_EXPR->[CAST(REX_EXTRACT($0, '(?^[A-Z])', 'lastname')):VARCHAR ASCENDING NULLS_LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["lastname"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQESHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVYX0VYVFJBQ1QiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAyMDAwCiAgICAgIH0sCiAgICAgICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAgICAgImR5bmFtaWMiOiBmYWxzZQogICAgfQogIF0sCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAicHJlY2lzaW9uIjogLTEKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":true,"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["lastname","(?^[A-Z])","lastname"]}},"type":"string","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->50000, SORT->[6]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"lastname":{"terms":{"field":"lastname","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["lastname","account_number","firstname","address","birthdate","gender","city","balance","employer","state","age","email","male"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_then_field_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_then_field_sort.yaml index d9726a2beb1..d139aad63e9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_then_field_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_then_field_sort.yaml @@ -1,12 +1,11 @@ calcite: logical: | LogicalSystemLimit(sort0=[$10], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], age2=[$19], balance2=[$20]) - LogicalSort(sort0=[$10], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[$19], balance2=[ABS($7)]) - LogicalSort(sort0=[$19], sort1=[$10], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSort(sort0=[$10], dir0=[ASC-nulls-first]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], age2=[$19], balance2=[ABS($7)]) + LogicalSort(sort0=[$19], sort1=[$10], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..12=[{inputs}], expr#13=[+($t10, $t7)], expr#14=[ABS($t7)], proj#0..14=[{exprs}]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT->[{ diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push.json new file mode 100644 index 00000000000..62e3523f96b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push.json @@ -0,0 +1,7 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], cnt=[COUNT()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push1.yaml index e0d0a3d0070..990ad2adce1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push1.yaml @@ -4,4 +4,4 @@ calcite: LogicalAggregate(group=[{}], cnt=[COUNT()]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#494:LogicalAggregate.NONE.[](input=RelSubset#493,group={},cnt=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push10.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push10.yaml index 26ab37adc53..50d0c9a1c1d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push10.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push10.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalAggregate(group=[{}], count(firstname)=[COUNT($0)], count(name)=[COUNT($1)]) - LogicalProject(firstname=[$1], name=[$10]) + LogicalProject(firstname=[$1], lastname=[$10]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(firstname)=COUNT($0),count(name)=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"count(firstname)":{"value_count":{"field":"firstname.keyword"}},"count(name)":{"value_count":{"field":"lastname.keyword"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#1284:LogicalAggregate.NONE.[](input=RelSubset#1283,group={},count(firstname)=COUNT($0),count(name)=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"count(firstname)":{"value_count":{"field":"firstname.keyword"}},"count(name)":{"value_count":{"field":"lastname.keyword"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push4.yaml index b191f90b170..2a90b9785dd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push4.yaml @@ -2,8 +2,8 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(c1=[$0], c2=[$0]) - LogicalAggregate(group=[{}], c1=[COUNT()]) + LogicalAggregate(group=[{}], c2=[COUNT()]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0=[{inputs}], c1=[$t0], c2=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},c1=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#783:LogicalAggregate.NONE.[](input=RelSubset#782,group={},c2=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push5.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push5.yaml index 933c7bab9b8..f9c2458af72 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push5.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push5.yaml @@ -2,10 +2,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(c1=[$0], c2=[$0]) - LogicalAggregate(group=[{}], c1=[COUNT($0)]) + LogicalAggregate(group=[{}], c2=[COUNT($0)]) LogicalProject(lastname=[$10]) LogicalFilter(condition=[IS NOT NULL($10)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0=[{inputs}], c1=[$t0], c2=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},c1=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"lastname","boost":1.0}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},c2=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"lastname","boost":1.0}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml index e1328084f77..552389b0b22 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalAggregate(group=[{}], cnt=[COUNT($0)]) - LogicalProject($f1=[+($3, 1)]) + LogicalProject($f0=[+($3, 1)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"cnt":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#1105:LogicalAggregate.NONE.[](input=RelSubset#1104,group={},cnt=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"cnt":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp":1765952821451990000,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push8.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push8.yaml index beaac7cbe69..1fa091fa7ed 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push8.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push8.yaml @@ -5,4 +5,4 @@ calcite: LogicalProject(lastname=[$10]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},c1=COUNT(),c2=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"c1":{"value_count":{"field":"_index"}},"c2":{"value_count":{"field":"lastname.keyword"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#1169:LogicalAggregate.NONE.[](input=RelSubset#1168,group={},c1=COUNT(),c2=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"c1":{"value_count":{"field":"_index"}},"c2":{"value_count":{"field":"lastname.keyword"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push9.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push9.yaml index b5548293b7f..5c0047753e6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push9.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push9.yaml @@ -5,4 +5,4 @@ calcite: LogicalProject(firstname=[$1], lastname=[$10]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(firstname)=COUNT($0),count(lastname)=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"count(firstname)":{"value_count":{"field":"firstname.keyword"}},"count(lastname)":{"value_count":{"field":"lastname.keyword"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#1226:LogicalAggregate.NONE.[](input=RelSubset#1225,group={},count(firstname)=COUNT($0),count(lastname)=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"count(firstname)":{"value_count":{"field":"firstname.keyword"}},"count(lastname)":{"value_count":{"field":"lastname.keyword"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_eval_complex_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_eval_complex_push.json deleted file mode 100644 index 8e429a7f610..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_eval_complex_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], mature_count=[COUNT($0)])\n LogicalProject($f1=[CASE(SEARCH($10, Sarg[(30..50)]), 1, null:NULL)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},mature_count=COUNT() FILTER $0)], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"mature_count\":{\"filter\":{\"range\":{\"age\":{\"from\":30.0,\"to\":50.0,\"include_lower\":false,\"include_upper\":false,\"boost\":1.0}}},\"aggregations\":{\"mature_count\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_eval_complex_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_eval_complex_push.yaml new file mode 100644 index 00000000000..5e6380cb217 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_eval_complex_push.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], mature_count=[COUNT($0)]) + LogicalProject($f0=[CASE(SEARCH($10, Sarg[(30..50)]), 1, null:INTEGER)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},mature_count=COUNT() FILTER $0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"mature_count":{"filter":{"range":{"age":{"from":30.0,"to":50.0,"include_lower":false,"include_upper":false,"boost":1.0}}},"aggregations":{"mature_count":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_eval_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_eval_push.json deleted file mode 100644 index f0b75595a56..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_eval_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], mature_count=[COUNT($0)])\n LogicalProject($f1=[CASE(>($10, 30), 1, null:NULL)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},mature_count=COUNT() FILTER $0)], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"mature_count\":{\"filter\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"aggregations\":{\"mature_count\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_eval_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_eval_push.yaml new file mode 100644 index 00000000000..581ef9b81ec --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_eval_push.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], mature_count=[COUNT($0)]) + LogicalProject($f0=[CASE(>($10, 30), 1, null:INTEGER)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#11421:LogicalAggregate.NONE.[](input=RelSubset#11420,group={},mature_count=COUNT() FILTER $0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"mature_count":{"filter":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"mature_count":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_push.yaml new file mode 100644 index 00000000000..f9c2458af72 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_push.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(c1=[$0], c2=[$0]) + LogicalAggregate(group=[{}], c2=[COUNT($0)]) + LogicalProject(lastname=[$10]) + LogicalFilter(condition=[IS NOT NULL($10)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0=[{inputs}], c1=[$t0], c2=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},c2=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"lastname","boost":1.0}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex1.yaml index 17aef474306..2aae73e3bb5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex1.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[<=($17, 1)]) + LogicalFilter(condition=[<=($17, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4)]) LogicalFilter(condition=[IS NOT NULL($4)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex2.yaml index ec8172633bb..818923e646c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex2.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3]) - LogicalFilter(condition=[<=($4, 1)]) + LogicalFilter(condition=[<=($4, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $1, $3)]) LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($3))]) LogicalProject(account_number=[$0], gender=[$4], age=[$8], state=[$7]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex3.yaml index c31ff1a3eb3..5d10805d04b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex3.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex3.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[<=($17, 2)]) + LogicalFilter(condition=[<=($17, SAFE_CAST(2:BIGINT))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4, $7)]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex4.yaml index 345d3829266..0e9bffb2c34 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex4.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3]) - LogicalFilter(condition=[<=($4, 2)]) + LogicalFilter(condition=[<=($4, SAFE_CAST(2:BIGINT))]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $1, $3)]) LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($3))]) LogicalProject(account_number=[$0], gender=[$4], age=[$8], state=[$7]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr.yaml new file mode 100644 index 00000000000..aa2cc131528 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5]) + LogicalFilter(condition=[<=($6, SAFE_CAST(2:BIGINT))]) + LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4, $5)]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($5))]) + LogicalSort(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) + LogicalProject(account_number=[$0], gender=[$4], age=[$8], state=[$7], new_gender=[LOWER($4)], new_state=[LOWER($7)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"new_gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"missing_bucket":false,"order":"asc"}}},{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["account_number","gender","age","state"],"excludes":[]},"script_fields":{"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false},"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr1.yaml index a84b664804e..075816d6ef8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr1.yaml @@ -2,10 +2,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], new_gender=[$17]) - LogicalFilter(condition=[<=($18, 1)]) + LogicalFilter(condition=[<=($18, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_gender=[$17], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $17)]) LogicalFilter(condition=[IS NOT NULL($17)]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_gender=[LOWER($4)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"new_gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]},"script_fields":{"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"new_gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]},"script_fields":{"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr2.yaml index a892b5c4e26..9e68eda5272 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr2.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5]) - LogicalFilter(condition=[<=($6, 1)]) + LogicalFilter(condition=[<=($6, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4, $5)]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($5))]) LogicalProject(account_number=[$0], gender=[$4], age=[$8], state=[$7], new_gender=[LOWER($4)], new_state=[LOWER($7)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr2_alternative.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr2_alternative.yaml index ffa6bc1aa52..b6c7f9dab27 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr2_alternative.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr2_alternative.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5]) - LogicalFilter(condition=[<=($6, 1)]) + LogicalFilter(condition=[<=($6, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4, $5)]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($5))]) LogicalProject(account_number=[$0], gender=[$4], age=[$8], state=[$7], new_gender=[LOWER($4)], new_state=[LOWER($7)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr3.yaml index ce4f25d3892..7da13bc93fb 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr3.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr3.yaml @@ -1,11 +1,12 @@ + calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], new_gender=[$17], new_state=[$18]) - LogicalFilter(condition=[<=($19, 2)]) + LogicalFilter(condition=[<=($19, SAFE_CAST(2:BIGINT))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_gender=[$17], new_state=[$18], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $17, $18)]) LogicalFilter(condition=[AND(IS NOT NULL($17), IS NOT NULL($18))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_gender=[LOWER($4)], new_state=[LOWER($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"new_gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"missing_bucket":false,"order":"asc"}}},{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]},"script_fields":{"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false},"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"new_gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"missing_bucket":false,"order":"asc"}}},{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]},"script_fields":{"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false},"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr3_alternative.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr3_alternative.yaml index a7009a3751b..a56e889cc0d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr3_alternative.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr3_alternative.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], new_gender=[$17], new_state=[$18]) - LogicalFilter(condition=[<=($19, 2)]) + LogicalFilter(condition=[<=($19, SAFE_CAST(2:BIGINT))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_gender=[$17], new_state=[$18], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $17, $18)]) LogicalFilter(condition=[AND(IS NOT NULL($17), IS NOT NULL($18))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_gender=[LOWER($4)], new_state=[LOWER($7)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr4.yaml index 54b69192b13..3d00926aa68 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr4.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5]) - LogicalFilter(condition=[<=($6, 2)]) + LogicalFilter(condition=[<=($6, SAFE_CAST(2:BIGINT))]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4, $5)]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($5))]) LogicalSort(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr4_alternative.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr4_alternative.yaml index ea9c46e976c..aa2cc131528 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr4_alternative.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr4_alternative.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5]) - LogicalFilter(condition=[<=($6, 2)]) + LogicalFilter(condition=[<=($6, SAFE_CAST(2:BIGINT))]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4, $5)]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($5))]) LogicalSort(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr_complex1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr_complex1.yaml index d7354c8b6d1..4518ee85ac2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr_complex1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr_complex1.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(sort0=[$4], sort1=[$5], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5]) - LogicalFilter(condition=[<=($6, 2)]) + LogicalFilter(condition=[<=($6, SAFE_CAST(2:BIGINT))]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4, $5)]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($5))]) LogicalSort(sort0=[$4], sort1=[$5], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) @@ -10,4 +10,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), SORT->[4 ASC FIRST, 5 DESC LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"new_gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"missing_bucket":false,"order":"asc"}}},{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"missing_bucket":false,"order":"desc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["account_number","gender","age","state"],"excludes":[]},"script_fields":{"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false},"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), SORT->[4 ASC FIRST, 5 DESC LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"new_gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"missing_bucket":false,"order":"asc"}}},{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"missing_bucket":false,"order":"desc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["account_number","gender","age","state"],"excludes":[]},"script_fields":{"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false},"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr_complex1_alternative.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr_complex1_alternative.yaml index fdd427f5f9b..e864cf636dd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr_complex1_alternative.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr_complex1_alternative.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(sort0=[$4], sort1=[$5], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5]) - LogicalFilter(condition=[<=($6, 2)]) + LogicalFilter(condition=[<=($6, SAFE_CAST(2:BIGINT))]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4, $5)]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($5))]) LogicalSort(sort0=[$4], sort1=[$5], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) @@ -10,4 +10,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), SORT->[4 ASC FIRST, 5 DESC LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"new_gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"missing_bucket":false,"order":"asc"}}},{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"missing_bucket":false,"order":"desc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["account_number","gender","age","state"],"excludes":[]},"script_fields":{"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false},"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), SORT->[4 ASC FIRST, 5 DESC LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"new_gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"missing_bucket":false,"order":"asc"}}},{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"missing_bucket":false,"order":"desc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["account_number","gender","age","state"],"excludes":[]},"script_fields":{"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false},"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr_complex2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr_complex2.yaml index 53462af4bce..a913f43e8fd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr_complex2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr_complex2.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$4], sort1=[$5], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$4], sort1=[$5], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5]) - LogicalFilter(condition=[<=($6, 2)]) + LogicalFilter(condition=[<=($6, SAFE_CAST(2:BIGINT))]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4, $5)]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($5))]) LogicalProject(account_number=[$0], gender=[$4], age=[$8], state=[$7], new_gender=[LOWER($4)], new_state=[LOWER($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), SORT->[4 ASC FIRST, 5 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"new_gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"missing_bucket":false,"order":"asc"}}},{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"missing_bucket":false,"order":"desc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["account_number","gender","age","state"],"excludes":[]},"script_fields":{"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false},"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), SORT->[4 ASC FIRST, 5 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"new_gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"missing_bucket":false,"order":"asc"}}},{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"missing_bucket":false,"order":"desc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["account_number","gender","age","state"],"excludes":[]},"script_fields":{"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false},"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr_complex2_alternative.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr_complex2_alternative.yaml index b9cb684053f..0e367468e1b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr_complex2_alternative.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr_complex2_alternative.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$4], sort1=[$5], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$4], sort1=[$5], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5]) - LogicalFilter(condition=[<=($6, 2)]) + LogicalFilter(condition=[<=($6, SAFE_CAST(2:BIGINT))]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4, $5)]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($5))]) LogicalProject(account_number=[$0], gender=[$4], age=[$8], state=[$7], new_gender=[LOWER($4)], new_state=[LOWER($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), SORT->[4 ASC FIRST, 5 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"new_gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"missing_bucket":false,"order":"asc"}}},{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"missing_bucket":false,"order":"desc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["account_number","gender","age","state"],"excludes":[]},"script_fields":{"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false},"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), SORT->[4 ASC FIRST, 5 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"new_gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"missing_bucket":false,"order":"asc"}}},{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"missing_bucket":false,"order":"desc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["account_number","gender","age","state"],"excludes":[]},"script_fields":{"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false},"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keep_empty_false_pushdown.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keep_empty_false_pushdown.yaml new file mode 100644 index 00000000000..c9a73af550b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keep_empty_false_pushdown.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], gender=[$1], age=[$2]) + LogicalFilter(condition=[<=($3, SAFE_CAST(1:BIGINT))]) + LogicalProject(account_number=[$0], gender=[$1], age=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $1)]) + LogicalFilter(condition=[IS NOT NULL($1)]) + LogicalProject(account_number=[$0], gender=[$4], age=[$8]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, gender, age], AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","_source":{"includes":["account_number","gender","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["gender","account_number","age"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keep_empty_true_not_pushed.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keep_empty_true_not_pushed.yaml new file mode 100644 index 00000000000..bb4f5ab388a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keep_empty_true_not_pushed.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], gender=[$1], age=[$2]) + LogicalFilter(condition=[OR(IS NULL($1), <=($3, SAFE_CAST(1:BIGINT)))]) + LogicalProject(account_number=[$0], gender=[$4], age=[$8], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t1)], expr#5=[1:BIGINT], expr#6=[<=($t3, $t5)], expr#7=[OR($t4, $t6)], proj#0..2=[{exprs}], $condition=[$t7]) + EnumerableWindow(window#0=[window(partition {1} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keepempty_false_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keepempty_false_push.yaml index ea1dbba807d..b12c113edde 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keepempty_false_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keepempty_false_push.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], gender=[$1], age=[$2]) - LogicalFilter(condition=[<=($3, 1)]) + LogicalFilter(condition=[<=($3, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $1)]) LogicalFilter(condition=[IS NOT NULL($1)]) LogicalProject(account_number=[$0], gender=[$4], age=[$8]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keepempty_false_pushdown.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keepempty_false_pushdown.yaml new file mode 100644 index 00000000000..b12c113edde --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keepempty_false_pushdown.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], gender=[$1], age=[$2]) + LogicalFilter(condition=[<=($3, SAFE_CAST(1:BIGINT))]) + LogicalProject(account_number=[$0], gender=[$1], age=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $1)]) + LogicalFilter(condition=[IS NOT NULL($1)]) + LogicalProject(account_number=[$0], gender=[$4], age=[$8]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["gender","account_number","age"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keepempty_true_not_pushed.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keepempty_true_not_pushed.yaml index 9f15c0d120b..bb4f5ab388a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keepempty_true_not_pushed.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keepempty_true_not_pushed.yaml @@ -2,11 +2,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], gender=[$1], age=[$2]) - LogicalFilter(condition=[OR(IS NULL($1), <=($3, 1))]) + LogicalFilter(condition=[OR(IS NULL($1), <=($3, SAFE_CAST(1:BIGINT)))]) LogicalProject(account_number=[$0], gender=[$4], age=[$8], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t1)], expr#5=[1], expr#6=[<=($t3, $t5)], expr#7=[OR($t4, $t6)], proj#0..2=[{exprs}], $condition=[$t7]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t1)], expr#5=[1:BIGINT], expr#6=[<=($t3, $t5)], expr#7=[OR($t4, $t6)], proj#0..2=[{exprs}], $condition=[$t7]) EnumerableWindow(window#0=[window(partition {1} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_push.yaml index ea1dbba807d..b12c113edde 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_push.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], gender=[$1], age=[$2]) - LogicalFilter(condition=[<=($3, 1)]) + LogicalFilter(condition=[<=($3, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $1)]) LogicalFilter(condition=[IS NOT NULL($1)]) LogicalProject(account_number=[$0], gender=[$4], age=[$8]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_pushdown.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_pushdown.yaml new file mode 100644 index 00000000000..b12c113edde --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_pushdown.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], gender=[$1], age=[$2]) + LogicalFilter(condition=[<=($3, SAFE_CAST(1:BIGINT))]) + LogicalProject(account_number=[$0], gender=[$1], age=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $1)]) + LogicalFilter(condition=[IS NOT NULL($1)]) + LogicalProject(account_number=[$0], gender=[$4], age=[$8]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["gender","account_number","age"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_rename.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_rename.yaml new file mode 100644 index 00000000000..3d00926aa68 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_rename.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5]) + LogicalFilter(condition=[<=($6, SAFE_CAST(2:BIGINT))]) + LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4, $5)]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($5))]) + LogicalSort(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) + LogicalProject(account_number=[$0], gender=[$4], age=[$8], state=[$7], new_gender=[LOWER($4)], new_state=[LOWER($7)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"new_gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"missing_bucket":false,"order":"asc"}}},{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["account_number","gender","age","state"],"excludes":[]},"script_fields":{"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false},"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_text_type_no_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_text_type_no_push.yaml index 80599e8a338..77dd09f164d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_text_type_no_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_text_type_no_push.yaml @@ -2,12 +2,12 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) - LogicalFilter(condition=[<=($19, 1)]) + LogicalFilter(condition=[<=($19, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $11)]) LogicalFilter(condition=[IS NOT NULL($11)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..13=[{inputs}], expr#14=[1], expr#15=[<=($t13, $t14)], proj#0..12=[{exprs}], $condition=[$t15]) + EnumerableCalc(expr#0..13=[{inputs}], expr#14=[1:BIGINT], expr#15=[<=($t13, $t14)], proj#0..12=[{exprs}], $condition=[$t15]) EnumerableWindow(window#0=[window(partition {11} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], FILTER->IS NOT NULL($11)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"email","boost":1.0}},"_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], FILTER->IS NOT NULL($11)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"email","boost":1.0}},"_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr1.yaml index 45034087d0f..b80152ba712 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr1.yaml @@ -2,10 +2,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], new_gender=[$17]) - LogicalFilter(condition=[<=($18, 1)]) + LogicalFilter(condition=[<=($18, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_gender=[$17], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $8)]) LogicalFilter(condition=[IS NOT NULL($8)]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_gender=[LOWER($4)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age":{"terms":{"field":"age","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["age","account_number","firstname","address","balance","gender","city","employer","state","email","lastname"],"excludes":[]},"script_fields":{"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age":{"terms":{"field":"age","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["age","account_number","firstname","address","balance","gender","city","employer","state","email","lastname"],"excludes":[]},"script_fields":{"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr2.yaml index eadbe165e38..80a16284fb3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr2.yaml @@ -2,10 +2,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5]) - LogicalFilter(condition=[<=($6, 1)]) + LogicalFilter(condition=[<=($6, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2, $5)]) LogicalFilter(condition=[AND(IS NOT NULL($2), IS NOT NULL($5))]) LogicalProject(account_number=[$0], gender=[$4], age=[$8], state=[$7], new_gender=[LOWER($4)], new_state=[LOWER($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age":{"terms":{"field":"age","missing_bucket":false,"order":"asc"}}},{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["age","account_number","gender","state"],"excludes":[]},"script_fields":{"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false},"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age":{"terms":{"field":"age","missing_bucket":false,"order":"asc"}}},{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["age","account_number","gender","state"],"excludes":[]},"script_fields":{"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false},"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr2_alternative.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr2_alternative.yaml index 4c7afb236a8..18a715e8854 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr2_alternative.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr2_alternative.yaml @@ -2,10 +2,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5]) - LogicalFilter(condition=[<=($6, 1)]) + LogicalFilter(condition=[<=($6, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2, $5)]) LogicalFilter(condition=[AND(IS NOT NULL($2), IS NOT NULL($5))]) LogicalProject(account_number=[$0], gender=[$4], age=[$8], state=[$7], new_gender=[LOWER($4)], new_state=[LOWER($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age":{"terms":{"field":"age","missing_bucket":false,"order":"asc"}}},{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["age","account_number","gender","state"],"excludes":[]},"script_fields":{"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false},"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age":{"terms":{"field":"age","missing_bucket":false,"order":"asc"}}},{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["age","account_number","gender","state"],"excludes":[]},"script_fields":{"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false},"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr3.yaml index 6f84e2f856d..eb36e333b49 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr3.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr3.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], new_gender=[$17], new_state=[$18]) - LogicalFilter(condition=[<=($19, 2)]) + LogicalFilter(condition=[<=($19, SAFE_CAST(2:BIGINT))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_gender=[$17], new_state=[$18], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $8, $0)]) LogicalFilter(condition=[AND(IS NOT NULL($8), IS NOT NULL($0))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_gender=[LOWER($4)], new_state=[LOWER($7)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr3_alternative.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr3_alternative.yaml index 3e647b1d4b6..8d7415320d3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr3_alternative.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr3_alternative.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], new_gender=[$17], new_state=[$18]) - LogicalFilter(condition=[<=($19, 2)]) + LogicalFilter(condition=[<=($19, SAFE_CAST(2:BIGINT))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_gender=[$17], new_state=[$18], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $8, $0)]) LogicalFilter(condition=[AND(IS NOT NULL($8), IS NOT NULL($0))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_gender=[LOWER($4)], new_state=[LOWER($7)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr4.yaml index 0767a8357e1..88da6d33a24 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr4.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5]) - LogicalFilter(condition=[<=($6, 2)]) + LogicalFilter(condition=[<=($6, SAFE_CAST(2:BIGINT))]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $1, $3)]) LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($3))]) LogicalSort(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr4_alternative.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr4_alternative.yaml index 4527da4f140..a35d2d8ded2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr4_alternative.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr4_alternative.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5]) - LogicalFilter(condition=[<=($6, 2)]) + LogicalFilter(condition=[<=($6, SAFE_CAST(2:BIGINT))]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $1, $3)]) LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($3))]) LogicalSort(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_eval_max.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_eval_max.yaml index bd517738a0b..d4a35f98447 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_eval_max.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_eval_max.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], new=[SCALAR_MAX(1, 2, 3, $8, 'banana':VARCHAR)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], new=[SCALAR_MAX(1, 2, 3, $8, 'banana')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..10=[{inputs}], expr#11=[1], expr#12=[2], expr#13=[3], expr#14=['banana':VARCHAR], expr#15=[SCALAR_MAX($t11, $t12, $t13, $t8, $t14)], proj#0..10=[{exprs}], $f11=[$t15]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..10=[{inputs}], expr#11=[1], expr#12=[2], expr#13=[3], expr#14=['banana'], expr#15=[SCALAR_MAX($t11, $t12, $t13, $t8, $t14)], proj#0..10=[{exprs}], $f11=[$t15]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_eval_min.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_eval_min.yaml index d28a4ef30ec..05da151c99a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_eval_min.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_eval_min.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], new=[SCALAR_MIN(1, 2, 3, $8, 'banana':VARCHAR)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], new=[SCALAR_MIN(1, 2, 3, $8, 'banana')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..10=[{inputs}], expr#11=[1], expr#12=[2], expr#13=[3], expr#14=['banana':VARCHAR], expr#15=[SCALAR_MIN($t11, $t12, $t13, $t8, $t14)], proj#0..10=[{exprs}], $f11=[$t15]) + EnumerableCalc(expr#0..10=[{inputs}], expr#11=[1], expr#12=[2], expr#13=[3], expr#14=['banana'], expr#15=[SCALAR_MIN($t11, $t12, $t13, $t8, $t14)], proj#0..10=[{exprs}], $f11=[$t15]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_correlated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_correlated_subquery.yaml index 05dd3778b8f..a344176a839 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_correlated_subquery.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_correlated_subquery.yaml @@ -1,21 +1,21 @@ calcite: logical: | LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(id=[$2], name=[$0], salary=[$4]) - LogicalSort(sort0=[$4], dir0=[DESC-nulls-last]) + LogicalSort(sort0=[$2], dir0=[DESC-nulls-last]) + LogicalProject(id=[$2], name=[$0], salary=[$4]) LogicalFilter(condition=[EXISTS({ - LogicalProject(name=[$0], uid=[$1], occupation=[$2], department=[$3]) - LogicalFilter(condition=[=($cor0.id, $1)]) - LogicalSystemLimit(fetch=[10000], type=[SUBSEARCH_MAXOUT]) + LogicalFilter(condition=[=($cor1.id, $1)]) + LogicalSort(fetch=[10000]) + LogicalProject(name=[$0], uid=[$1], occupation=[$2], department=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9]) LogicalFilter(condition=[=($0, 'Tom')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) - })], variablesSet=[[$cor0]]) + })], variablesSet=[[$cor1]]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | - EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) - CalciteEnumerableTopK(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000]) - EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) + CalciteEnumerableTopK(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) + EnumerableCorrelate(correlation=[$cor1], joinType=[inner], requiredColumns=[{1}]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id, salary]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id","salary"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableAggregate(group=[{0}]) - EnumerableCalc(expr#0=[{inputs}], expr#1=[true], expr#2=[$cor0], expr#3=[$t2.id], expr#4=[=($t3, $t0)], i=[$t1], $condition=[$t4]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[name, uid], FILTER->=($0, 'Tom'), LIMIT->10000, PROJECT->[uid]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"term":{"name":{"value":"Tom","boost":1.0}}},"_source":{"includes":["uid"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0=[{inputs}], expr#1=[true], expr#2=[$cor1], expr#3=[$t2.id], expr#4=[=($t3, $t0)], i=[$t1], $condition=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[name, uid], FILTER->=($0, 'Tom'), PROJECT->[uid], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"term":{"name":{"value":"Tom","boost":1.0}}},"_source":{"includes":["uid"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_uncorrelated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_uncorrelated_subquery.yaml index f54be57fefd..a0b3b88d630 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_uncorrelated_subquery.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_uncorrelated_subquery.yaml @@ -1,20 +1,20 @@ calcite: logical: | LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(id=[$2], name=[$0], salary=[$4]) - LogicalSort(sort0=[$4], dir0=[DESC-nulls-last]) + LogicalSort(sort0=[$2], dir0=[DESC-nulls-last]) + LogicalProject(id=[$2], name=[$0], salary=[$4]) LogicalFilter(condition=[EXISTS({ - LogicalSystemLimit(fetch=[10000], type=[SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[10000]) LogicalProject(name=[$0], uid=[$1], occupation=[$2], department=[$3]) LogicalFilter(condition=[=($0, 'Tom')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) - })], variablesSet=[[$cor0]]) + })]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | - EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) - CalciteEnumerableTopK(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000]) + CalciteEnumerableTopK(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id, salary]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id","salary"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableAggregate(group=[{0}]) EnumerableCalc(expr#0=[{inputs}], expr#1=[true], i=[$t1]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[name], FILTER->=($0, 'Tom'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"term":{"name":{"value":"Tom","boost":1.0}}},"_source":{"includes":["name"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[name], FILTER->=($0, 'Tom'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"term":{"name":{"value":"Tom","boost":1.0}}},"_source":{"includes":["name"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.json deleted file mode 100644 index 6c60de5dccc..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(avg_age=[$1], age_range=[$0])\n LogicalAggregate(group=[{0}], avg_age=[AVG($1)])\n LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[30..40]]), 'u40':VARCHAR, 'u100':VARCHAR)], age=[$10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg_age=AVG($1)), PROJECT->[avg_age, age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"age_range\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"CASE\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"CASE\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"SPECIAL\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"<\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"LESS_THAN\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 0,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 1,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n }\\\\n ]\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 2,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"AND\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"AND\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"<=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"LESS_THAN_OR_EQUAL\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 3,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 4,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n }\\\\n ]\\\\n },\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"<=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"LESS_THAN_OR_EQUAL\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 5,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 6,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 7,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 8,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":1765429962288092000,\"SOURCES\":[0,2,2,2,0,0,2,2,2],\"DIGESTS\":[\"age\",30,\"u30\",30,\"age\",\"age\",40,\"u40\",\"u100\"]}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n", - "extended": "public org.apache.calcite.linq4j.Enumerable bind(final org.apache.calcite.DataContext root) {\n final org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan v1stashed = (org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan) root.get(\"v1stashed\");\n return v1stashed.scan();\n}\n\n\npublic Class getElementType() {\n return java.lang.Object[].class;\n}\n\n\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.yaml new file mode 100644 index 00000000000..cb358fb9863 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg_age=[$1], age_range=[$0]) + LogicalAggregate(group=[{0}], avg_age=[AVG($1)]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[30..40]]), 'u40':VARCHAR, 'u100':VARCHAR)], age=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg_age=AVG($1)), PROJECT->[avg_age, age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQITHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiPCIsCiAgICAgICAgImtpbmQiOiAiTEVTU19USEFOIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiQU5EIiwKICAgICAgICAia2luZCI6ICJBTkQiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgIm9wIjogewogICAgICAgICAgICAibmFtZSI6ICI8PSIsCiAgICAgICAgICAgICJraW5kIjogIkxFU1NfVEhBTl9PUl9FUVVBTCIsCiAgICAgICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICAgICAgfSwKICAgICAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICAgICAgewogICAgICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAzLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogNCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9CiAgICAgICAgICBdCiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAib3AiOiB7CiAgICAgICAgICAgICJuYW1lIjogIjw9IiwKICAgICAgICAgICAgImtpbmQiOiAiTEVTU19USEFOX09SX0VRVUFMIiwKICAgICAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgICAgICB9LAogICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDUsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgICAgIH0KICAgICAgICAgICAgfSwKICAgICAgICAgICAgewogICAgICAgICAgICAgICJkeW5hbWljUGFyYW0iOiA2LAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiA3LAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogOCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2,2,0,0,2,2,2],"DIGESTS":["age",30,"u30",30,"age","age",40,"u40","u100"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_fillnull_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_fillnull_push.json deleted file mode 100644 index cdc0c4c27fd..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_fillnull_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[COALESCE($8, -1)], balance=[COALESCE($3, -1)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0..1=[{inputs}], expr#2=[-1], expr#3=[COALESCE($t0, $t2)], expr#4=[COALESCE($t1, $t2)], $f0=[$t3], $f1=[$t4])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age, balance], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\",\"balance\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_fillnull_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_fillnull_push.yaml new file mode 100644 index 00000000000..b57a72289c9 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_fillnull_push.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[CASE(IS NOT NULL($8), CAST($8):BIGINT NOT NULL, -1:BIGINT)], balance=[CASE(IS NOT NULL($3), CAST($3):BIGINT NOT NULL, -1:BIGINT)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NOT NULL($t0)], expr#3=[CAST($t0):BIGINT NOT NULL], expr#4=[-1:BIGINT], expr#5=[CASE($t2, $t3, $t4)], expr#6=[IS NOT NULL($t1)], expr#7=[CAST($t1):BIGINT NOT NULL], expr#8=[CASE($t6, $t7, $t4)], $f0=[$t5], $f1=[$t8]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age, balance], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_fillnull_value_syntax.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_fillnull_value_syntax.yaml index 0f2ba239d73..1740c789956 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_fillnull_value_syntax.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_fillnull_value_syntax.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[COALESCE($8, 0)], balance=[COALESCE($3, 0)]) + LogicalProject(age=[CASE(IS NOT NULL($8), CAST($8):BIGINT NOT NULL, 0:BIGINT)], balance=[CASE(IS NOT NULL($3), CAST($3):BIGINT NOT NULL, 0:BIGINT)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[0], expr#3=[COALESCE($t0, $t2)], expr#4=[COALESCE($t1, $t2)], $f0=[$t3], $f1=[$t4]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NOT NULL($t0)], expr#3=[CAST($t0):BIGINT NOT NULL], expr#4=[0:BIGINT], expr#5=[CASE($t2, $t3, $t4)], expr#6=[IS NOT NULL($t1)], expr#7=[CAST($t1):BIGINT NOT NULL], expr#8=[CASE($t6, $t7, $t4)], $f0=[$t5], $f1=[$t8]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age, balance], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter.yaml new file mode 100644 index 00000000000..0e4632c9595 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(name=[$0], age=[$1]) + LogicalFilter(condition=[=($1, SAFE_CAST(20:BIGINT))]) + CalciteLogicalIndexScan(table=[[OpenSearch, test]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, test]], PushDownContext=[[PROJECT->[name, age], FILTER->=($1, 20), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"term":{"age":{"value":20,"boost":1.0}}},"_source":{"includes":["name","age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml index c3db9d4f6b9..63d4d0b4a98 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml @@ -4,7 +4,7 @@ calcite: LogicalProject(avg_age=[$2], state=[$0], city=[$1]) LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) LogicalProject(state=[$7], city=[$5], age=[$8]) - LogicalFilter(condition=[>($8, 30)]) + LogicalFilter(condition=[>($8, SAFE_CAST(30:BIGINT))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state, city], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ip.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ip.json deleted file mode 100644 index bb62be7b990..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ip.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(host=[$0])\n LogicalFilter(condition=[GREATER_IP($0, IP('1.1.1.1':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], FILTER->GREATER_IP($0, IP('1.1.1.1':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"host\":{\"from\":\"1.1.1.1\",\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ip.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ip.yaml new file mode 100644 index 00000000000..7a5f6a58194 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ip.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(host=[$0]) + LogicalFilter(condition=[GREATER_IP($0, IP('1.1.1.1'))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], FILTER->GREATER_IP($0, IP('1.1.1.1')), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"host":{"from":"1.1.1.1","to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["host"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ipv6_swapped.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ipv6_swapped.json deleted file mode 100644 index 8ebbbbe885d..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ipv6_swapped.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(host=[$0])\n LogicalFilter(condition=[LTE_IP(IP('::ffff:1234':VARCHAR), $0)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], FILTER->LTE_IP(IP('::ffff:1234':VARCHAR), $0), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"host\":{\"from\":\"::ffff:1234\",\"to\":null,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ipv6_swapped.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ipv6_swapped.yaml new file mode 100644 index 00000000000..43c837202c5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ipv6_swapped.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(host=[$0]) + LogicalFilter(condition=[LTE_IP(IP('::ffff:1234'), $0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], FILTER->LTE_IP(IP('::ffff:1234'), $0), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"host":{"from":"::ffff:1234","to":null,"include_lower":true,"include_upper":true,"boost":1.0}}},"_source":{"includes":["host"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_function_script_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_function_script_push.yaml index f72238d95a0..2ca7953b4cb 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_function_script_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_function_script_push.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(firstname=[$1], age=[$8]) - LogicalFilter(condition=[AND(=(CHAR_LENGTH($1), 5), =(ABS($8), 32), =($3, 39225))]) + LogicalFilter(condition=[AND(=(CHAR_LENGTH($1), 5), =(ABS($8), SAFE_CAST(32:BIGINT)), =($3, SAFE_CAST(39225:BIGINT)))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, balance, age], SCRIPT->AND(=(CHAR_LENGTH($0), 5), =(ABS($2), 32), =($1, 39225)), PROJECT->[firstname, age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCGXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNIQVJfTEVOR1RIIiwKICAgICAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["firstname.keyword",5]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQB9nsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkFCUyIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["age",32]}},"boost":1.0}},{"term":{"balance":{"value":39225,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push.yaml index eb020cf0f92..c50d6154eab 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age=[$8]) - LogicalFilter(condition=[AND(SEARCH($8, Sarg[(30..40)]), >($3, 10000))]) + LogicalFilter(condition=[AND(>($8, SAFE_CAST(30:BIGINT)), <($8, SAFE_CAST(40:BIGINT)), >($3, SAFE_CAST(10000:BIGINT)))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[balance, age], FILTER->AND(SEARCH($1, Sarg[(30..40)]), >($0, 10000)), PROJECT->[age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"range":{"age":{"from":30.0,"to":40.0,"include_lower":false,"include_upper":false,"boost":1.0}}},{"range":{"balance":{"from":10000,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[balance, age], FILTER->AND(SEARCH($1, Sarg[(30L:BIGINT..40L:BIGINT)]:BIGINT), >($0, 10000)), PROJECT->[age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"range":{"age":{"from":30.0,"to":40.0,"include_lower":false,"include_upper":false,"boost":1.0}}},{"range":{"balance":{"from":10000,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_date_string.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_date_string.yaml index 7f604d806ee..4d4242d31eb 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_date_string.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_date_string.yaml @@ -1,8 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalFilter(condition=[AND(>($0, DATE('2016-12-08 00:00:00.123456789':VARCHAR)), <($0, DATE('2018-11-09 00:00:00.000000000':VARCHAR)))]) - LogicalProject(yyyy-MM-dd=[$83]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]]) + LogicalProject(yyyy-MM-dd=[$0]) + LogicalFilter(condition=[AND(>($0, DATE('2016-12-08 00:00:00.123456789')), <($0, DATE('2018-11-09 00:00:00.000000000')))]) + LogicalProject(yyyy-MM-dd=[$83]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]], PushDownContext=[[PROJECT->[yyyy-MM-dd], FILTER->SEARCH($0, Sarg[('2016-12-08':VARCHAR..'2018-11-09':VARCHAR)]:VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"yyyy-MM-dd":{"from":"2016-12-08","to":"2018-11-09","include_lower":false,"include_upper":false,"boost":1.0}}},"_source":{"includes":["yyyy-MM-dd"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]], PushDownContext=[[PROJECT->[yyyy-MM-dd], FILTER->SEARCH($0, Sarg[('2016-12-08':EXPR_DATE VARCHAR..'2018-11-09':EXPR_DATE VARCHAR)]:EXPR_DATE VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"yyyy-MM-dd":{"from":"2016-12-08","to":"2018-11-09","include_lower":false,"include_upper":false,"boost":1.0}}},"_source":{"includes":["yyyy-MM-dd"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_time_string.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_time_string.yaml index 475bc85ecb2..df3136fe3e8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_time_string.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_time_string.yaml @@ -1,8 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalFilter(condition=[AND(>($0, TIME('2016-12-08 12:00:00.123456789':VARCHAR)), <($0, TIME('2018-11-09 19:00:00.123456789':VARCHAR)))]) - LogicalProject(custom_time=[$49]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]]) + LogicalProject(custom_time=[$0]) + LogicalFilter(condition=[AND(>($0, TIME('2016-12-08 12:00:00.123456789')), <($0, TIME('2018-11-09 19:00:00.123456789')))]) + LogicalProject(custom_time=[$49]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]], PushDownContext=[[PROJECT->[custom_time], FILTER->SEARCH($0, Sarg[('12:00:00.123456789':VARCHAR..'19:00:00.123456789':VARCHAR)]:VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"custom_time":{"from":"12:00:00.123456789","to":"19:00:00.123456789","include_lower":false,"include_upper":false,"boost":1.0}}},"_source":{"includes":["custom_time"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]], PushDownContext=[[PROJECT->[custom_time], FILTER->SEARCH($0, Sarg[('12:00:00.123456789':EXPR_TIME VARCHAR..'19:00:00.123456789':EXPR_TIME VARCHAR)]:EXPR_TIME VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"custom_time":{"from":"12:00:00.123456789","to":"19:00:00.123456789","include_lower":false,"include_upper":false,"boost":1.0}}},"_source":{"includes":["custom_time"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_timestamp_string.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_timestamp_string.yaml index 954fab09d05..3dc548fc7cf 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_timestamp_string.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_timestamp_string.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) - LogicalFilter(condition=[AND(>($3, TIMESTAMP('2016-12-08 00:00:00.000000000':VARCHAR)), <($3, TIMESTAMP('2018-11-09 00:00:00.000000000':VARCHAR)))]) + LogicalFilter(condition=[AND(>($3, TIMESTAMP('2016-12-08 00:00:00.000000000')), <($3, TIMESTAMP('2018-11-09 00:00:00.000000000')))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], FILTER->SEARCH($3, Sarg[('2016-12-08 00:00:00':VARCHAR..'2018-11-09 00:00:00':VARCHAR)]:VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"birthdate":{"from":"2016-12-08T00:00:00.000Z","to":"2018-11-09T00:00:00.000Z","include_lower":false,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], FILTER->SEARCH($3, Sarg[('2016-12-08 00:00:00':EXPR_TIMESTAMP VARCHAR..'2018-11-09 00:00:00':EXPR_TIMESTAMP VARCHAR)]:EXPR_TIMESTAMP VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"birthdate":{"from":"2016-12-08T00:00:00.000Z","to":"2018-11-09T00:00:00.000Z","include_lower":false,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_ip_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_ip_push.yaml index a4e8e998ffd..64da01ac0fd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_ip_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_ip_push.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(host=[$0]) - LogicalFilter(condition=[CIDRMATCH($0, '0.0.0.0/24':VARCHAR)]) + LogicalFilter(condition=[CIDRMATCH($0, '0.0.0.0/24')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], SCRIPT->CIDRMATCH($0, '0.0.0.0/24':VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCRnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSURSTUFUQ0giLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ1ZHQiOiAiRVhQUl9JUCIsCiAgICAgICAgInR5cGUiOiAiT1RIRVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdLAogICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkJPT0xFQU4iLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["host","0.0.0.0/24"]}},"boost":1.0}},"_source":{"includes":["host"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], SCRIPT->CIDRMATCH($0, '0.0.0.0/24'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCRnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSURSTUFUQ0giLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ1ZHQiOiAiRVhQUl9JUCIsCiAgICAgICAgInR5cGUiOiAiT1RIRVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdLAogICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkJPT0xFQU4iLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["host","0.0.0.0/24"]}},"boost":1.0}},"_source":{"includes":["host"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml index 519377d3dde..929ed22b9c3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(firstname=[$1], age=[$8]) - LogicalFilter(condition=[AND(=($1, 'Amber'), =(-($8, 2), 30))]) + LogicalFilter(condition=[AND(=($1, 'Amber'), =(-($8, 2), SAFE_CAST(30:BIGINT)))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, age], SCRIPT->AND(=($0, 'Amber'), =(-($1, 2), 30)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"term":{"firstname.keyword":{"value":"Amber","boost":1.0}}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCwnsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIi0iLAogICAgICAgICJraW5kIjogIk1JTlVTIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["age",2,30]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_then_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_then_limit_push.yaml index 802867dfc5a..1a6c4bcfdc8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_then_limit_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_then_limit_push.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(fetch=[5]) - LogicalFilter(condition=[>($8, 30)]) + LogicalSort(fetch=[5]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[>($8, SAFE_CAST(30:BIGINT))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], FILTER->>($0, 30), LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_w_pushdown.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_w_pushdown.json deleted file mode 100644 index c99e6b323e2..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_w_pushdown.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(name=[$0], age=[$1])\n LogicalFilter(condition=[=($1, 20)])\n CalciteLogicalIndexScan(table=[[OpenSearch, test]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, test]], PushDownContext=[[PROJECT->[name, age], FILTER->=($1, 20), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"term\":{\"age\":{\"value\":20,\"boost\":1.0}}},\"_source\":{\"includes\":[\"name\",\"age\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search.yaml index b6dd067dc2e..9e7fc83ac54 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search.yaml @@ -4,7 +4,7 @@ calcite: LogicalProject(count()=[$1], span(birthdate,1d)=[$0]) LogicalAggregate(group=[{0}], count()=[COUNT()]) LogicalProject(span(birthdate,1d)=[SPAN($3, 1, 'd')]) - LogicalFilter(condition=[AND(>=($3, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($3, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) + LogicalFilter(condition=[AND(>=($3, TIMESTAMP('2023-01-01 00:00:00')), <($3, TIMESTAMP('2023-01-03 00:00:00')))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->SEARCH($0, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"birthdate":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1d)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->SEARCH($0, Sarg[['2023-01-01 00:00:00':EXPR_TIMESTAMP VARCHAR..'2023-01-03 00:00:00':EXPR_TIMESTAMP VARCHAR)]:EXPR_TIMESTAMP VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"birthdate":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1d)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search_call.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search_call.yaml new file mode 100644 index 00000000000..9e7fc83ac54 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search_call.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], span(birthdate,1d)=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(span(birthdate,1d)=[SPAN($3, 1, 'd')]) + LogicalFilter(condition=[AND(>=($3, TIMESTAMP('2023-01-01 00:00:00')), <($3, TIMESTAMP('2023-01-03 00:00:00')))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->SEARCH($0, Sarg[['2023-01-01 00:00:00':EXPR_TIMESTAMP VARCHAR..'2023-01-03 00:00:00':EXPR_TIMESTAMP VARCHAR)]:EXPR_TIMESTAMP VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"birthdate":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1d)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_wo_pushdown.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_wo_pushdown.json deleted file mode 100644 index 6771c06a5bf..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_wo_pushdown.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(name=[$0], age=[$1])\n LogicalFilter(condition=[=($1, 20)])\n CalciteLogicalIndexScan(table=[[OpenSearch, test]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..7=[{inputs}], expr#8=[20], expr#9=[=($t1, $t8)], proj#0..1=[{exprs}], $condition=[$t9])\n CalciteEnumerableIndexScan(table=[[OpenSearch, test]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_in_correlated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_in_correlated_subquery.yaml index 18cd0c5147a..6dfc8898764 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_in_correlated_subquery.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_in_correlated_subquery.yaml @@ -1,22 +1,23 @@ calcite: logical: | LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(id=[$2], name=[$0], salary=[$4]) - LogicalSort(sort0=[$4], dir0=[DESC-nulls-last]) + LogicalSort(sort0=[$2], dir0=[DESC-nulls-last]) + LogicalProject(id=[$2], name=[$0], salary=[$4]) LogicalFilter(condition=[IN($0, { LogicalProject(name=[$0]) - LogicalFilter(condition=[=($cor0.id, $1)]) - LogicalSystemLimit(fetch=[10000], type=[SUBSEARCH_MAXOUT]) - LogicalFilter(condition=[=($0, 'Tom')]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) - })], variablesSet=[[$cor0]]) + LogicalFilter(condition=[=($cor1.id, $1)]) + LogicalSort(fetch=[10000]) + LogicalProject(name=[$0], uid=[$1], occupation=[$2], department=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9]) + LogicalFilter(condition=[=($0, 'Tom')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + })], variablesSet=[[$cor1]]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) CalciteEnumerableTopK(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[=($t0, $t3)], proj#0..3=[{exprs}], $condition=[$t4]) - EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) + EnumerableCorrelate(correlation=[$cor1], joinType=[inner], requiredColumns=[{1}]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id, salary]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id","salary"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableAggregate(group=[{0}]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[$cor0], expr#3=[$t2.id], expr#4=[=($t3, $t1)], proj#0..1=[{exprs}], $condition=[$t4]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[name, uid], FILTER->=($0, 'Tom'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"term":{"name":{"value":"Tom","boost":1.0}}},"_source":{"includes":["name","uid"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[$cor1], expr#3=[$t2.id], expr#4=[=($t3, $t1)], proj#0..1=[{exprs}], $condition=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[name, uid], FILTER->=($0, 'Tom'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"term":{"name":{"value":"Tom","boost":1.0}}},"_source":{"includes":["name","uid"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_in_uncorrelated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_in_uncorrelated_subquery.yaml index ae5e866620b..6092891a4c1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_in_uncorrelated_subquery.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_in_uncorrelated_subquery.yaml @@ -1,17 +1,17 @@ calcite: logical: | LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(id=[$2], name=[$0], salary=[$4]) - LogicalSort(sort0=[$4], dir0=[DESC-nulls-last]) + LogicalSort(sort0=[$2], dir0=[DESC-nulls-last]) + LogicalProject(id=[$2], name=[$0], salary=[$4]) LogicalFilter(condition=[IN($2, { - LogicalSystemLimit(fetch=[10000], type=[SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[10000]) LogicalProject(uid=[$1]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) - })], variablesSet=[[$cor0]]) + })]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | - EnumerableCalc(expr#0..2=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) - CalciteEnumerableTopK(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000]) + CalciteEnumerableTopK(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) EnumerableHashJoin(condition=[=($1, $3)], joinType=[semi]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id, salary]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id","salary"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[uid], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["uid"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[uid], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["uid"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_isblank.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_isblank.yaml index d262a4af76d..957d5e8305b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_isblank.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_isblank.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[OR(IS NULL($1), IS EMPTY(TRIM(FLAG(BOTH), ' ', $1)))]) + LogicalFilter(condition=[OR(IS NULL($1), =(TRIM(FLAG(BOTH), ' ', $1), ''))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), IS EMPTY(TRIM(FLAG(BOTH), ' ', $1))), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQFHHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJPUiIsCiAgICAia2luZCI6ICJPUiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIE5VTEwiLAogICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJJUyBFTVBUWSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVIiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiVFJJTSIsCiAgICAgICAgICAgICJraW5kIjogIlRSSU0iLAogICAgICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICAgICAgfSwKICAgICAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICAgICAgewogICAgICAgICAgICAgICJsaXRlcmFsIjogIkJPVEgiLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiU1lNQk9MIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgICAgIH0KICAgICAgICAgICAgfQogICAgICAgICAgXQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,0],"DIGESTS":["firstname.keyword"," ","firstname.keyword"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), =(TRIM(FLAG(BOTH), ' ', $1), '')), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"should":[{"bool":{"must_not":[{"exists":{"field":"firstname","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDXnsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlRSSU0iLAogICAgICAgICJraW5kIjogIlRSSU0iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAibGl0ZXJhbCI6ICJCT1RIIiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJTWU1CT0wiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[2,0,2],"DIGESTS":[" ","firstname.keyword",""]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty.yaml index df97332ae98..36cdf0ff187 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[OR(IS NULL($1), IS EMPTY($1))]) + LogicalFilter(condition=[OR(IS NULL($1), =($1, ''))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), IS EMPTY($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQC13sKICAib3AiOiB7CiAgICAibmFtZSI6ICJPUiIsCiAgICAia2luZCI6ICJPUiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIE5VTEwiLAogICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJJUyBFTVBUWSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVIiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["firstname.keyword","firstname.keyword"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->SEARCH($1, Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"should":[{"terms":{"firstname.keyword":[""],"boost":1.0}},{"bool":{"must_not":[{"exists":{"field":"firstname","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty_or_others.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty_or_others.yaml index 06215911134..10c0b22b7ed 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty_or_others.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty_or_others.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[OR(=($4, 'M'), IS NULL($1), IS EMPTY($1))]) + LogicalFilter(condition=[OR(=($4, 'M'), IS NULL($1), =($1, ''))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), =($4, 'M'), IS EMPTY($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQEtnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJPUiIsCiAgICAia2luZCI6ICJPUiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIE5VTEwiLAogICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICI9IiwKICAgICAgICAia2luZCI6ICJFUVVBTFMiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfSwKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIEVNUFRZIiwKICAgICAgICAia2luZCI6ICJPVEhFUiIsCiAgICAgICAgInN5bnRheCI6ICJQT1NURklYIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDMsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,0,2,0],"DIGESTS":["firstname.keyword","gender.keyword","M","firstname.keyword"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->OR(=($4, 'M'), SEARCH($1, Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"should":[{"term":{"gender.keyword":{"value":"M","boost":1.0}}},{"bool":{"should":[{"terms":{"firstname.keyword":[""],"boost":1.0}},{"bool":{"must_not":[{"exists":{"field":"firstname","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_isnull_or_others.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_isnull_or_others.json deleted file mode 100644 index e4d43e1690e..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_isnull_or_others.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalFilter(condition=[OR(IS NULL($1), =($4, 'M'))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->OR(IS NULL($1), =($4, 'M')), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"should\":[{\"bool\":{\"must_not\":[{\"exists\":{\"field\":\"firstname\",\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"term\":{\"gender.keyword\":{\"value\":\"M\",\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_isnull_or_others.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_isnull_or_others.yaml new file mode 100644 index 00000000000..673ec496320 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_isnull_or_others.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[OR(IS NULL($1), =($4, 'M'))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->OR(IS NULL($1), =($4, 'M')), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"should":[{"bool":{"must_not":[{"exists":{"field":"firstname","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"term":{"gender.keyword":{"value":"M","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_criteria_max_option.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_criteria_max_option.yaml index 0c3f8565335..85c43e4637d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_criteria_max_option.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_criteria_max_option.yaml @@ -5,9 +5,9 @@ calcite: LogicalJoin(condition=[=($0, $13)], joinType=[inner]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[50000]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) - LogicalFilter(condition=[<=($13, 1)]) + LogicalFilter(condition=[<=($13, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _row_number_join_max_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_field_list_max_option.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_field_list_max_option.yaml new file mode 100644 index 00000000000..a6ad73cb6b5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_field_list_max_option.yaml @@ -0,0 +1,20 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$13], firstname=[$14], address=[$15], birthdate=[$16], gender=[$17], city=[$18], lastname=[$19], balance=[$20], employer=[$21], state=[$22], age=[$23], email=[$24], male=[$25]) + LogicalJoin(condition=[=($0, $13)], joinType=[inner]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSort(fetch=[50000]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + LogicalFilter(condition=[<=($13, SAFE_CAST(1:BIGINT))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _row_number_join_max_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableCalc(expr#0..13=[{inputs}], proj#0..12=[{exprs}]) + EnumerableLimit(fetch=[10000]) + EnumerableHashJoin(condition=[=($0, $13)], joinType=[inner]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->50000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"account_number":{"terms":{"field":"account_number","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_fields.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_fields.yaml index 12259a8e5ae..f28687f181c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_fields.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_fields.yaml @@ -5,7 +5,7 @@ calcite: LogicalJoin(condition=[=($0, $13)], joinType=[left]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[50000]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_fields_max_option.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_fields_max_option.yaml index 4b5597cc6b2..a6ad73cb6b5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_fields_max_option.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_fields_max_option.yaml @@ -5,9 +5,9 @@ calcite: LogicalJoin(condition=[=($0, $13)], joinType=[inner]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[50000]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) - LogicalFilter(condition=[<=($13, 1)]) + LogicalFilter(condition=[<=($13, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _row_number_join_max_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_5_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_5_push.yaml index dee2d92c7a3..19d30cfb035 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_5_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_5_push.yaml @@ -1,9 +1,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(fetch=[5]) + LogicalSort(fetch=[5]) + LogicalProject(age=[$8]) LogicalSort(fetch=[10]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->10, LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_filter_5_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_filter_5_push.yaml index 21f46d7f96a..4ab5718de6e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_filter_5_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_filter_5_push.yaml @@ -1,13 +1,14 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(fetch=[5]) - LogicalFilter(condition=[>($8, 30)]) + LogicalSort(fetch=[5]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[>($8, SAFE_CAST(30:BIGINT))]) LogicalSort(fetch=[10]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableLimit(fetch=[5]) - EnumerableCalc(expr#0=[{inputs}], expr#1=[30], expr#2=[>($t0, $t1)], age=[$t0], $condition=[$t2]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[30:BIGINT], expr#2=[>($t0, $t1)], age=[$t0], $condition=[$t2]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->10], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10from1_10from2_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10from1_10from2_push.yaml index e17b61abc78..7d587bcb13c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10from1_10from2_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10from1_10from2_push.yaml @@ -1,9 +1,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(offset=[2], fetch=[10]) + LogicalSort(offset=[2], fetch=[10]) + LogicalProject(age=[$8]) LogicalSort(offset=[1], fetch=[10]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->[10 from 1], LIMIT->[10 from 2], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":3,"size":8,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=8, pageSize=null, startFrom=3)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_5_10_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_5_10_push.yaml index 7b936024c74..7e0352131fc 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_5_10_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_5_10_push.yaml @@ -1,9 +1,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(fetch=[10]) + LogicalSort(fetch=[10]) + LogicalProject(age=[$8]) LogicalSort(fetch=[5]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5, LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown.json deleted file mode 100644 index 82c72695d0b..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown.yaml new file mode 100644 index 00000000000..215000ebf4b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], state=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown3.yaml new file mode 100644 index 00000000000..10e59b872bc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown3.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(offset=[10], fetch=[10]) + LogicalProject(count()=[$0], state=[$1]) + LogicalSort(fetch=[100]) + LogicalProject(count()=[$1], state=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(offset=[10], fetch=[10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":20,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=20, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.yaml index 4df04144f06..b3b9952ff35 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.yaml @@ -2,12 +2,13 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(offset=[10], fetch=[10]) - LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[100]) - LogicalProject(count()=[$1], state=[$0]) - LogicalAggregate(group=[{0}], count()=[COUNT()]) - LogicalProject(state=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(count()=[$0], state=[$1]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[100]) + LogicalProject(count()=[$1], state=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableLimit(offset=[10], fetch=[10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], SORT->[1 ASC FIRST], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":20,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=20, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], SORT->[1 ASC FIRST], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":20,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=20, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown5.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown5.json deleted file mode 100644 index 3b0fede88bf..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown5.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(offset=[10], fetch=[10])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[100])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown5.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown5.yaml new file mode 100644 index 00000000000..ab5d60b80af --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown5.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(offset=[10], fetch=[10]) + LogicalProject(count()=[$0], state=[$1]) + LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[100]) + LogicalProject(count()=[$1], state=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(offset=[10], fetch=[10]) + CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[100]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.yaml index bab348f5897..c689be7914a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.yaml @@ -2,12 +2,13 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(offset=[10], fetch=[10]) - LogicalSort(fetch=[100]) - LogicalProject(count()=[$1], state=[$0]) - LogicalAggregate(group=[{0}], count()=[COUNT()]) - LogicalProject(state=[$7]) - LogicalFilter(condition=[IS NOT NULL($7)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(count()=[$0], state=[$1]) + LogicalSort(fetch=[100]) + LogicalProject(count()=[$1], state=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(state=[$7]) + LogicalFilter(condition=[IS NOT NULL($7)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableLimit(offset=[10], fetch=[10]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.yaml index e4f66cf442d..8315f911269 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.yaml @@ -2,12 +2,13 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(offset=[10], fetch=[10]) - LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[100]) - LogicalProject(count()=[$1], state=[$0]) - LogicalAggregate(group=[{0}], count()=[COUNT()]) - LogicalProject(state=[$7]) - LogicalFilter(condition=[IS NOT NULL($7)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(count()=[$0], state=[$1]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[100]) + LogicalProject(count()=[$1], state=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(state=[$7]) + LogicalFilter(condition=[IS NOT NULL($7)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableLimit(offset=[10], fetch=[10]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_offsets_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_offsets_push.yaml index 1a5003d8a2b..4b815d5f9e0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_offsets_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_offsets_push.yaml @@ -1,9 +1,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(offset=[2], fetch=[5]) + LogicalSort(offset=[2], fetch=[5]) + LogicalProject(age=[$8]) LogicalSort(offset=[1], fetch=[10]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->[10 from 1], LIMIT->[5 from 2], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":3,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=3)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml index 187f948a6e4..485c1837d54 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml @@ -1,10 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(ageMinus=[$17]) - LogicalSort(fetch=[5]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], ageMinus=[-($8, 30)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalSort(fetch=[5]) + LogicalProject(ageMinus=[-($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0=[{inputs}], expr#1=[30], expr#2=[-($t0, $t1)], $f0=[$t2]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_filter_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_filter_push.yaml index aed8746d78b..ab587e8e156 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_filter_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_filter_push.yaml @@ -2,10 +2,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age=[$8]) - LogicalFilter(condition=[>($8, 30)]) + LogicalFilter(condition=[>($8, SAFE_CAST(30:BIGINT))]) LogicalSort(fetch=[5]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0=[{inputs}], expr#1=[30], expr#2=[>($t0, $t1)], age=[$t0], $condition=[$t2]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[30:BIGINT], expr#2=[>($t0, $t1)], age=[$t0], $condition=[$t2]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_sort_push.yaml index 7ab0399e85f..e2a7ad52687 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_sort_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_sort_push.yaml @@ -1,10 +1,11 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(sort0=[$8], dir0=[ASC-nulls-first]) + LogicalSort(sort0=[$0], dir0=[ASC-nulls-first]) + LogicalProject(age=[$8]) LogicalSort(fetch=[5]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5, SORT->[{ "age" : { diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_merge_join_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_merge_join_sort_push.yaml index 7c5f95fba77..eadbe1191b1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_merge_join_sort_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_merge_join_sort_push.yaml @@ -5,7 +5,7 @@ calcite: LogicalJoin(condition=[=($0, $13)], joinType=[inner]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[50000]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_min_max_agg_on_derived_field.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_min_max_agg_on_derived_field.yaml index c32f7d1a2d6..85dd4f618dd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_min_max_agg_on_derived_field.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_min_max_agg_on_derived_field.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalAggregate(group=[{}], MIN(balance2)=[MIN($0)], MAX(balance2)=[MAX($0)]) - LogicalProject(balance2=[CEIL(DIVIDE($3, 10000.0:DECIMAL(6, 1)))]) + LogicalProject($f0=[CEIL(DIVIDE($3, 10000.0:DECIMAL(6, 1)))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},MIN(balance2)=MIN($0),MAX(balance2)=MAX($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"MIN(balance2)":{"min":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDUnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDRUlMIiwKICAgICJraW5kIjogIkNFSUwiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkRJVklERSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJERUNJTUFMIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IDYsCiAgICAgICAgICAgICJzY2FsZSI6IDEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiREVDSU1BTCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogMjcsCiAgICAgICAgInNjYWxlIjogNwogICAgICB9LAogICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICJkeW5hbWljIjogZmFsc2UKICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",10000.0]}}}},"MAX(balance2)":{"max":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDUnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDRUlMIiwKICAgICJraW5kIjogIkNFSUwiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkRJVklERSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJERUNJTUFMIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IDYsCiAgICAgICAgICAgICJzY2FsZSI6IDEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiREVDSU1BTCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogMjcsCiAgICAgICAgInNjYWxlIjogNwogICAgICB9LAogICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICJkeW5hbWljIjogZmFsc2UKICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",10000.0]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},MIN(balance2)=MIN($0),MAX(balance2)=MAX($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"MIN(balance2)":{"min":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDFXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDRUlMIiwKICAgICJraW5kIjogIkNFSUwiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkRJVklERSIsCiAgICAgICAgImtpbmQiOiAiRElWSURFIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJERUNJTUFMIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAyNywKICAgICAgICAic2NhbGUiOiA3CiAgICAgIH0sCiAgICAgICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAgICAgImR5bmFtaWMiOiBmYWxzZQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",10000.0]}}}},"MAX(balance2)":{"max":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDFXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDRUlMIiwKICAgICJraW5kIjogIkNFSUwiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkRJVklERSIsCiAgICAgICAgImtpbmQiOiAiRElWSURFIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJERUNJTUFMIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAyNywKICAgICAgICAic2NhbGUiOiA3CiAgICAgIH0sCiAgICAgICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAgICAgImR5bmFtaWMiOiBmYWxzZQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",10000.0]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_fields_relevance_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_fields_relevance_push.json deleted file mode 100644 index 46216cff058..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_fields_relevance_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalFilter(condition=[simple_query_string(MAP('fields', MAP('name':VARCHAR, 4.0E0:DOUBLE, 'email':VARCHAR, 1.0E0:DOUBLE)), MAP('query', 'gmail':VARCHAR), MAP('default_operator', 'or':VARCHAR), MAP('analyzer', 'english':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->simple_query_string(MAP('fields', MAP('name':VARCHAR, 4.0E0:DOUBLE, 'email':VARCHAR, 1.0E0:DOUBLE)), MAP('query', 'gmail':VARCHAR), MAP('default_operator', 'or':VARCHAR), MAP('analyzer', 'english':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"simple_query_string\":{\"query\":\"gmail\",\"fields\":[\"name^4.0\",\"email^1.0\"],\"analyzer\":\"english\",\"flags\":-1,\"default_operator\":\"or\",\"analyze_wildcard\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_fields_relevance_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_fields_relevance_push.yaml new file mode 100644 index 00000000000..e8eb753aaee --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_fields_relevance_push.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[simple_query_string(MAP('fields':VARCHAR, MAP('name':VARCHAR, 4.0E0:DOUBLE, 'email':VARCHAR, 1.0E0:DOUBLE)), MAP('query':VARCHAR, 'gmail':VARCHAR), MAP('default_operator':VARCHAR, 'or':VARCHAR), MAP('analyzer':VARCHAR, 'english':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->simple_query_string(MAP('fields':VARCHAR, MAP('name':VARCHAR, 4.0E0:DOUBLE, 'email':VARCHAR, 1.0E0:DOUBLE)), MAP('query':VARCHAR, 'gmail':VARCHAR), MAP('default_operator':VARCHAR, 'or':VARCHAR), MAP('analyzer':VARCHAR, 'english':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"simple_query_string":{"query":"gmail","fields":["name^4.0","email^1.0"],"analyzer":"english","flags":-1,"default_operator":"or","analyze_wildcard":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_sort_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_sort_push.json deleted file mode 100644 index c2fa77194f5..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_sort_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$3], sort1=[$4], sort2=[$0], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4])\n LogicalSort(sort0=[$3], sort1=[$4], sort2=[$0], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first])\n LogicalSort(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], dir2=[ASC-nulls-first], dir3=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender], SORT->[{\n \"balance\" : {\n \"order\" : \"desc\",\n \"missing\" : \"_last\"\n }\n}, {\n \"gender.keyword\" : {\n \"order\" : \"desc\",\n \"missing\" : \"_last\"\n }\n}, {\n \"account_number\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\"],\"excludes\":[]},\"sort\":[{\"balance\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"gender.keyword\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"account_number\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_sort_push.yaml new file mode 100644 index 00000000000..7782dba1894 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_sort_push.yaml @@ -0,0 +1,26 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$3], sort1=[$4], sort2=[$0], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$3], sort1=[$4], sort2=[$0], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4]) + LogicalSort(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], dir2=[ASC-nulls-first], dir3=[ASC-nulls-first]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender], SORT->[{ + "balance" : { + "order" : "desc", + "missing" : "_last" + } + }, { + "gender.keyword" : { + "order" : "desc", + "missing" : "_last" + } + }, { + "account_number" : { + "order" : "asc", + "missing" : "_first" + } + }]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender"],"excludes":[]},"sort":[{"balance":{"order":"desc","missing":"_last"}},{"gender.keyword":{"order":"desc","missing":"_last"}},{"account_number":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multisearch_basic.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multisearch_basic.yaml index 8fe5241ced4..381b76384e7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_multisearch_basic.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multisearch_basic.yaml @@ -5,18 +5,18 @@ calcite: LogicalAggregate(group=[{0}], count=[COUNT()]) LogicalProject(age_group=[$11]) LogicalUnion(all=[true]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], age_group=['young':VARCHAR]) - LogicalFilter(condition=[<($8, 30)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], age_group=['young']) + LogicalFilter(condition=[<($8, SAFE_CAST(30:BIGINT))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], age_group=['adult':VARCHAR]) - LogicalFilter(condition=[>=($8, 30)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], age_group=['adult']) + LogicalFilter(condition=[>=($8, SAFE_CAST(30:BIGINT))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..1=[{inputs}], count=[$t1], age_group=[$t0]) EnumerableAggregate(group=[{0}], count=[COUNT()]) EnumerableUnion(all=[true]) - EnumerableCalc(expr#0=[{inputs}], expr#1=['young':VARCHAR], age_group=[$t1]) + EnumerableCalc(expr#0=[{inputs}], expr#1=['young'], age_group=[$t1]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], FILTER-><($0, 30)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"range":{"age":{"from":null,"to":30,"include_lower":true,"include_upper":false,"boost":1.0}}},"_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0=[{inputs}], expr#1=['adult':VARCHAR], age_group=[$t1]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], FILTER->>=($0, 30)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":true,"include_upper":true,"boost":1.0}}},"_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0=[{inputs}], expr#1=['adult'], age_group=[$t1]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], FILTER->>=($0, 30)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":true,"include_upper":true,"boost":1.0}}},"_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multisearch_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multisearch_timestamp.yaml index 92358db2bf8..a8e30b6ff28 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_multisearch_timestamp.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multisearch_timestamp.yaml @@ -2,13 +2,14 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[DESC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC], fetch=[5]) - LogicalUnion(all=[true]) - LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) - LogicalFilter(condition=[SEARCH($1, Sarg['A':VARCHAR, 'B':VARCHAR]:VARCHAR)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) - LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) - LogicalFilter(condition=[SEARCH($1, Sarg['E':VARCHAR, 'F':VARCHAR]:VARCHAR)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data2]]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) + LogicalUnion(all=[true]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) + LogicalFilter(condition=[OR(=($1, 'A'), =($1, 'B'))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) + LogicalFilter(condition=[OR(=($1, 'E'), =($1, 'F'))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data2]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableLimit(fetch=[5]) @@ -24,4 +25,4 @@ calcite: "order" : "desc", "missing" : "_first" } - }], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"terms":{"category":["E","F"],"boost":1.0}},"_source":{"includes":["@timestamp","category","value","timestamp"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_first"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"terms":{"category":["E","F"],"boost":1.0}},"_source":{"includes":["@timestamp","category","value","timestamp"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_first"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_mvjoin.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_mvjoin.json deleted file mode 100644 index a539122e998..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_mvjoin.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(fetch=[1])\n LogicalProject(result=[ARRAY_JOIN(array('a', 'b', 'c'), ',')])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0..16=[{inputs}], expr#17=['a'], expr#18=['b'], expr#19=['c'], expr#20=[array($t17, $t18, $t19)], expr#21=[','], expr#22=[ARRAY_JOIN($t20, $t21)], result=[$t22])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->1, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":1,\"timeout\":\"1m\"}, requestedTotalSize=1, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_mvjoin.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_mvjoin.yaml new file mode 100644 index 00000000000..1386211d690 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_mvjoin.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(fetch=[1]) + LogicalProject(result=[ARRAY_JOIN(array('a':VARCHAR, 'b':VARCHAR, 'c':VARCHAR), ',')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['a':VARCHAR], expr#18=['b':VARCHAR], expr#19=['c':VARCHAR], expr#20=[array($t17, $t18, $t19)], expr#21=[','], expr#22=[ARRAY_JOIN($t20, $t21)], result=[$t22]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->1, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":1,"timeout":"1m"}, requestedTotalSize=1, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml index 73a3fefe705..e487a622561 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml @@ -2,19 +2,19 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age2=[$2]) - LogicalFilter(condition=[<=($3, 1)]) + LogicalFilter(condition=[<=($3, SAFE_CAST(1:BIGINT))]) LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2)]) LogicalFilter(condition=[IS NOT NULL($2)]) - LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]) - LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) - LogicalProject(avg_age=[$2], state=[$0], city=[$1]) - LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) - LogicalProject(state=[$7], city=[$5], age=[$8]) - LogicalFilter(condition=[>($8, 30)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) + LogicalProject(avg_age=[$2], state=[$0], age2=[+($2, 2)]) + LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) + LogicalProject(state=[$7], city=[$5], age=[$8]) + LogicalFilter(condition=[>($8, SAFE_CAST(30:BIGINT))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[<=($t1, $t2)], age2=[$t0], $condition=[$t3]) - EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], age2=[$t2], $condition=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), SORT->[0 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], age2=[$t0]) + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1:BIGINT], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3]) + EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], age2=[$t2], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), SORT->[1 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_paginating_agg_for_join.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_paginating_agg_for_join.yaml new file mode 100644 index 00000000000..83eb7a68f9a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_paginating_agg_for_join.yaml @@ -0,0 +1,21 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(c=[$2], state=[$3]) + LogicalJoin(condition=[=($1, $3)], joinType=[inner]) + LogicalSort(fetch=[10]) + LogicalProject(c=[$1], state=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()]) + LogicalProject(state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalSort(fetch=[50000]) + LogicalProject(c=[$1], state=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()]) + LogicalProject(state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], c=[$t1], state=[$t2]) + EnumerableHashJoin(condition=[=($0, $2)], joinType=[inner]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0}), LIMIT->10], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}}]}}}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state], LIMIT->50000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}}]}}}}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_partial_filter_isnull.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_partial_filter_isnull.json deleted file mode 100644 index 52c53337d61..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_partial_filter_isnull.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(name=[$0], address=[$1], id=[$6], age=[$7])\n LogicalFilter(condition=[AND(IS NULL($1), =($0, 'david'))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t1)], proj#0..3=[{exprs}], $condition=[$t4])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]], PushDownContext=[[PROJECT->[name, address, id, age], FILTER->=($0, 'david')], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"term\":{\"name.keyword\":{\"value\":\"david\",\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"name\",\"address\",\"id\",\"age\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_partial_filter_isnull.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_partial_filter_isnull.yaml new file mode 100644 index 00000000000..45d02f6d7a0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_partial_filter_isnull.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(name=[$0], address=[$1], id=[$6], age=[$7]) + LogicalFilter(condition=[AND(IS NULL($1), =($0, 'david'))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t1)], proj#0..3=[{exprs}], $condition=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]], PushDownContext=[[PROJECT->[name, address, id, age], FILTER->=($0, 'david')], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"name.keyword":{"value":"david","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["name","address","id","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern.yaml index 5b6fdebfd14..38f514b13f5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], patterns_field=[CASE(SEARCH($9, Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR), '':VARCHAR, REGEXP_REPLACE($9, '[a-zA-Z0-9]+':VARCHAR, '<*>':VARCHAR))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], patterns_field=[CASE(SEARCH($9, Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR), '':VARCHAR, REGEXP_REPLACE($9, '[a-zA-Z0-9]+', '<*>'))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..10=[{inputs}], expr#11=[Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR], expr#12=[SEARCH($t9, $t11)], expr#13=['':VARCHAR], expr#14=['[a-zA-Z0-9]+':VARCHAR], expr#15=['<*>':VARCHAR], expr#16=[REGEXP_REPLACE($t9, $t14, $t15)], expr#17=[CASE($t12, $t13, $t16)], proj#0..10=[{exprs}], $f11=[$t17]) + EnumerableCalc(expr#0..10=[{inputs}], expr#11=[Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR], expr#12=[SEARCH($t9, $t11)], expr#13=['':VARCHAR], expr#14=['[a-zA-Z0-9]+'], expr#15=['<*>'], expr#16=[REGEXP_REPLACE($t9, $t14, $t15)], expr#17=[CASE($t12, $t13, $t16)], proj#0..10=[{exprs}], $f11=[$t17]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml index b2185274499..18febc84793 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml @@ -1,10 +1,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'pattern'))], pattern_count=[$1], tokens=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'tokens'))], sample_logs=[$2]) - LogicalAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0, $2)]) - LogicalProject(email=[$9], patterns_field=[CASE(SEARCH($9, Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR), '':VARCHAR, REGEXP_REPLACE($9, '[a-zA-Z0-9]+':VARCHAR, '<*>':VARCHAR))], $f18=[10]) + LogicalProject(patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $1), 'pattern'))], pattern_count=[$2], tokens=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $1), 'tokens'))], sample_logs=[$1]) + LogicalAggregate(group=[{0}], sample_logs=[TAKE($1, $2)], pattern_count=[COUNT($0)]) + LogicalProject($f0=[CASE(SEARCH($9, Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR), '':VARCHAR, REGEXP_REPLACE($9, '[a-zA-Z0-9]+', '<*>'))], email=[$9], $f2=[10]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[PATTERN_PARSER($t0, $t2)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], expr#7=['tokens'], expr#8=[ITEM($t3, $t7)], expr#9=[SAFE_CAST($t8)], patterns_field=[$t6], pattern_count=[$t1], tokens=[$t9], sample_logs=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},pattern_count=COUNT($1),sample_logs=TAKE($0, $2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"patterns_field":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQH2nsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiT1IiLAogICAgICAgICJraW5kIjogIk9SIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiSVMgTlVMTCIsCiAgICAgICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICAgICAic3ludGF4IjogIlBPU1RGSVgiCiAgICAgICAgICB9LAogICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiPSIsCiAgICAgICAgICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAgICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICAgICAgfSwKICAgICAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICAgICAgewogICAgICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAzLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVHRVhQX1JFUExBQ0UiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDQsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDUsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDYsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,0,2,2,0,2,2],"DIGESTS":["email.keyword","email.keyword","","","email.keyword","[a-zA-Z0-9]+","<*>"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"pattern_count":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQH2nsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiT1IiLAogICAgICAgICJraW5kIjogIk9SIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiSVMgTlVMTCIsCiAgICAgICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICAgICAic3ludGF4IjogIlBPU1RGSVgiCiAgICAgICAgICB9LAogICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiPSIsCiAgICAgICAgICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAgICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICAgICAgfSwKICAgICAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICAgICAgewogICAgICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAzLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVHRVhQX1JFUExBQ0UiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDQsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDUsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDYsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,0,2,2,0,2,2],"DIGESTS":["email.keyword","email.keyword","","","email.keyword","[a-zA-Z0-9]+","<*>"]}}}},"sample_logs":{"top_hits":{"from":0,"size":10,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"email.keyword"}]}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[PATTERN_PARSER($t0, $t1)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], expr#7=['tokens'], expr#8=[ITEM($t3, $t7)], expr#9=[SAFE_CAST($t8)], patterns_field=[$t6], pattern_count=[$t2], tokens=[$t9], sample_logs=[$t1]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sample_logs=TAKE($1, $2),pattern_count=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"$f0":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQH2nsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiT1IiLAogICAgICAgICJraW5kIjogIk9SIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiSVMgTlVMTCIsCiAgICAgICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICAgICAic3ludGF4IjogIlBPU1RGSVgiCiAgICAgICAgICB9LAogICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiPSIsCiAgICAgICAgICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAgICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICAgICAgfSwKICAgICAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICAgICAgewogICAgICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAzLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVHRVhQX1JFUExBQ0UiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDQsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDUsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDYsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,0,2,2,0,2,2],"DIGESTS":["email.keyword","email.keyword","","","email.keyword","[a-zA-Z0-9]+","<*>"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sample_logs":{"top_hits":{"from":0,"size":10,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"email.keyword"}]}},"pattern_count":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQH2nsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiT1IiLAogICAgICAgICJraW5kIjogIk9SIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiSVMgTlVMTCIsCiAgICAgICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICAgICAic3ludGF4IjogIlBPU1RGSVgiCiAgICAgICAgICB9LAogICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiPSIsCiAgICAgICAgICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAgICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICAgICAgfSwKICAgICAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICAgICAgewogICAgICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAzLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVHRVhQX1JFUExBQ0UiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDQsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDUsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDYsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,0,2,2,0,2,2],"DIGESTS":["email.keyword","email.keyword","","","email.keyword","[a-zA-Z0-9]+","<*>"]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_percentile.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_percentile.json deleted file mode 100644 index ffe6fae5517..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_percentile.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], p50=[percentile_approx($0, $1, $2)], p90=[percentile_approx($0, $3, $2)])\n LogicalProject(balance=[$3], $f2=[50], $f3=[FLAG(BIGINT)], $f4=[90])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},p50=percentile_approx($0, $1, $2),p90=percentile_approx($0, $3, $2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"p50\":{\"percentiles\":{\"field\":\"balance\",\"percents\":[50.0],\"keyed\":true,\"tdigest\":{\"compression\":100.0}}},\"p90\":{\"percentiles\":{\"field\":\"balance\",\"percents\":[90.0],\"keyed\":true,\"tdigest\":{\"compression\":100.0}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_percentile.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_percentile.yaml new file mode 100644 index 00000000000..8041947b00b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_percentile.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], p50=[percentile_approx($0, $1, $2)], p90=[percentile_approx($0, $3, $2)]) + LogicalProject(balance=[$3], $f1=[50], $f2=[FLAG(BIGINT)], $f3=[90]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},p50=percentile_approx($0, $1, $2),p90=percentile_approx($0, $3, $2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"p50":{"percentiles":{"field":"balance","percents":[50.0],"keyed":true,"tdigest":{"compression":100.0}}},"p90":{"percentiles":{"field":"balance","percents":[90.0],"keyed":true,"tdigest":{"compression":100.0}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_prevent_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_prevent_limit_push.yaml index e7019b44d7d..fd794d44307 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_prevent_limit_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_prevent_limit_push.yaml @@ -1,10 +1,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalSort(offset=[1], fetch=[1]) + LogicalSort(offset=[1], fetch=[1]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableLimit(offset=[1], fetch=[1]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_pushdown_limit_into_aggregation.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_pushdown_limit_into_aggregation.yaml new file mode 100644 index 00000000000..10e59b872bc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_pushdown_limit_into_aggregation.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(offset=[10], fetch=[10]) + LogicalProject(count()=[$0], state=[$1]) + LogicalSort(fetch=[100]) + LogicalProject(count()=[$1], state=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(offset=[10], fetch=[10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":20,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=20, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_rare_usenull_false.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_rare_usenull_false.yaml index 9c5157c72bc..b9c455cdfa1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_rare_usenull_false.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_rare_usenull_false.yaml @@ -2,11 +2,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(gender=[$0], state=[$1], count=[$2]) - LogicalFilter(condition=[<=($3, 2)]) - LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2)]) + LogicalFilter(condition=[<=($3, SAFE_CAST(2:BIGINT))]) + LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 NULLS LAST)]) LogicalAggregate(group=[{0, 1}], count=[COUNT()]) LogicalProject(gender=[$4], state=[$7]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count=COUNT()), RARE_TOP->rare 2 state by gender, PROJECT->[gender, state, count], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender":{"terms":{"field":"gender.keyword","size":10000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]},"aggregations":{"state":{"terms":{"field":"state.keyword","size":2,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"asc"},{"_key":"asc"}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count=COUNT()), RARE_TOP->rare 2 state by gender, PROJECT->[gender, state, count], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender":{"terms":{"field":"gender.keyword","size":10000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]},"aggregations":{"state":{"terms":{"field":"state.keyword","size":2,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"asc"},{"_key":"asc"}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_rare_usenull_true.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_rare_usenull_true.yaml index 7bc4aa1e21c..3cdce2dd92a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_rare_usenull_true.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_rare_usenull_true.yaml @@ -2,13 +2,13 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(gender=[$0], state=[$1], count=[$2]) - LogicalFilter(condition=[<=($3, 2)]) - LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2)]) + LogicalFilter(condition=[<=($3, SAFE_CAST(2:BIGINT))]) + LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 NULLS LAST)]) LogicalAggregate(group=[{0, 1}], count=[COUNT()]) LogicalProject(gender=[$4], state=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[2], expr#5=[<=($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[2:BIGINT], expr#5=[<=($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) EnumerableWindow(window#0=[window(partition {0} order by [2] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_regex.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_regex.yaml index 5d76ec9ab64..c5e8221a336 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_regex.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_regex.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalSort(fetch=[5]) - LogicalFilter(condition=[REGEXP_CONTAINS($10, '^[A-Z][a-z]+$':VARCHAR)]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[REGEXP_CONTAINS($10, '^[A-Z][a-z]+$')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->REGEXP_CONTAINS($10, '^[A-Z][a-z]+$':VARCHAR), LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBkXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfQ09OVEFJTlMiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["lastname.keyword","^[A-Z][a-z]+$"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->REGEXP_CONTAINS($10, '^[A-Z][a-z]+$'), LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBkXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfQ09OVEFJTlMiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["lastname.keyword","^[A-Z][a-z]+$"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_regex_negated.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_regex_negated.yaml index 77cedbc871d..ce258196959 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_regex_negated.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_regex_negated.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalSort(fetch=[5]) - LogicalFilter(condition=[NOT(REGEXP_CONTAINS($10, '.*son$':VARCHAR))]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[NOT(REGEXP_CONTAINS($10, '.*son$'))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->NOT(REGEXP_CONTAINS($10, '.*son$':VARCHAR)), LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCWXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJOT1QiLAogICAgImtpbmQiOiAiTk9UIiwKICAgICJzeW50YXgiOiAiUFJFRklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVHRVhQX0NPTlRBSU5TIiwKICAgICAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["lastname.keyword",".*son$"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->NOT(REGEXP_CONTAINS($10, '.*son$')), LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCWXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJOT1QiLAogICAgImtpbmQiOiAiTk9UIiwKICAgICJzeW50YXgiOiAiUFJFRklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVHRVhQX0NPTlRBSU5TIiwKICAgICAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["lastname.keyword",".*son$"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_regexp_match_in_eval.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_regexp_match_in_eval.json deleted file mode 100644 index cbd608971fd..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_regexp_match_in_eval.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(has_hello=[REGEXP_CONTAINS($0, 'hello':VARCHAR)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_strings]])\n", - "physical": "EnumerableCalc(expr#0=[{inputs}], expr#1=['hello':VARCHAR], expr#2=[REGEXP_CONTAINS($t0, $t1)], $f0=[$t2])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_strings]], PushDownContext=[[PROJECT->[name], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"name\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_regexp_match_in_eval.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_regexp_match_in_eval.yaml new file mode 100644 index 00000000000..eee0ffc90a3 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_regexp_match_in_eval.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(has_hello=[REGEXP_CONTAINS($0, 'hello')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_strings]]) + physical: | + EnumerableCalc(expr#0=[{inputs}], expr#1=['hello'], expr#2=[REGEXP_CONTAINS($t0, $t1)], $f0=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_strings]], PushDownContext=[[PROJECT->[name], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["name"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_regexp_match_in_where.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_regexp_match_in_where.yaml index 664719d5411..7fd4347ef6b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_regexp_match_in_where.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_regexp_match_in_where.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(name=[$0]) - LogicalFilter(condition=[REGEXP_CONTAINS($0, 'hello':VARCHAR)]) + LogicalFilter(condition=[REGEXP_CONTAINS($0, 'hello')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_strings]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_strings]], PushDownContext=[[PROJECT->[name], SCRIPT->REGEXP_CONTAINS($0, 'hello':VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBkXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfQ09OVEFJTlMiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2],"DIGESTS":["name","hello"]}},"boost":1.0}},"_source":{"includes":["name"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_strings]], PushDownContext=[[PROJECT->[name], SCRIPT->REGEXP_CONTAINS($0, 'hello'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBkXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfQ09OVEFJTlMiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2],"DIGESTS":["name","hello"]}},"boost":1.0}},"_source":{"includes":["name"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_replace_command.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_replace_command.yaml index a867c569168..0205f6dc442 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_replace_command.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_replace_command.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(state=[REPLACE($7, 'IL':VARCHAR, 'Illinois':VARCHAR)]) + LogicalProject(state=[REPLACE($7, 'IL', 'Illinois')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0=[{inputs}], expr#1=['IL':VARCHAR], expr#2=['Illinois':VARCHAR], expr#3=[REPLACE($t0, $t1, $t2)], $f0=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["state"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0=[{inputs}], expr#1=['IL'], expr#2=['Illinois'], expr#3=[REPLACE($t0, $t1, $t2)], $f0=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["state"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_replace_wildcard.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_replace_wildcard.yaml index 0407849a472..d274087b5f9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_replace_wildcard.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_replace_wildcard.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(state=[REGEXP_REPLACE($7, '^\Q\E(.*?)\QL\E$':VARCHAR, 'STATE_IL':VARCHAR)]) + LogicalProject(state=[REGEXP_REPLACE($7, '^\Q\E(.*?)\QL\E$', 'STATE_IL')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0=[{inputs}], expr#1=['^\Q\E(.*?)\QL\E$':VARCHAR], expr#2=['STATE_IL':VARCHAR], expr#3=[REGEXP_REPLACE($t0, $t1, $t2)], $f0=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["state"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0=[{inputs}], expr#1=['^\Q\E(.*?)\QL\E$'], expr#2=['STATE_IL'], expr#3=[REGEXP_REPLACE($t0, $t1, $t2)], $f0=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["state"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_rex.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_rex.yaml index d420fca0baf..d92da85be2b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_rex.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_rex.yaml @@ -1,10 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], initial=[$17]) - LogicalSort(fetch=[5]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], initial=[REX_EXTRACT($10, '(?^[A-Z])', 'initial')]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], initial=[REX_EXTRACT($10, '(?^[A-Z])', 'initial')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..10=[{inputs}], expr#11=['(?^[A-Z])'], expr#12=['initial'], expr#13=[REX_EXTRACT($t10, $t11, $t12)], proj#0..10=[{exprs}], $f11=[$t13]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_multi_range.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_multi_range.json deleted file mode 100644 index b9b6fc9c2b7..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_multi_range.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalFilter(condition=[SEARCH($8, Sarg[0, [1..10], (20..30)])])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], FILTER->SEARCH($0, Sarg[0, [1..10], (20..30)]), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"should\":[{\"term\":{\"age\":{\"value\":0.0,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":1.0,\"to\":10.0,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":20.0,\"to\":30.0,\"include_lower\":false,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_multi_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_multi_range.yaml new file mode 100644 index 00000000000..2b65c7dce4c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_multi_range.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[OR(=($8, SAFE_CAST(0:BIGINT)), AND(>=($8, SAFE_CAST(1:BIGINT)), <=($8, SAFE_CAST(10:BIGINT))), AND(>($8, SAFE_CAST(20:BIGINT)), <($8, SAFE_CAST(30:BIGINT))))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], FILTER->SEARCH($0, Sarg[0L:BIGINT, [1L:BIGINT..10L:BIGINT], (20L:BIGINT..30L:BIGINT)]:BIGINT), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"should":[{"term":{"age":{"value":0.0,"boost":1.0}}},{"range":{"age":{"from":1.0,"to":10.0,"include_lower":true,"include_upper":true,"boost":1.0}}},{"range":{"age":{"from":20.0,"to":30.0,"include_lower":false,"include_upper":false,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_single_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_single_range.yaml index f6debc48fcc..5b8236876b2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_single_range.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_single_range.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age=[$8]) - LogicalFilter(condition=[SEARCH($8, Sarg[[1.0:DECIMAL(11, 1)..10:DECIMAL(11, 1))]:DECIMAL(11, 1))]) + LogicalFilter(condition=[AND(>=(SAFE_CAST($8), SAFE_CAST(1.0:DECIMAL(20, 1))), <(SAFE_CAST($8), SAFE_CAST(10.0:DECIMAL(20, 1))))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], FILTER->SEARCH($0, Sarg[[1.0:DECIMAL(11, 1)..10:DECIMAL(11, 1))]:DECIMAL(11, 1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"age":{"from":1.0,"to":10.0,"include_lower":true,"include_upper":false,"boost":1.0}}},"_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], SCRIPT->SEARCH(SAFE_CAST($0), Sarg[[1.0:DECIMAL(20, 1)..10.0:DECIMAL(20, 1))]:DECIMAL(20, 1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQGhnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJBTkQiLAogICAgImtpbmQiOiAiQU5EIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiPD0iLAogICAgICAgICJraW5kIjogIkxFU1NfVEhBTl9PUl9FUVVBTCIsCiAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJET1VCTEUiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAib3AiOiB7CiAgICAgICAgICAgICJuYW1lIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgICAgICJraW5kIjogIlNBRkVfQ0FTVCIsCiAgICAgICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9CiAgICAgICAgICBdLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkRFQ0lNQUwiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMjAsCiAgICAgICAgICAgICJzY2FsZSI6IDEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICI8IiwKICAgICAgICAia2luZCI6ICJMRVNTX1RIQU4iLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgIm9wIjogewogICAgICAgICAgICAibmFtZSI6ICJTQUZFX0NBU1QiLAogICAgICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgICAgICB9LAogICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgICAgIH0KICAgICAgICAgICAgfQogICAgICAgICAgXSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJERUNJTUFMIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IDIwLAogICAgICAgICAgICAic2NhbGUiOiAxCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMywKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJET1VCTEUiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[2,0,0,2],"DIGESTS":[1.0,"age","age",10.0]}},"boost":1.0}},"_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_time_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_time_range.yaml index cfb07502429..953b97de4b3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_time_range.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_time_range.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) - LogicalFilter(condition=[AND(>=($3, TIMESTAMP('2016-12-08 00:00:00.000000000':VARCHAR)), <($3, TIMESTAMP('2018-11-09 00:00:00.000000000':VARCHAR)))]) + LogicalFilter(condition=[AND(>=($3, TIMESTAMP('2016-12-08 00:00:00.000000000')), <($3, TIMESTAMP('2018-11-09 00:00:00.000000000')))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], FILTER->SEARCH($3, Sarg[['2016-12-08 00:00:00':VARCHAR..'2018-11-09 00:00:00':VARCHAR)]:VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"birthdate":{"from":"2016-12-08T00:00:00.000Z","to":"2018-11-09T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], FILTER->SEARCH($3, Sarg[['2016-12-08 00:00:00':EXPR_TIMESTAMP VARCHAR..'2018-11-09 00:00:00':EXPR_TIMESTAMP VARCHAR)]:EXPR_TIMESTAMP VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"birthdate":{"from":"2016-12-08T00:00:00.000Z","to":"2018-11-09T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_select.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_select.yaml index 027e7e40c43..da2ef173bbd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_select.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_select.yaml @@ -1,12 +1,13 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(variablesSet=[[$cor0]], id=[$2], name=[$0], count_dept=[$SCALAR_QUERY({ + LogicalProject(variablesSet=[[$cor1]], id=[$2], name=[$0], count_dept=[$SCALAR_QUERY({ LogicalAggregate(group=[{}], count(name)=[COUNT($0)]) LogicalProject(name=[$0]) LogicalFilter(condition=[IS NOT NULL($0)]) - LogicalFilter(condition=[=($cor0.id, $1)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + LogicalProject(name=[$0], uid=[$1], occupation=[$2], department=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9]) + LogicalFilter(condition=[=($cor1.id, $1)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) })]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | @@ -14,7 +15,7 @@ calcite: EnumerableLimit(fetch=[10000]) EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[left]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], expr#4=[0], expr#5=[CASE($t3, $t2, $t4)], uid=[$t0], count(name)=[$t5]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], expr#4=[0], expr#5=[CASE($t3, $t2, $t4)], uid10=[$t0], count(name)=[$t5]) EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0})], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"id":{"terms":{"field":"id","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},count(name)=COUNT($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"uid","boost":1.0}},{"exists":{"field":"name","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"uid":{"terms":{"field":"uid","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(name)":{"value_count":{"field":"name"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_where.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_where.yaml index 42e5e196667..dc2493d2a02 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_where.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_where.yaml @@ -5,9 +5,9 @@ calcite: LogicalFilter(condition=[=($2, $SCALAR_QUERY({ LogicalAggregate(group=[{}], max(uid)=[MAX($0)]) LogicalProject(uid=[$1]) - LogicalFilter(condition=[=($cor0.id, $1)]) + LogicalFilter(condition=[=($cor1.id, $1)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) - }))], variablesSet=[[$cor0]]) + }))], variablesSet=[[$cor1]]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | EnumerableLimit(fetch=[10000]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_subquery_select.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_subquery_select.yaml new file mode 100644 index 00000000000..da2ef173bbd --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_subquery_select.yaml @@ -0,0 +1,21 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(variablesSet=[[$cor1]], id=[$2], name=[$0], count_dept=[$SCALAR_QUERY({ + LogicalAggregate(group=[{}], count(name)=[COUNT($0)]) + LogicalProject(name=[$0]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(name=[$0], uid=[$1], occupation=[$2], department=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9]) + LogicalFilter(condition=[=($cor1.id, $1)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + })]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + physical: | + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[0:BIGINT], expr#6=[CASE($t4, $t5, $t3)], id=[$t1], name=[$t0], count_dept=[$t6]) + EnumerableLimit(fetch=[10000]) + EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[left]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], expr#4=[0], expr#5=[CASE($t3, $t2, $t4)], uid10=[$t0], count(name)=[$t5]) + EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0})], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"id":{"terms":{"field":"id","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},count(name)=COUNT($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"uid","boost":1.0}},{"exists":{"field":"name","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"uid":{"terms":{"field":"uid","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(name)":{"value_count":{"field":"name"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_subquery_where.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_subquery_where.yaml new file mode 100644 index 00000000000..dc2493d2a02 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_subquery_where.yaml @@ -0,0 +1,18 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(id=[$2], name=[$0]) + LogicalFilter(condition=[=($2, $SCALAR_QUERY({ + LogicalAggregate(group=[{}], max(uid)=[MAX($0)]) + LogicalProject(uid=[$1]) + LogicalFilter(condition=[=($cor1.id, $1)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + }))], variablesSet=[[$cor1]]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], id=[$t1], name=[$t0]) + EnumerableHashJoin(condition=[=($1, $2)], joinType=[semi]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[=($t0, $t1)], proj#0..1=[{exprs}], $condition=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},max(uid)=MAX($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"uid","boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"uid1":{"terms":{"field":"uid","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(uid)":{"max":{"field":"uid"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_select.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_select.yaml index 70fcf1c804d..e548f24def8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_select.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_select.yaml @@ -1,7 +1,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(variablesSet=[[$cor0]], name=[$0], count_dept=[$SCALAR_QUERY({ + LogicalProject(name=[$0], count_dept=[$SCALAR_QUERY({ LogicalAggregate(group=[{}], count(name)=[COUNT($0)]) LogicalProject(name=[$0]) LogicalFilter(condition=[IS NOT NULL($0)]) @@ -12,4 +12,4 @@ calcite: EnumerableLimit(fetch=[10000]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(name)=COUNT($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"name","boost":1.0}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(name)=COUNT($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"name","boost":1.0}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml index a14c422cfb5..8456cc5c33d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml @@ -2,16 +2,16 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(name=[$0]) - LogicalFilter(condition=[>($2, +($SCALAR_QUERY({ + LogicalFilter(condition=[>(SAFE_CAST($2), +($SCALAR_QUERY({ LogicalAggregate(group=[{}], count(name)=[COUNT($0)]) LogicalProject(name=[$0]) LogicalFilter(condition=[IS NOT NULL($0)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) - }), 999))], variablesSet=[[$cor0]]) + }), 999))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..2=[{inputs}], name=[$t0]) - EnumerableNestedLoopJoin(condition=[>($1, +($2, 999))], joinType=[inner]) + EnumerableNestedLoopJoin(condition=[>(SAFE_CAST($1), +($2, 999))], joinType=[inner]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(name)=COUNT($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"name","boost":1.0}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(name)=COUNT($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"name","boost":1.0}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_search_basic_text.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_search_basic_text.json deleted file mode 100644 index 85494ecb282..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_search_basic_text.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165])\n LogicalFilter(condition=[query_string(MAP('query', 'ERROR':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[spanId, traceId, @timestamp, instrumentationScope, severityText, resource, flags, attributes, droppedAttributesCount, severityNumber, time, body], FILTER->query_string(MAP('query', 'ERROR':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"ERROR\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"spanId\",\"traceId\",\"@timestamp\",\"instrumentationScope\",\"severityText\",\"resource\",\"flags\",\"attributes\",\"droppedAttributesCount\",\"severityNumber\",\"time\",\"body\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_search_basic_text.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_search_basic_text.yaml new file mode 100644 index 00000000000..e36d9dfaf58 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_search_basic_text.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, 'ERROR':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[spanId, traceId, @timestamp, instrumentationScope, severityText, resource, flags, attributes, droppedAttributesCount, severityNumber, time, body], FILTER->query_string(MAP('query':VARCHAR, 'ERROR':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"query_string":{"query":"ERROR","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["spanId","traceId","@timestamp","instrumentationScope","severityText","resource","flags","attributes","droppedAttributesCount","severityNumber","time","body"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_search_numeric_comparison.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_search_numeric_comparison.json deleted file mode 100644 index 1883a3503fc..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_search_numeric_comparison.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165])\n LogicalFilter(condition=[query_string(MAP('query', 'severityNumber:>15':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[spanId, traceId, @timestamp, instrumentationScope, severityText, resource, flags, attributes, droppedAttributesCount, severityNumber, time, body], FILTER->query_string(MAP('query', 'severityNumber:>15':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"severityNumber:>15\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"spanId\",\"traceId\",\"@timestamp\",\"instrumentationScope\",\"severityText\",\"resource\",\"flags\",\"attributes\",\"droppedAttributesCount\",\"severityNumber\",\"time\",\"body\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_search_numeric_comparison.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_search_numeric_comparison.yaml new file mode 100644 index 00000000000..d4f612830ae --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_search_numeric_comparison.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, 'severityNumber:>15':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[spanId, traceId, @timestamp, instrumentationScope, severityText, resource, flags, attributes, droppedAttributesCount, severityNumber, time, body], FILTER->query_string(MAP('query':VARCHAR, 'severityNumber:>15':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"query_string":{"query":"severityNumber:>15","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["spanId","traceId","@timestamp","instrumentationScope","severityText","resource","flags","attributes","droppedAttributesCount","severityNumber","time","body"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_search_wildcard_star.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_search_wildcard_star.json deleted file mode 100644 index 835dfca0835..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_search_wildcard_star.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165])\n LogicalFilter(condition=[query_string(MAP('query', 'severityText:ERR*':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[spanId, traceId, @timestamp, instrumentationScope, severityText, resource, flags, attributes, droppedAttributesCount, severityNumber, time, body], FILTER->query_string(MAP('query', 'severityText:ERR*':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"severityText:ERR*\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"spanId\",\"traceId\",\"@timestamp\",\"instrumentationScope\",\"severityText\",\"resource\",\"flags\",\"attributes\",\"droppedAttributesCount\",\"severityNumber\",\"time\",\"body\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_search_wildcard_star.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_search_wildcard_star.yaml new file mode 100644 index 00000000000..4d6d7fcde25 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_search_wildcard_star.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, 'severityText:ERR*':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[spanId, traceId, @timestamp, instrumentationScope, severityText, resource, flags, attributes, droppedAttributesCount, severityNumber, time, body], FILTER->query_string(MAP('query':VARCHAR, 'severityText:ERR*':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"query_string":{"query":"severityText:ERR*","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["spanId","traceId","@timestamp","instrumentationScope","severityText","resource","flags","attributes","droppedAttributesCount","severityNumber","time","body"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml index 3c6bf5d725d..b310fee5d8c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml @@ -1,18 +1,19 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], b.account_number=[$13], b.firstname=[$14], b.address=[$15], b.birthdate=[$16], b.gender=[$17], b.city=[$18], b.lastname=[$19], b.balance=[$20], b.employer=[$21], b.state=[$22], b.age=[$23], b.email=[$24], b.male=[$25]) - LogicalJoin(condition=[=(+($10, 1), -($20, 20))], joinType=[inner]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], b.account_number=[$14], b.firstname=[$15], b.address=[$16], b.birthdate=[$17], b.gender=[$18], b.city=[$19], b.lastname=[$20], b.balance=[$21], b.employer=[$22], b.state=[$23], b.age=[$24], b.email=[$25], b.male=[$26]) + LogicalJoin(condition=[=($13, $27)], joinType=[inner]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], $f13=[SAFE_CAST(+($10, 1))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], $f13=[-($7, 20)]) + LogicalSort(fetch=[50000]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..27=[{inputs}], proj#0..12=[{exprs}], b.account_number=[$t14], b.firstname=[$t15], b.address=[$t16], b.birthdate=[$t17], b.gender=[$t18], b.city=[$t19], b.lastname=[$t20], b.balance=[$t21], b.employer=[$t22], b.state=[$t23], b.age=[$t24], b.email=[$t25], b.male=[$t26]) EnumerableLimit(fetch=[10000]) EnumerableMergeJoin(condition=[=($13, $27)], joinType=[inner]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[1], expr#14=[+($t10, $t13)], proj#0..12=[{exprs}], $f13=[$t14]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[1], expr#14=[+($t10, $t13)], expr#15=[SAFE_CAST($t14)], proj#0..12=[{exprs}], $f13=[$t15]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT->[{ "age" : { "order" : "asc", diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_single_field_relevance_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_single_field_relevance_push.json deleted file mode 100644 index f9f084b81a9..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_single_field_relevance_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalFilter(condition=[match(MAP('field', $9), MAP('query', '*@gmail.com':VARCHAR), MAP('boost', '1.0':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->match(MAP('field', $9), MAP('query', '*@gmail.com':VARCHAR), MAP('boost', '1.0':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"match\":{\"email\":{\"query\":\"*@gmail.com\",\"operator\":\"OR\",\"prefix_length\":0,\"max_expansions\":50,\"fuzzy_transpositions\":true,\"lenient\":false,\"zero_terms_query\":\"NONE\",\"auto_generate_synonyms_phrase_query\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_single_field_relevance_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_single_field_relevance_push.yaml new file mode 100644 index 00000000000..a552e25a7d5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_single_field_relevance_push.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[match(MAP('field':VARCHAR, $9), MAP('query':VARCHAR, '*@gmail.com':VARCHAR), MAP('boost':VARCHAR, '1.0':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->match(MAP('field':VARCHAR, $9), MAP('query':VARCHAR, '*@gmail.com':VARCHAR), MAP('boost':VARCHAR, '1.0':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"match":{"email":{"query":"*@gmail.com","operator":"OR","prefix_length":0,"max_expansions":50,"fuzzy_transpositions":true,"lenient":false,"zero_terms_query":"NONE","auto_generate_synonyms_phrase_query":true,"boost":1.0}}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json deleted file mode 100644 index 99aa2e60259..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(firstname=[$1], age=[$8], address=[$2])\n LogicalFilter(condition=[AND(=($2, '671 Bristol Street'), =(-($8, 2), 30))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, address, age], SCRIPT->AND(=($1, '671 Bristol Street'), =(-($2, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 0,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 1,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*,\"SOURCES\":[1,2],\"DIGESTS\":[\"address\",\"671 Bristol Street\"]}},\"boost\":1.0}},{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"-\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"MINUS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 0,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 1,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n }\\\\n ],\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 2,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*,\"SOURCES\":[0,2,2],\"DIGESTS\":[\"age\",2,30]}},\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"firstname\",\"age\",\"address\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n","extended":"public org.apache.calcite.linq4j.Enumerable bind(final org.apache.calcite.DataContext root) {\n final org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan v1stashed = (org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan) root.get(\"v1stashed\");\n return v1stashed.scan();\n}\n\n\npublic Class getElementType() {\n return java.lang.Object[].class;\n}\n\n\n"}} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml new file mode 100644 index 00000000000..9c879a4c601 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(firstname=[$1], age=[$8], address=[$2]) + LogicalFilter(condition=[AND(=($2, '671 Bristol Street'), =(-($8, 2), SAFE_CAST(30:BIGINT)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, address, age], SCRIPT->AND(=($1, '671 Bristol Street'), =(-($2, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBeXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2],"DIGESTS":["address","671 Bristol Street"]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCwnsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIi0iLAogICAgICAgICJraW5kIjogIk1JTlVTIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["age",2,30]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age","address"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_agg_push.yaml new file mode 100644 index 00000000000..cff61992fb8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_agg_push.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(balance)=[$1], state=[$0]) + LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) + LogicalProject(state=[$0], balance=[$1]) + LogicalSort(sort0=[$1], sort1=[$2], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) + LogicalProject(state=[$7], balance=[$3], age=[$8]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), PROJECT->[avg(balance), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_count_push.yaml index ea09a6f976f..b1b013ec67e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_count_push.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(sort0=[$8], dir0=[ASC-nulls-first], fetch=[5]) + LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[5]) + LogicalProject(age=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], SORT->[{ diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_desc_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_desc_push.json deleted file mode 100644 index 204daa39ab0..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_desc_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8], firstname=[$1])\n LogicalSort(sort0=[$8], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, age], SORT->[{\n \"age\" : {\n \"order\" : \"desc\",\n \"missing\" : \"_last\"\n }\n}, {\n \"firstname.keyword\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], PROJECT->[age, firstname], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\",\"firstname\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"firstname.keyword\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_desc_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_desc_push.yaml new file mode 100644 index 00000000000..a0a99f4c2bc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_desc_push.yaml @@ -0,0 +1,18 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first]) + LogicalProject(age=[$8], firstname=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age, firstname], SORT->[{ + "age" : { + "order" : "desc", + "missing" : "_last" + } + }, { + "firstname.keyword" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","firstname"],"excludes":[]},"sort":[{"age":{"order":"desc","missing":"_last"}},{"firstname.keyword":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_pass_through_join_then_pushdown.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_pass_through_join_then_pushdown.yaml index 36ae1037150..59dc21e4579 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_pass_through_join_then_pushdown.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_pass_through_join_then_pushdown.yaml @@ -2,19 +2,20 @@ calcite: logical: | LogicalSystemLimit(sort0=[$13], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$13], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[$13], b.account_number=[$14], b.firstname=[$15], b.address=[$16], b.birthdate=[$17], b.gender=[$18], b.city=[$19], b.lastname=[$20], b.balance=[$21], b.employer=[$22], b.state=[$23], b.age=[$24], b.email=[$25], b.male=[$26]) - LogicalJoin(condition=[=($13, $15)], joinType=[left]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[REX_EXTRACT($6, '(?^[A-Z])', 'initial')]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[$13], b.account_number=[$15], b.firstname=[$16], b.address=[$17], b.birthdate=[$18], b.gender=[$19], b.city=[$20], b.lastname=[$21], b.balance=[$22], b.employer=[$23], b.state=[$24], b.age=[$25], b.email=[$26], b.male=[$27]) + LogicalJoin(condition=[=($14, $16)], joinType=[left]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[REX_EXTRACT($6, '(?^[A-Z])', 'initial')], $f14=[CAST(REX_EXTRACT($6, '(?^[A-Z])', 'initial')):VARCHAR]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[50000]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableTopK(sort0=[$13], dir0=[ASC-nulls-first], fetch=[10000]) - EnumerableMergeJoin(condition=[=($13, $15)], joinType=[left]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=['(?^[A-Z])'], expr#14=['initial'], expr#15=[REX_EXTRACT($t6, $t13, $t14)], proj#0..12=[{exprs}], $f13=[$t15]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[REX_EXTRACT($6, '(?^[A-Z])', 'initial') ASCENDING NULLS_LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQC63sKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRVhfRVhUUkFDVCIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAicHJlY2lzaW9uIjogMjAwMAogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":true,"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["lastname","(?^[A-Z])","initial"]}},"type":"string","order":"asc"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], LIMIT->50000, SORT->[{ + EnumerableCalc(expr#0..27=[{inputs}], proj#0..13=[{exprs}], b.account_number=[$t15], b.firstname=[$t16], b.address=[$t17], b.birthdate=[$t18], b.gender=[$t19], b.city=[$t20], b.lastname=[$t21], b.balance=[$t22], b.employer=[$t23], b.state=[$t24], b.age=[$t25], b.email=[$t26], b.male=[$t27]) + CalciteEnumerableTopK(sort0=[$13], dir0=[ASC-nulls-first], fetch=[10000]) + EnumerableMergeJoin(condition=[=($14, $16)], joinType=[left]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=['(?^[A-Z])'], expr#14=['initial'], expr#15=[REX_EXTRACT($t6, $t13, $t14)], expr#16=[CAST($t15):VARCHAR], proj#0..12=[{exprs}], $f13=[$t15], $f14=[$t16]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[CAST(REX_EXTRACT($6, '(?^[A-Z])', 'initial')):VARCHAR ASCENDING NULLS_LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQESHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVYX0VYVFJBQ1QiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAyMDAwCiAgICAgIH0sCiAgICAgICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAgICAgImR5bmFtaWMiOiBmYWxzZQogICAgfQogIF0sCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAicHJlY2lzaW9uIjogLTEKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":true,"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["lastname","(?^[A-Z])","initial"]}},"type":"string","order":"asc"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], LIMIT->50000, SORT->[{ "firstname" : { "order" : "asc", "missing" : "_last" diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_push.json deleted file mode 100644 index 7b75e10b17d..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalFilter(condition=[>($8, 30)])\n LogicalSort(sort0=[$8], dir0=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], SORT->[{\n \"age\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], FILTER->>($0, 30), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_push.yaml new file mode 100644 index 00000000000..f07b0539054 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_push.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[>($8, SAFE_CAST(30:BIGINT))]) + LogicalSort(sort0=[$8], dir0=[ASC-nulls-first]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], SORT->[{ + "age" : { + "order" : "asc", + "missing" : "_first" + } + }], FILTER->>($0, 30), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["age"],"excludes":[]},"sort":[{"age":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_rename_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_rename_push.json deleted file mode 100644 index 6281b1825b6..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_rename_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(alias=[$17])\n LogicalSort(sort0=[$17], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], name=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], alias=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname], SORT->[{\n \"firstname.keyword\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]},\"sort\":[{\"firstname.keyword\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_rename_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_rename_push.yaml new file mode 100644 index 00000000000..041ff235a8b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_rename_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC-nulls-first]) + LogicalProject(alias=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname], SORT->[{ + "firstname.keyword" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["firstname"],"excludes":[]},"sort":[{"firstname.keyword":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.json deleted file mode 100644 index 0d6b8c4b9cf..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(avg(balance)=[$1], state=[$0])\n LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)])\n LogicalProject(state=[$7], balance=[$3])\n LogicalSort(sort0=[$3], sort1=[$8], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},avg(balance)=AVG($0)), PROJECT->[avg(balance), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(balance)\":{\"avg\":{\"field\":\"balance\"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.yaml new file mode 100644 index 00000000000..cff61992fb8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(balance)=[$1], state=[$0]) + LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) + LogicalProject(state=[$0], balance=[$1]) + LogicalSort(sort0=[$1], sort1=[$2], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) + LogicalProject(state=[$7], balance=[$3], age=[$8]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), PROJECT->[avg(balance), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_limit_push.yaml index ea09a6f976f..b1b013ec67e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_limit_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_limit_push.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(sort0=[$8], dir0=[ASC-nulls-first], fetch=[5]) + LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[5]) + LogicalProject(age=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], SORT->[{ diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span.json deleted file mode 100644 index 094516a934a..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(age,10)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(age,10)=[SPAN($10, 10, null:NULL)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(age,10)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(age,10)\":{\"histogram\":{\"field\":\"age\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\",\"interval\":10.0}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span.yaml new file mode 100644 index 00000000000..6eac4265e94 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], span(age,10)=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(span(age,10)=[SPAN($10, 10, null:ANY)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(age,10)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(age,10)":{"histogram":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc","interval":10.0}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span_non_bucket_nullable.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span_non_bucket_nullable.json deleted file mode 100644 index 6104070d6d4..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span_non_bucket_nullable.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(age,10)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(age,10)=[SPAN($10, 10, null:NULL)])\n LogicalFilter(condition=[IS NOT NULL($10)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(age,10)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(age,10)\":{\"histogram\":{\"field\":\"age\",\"missing_bucket\":false,\"order\":\"asc\",\"interval\":10.0}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span_non_bucket_nullable.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span_non_bucket_nullable.yaml new file mode 100644 index 00000000000..662a5954571 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span_non_bucket_nullable.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], span(age,10)=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(span(age,10)=[SPAN($10, 10, null:ANY)]) + LogicalFilter(condition=[IS NOT NULL($10)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(age,10)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(age,10)":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":10.0}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.yaml index 0e1e29e9370..3cc9e576e8f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.yaml @@ -7,4 +7,4 @@ calcite: LogicalFilter(condition=[IS NOT NULL($3)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1m)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1m)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1m)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1m)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.yaml index c9e64c16e80..54f9b1ea7ac 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.yaml @@ -7,4 +7,4 @@ calcite: LogicalFilter(condition=[IS NOT NULL($3)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1M)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1M)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","calendar_interval":"1M"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1M)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1M)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","calendar_interval":"1M"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_distinct_count.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_distinct_count.yaml index d464ea5b6d8..ccef8e584e5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_distinct_count.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_distinct_count.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], distinct_states=[$18]) - LogicalSort(sort0=[$17], dir0=[ASC]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], distinct_states=[DISTINCT_COUNT_APPROX($7) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], distinct_states=[$11]) + LogicalSort(sort0=[$12], dir0=[ASC]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], distinct_states=[DISTINCT_COUNT_APPROX($7) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)], __stream_seq__=[$17]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | @@ -11,4 +11,4 @@ calcite: CalciteEnumerableTopK(sort0=[$11], dir0=[ASC], fetch=[10000]) EnumerableWindow(window#0=[window(partition {4} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [DISTINCT_COUNT_APPROX($7)])]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_earliest_latest.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_earliest_latest.yaml index b8ddae83a2e..126cdddd906 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_earliest_latest.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_earliest_latest.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[$12], latest_message=[$13]) - LogicalSort(sort0=[$11], dir0=[ASC]) - LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], _id=[$5], _index=[$6], _score=[$7], _maxscore=[$8], _sort=[$9], _routing=[$10], __stream_seq__=[$11], earliest_message=[ARG_MIN($3, $2) OVER (PARTITION BY $1 ROWS UNBOUNDED PRECEDING)], latest_message=[ARG_MAX($3, $2) OVER (PARTITION BY $1 ROWS UNBOUNDED PRECEDING)]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[$5], latest_message=[$6]) + LogicalSort(sort0=[$7], dir0=[ASC]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[ARG_MIN($3, $2) OVER (PARTITION BY $1 ROWS UNBOUNDED PRECEDING)], latest_message=[ARG_MAX($3, $2) OVER (PARTITION BY $1 ROWS UNBOUNDED PRECEDING)], __stream_seq__=[$11]) LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], _id=[$5], _index=[$6], _score=[$7], _maxscore=[$8], _sort=[$9], _routing=[$10], __stream_seq__=[ROW_NUMBER() OVER ()]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) physical: | @@ -11,4 +11,4 @@ calcite: CalciteEnumerableTopK(sort0=[$5], dir0=[ASC], fetch=[10000]) EnumerableWindow(window#0=[window(partition {1} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ARG_MIN($3, $2), ARG_MAX($3, $2)])]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[PROJECT->[created_at, server, @timestamp, message, level]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["created_at","server","@timestamp","message","level"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[PROJECT->[created_at, server, @timestamp, message, level]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["created_at","server","@timestamp","message","level"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_earliest_latest_custom_time.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_earliest_latest_custom_time.yaml index 86666bdaf8a..c16c462a86b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_earliest_latest_custom_time.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_earliest_latest_custom_time.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[$12], latest_message=[$13]) - LogicalSort(sort0=[$11], dir0=[ASC]) - LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], _id=[$5], _index=[$6], _score=[$7], _maxscore=[$8], _sort=[$9], _routing=[$10], __stream_seq__=[$11], earliest_message=[ARG_MIN($3, $0) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)], latest_message=[ARG_MAX($3, $0) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[$5], latest_message=[$6]) + LogicalSort(sort0=[$7], dir0=[ASC]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[ARG_MIN($3, $0) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)], latest_message=[ARG_MAX($3, $0) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)], __stream_seq__=[$11]) LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], _id=[$5], _index=[$6], _score=[$7], _maxscore=[$8], _sort=[$9], _routing=[$10], __stream_seq__=[ROW_NUMBER() OVER ()]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) physical: | @@ -11,4 +11,4 @@ calcite: CalciteEnumerableTopK(sort0=[$5], dir0=[ASC], fetch=[10000]) EnumerableWindow(window#0=[window(partition {4} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ARG_MIN($3, $0), ARG_MAX($3, $0)])]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[PROJECT->[created_at, server, @timestamp, message, level]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["created_at","server","@timestamp","message","level"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[PROJECT->[created_at, server, @timestamp, message, level]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["created_at","server","@timestamp","message","level"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml index 97703d849a7..a3dd4335c7e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml @@ -1,16 +1,17 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$18]) - LogicalSort(sort0=[$17], dir0=[ASC]) - LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{4, 17}]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalAggregate(group=[{}], avg_age=[AVG($0)]) - LogicalProject(age=[$8]) - LogicalFilter(condition=[AND(>=($17, -($cor0.__stream_seq__, 1)), <=($17, $cor0.__stream_seq__), OR(=($4, $cor0.gender), AND(IS NULL($4), IS NULL($cor0.gender))))]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$11]) + LogicalSort(sort0=[$12], dir0=[ASC]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$18], __stream_seq__=[$17]) + LogicalCorrelate(correlation=[$cor1], joinType=[left], requiredColumns=[{4, 17}]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalAggregate(group=[{}], avg_age=[AVG($0)]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[AND(>=($17, -($cor1.__stream_seq__, 1)), <=($17, $cor1.__stream_seq__), OR(=($4, $cor1.gender), AND(IS NULL($4), IS NULL($cor1.gender))))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) EnumerableLimit(fetch=[10000]) @@ -28,4 +29,4 @@ calcite: EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml index 3959777f707..d65b0770d68 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml @@ -1,16 +1,17 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$18]) - LogicalSort(sort0=[$17], dir0=[ASC]) - LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{4, 17}]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalAggregate(group=[{}], avg_age=[AVG($0)]) - LogicalProject(age=[$8]) - LogicalFilter(condition=[AND(>=($17, -($cor0.__stream_seq__, 1)), <=($17, $cor0.__stream_seq__), =($4, $cor0.gender))]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$11]) + LogicalSort(sort0=[$12], dir0=[ASC]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$18], __stream_seq__=[$17]) + LogicalCorrelate(correlation=[$cor1], joinType=[left], requiredColumns=[{4, 17}]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalAggregate(group=[{}], avg_age=[AVG($0)]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[AND(>=($17, -($cor1.__stream_seq__, 1)), <=($17, $cor1.__stream_seq__), =($4, $cor1.gender))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t16]) CalciteEnumerableTopK(sort0=[$11], dir0=[ASC], fetch=[10000]) @@ -28,4 +29,4 @@ calcite: EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_null_bucket.yaml index c4b49653614..0726b099116 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_null_bucket.yaml @@ -1,15 +1,15 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$18]) - LogicalSort(sort0=[$17], dir0=[ASC]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], avg_age=[CASE(IS NOT NULL($4), /(SUM($8) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING), CAST(COUNT($8) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)):DOUBLE NOT NULL), null:DOUBLE)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$11]) + LogicalSort(sort0=[$12], dir0=[ASC]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[CASE(IS NOT NULL($4), /(CASE(>(COUNT($8) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING), 0), SUM($8) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING), null:BIGINT), CAST(COUNT($8) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)):DOUBLE NOT NULL), null:DOUBLE)], __stream_seq__=[$17]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..14=[{inputs}], expr#15=[CAST($t14):DOUBLE NOT NULL], expr#16=[/($t13, $t15)], expr#17=[null:DOUBLE], expr#18=[CASE($t12, $t16, $t17)], proj#0..10=[{exprs}], avg_age=[$t18]) + EnumerableCalc(expr#0..14=[{inputs}], expr#15=[0:BIGINT], expr#16=[>($t13, $t15)], expr#17=[null:BIGINT], expr#18=[CASE($t16, $t14, $t17)], expr#19=[CAST($t13):DOUBLE NOT NULL], expr#20=[/($t18, $t19)], expr#21=[null:DOUBLE], expr#22=[CASE($t12, $t20, $t21)], proj#0..10=[{exprs}], avg_age=[$t22]) CalciteEnumerableTopK(sort0=[$11], dir0=[ASC], fetch=[10000]) - EnumerableWindow(window#0=[window(partition {4} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($8), COUNT($8)])]) + EnumerableWindow(window#0=[window(partition {4} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [COUNT($8), $SUM0($8)])]) EnumerableCalc(expr#0..11=[{inputs}], expr#12=[IS NOT NULL($t4)], proj#0..12=[{exprs}]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml index fa6654252f4..8457d9ad27e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml @@ -1,26 +1,27 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$21]) - LogicalSort(sort0=[$17], dir0=[ASC]) - LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{4, 17, 20}]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), COALESCE(SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0))]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalAggregate(group=[{}], avg_age=[AVG($0)]) - LogicalProject(age=[$8]) - LogicalFilter(condition=[AND(<($17, $cor0.__stream_seq__), =($20, $cor0.__seg_id__), OR(=($4, $cor0.gender), AND(IS NULL($4), IS NULL($cor0.gender))))]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), COALESCE(SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0))]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$11]) + LogicalSort(sort0=[$12], dir0=[ASC]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$21], __stream_seq__=[$17]) + LogicalCorrelate(correlation=[$cor1], joinType=[left], requiredColumns=[{4, 17, 20}]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), CASE(IS NOT NULL(CASE(>(COUNT($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0), SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), null:BIGINT)), CAST(CASE(>(COUNT($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0), SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), null:BIGINT)):BIGINT NOT NULL, 0:BIGINT))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalAggregate(group=[{}], avg_age=[AVG($0)]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[AND(<($17, $cor1.__stream_seq__), =($20, $cor1.__seg_id__), OR(=($4, $cor1.gender), AND(IS NULL($4), IS NULL($cor1.gender))))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), CASE(IS NOT NULL(CASE(>(COUNT($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0), SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), null:BIGINT)), CAST(CASE(>(COUNT($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0), SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), null:BIGINT)):BIGINT NOT NULL, 0:BIGINT))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) EnumerableLimit(fetch=[10000]) EnumerableMergeJoin(condition=[AND(=($11, $15), =($12, $16), =($13, $17), IS NOT DISTINCT FROM($4, $14))], joinType=[left]) EnumerableSort(sort0=[$11], sort1=[$12], sort2=[$13], dir0=[ASC], dir1=[ASC], dir2=[ASC]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[0], expr#18=[COALESCE($t16, $t17)], expr#19=[+($t15, $t18)], proj#0..11=[{exprs}], __seg_id__=[$t19], $f16=[$t14]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($12)])], window#1=[window(rows between UNBOUNDED PRECEDING and $15 PRECEDING aggs [$SUM0($13)])], constants=[[1]]) - EnumerableCalc(expr#0..11=[{inputs}], expr#12=[34], expr#13=[>($t8, $t12)], expr#14=[1], expr#15=[0], expr#16=[CASE($t13, $t14, $t15)], expr#17=[25], expr#18=[<($t8, $t17)], expr#19=[CASE($t18, $t14, $t15)], expr#20=[IS NULL($t4)], proj#0..11=[{exprs}], __reset_before_flag__=[$t16], __reset_after_flag__=[$t19], $14=[$t20]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[0:BIGINT], expr#19=[>($t16, $t18)], expr#20=[null:BIGINT], expr#21=[CASE($t19, $t17, $t20)], expr#22=[IS NOT NULL($t21)], expr#23=[CAST($t21):BIGINT NOT NULL], expr#24=[CASE($t22, $t23, $t18)], expr#25=[+($t15, $t24)], proj#0..11=[{exprs}], __seg_id__=[$t25], $f16=[$t14]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($12)])], window#1=[window(rows between UNBOUNDED PRECEDING and $15 PRECEDING aggs [COUNT($13), $SUM0($13)])], constants=[[1]]) + EnumerableCalc(expr#0..11=[{inputs}], expr#12=[34:BIGINT], expr#13=[>($t8, $t12)], expr#14=[1], expr#15=[0], expr#16=[CASE($t13, $t14, $t15)], expr#17=[25:BIGINT], expr#18=[<($t8, $t17)], expr#19=[CASE($t18, $t14, $t15)], expr#20=[IS NULL($t4)], proj#0..11=[{exprs}], __reset_before_flag__=[$t16], __reset_after_flag__=[$t19], $14=[$t20]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$1], sort1=[$2], sort2=[$3], dir0=[ASC], dir1=[ASC], dir2=[ASC]) @@ -28,13 +29,13 @@ calcite: EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) EnumerableAggregate(group=[{0, 1, 2, 3}]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], expr#11=[IS NULL($t0)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10], $4=[$t11]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0:BIGINT], expr#9=[>($t6, $t8)], expr#10=[null:BIGINT], expr#11=[CASE($t9, $t7, $t10)], expr#12=[IS NOT NULL($t11)], expr#13=[CAST($t11):BIGINT NOT NULL], expr#14=[CASE($t12, $t13, $t8)], expr#15=[+($t5, $t14)], proj#0..1=[{exprs}], __seg_id__=[$t15], $f16=[$t4]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [COUNT($3), $SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34:BIGINT], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25:BIGINT], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], expr#11=[IS NULL($t0)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10], $4=[$t11]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0:BIGINT], expr#9=[>($t6, $t8)], expr#10=[null:BIGINT], expr#11=[CASE($t9, $t7, $t10)], expr#12=[IS NOT NULL($t11)], expr#13=[CAST($t11):BIGINT NOT NULL], expr#14=[CASE($t12, $t13, $t8)], expr#15=[+($t5, $t14)], proj#0..2=[{exprs}], __seg_id__=[$t15]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [COUNT($4), $SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34:BIGINT], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25:BIGINT], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml index 7be62745372..2eb07d20556 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml @@ -1,26 +1,27 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$21]) - LogicalSort(sort0=[$17], dir0=[ASC]) - LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{4, 17, 20}]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), COALESCE(SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0))]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalAggregate(group=[{}], avg_age=[AVG($0)]) - LogicalProject(age=[$8]) - LogicalFilter(condition=[AND(<($17, $cor0.__stream_seq__), =($20, $cor0.__seg_id__), =($4, $cor0.gender))]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), COALESCE(SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0))]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$11]) + LogicalSort(sort0=[$12], dir0=[ASC]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$21], __stream_seq__=[$17]) + LogicalCorrelate(correlation=[$cor1], joinType=[left], requiredColumns=[{4, 17, 20}]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), CASE(IS NOT NULL(CASE(>(COUNT($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0), SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), null:BIGINT)), CAST(CASE(>(COUNT($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0), SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), null:BIGINT)):BIGINT NOT NULL, 0:BIGINT))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalAggregate(group=[{}], avg_age=[AVG($0)]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[AND(<($17, $cor1.__stream_seq__), =($20, $cor1.__seg_id__), =($4, $cor1.gender))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), CASE(IS NOT NULL(CASE(>(COUNT($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0), SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), null:BIGINT)), CAST(CASE(>(COUNT($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0), SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), null:BIGINT)):BIGINT NOT NULL, 0:BIGINT))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t16]) CalciteEnumerableTopK(sort0=[$11], dir0=[ASC], fetch=[10000]) EnumerableMergeJoin(condition=[AND(=($4, $13), =($11, $14), =($12, $15))], joinType=[left]) EnumerableSort(sort0=[$4], sort1=[$11], sort2=[$12], dir0=[ASC], dir1=[ASC], dir2=[ASC]) - EnumerableCalc(expr#0..15=[{inputs}], expr#16=[0], expr#17=[COALESCE($t15, $t16)], expr#18=[+($t14, $t17)], proj#0..11=[{exprs}], __seg_id__=[$t18]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($12)])], window#1=[window(rows between UNBOUNDED PRECEDING and $14 PRECEDING aggs [$SUM0($13)])], constants=[[1]]) - EnumerableCalc(expr#0..11=[{inputs}], expr#12=[34], expr#13=[>($t8, $t12)], expr#14=[1], expr#15=[0], expr#16=[CASE($t13, $t14, $t15)], expr#17=[25], expr#18=[<($t8, $t17)], expr#19=[CASE($t18, $t14, $t15)], proj#0..11=[{exprs}], __reset_before_flag__=[$t16], __reset_after_flag__=[$t19]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[0:BIGINT], expr#18=[>($t15, $t17)], expr#19=[null:BIGINT], expr#20=[CASE($t18, $t16, $t19)], expr#21=[IS NOT NULL($t20)], expr#22=[CAST($t20):BIGINT NOT NULL], expr#23=[CASE($t21, $t22, $t17)], expr#24=[+($t14, $t23)], proj#0..11=[{exprs}], __seg_id__=[$t24]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($12)])], window#1=[window(rows between UNBOUNDED PRECEDING and $14 PRECEDING aggs [COUNT($13), $SUM0($13)])], constants=[[1]]) + EnumerableCalc(expr#0..11=[{inputs}], expr#12=[34:BIGINT], expr#13=[>($t8, $t12)], expr#14=[1], expr#15=[0], expr#16=[CASE($t13, $t14, $t15)], expr#17=[25:BIGINT], expr#18=[<($t8, $t17)], expr#19=[CASE($t18, $t14, $t15)], proj#0..11=[{exprs}], __reset_before_flag__=[$t16], __reset_after_flag__=[$t19]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], sort1=[$1], sort2=[$2], dir0=[ASC], dir1=[ASC], dir2=[ASC]) @@ -28,13 +29,13 @@ calcite: EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($4)], agg#1=[COUNT($4)]) EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) EnumerableAggregate(group=[{0, 1, 2}]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0:BIGINT], expr#8=[>($t5, $t7)], expr#9=[null:BIGINT], expr#10=[CASE($t8, $t6, $t9)], expr#11=[IS NOT NULL($t10)], expr#12=[CAST($t10):BIGINT NOT NULL], expr#13=[CASE($t11, $t12, $t7)], expr#14=[+($t4, $t13)], proj#0..1=[{exprs}], __seg_id__=[$t14]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [COUNT($3), $SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34:BIGINT], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25:BIGINT], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0:BIGINT], expr#9=[>($t6, $t8)], expr#10=[null:BIGINT], expr#11=[CASE($t9, $t7, $t10)], expr#12=[IS NOT NULL($t11)], expr#13=[CAST($t11):BIGINT NOT NULL], expr#14=[CASE($t12, $t13, $t8)], expr#15=[+($t5, $t14)], proj#0..2=[{exprs}], __seg_id__=[$t15]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [COUNT($4), $SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34:BIGINT], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25:BIGINT], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_strftime_function.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_strftime_function.json deleted file mode 100644 index 55af94637f0..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_strftime_function.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(fetch=[1])\n LogicalProject(formatted_date=[STRFTIME(1521467703, '%Y-%m-%d':VARCHAR)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0..16=[{inputs}], expr#17=[1521467703], expr#18=['%Y-%m-%d':VARCHAR], expr#19=[STRFTIME($t17, $t18)], formatted_date=[$t19])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->1, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":1,\"timeout\":\"1m\"}, requestedTotalSize=1, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_strftime_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_strftime_function.yaml new file mode 100644 index 00000000000..26e895408ba --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_strftime_function.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(fetch=[1]) + LogicalProject(formatted_date=[STRFTIME(1521467703, '%Y-%m-%d')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[1521467703], expr#18=['%Y-%m-%d'], expr#19=[STRFTIME($t17, $t18)], formatted_date=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->1, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":1,"timeout":"1m"}, requestedTotalSize=1, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml index 5dd080d68b5..ad352529e11 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml @@ -3,37 +3,37 @@ calcite: LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[AVG($2)]) - LogicalProject(@timestamp=[$0], host=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], avg(cpu_usage)=[$2]) + LogicalProject(@timestamp=[$0], host=[CASE(IS NULL($1), 'NULL':VARCHAR, <=($5, 10), $1, 'OTHER':VARCHAR)], avg(cpu_usage)=[$2]) LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalProject(@timestamp=[$1], host=[$0], avg(cpu_usage)=[$2]) - LogicalAggregate(group=[{0, 2}], avg(cpu_usage)=[AVG($1)]) - LogicalProject(host=[$4], cpu_usage=[$7], @timestamp0=[SPAN($1, 1, 'm')]) + LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[AVG($2)]) + LogicalProject(host=[$4], @timestamp=[SPAN($1, 1, 'm')], cpu_usage=[$7]) LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, events]]) LogicalProject(host=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(host=[$0], avg(cpu_usage)=[$2]) - LogicalAggregate(group=[{0, 2}], avg(cpu_usage)=[AVG($1)]) - LogicalProject(host=[$4], cpu_usage=[$7], @timestamp0=[SPAN($1, 1, 'm')]) + LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[AVG($2)]) + LogicalProject(host=[$4], $f1=[SPAN($1, 1, 'm')], cpu_usage=[$7]) LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, events]]) physical: | CalciteEnumerableTopK(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(cpu_usage)=[$t8]) EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL':VARCHAR], expr#7=[10:BIGINT], expr#8=[<=($t4, $t7)], expr#9=['OTHER':VARCHAR], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], @timestamp=[$t1], host=[$t0], avg(cpu_usage)=[$t8]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t0, $t3, $t4)], host=[$t1], cpu_usage=[$t2], @timestamp0=[$t5]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t0, $t3, $t4)], host=[$t1], @timestamp=[$t5], cpu_usage=[$t2]) CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host, cpu_usage], FILTER->AND(IS NOT NULL($0), IS NOT NULL($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["@timestamp","host","cpu_usage"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], host=[$t0], avg(cpu_usage)=[$t8]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host, cpu_usage], FILTER->AND(IS NOT NULL($0), IS NOT NULL($2)), PROJECT->[host, cpu_usage, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"host","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["host","cpu_usage","@timestamp"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t1, $t3, $t4)], host=[$t0], $f1=[$t5], cpu_usage=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host, cpu_usage], FILTER->AND(IS NOT NULL($0), IS NOT NULL($2)), PROJECT->[host, @timestamp, cpu_usage], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"host","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["host","@timestamp","cpu_usage"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_avg.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_avg.yaml new file mode 100644 index 00000000000..47802b7c09f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_avg.yaml @@ -0,0 +1,40 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[AVG($2)]) + LogicalProject(@timestamp=[$0], host=[CASE(IS NULL($1), 'NULL':VARCHAR, <=($5, 10), $1, 'OTHER':VARCHAR)], avg(cpu_usage)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(@timestamp=[$1], host=[$0], avg(cpu_usage)=[$2]) + LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[AVG($2)]) + LogicalProject(host=[$4], @timestamp=[SPAN($1, 1, 'm')], cpu_usage=[$7]) + LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($7))]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + LogicalProject(host=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(host=[$0], avg(cpu_usage)=[$2]) + LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[AVG($2)]) + LogicalProject(host=[$4], $f1=[SPAN($1, 1, 'm')], cpu_usage=[$7]) + LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($7))]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(cpu_usage)=[$t8]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL':VARCHAR], expr#7=[10:BIGINT], expr#8=[<=($t4, $t7)], expr#9=['OTHER':VARCHAR], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], @timestamp=[$t1], host=[$t0], avg(cpu_usage)=[$t8]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t0, $t3, $t4)], host=[$t1], @timestamp=[$t5], cpu_usage=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host, cpu_usage], FILTER->AND(IS NOT NULL($0), IS NOT NULL($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["@timestamp","host","cpu_usage"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], host=[$t0], avg(cpu_usage)=[$t8]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t1, $t3, $t4)], host=[$t0], $f1=[$t5], cpu_usage=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host, cpu_usage], FILTER->AND(IS NOT NULL($0), IS NOT NULL($2)), PROJECT->[host, @timestamp, cpu_usage], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"host","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["host","@timestamp","cpu_usage"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml index 9e462061605..8d441e7a937 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml @@ -3,11 +3,11 @@ calcite: LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) LogicalAggregate(group=[{0, 1}], count()=[SUM($2)]) - LogicalProject(@timestamp=[$0], host=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], count()=[$2]) + LogicalProject(@timestamp=[$0], host=[CASE(IS NULL($1), 'NULL':VARCHAR, <=($5, 10), $1, 'OTHER':VARCHAR)], count()=[$2]) LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalProject(@timestamp=[$1], host=[$0], count()=[$2]) LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) - LogicalProject(host=[$4], @timestamp0=[SPAN($1, 1, 'm')]) + LogicalProject(host=[$4], @timestamp=[SPAN($1, 1, 'm')]) LogicalFilter(condition=[IS NOT NULL($1)]) CalciteLogicalIndexScan(table=[[OpenSearch, events]]) LogicalProject(host=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) @@ -15,21 +15,21 @@ calcite: LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(host=[$0], count()=[$2]) LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) - LogicalProject(host=[$4], @timestamp0=[SPAN($1, 1, 'm')]) + LogicalProject(host=[$4], $f1=[SPAN($1, 1, 'm')]) LogicalFilter(condition=[IS NOT NULL($1)]) CalciteLogicalIndexScan(table=[[OpenSearch, events]]) physical: | CalciteEnumerableTopK(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000]) EnumerableAggregate(group=[{0, 1}], count()=[$SUM0($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], @timestamp=[$t0], host=[$t10], count()=[$t2]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL':VARCHAR], expr#7=[10:BIGINT], expr#8=[<=($t4, $t7)], expr#9=['OTHER':VARCHAR], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], @timestamp=[$t0], host=[$t10], count()=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], @timestamp=[$t1], host=[$t0], count()=[$t2]) EnumerableAggregate(group=[{0, 1}], count()=[COUNT()]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t0, $t2, $t3)], host=[$t1], @timestamp0=[$t4]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t0, $t2, $t3)], host=[$t1], @timestamp=[$t4]) CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"@timestamp","boost":1.0}},"_source":{"includes":["@timestamp","host"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[COUNT()]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host], FILTER->IS NOT NULL($0), PROJECT->[host, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"filter":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"host","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["host","@timestamp"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host], FILTER->IS NOT NULL($0), PROJECT->[host, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"filter":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"host","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["host","@timestamp"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_top_k_then_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_top_k_then_sort_push.yaml index cba5870494c..b87d3fb45c5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_top_k_then_sort_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_top_k_then_sort_push.yaml @@ -1,16 +1,16 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(sort0=[$8], dir0=[ASC-nulls-first]) + LogicalSort(sort0=[$0], dir0=[ASC-nulls-first]) + LogicalProject(age=[$8]) LogicalSort(sort0=[$3], dir0=[ASC-nulls-first], fetch=[5]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], age=[$t1]) - CalciteEnumerableTopK(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[balance, age], SORT->[{ + CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[balance, age], SORT->[{ "balance" : { "order" : "asc", "missing" : "_first" } - }], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["balance","age"],"excludes":[]},"sort":[{"balance":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->5, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]},"sort":[{"balance":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_top_usenull_false.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_top_usenull_false.yaml index 21457c6170f..17813714925 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_top_usenull_false.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_top_usenull_false.yaml @@ -2,11 +2,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(gender=[$0], state=[$1], count=[$2]) - LogicalFilter(condition=[<=($3, 2)]) - LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC)]) + LogicalFilter(condition=[<=($3, SAFE_CAST(2:BIGINT))]) + LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC NULLS FIRST)]) LogicalAggregate(group=[{0, 1}], count=[COUNT()]) LogicalProject(gender=[$4], state=[$7]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count=COUNT()), RARE_TOP->top 2 state by gender, PROJECT->[gender, state, count], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender":{"terms":{"field":"gender.keyword","size":10000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]},"aggregations":{"state":{"terms":{"field":"state.keyword","size":2,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count=COUNT()), RARE_TOP->top 2 state by gender, PROJECT->[gender, state, count], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender":{"terms":{"field":"gender.keyword","size":10000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]},"aggregations":{"state":{"terms":{"field":"state.keyword","size":2,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_top_usenull_true.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_top_usenull_true.yaml index 51ffb883407..db8202b26cf 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_top_usenull_true.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_top_usenull_true.yaml @@ -2,13 +2,13 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(gender=[$0], state=[$1], count=[$2]) - LogicalFilter(condition=[<=($3, 2)]) - LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC)]) + LogicalFilter(condition=[<=($3, SAFE_CAST(2:BIGINT))]) + LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC NULLS FIRST)]) LogicalAggregate(group=[{0, 1}], count=[COUNT()]) LogicalProject(gender=[$4], state=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[2], expr#5=[<=($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[2:BIGINT], expr#5=[<=($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) EnumerableWindow(window#0=[window(partition {0} order by [2 DESC] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_push.yaml index 683bfe610cd..600e62d526b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_push.yaml @@ -1,13 +1,14 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(ageTrend=[CASE(>(COUNT() OVER (ROWS 1 PRECEDING), 1), /(SUM($8) OVER (ROWS 1 PRECEDING), CAST(COUNT($8) OVER (ROWS 1 PRECEDING)):DOUBLE NOT NULL), null:NULL)]) + LogicalProject(ageTrend=[CASE(>(COUNT() OVER (ROWS 1 PRECEDING), 1), /(CASE(>(COUNT($8) OVER (ROWS 1 PRECEDING), 0), SUM($8) OVER (ROWS 1 PRECEDING), null:BIGINT), CAST(COUNT($8) OVER (ROWS 1 PRECEDING)):DOUBLE NOT NULL), null:DOUBLE)]) LogicalFilter(condition=[IS NOT NULL($8)]) LogicalSort(fetch=[5]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[>($t1, $t4)], expr#6=[CAST($t3):DOUBLE NOT NULL], expr#7=[/($t2, $t6)], expr#8=[null:NULL], expr#9=[CASE($t5, $t7, $t8)], ageTrend=[$t9]) - EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])], constants=[[1]]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1:BIGINT], expr#5=[>($t1, $t4)], expr#6=[0:BIGINT], expr#7=[>($t2, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t3, $t8)], expr#10=[CAST($t2):DOUBLE NOT NULL], expr#11=[/($t9, $t10)], expr#12=[null:DOUBLE], expr#13=[CASE($t5, $t11, $t12)], ageTrend=[$t13]) + EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), COUNT($0), $SUM0($0)])], constants=[[1]]) EnumerableCalc(expr#0=[{inputs}], expr#1=[IS NOT NULL($t0)], age=[$t0], $condition=[$t1]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_sort_push.yaml index 94265227c8e..2df541cac73 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_sort_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_sort_push.yaml @@ -1,19 +1,21 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(ageTrend=[CASE(>(COUNT() OVER (ROWS 1 PRECEDING), 1), /(SUM($8) OVER (ROWS 1 PRECEDING), CAST(COUNT($8) OVER (ROWS 1 PRECEDING)):DOUBLE NOT NULL), null:NULL)]) + LogicalProject(ageTrend=[CASE(>(COUNT() OVER (ROWS 1 PRECEDING), 1), /(CASE(>(COUNT($8) OVER (ROWS 1 PRECEDING), 0), SUM($8) OVER (ROWS 1 PRECEDING), null:BIGINT), CAST(COUNT($8) OVER (ROWS 1 PRECEDING)):DOUBLE NOT NULL), null:DOUBLE)]) LogicalFilter(condition=[IS NOT NULL($8)]) LogicalSort(sort0=[$8], dir0=[ASC]) - LogicalSort(fetch=[5]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[>($t1, $t4)], expr#6=[CAST($t3):DOUBLE NOT NULL], expr#7=[/($t2, $t6)], expr#8=[null:NULL], expr#9=[CASE($t5, $t7, $t8)], ageTrend=[$t9]) - EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])], constants=[[1]]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1:BIGINT], expr#5=[>($t1, $t4)], expr#6=[0:BIGINT], expr#7=[>($t2, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t3, $t8)], expr#10=[CAST($t2):DOUBLE NOT NULL], expr#11=[/($t9, $t10)], expr#12=[null:DOUBLE], expr#13=[CASE($t5, $t11, $t12)], ageTrend=[$t13]) + EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), COUNT($0), $SUM0($0)])], constants=[[1]]) EnumerableCalc(expr#0=[{inputs}], expr#1=[IS NOT NULL($t0)], age=[$t0], $condition=[$t1]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5, SORT->[{ "age" : { "order" : "asc", "missing" : "_last" } - }]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]},"sort":[{"age":{"order":"asc","missing":"_last"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)]) \ No newline at end of file + }]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]},"sort":[{"age":{"order":"asc","missing":"_last"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_week_argument_coercion.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_week_argument_coercion.json deleted file mode 100644 index 25891f0e629..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_week_argument_coercion.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(w=[WEEK(TIMESTAMP('2024-12-10':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0..16=[{inputs}], expr#17=['2024-12-10':VARCHAR], expr#18=[TIMESTAMP($t17)], expr#19=[WEEK($t18)], w=[$t19])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_week_argument_coercion.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_week_argument_coercion.yaml new file mode 100644 index 00000000000..6ebdc149edb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_week_argument_coercion.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(w=[WEEK(TIMESTAMP('2024-12-10'))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['2024-12-10'], expr#18=[TIMESTAMP($t17)], expr#19=[WEEK($t18)], w=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m"}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/filter_then_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/filter_then_limit_push.yaml new file mode 100644 index 00000000000..fd240348962 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/filter_then_limit_push.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(fetch=[5]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[>($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], FILTER->>($0, 30), LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/search_with_absolute_time_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite/search_with_absolute_time_range.yaml index a3bcf11b79e..d9be2c4bb74 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/search_with_absolute_time_range.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/search_with_absolute_time_range.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) - LogicalFilter(condition=[query_string(MAP('query', '(@timestamp:>=2022\-12\-10T13\:11\:04Z) AND (@timestamp:<=2025\-09\-03T15\:10\:00Z)':VARCHAR))]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, '(@timestamp:>=2022\-12\-10T13\:11\:04Z) AND (@timestamp:<=2025\-09\-03T15\:10\:00Z)':VARCHAR))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[@timestamp, category, value, timestamp], FILTER->query_string(MAP('query', '(@timestamp:>=2022\-12\-10T13\:11\:04Z) AND (@timestamp:<=2025\-09\-03T15\:10\:00Z)':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=2022\\-12\\-10T13\\:11\\:04Z) AND (@timestamp:<=2025\\-09\\-03T15\\:10\\:00Z)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["@timestamp","category","value","timestamp"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[@timestamp, category, value, timestamp], FILTER->query_string(MAP('query':VARCHAR, '(@timestamp:>=2022\-12\-10T13\:11\:04Z) AND (@timestamp:<=2025\-09\-03T15\:10\:00Z)':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=2022\\-12\\-10T13\\:11\\:04Z) AND (@timestamp:<=2025\\-09\\-03T15\\:10\\:00Z)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["@timestamp","category","value","timestamp"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/search_with_chained_time_modifier.yaml b/integ-test/src/test/resources/expectedOutput/calcite/search_with_chained_time_modifier.yaml index 85578283b39..b48f3966475 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/search_with_chained_time_modifier.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/search_with_chained_time_modifier.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) - LogicalFilter(condition=[query_string(MAP('query', '(@timestamp:>=now\-3d\/d\-2h\+10m) AND (@timestamp:<=now\-1d\+1y\/M)':VARCHAR))]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, '(@timestamp:>=now\-3d\/d\-2h\+10m) AND (@timestamp:<=now\-1d\+1y\/M)':VARCHAR))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[@timestamp, category, value, timestamp], FILTER->query_string(MAP('query', '(@timestamp:>=now\-3d\/d\-2h\+10m) AND (@timestamp:<=now\-1d\+1y\/M)':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=now\\-3d\\/d\\-2h\\+10m) AND (@timestamp:<=now\\-1d\\+1y\\/M)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["@timestamp","category","value","timestamp"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[@timestamp, category, value, timestamp], FILTER->query_string(MAP('query':VARCHAR, '(@timestamp:>=now\-3d\/d\-2h\+10m) AND (@timestamp:<=now\-1d\+1y\/M)':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=now\\-3d\\/d\\-2h\\+10m) AND (@timestamp:<=now\\-1d\\+1y\\/M)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["@timestamp","category","value","timestamp"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/search_with_numeric_time_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite/search_with_numeric_time_range.yaml index 464a0e108fd..81f834b8058 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/search_with_numeric_time_range.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/search_with_numeric_time_range.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) - LogicalFilter(condition=[query_string(MAP('query', '(@timestamp:>=1000) AND (@timestamp:<=1754020061123.456)':VARCHAR))]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, '(@timestamp:>=1000) AND (@timestamp:<=1754020061123.456)':VARCHAR))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[@timestamp, category, value, timestamp], FILTER->query_string(MAP('query', '(@timestamp:>=1000) AND (@timestamp:<=1754020061123.456)':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=1000) AND (@timestamp:<=1754020061123.456)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["@timestamp","category","value","timestamp"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[@timestamp, category, value, timestamp], FILTER->query_string(MAP('query':VARCHAR, '(@timestamp:>=1000) AND (@timestamp:<=1754020061123.456)':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=1000) AND (@timestamp:<=1754020061123.456)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["@timestamp","category","value","timestamp"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/search_with_relative_time_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite/search_with_relative_time_range.yaml index 10badf7a8af..eef5ad89293 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/search_with_relative_time_range.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/search_with_relative_time_range.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) - LogicalFilter(condition=[query_string(MAP('query', '(@timestamp:>=now\-3M) AND (@timestamp:<=now\+30d)':VARCHAR))]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, '(@timestamp:>=now\-3M) AND (@timestamp:<=now\+30d)':VARCHAR))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[@timestamp, category, value, timestamp], FILTER->query_string(MAP('query', '(@timestamp:>=now\-3M) AND (@timestamp:<=now\+30d)':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=now\\-3M) AND (@timestamp:<=now\\+30d)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["@timestamp","category","value","timestamp"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[@timestamp, category, value, timestamp], FILTER->query_string(MAP('query':VARCHAR, '(@timestamp:>=now\-3M) AND (@timestamp:<=now\+30d)':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=now\\-3M) AND (@timestamp:<=now\\+30d)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["@timestamp","category","value","timestamp"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/search_with_relative_time_snap.yaml b/integ-test/src/test/resources/expectedOutput/calcite/search_with_relative_time_snap.yaml index 04b2b245ef1..a16f1520f5a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/search_with_relative_time_snap.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/search_with_relative_time_snap.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) - LogicalFilter(condition=[query_string(MAP('query', '(@timestamp:>=now\-3M\/y) AND (@timestamp:<=now)':VARCHAR))]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, '(@timestamp:>=now\-3M\/y) AND (@timestamp:<=now)':VARCHAR))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[@timestamp, category, value, timestamp], FILTER->query_string(MAP('query', '(@timestamp:>=now\-3M\/y) AND (@timestamp:<=now)':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=now\\-3M\\/y) AND (@timestamp:<=now)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["@timestamp","category","value","timestamp"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[@timestamp, category, value, timestamp], FILTER->query_string(MAP('query':VARCHAR, '(@timestamp:>=now\-3M\/y) AND (@timestamp:<=now)':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=now\\-3M\\/y) AND (@timestamp:<=now)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["@timestamp","category","value","timestamp"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/test_output_big5.log b/integ-test/src/test/resources/expectedOutput/calcite/test_output_big5.log new file mode 100644 index 00000000000..7b023e69f5d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/test_output_big5.log @@ -0,0 +1 @@ +(eval):1: no such file or directory: ./gradlew diff --git a/integ-test/src/test/resources/expectedOutput/calcite/udf_geoip_in_agg_pushed.yaml b/integ-test/src/test/resources/expectedOutput/calcite/udf_geoip_in_agg_pushed.yaml index 7cd4f5b4bf0..0d6194157c7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/udf_geoip_in_agg_pushed.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/udf_geoip_in_agg_pushed.yaml @@ -3,7 +3,7 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(count()=[$1], info.city=[$0]) LogicalAggregate(group=[{0}], count()=[COUNT()]) - LogicalProject(info.city=[ITEM(GEOIP('my-datasource':VARCHAR, $0), 'city')]) + LogicalProject(info.city=[ITEM(GEOIP('my-datasource', $0), 'city')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), info.city], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"info.city":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQEr3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJJVEVNIiwKICAgICJraW5kIjogIklURU0iLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiR0VPSVAiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInVkdCI6ICJFWFBSX0lQIiwKICAgICAgICAgICAgInR5cGUiOiAiT1RIRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdLAogICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIk1BUCIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgImtleSI6IHsKICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICB9LAogICAgICAgICJ2YWx1ZSI6IHsKICAgICAgICAgICJ0eXBlIjogIkFOWSIsCiAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgICAgICJzY2FsZSI6IC0yMTQ3NDgzNjQ4CiAgICAgICAgfQogICAgICB9LAogICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICJkeW5hbWljIjogZmFsc2UKICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[2,0,2],"DIGESTS":["my-datasource","host","city"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/access_struct_subfield_with_item.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/access_struct_subfield_with_item.yaml index afd78ed8c22..64a5b5f46ea 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/access_struct_subfield_with_item.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/access_struct_subfield_with_item.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(host=[$0], info=[GEOIP('dummy-datasource':VARCHAR, $0)], info.dummy_sub_field=[ITEM(GEOIP('dummy-datasource':VARCHAR, $0), 'dummy_sub_field')]) + LogicalProject(host=[$0], info=[GEOIP('dummy-datasource', $0)], info.dummy_sub_field=[ITEM(GEOIP('dummy-datasource', $0), 'dummy_sub_field')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..11=[{inputs}], expr#12=['dummy-datasource':VARCHAR], expr#13=[GEOIP($t12, $t0)], expr#14=['dummy_sub_field'], expr#15=[ITEM($t13, $t14)], host=[$t0], info=[$t13], info.dummy_sub_field=[$t15]) + EnumerableCalc(expr#0..11=[{inputs}], expr#12=['dummy-datasource'], expr#13=[GEOIP($t12, $t0)], expr#14=['dummy_sub_field'], expr#15=[ITEM($t13, $t14)], host=[$t0], info=[$t13], info.dummy_sub_field=[$t15]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_date_range_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_date_range_push.yaml index f99713d9aaa..a9cb1b237fd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_date_range_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_date_range_push.yaml @@ -2,14 +2,14 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(avg(value)=[$2], span(@timestamp,1h)=[$1], value_range=[$0]) - LogicalAggregate(group=[{0, 2}], avg(value)=[AVG($1)]) - LogicalProject(value_range=[$10], value=[$2], span(@timestamp,1h)=[SPAN($0, 1, 'h')]) + LogicalAggregate(group=[{0, 1}], avg(value)=[AVG($2)]) + LogicalProject(value_range=[$10], span(@timestamp,1h)=[SPAN($0, 1, 'h')], value=[$2]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9], value_range=[CASE(<($2, 7000), 'small':VARCHAR, 'large':VARCHAR)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg(value)=[$t9], span(@timestamp,1h)=[$t1], value_range=[$t0]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..9=[{inputs}], expr#10=[7000], expr#11=[<($t2, $t10)], expr#12=['small':VARCHAR], expr#13=['large':VARCHAR], expr#14=[CASE($t11, $t12, $t13)], expr#15=[1], expr#16=['h'], expr#17=[SPAN($t0, $t15, $t16)], expr#18=[IS NOT NULL($t0)], value_range=[$t14], value=[$t2], span(@timestamp,1h)=[$t17], $condition=[$t18]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[7000], expr#11=[<($t2, $t10)], expr#12=['small':VARCHAR], expr#13=['large':VARCHAR], expr#14=[CASE($t11, $t12, $t13)], expr#15=[1], expr#16=['h'], expr#17=[SPAN($t0, $t15, $t16)], expr#18=[IS NOT NULL($t0)], value_range=[$t14], span(@timestamp,1h)=[$t17], value=[$t2], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_sort_agg_measure_not_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_sort_agg_measure_not_push.yaml new file mode 100644 index 00000000000..9d0ec052be4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_sort_agg_measure_not_push.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) + LogicalProject(avg(age)=[$2], age_range=[$0], balance_range=[$1]) + LogicalAggregate(group=[{0, 1}], avg(age)=[AVG($2)]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, <($10, 40), 'u40':VARCHAR, 'u100':VARCHAR)], balance_range=[CASE(<($7, 20000), 'medium':VARCHAR, 'high':VARCHAR)], age=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg(age)=[$t9], age_range=[$t0], balance_range=[$t1]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[40], expr#23=[<($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], expr#27=[20000], expr#28=[<($t7, $t27)], expr#29=['medium':VARCHAR], expr#30=['high':VARCHAR], expr#31=[CASE($t28, $t29, $t30)], age_range=[$t26], balance_range=[$t31], age=[$t10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml index 67ad0f0fd07..1ae9205fa10 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml @@ -10,4 +10,4 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(age)=[$t8], age_range=[$t0]) EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[Sarg[[30..40)]], expr#23=[SEARCH($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], age_range=[$t26], age=[$t10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml index 10ead7ad449..6da3fa554ec 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml @@ -3,11 +3,11 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(avg(balance)=[$1], age_range=[$0]) LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) - LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[35..40), [80..+∞)]), '30-40 or >=80':VARCHAR, null:NULL)], balance=[$7]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[35..40), [80..+∞)]), '30-40 or >=80':VARCHAR, null:VARCHAR)], balance=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(balance)=[$t8], age_range=[$t0]) EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[Sarg[[35..40), [80..+∞)]], expr#23=[SEARCH($t10, $t22)], expr#24=['30-40 or >=80':VARCHAR], expr#25=[null:NULL], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], age_range=[$t26], balance=[$t7]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[Sarg[[35..40), [80..+∞)]], expr#23=[SEARCH($t10, $t22)], expr#24=['30-40 or >=80':VARCHAR], expr#25=[null:VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], age_range=[$t26], balance=[$t7]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml index a81e208bdbf..6a33abfd5df 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml @@ -10,4 +10,4 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg_age=[$t8], age_range=[$t0]) EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[40], expr#23=[<($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], age_range=[$t26], age=[$t10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml index 404726f6083..86dcdf896b7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml @@ -10,4 +10,4 @@ calcite: EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_balance=[$t9], age_range=[$t0], balance_range=[$t1]) EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[40], expr#23=[<($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], expr#27=[20000], expr#28=[<($t7, $t27)], expr#29=['medium':VARCHAR], expr#30=['high':VARCHAR], expr#31=[CASE($t28, $t29, $t30)], age_range=[$t26], balance_range=[$t31], balance=[$t7]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml index 95e83cdcd19..ac9636c8ffe 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml @@ -3,7 +3,7 @@ calcite: LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) + LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'NULL':VARCHAR, <=($5, 10), $1, 'OTHER':VARCHAR)], avg(balance)=[$2]) LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) @@ -23,7 +23,7 @@ calcite: EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL':VARCHAR], expr#7=[10:BIGINT], expr#8=[<=($t4, $t7)], expr#9=['OTHER':VARCHAR], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_groups.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_groups.yaml index df3fd8391d5..0fec3887ff3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_groups.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_groups.yaml @@ -14,7 +14,6 @@ logical: | LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) LogicalProject(gender=[$4], age=[$10], balance=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - physical: | EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) EnumerableLimit(fetch=[10000]) @@ -34,4 +33,3 @@ physical: | EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[SAFE_CAST($t1)], avg(balance)=[$t9], age=[$t10]) EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml index 274186e377e..d3af3cf4001 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml @@ -3,19 +3,19 @@ calcite: LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'nil', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) + LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'nil':VARCHAR, <=($5, 10), $1, 'OTHER':VARCHAR)], avg(balance)=[$2]) LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], $f1=[SPAN($5, 10, null:ANY)], balance=[$3]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($3))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], $f1=[SPAN($5, 10, null:ANY)], balance=[$3]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($3))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) physical: | @@ -23,18 +23,18 @@ calcite: EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil':VARCHAR], expr#7=[10:BIGINT], expr#8=[<=($t4, $t7)], expr#9=['OTHER':VARCHAR], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], expr#17=[IS NOT NULL($t3)], expr#18=[AND($t16, $t17)], gender=[$t4], balance=[$t3], age0=[$t15], $condition=[$t18]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:ANY], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], expr#17=[IS NOT NULL($t3)], expr#18=[AND($t16, $t17)], gender=[$t4], $f1=[$t15], balance=[$t3], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], expr#17=[IS NOT NULL($t3)], expr#18=[SAFE_CAST($t15)], expr#19=[IS NOT NULL($t18)], expr#20=[AND($t16, $t17, $t19)], gender=[$t4], balance=[$t3], age0=[$t15], $condition=[$t20]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:ANY], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], expr#17=[IS NOT NULL($t3)], expr#18=[SAFE_CAST($t15)], expr#19=[IS NOT NULL($t18)], expr#20=[AND($t16, $t17, $t19)], gender=[$t4], $f1=[$t15], balance=[$t3], $condition=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group.yaml index 208fdf99935..174bcb92567 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group.yaml @@ -5,11 +5,9 @@ logical: | LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) LogicalProject(gender=[$4], balance=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - physical: | EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(balance)=[$t8], gender=[$t0]) EnumerableAggregate(group=[{4}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_timestamp_span_and_category.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_timestamp_span_and_category.yaml index 76b833ce3f1..37e08f8968a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_timestamp_span_and_category.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_timestamp_span_and_category.yaml @@ -3,36 +3,36 @@ calcite: LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) LogicalAggregate(group=[{0, 1}], max(value)=[MAX($2)]) - LogicalProject(timestamp=[$0], category=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], max(value)=[$2]) + LogicalProject(timestamp=[$0], category=[CASE(IS NULL($1), 'NULL':VARCHAR, <=($5, 10), $1, 'OTHER':VARCHAR)], max(value)=[$2]) LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalProject(timestamp=[$1], category=[$0], max(value)=[$2]) - LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) - LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) + LogicalAggregate(group=[{0, 1}], max(value)=[MAX($2)]) + LogicalProject(category=[$1], timestamp=[SPAN($3, 1, 'w')], value=[$2]) LogicalFilter(condition=[AND(IS NOT NULL($3), IS NOT NULL($2))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) LogicalProject(category=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(category=[$0], max(value)=[$2]) - LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) - LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) + LogicalAggregate(group=[{0, 1}], max(value)=[MAX($2)]) + LogicalProject(category=[$1], $f1=[SPAN($3, 1, 'w')], value=[$2]) LogicalFilter(condition=[AND(IS NOT NULL($3), IS NOT NULL($2))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) EnumerableAggregate(group=[{0, 1}], max(value)=[MAX($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], timestamp=[$t0], category=[$t10], max(value)=[$t2]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL':VARCHAR], expr#7=[10:BIGINT], expr#8=[<=($t4, $t7)], expr#9=['OTHER':VARCHAR], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], timestamp=[$t0], category=[$t10], max(value)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], timestamp=[$t1], category=[$t0], max(value)=[$t2]) - EnumerableAggregate(group=[{0, 2}], max(value)=[MAX($1)]) - EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['w'], expr#12=[SPAN($t3, $t10, $t11)], expr#13=[IS NOT NULL($t3)], expr#14=[IS NOT NULL($t2)], expr#15=[AND($t13, $t14)], category=[$t1], value=[$t2], timestamp0=[$t12], $condition=[$t15]) + EnumerableAggregate(group=[{0, 1}], max(value)=[MAX($2)]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['w'], expr#12=[SPAN($t3, $t10, $t11)], expr#13=[IS NOT NULL($t3)], expr#14=[IS NOT NULL($t2)], expr#15=[AND($t13, $t14)], category=[$t1], timestamp=[$t12], value=[$t2], $condition=[$t15]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($2)]) - EnumerableAggregate(group=[{0, 2}], max(value)=[MAX($1)]) - EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['w'], expr#12=[SPAN($t3, $t10, $t11)], expr#13=[IS NOT NULL($t3)], expr#14=[IS NOT NULL($t2)], expr#15=[IS NOT NULL($t1)], expr#16=[AND($t13, $t14, $t15)], category=[$t1], value=[$t2], timestamp0=[$t12], $condition=[$t16]) + EnumerableAggregate(group=[{0, 1}], max(value)=[MAX($2)]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['w'], expr#12=[SPAN($t3, $t10, $t11)], expr#13=[IS NOT NULL($t3)], expr#14=[IS NOT NULL($t2)], expr#15=[IS NOT NULL($t1)], expr#16=[AND($t13, $t14, $t15)], category=[$t1], timestamp=[$t12], value=[$t2], $condition=[$t16]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_use_other.yaml index 027d0e30124..897b108a9c6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_use_other.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_use_other.yaml @@ -3,7 +3,7 @@ calcite: LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - LogicalProject(flags=[$0], severityText=[CASE(IS NULL($1), 'NULL', <=($5, 2), $1, 'max_among_other')], max(severityNumber)=[$2]) + LogicalProject(flags=[$0], severityText=[CASE(IS NULL($1), 'NULL':VARCHAR, <=($5, 2), $1, 'max_among_other':VARCHAR)], max(severityNumber)=[$2]) LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) @@ -21,7 +21,7 @@ calcite: EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) EnumerableAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[2], expr#8=[<=($t4, $t7)], expr#9=['max_among_other'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], flags=[$t0], severityText=[$t10], max(severityNumber)=[$t2]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL':VARCHAR], expr#7=[2:BIGINT], expr#8=[<=($t4, $t7)], expr#9=['max_among_other':VARCHAR], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], flags=[$t0], severityText=[$t10], max(severityNumber)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], flags=[$t1], severityText=[$t0], max(severityNumber)=[$t2]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_integer_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_integer_span.yaml index 5e3a5d0ba33..391f70a8196 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_integer_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_integer_span.yaml @@ -2,14 +2,13 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[ASC]) - LogicalProject(age=[$0], max(balance)=[$1]) - LogicalAggregate(group=[{1}], max(balance)=[MAX($0)]) - LogicalProject(balance=[$7], age0=[SPAN($10, 10, null:NULL)]) - LogicalFilter(condition=[AND(IS NOT NULL($10), IS NOT NULL($7))]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalAggregate(group=[{0}], max(balance)=[MAX($1)]) + LogicalProject(age=[SPAN($10, 10, null:ANY)], balance=[$7]) + LogicalFilter(condition=[AND(IS NOT NULL($10), IS NOT NULL($7))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableAggregate(group=[{1}], max(balance)=[MAX($0)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[10], expr#20=[null:NULL], expr#21=[SPAN($t10, $t19, $t20)], expr#22=[IS NOT NULL($t10)], expr#23=[IS NOT NULL($t7)], expr#24=[AND($t22, $t23)], balance=[$t7], age0=[$t21], $condition=[$t24]) + EnumerableAggregate(group=[{0}], max(balance)=[MAX($1)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[10], expr#20=[null:ANY], expr#21=[SPAN($t10, $t19, $t20)], expr#22=[IS NOT NULL($t10)], expr#23=[IS NOT NULL($t7)], expr#24=[AND($t22, $t23)], age=[$t21], balance=[$t7], $condition=[$t24]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_timestamp_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_timestamp_span.yaml index a8bd9d61f77..6d65fb4afa1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_timestamp_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_timestamp_span.yaml @@ -2,14 +2,13 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[ASC]) - LogicalProject(@timestamp=[$0], count=[$1]) - LogicalAggregate(group=[{0}], count=[COUNT()]) - LogicalProject(@timestamp0=[SPAN($0, 1, 'd')]) - LogicalFilter(condition=[IS NOT NULL($0)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalAggregate(group=[{0}], count=[COUNT()]) + LogicalProject(@timestamp=[SPAN($0, 1, 'd')]) + LogicalFilter(condition=[IS NOT NULL($0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableAggregate(group=[{0}], count=[COUNT()]) - EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['d'], expr#12=[SPAN($t0, $t10, $t11)], expr#13=[IS NOT NULL($t0)], @timestamp0=[$t12], $condition=[$t13]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['d'], expr#12=[SPAN($t0, $t10, $t11)], expr#13=[IS NOT NULL($t0)], @timestamp=[$t12], $condition=[$t13]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_on_window.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_on_window.yaml new file mode 100644 index 00000000000..73d6c9b85a0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_on_window.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], gender=[$1], age=[$2]) + LogicalFilter(condition=[<=($3, 1)]) + LogicalProject(account_number=[$0], gender=[$1], age=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $1)]) + LogicalFilter(condition=[IS NOT NULL($1)]) + LogicalProject(account_number=[$0], gender=[$4], age=[$8]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[1:BIGINT], expr#19=[<=($t17, $t18)], account_number=[$t0], gender=[$t4], age=[$t8], $condition=[$t19]) + EnumerableWindow(window#0=[window(partition {4} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t4)], proj#0..16=[{exprs}], $condition=[$t17]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml index 285d0b221e1..7ac6792d4c9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml @@ -3,7 +3,7 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(sum=[$2], len=[$0], gender=[$1]) LogicalAggregate(group=[{0, 1}], sum=[SUM($2)]) - LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f3=[+($7, 100)]) + LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f2=[+($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml index bf861c337b9..72c5b7c8017 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml @@ -3,7 +3,7 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0]) LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)]) - LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)]) + LogicalProject(gender=[$4], balance=[$7], $f2=[+($7, 100)], $f3=[-($7, 100)], $f4=[*($7, 100)], $f5=[DIVIDE($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_append_command.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_append_command.yaml index eb4b6f09d58..7f76e789b4e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_append_command.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_append_command.yaml @@ -17,4 +17,4 @@ calcite: CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) EnumerableCalc(expr#0=[{inputs}], expr#1=[null:VARCHAR], proj#0..1=[{exprs}]) EnumerableAggregate(group=[{}], cnt=[COUNT()]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_appendpipe_command.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_appendpipe_command.json deleted file mode 100644 index 2b111e119db..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_appendpipe_command.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], cnt=[$19])\n LogicalUnion(all=[true])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], cnt=[null:BIGINT])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n LogicalProject(account_number=[null:BIGINT], firstname=[null:VARCHAR], address=[null:VARCHAR], birthdate=[null:EXPR_TIMESTAMP VARCHAR], gender=[$0], city=[null:VARCHAR], lastname=[null:VARCHAR], balance=[null:BIGINT], employer=[null:VARCHAR], state=[null:VARCHAR], age=[null:INTEGER], email=[null:VARCHAR], male=[null:BOOLEAN], _id=[null:VARCHAR], _index=[null:VARCHAR], _score=[null:REAL], _maxscore=[null:REAL], _sort=[null:BIGINT], _routing=[null:VARCHAR], cnt=[$1])\n LogicalAggregate(group=[{0}], cnt=[COUNT($1)])\n LogicalProject(gender=[$4], balance=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical":"EnumerableLimit(fetch=[10000])\n EnumerableUnion(all=[true])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[null:BIGINT], proj#0..12=[{exprs}], cnt=[$t19])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[null:BIGINT], expr#3=[null:VARCHAR], expr#4=[null:EXPR_TIMESTAMP VARCHAR], expr#5=[null:INTEGER], expr#6=[null:BOOLEAN], account_number=[$t2], firstname=[$t3], address=[$t3], birthdate=[$t4], gender=[$t0], city=[$t3], lastname=[$t3], balance=[$t2], employer=[$t3], state=[$t3], age=[$t5], email=[$t3], male=[$t6], cnt=[$t1])\n EnumerableAggregate(group=[{4}], cnt=[COUNT($7)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_appendpipe_command.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_appendpipe_command.yaml new file mode 100644 index 00000000000..906e4693dfb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_appendpipe_command.yaml @@ -0,0 +1,19 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], cnt=[$19]) + LogicalUnion(all=[true]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], cnt=[null:BIGINT]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalProject(account_number=[null:BIGINT], firstname=[null:VARCHAR], address=[null:VARCHAR], birthdate=[null:VARCHAR], gender=[$0], city=[null:VARCHAR], lastname=[null:VARCHAR], balance=[null:BIGINT], employer=[null:VARCHAR], state=[null:VARCHAR], age=[null:INTEGER], email=[null:VARCHAR], male=[null:BOOLEAN], _id=[null:VARCHAR], _index=[null:VARCHAR], _score=[SAFE_CAST(null:REAL)], _maxscore=[SAFE_CAST(null:REAL)], _sort=[null:BIGINT], _routing=[null:VARCHAR], cnt=[$1]) + LogicalAggregate(group=[{0}], cnt=[COUNT($1)]) + LogicalProject(gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableUnion(all=[true]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[null:BIGINT], proj#0..12=[{exprs}], cnt=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[null:BIGINT], expr#3=[null:VARCHAR], expr#4=[null:INTEGER], expr#5=[null:BOOLEAN], account_number=[$t2], firstname=[$t3], address=[$t3], birthdate=[$t3], gender=[$t0], city=[$t3], lastname=[$t3], balance=[$t2], employer=[$t3], state=[$t3], age=[$t4], email=[$t3], male=[$t5], cnt=[$t1]) + EnumerableAggregate(group=[{4}], cnt=[COUNT($7)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_aligntime.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_aligntime.yaml index e8a727cf1e3..32dd22a9622 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_aligntime.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_aligntime.yaml @@ -1,12 +1,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(category=[$0], value=[$1], timestamp=[$2], @timestamp=[$9]) - LogicalSort(fetch=[5]) - LogicalProject(category=[$1], value=[$2], timestamp=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9], @timestamp=[FROM_UNIXTIME(*(*(FLOOR(DIVIDE(DIVIDE(UNIX_TIMESTAMP($0), 3600), 2)), 2), 3600))]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalSort(fetch=[5]) + LogicalProject(category=[$1], value=[$2], timestamp=[$3], @timestamp=[FROM_UNIXTIME(*(*(FLOOR(DIVIDE(DIVIDE(UNIX_TIMESTAMP($0), 3600), 2)), 2), 3600))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..9=[{inputs}], expr#10=[UNIX_TIMESTAMP($t0)], expr#11=[3600], expr#12=[DIVIDE($t10, $t11)], expr#13=[2], expr#14=[DIVIDE($t12, $t13)], expr#15=[FLOOR($t14)], expr#16=[*($t15, $t13)], expr#17=[*($t16, $t11)], expr#18=[FROM_UNIXTIME($t17)], category=[$t1], value=[$t2], timestamp=[$t3], @timestamp=[$t18]) EnumerableLimit(fetch=[5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_bins.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_bins.json deleted file mode 100644 index bbdde96acf1..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_bins.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[WIDTH_BUCKET($8, 3, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[3], expr#14=[-($t11, $t12)], expr#15=[WIDTH_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_bins.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_bins.yaml new file mode 100644 index 00000000000..40c05d0e29e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_bins.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], age=[WIDTH_BUCKET($8, 3, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[3], expr#14=[-($t11, $t12)], expr#15=[WIDTH_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15]) + EnumerableLimit(fetch=[5]) + EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])]) + EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json deleted file mode 100644 index a31d2acfc61..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.yaml new file mode 100644 index 00000000000..2cd36a4f75a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15]) + EnumerableLimit(fetch=[5]) + EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])]) + EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_span.yaml index 13a1b2e5029..e1253f57f4c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_span.yaml @@ -1,10 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16]) - LogicalSort(fetch=[5]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[SPAN_BUCKET($8, 10)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], age=[SPAN_BUCKET($8, 10)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[10], expr#18=[SPAN_BUCKET($t8, $t17)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t18]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_start_end.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_start_end.json deleted file mode 100644 index 2a6027db5ee..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_start_end.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], gender=[$3], city=[$4], employer=[$5], state=[$6], age=[$7], email=[$8], lastname=[$9], balance=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], balance=[RANGE_BUCKET($3, MIN($3) OVER (), MAX($3) OVER (), 0, 100001)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[0], expr#14=[100001], expr#15=[RANGE_BUCKET($t3, $t11, $t12, $t13, $t14)], proj#0..2=[{exprs}], gender=[$t4], city=[$t5], employer=[$t6], state=[$t7], age=[$t8], email=[$t9], lastname=[$t10], balance=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MIN($3), MAX($3)])])\n EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_start_end.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_start_end.yaml new file mode 100644 index 00000000000..5cc8fbf2ba1 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_start_end.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], balance=[RANGE_BUCKET($3, MIN($3) OVER (), MAX($3) OVER (), 0, 100001)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[0], expr#14=[100001], expr#15=[RANGE_BUCKET($t3, $t11, $t12, $t13, $t14)], proj#0..2=[{exprs}], gender=[$t4], city=[$t5], employer=[$t6], state=[$t7], age=[$t8], email=[$t9], lastname=[$t10], balance=[$t15]) + EnumerableLimit(fetch=[5]) + EnumerableWindow(window#0=[window(aggs [MIN($3), MAX($3)])]) + EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_pushdown_for_smj.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_pushdown_for_smj.yaml index 9b357fbbfed..e35f2db7b58 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_pushdown_for_smj.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_pushdown_for_smj.yaml @@ -1,20 +1,21 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[$13], b.account_number=[$14], b.firstname=[$15], b.address=[$16], b.birthdate=[$17], b.gender=[$18], b.city=[$19], b.lastname=[$20], b.balance=[$21], b.employer=[$22], b.state=[$23], b.age=[$24], b.email=[$25], b.male=[$26]) - LogicalJoin(condition=[=($13, $15)], joinType=[inner]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[REX_EXTRACT($6, '(?^[A-Z])', 'initial')]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[$13], b.account_number=[$15], b.firstname=[$16], b.address=[$17], b.birthdate=[$18], b.gender=[$19], b.city=[$20], b.lastname=[$21], b.balance=[$22], b.employer=[$23], b.state=[$24], b.age=[$25], b.email=[$26], b.male=[$27]) + LogicalJoin(condition=[=($14, $16)], joinType=[inner]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[REX_EXTRACT($6, '(?^[A-Z])', 'initial')], $f14=[CAST(REX_EXTRACT($6, '(?^[A-Z])', 'initial')):VARCHAR]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[50000]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableLimit(fetch=[10000]) - EnumerableMergeJoin(condition=[=($13, $15)], joinType=[inner]) - EnumerableSort(sort0=[$13], dir0=[ASC]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=['(?^[A-Z])'], expr#20=['initial'], expr#21=[REX_EXTRACT($t6, $t19, $t20)], proj#0..12=[{exprs}], initial=[$t21]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableLimit(fetch=[50000]) - EnumerableCalc(expr#0..18=[{inputs}], proj#0..12=[{exprs}]) + EnumerableCalc(expr#0..27=[{inputs}], proj#0..13=[{exprs}], b.account_number=[$t15], b.firstname=[$t16], b.address=[$t17], b.birthdate=[$t18], b.gender=[$t19], b.city=[$t20], b.lastname=[$t21], b.balance=[$t22], b.employer=[$t23], b.state=[$t24], b.age=[$t25], b.email=[$t26], b.male=[$t27]) + EnumerableLimit(fetch=[10000]) + EnumerableMergeJoin(condition=[=($14, $16)], joinType=[inner]) + EnumerableSort(sort0=[$14], dir0=[ASC]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=['(?^[A-Z])'], expr#20=['initial'], expr#21=[REX_EXTRACT($t6, $t19, $t20)], expr#22=[CAST($t21):VARCHAR], proj#0..12=[{exprs}], initial=[$t21], $f14=[$t22]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableLimit(fetch=[50000]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..12=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_pushdown_for_smj_w_max_option.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_pushdown_for_smj_w_max_option.yaml index 07684889396..58f53e4f763 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_pushdown_for_smj_w_max_option.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_pushdown_for_smj_w_max_option.yaml @@ -1,13 +1,13 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$13], firstname=[$14], address=[$15], birthdate=[$16], gender=[$17], city=[$18], lastname=[$19], balance=[$20], employer=[$21], state=[$22], age=[$23], email=[$24], male=[$25]) - LogicalJoin(condition=[=($12, $19)], joinType=[left]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], lastname=[REX_EXTRACT($6, '(?^[A-Z])', 'lastname')]) + LogicalProject(account_number=[$14], firstname=[$15], address=[$16], birthdate=[$17], gender=[$18], city=[$19], lastname=[$20], balance=[$21], employer=[$22], state=[$23], age=[$24], email=[$25], male=[$26]) + LogicalJoin(condition=[=($13, $20)], joinType=[left]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], lastname=[REX_EXTRACT($6, '(?^[A-Z])', 'lastname')], $f13=[CAST(REX_EXTRACT($6, '(?^[A-Z])', 'lastname')):VARCHAR]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[50000]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) - LogicalFilter(condition=[<=($13, 1)]) + LogicalFilter(condition=[<=($13, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _row_number_join_max_dedup_=[ROW_NUMBER() OVER (PARTITION BY $6)]) LogicalFilter(condition=[IS NOT NULL($6)]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) @@ -17,11 +17,11 @@ calcite: EnumerableLimit(fetch=[10000]) EnumerableMergeJoin(condition=[=($0, $7)], joinType=[left]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=['(?^[A-Z])'], expr#20=['lastname'], expr#21=[REX_EXTRACT($t6, $t19, $t20)], lastname=[$t21]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=['(?^[A-Z])'], expr#20=['lastname'], expr#21=[REX_EXTRACT($t6, $t19, $t20)], expr#22=[CAST($t21):VARCHAR], $f13=[$t22]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) EnumerableSort(sort0=[$6], dir0=[ASC]) EnumerableLimit(fetch=[50000]) - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[1], expr#21=[<=($t19, $t20)], proj#0..12=[{exprs}], $condition=[$t21]) + EnumerableCalc(expr#0..19=[{inputs}], expr#20=[1:BIGINT], expr#21=[<=($t19, $t20)], proj#0..12=[{exprs}], $condition=[$t21]) EnumerableWindow(window#0=[window(partition {6} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t6)], proj#0..18=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_then_field_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_then_field_sort.yaml index 362f847ae6e..b8af5829400 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_then_field_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_then_field_sort.yaml @@ -1,12 +1,11 @@ calcite: logical: | LogicalSystemLimit(sort0=[$10], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], age2=[$19], balance2=[$20]) - LogicalSort(sort0=[$10], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[$19], balance2=[ABS($7)]) - LogicalSort(sort0=[$19], sort1=[$10], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSort(sort0=[$10], dir0=[ASC-nulls-first]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], age2=[$19], balance2=[ABS($7)]) + LogicalSort(sort0=[$19], sort1=[$10], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..13=[{inputs}], expr#14=[ABS($t7)], proj#0..14=[{exprs}]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_count_eval_complex_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_count_eval_complex_push.json deleted file mode 100644 index 1c4cb0bf63e..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_count_eval_complex_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], mature_count=[COUNT($0)])\n LogicalProject($f1=[CASE(SEARCH($10, Sarg[(30..50)]), 1, null:NULL)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableAggregate(group=[{}], mature_count=[COUNT() FILTER $0])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[Sarg[(30..50)]], expr#20=[SEARCH($t10, $t19)], expr#21=[IS TRUE($t20)], $f1=[$t21])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_count_eval_complex_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_count_eval_complex_push.yaml new file mode 100644 index 00000000000..a0761810b51 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_count_eval_complex_push.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], mature_count=[COUNT($0)]) + LogicalProject($f0=[CASE(SEARCH($10, Sarg[(30..50)]), 1, null:INTEGER)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableAggregate(group=[{}], mature_count=[COUNT() FILTER $0]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[Sarg[(30..50)]], expr#20=[SEARCH($t10, $t19)], expr#21=[IS TRUE($t20)], $f1=[$t21]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_count_eval_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_count_eval_push.json deleted file mode 100644 index 879da821403..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_count_eval_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], mature_count=[COUNT($0)])\n LogicalProject($f1=[CASE(>($10, 30), 1, null:NULL)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableAggregate(group=[{}], mature_count=[COUNT() FILTER $0])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[>($t10, $t19)], expr#21=[IS TRUE($t20)], $f1=[$t21])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_count_eval_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_count_eval_push.yaml new file mode 100644 index 00000000000..9687acb4119 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_count_eval_push.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], mature_count=[COUNT($0)]) + LogicalProject($f0=[CASE(>($10, 30), 1, null:INTEGER)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableAggregate(group=[{}], mature_count=[COUNT() FILTER $0]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[>($t10, $t19)], expr#21=[IS TRUE($t20)], $f1=[$t21]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_dedup_keepempty_false_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_dedup_keepempty_false_push.yaml index 6bb96baa243..02e23dedb2d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_dedup_keepempty_false_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_dedup_keepempty_false_push.yaml @@ -2,14 +2,14 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], gender=[$1], age=[$2]) - LogicalFilter(condition=[<=($3, 1)]) + LogicalFilter(condition=[<=($3, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $1)]) LogicalFilter(condition=[IS NOT NULL($1)]) LogicalProject(account_number=[$0], gender=[$4], age=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[1], expr#19=[<=($t17, $t18)], account_number=[$t0], gender=[$t4], age=[$t8], $condition=[$t19]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[1:BIGINT], expr#19=[<=($t17, $t18)], account_number=[$t0], gender=[$t4], age=[$t8], $condition=[$t19]) EnumerableWindow(window#0=[window(partition {4} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t4)], proj#0..16=[{exprs}], $condition=[$t17]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_dedup_keepempty_true_not_pushed.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_dedup_keepempty_true_not_pushed.yaml index 52d30ddf078..08afc693196 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_dedup_keepempty_true_not_pushed.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_dedup_keepempty_true_not_pushed.yaml @@ -2,11 +2,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], gender=[$1], age=[$2]) - LogicalFilter(condition=[OR(IS NULL($1), <=($3, 1))]) + LogicalFilter(condition=[OR(IS NULL($1), <=($3, SAFE_CAST(1:BIGINT)))]) LogicalProject(account_number=[$0], gender=[$4], age=[$8], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[IS NULL($t4)], expr#19=[1], expr#20=[<=($t17, $t19)], expr#21=[OR($t18, $t20)], account_number=[$t0], gender=[$t4], age=[$t8], $condition=[$t21]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[IS NULL($t4)], expr#19=[1:BIGINT], expr#20=[<=($t17, $t19)], expr#21=[OR($t18, $t20)], account_number=[$t0], gender=[$t4], age=[$t8], $condition=[$t21]) EnumerableWindow(window#0=[window(partition {4} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_dedup_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_dedup_push.yaml index 6bb96baa243..02e23dedb2d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_dedup_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_dedup_push.yaml @@ -2,14 +2,14 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], gender=[$1], age=[$2]) - LogicalFilter(condition=[<=($3, 1)]) + LogicalFilter(condition=[<=($3, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], gender=[$1], age=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $1)]) LogicalFilter(condition=[IS NOT NULL($1)]) LogicalProject(account_number=[$0], gender=[$4], age=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[1], expr#19=[<=($t17, $t18)], account_number=[$t0], gender=[$t4], age=[$t8], $condition=[$t19]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[1:BIGINT], expr#19=[<=($t17, $t18)], account_number=[$t0], gender=[$t4], age=[$t8], $condition=[$t19]) EnumerableWindow(window#0=[window(partition {4} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t4)], proj#0..16=[{exprs}], $condition=[$t17]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eval_max.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eval_max.yaml index cae22c2a7f4..5737a05a4ad 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eval_max.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eval_max.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], new=[SCALAR_MAX(1, 2, 3, $8, 'banana':VARCHAR)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], new=[SCALAR_MAX(1, 2, 3, $8, 'banana')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[1], expr#18=[2], expr#19=[3], expr#20=['banana':VARCHAR], expr#21=[SCALAR_MAX($t17, $t18, $t19, $t8, $t20)], proj#0..10=[{exprs}], new=[$t21]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[1], expr#18=[2], expr#19=[3], expr#20=['banana'], expr#21=[SCALAR_MAX($t17, $t18, $t19, $t8, $t20)], proj#0..10=[{exprs}], new=[$t21]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eval_min.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eval_min.yaml index 6468a521e38..b092828706d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eval_min.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eval_min.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], new=[SCALAR_MIN(1, 2, 3, $8, 'banana':VARCHAR)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], new=[SCALAR_MIN(1, 2, 3, $8, 'banana')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[1], expr#18=[2], expr#19=[3], expr#20=['banana':VARCHAR], expr#21=[SCALAR_MIN($t17, $t18, $t19, $t8, $t20)], proj#0..10=[{exprs}], new=[$t21]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[1], expr#18=[2], expr#19=[3], expr#20=['banana'], expr#21=[SCALAR_MIN($t17, $t18, $t19, $t8, $t20)], proj#0..10=[{exprs}], new=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_avg.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_avg.json index 119aaaf8f55..8d1a1e511a3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_avg.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_avg.json @@ -3,4 +3,4 @@ "logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_balance=[/(SUM($3) OVER (), CAST(COUNT($3) OVER ()):DOUBLE NOT NULL)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", "physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t18):DOUBLE NOT NULL], expr#20=[/($t17, $t19)], proj#0..10=[{exprs}], avg_balance=[$t20])\n EnumerableWindow(window#0=[window(aggs [$SUM0($3), COUNT($3)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_null_bucket.yaml index ad8f22e9421..8a7375bd477 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_null_bucket.yaml @@ -8,4 +8,4 @@ calcite: EnumerableCalc(expr#0..12=[{inputs}], expr#13=[null:BIGINT], expr#14=[CASE($t11, $t12, $t13)], proj#0..10=[{exprs}], count()=[$t14]) EnumerableWindow(window#0=[window(partition {7} aggs [COUNT()])]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t7)], proj#0..10=[{exprs}], $11=[$t17]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_correlated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_correlated_subquery.yaml index 400bd549ee8..737579d9dd0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_correlated_subquery.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_correlated_subquery.yaml @@ -1,25 +1,25 @@ calcite: logical: | LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(id=[$2], name=[$0], salary=[$4]) - LogicalSort(sort0=[$4], dir0=[DESC-nulls-last]) + LogicalSort(sort0=[$2], dir0=[DESC-nulls-last]) + LogicalProject(id=[$2], name=[$0], salary=[$4]) LogicalFilter(condition=[EXISTS({ - LogicalProject(name=[$0], uid=[$1], occupation=[$2], department=[$3]) - LogicalFilter(condition=[=($cor0.id, $1)]) - LogicalSystemLimit(fetch=[10000], type=[SUBSEARCH_MAXOUT]) + LogicalFilter(condition=[=($cor1.id, $1)]) + LogicalSort(fetch=[10000]) + LogicalProject(name=[$0], uid=[$1], occupation=[$2], department=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9]) LogicalFilter(condition=[=($0, 'Tom')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) - })], variablesSet=[[$cor0]]) + })], variablesSet=[[$cor1]]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) - EnumerableSort(sort0=[$2], dir0=[DESC-nulls-last]) - EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) + EnumerableSort(sort0=[$2], dir0=[DESC-nulls-last]) + EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) + EnumerableCorrelate(correlation=[$cor1], joinType=[inner], requiredColumns=[{1}]) EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2], salary=[$t4]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) EnumerableAggregate(group=[{0}]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], expr#3=[$cor0], expr#4=[$t3.id], expr#5=[=($t4, $t1)], i=[$t2], $condition=[$t5]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[true], expr#2=[$cor1], expr#3=[$t2.id], expr#4=[=($t3, $t0)], i=[$t1], $condition=[$t4]) EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..9=[{inputs}], expr#10=['Tom':VARCHAR], expr#11=[=($t0, $t10)], proj#0..1=[{exprs}], $condition=[$t11]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file + EnumerableCalc(expr#0..9=[{inputs}], expr#10=['Tom':VARCHAR], expr#11=[=($t0, $t10)], uid=[$t1], $condition=[$t11]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_uncorrelated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_uncorrelated_subquery.yaml index 3b4e34539c7..9cb111a6190 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_uncorrelated_subquery.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_uncorrelated_subquery.yaml @@ -1,14 +1,14 @@ calcite: logical: | LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(id=[$2], name=[$0], salary=[$4]) - LogicalSort(sort0=[$4], dir0=[DESC-nulls-last]) + LogicalSort(sort0=[$2], dir0=[DESC-nulls-last]) + LogicalProject(id=[$2], name=[$0], salary=[$4]) LogicalFilter(condition=[EXISTS({ - LogicalSystemLimit(fetch=[10000], type=[SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[10000]) LogicalProject(name=[$0], uid=[$1], occupation=[$2], department=[$3]) LogicalFilter(condition=[=($0, 'Tom')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) - })], variablesSet=[[$cor0]]) + })]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | EnumerableLimit(fetch=[10000]) @@ -21,4 +21,4 @@ calcite: EnumerableCalc(expr#0..9=[{inputs}], expr#10=[true], i=[$t10]) EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..9=[{inputs}], expr#10=['Tom':VARCHAR], expr#11=[=($t0, $t10)], proj#0..9=[{exprs}], $condition=[$t11]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_fillnull_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_fillnull_push.json deleted file mode 100644 index 5042ab5b94f..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_fillnull_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[COALESCE($8, -1)], balance=[COALESCE($3, -1)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[-1], expr#18=[COALESCE($t8, $t17)], expr#19=[COALESCE($t3, $t17)], age=[$t18], balance=[$t19])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_fillnull_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_fillnull_push.yaml new file mode 100644 index 00000000000..3ac3c1ecc71 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_fillnull_push.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[CASE(IS NOT NULL($8), CAST($8):BIGINT NOT NULL, -1:BIGINT)], balance=[CASE(IS NOT NULL($3), CAST($3):BIGINT NOT NULL, -1:BIGINT)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t8)], expr#18=[CAST($t8):BIGINT NOT NULL], expr#19=[-1:BIGINT], expr#20=[CASE($t17, $t18, $t19)], expr#21=[IS NOT NULL($t3)], expr#22=[CAST($t3):BIGINT NOT NULL], expr#23=[CASE($t21, $t22, $t19)], age=[$t20], balance=[$t23]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_fillnull_value_syntax.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_fillnull_value_syntax.yaml index 740047e9805..02b90cd23d8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_fillnull_value_syntax.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_fillnull_value_syntax.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[COALESCE($8, 0)], balance=[COALESCE($3, 0)]) + LogicalProject(age=[CASE(IS NOT NULL($8), CAST($8):BIGINT NOT NULL, 0:BIGINT)], balance=[CASE(IS NOT NULL($3), CAST($3):BIGINT NOT NULL, 0:BIGINT)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[0], expr#18=[COALESCE($t8, $t17)], expr#19=[COALESCE($t3, $t17)], age=[$t18], balance=[$t19]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t8)], expr#18=[CAST($t8):BIGINT NOT NULL], expr#19=[0:BIGINT], expr#20=[CASE($t17, $t18, $t19)], expr#21=[IS NOT NULL($t3)], expr#22=[CAST($t3):BIGINT NOT NULL], expr#23=[CASE($t21, $t22, $t19)], age=[$t20], balance=[$t23]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter.yaml new file mode 100644 index 00000000000..c26eafacd9d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(name=[$0], age=[$1]) + LogicalFilter(condition=[=($1, SAFE_CAST(20:BIGINT))]) + CalciteLogicalIndexScan(table=[[OpenSearch, test]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[20:BIGINT], expr#9=[=($t1, $t8)], proj#0..1=[{exprs}], $condition=[$t9]) + CalciteEnumerableIndexScan(table=[[OpenSearch, test]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml index ac3728eacb9..93baa219935 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml @@ -4,11 +4,11 @@ calcite: LogicalProject(avg_age=[$2], state=[$0], city=[$1]) LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) LogicalProject(state=[$7], city=[$5], age=[$8]) - LogicalFilter(condition=[>($8, 30)]) + LogicalFilter(condition=[>($8, SAFE_CAST(30:BIGINT))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_age=[$t9], state=[$t1], city=[$t0]) EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30:BIGINT], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ip.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ip.json deleted file mode 100644 index 9d963dd5747..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ip.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(host=[$0])\n LogicalFilter(condition=[GREATER_IP($0, IP('1.1.1.1':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..11=[{inputs}], expr#12=['1.1.1.1':VARCHAR], expr#13=[IP($t12)], expr#14=[GREATER_IP($t0, $t13)], host=[$t0], $condition=[$t14])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ip.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ip.yaml new file mode 100644 index 00000000000..91a90f30ea9 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ip.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(host=[$0]) + LogicalFilter(condition=[GREATER_IP($0, IP('1.1.1.1'))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..11=[{inputs}], expr#12=['1.1.1.1'], expr#13=[IP($t12)], expr#14=[GREATER_IP($t0, $t13)], host=[$t0], $condition=[$t14]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ipv6_swapped.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ipv6_swapped.json deleted file mode 100644 index bf130cd5789..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ipv6_swapped.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(host=[$0])\n LogicalFilter(condition=[LTE_IP(IP('::ffff:1234':VARCHAR), $0)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..11=[{inputs}], expr#12=['::ffff:1234':VARCHAR], expr#13=[IP($t12)], expr#14=[LTE_IP($t13, $t0)], host=[$t0], $condition=[$t14])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ipv6_swapped.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ipv6_swapped.yaml new file mode 100644 index 00000000000..1cab09f4619 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ipv6_swapped.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(host=[$0]) + LogicalFilter(condition=[LTE_IP(IP('::ffff:1234'), $0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..11=[{inputs}], expr#12=['::ffff:1234'], expr#13=[IP($t12)], expr#14=[LTE_IP($t13, $t0)], host=[$t0], $condition=[$t14]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_function_script_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_function_script_push.yaml index 0cc53c7287f..fe5598cf48c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_function_script_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_function_script_push.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(firstname=[$1], age=[$8]) - LogicalFilter(condition=[AND(=(CHAR_LENGTH($1), 5), =(ABS($8), 32), =($3, 39225))]) + LogicalFilter(condition=[AND(=(CHAR_LENGTH($1), 5), =(ABS($8), SAFE_CAST(32:BIGINT)), =($3, SAFE_CAST(39225:BIGINT)))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[CHAR_LENGTH($t1)], expr#18=[5], expr#19=[=($t17, $t18)], expr#20=[ABS($t8)], expr#21=[32], expr#22=[=($t20, $t21)], expr#23=[39225], expr#24=[=($t3, $t23)], expr#25=[AND($t19, $t22, $t24)], firstname=[$t1], age=[$t8], $condition=[$t25]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[CHAR_LENGTH($t1)], expr#18=[5], expr#19=[=($t17, $t18)], expr#20=[ABS($t8)], expr#21=[32:BIGINT], expr#22=[=($t20, $t21)], expr#23=[39225:BIGINT], expr#24=[=($t3, $t23)], expr#25=[AND($t19, $t22, $t24)], firstname=[$t1], age=[$t8], $condition=[$t25]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push.yaml index d1f0cead8f6..712035468e1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age=[$8]) - LogicalFilter(condition=[AND(SEARCH($8, Sarg[(30..40)]), >($3, 10000))]) + LogicalFilter(condition=[AND(>($8, SAFE_CAST(30:BIGINT)), <($8, SAFE_CAST(40:BIGINT)), >($3, SAFE_CAST(10000:BIGINT)))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[Sarg[(30..40)]], expr#18=[SEARCH($t8, $t17)], expr#19=[10000], expr#20=[>($t3, $t19)], expr#21=[AND($t18, $t20)], age=[$t8], $condition=[$t21]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[Sarg[(30L:BIGINT..40L:BIGINT)]:BIGINT], expr#18=[SEARCH($t8, $t17)], expr#19=[10000:BIGINT], expr#20=[>($t3, $t19)], expr#21=[AND($t18, $t20)], age=[$t8], $condition=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_date_string.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_date_string.yaml index f8fcc3a23a7..88c7cb6da44 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_date_string.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_date_string.yaml @@ -1,10 +1,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalFilter(condition=[AND(>($0, DATE('2016-12-08 00:00:00.123456789':VARCHAR)), <($0, DATE('2018-11-09 00:00:00.000000000':VARCHAR)))]) - LogicalProject(yyyy-MM-dd=[$83]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]]) + LogicalProject(yyyy-MM-dd=[$0]) + LogicalFilter(condition=[AND(>($0, DATE('2016-12-08 00:00:00.123456789')), <($0, DATE('2018-11-09 00:00:00.000000000')))]) + LogicalProject(yyyy-MM-dd=[$83]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..94=[{inputs}], expr#95=[Sarg[('2016-12-08':VARCHAR..'2018-11-09':VARCHAR)]:VARCHAR], expr#96=[SEARCH($t83, $t95)], yyyy-MM-dd=[$t83], $condition=[$t96]) + EnumerableCalc(expr#0..94=[{inputs}], expr#95=[Sarg[('2016-12-08':EXPR_DATE VARCHAR..'2018-11-09':EXPR_DATE VARCHAR)]:EXPR_DATE VARCHAR], expr#96=[SEARCH($t83, $t95)], yyyy-MM-dd=[$t83], $condition=[$t96]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_time_string.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_time_string.yaml index 4634cfaaa47..5bf27a3a0c9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_time_string.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_time_string.yaml @@ -1,10 +1,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalFilter(condition=[AND(>($0, TIME('2016-12-08 12:00:00.123456789':VARCHAR)), <($0, TIME('2018-11-09 19:00:00.123456789':VARCHAR)))]) - LogicalProject(custom_time=[$49]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]]) + LogicalProject(custom_time=[$0]) + LogicalFilter(condition=[AND(>($0, TIME('2016-12-08 12:00:00.123456789')), <($0, TIME('2018-11-09 19:00:00.123456789')))]) + LogicalProject(custom_time=[$49]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..94=[{inputs}], expr#95=[Sarg[('12:00:00.123456789':VARCHAR..'19:00:00.123456789':VARCHAR)]:VARCHAR], expr#96=[SEARCH($t49, $t95)], custom_time=[$t49], $condition=[$t96]) + EnumerableCalc(expr#0..94=[{inputs}], expr#95=[Sarg[('12:00:00.123456789':EXPR_TIME VARCHAR..'19:00:00.123456789':EXPR_TIME VARCHAR)]:EXPR_TIME VARCHAR], expr#96=[SEARCH($t49, $t95)], custom_time=[$t49], $condition=[$t96]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_timestamp_string.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_timestamp_string.yaml index 20f0cbf4238..6270997ffed 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_timestamp_string.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_timestamp_string.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) - LogicalFilter(condition=[AND(>($3, TIMESTAMP('2016-12-08 00:00:00.000000000':VARCHAR)), <($3, TIMESTAMP('2018-11-09 00:00:00.000000000':VARCHAR)))]) + LogicalFilter(condition=[AND(>($3, TIMESTAMP('2016-12-08 00:00:00.000000000')), <($3, TIMESTAMP('2018-11-09 00:00:00.000000000')))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[Sarg[('2016-12-08 00:00:00':VARCHAR..'2018-11-09 00:00:00':VARCHAR)]:VARCHAR], expr#20=[SEARCH($t3, $t19)], proj#0..12=[{exprs}], $condition=[$t20]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[Sarg[('2016-12-08 00:00:00':EXPR_TIMESTAMP VARCHAR..'2018-11-09 00:00:00':EXPR_TIMESTAMP VARCHAR)]:EXPR_TIMESTAMP VARCHAR], expr#20=[SEARCH($t3, $t19)], proj#0..12=[{exprs}], $condition=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_ip_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_ip_push.yaml index af7a44c2f6d..de83c13604b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_ip_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_ip_push.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(host=[$0]) - LogicalFilter(condition=[CIDRMATCH($0, '0.0.0.0/24':VARCHAR)]) + LogicalFilter(condition=[CIDRMATCH($0, '0.0.0.0/24')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..11=[{inputs}], expr#12=['0.0.0.0/24':VARCHAR], expr#13=[CIDRMATCH($t0, $t12)], host=[$t0], $condition=[$t13]) + EnumerableCalc(expr#0..11=[{inputs}], expr#12=['0.0.0.0/24'], expr#13=[CIDRMATCH($t0, $t12)], host=[$t0], $condition=[$t13]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml index 90492abbaf8..3fd6f43dee0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(firstname=[$1], age=[$8]) - LogicalFilter(condition=[AND(=($1, 'Amber'), =(-($8, 2), 30))]) + LogicalFilter(condition=[AND(=($1, 'Amber'), =(-($8, 2), SAFE_CAST(30:BIGINT)))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['Amber':VARCHAR], expr#18=[=($t1, $t17)], expr#19=[2], expr#20=[-($t8, $t19)], expr#21=[30], expr#22=[=($t20, $t21)], expr#23=[AND($t18, $t22)], firstname=[$t1], age=[$t8], $condition=[$t23]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['Amber':VARCHAR], expr#18=[=($t1, $t17)], expr#19=[2], expr#20=[-($t8, $t19)], expr#21=[30:BIGINT], expr#22=[=($t20, $t21)], expr#23=[AND($t18, $t22)], firstname=[$t1], age=[$t8], $condition=[$t23]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_then_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_then_limit_push.yaml index 8470ec7d765..b8c2251c6c7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_then_limit_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_then_limit_push.yaml @@ -1,13 +1,13 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(fetch=[5]) - LogicalFilter(condition=[>($8, 30)]) + LogicalSort(fetch=[5]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[>($8, SAFE_CAST(30:BIGINT))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..16=[{inputs}], age=[$t8]) EnumerableLimit(fetch=[5]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30:BIGINT], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_with_search.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_with_search.yaml new file mode 100644 index 00000000000..3bf50337fe9 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_with_search.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], patterns_field=[CASE(SEARCH($9, Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR), '':VARCHAR, REGEXP_REPLACE($9, '[a-zA-Z0-9]+', '<*>'))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR], expr#18=[SEARCH($t9, $t17)], expr#19=['':VARCHAR], expr#20=['[a-zA-Z0-9]+'], expr#21=['<*>'], expr#22=[REGEXP_REPLACE($t9, $t20, $t21)], expr#23=[CASE($t18, $t19, $t22)], proj#0..10=[{exprs}], patterns_field=[$t23]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_correlated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_correlated_subquery.yaml index cb17a67d1cd..205feb719a7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_correlated_subquery.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_correlated_subquery.yaml @@ -1,26 +1,27 @@ calcite: logical: | LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(id=[$2], name=[$0], salary=[$4]) - LogicalSort(sort0=[$4], dir0=[DESC-nulls-last]) + LogicalSort(sort0=[$2], dir0=[DESC-nulls-last]) + LogicalProject(id=[$2], name=[$0], salary=[$4]) LogicalFilter(condition=[IN($0, { LogicalProject(name=[$0]) - LogicalFilter(condition=[=($cor0.id, $1)]) - LogicalSystemLimit(fetch=[10000], type=[SUBSEARCH_MAXOUT]) - LogicalFilter(condition=[=($0, 'Tom')]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) - })], variablesSet=[[$cor0]]) + LogicalFilter(condition=[=($cor1.id, $1)]) + LogicalSort(fetch=[10000]) + LogicalProject(name=[$0], uid=[$1], occupation=[$2], department=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9]) + LogicalFilter(condition=[=($0, 'Tom')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + })], variablesSet=[[$cor1]]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$2], dir0=[DESC-nulls-last]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[=($t0, $t3)], proj#0..3=[{exprs}], $condition=[$t4]) - EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) + EnumerableCorrelate(correlation=[$cor1], joinType=[inner], requiredColumns=[{1}]) EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2], salary=[$t4]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) EnumerableAggregate(group=[{0}]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[$cor0], expr#3=[$t2.id], expr#4=[=($t3, $t1)], proj#0..1=[{exprs}], $condition=[$t4]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[$cor1], expr#3=[$t2.id], expr#4=[=($t3, $t1)], proj#0..1=[{exprs}], $condition=[$t4]) EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..9=[{inputs}], expr#10=['Tom':VARCHAR], expr#11=[=($t0, $t10)], proj#0..1=[{exprs}], $condition=[$t11]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_uncorrelated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_uncorrelated_subquery.yaml index e94a46d70d8..5cecdff5330 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_uncorrelated_subquery.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_uncorrelated_subquery.yaml @@ -1,20 +1,20 @@ calcite: logical: | LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(id=[$2], name=[$0], salary=[$4]) - LogicalSort(sort0=[$4], dir0=[DESC-nulls-last]) + LogicalSort(sort0=[$2], dir0=[DESC-nulls-last]) + LogicalProject(id=[$2], name=[$0], salary=[$4]) LogicalFilter(condition=[IN($2, { - LogicalSystemLimit(fetch=[10000], type=[SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[10000]) LogicalProject(uid=[$1]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) - })], variablesSet=[[$cor0]]) + })]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) - EnumerableSort(sort0=[$2], dir0=[DESC-nulls-last]) + EnumerableSort(sort0=[$2], dir0=[DESC-nulls-last]) + EnumerableCalc(expr#0..2=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) EnumerableHashJoin(condition=[=($1, $4)], joinType=[semi]) EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2], salary=[$t4]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) EnumerableLimit(fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isblank.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isblank.yaml index 887fd96408b..b854577fc5b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isblank.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isblank.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[OR(IS NULL($1), IS EMPTY(TRIM(FLAG(BOTH), ' ', $1)))]) + LogicalFilter(condition=[OR(IS NULL($1), =(TRIM(FLAG(BOTH), ' ', $1), ''))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NULL($t1)], expr#18=[FLAG(BOTH)], expr#19=[' '], expr#20=[TRIM($t18, $t19, $t1)], expr#21=[IS EMPTY($t20)], expr#22=[OR($t17, $t21)], proj#0..10=[{exprs}], $condition=[$t22]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NULL($t1)], expr#18=[FLAG(BOTH)], expr#19=[' '], expr#20=[TRIM($t18, $t19, $t1)], expr#21=['':VARCHAR], expr#22=[=($t20, $t21)], expr#23=[OR($t17, $t22)], proj#0..10=[{exprs}], $condition=[$t23]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty.yaml index 6115f98e23f..29aa5686ddc 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[OR(IS NULL($1), IS EMPTY($1))]) + LogicalFilter(condition=[OR(IS NULL($1), =($1, ''))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NULL($t1)], expr#18=[IS EMPTY($t1)], expr#19=[OR($t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR], expr#18=[SEARCH($t1, $t17)], proj#0..10=[{exprs}], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty_or_others.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty_or_others.yaml index 7f43f48dc57..091b9260f57 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty_or_others.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty_or_others.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[OR(=($4, 'M'), IS NULL($1), IS EMPTY($1))]) + LogicalFilter(condition=[OR(=($4, 'M'), IS NULL($1), =($1, ''))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NULL($t1)], expr#18=['M'], expr#19=[=($t4, $t18)], expr#20=[IS EMPTY($t1)], expr#21=[OR($t17, $t19, $t20)], proj#0..10=[{exprs}], $condition=[$t21]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['M':VARCHAR], expr#18=[=($t4, $t17)], expr#19=[Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR], expr#20=[SEARCH($t1, $t19)], expr#21=[OR($t18, $t20)], proj#0..10=[{exprs}], $condition=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isnull_or_others.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isnull_or_others.json deleted file mode 100644 index 2355fb0c4bb..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isnull_or_others.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalFilter(condition=[OR(IS NULL($1), =($4, 'M'))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NULL($t1)], expr#18=['M'], expr#19=[=($t4, $t18)], expr#20=[OR($t17, $t19)], proj#0..10=[{exprs}], $condition=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isnull_or_others.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isnull_or_others.yaml new file mode 100644 index 00000000000..83aec7aa97f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isnull_or_others.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[OR(IS NULL($1), =($4, 'M'))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NULL($t1)], expr#18=['M':VARCHAR], expr#19=[=($t4, $t18)], expr#20=[OR($t17, $t19)], proj#0..10=[{exprs}], $condition=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_join_with_criteria_max_option.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_join_with_criteria_max_option.yaml index be2c210caf2..90f40a34a40 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_join_with_criteria_max_option.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_join_with_criteria_max_option.yaml @@ -5,9 +5,9 @@ calcite: LogicalJoin(condition=[=($0, $13)], joinType=[inner]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[50000]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) - LogicalFilter(condition=[<=($13, 1)]) + LogicalFilter(condition=[<=($13, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _row_number_join_max_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) @@ -20,7 +20,7 @@ calcite: CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableLimit(fetch=[50000]) - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[1], expr#21=[<=($t19, $t20)], proj#0..12=[{exprs}], $condition=[$t21]) + EnumerableCalc(expr#0..19=[{inputs}], expr#20=[1:BIGINT], expr#21=[<=($t19, $t20)], proj#0..12=[{exprs}], $condition=[$t21]) EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t0)], proj#0..18=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_join_with_fields.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_join_with_fields.yaml index bf397010f6b..0dbca894e09 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_join_with_fields.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_join_with_fields.yaml @@ -5,7 +5,7 @@ calcite: LogicalJoin(condition=[=($0, $13)], joinType=[left]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[50000]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | @@ -18,4 +18,4 @@ calcite: EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableLimit(fetch=[50000]) EnumerableCalc(expr#0..18=[{inputs}], proj#0..12=[{exprs}]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_join_with_fields_max_option.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_join_with_fields_max_option.yaml index 84ce94c02e8..d7aae14f46a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_join_with_fields_max_option.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_join_with_fields_max_option.yaml @@ -5,9 +5,9 @@ calcite: LogicalJoin(condition=[=($0, $13)], joinType=[inner]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[50000]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) - LogicalFilter(condition=[<=($13, 1)]) + LogicalFilter(condition=[<=($13, SAFE_CAST(1:BIGINT))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _row_number_join_max_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) @@ -21,7 +21,7 @@ calcite: CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableLimit(fetch=[50000]) - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[1], expr#21=[<=($t19, $t20)], proj#0..12=[{exprs}], $condition=[$t21]) + EnumerableCalc(expr#0..19=[{inputs}], expr#20=[1:BIGINT], expr#21=[<=($t19, $t20)], proj#0..12=[{exprs}], $condition=[$t21]) EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t0)], proj#0..18=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_ilike_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_ilike_function.yaml index f8b576cb814..4bf3888c8f2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_ilike_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_ilike_function.yaml @@ -7,4 +7,4 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%mbe%'], expr#18=['\'], expr#19=[ILIKE($t1, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function.yaml index 2d164b50d29..429ecfb8189 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function.yaml @@ -7,4 +7,4 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%mbe%'], expr#18=['\'], expr#19=[LIKE($t1, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function_case_insensitive.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function_case_insensitive.yaml index f8b576cb814..4bf3888c8f2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function_case_insensitive.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function_case_insensitive.yaml @@ -7,4 +7,4 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%mbe%'], expr#18=['\'], expr#19=[ILIKE($t1, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_5_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_5_push.yaml index ebdf44514de..5788bee8d3c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_5_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_5_push.yaml @@ -1,10 +1,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(fetch=[5]) + LogicalSort(fetch=[5]) + LogicalProject(age=[$8]) LogicalSort(fetch=[10]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..16=[{inputs}], age=[$t8]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_filter_5_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_filter_5_push.yaml index 54bf835b3ef..0a50d218c26 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_filter_5_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_filter_5_push.yaml @@ -1,14 +1,15 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(fetch=[5]) - LogicalFilter(condition=[>($8, 30)]) + LogicalSort(fetch=[5]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[>($8, SAFE_CAST(30:BIGINT))]) LogicalSort(fetch=[10]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableLimit(fetch=[5]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], age=[$t8], $condition=[$t18]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30:BIGINT], expr#18=[>($t8, $t17)], age=[$t8], $condition=[$t18]) EnumerableLimit(fetch=[10]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10from1_10from2_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10from1_10from2_push.yaml index 544f0ac88bb..7e62e2d5bcb 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10from1_10from2_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10from1_10from2_push.yaml @@ -1,10 +1,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(offset=[2], fetch=[10]) + LogicalSort(offset=[2], fetch=[10]) + LogicalProject(age=[$8]) LogicalSort(offset=[1], fetch=[10]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..16=[{inputs}], age=[$t8]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_5_10_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_5_10_push.yaml index 17801ae089a..6acd7f4778f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_5_10_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_5_10_push.yaml @@ -1,10 +1,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(fetch=[10]) + LogicalSort(fetch=[10]) + LogicalProject(age=[$8]) LogicalSort(fetch=[5]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableLimit(fetch=[10]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_offsets_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_offsets_push.yaml index 6f901ac6276..efa44139c1c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_offsets_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_offsets_push.yaml @@ -1,10 +1,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(offset=[2], fetch=[5]) + LogicalSort(offset=[2], fetch=[5]) + LogicalProject(age=[$8]) LogicalSort(offset=[1], fetch=[10]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..16=[{inputs}], age=[$t8]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml index fb3daa06769..9715ff157c2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml @@ -1,10 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(ageMinus=[$17]) - LogicalSort(fetch=[5]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], ageMinus=[-($8, 30)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalSort(fetch=[5]) + LogicalProject(ageMinus=[-($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[-($t8, $t17)], ageMinus=[$t18]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_filter_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_filter_push.yaml index 6bae1a225fb..1bfe7e86b3b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_filter_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_filter_push.yaml @@ -2,11 +2,12 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age=[$8]) - LogicalFilter(condition=[>($8, 30)]) + LogicalFilter(condition=[>($8, SAFE_CAST(30:BIGINT))]) LogicalSort(fetch=[5]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], age=[$t8], $condition=[$t18]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30:BIGINT], expr#18=[>($t8, $t17)], age=[$t8], $condition=[$t18]) EnumerableLimit(fetch=[5]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_sort_push.yaml index 9b7aa3c87b0..51295614f94 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_sort_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_sort_push.yaml @@ -1,10 +1,11 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(sort0=[$8], dir0=[ASC-nulls-first]) + LogicalSort(sort0=[$0], dir0=[ASC-nulls-first]) + LogicalProject(age=[$8]) LogicalSort(fetch=[5]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_list_aggregation.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_list_aggregation.json index 7da65bb8d4f..f777c6495a7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_list_aggregation.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_list_aggregation.json @@ -3,4 +3,4 @@ "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], age_list=[LIST($0)])\n LogicalProject(age=[$8])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", "physical": "EnumerableLimit(fetch=[10000])\n EnumerableAggregate(group=[{}], age_list=[LIST($8)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_merge_join_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_merge_join_sort_push.yaml index 843fa505511..702d1adde13 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_merge_join_sort_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_merge_join_sort_push.yaml @@ -5,7 +5,7 @@ calcite: LogicalJoin(condition=[=($0, $13)], joinType=[inner]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[50000]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | @@ -17,4 +17,4 @@ calcite: EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableLimit(fetch=[50000]) EnumerableCalc(expr#0..18=[{inputs}], proj#0..12=[{exprs}]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_min_max_agg_on_derived_field.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_min_max_agg_on_derived_field.yaml new file mode 100644 index 00000000000..8224f075819 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_min_max_agg_on_derived_field.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$7]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], gender=[$t0], avg(balance)=[$t8]) + EnumerableAggregate(group=[{4}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multi_sort_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multi_sort_push.json deleted file mode 100644 index 0871cef3d5a..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multi_sort_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$3], sort1=[$4], sort2=[$0], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4])\n LogicalSort(sort0=[$3], sort1=[$4], sort2=[$0], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first])\n LogicalSort(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], dir2=[ASC-nulls-first], dir3=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$3], sort1=[$4], sort2=[$0], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first])\n EnumerableCalc(expr#0..16=[{inputs}], proj#0..4=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multi_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multi_sort_push.yaml new file mode 100644 index 00000000000..6affc6dee06 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multi_sort_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$3], sort1=[$4], sort2=[$0], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$3], sort1=[$4], sort2=[$0], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4]) + LogicalSort(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], dir2=[ASC-nulls-first], dir3=[ASC-nulls-first]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$3], sort1=[$4], sort2=[$0], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first]) + EnumerableCalc(expr#0..16=[{inputs}], proj#0..4=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multisearch_basic.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multisearch_basic.yaml index 4910dc0a253..a9b01105f2a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multisearch_basic.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multisearch_basic.yaml @@ -5,18 +5,18 @@ calcite: LogicalAggregate(group=[{0}], count=[COUNT()]) LogicalProject(age_group=[$11]) LogicalUnion(all=[true]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], age_group=['young':VARCHAR]) - LogicalFilter(condition=[<($8, 30)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], age_group=['young']) + LogicalFilter(condition=[<($8, SAFE_CAST(30:BIGINT))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], age_group=['adult':VARCHAR]) - LogicalFilter(condition=[>=($8, 30)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], age_group=['adult']) + LogicalFilter(condition=[>=($8, SAFE_CAST(30:BIGINT))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..1=[{inputs}], count=[$t1], age_group=[$t0]) EnumerableAggregate(group=[{0}], count=[COUNT()]) EnumerableUnion(all=[true]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['young':VARCHAR], expr#18=[30], expr#19=[<($t8, $t18)], age_group=[$t17], $condition=[$t19]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['young'], expr#18=[30:BIGINT], expr#19=[<($t8, $t18)], age_group=[$t17], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['adult'], expr#18=[30:BIGINT], expr#19=[>=($t8, $t18)], age_group=[$t17], $condition=[$t19]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['adult':VARCHAR], expr#18=[30], expr#19=[>=($t8, $t18)], age_group=[$t17], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multisearch_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multisearch_timestamp.yaml index 64b0ff25050..4b664fe7f73 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multisearch_timestamp.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multisearch_timestamp.yaml @@ -2,13 +2,14 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[DESC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC], fetch=[5]) - LogicalUnion(all=[true]) - LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) - LogicalFilter(condition=[SEARCH($1, Sarg['A':VARCHAR, 'B':VARCHAR]:VARCHAR)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) - LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) - LogicalFilter(condition=[SEARCH($1, Sarg['E':VARCHAR, 'F':VARCHAR]:VARCHAR)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data2]]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) + LogicalUnion(all=[true]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) + LogicalFilter(condition=[OR(=($1, 'A'), =($1, 'B'))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) + LogicalFilter(condition=[OR(=($1, 'E'), =($1, 'F'))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data2]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableLimit(fetch=[5]) @@ -22,4 +23,4 @@ calcite: EnumerableLimit(fetch=[5]) EnumerableSort(sort0=[$0], dir0=[DESC]) EnumerableCalc(expr#0..9=[{inputs}], expr#10=[Sarg['E':VARCHAR, 'F':VARCHAR]:VARCHAR], expr#11=[SEARCH($t1, $t10)], proj#0..9=[{exprs}], $condition=[$t11]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data2]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data2]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_mvjoin.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_mvjoin.json deleted file mode 100644 index f8c0bc1c908..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_mvjoin.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(fetch=[1])\n LogicalProject(result=[ARRAY_JOIN(array('a', 'b', 'c'), ',')])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=['a'], expr#18=['b'], expr#19=['c'], expr#20=[array($t17, $t18, $t19)], expr#21=[','], expr#22=[ARRAY_JOIN($t20, $t21)], result=[$t22])\n EnumerableLimit(fetch=[1])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_mvjoin.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_mvjoin.yaml new file mode 100644 index 00000000000..5f68b415151 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_mvjoin.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(fetch=[1]) + LogicalProject(result=[ARRAY_JOIN(array('a':VARCHAR, 'b':VARCHAR, 'c':VARCHAR), ',')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['a':VARCHAR], expr#18=['b':VARCHAR], expr#19=['c':VARCHAR], expr#20=[array($t17, $t18, $t19)], expr#21=[','], expr#22=[ARRAY_JOIN($t20, $t21)], result=[$t22]) + EnumerableLimit(fetch=[1]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml index 2a1449e4b1f..67399ce780f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml @@ -2,23 +2,22 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age2=[$2]) - LogicalFilter(condition=[<=($3, 1)]) + LogicalFilter(condition=[<=($3, SAFE_CAST(1:BIGINT))]) LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2)]) LogicalFilter(condition=[IS NOT NULL($2)]) - LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]) - LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) - LogicalProject(avg_age=[$2], state=[$0], city=[$1]) - LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) - LogicalProject(state=[$7], city=[$5], age=[$8]) - LogicalFilter(condition=[>($8, 30)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) + LogicalProject(avg_age=[$2], state=[$0], age2=[+($2, 2)]) + LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) + LogicalProject(state=[$7], city=[$5], age=[$8]) + LogicalFilter(condition=[>($8, SAFE_CAST(30:BIGINT))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..2=[{inputs}], age2=[$t1]) EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1:BIGINT], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) EnumerableWindow(window#0=[window(partition {1} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[2], expr#11=[+($t9, $t10)], expr#12=[IS NOT NULL($t8)], state=[$t1], age2=[$t11], $condition=[$t12]) EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first]) EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30:BIGINT], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_partial_filter_isnull.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_partial_filter_isnull.json deleted file mode 100644 index 75c84e80b6c..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_partial_filter_isnull.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(name=[$0], address=[$1], id=[$6], age=[$7])\n LogicalFilter(condition=[AND(IS NULL($1), =($0, 'david'))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..13=[{inputs}], expr#14=[IS NULL($t1)], expr#15=['david':VARCHAR], expr#16=[=($t0, $t15)], expr#17=[AND($t14, $t16)], proj#0..1=[{exprs}], id=[$t6], age=[$t7], $condition=[$t17])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_partial_filter_isnull.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_partial_filter_isnull.yaml new file mode 100644 index 00000000000..9415e23d541 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_partial_filter_isnull.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(name=[$0], address=[$1], id=[$6], age=[$7]) + LogicalFilter(condition=[AND(IS NULL($1), =($0, 'david'))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..13=[{inputs}], expr#14=[IS NULL($t1)], expr#15=['david':VARCHAR], expr#16=[=($t0, $t15)], expr#17=[AND($t14, $t16)], proj#0..1=[{exprs}], id=[$t6], age=[$t7], $condition=[$t17]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_partial_filter_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_partial_filter_push.json new file mode 100644 index 00000000000..51295614f94 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_partial_filter_push.json @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC-nulls-first]) + LogicalProject(age=[$8]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) + EnumerableCalc(expr#0..16=[{inputs}], age=[$t8]) + EnumerableLimit(fetch=[5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_partial_filter_push2.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_partial_filter_push2.json new file mode 100644 index 00000000000..9715ff157c2 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_partial_filter_push2.json @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(fetch=[5]) + LogicalProject(ageMinus=[-($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[-($t8, $t17)], ageMinus=[$t18]) + EnumerableLimit(fetch=[5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_partial_filter_script_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_partial_filter_script_push.json new file mode 100644 index 00000000000..81e88ea57b9 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_partial_filter_script_push.json @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[CASE(IS NOT NULL($8), CAST($8):BIGINT NOT NULL, -1:BIGINT)], balance=[CASE(IS NOT NULL($3), CAST($3):BIGINT NOT NULL, -1:BIGINT)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t8)], expr#18=[CAST($t8):BIGINT NOT NULL], expr#19=[-1:BIGINT], expr#20=[CASE($t17, $t18, $t19)], expr#21=[IS NOT NULL($t3)], expr#22=[CAST($t3):BIGINT NOT NULL], expr#23=[CASE($t21, $t22, $t19)], age=[$t20], balance=[$t23]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_simple_pattern.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_simple_pattern.yaml index 056b8a463a7..3bf50337fe9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_simple_pattern.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_simple_pattern.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], patterns_field=[CASE(SEARCH($9, Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR), '':VARCHAR, REGEXP_REPLACE($9, '[a-zA-Z0-9]+':VARCHAR, '<*>':VARCHAR))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], patterns_field=[CASE(SEARCH($9, Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR), '':VARCHAR, REGEXP_REPLACE($9, '[a-zA-Z0-9]+', '<*>'))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR], expr#18=[SEARCH($t9, $t17)], expr#19=['':VARCHAR], expr#20=['[a-zA-Z0-9]+':VARCHAR], expr#21=['<*>':VARCHAR], expr#22=[REGEXP_REPLACE($t9, $t20, $t21)], expr#23=[CASE($t18, $t19, $t22)], proj#0..10=[{exprs}], patterns_field=[$t23]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR], expr#18=[SEARCH($t9, $t17)], expr#19=['':VARCHAR], expr#20=['[a-zA-Z0-9]+'], expr#21=['<*>'], expr#22=[REGEXP_REPLACE($t9, $t20, $t21)], expr#23=[CASE($t18, $t19, $t22)], proj#0..10=[{exprs}], patterns_field=[$t23]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_simple_pattern_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_simple_pattern_agg_push.yaml index bd40ed4bb7a..196987a0dad 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_simple_pattern_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_simple_pattern_agg_push.yaml @@ -1,13 +1,13 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'pattern'))], pattern_count=[$1], tokens=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'tokens'))], sample_logs=[$2]) - LogicalAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0, $2)]) - LogicalProject(email=[$9], patterns_field=[CASE(SEARCH($9, Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR), '':VARCHAR, REGEXP_REPLACE($9, '[a-zA-Z0-9]+':VARCHAR, '<*>':VARCHAR))], $f18=[10]) + LogicalProject(patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $1), 'pattern'))], pattern_count=[$2], tokens=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $1), 'tokens'))], sample_logs=[$1]) + LogicalAggregate(group=[{0}], sample_logs=[TAKE($1, $2)], pattern_count=[COUNT($0)]) + LogicalProject($f0=[CASE(SEARCH($9, Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR), '':VARCHAR, REGEXP_REPLACE($9, '[a-zA-Z0-9]+', '<*>'))], email=[$9], $f2=[10]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[PATTERN_PARSER($t0, $t2)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], expr#7=['tokens'], expr#8=[ITEM($t3, $t7)], expr#9=[SAFE_CAST($t8)], patterns_field=[$t6], pattern_count=[$t1], tokens=[$t9], sample_logs=[$t2]) - EnumerableAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0, $2)]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR], expr#18=[SEARCH($t9, $t17)], expr#19=['':VARCHAR], expr#20=['[a-zA-Z0-9]+':VARCHAR], expr#21=['<*>':VARCHAR], expr#22=[REGEXP_REPLACE($t9, $t20, $t21)], expr#23=[CASE($t18, $t19, $t22)], expr#24=[10], email=[$t9], patterns_field=[$t23], $f18=[$t24]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[PATTERN_PARSER($t0, $t1)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], expr#7=['tokens'], expr#8=[ITEM($t3, $t7)], expr#9=[SAFE_CAST($t8)], patterns_field=[$t6], pattern_count=[$t2], tokens=[$t9], sample_logs=[$t1]) + EnumerableAggregate(group=[{0}], sample_logs=[TAKE($1, $2)], pattern_count=[COUNT($0)]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[Sarg['':VARCHAR; NULL AS TRUE]:VARCHAR], expr#18=[SEARCH($t9, $t17)], expr#19=['':VARCHAR], expr#20=['[a-zA-Z0-9]+'], expr#21=['<*>'], expr#22=[REGEXP_REPLACE($t9, $t20, $t21)], expr#23=[CASE($t18, $t19, $t22)], expr#24=[10], $f0=[$t23], email=[$t9], $f2=[$t24]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_percentile.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_percentile.json deleted file mode 100644 index f96489564ff..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_percentile.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], p50=[percentile_approx($0, $1, $2)], p90=[percentile_approx($0, $3, $2)])\n LogicalProject(balance=[$3], $f2=[50], $f3=[FLAG(BIGINT)], $f4=[90])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableAggregate(group=[{}], p50=[percentile_approx($0, $1, $2)], p90=[percentile_approx($0, $3, $2)])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[50], expr#18=[FLAG(BIGINT)], expr#19=[90], balance=[$t3], $f2=[$t17], $f3=[$t18], $f4=[$t19])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_percentile.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_percentile.yaml new file mode 100644 index 00000000000..47a5b534927 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_percentile.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], p50=[percentile_approx($0, $1, $2)], p90=[percentile_approx($0, $3, $2)]) + LogicalProject(balance=[$3], $f1=[50], $f2=[FLAG(BIGINT)], $f3=[90]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableAggregate(group=[{}], p50=[percentile_approx($0, $1, $2)], p90=[percentile_approx($0, $3, $2)]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[50], expr#18=[FLAG(BIGINT)], expr#19=[90], balance=[$t3], $f1=[$t17], $f2=[$t18], $f3=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rare_usenull_false.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rare_usenull_false.yaml index 49a464287ff..ab8c08cd319 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rare_usenull_false.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rare_usenull_false.yaml @@ -2,16 +2,16 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(gender=[$0], state=[$1], count=[$2]) - LogicalFilter(condition=[<=($3, 2)]) - LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2)]) + LogicalFilter(condition=[<=($3, SAFE_CAST(2:BIGINT))]) + LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 NULLS LAST)]) LogicalAggregate(group=[{0, 1}], count=[COUNT()]) LogicalProject(gender=[$4], state=[$7]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[2], expr#5=[<=($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[2:BIGINT], expr#5=[<=($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) EnumerableWindow(window#0=[window(partition {0} order by [2] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{4, 7}], count=[COUNT()]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t4)], expr#18=[IS NOT NULL($t7)], expr#19=[AND($t17, $t18)], proj#0..16=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rare_usenull_true.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rare_usenull_true.yaml index d7b401feb17..51cd09cf218 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rare_usenull_true.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rare_usenull_true.yaml @@ -2,14 +2,14 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(gender=[$0], state=[$1], count=[$2]) - LogicalFilter(condition=[<=($3, 2)]) - LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2)]) + LogicalFilter(condition=[<=($3, SAFE_CAST(2:BIGINT))]) + LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 NULLS LAST)]) LogicalAggregate(group=[{0, 1}], count=[COUNT()]) LogicalProject(gender=[$4], state=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[2], expr#5=[<=($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[2:BIGINT], expr#5=[<=($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) EnumerableWindow(window#0=[window(partition {0} order by [2] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{4, 7}], count=[COUNT()]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex.yaml index 1584ec71f53..181862d6c9c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex.yaml @@ -1,13 +1,13 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalSort(fetch=[5]) - LogicalFilter(condition=[REGEXP_CONTAINS($10, '^[A-Z][a-z]+$':VARCHAR)]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[REGEXP_CONTAINS($10, '^[A-Z][a-z]+$')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}]) EnumerableLimit(fetch=[5]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['^[A-Z][a-z]+$':VARCHAR], expr#18=[REGEXP_CONTAINS($t10, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['^[A-Z][a-z]+$'], expr#18=[REGEXP_CONTAINS($t10, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex_negated.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex_negated.yaml index b16c26d4169..3f15dab4141 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex_negated.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex_negated.yaml @@ -1,13 +1,13 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalSort(fetch=[5]) - LogicalFilter(condition=[NOT(REGEXP_CONTAINS($10, '.*son$':VARCHAR))]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[NOT(REGEXP_CONTAINS($10, '.*son$'))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}]) EnumerableLimit(fetch=[5]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['.*son$':VARCHAR], expr#18=[REGEXP_CONTAINS($t10, $t17)], expr#19=[NOT($t18)], proj#0..16=[{exprs}], $condition=[$t19]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['.*son$'], expr#18=[REGEXP_CONTAINS($t10, $t17)], expr#19=[NOT($t18)], proj#0..16=[{exprs}], $condition=[$t19]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regexp_match_in_where.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regexp_match_in_where.yaml new file mode 100644 index 00000000000..9ef9d577892 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regexp_match_in_where.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$4], age1=[$0], age2=[$1], age3=[$2], age=[$3]) + LogicalAggregate(group=[{0, 1, 2, 3}], count()=[COUNT()]) + LogicalProject($f0=[*($8, 10)], $f1=[+($8, 10)], age3=[10], age=[$8]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10], expr#3=[*($t0, $t2)], expr#4=[+($t0, $t2)], count()=[$t1], age1=[$t3], age2=[$t4], age3=[$t2], age=[$t0]) + EnumerableAggregate(group=[{8}], count()=[COUNT()]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_replace_command.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_replace_command.yaml index bbebbf37f80..0d2c7e6d76c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_replace_command.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_replace_command.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(state=[REPLACE($7, 'IL':VARCHAR, 'Illinois':VARCHAR)]) + LogicalProject(state=[REPLACE($7, 'IL', 'Illinois')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['IL':VARCHAR], expr#18=['Illinois':VARCHAR], expr#19=[REPLACE($t7, $t17, $t18)], state=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['IL'], expr#18=['Illinois'], expr#19=[REPLACE($t7, $t17, $t18)], state=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_replace_wildcard.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_replace_wildcard.yaml index 194f680adf2..af956e564b3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_replace_wildcard.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_replace_wildcard.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(state=[REGEXP_REPLACE($7, '^\Q\E(.*?)\QL\E$':VARCHAR, 'STATE_IL':VARCHAR)]) + LogicalProject(state=[REGEXP_REPLACE($7, '^\Q\E(.*?)\QL\E$', 'STATE_IL')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['^\Q\E(.*?)\QL\E$':VARCHAR], expr#18=['STATE_IL':VARCHAR], expr#19=[REGEXP_REPLACE($t7, $t17, $t18)], state=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['^\Q\E(.*?)\QL\E$'], expr#18=['STATE_IL'], expr#19=[REGEXP_REPLACE($t7, $t17, $t18)], state=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rex.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rex.yaml index 56fd60a0abe..ac76a9c805d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rex.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rex.yaml @@ -1,10 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], initial=[$17]) - LogicalSort(fetch=[5]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], initial=[REX_EXTRACT($10, '(?^[A-Z])', 'initial')]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], initial=[REX_EXTRACT($10, '(?^[A-Z])', 'initial')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=['(?^[A-Z])'], expr#18=['initial'], expr#19=[REX_EXTRACT($t10, $t17, $t18)], proj#0..10=[{exprs}], initial=[$t19]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_multi_range.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_multi_range.json deleted file mode 100644 index cb37553feef..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_multi_range.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalFilter(condition=[SEARCH($8, Sarg[0, [1..10], (20..30)])])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[Sarg[0, [1..10], (20..30)]], expr#18=[SEARCH($t8, $t17)], age=[$t8], $condition=[$t18])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_multi_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_multi_range.yaml new file mode 100644 index 00000000000..49353a5c4c1 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_multi_range.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[OR(=($8, SAFE_CAST(0:BIGINT)), AND(>=($8, SAFE_CAST(1:BIGINT)), <=($8, SAFE_CAST(10:BIGINT))), AND(>($8, SAFE_CAST(20:BIGINT)), <($8, SAFE_CAST(30:BIGINT))))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[Sarg[0L:BIGINT, [1L:BIGINT..10L:BIGINT], (20L:BIGINT..30L:BIGINT)]:BIGINT], expr#18=[SEARCH($t8, $t17)], age=[$t8], $condition=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_single_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_single_range.yaml index 834e267b63f..56de7d378a3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_single_range.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_single_range.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age=[$8]) - LogicalFilter(condition=[SEARCH($8, Sarg[[1.0:DECIMAL(11, 1)..10:DECIMAL(11, 1))]:DECIMAL(11, 1))]) + LogicalFilter(condition=[AND(>=(SAFE_CAST($8), SAFE_CAST(1.0:DECIMAL(20, 1))), <(SAFE_CAST($8), SAFE_CAST(10.0:DECIMAL(20, 1))))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[Sarg[[1.0:DECIMAL(11, 1)..10:DECIMAL(11, 1))]:DECIMAL(11, 1)], expr#18=[SEARCH($t8, $t17)], age=[$t8], $condition=[$t18]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[SAFE_CAST($t8)], expr#18=[Sarg[[1.0:DECIMAL(20, 1)..10.0:DECIMAL(20, 1))]:DECIMAL(20, 1)], expr#19=[SEARCH($t17, $t18)], age=[$t8], $condition=[$t19]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_time_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_time_range.yaml index d6d96c9e057..15a2602b8c9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_time_range.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_time_range.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) - LogicalFilter(condition=[AND(>=($3, TIMESTAMP('2016-12-08 00:00:00.000000000':VARCHAR)), <($3, TIMESTAMP('2018-11-09 00:00:00.000000000':VARCHAR)))]) + LogicalFilter(condition=[AND(>=($3, TIMESTAMP('2016-12-08 00:00:00.000000000')), <($3, TIMESTAMP('2018-11-09 00:00:00.000000000')))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[Sarg[['2016-12-08 00:00:00':VARCHAR..'2018-11-09 00:00:00':VARCHAR)]:VARCHAR], expr#20=[SEARCH($t3, $t19)], proj#0..12=[{exprs}], $condition=[$t20]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[Sarg[['2016-12-08 00:00:00':EXPR_TIMESTAMP VARCHAR..'2018-11-09 00:00:00':EXPR_TIMESTAMP VARCHAR)]:EXPR_TIMESTAMP VARCHAR], expr#20=[SEARCH($t3, $t19)], proj#0..12=[{exprs}], $condition=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_select.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_select.yaml index 5e76c380ff2..c1574c647a3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_select.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_select.yaml @@ -1,12 +1,13 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(variablesSet=[[$cor0]], id=[$2], name=[$0], count_dept=[$SCALAR_QUERY({ + LogicalProject(variablesSet=[[$cor1]], id=[$2], name=[$0], count_dept=[$SCALAR_QUERY({ LogicalAggregate(group=[{}], count(name)=[COUNT($0)]) LogicalProject(name=[$0]) LogicalFilter(condition=[IS NOT NULL($0)]) - LogicalFilter(condition=[=($cor0.id, $1)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + LogicalProject(name=[$0], uid=[$1], occupation=[$2], department=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9]) + LogicalFilter(condition=[=($cor1.id, $1)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) })]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | @@ -15,10 +16,10 @@ calcite: EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[left]) EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], expr#4=[0], expr#5=[CASE($t3, $t2, $t4)], uid=[$t0], count(name)=[$t5]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], expr#4=[0], expr#5=[CASE($t3, $t2, $t4)], uid10=[$t0], count(name)=[$t5]) EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) EnumerableAggregate(group=[{2}]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) EnumerableAggregate(group=[{1}], count(name)=[COUNT($0)]) EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NOT NULL($t1)], expr#11=[IS NOT NULL($t0)], expr#12=[AND($t10, $t11)], proj#0..9=[{exprs}], $condition=[$t12]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_where.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_where.yaml index ed28cb93b50..6d275dd8056 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_where.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_where.yaml @@ -5,9 +5,9 @@ calcite: LogicalFilter(condition=[=($2, $SCALAR_QUERY({ LogicalAggregate(group=[{}], max(uid)=[MAX($0)]) LogicalProject(uid=[$1]) - LogicalFilter(condition=[=($cor0.id, $1)]) + LogicalFilter(condition=[=($cor1.id, $1)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) - }))], variablesSet=[[$cor0]]) + }))], variablesSet=[[$cor1]]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | EnumerableLimit(fetch=[10000]) @@ -18,4 +18,4 @@ calcite: EnumerableCalc(expr#0..1=[{inputs}], expr#2=[=($t0, $t1)], proj#0..1=[{exprs}], $condition=[$t2]) EnumerableAggregate(group=[{1}], max(uid)=[MAX($1)]) EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NOT NULL($t1)], proj#0..9=[{exprs}], $condition=[$t10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_select.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_select.yaml index a229cdc7bc0..bf2fd2e405f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_select.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_select.yaml @@ -1,7 +1,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(variablesSet=[[$cor0]], name=[$0], count_dept=[$SCALAR_QUERY({ + LogicalProject(name=[$0], count_dept=[$SCALAR_QUERY({ LogicalAggregate(group=[{}], count(name)=[COUNT($0)]) LogicalProject(name=[$0]) LogicalFilter(condition=[IS NOT NULL($0)]) @@ -15,4 +15,4 @@ calcite: CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) EnumerableAggregate(group=[{}], count(name)=[COUNT($0)]) EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NOT NULL($t0)], proj#0..9=[{exprs}], $condition=[$t10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_where.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_where.yaml index ba13359c44d..4210eea48f7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_where.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_where.yaml @@ -2,19 +2,19 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(name=[$0]) - LogicalFilter(condition=[>($2, +($SCALAR_QUERY({ + LogicalFilter(condition=[>(SAFE_CAST($2), +($SCALAR_QUERY({ LogicalAggregate(group=[{}], count(name)=[COUNT($0)]) LogicalProject(name=[$0]) LogicalFilter(condition=[IS NOT NULL($0)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) - }), 999))], variablesSet=[[$cor0]]) + }), 999))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..2=[{inputs}], name=[$t0]) - EnumerableNestedLoopJoin(condition=[>($1, +($2, 999))], joinType=[inner]) + EnumerableNestedLoopJoin(condition=[>(SAFE_CAST($1), +($2, 999))], joinType=[inner]) EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) EnumerableAggregate(group=[{}], count(name)=[COUNT($0)]) EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NOT NULL($t0)], proj#0..9=[{exprs}], $condition=[$t10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_script_push_on_text.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_script_push_on_text.yaml new file mode 100644 index 00000000000..64d8f9e43a2 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_script_push_on_text.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], gender=[$1], age=[$2]) + LogicalFilter(condition=[OR(IS NULL($1), <=($3, 1))]) + LogicalProject(account_number=[$0], gender=[$4], age=[$8], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[IS NULL($t4)], expr#19=[1:BIGINT], expr#20=[<=($t17, $t19)], expr#21=[OR($t18, $t20)], account_number=[$t0], gender=[$t4], age=[$t8], $condition=[$t21]) + EnumerableWindow(window#0=[window(partition {4} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_basic_text.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_basic_text.json deleted file mode 100644 index 4211405963b..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_basic_text.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165])\n LogicalFilter(condition=[query_string(MAP('query', 'ERROR':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..171=[{inputs}], proj#0..3=[{exprs}], severityText=[$t7], resource=[$t8], flags=[$t23], attributes=[$t24], droppedAttributesCount=[$t162], severityNumber=[$t163], time=[$t164], body=[$t165])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->query_string(MAP('query', 'ERROR':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"ERROR\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_basic_text.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_basic_text.yaml new file mode 100644 index 00000000000..bd07860c2f0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_basic_text.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, 'ERROR':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..171=[{inputs}], proj#0..3=[{exprs}], severityText=[$t7], resource=[$t8], flags=[$t23], attributes=[$t24], droppedAttributesCount=[$t162], severityNumber=[$t163], time=[$t164], body=[$t165]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->query_string(MAP('query':VARCHAR, 'ERROR':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"query_string":{"query":"ERROR","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_numeric_comparison.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_numeric_comparison.json deleted file mode 100644 index bc2050cb42b..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_numeric_comparison.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165])\n LogicalFilter(condition=[query_string(MAP('query', 'severityNumber:>15':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..171=[{inputs}], proj#0..3=[{exprs}], severityText=[$t7], resource=[$t8], flags=[$t23], attributes=[$t24], droppedAttributesCount=[$t162], severityNumber=[$t163], time=[$t164], body=[$t165])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->query_string(MAP('query', 'severityNumber:>15':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"severityNumber:>15\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_numeric_comparison.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_numeric_comparison.yaml new file mode 100644 index 00000000000..23ff277d624 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_numeric_comparison.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, 'severityNumber:>15':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..171=[{inputs}], proj#0..3=[{exprs}], severityText=[$t7], resource=[$t8], flags=[$t23], attributes=[$t24], droppedAttributesCount=[$t162], severityNumber=[$t163], time=[$t164], body=[$t165]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->query_string(MAP('query':VARCHAR, 'severityNumber:>15':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"query_string":{"query":"severityNumber:>15","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_wildcard_star.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_wildcard_star.json deleted file mode 100644 index 470c8b86aa5..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_wildcard_star.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165])\n LogicalFilter(condition=[query_string(MAP('query', 'severityText:ERR*':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..171=[{inputs}], proj#0..3=[{exprs}], severityText=[$t7], resource=[$t8], flags=[$t23], attributes=[$t24], droppedAttributesCount=[$t162], severityNumber=[$t163], time=[$t164], body=[$t165])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->query_string(MAP('query', 'severityText:ERR*':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"severityText:ERR*\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_wildcard_star.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_wildcard_star.yaml new file mode 100644 index 00000000000..435f08da084 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_wildcard_star.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, 'severityText:ERR*':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..171=[{inputs}], proj#0..3=[{exprs}], severityText=[$t7], resource=[$t8], flags=[$t23], attributes=[$t24], droppedAttributesCount=[$t162], severityNumber=[$t163], time=[$t164], body=[$t165]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->query_string(MAP('query':VARCHAR, 'severityText:ERR*':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"query_string":{"query":"severityText:ERR*","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_push.json index adb4cb6244d..1acb0f0be66 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_push.json @@ -3,4 +3,4 @@ "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$10], age2=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, 2)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[2], expr#20=[+($t10, $t19)], age=[$t10], age2=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml index 8897a1023cc..60a3d27a913 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml @@ -1,19 +1,20 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], b.account_number=[$13], b.firstname=[$14], b.address=[$15], b.birthdate=[$16], b.gender=[$17], b.city=[$18], b.lastname=[$19], b.balance=[$20], b.employer=[$21], b.state=[$22], b.age=[$23], b.email=[$24], b.male=[$25]) - LogicalJoin(condition=[=(+($10, 1), -($20, 20))], joinType=[inner]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], b.account_number=[$14], b.firstname=[$15], b.address=[$16], b.birthdate=[$17], b.gender=[$18], b.city=[$19], b.lastname=[$20], b.balance=[$21], b.employer=[$22], b.state=[$23], b.age=[$24], b.email=[$25], b.male=[$26]) + LogicalJoin(condition=[=($13, $27)], joinType=[inner]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], $f13=[SAFE_CAST(+($10, 1))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], $f13=[-($7, 20)]) + LogicalSort(fetch=[50000]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..27=[{inputs}], proj#0..12=[{exprs}], b.account_number=[$t14], b.firstname=[$t15], b.address=[$t16], b.birthdate=[$t17], b.gender=[$t18], b.city=[$t19], b.lastname=[$t20], b.balance=[$t21], b.employer=[$t22], b.state=[$t23], b.age=[$t24], b.email=[$t25], b.male=[$t26]) EnumerableLimit(fetch=[10000]) EnumerableMergeJoin(condition=[=($13, $27)], joinType=[inner]) EnumerableSort(sort0=[$13], dir0=[ASC]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=[+($t10, $t19)], proj#0..12=[{exprs}], $f13=[$t20]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=[+($t10, $t19)], expr#21=[SAFE_CAST($t20)], proj#0..12=[{exprs}], $f13=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) EnumerableSort(sort0=[$13], dir0=[ASC]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[20], expr#20=[-($t7, $t19)], proj#0..12=[{exprs}], $f13=[$t20]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_single_expr_output_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_single_expr_output_push.json index 67cf82580e9..20517b1f960 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_single_expr_output_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_single_expr_output_push.json @@ -3,4 +3,4 @@ "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(b=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], b=[+($7, 1)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=[+($t7, $t19)], b=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_skip_script_encoding.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_skip_script_encoding.yaml new file mode 100644 index 00000000000..4296f43678e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_skip_script_encoding.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(firstname=[$1], age=[$8]) + LogicalFilter(condition=[AND(=(CHAR_LENGTH($1), 5), =(ABS($8), 32), =($3, 39225))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[CHAR_LENGTH($t1)], expr#18=[5], expr#19=[=($t17, $t18)], expr#20=[ABS($t8)], expr#21=[32:BIGINT], expr#22=[=($t20, $t21)], expr#23=[39225:BIGINT], expr#24=[=($t3, $t23)], expr#25=[AND($t19, $t22, $t24)], firstname=[$t1], age=[$t8], $condition=[$t25]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_count_push.yaml index c39d769eb4b..1f2f9786e29 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_count_push.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(sort0=[$8], dir0=[ASC-nulls-first], fetch=[5]) + LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[5]) + LogicalProject(age=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_desc_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_desc_push.json deleted file mode 100644 index dd8d9abb55d..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_desc_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8], firstname=[$1])\n LogicalSort(sort0=[$8], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], age=[$t8], firstname=[$t1])\n EnumerableSort(sort0=[$8], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_desc_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_desc_push.yaml new file mode 100644 index 00000000000..88a12f2d94a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_desc_push.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first]) + LogicalProject(age=[$8], firstname=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first]) + EnumerableCalc(expr#0..16=[{inputs}], age=[$t8], firstname=[$t1]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_pass_through_join_then_pushdown.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_pass_through_join_then_pushdown.yaml index 5df0b4c36a3..8b22921fd47 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_pass_through_join_then_pushdown.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_pass_through_join_then_pushdown.yaml @@ -2,19 +2,20 @@ calcite: logical: | LogicalSystemLimit(sort0=[$13], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$13], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[$13], b.account_number=[$14], b.firstname=[$15], b.address=[$16], b.birthdate=[$17], b.gender=[$18], b.city=[$19], b.lastname=[$20], b.balance=[$21], b.employer=[$22], b.state=[$23], b.age=[$24], b.email=[$25], b.male=[$26]) - LogicalJoin(condition=[=($13, $15)], joinType=[left]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[REX_EXTRACT($6, '(?^[A-Z])', 'initial')]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[$13], b.account_number=[$15], b.firstname=[$16], b.address=[$17], b.birthdate=[$18], b.gender=[$19], b.city=[$20], b.lastname=[$21], b.balance=[$22], b.employer=[$23], b.state=[$24], b.age=[$25], b.email=[$26], b.male=[$27]) + LogicalJoin(condition=[=($14, $16)], joinType=[left]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[REX_EXTRACT($6, '(?^[A-Z])', 'initial')], $f14=[CAST(REX_EXTRACT($6, '(?^[A-Z])', 'initial')):VARCHAR]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalSort(fetch=[50000]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableLimit(fetch=[10000]) - EnumerableHashJoin(condition=[=($13, $15)], joinType=[left]) - EnumerableSort(sort0=[$13], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=['(?^[A-Z])'], expr#20=['initial'], expr#21=[REX_EXTRACT($t6, $t19, $t20)], proj#0..12=[{exprs}], initial=[$t21]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - EnumerableLimit(fetch=[50000]) - EnumerableCalc(expr#0..18=[{inputs}], proj#0..12=[{exprs}]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableCalc(expr#0..27=[{inputs}], proj#0..13=[{exprs}], b.account_number=[$t15], b.firstname=[$t16], b.address=[$t17], b.birthdate=[$t18], b.gender=[$t19], b.city=[$t20], b.lastname=[$t21], b.balance=[$t22], b.employer=[$t23], b.state=[$t24], b.age=[$t25], b.email=[$t26], b.male=[$t27]) + EnumerableLimit(fetch=[10000]) + EnumerableHashJoin(condition=[=($14, $16)], joinType=[left]) + EnumerableSort(sort0=[$13], dir0=[ASC-nulls-first]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=['(?^[A-Z])'], expr#20=['initial'], expr#21=[REX_EXTRACT($t6, $t19, $t20)], expr#22=[CAST($t21):VARCHAR], proj#0..12=[{exprs}], initial=[$t21], $f14=[$t22]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableLimit(fetch=[50000]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..12=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_push.json deleted file mode 100644 index 8257acea073..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalFilter(condition=[>($8, 30)])\n LogicalSort(sort0=[$8], dir0=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], age=[$t8], $condition=[$t18])\n EnumerableSort(sort0=[$8], dir0=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_push.yaml new file mode 100644 index 00000000000..c094305e591 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[>($8, SAFE_CAST(30:BIGINT))]) + LogicalSort(sort0=[$8], dir0=[ASC-nulls-first]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30:BIGINT], expr#18=[>($t8, $t17)], age=[$t8], $condition=[$t18]) + EnumerableSort(sort0=[$8], dir0=[ASC-nulls-first]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_rename_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_rename_push.json deleted file mode 100644 index 1051442f396..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_rename_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(alias=[$17])\n LogicalSort(sort0=[$17], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], name=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], alias=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..16=[{inputs}], alias=[$t1])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_rename_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_rename_push.yaml new file mode 100644 index 00000000000..6b2955654bd --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_rename_push.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC-nulls-first]) + LogicalProject(alias=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) + EnumerableCalc(expr#0..16=[{inputs}], alias=[$t1]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json deleted file mode 100644 index 9d64b554b18..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(avg(balance)=[$1], state=[$0])\n LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)])\n LogicalProject(state=[$7], balance=[$3])\n LogicalSort(sort0=[$3], sort1=[$8], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(balance)=[$t8], state=[$t0])\n EnumerableAggregate(group=[{7}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.yaml new file mode 100644 index 00000000000..e3d81b8cf94 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(balance)=[$1], state=[$0]) + LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) + LogicalProject(state=[$0], balance=[$1]) + LogicalSort(sort0=[$1], sort1=[$2], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) + LogicalProject(state=[$7], balance=[$3], age=[$8]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(balance)=[$t8], state=[$t0]) + EnumerableAggregate(group=[{7}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_limit_push.yaml index c39d769eb4b..1f2f9786e29 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_limit_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_limit_push.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$8]) - LogicalSort(sort0=[$8], dir0=[ASC-nulls-first], fetch=[5]) + LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[5]) + LogicalProject(age=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_type_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_type_push.json index 1e075f5e3bf..db71b50ed21 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_type_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_type_push.json @@ -3,4 +3,4 @@ "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(sort0=[$17], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], $f17=[SAFE_CAST($8)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", "physical": "EnumerableCalc(expr#0..1=[{inputs}], age=[$t0])\n EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[SAFE_CAST($t8)], age=[$t8], $f17=[$t17])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_span.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_span.json deleted file mode 100644 index 0d3dd2ba191..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_span.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(age,10)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(age,10)=[SPAN($10, 10, null:NULL)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(age,10)=[$t0])\n EnumerableAggregate(group=[{0}], count()=[COUNT()])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[10], expr#20=[null:NULL], expr#21=[SPAN($t10, $t19, $t20)], span(age,10)=[$t21])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_span.yaml new file mode 100644 index 00000000000..d7e77b6805d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_span.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], span(age,10)=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(span(age,10)=[SPAN($10, 10, null:ANY)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(age,10)=[$t0]) + EnumerableAggregate(group=[{0}], count()=[COUNT()]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[10], expr#20=[null:ANY], expr#21=[SPAN($t10, $t19, $t20)], span(age,10)=[$t21]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_span_non_bucket_nullable.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_span_non_bucket_nullable.json deleted file mode 100644 index ad39983d50d..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_span_non_bucket_nullable.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(age,10)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(age,10)=[SPAN($10, 10, null:NULL)])\n LogicalFilter(condition=[IS NOT NULL($10)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(age,10)=[$t0])\n EnumerableAggregate(group=[{0}], count()=[COUNT()])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[10], expr#20=[null:NULL], expr#21=[SPAN($t10, $t19, $t20)], expr#22=[IS NOT NULL($t10)], span(age,10)=[$t21], $condition=[$t22])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_span_non_bucket_nullable.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_span_non_bucket_nullable.yaml new file mode 100644 index 00000000000..3b310db9778 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_span_non_bucket_nullable.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], span(age,10)=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(span(age,10)=[SPAN($10, 10, null:ANY)]) + LogicalFilter(condition=[IS NOT NULL($10)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(age,10)=[$t0]) + EnumerableAggregate(group=[{0}], count()=[COUNT()]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[10], expr#20=[null:ANY], expr#21=[SPAN($t10, $t19, $t20)], expr#22=[IS NOT NULL($t10)], span(age,10)=[$t21], $condition=[$t22]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan.yaml index 3843b2bce4a..194654fe2b4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan.yaml @@ -11,4 +11,4 @@ calcite: EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(birthdate,1m)=[$t0]) EnumerableAggregate(group=[{0}], count()=[COUNT()]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=['m'], expr#21=[SPAN($t3, $t19, $t20)], expr#22=[IS NOT NULL($t3)], span(birthdate,1m)=[$t21], $condition=[$t22]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan2.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan2.yaml index af739c44d85..ea76d66182b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan2.yaml @@ -11,4 +11,4 @@ calcite: EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(birthdate,1M)=[$t0]) EnumerableAggregate(group=[{0}], count()=[COUNT()]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=['M'], expr#21=[SPAN($t3, $t19, $t20)], expr#22=[IS NOT NULL($t3)], span(birthdate,1M)=[$t21], $condition=[$t22]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_distinct_count.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_distinct_count.yaml index 550cf0ea9cb..28e74cd3dad 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_distinct_count.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_distinct_count.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], distinct_states=[$18]) - LogicalSort(sort0=[$17], dir0=[ASC]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], distinct_states=[DISTINCT_COUNT_APPROX($7) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], distinct_states=[$11]) + LogicalSort(sort0=[$12], dir0=[ASC]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], distinct_states=[DISTINCT_COUNT_APPROX($7) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)], __stream_seq__=[$17]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | @@ -12,4 +12,4 @@ calcite: EnumerableSort(sort0=[$17], dir0=[ASC]) EnumerableWindow(window#0=[window(partition {4} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [DISTINCT_COUNT_APPROX($7)])]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest.yaml index c37fae48771..fd6807d6892 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[$12], latest_message=[$13]) - LogicalSort(sort0=[$11], dir0=[ASC]) - LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], _id=[$5], _index=[$6], _score=[$7], _maxscore=[$8], _sort=[$9], _routing=[$10], __stream_seq__=[$11], earliest_message=[ARG_MIN($3, $2) OVER (PARTITION BY $1 ROWS UNBOUNDED PRECEDING)], latest_message=[ARG_MAX($3, $2) OVER (PARTITION BY $1 ROWS UNBOUNDED PRECEDING)]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[$5], latest_message=[$6]) + LogicalSort(sort0=[$7], dir0=[ASC]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[ARG_MIN($3, $2) OVER (PARTITION BY $1 ROWS UNBOUNDED PRECEDING)], latest_message=[ARG_MAX($3, $2) OVER (PARTITION BY $1 ROWS UNBOUNDED PRECEDING)], __stream_seq__=[$11]) LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], _id=[$5], _index=[$6], _score=[$7], _maxscore=[$8], _sort=[$9], _routing=[$10], __stream_seq__=[ROW_NUMBER() OVER ()]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) physical: | @@ -12,4 +12,4 @@ calcite: EnumerableSort(sort0=[$11], dir0=[ASC]) EnumerableWindow(window#0=[window(partition {1} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ARG_MIN($3, $2), ARG_MAX($3, $2)])]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_custom_time.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_custom_time.yaml index b85e4b6b7bb..f0320664650 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_custom_time.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_custom_time.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[$12], latest_message=[$13]) - LogicalSort(sort0=[$11], dir0=[ASC]) - LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], _id=[$5], _index=[$6], _score=[$7], _maxscore=[$8], _sort=[$9], _routing=[$10], __stream_seq__=[$11], earliest_message=[ARG_MIN($3, $0) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)], latest_message=[ARG_MAX($3, $0) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[$5], latest_message=[$6]) + LogicalSort(sort0=[$7], dir0=[ASC]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[ARG_MIN($3, $0) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)], latest_message=[ARG_MAX($3, $0) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)], __stream_seq__=[$11]) LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], _id=[$5], _index=[$6], _score=[$7], _maxscore=[$8], _sort=[$9], _routing=[$10], __stream_seq__=[ROW_NUMBER() OVER ()]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) physical: | @@ -12,4 +12,4 @@ calcite: EnumerableSort(sort0=[$11], dir0=[ASC]) EnumerableWindow(window#0=[window(partition {4} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ARG_MIN($3, $0), ARG_MAX($3, $0)])]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_no_group.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_no_group.yaml index 79dcbca7555..ef826c16039 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_no_group.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_no_group.yaml @@ -7,4 +7,4 @@ calcite: EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..12=[{inputs}], proj#0..4=[{exprs}], $5=[$t11], $6=[$t12]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ARG_MIN($3, $2), ARG_MAX($3, $2)])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml index 522e7922e68..27ed036a7f7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml @@ -1,16 +1,17 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$18]) - LogicalSort(sort0=[$17], dir0=[ASC]) - LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{4, 17}]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalAggregate(group=[{}], avg_age=[AVG($0)]) - LogicalProject(age=[$8]) - LogicalFilter(condition=[AND(>=($17, -($cor0.__stream_seq__, 1)), <=($17, $cor0.__stream_seq__), OR(=($4, $cor0.gender), AND(IS NULL($4), IS NULL($cor0.gender))))]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$11]) + LogicalSort(sort0=[$12], dir0=[ASC]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$18], __stream_seq__=[$17]) + LogicalCorrelate(correlation=[$cor1], joinType=[left], requiredColumns=[{4, 17}]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalAggregate(group=[{}], avg_age=[AVG($0)]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[AND(>=($17, -($cor1.__stream_seq__, 1)), <=($17, $cor1.__stream_seq__), OR(=($4, $cor1.gender), AND(IS NULL($4), IS NULL($cor1.gender))))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) EnumerableLimit(fetch=[10000]) @@ -29,4 +30,4 @@ calcite: CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) EnumerableCalc(expr#0..17=[{inputs}], gender=[$t4], age=[$t8], $2=[$t17]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml index a0634448b5e..3d60cd4bffb 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml @@ -1,16 +1,17 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$18]) - LogicalSort(sort0=[$17], dir0=[ASC]) - LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{4, 17}]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalAggregate(group=[{}], avg_age=[AVG($0)]) - LogicalProject(age=[$8]) - LogicalFilter(condition=[AND(>=($17, -($cor0.__stream_seq__, 1)), <=($17, $cor0.__stream_seq__), =($4, $cor0.gender))]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$11]) + LogicalSort(sort0=[$12], dir0=[ASC]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$18], __stream_seq__=[$17]) + LogicalCorrelate(correlation=[$cor1], joinType=[left], requiredColumns=[{4, 17}]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalAggregate(group=[{}], avg_age=[AVG($0)]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[AND(>=($17, -($cor1.__stream_seq__, 1)), <=($17, $cor1.__stream_seq__), =($4, $cor1.gender))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t16]) EnumerableLimit(fetch=[10000]) @@ -28,4 +29,4 @@ calcite: CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) EnumerableCalc(expr#0..17=[{inputs}], gender=[$t4], age=[$t8], $2=[$t17]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_null_bucket.yaml index 08876045225..ef7184239e8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_null_bucket.yaml @@ -1,16 +1,16 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$18]) - LogicalSort(sort0=[$17], dir0=[ASC]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], avg_age=[CASE(IS NOT NULL($4), /(SUM($8) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING), CAST(COUNT($8) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)):DOUBLE NOT NULL), null:DOUBLE)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$11]) + LogicalSort(sort0=[$12], dir0=[ASC]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[CASE(IS NOT NULL($4), /(CASE(>(COUNT($8) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING), 0), SUM($8) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING), null:BIGINT), CAST(COUNT($8) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)):DOUBLE NOT NULL), null:DOUBLE)], __stream_seq__=[$17]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..14=[{inputs}], expr#15=[CAST($t14):DOUBLE NOT NULL], expr#16=[/($t13, $t15)], expr#17=[null:DOUBLE], expr#18=[CASE($t12, $t16, $t17)], proj#0..10=[{exprs}], avg_age=[$t18]) + EnumerableCalc(expr#0..14=[{inputs}], expr#15=[0:BIGINT], expr#16=[>($t13, $t15)], expr#17=[null:BIGINT], expr#18=[CASE($t16, $t14, $t17)], expr#19=[CAST($t13):DOUBLE NOT NULL], expr#20=[/($t18, $t19)], expr#21=[null:DOUBLE], expr#22=[CASE($t12, $t20, $t21)], proj#0..10=[{exprs}], avg_age=[$t22]) EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$11], dir0=[ASC]) - EnumerableWindow(window#0=[window(partition {4} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($8), COUNT($8)])]) + EnumerableWindow(window#0=[window(partition {4} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [COUNT($8), $SUM0($8)])]) EnumerableCalc(expr#0..17=[{inputs}], expr#18=[IS NOT NULL($t4)], proj#0..10=[{exprs}], __stream_seq__=[$t17], $12=[$t18]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml index 5664cc6aa87..7b5e55de709 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml @@ -1,26 +1,27 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$21]) - LogicalSort(sort0=[$17], dir0=[ASC]) - LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{4, 17, 20}]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), COALESCE(SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0))]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalAggregate(group=[{}], avg_age=[AVG($0)]) - LogicalProject(age=[$8]) - LogicalFilter(condition=[AND(<($17, $cor0.__stream_seq__), =($20, $cor0.__seg_id__), OR(=($4, $cor0.gender), AND(IS NULL($4), IS NULL($cor0.gender))))]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), COALESCE(SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0))]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$11]) + LogicalSort(sort0=[$12], dir0=[ASC]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$21], __stream_seq__=[$17]) + LogicalCorrelate(correlation=[$cor1], joinType=[left], requiredColumns=[{4, 17, 20}]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), CASE(IS NOT NULL(CASE(>(COUNT($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0), SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), null:BIGINT)), CAST(CASE(>(COUNT($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0), SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), null:BIGINT)):BIGINT NOT NULL, 0:BIGINT))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalAggregate(group=[{}], avg_age=[AVG($0)]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[AND(<($17, $cor1.__stream_seq__), =($20, $cor1.__seg_id__), OR(=($4, $cor1.gender), AND(IS NULL($4), IS NULL($cor1.gender))))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), CASE(IS NOT NULL(CASE(>(COUNT($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0), SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), null:BIGINT)), CAST(CASE(>(COUNT($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0), SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), null:BIGINT)):BIGINT NOT NULL, 0:BIGINT))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) EnumerableLimit(fetch=[10000]) EnumerableMergeJoin(condition=[AND(=($11, $15), =($12, $16), =($13, $17), IS NOT DISTINCT FROM($4, $14))], joinType=[left]) EnumerableSort(sort0=[$11], sort1=[$12], sort2=[$13], dir0=[ASC], dir1=[ASC], dir2=[ASC]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[0], expr#18=[COALESCE($t16, $t17)], expr#19=[+($t15, $t18)], proj#0..11=[{exprs}], __seg_id__=[$t19], $f16=[$t14]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($12)])], window#1=[window(rows between UNBOUNDED PRECEDING and $15 PRECEDING aggs [$SUM0($13)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], proj#0..10=[{exprs}], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $14=[$t26]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[0:BIGINT], expr#19=[>($t16, $t18)], expr#20=[null:BIGINT], expr#21=[CASE($t19, $t17, $t20)], expr#22=[IS NOT NULL($t21)], expr#23=[CAST($t21):BIGINT NOT NULL], expr#24=[CASE($t22, $t23, $t18)], expr#25=[+($t15, $t24)], proj#0..11=[{exprs}], __seg_id__=[$t25], $f16=[$t14]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($12)])], window#1=[window(rows between UNBOUNDED PRECEDING and $15 PRECEDING aggs [COUNT($13), $SUM0($13)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34:BIGINT], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25:BIGINT], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], proj#0..10=[{exprs}], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $14=[$t26]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) EnumerableSort(sort0=[$1], sort1=[$2], sort2=[$3], dir0=[ASC], dir1=[ASC], dir2=[ASC]) @@ -28,13 +29,13 @@ calcite: EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) EnumerableAggregate(group=[{0, 1, 2, 3}]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $4=[$t26]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0:BIGINT], expr#9=[>($t6, $t8)], expr#10=[null:BIGINT], expr#11=[CASE($t9, $t7, $t10)], expr#12=[IS NOT NULL($t11)], expr#13=[CAST($t11):BIGINT NOT NULL], expr#14=[CASE($t12, $t13, $t8)], expr#15=[+($t5, $t14)], proj#0..1=[{exprs}], __seg_id__=[$t15], $f16=[$t4]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [COUNT($3), $SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34:BIGINT], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25:BIGINT], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $4=[$t26]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0:BIGINT], expr#9=[>($t6, $t8)], expr#10=[null:BIGINT], expr#11=[CASE($t9, $t7, $t10)], expr#12=[IS NOT NULL($t11)], expr#13=[CAST($t11):BIGINT NOT NULL], expr#14=[CASE($t12, $t13, $t8)], expr#15=[+($t5, $t14)], proj#0..2=[{exprs}], __seg_id__=[$t15]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [COUNT($4), $SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34:BIGINT], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25:BIGINT], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml index 40fb4087001..dd5b3453bfa 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml @@ -1,39 +1,40 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$21]) - LogicalSort(sort0=[$17], dir0=[ASC]) - LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{4, 17, 20}]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), COALESCE(SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0))]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalAggregate(group=[{}], avg_age=[AVG($0)]) - LogicalProject(age=[$8]) - LogicalFilter(condition=[AND(<($17, $cor0.__stream_seq__), =($20, $cor0.__seg_id__), =($4, $cor0.gender))]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), COALESCE(SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0))]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$11]) + LogicalSort(sort0=[$12], dir0=[ASC]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$21], __stream_seq__=[$17]) + LogicalCorrelate(correlation=[$cor1], joinType=[left], requiredColumns=[{4, 17, 20}]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), CASE(IS NOT NULL(CASE(>(COUNT($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0), SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), null:BIGINT)), CAST(CASE(>(COUNT($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0), SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), null:BIGINT)):BIGINT NOT NULL, 0:BIGINT))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalAggregate(group=[{}], avg_age=[AVG($0)]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[AND(<($17, $cor1.__stream_seq__), =($20, $cor1.__seg_id__), =($4, $cor1.gender))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), CASE(IS NOT NULL(CASE(>(COUNT($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0), SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), null:BIGINT)), CAST(CASE(>(COUNT($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0), SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), null:BIGINT)):BIGINT NOT NULL, 0:BIGINT))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t16]) EnumerableLimit(fetch=[10000]) EnumerableHashJoin(condition=[AND(=($4, $13), =($11, $14), =($12, $15))], joinType=[left]) EnumerableSort(sort0=[$11], dir0=[ASC]) - EnumerableCalc(expr#0..15=[{inputs}], expr#16=[0], expr#17=[COALESCE($t15, $t16)], expr#18=[+($t14, $t17)], proj#0..11=[{exprs}], __seg_id__=[$t18]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($12)])], window#1=[window(rows between UNBOUNDED PRECEDING and $14 PRECEDING aggs [$SUM0($13)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], proj#0..10=[{exprs}], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[0:BIGINT], expr#18=[>($t15, $t17)], expr#19=[null:BIGINT], expr#20=[CASE($t18, $t16, $t19)], expr#21=[IS NOT NULL($t20)], expr#22=[CAST($t20):BIGINT NOT NULL], expr#23=[CASE($t21, $t22, $t17)], expr#24=[+($t14, $t23)], proj#0..11=[{exprs}], __seg_id__=[$t24]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($12)])], window#1=[window(rows between UNBOUNDED PRECEDING and $14 PRECEDING aggs [COUNT($13), $SUM0($13)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34:BIGINT], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25:BIGINT], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], proj#0..10=[{exprs}], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t3)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t4)], proj#0..2=[{exprs}], avg_age=[$t10]) EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($4)], agg#1=[COUNT($4)]) EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) EnumerableAggregate(group=[{0, 1, 2}]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0:BIGINT], expr#8=[>($t5, $t7)], expr#9=[null:BIGINT], expr#10=[CASE($t8, $t6, $t9)], expr#11=[IS NOT NULL($t10)], expr#12=[CAST($t10):BIGINT NOT NULL], expr#13=[CASE($t11, $t12, $t7)], expr#14=[+($t4, $t13)], proj#0..1=[{exprs}], __seg_id__=[$t14]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [COUNT($3), $SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34:BIGINT], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25:BIGINT], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0:BIGINT], expr#9=[>($t6, $t8)], expr#10=[null:BIGINT], expr#11=[CASE($t9, $t7, $t10)], expr#12=[IS NOT NULL($t11)], expr#13=[CAST($t11):BIGINT NOT NULL], expr#14=[CASE($t12, $t13, $t8)], expr#15=[+($t5, $t14)], proj#0..2=[{exprs}], __seg_id__=[$t15]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [COUNT($4), $SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34:BIGINT], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25:BIGINT], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_strftime_function.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_strftime_function.json deleted file mode 100644 index a48f7e114c8..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_strftime_function.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(fetch=[1])\n LogicalProject(formatted_date=[STRFTIME(1521467703, '%Y-%m-%d':VARCHAR)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[1521467703], expr#18=['%Y-%m-%d':VARCHAR], expr#19=[STRFTIME($t17, $t18)], formatted_date=[$t19])\n EnumerableLimit(fetch=[1])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_strftime_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_strftime_function.yaml new file mode 100644 index 00000000000..d2a90698f9b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_strftime_function.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(fetch=[1]) + LogicalProject(formatted_date=[STRFTIME(1521467703, '%Y-%m-%d')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[1521467703], expr#18=['%Y-%m-%d'], expr#19=[STRFTIME($t17, $t18)], formatted_date=[$t19]) + EnumerableLimit(fetch=[1]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_ilike_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_ilike_function.yaml index 41638cd1b16..7a0c26ff5a2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_ilike_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_ilike_function.yaml @@ -7,4 +7,4 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%Holmes%'], expr#18=['\'], expr#19=[ILIKE($t2, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function.yaml index 6be02086bb0..25e928813b9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function.yaml @@ -7,4 +7,4 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%Holmes%'], expr#18=['\'], expr#19=[LIKE($t2, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function_case_insensitive.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function_case_insensitive.yaml index 41638cd1b16..7a0c26ff5a2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function_case_insensitive.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function_case_insensitive.yaml @@ -7,4 +7,4 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%Holmes%'], expr#18=['\'], expr#19=[ILIKE($t2, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml index e982ce038e2..33b4a53b968 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml @@ -3,19 +3,19 @@ calcite: LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[AVG($2)]) - LogicalProject(@timestamp=[$0], host=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], avg(cpu_usage)=[$2]) + LogicalProject(@timestamp=[$0], host=[CASE(IS NULL($1), 'NULL':VARCHAR, <=($5, 10), $1, 'OTHER':VARCHAR)], avg(cpu_usage)=[$2]) LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalProject(@timestamp=[$1], host=[$0], avg(cpu_usage)=[$2]) - LogicalAggregate(group=[{0, 2}], avg(cpu_usage)=[AVG($1)]) - LogicalProject(host=[$4], cpu_usage=[$7], @timestamp0=[SPAN($1, 1, 'm')]) + LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[AVG($2)]) + LogicalProject(host=[$4], @timestamp=[SPAN($1, 1, 'm')], cpu_usage=[$7]) LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, events]]) LogicalProject(host=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(host=[$0], avg(cpu_usage)=[$2]) - LogicalAggregate(group=[{0, 2}], avg(cpu_usage)=[AVG($1)]) - LogicalProject(host=[$4], cpu_usage=[$7], @timestamp0=[SPAN($1, 1, 'm')]) + LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[AVG($2)]) + LogicalProject(host=[$4], $f1=[SPAN($1, 1, 'm')], cpu_usage=[$7]) LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, events]]) physical: | @@ -23,18 +23,18 @@ calcite: EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(cpu_usage)=[$t8]) EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL':VARCHAR], expr#7=[10:BIGINT], expr#8=[<=($t4, $t7)], expr#9=['OTHER':VARCHAR], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], @timestamp=[$t1], host=[$t0], avg(cpu_usage)=[$t8]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], expr#19=[IS NOT NULL($t1)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], host=[$t4], cpu_usage=[$t7], @timestamp0=[$t18], $condition=[$t21]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], expr#19=[IS NOT NULL($t1)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], host=[$t4], @timestamp=[$t18], cpu_usage=[$t7], $condition=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($2)]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(cpu_usage)=[$t8]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], expr#19=[IS NOT NULL($t1)], expr#20=[IS NOT NULL($t7)], expr#21=[IS NOT NULL($t4)], expr#22=[AND($t19, $t20, $t21)], host=[$t4], cpu_usage=[$t7], @timestamp0=[$t18], $condition=[$t22]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], expr#19=[IS NOT NULL($t1)], expr#20=[IS NOT NULL($t7)], expr#21=[IS NOT NULL($t4)], expr#22=[AND($t19, $t20, $t21)], host=[$t4], @timestamp=[$t18], cpu_usage=[$t7], $condition=[$t22]) CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart_count.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart_count.yaml index 2979778506a..c2fcb2c3345 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart_count.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart_count.yaml @@ -3,11 +3,11 @@ calcite: LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) LogicalAggregate(group=[{0, 1}], count()=[SUM($2)]) - LogicalProject(@timestamp=[$0], host=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], count()=[$2]) + LogicalProject(@timestamp=[$0], host=[CASE(IS NULL($1), 'NULL':VARCHAR, <=($5, 10), $1, 'OTHER':VARCHAR)], count()=[$2]) LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalProject(@timestamp=[$1], host=[$0], count()=[$2]) LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) - LogicalProject(host=[$4], @timestamp0=[SPAN($1, 1, 'm')]) + LogicalProject(host=[$4], @timestamp=[SPAN($1, 1, 'm')]) LogicalFilter(condition=[IS NOT NULL($1)]) CalciteLogicalIndexScan(table=[[OpenSearch, events]]) LogicalProject(host=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) @@ -15,19 +15,19 @@ calcite: LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(host=[$0], count()=[$2]) LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) - LogicalProject(host=[$4], @timestamp0=[SPAN($1, 1, 'm')]) + LogicalProject(host=[$4], $f1=[SPAN($1, 1, 'm')]) LogicalFilter(condition=[IS NOT NULL($1)]) CalciteLogicalIndexScan(table=[[OpenSearch, events]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) EnumerableAggregate(group=[{0, 1}], count()=[$SUM0($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], @timestamp=[$t0], host=[$t10], count()=[$t2]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL':VARCHAR], expr#7=[10:BIGINT], expr#8=[<=($t4, $t7)], expr#9=['OTHER':VARCHAR], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], @timestamp=[$t0], host=[$t10], count()=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], @timestamp=[$t1], host=[$t0], count()=[$t2]) EnumerableAggregate(group=[{0, 1}], count()=[COUNT()]) - EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], expr#19=[IS NOT NULL($t1)], host=[$t4], @timestamp0=[$t18], $condition=[$t19]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], expr#19=[IS NOT NULL($t1)], host=[$t4], @timestamp=[$t18], $condition=[$t19]) CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], $1=[$t2]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_top_usenull_false.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_top_usenull_false.yaml index 17a17081e61..60e923ec9fd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_top_usenull_false.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_top_usenull_false.yaml @@ -2,16 +2,16 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(gender=[$0], state=[$1], count=[$2]) - LogicalFilter(condition=[<=($3, 2)]) - LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC)]) + LogicalFilter(condition=[<=($3, SAFE_CAST(2:BIGINT))]) + LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC NULLS FIRST)]) LogicalAggregate(group=[{0, 1}], count=[COUNT()]) LogicalProject(gender=[$4], state=[$7]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[2], expr#5=[<=($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[2:BIGINT], expr#5=[<=($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) EnumerableWindow(window#0=[window(partition {0} order by [2 DESC] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{4, 7}], count=[COUNT()]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t4)], expr#18=[IS NOT NULL($t7)], expr#19=[AND($t17, $t18)], proj#0..16=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_top_usenull_true.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_top_usenull_true.yaml index 5fb27c851af..c869e1ebfdd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_top_usenull_true.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_top_usenull_true.yaml @@ -2,14 +2,14 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(gender=[$0], state=[$1], count=[$2]) - LogicalFilter(condition=[<=($3, 2)]) - LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC)]) + LogicalFilter(condition=[<=($3, SAFE_CAST(2:BIGINT))]) + LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC NULLS FIRST)]) LogicalAggregate(group=[{0, 1}], count=[COUNT()]) LogicalProject(gender=[$4], state=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[2], expr#5=[<=($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[2:BIGINT], expr#5=[<=($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) EnumerableWindow(window#0=[window(partition {0} order by [2 DESC] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{4, 7}], count=[COUNT()]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_push.yaml index 0d929025da2..ea4a2f6f40a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_push.yaml @@ -1,14 +1,15 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(ageTrend=[CASE(>(COUNT() OVER (ROWS 1 PRECEDING), 1), /(SUM($8) OVER (ROWS 1 PRECEDING), CAST(COUNT($8) OVER (ROWS 1 PRECEDING)):DOUBLE NOT NULL), null:NULL)]) + LogicalProject(ageTrend=[CASE(>(COUNT() OVER (ROWS 1 PRECEDING), 1), /(CASE(>(COUNT($8) OVER (ROWS 1 PRECEDING), 0), SUM($8) OVER (ROWS 1 PRECEDING), null:BIGINT), CAST(COUNT($8) OVER (ROWS 1 PRECEDING)):DOUBLE NOT NULL), null:DOUBLE)]) LogicalFilter(condition=[IS NOT NULL($8)]) LogicalSort(fetch=[5]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[>($t1, $t4)], expr#6=[CAST($t3):DOUBLE NOT NULL], expr#7=[/($t2, $t6)], expr#8=[null:NULL], expr#9=[CASE($t5, $t7, $t8)], ageTrend=[$t9]) - EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])], constants=[[1]]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1:BIGINT], expr#5=[>($t1, $t4)], expr#6=[0:BIGINT], expr#7=[>($t2, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t3, $t8)], expr#10=[CAST($t2):DOUBLE NOT NULL], expr#11=[/($t9, $t10)], expr#12=[null:DOUBLE], expr#13=[CASE($t5, $t11, $t12)], ageTrend=[$t13]) + EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), COUNT($0), $SUM0($0)])], constants=[[1]]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t8)], age=[$t8], $condition=[$t17]) EnumerableLimit(fetch=[5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_sort_push.yaml index 2427a30e1a7..257df46ee94 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_sort_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_sort_push.yaml @@ -1,16 +1,18 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(ageTrend=[CASE(>(COUNT() OVER (ROWS 1 PRECEDING), 1), /(SUM($8) OVER (ROWS 1 PRECEDING), CAST(COUNT($8) OVER (ROWS 1 PRECEDING)):DOUBLE NOT NULL), null:NULL)]) + LogicalProject(ageTrend=[CASE(>(COUNT() OVER (ROWS 1 PRECEDING), 1), /(CASE(>(COUNT($8) OVER (ROWS 1 PRECEDING), 0), SUM($8) OVER (ROWS 1 PRECEDING), null:BIGINT), CAST(COUNT($8) OVER (ROWS 1 PRECEDING)):DOUBLE NOT NULL), null:DOUBLE)]) LogicalFilter(condition=[IS NOT NULL($8)]) LogicalSort(sort0=[$8], dir0=[ASC]) - LogicalSort(fetch=[5]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[>($t1, $t4)], expr#6=[CAST($t3):DOUBLE NOT NULL], expr#7=[/($t2, $t6)], expr#8=[null:NULL], expr#9=[CASE($t5, $t7, $t8)], ageTrend=[$t9]) - EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])], constants=[[1]]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1:BIGINT], expr#5=[>($t1, $t4)], expr#6=[0:BIGINT], expr#7=[>($t2, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t3, $t8)], expr#10=[CAST($t2):DOUBLE NOT NULL], expr#11=[/($t9, $t10)], expr#12=[null:DOUBLE], expr#13=[CASE($t5, $t11, $t12)], ageTrend=[$t13]) + EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), COUNT($0), $SUM0($0)])], constants=[[1]]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t8)], age=[$t8], $condition=[$t17]) EnumerableSort(sort0=[$8], dir0=[ASC]) EnumerableLimit(fetch=[5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_values_aggregation.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_values_aggregation.json index 7fc9ba40c61..c5a4f3714ad 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_values_aggregation.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_values_aggregation.json @@ -3,4 +3,4 @@ "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], age_values=[VALUES($0)])\n LogicalProject(age=[$8])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", "physical": "EnumerableLimit(fetch=[10000])\n EnumerableAggregate(group=[{}], age_values=[VALUES($8)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_week_argument_coercion.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_week_argument_coercion.json deleted file mode 100644 index c475102597f..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_week_argument_coercion.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(w=[WEEK(TIMESTAMP('2024-12-10':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=['2024-12-10':VARCHAR], expr#18=[TIMESTAMP($t17)], expr#19=[WEEK($t18)], w=[$t19])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_week_argument_coercion.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_week_argument_coercion.yaml new file mode 100644 index 00000000000..7cad776b702 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_week_argument_coercion.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(w=[WEEK(TIMESTAMP('2024-12-10'))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['2024-12-10'], expr#18=[TIMESTAMP($t17)], expr#19=[WEEK($t18)], w=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_absolute_time_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_absolute_time_range.yaml index 4cd552b198c..2f04c94fe2c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_absolute_time_range.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_absolute_time_range.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) - LogicalFilter(condition=[query_string(MAP('query', '(@timestamp:>=2022\-12\-10T13\:11\:04Z) AND (@timestamp:<=2025\-09\-03T15\:10\:00Z)':VARCHAR))]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, '(@timestamp:>=2022\-12\-10T13\:11\:04Z) AND (@timestamp:<=2025\-09\-03T15\:10\:00Z)':VARCHAR))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..9=[{inputs}], proj#0..3=[{exprs}]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->query_string(MAP('query', '(@timestamp:>=2022\-12\-10T13\:11\:04Z) AND (@timestamp:<=2025\-09\-03T15\:10\:00Z)':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=2022\\-12\\-10T13\\:11\\:04Z) AND (@timestamp:<=2025\\-09\\-03T15\\:10\\:00Z)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->query_string(MAP('query':VARCHAR, '(@timestamp:>=2022\-12\-10T13\:11\:04Z) AND (@timestamp:<=2025\-09\-03T15\:10\:00Z)':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=2022\\-12\\-10T13\\:11\\:04Z) AND (@timestamp:<=2025\\-09\\-03T15\\:10\\:00Z)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_chained_time_modifier.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_chained_time_modifier.yaml index b23c0a66e83..ff7643187d2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_chained_time_modifier.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_chained_time_modifier.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) - LogicalFilter(condition=[query_string(MAP('query', '(@timestamp:>=now\-3d\/d\-2h\+10m) AND (@timestamp:<=now\-1d\+1y\/M)':VARCHAR))]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, '(@timestamp:>=now\-3d\/d\-2h\+10m) AND (@timestamp:<=now\-1d\+1y\/M)':VARCHAR))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..9=[{inputs}], proj#0..3=[{exprs}]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->query_string(MAP('query', '(@timestamp:>=now\-3d\/d\-2h\+10m) AND (@timestamp:<=now\-1d\+1y\/M)':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=now\\-3d\\/d\\-2h\\+10m) AND (@timestamp:<=now\\-1d\\+1y\\/M)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->query_string(MAP('query':VARCHAR, '(@timestamp:>=now\-3d\/d\-2h\+10m) AND (@timestamp:<=now\-1d\+1y\/M)':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=now\\-3d\\/d\\-2h\\+10m) AND (@timestamp:<=now\\-1d\\+1y\\/M)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_numeric_time_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_numeric_time_range.yaml index e7392a5f261..44f2d2173ff 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_numeric_time_range.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_numeric_time_range.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) - LogicalFilter(condition=[query_string(MAP('query', '(@timestamp:>=1000) AND (@timestamp:<=1754020061123.456)':VARCHAR))]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, '(@timestamp:>=1000) AND (@timestamp:<=1754020061123.456)':VARCHAR))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..9=[{inputs}], proj#0..3=[{exprs}]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->query_string(MAP('query', '(@timestamp:>=1000) AND (@timestamp:<=1754020061123.456)':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=1000) AND (@timestamp:<=1754020061123.456)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->query_string(MAP('query':VARCHAR, '(@timestamp:>=1000) AND (@timestamp:<=1754020061123.456)':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=1000) AND (@timestamp:<=1754020061123.456)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_relative_time_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_relative_time_range.yaml index 18f66a611be..36bf49dfc9b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_relative_time_range.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_relative_time_range.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) - LogicalFilter(condition=[query_string(MAP('query', '(@timestamp:>=now\-3M) AND (@timestamp:<=now\+30d)':VARCHAR))]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, '(@timestamp:>=now\-3M) AND (@timestamp:<=now\+30d)':VARCHAR))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..9=[{inputs}], proj#0..3=[{exprs}]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->query_string(MAP('query', '(@timestamp:>=now\-3M) AND (@timestamp:<=now\+30d)':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=now\\-3M) AND (@timestamp:<=now\\+30d)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->query_string(MAP('query':VARCHAR, '(@timestamp:>=now\-3M) AND (@timestamp:<=now\+30d)':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=now\\-3M) AND (@timestamp:<=now\\+30d)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_relative_time_snap.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_relative_time_snap.yaml index e864b18eeab..4de2f2c833f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_relative_time_snap.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/search_with_relative_time_snap.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) - LogicalFilter(condition=[query_string(MAP('query', '(@timestamp:>=now\-3M\/y) AND (@timestamp:<=now)':VARCHAR))]) + LogicalFilter(condition=[query_string(MAP('query':VARCHAR, '(@timestamp:>=now\-3M\/y) AND (@timestamp:<=now)':VARCHAR))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..9=[{inputs}], proj#0..3=[{exprs}]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->query_string(MAP('query', '(@timestamp:>=now\-3M\/y) AND (@timestamp:<=now)':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=now\\-3M\\/y) AND (@timestamp:<=now)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->query_string(MAP('query':VARCHAR, '(@timestamp:>=now\-3M\/y) AND (@timestamp:<=now)':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"query_string":{"query":"(@timestamp:>=now\\-3M\\/y) AND (@timestamp:<=now)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/udf_geoip_in_agg_pushed.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/udf_geoip_in_agg_pushed.yaml index 0dbea4e19e2..d78b7778928 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/udf_geoip_in_agg_pushed.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/udf_geoip_in_agg_pushed.yaml @@ -3,11 +3,11 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(count()=[$1], info.city=[$0]) LogicalAggregate(group=[{0}], count()=[COUNT()]) - LogicalProject(info.city=[ITEM(GEOIP('my-datasource':VARCHAR, $0), 'city')]) + LogicalProject(info.city=[ITEM(GEOIP('my-datasource', $0), 'city')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], info.city=[$t0]) EnumerableAggregate(group=[{0}], count()=[COUNT()]) - EnumerableCalc(expr#0..11=[{inputs}], expr#12=['my-datasource':VARCHAR], expr#13=[GEOIP($t12, $t0)], expr#14=['city'], expr#15=[ITEM($t13, $t14)], info.city=[$t15]) + EnumerableCalc(expr#0..11=[{inputs}], expr#12=['my-datasource'], expr#13=[GEOIP($t12, $t0)], expr#14=['city'], expr#15=[ITEM($t13, $t14)], info.city=[$t15]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_fillnull_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_fillnull_push.json deleted file mode 100644 index 4bf6dd22b0c..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_fillnull_push.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[age, balance]" - }, - "children": [ - { - "name": "OpenSearchEvalOperator", - "description": { - "expressions": { - "balance": "ifnull(balance, -1)", - "age": "ifnull(age, -1)" - } - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_fillnull_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_fillnull_push.yaml new file mode 100644 index 00000000000..55eb810d8bb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_fillnull_push.yaml @@ -0,0 +1,17 @@ +root: + name: ProjectOperator + description: + fields: "[age, balance]" + children: + - name: OpenSearchEvalOperator + description: + expressions: + balance: "ifnull(balance, -1)" + age: "ifnull(age, -1)" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ip.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ip.json deleted file mode 100644 index 7afd6497b9a..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ip.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[host]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_weblogs, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"host\":{\"from\":\"1.1.1.1\",\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ip.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ip.yaml new file mode 100644 index 00000000000..a5c6775f317 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ip.yaml @@ -0,0 +1,14 @@ +root: + name: ProjectOperator + description: + fields: "[host]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_weblogs,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\"\ + :{\"range\":{\"host\":{\"from\":\"1.1.1.1\",\"to\":null,\"include_lower\"\ + :false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\"\ + :[\"host\"],\"excludes\":[]}}, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ipv6_swapped.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ipv6_swapped.json deleted file mode 100644 index ff004cfeb4c..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ipv6_swapped.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[host]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_weblogs, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IANG9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMJDI9znTUIQE9bAIABUwADXZhbCRhcmd1bWVudHNxAH4AAUwADHZhbCRmdW5jdGlvbnQAP0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0wAEHZhbCRmdW5jdGlvbk5hbWVxAH4AA0wAFnZhbCRmdW5jdGlvblByb3BlcnRpZXNxAH4ABEwADnZhbCRyZXR1cm5UeXBlcQB+AAV4cQB+AAZzcQB+AAgAAAABdwQAAAABc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJTdHJpbmdWYWx1ZQBBMiVziQ4TAgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL1N0cmluZzt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwdAALOjpmZmZmOjEyMzR4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+ABJ4cHQACmNhc3RfdG9faXBxAH4ADXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzdAATW0xqYXZhL2xhbmcvT2JqZWN0O0wADmNhcHR1cmluZ0NsYXNzdAARTGphdmEvbGFuZy9DbGFzcztMABhmdW5jdGlvbmFsSW50ZXJmYWNlQ2xhc3NxAH4AEkwAHWZ1bmN0aW9uYWxJbnRlcmZhY2VNZXRob2ROYW1lcQB+ABJMACJmdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kU2lnbmF0dXJlcQB+ABJMAAlpbXBsQ2xhc3NxAH4AEkwADmltcGxNZXRob2ROYW1lcQB+ABJMABNpbXBsTWV0aG9kU2lnbmF0dXJlcQB+ABJMABZpbnN0YW50aWF0ZWRNZXRob2RUeXBlcQB+ABJ4cAAAAAZ1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAABc3EAfgAZAAAABnVxAH4AHQAAAAFzcQB+ABkAAAAGdXEAfgAdAAAAAHZyAEBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5vcGVyYXRvci5jb252ZXJ0LlR5cGVDYXN0T3BlcmF0b3JzAAAAAAAAAAAAAAB4cHQAO29yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUZ1bmN0aW9udAAFYXBwbHl0ACYoTGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3QAQG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL29wZXJhdG9yL2NvbnZlcnQvVHlwZUNhc3RPcGVyYXRvcnN0ABpsYW1iZGEkY2FzdFRvSXAkMTJjN2RjNDgkMXQAVChMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAqdnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAlcQB+ACZxAH4AJ3QAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckODc4MDY5YzgkMXQAkShMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AKnEAfgAsdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnEAfgAmdAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDtxAH4ALXQAFmxhbWJkYSRpbXBsJDhkNTg2Y2RjJDF0AMwoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0AI8oTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAXc3IAOW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uUHJvcGVydGllc888WWObqPmzAgADTAANY3VycmVudFpvbmVJZHQAEkxqYXZhL3RpbWUvWm9uZUlkO0wACm5vd0luc3RhbnR0ABNMamF2YS90aW1lL0luc3RhbnQ7TAAJcXVlcnlUeXBldAAnTG9yZy9vcGVuc2VhcmNoL3NxbC9leGVjdXRvci9RdWVyeVR5cGU7eHBzcgANamF2YS50aW1lLlNlcpVdhLobIkiyDAAAeHB3AggAeHNxAH4AOncNAgAAAABoifJmKvyT4Hh+cgAlb3JnLm9wZW5zZWFyY2guc3FsLmV4ZWN1dG9yLlF1ZXJ5VHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAA1BQTH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHEAfgA+dAACSVBzcgAxb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uUmVmZXJlbmNlRXhwcmVzc2lvbqtE71wSB4XWAgAETAAEYXR0cnEAfgASTAAFcGF0aHNxAH4AAUwAB3Jhd1BhdGhxAH4AEkwABHR5cGVxAH4ABXhwdAAEaG9zdHNyABpqYXZhLnV0aWwuQXJyYXlzJEFycmF5TGlzdNmkPL7NiAbSAgABWwABYXEAfgAaeHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AEZxAH4ARnEAfgBCeHNxAH4AFnQAAjw9cQB+AAlzcQB+ABkAAAAGdXEAfgAdAAAAAXNxAH4AGQAAAAZ1cQB+AB0AAAABc3EAfgAZAAAABnVxAH4AHQAAAAB2cgBJb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IucHJlZGljYXRlLkJpbmFyeVByZWRpY2F0ZU9wZXJhdG9ycwAAAAAAAAAAAAAAeHBxAH4AMHEAfgAmcQB+ADF0AElvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9wcmVkaWNhdGUvQmluYXJ5UHJlZGljYXRlT3BlcmF0b3JzdAAVbGFtYmRhJGx0ZSQ5NTA0OGZjMSQxdAB9KExvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AV3EAfgAscQB+ADBxAH4AJnEAfgAxcQB+AC10ACVsYW1iZGEkbnVsbE1pc3NpbmdIYW5kbGluZyRhNTAwNTI4MSQxdAC8KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AV3EAfgAsdAA+b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlVHJpRnVuY3Rpb25xAH4AJnQASihMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7cQB+AC10ABZsYW1iZGEkaW1wbCRhMGZiMzRkNCQxdAD3KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3QAuChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AS3EAfgA5fnEAfgBBdAAHQk9PTEVBTg==\\\"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ipv6_swapped.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ipv6_swapped.yaml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_multi_fields_relevance_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_multi_fields_relevance_push.json deleted file mode 100644 index 98dc18c5a3e..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_multi_fields_relevance_push.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]" - }, - "children": [{ - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"simple_query_string\":{\"query\":\"gmail\",\"fields\":[\"name^4.0\",\"email^1.0\"],\"analyzer\":\"english\",\"flags\":-1,\"default_operator\":\"or\",\"analyze_wildcard\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - }] - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_multi_fields_relevance_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_multi_fields_relevance_push.yaml new file mode 100644 index 00000000000..96a5d82d523 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_multi_fields_relevance_push.yaml @@ -0,0 +1,19 @@ +root: + name: ProjectOperator + description: + fields: "[account_number, firstname, address, balance, gender, city, employer,\ + \ state, age, email, lastname]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\"\ + :{\"simple_query_string\":{\"query\":\"gmail\",\"fields\":[\"name^4.0\"\ + ,\"email^1.0\"],\"analyzer\":\"english\",\"flags\":-1,\"default_operator\"\ + :\"or\",\"analyze_wildcard\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"fuzzy_transpositions\"\ + :true,\"boost\":1.0}},\"_source\":{\"includes\":[\"account_number\",\"firstname\"\ + ,\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\"\ + ,\"email\",\"lastname\"],\"excludes\":[]}}, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_multi_sort_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_multi_sort_push.json deleted file mode 100644 index 147184ed5e4..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_multi_sort_push.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[account_number, firstname, address, balance, gender]" - }, - "children": [{ - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\"],\"excludes\":[]},\"sort\":[{\"account_number\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"firstname.keyword\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"address\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"balance\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"balance\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"gender.keyword\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"account_number\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - }] - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_multi_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_multi_sort_push.yaml new file mode 100644 index 00000000000..9299a9b3d40 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_multi_sort_push.yaml @@ -0,0 +1,19 @@ +root: + name: ProjectOperator + description: + fields: "[account_number, firstname, address, balance, gender]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\"\ + :{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\"\ + ,\"gender\"],\"excludes\":[]},\"sort\":[{\"account_number\":{\"order\":\"\ + asc\",\"missing\":\"_first\"}},{\"firstname.keyword\":{\"order\":\"asc\"\ + ,\"missing\":\"_first\"}},{\"address\":{\"order\":\"asc\",\"missing\":\"\ + _first\"}},{\"balance\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"\ + balance\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"gender.keyword\"\ + :{\"order\":\"desc\",\"missing\":\"_last\"}},{\"account_number\":{\"order\"\ + :\"asc\",\"missing\":\"_first\"}}]}, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_percentile.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_percentile.json deleted file mode 100644 index c55e236e3c1..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_percentile.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[p50, p90]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"p50\":{\"percentiles\":{\"field\":\"balance\",\"percents\":[50.0],\"keyed\":true,\"tdigest\":{\"compression\":100.0}}},\"p90\":{\"percentiles\":{\"field\":\"balance\",\"percents\":[90.0],\"keyed\":true,\"tdigest\":{\"compression\":100.0}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_percentile.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_percentile.yaml new file mode 100644 index 00000000000..638f009509f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_percentile.yaml @@ -0,0 +1,15 @@ +root: + name: ProjectOperator + description: + fields: "[p50, p90]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\"\ + :{\"p50\":{\"percentiles\":{\"field\":\"balance\",\"percents\":[50.0],\"\ + keyed\":true,\"tdigest\":{\"compression\":100.0}}},\"p90\":{\"percentiles\"\ + :{\"field\":\"balance\",\"percents\":[90.0],\"keyed\":true,\"tdigest\":{\"\ + compression\":100.0}}}}}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_search_basic_text.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_search_basic_text.json deleted file mode 100644 index 5ecf576cd1d..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_search_basic_text.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[spanId, traceId, @timestamp, instrumentationScope, severityText, resource, flags, attributes, droppedAttributesCount, severityNumber, time, body]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_otel_logs, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"ERROR\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"spanId\",\"traceId\",\"@timestamp\",\"instrumentationScope\",\"severityText\",\"resource\",\"flags\",\"attributes\",\"droppedAttributesCount\",\"severityNumber\",\"time\",\"body\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_search_basic_text.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_search_basic_text.yaml new file mode 100644 index 00000000000..a025fde6f98 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_search_basic_text.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[spanId, traceId, @timestamp, instrumentationScope, severityText, resource,\ + \ flags, attributes, droppedAttributesCount, severityNumber, time, body]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_otel_logs,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\"\ + :{\"query_string\":{\"query\":\"ERROR\",\"fields\":[],\"type\":\"best_fields\"\ + ,\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\"\ + :true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\"\ + :50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"spanId\",\"traceId\",\"@timestamp\",\"instrumentationScope\",\"severityText\"\ + ,\"resource\",\"flags\",\"attributes\",\"droppedAttributesCount\",\"severityNumber\"\ + ,\"time\",\"body\"],\"excludes\":[]}}, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_search_numeric_comparison.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_search_numeric_comparison.json deleted file mode 100644 index d6e10550c31..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_search_numeric_comparison.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[spanId, traceId, @timestamp, instrumentationScope, severityText, resource, flags, attributes, droppedAttributesCount, severityNumber, time, body]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_otel_logs, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"severityNumber:>15\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"spanId\",\"traceId\",\"@timestamp\",\"instrumentationScope\",\"severityText\",\"resource\",\"flags\",\"attributes\",\"droppedAttributesCount\",\"severityNumber\",\"time\",\"body\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_search_numeric_comparison.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_search_numeric_comparison.yaml new file mode 100644 index 00000000000..b6170c97437 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_search_numeric_comparison.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[spanId, traceId, @timestamp, instrumentationScope, severityText, resource,\ + \ flags, attributes, droppedAttributesCount, severityNumber, time, body]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_otel_logs,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\"\ + :{\"query_string\":{\"query\":\"severityNumber:>15\",\"fields\":[],\"type\"\ + :\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\"\ + :10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\"\ + :0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"spanId\",\"traceId\",\"@timestamp\",\"instrumentationScope\",\"severityText\"\ + ,\"resource\",\"flags\",\"attributes\",\"droppedAttributesCount\",\"severityNumber\"\ + ,\"time\",\"body\"],\"excludes\":[]}}, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_search_wildcard_star.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_search_wildcard_star.json deleted file mode 100644 index 8ba7e887cdf..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_search_wildcard_star.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[spanId, traceId, @timestamp, instrumentationScope, severityText, resource, flags, attributes, droppedAttributesCount, severityNumber, time, body]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_otel_logs, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"severityText:ERR*\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"spanId\",\"traceId\",\"@timestamp\",\"instrumentationScope\",\"severityText\",\"resource\",\"flags\",\"attributes\",\"droppedAttributesCount\",\"severityNumber\",\"time\",\"body\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_search_wildcard_star.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_search_wildcard_star.yaml new file mode 100644 index 00000000000..6b771c24ca6 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_search_wildcard_star.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[spanId, traceId, @timestamp, instrumentationScope, severityText, resource,\ + \ flags, attributes, droppedAttributesCount, severityNumber, time, body]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_otel_logs,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\"\ + :{\"query_string\":{\"query\":\"severityText:ERR*\",\"fields\":[],\"type\"\ + :\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\"\ + :10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\"\ + :0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"spanId\",\"traceId\",\"@timestamp\",\"instrumentationScope\",\"severityText\"\ + ,\"resource\",\"flags\",\"attributes\",\"droppedAttributesCount\",\"severityNumber\"\ + ,\"time\",\"body\"],\"excludes\":[]}}, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_single_field_relevance_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_single_field_relevance_push.json deleted file mode 100644 index de132c622f8..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_single_field_relevance_push.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]" - }, - "children": [{ - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"match\":{\"email\":{\"query\":\"*@gmail.com\",\"operator\":\"OR\",\"prefix_length\":0,\"max_expansions\":50,\"fuzzy_transpositions\":true,\"lenient\":false,\"zero_terms_query\":\"NONE\",\"auto_generate_synonyms_phrase_query\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - }] - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_single_field_relevance_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_single_field_relevance_push.yaml new file mode 100644 index 00000000000..66ece618c7a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_single_field_relevance_push.yaml @@ -0,0 +1,18 @@ +root: + name: ProjectOperator + description: + fields: "[account_number, firstname, address, balance, gender, city, employer,\ + \ state, age, email, lastname]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\"\ + :{\"match\":{\"email\":{\"query\":\"*@gmail.com\",\"operator\":\"OR\",\"\ + prefix_length\":0,\"max_expansions\":50,\"fuzzy_transpositions\":true,\"\ + lenient\":false,\"zero_terms_query\":\"NONE\",\"auto_generate_synonyms_phrase_query\"\ + :true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"account_number\",\"\ + firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"\ + state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_desc_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_desc_push.json deleted file mode 100644 index 408f860b2d2..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_desc_push.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[age, firstname]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\",\"firstname\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"firstname.keyword\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_desc_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_desc_push.yaml new file mode 100644 index 00000000000..f98514ac90d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_desc_push.yaml @@ -0,0 +1,14 @@ +root: + name: ProjectOperator + description: + fields: "[age, firstname]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\"\ + :{\"includes\":[\"age\",\"firstname\"],\"excludes\":[]},\"sort\":[{\"age\"\ + :{\"order\":\"desc\",\"missing\":\"_last\"}},{\"firstname.keyword\":{\"\ + order\":\"asc\",\"missing\":\"_first\"}}]}, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_push.json deleted file mode 100644 index c95519e037d..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_push.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[age]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=s9y3QQEhb3BlbnNlYXJjaC1zcWxfdGVzdF9pbmRleF9hY2NvdW50FkxkUmtkc296Ul9hY0ZyQWdnSklXTlEAFnJ6SVBMUjhrU3lTMHNMQXA1ckRnVWcAAAAAAAAAAAMWNEd5alM2OFhUZXVPYW1mSm1Gc1ZVUQEWTGRSa2Rzb3pSX2FjRnJBZ2dKSVdOUQAA, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_push.yaml new file mode 100644 index 00000000000..1cefe6e35fe --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_push.yaml @@ -0,0 +1,15 @@ +root: + name: ProjectOperator + description: + fields: "[age]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\"\ + :{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"\ + include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"\ + ],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"\ + _first\"}}]}, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_rename_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_rename_push.json deleted file mode 100644 index 27b7b6b27b8..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_rename_push.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[alias]" - }, - "children": [ - { - "name": "SortOperator", - "description": { - "sortList": { - "alias": { - "sortOrder": "ASC", - "nullOrder": "NULL_FIRST" - } - } - }, - "children": [ - { - "name": "OpenSearchEvalOperator", - "description": { - "expressions": { - "alias": "name" - } - }, - "children": [ - { - "name": "RenameOperator", - "description": { - "mapping": { - "firstname": "name" - } - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true, searchDone=false, pitId=s9y3QQEhb3BlbnNlYXJjaC1zcWxfdGVzdF9pbmRleF9hY2NvdW50Fndla1VpMi1kVHh5Qi1lYnhPQnlSbXcAFkU4Qm9UVURIUWI2a3pjNkhmQkxvc2cAAAAAAAAAAAsWcnUyVHZfNk1SeC1DMFNFdGtPcDN4QQEWd2VrVWkyLWRUeHlCLWVieE9CeVJtdwAA, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } - ] - } - ] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_rename_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_rename_push.yaml new file mode 100644 index 00000000000..2070e08a336 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_rename_push.yaml @@ -0,0 +1,29 @@ +root: + name: ProjectOperator + description: + fields: "[alias]" + children: + - name: SortOperator + description: + sortList: + alias: + sortOrder: ASC + nullOrder: NULL_FIRST + children: + - name: OpenSearchEvalOperator + description: + expressions: + alias: name + children: + - name: RenameOperator + description: + mapping: + firstname: name + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"\ + }, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_then_agg_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_then_agg_push.json deleted file mode 100644 index 42dbbf3483a..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_then_agg_push.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[avg(balance), state]" - }, - "children": [{ - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(balance)\":{\"avg\":{\"field\":\"balance\"}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - }] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_then_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_then_agg_push.yaml new file mode 100644 index 00000000000..ffd35a74b45 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_then_agg_push.yaml @@ -0,0 +1,15 @@ +root: + name: ProjectOperator + description: + fields: "[avg(balance), state]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"sort\":[],\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\"\ + :[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\"\ + :true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ + :{\"avg(balance)\":{\"avg\":{\"field\":\"balance\"}}}}}}, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_span.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_span.json deleted file mode 100644 index d311d8f9388..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_span.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[count(), span(age,10)]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(age,10)\":{\"histogram\":{\"field\":\"age\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\",\"interval\":10.0}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_span.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_span.yaml new file mode 100644 index 00000000000..7246fd36754 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_span.yaml @@ -0,0 +1,15 @@ +root: + name: ProjectOperator + description: + fields: "[count(), span(age,10)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank,\ + \ sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\"\ + :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(age,10)\"\ + :{\"histogram\":{\"field\":\"age\",\"missing_bucket\":true,\"missing_order\"\ + :\"first\",\"order\":\"asc\",\"interval\":10.0}}}]},\"aggregations\":{\"\ + count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_span_non_bucket_nullable.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_span_non_bucket_nullable.json deleted file mode 100644 index 35c137ab172..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_span_non_bucket_nullable.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[count(), span(age,10)]" - }, - "children": [{ - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(age,10)\":{\"histogram\":{\"field\":\"age\",\"missing_bucket\":false,\"order\":\"asc\",\"interval\":10.0}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - }] - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_span_non_bucket_nullable.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_span_non_bucket_nullable.yaml new file mode 100644 index 00000000000..a0f9965f7ed --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_span_non_bucket_nullable.yaml @@ -0,0 +1,15 @@ +root: + name: ProjectOperator + description: + fields: "[count(), span(age,10)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank,\ + \ sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\"\ + :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(age,10)\"\ + :{\"histogram\":{\"field\":\"age\",\"missing_bucket\":false,\"order\":\"\ + asc\",\"interval\":10.0}}}]},\"aggregations\":{\"count()\":{\"value_count\"\ + :{\"field\":\"_index\"}}}}}}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_week_argument_coercion.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_week_argument_coercion.json deleted file mode 100644 index bc88f5eedfa..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_week_argument_coercion.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[w]" - }, - "children": [ - { - "name": "OpenSearchEvalOperator", - "description": { - "expressions": { - "w": "week(\"2024-12-10\")" - } - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } - ] - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_week_argument_coercion.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_week_argument_coercion.yaml new file mode 100644 index 00000000000..2b4b314e5c3 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_week_argument_coercion.yaml @@ -0,0 +1,16 @@ +root: + name: ProjectOperator + description: + fields: "[w]" + children: + - name: OpenSearchEvalOperator + description: + expressions: + w: week("2024-12-10") + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4356.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4356.yml index e4a99a268b2..3513f960c59 100644 --- a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4356.yml +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4356.yml @@ -83,8 +83,9 @@ teardown: ppl: body: query: source=log00001 | rex field=v 'value=(?[\\w\\d\\.]*)' | eval m=digits * 10 | eval d=digits/10 | sort a | fields m, d - - match: {"schema": [{"name": "m", "type": "double"}, {"name": "d", "type": "double"}]} - - match: {"datarows": [[10.0, 0.1], [15.0, 0.15], [null, null], [null, null]]} + - match: {"schema": [{"name": "m", "type": "int"}, {"name": "d", "type": "double"}]} + # As the second operand is integer, the first string operand is cast to integer as well. "1.5" is not a valid string, so the second row becomes [null, 0.15] + - match: {"datarows": [[10, 0.1], [null, 0.15], [null, null], [null, null]]} - do: allowed_warnings: @@ -105,8 +106,9 @@ teardown: ppl: body: query: source=log00002 | eval m="5" - 10 | eval r=round("1.5", 1) | eval f=floor("5.2") | eval c=ceil("5.2") | fields m, r, f, c - - match: { "schema": [ { "name": "m", "type": "double" }, { "name": "r", "type": "double" }, { "name": "f", "type": "double" }, { "name": "c", "type": "double" }] } - - match: { "datarows": [ [ -5.0, 1.5, 5.0, 6.0] ] } + # "5" in `"5" - 10` is cast to integer as the second argument is integer. + - match: { "schema": [ { "name": "m", "type": "int" }, { "name": "r", "type": "double" }, { "name": "f", "type": "double" }, { "name": "c", "type": "double" }] } + - match: { "datarows": [ [ -5, 1.5, 5.0, 6.0] ] } --- "Extracted value participate in comparison operator": @@ -123,7 +125,8 @@ teardown: body: query: source=log00001 | rex field=v 'value=(?[\\w\\d\\.]*)' | eval i=digits==vint, d=digits==vdouble, b=digits==vboolean| fields i, d, b - match: {"schema": [{"name": "i", "type": "boolean"}, {"name": "d", "type": "boolean"}, {"name": "b", "type": "boolean"}]} - - match: {"datarows": [[true,true,null], [false,true,null], [null, null, true], [null, null, null]]} + # Same as the above arithmetic operator case, "1.5" fails to be cased to integer, so the second row becomes [null,true,null] + - match: {"datarows": [[true,true,null], [null,true,null], [null, null, true], [null, null, null]]} - do: allowed_warnings: diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4740.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4740.yml index 5fdb4198abe..a39c6cc842a 100644 --- a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4740.yml +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4740.yml @@ -29,23 +29,25 @@ setup: - '{"index":{}}' - '{"@timestamp":"2024-01-01T00:20:00.000Z","age":"45","balance":"5000.0","name":"Eve"}' ---- -"bin with numeric field using WIDTH_BUCKET - issue 4740": - - skip: - features: - - headers - - allowed_warnings - - do: - allowed_warnings: - - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' - headers: - Content-Type: 'application/json' - ppl: - body: - query: source=test_binning_4740 | bin age bins=3 | stats count() by age | sort age - - - match: { "schema": [ { "name": "count()", "type": "bigint" }, { "name": "age", "type": "string" } ] } - - match: { "datarows": [ [ 1, "20-30" ], [ 2, "30-40" ], [ 2, "40-50" ] ] } +# TODO: Enable after fixing https://github.com/opensearch-project/sql/issues/4973 +# problem: string minus string in the generated plan +#--- +#"bin with numeric field using WIDTH_BUCKET - issue 4740": +# - skip: +# features: +# - headers +# - allowed_warnings +# - do: +# allowed_warnings: +# - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' +# headers: +# Content-Type: 'application/json' +# ppl: +# body: +# query: source=test_binning_4740 | bin age bins=3 | stats count() by age | sort age +# +# - match: { "schema": [ { "name": "count()", "type": "bigint" }, { "name": "age", "type": "string" } ] } +# - match: { "datarows": [ [ 1, "20-30" ], [ 2, "30-40" ], [ 2, "40-50" ] ] } --- "bin with numeric span using SPAN_BUCKET - issue 4740": @@ -65,56 +67,62 @@ setup: - match: { "schema": [ { "name": "count()", "type": "bigint" }, { "name": "age", "type": "string" } ] } - match: { "datarows": [ [ 1, "20-30" ], [ 2, "30-40" ], [ 2, "40-50" ] ] } ---- -"bin with minspan using MINSPAN_BUCKET - issue 4740": - - skip: - features: - - headers - - allowed_warnings - - do: - allowed_warnings: - - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' - headers: - Content-Type: 'application/json' - ppl: - body: - query: source=test_binning_4740 | bin balance minspan=1000 | stats count() by balance | sort balance - - - match: { "schema": [ { "name": "count()", "type": "bigint" }, { "name": "balance", "type": "string" } ] } - - match: { "datarows": [ [ 1, "1000-2000" ], [ 1, "2000-3000" ], [ 1, "3000-4000" ], [ 1, "4000-5000" ], [ 1, "5000-6000" ] ] } +# TODO: Enable after fixing https://github.com/opensearch-project/sql/issues/4973 +# problem: string minus string in the generated plan +#--- +#"bin with minspan using MINSPAN_BUCKET - issue 4740": +# - skip: +# features: +# - headers +# - allowed_warnings +# - do: +# allowed_warnings: +# - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' +# headers: +# Content-Type: 'application/json' +# ppl: +# body: +# query: source=test_binning_4740 | bin balance minspan=1000 | stats count() by balance | sort balance +# +# - match: { "schema": [ { "name": "count()", "type": "bigint" }, { "name": "balance", "type": "string" } ] } +# - match: { "datarows": [ [ 1, "1000-2000" ], [ 1, "2000-3000" ], [ 1, "3000-4000" ], [ 1, "4000-5000" ], [ 1, "5000-6000" ] ] } ---- -"bin with start and end using RANGE_BUCKET - issue 4740": - - skip: - features: - - headers - - allowed_warnings - - do: - allowed_warnings: - - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' - headers: - Content-Type: 'application/json' - ppl: - body: - query: source=test_binning_4740 | bin age start=20 end=50 | stats count() by age | sort age - - - match: { "schema": [ { "name": "count()", "type": "bigint" }, { "name": "age", "type": "string" } ] } - - match: { "datarows": [ [ 1, "20-30" ], [ 2, "30-40" ], [ 2, "40-50" ] ] } - ---- -"bin with default binning (no parameters) on string field - issue 4740": - - skip: - features: - - headers - - allowed_warnings - - do: - allowed_warnings: - - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' - headers: - Content-Type: 'application/json' - ppl: - body: - query: source=test_binning_4740 | bin balance | stats count() by balance | sort balance +# TODO: Enable after fixing https://github.com/opensearch-project/sql/issues/4973 +# problem: cast string to number in the generated code +#--- +#"bin with start and end using RANGE_BUCKET - issue 4740": +# - skip: +# features: +# - headers +# - allowed_warnings +# - do: +# allowed_warnings: +# - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' +# headers: +# Content-Type: 'application/json' +# ppl: +# body: +# query: source=test_binning_4740 | bin age start=20 end=50 | stats count() by age | sort age +# +# - match: { "schema": [ { "name": "count()", "type": "bigint" }, { "name": "age", "type": "string" } ] } +# - match: { "datarows": [ [ 1, "20-30" ], [ 2, "30-40" ], [ 2, "40-50" ] ] } - - match: { "schema": [ { "name": "count()", "type": "bigint" }, { "name": "balance", "type": "string" } ] } - - match: { "datarows": [ [ 1, "1000.0-2000.0" ], [ 1, "2000.0-3000.0" ], [ 1, "3000.0-4000.0" ], [ 1, "4000.0-5000.0" ], [ 1, "5000.0-6000.0" ] ] } +# TODO: Enable after fixing https://github.com/opensearch-project/sql/issues/4973 +# problem: string minus string in the generated plan +#--- +#"bin with default binning (no parameters) on string field - issue 4740": +# - skip: +# features: +# - headers +# - allowed_warnings +# - do: +# allowed_warnings: +# - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' +# headers: +# Content-Type: 'application/json' +# ppl: +# body: +# query: source=test_binning_4740 | bin balance | stats count() by balance | sort balance +# +# - match: { "schema": [ { "name": "count()", "type": "bigint" }, { "name": "balance", "type": "string" } ] } +# - match: { "datarows": [ [ 1, "1000.0-2000.0" ], [ 1, "2000.0-3000.0" ], [ 1, "3000.0-4000.0" ], [ 1, "4000.0-5000.0" ], [ 1, "5000.0-6000.0" ] ] } diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4866.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4866.yml index e2ae4c86803..8d84a0b39ed 100644 --- a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4866.yml +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4866.yml @@ -27,39 +27,39 @@ teardown: transient: plugins.calcite.enabled : false - ---- -"Patterns with specified max_sample_count should return correct result": - - skip: - features: - - headers - - allowed_warnings - - do: - allowed_warnings: - - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' - headers: - Content-Type: 'application/json' - ppl: - body: - query: 'source=hdfs_logs | patterns content method=brain mode=aggregation max_sample_count=2 variable_count_threshold=3' - - match: {"total": 2} - - match: {"schema": [{"name": "patterns_field", "type": "string"}, {"name": "pattern_count", "type": "bigint"}, {"name": "sample_logs", "type": "array"}]} - - match: {"datarows": [ - [ - "PacketResponder failed for blk_<*>", - 2, - [ - "PacketResponder failed for blk_6996194389878584395", - "PacketResponder failed for blk_-1547954353065580372" - ] - ], - [ - "BLOCK* NameSystem.addStoredBlock: blockMap updated: <*IP*> is added to blk_<*> size <*>", - 2, - [ - "BLOCK* NameSystem.addStoredBlock: blockMap updated: 10.251.31.85:50010 is added to blk_-7017553867379051457 size 67108864", - "BLOCK* NameSystem.addStoredBlock: blockMap updated: 10.251.107.19:50010 is added to blk_-3249711809227781266 size 67108864" - ] - ] - ]} +# TODO: enable after fixing https://github.com/opensearch-project/sql/issues/4968 +#--- +#"Patterns with specified max_sample_count should return correct result": +# - skip: +# features: +# - headers +# - allowed_warnings +# - do: +# allowed_warnings: +# - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' +# headers: +# Content-Type: 'application/json' +# ppl: +# body: +# query: 'source=hdfs_logs | patterns content method=brain mode=aggregation max_sample_count=2 variable_count_threshold=3' +# - match: {"total": 2} +# - match: {"schema": [{"name": "patterns_field", "type": "string"}, {"name": "pattern_count", "type": "bigint"}, {"name": "sample_logs", "type": "array"}]} +# - match: {"datarows": [ +# [ +# "PacketResponder failed for blk_<*>", +# 2, +# [ +# "PacketResponder failed for blk_6996194389878584395", +# "PacketResponder failed for blk_-1547954353065580372" +# ] +# ], +# [ +# "BLOCK* NameSystem.addStoredBlock: blockMap updated: <*IP*> is added to blk_<*> size <*>", +# 2, +# [ +# "BLOCK* NameSystem.addStoredBlock: blockMap updated: 10.251.31.85:50010 is added to blk_-7017553867379051457 size 67108864", +# "BLOCK* NameSystem.addStoredBlock: blockMap updated: 10.251.107.19:50010 is added to blk_-3249711809227781266 size 67108864" +# ] +# ] +# ]} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java index 71f0c8667ff..38e0ec8b74a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java @@ -27,9 +27,14 @@ import org.apache.calcite.sql.SqlExplainLevel; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlOperatorTable; +import org.apache.calcite.sql.fun.SqlLibrary; +import org.apache.calcite.sql.fun.SqlLibraryOperatorTableFactory; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.util.ListSqlOperatorTable; +import org.apache.calcite.sql.util.SqlOperatorTables; import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction; import org.apache.calcite.sql.validate.SqlUserDefinedFunction; import org.apache.logging.log4j.LogManager; @@ -50,7 +55,9 @@ import org.opensearch.sql.executor.Explain; import org.opensearch.sql.executor.pagination.PlanSerializer; import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; import org.opensearch.sql.expression.function.PPLFuncImpTable; +import org.opensearch.sql.expression.function.UDFOperandMetadata; import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.executor.protector.ExecutionProtector; import org.opensearch.sql.opensearch.functions.DistinctCountApproxAggFunction; @@ -69,6 +76,10 @@ public class OpenSearchExecutionEngine implements ExecutionEngine { private final ExecutionProtector executionProtector; private final PlanSerializer planSerializer; + static { + CalcitePlanContext.setOperatorTableProvider(OperatorTable::getChainedOperatorTable); + } + public OpenSearchExecutionEngine( OpenSearchClient client, ExecutionProtector executionProtector, @@ -284,7 +295,7 @@ private void registerOpenSearchFunctions() { DistinctCountApproxAggFunction.class, BuiltinFunctionName.DISTINCT_COUNT_APPROX.name(), ReturnTypes.BIGINT_FORCE_NULLABLE, - null); + UDFOperandMetadata.wrap(OperandTypes.ANY)); PPLFuncImpTable.INSTANCE.registerExternalAggOperator( BuiltinFunctionName.DISTINCT_COUNT_APPROX, approxDistinctCountFunction); OperatorTable.addOperator( @@ -314,5 +325,20 @@ private ListSqlOperatorTable init() { public static synchronized void addOperator(String name, SqlOperator operator) { operators.put(name, operator); } + + /** + * Chain PPL's operator table with selected Calcite's built-in library operator tables. + * + *

This method should be called AFTER operators are initialized + */ + public static SqlOperatorTable getChainedOperatorTable() { + return SqlOperatorTables.chain( + PPLBuiltinOperators.instance(), + SqlStdOperatorTable.instance(), + OperatorTable.instance(), + // Add a list of necessary SqlLibrary if needed + SqlLibraryOperatorTableFactory.INSTANCE.getOperatorTable( + SqlLibrary.MYSQL, SqlLibrary.BIG_QUERY, SqlLibrary.POSTGRESQL, SqlLibrary.HIVE)); + } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java index 83b1915f6b5..157c46fdb0f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java @@ -16,15 +16,20 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.geospatial.action.IpEnrichmentActionClient; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; import org.opensearch.transport.client.node.NodeClient; @@ -58,10 +63,32 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return UDFOperandMetadata.wrapUDT( - List.of( - List.of(ExprCoreType.STRING, ExprCoreType.IP), - List.of(ExprCoreType.STRING, ExprCoreType.IP, ExprCoreType.STRING))); + return UDFOperandMetadata.wrap( + new SqlOperandTypeChecker() { + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + if (!getOperandCountRange().isValidCount(callBinding.getOperandCount())) { + return false; + } + boolean valid = + OpenSearchTypeUtil.isCharacter(callBinding.getOperandType(0)) + && OpenSearchTypeUtil.isIp(callBinding.getOperandType(1), true); + if (callBinding.getOperandCount() == 3) { + valid = valid && OpenSearchTypeUtil.isCharacter(callBinding.getOperandType(2)); + } + return valid; + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.between(2, 3); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return "GEOIP(, ), GEOIP(, , )"; + } + }); } public static class GeoIPImplementor implements NotNullImplementor { @@ -101,10 +128,15 @@ public Expression implement( ExprIpValue ipAddress, String commaSeparatedOptions, NodeClient nodeClient) { + return fetchIpEnrichment(dataSource, ipAddress.toString(), commaSeparatedOptions, nodeClient); + } + + public static Map fetchIpEnrichment( + String dataSource, String ipAddress, String commaSeparatedOptions, NodeClient nodeClient) { String unquotedOptions = StringUtils.unquoteText(commaSeparatedOptions); final Set options = Arrays.stream(unquotedOptions.split(",")).map(String::trim).collect(Collectors.toSet()); - return fetchIpEnrichment(dataSource, ipAddress.toString(), options, nodeClient); + return fetchIpEnrichment(dataSource, ipAddress, options, nodeClient); } private static Map fetchIpEnrichment( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/AggregateIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/AggregateIndexScanRule.java index 4154a10ce51..65e3a61033e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/AggregateIndexScanRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/AggregateIndexScanRule.java @@ -24,8 +24,8 @@ import org.apache.calcite.tools.RelBuilder; import org.immutables.value.Value; import org.opensearch.sql.calcite.plan.OpenSearchRuleConfig; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.calcite.utils.PlanUtils; -import org.opensearch.sql.expression.function.udf.binning.WidthBucketFunction; import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalIndexScan; @@ -243,8 +243,7 @@ static boolean containsWidthBucketFuncOnDate(LogicalProject project) { expr -> expr instanceof RexCall rexCall && rexCall.getOperator().equals(WIDTH_BUCKET) - && WidthBucketFunction.dateRelatedType( - rexCall.getOperands().getFirst().getType())); + && OpenSearchTypeUtil.isDatetime(rexCall.getOperands().getFirst().getType())); } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java index d77dee3e297..d7d9b79aac8 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java @@ -500,7 +500,7 @@ RexNode toRex(RelInput relInput, @PolyNull Object o) { } final RelDataType type = toType(typeFactory, get(map, "type")); if (literal instanceof Map && ((Map) literal).containsKey("rangeSet")) { - Sarg sarg = sargFromJson((Map) literal); + Sarg sarg = sargFromJson((Map) literal, type); return rexBuilder.makeSearchArgumentLiteral(sarg, type); } if (type.getSqlTypeName() == SqlTypeName.SYMBOL) { @@ -515,7 +515,7 @@ RexNode toRex(RelInput relInput, @PolyNull Object o) { return rexBuilder.makeNullLiteral(type); } final RelDataType type = toType(typeFactory, get(map, "type")); - Sarg sarg = sargFromJson((Map) sargObject); + Sarg sarg = sargFromJson((Map) sargObject, type); return rexBuilder.makeSearchArgumentLiteral(sarg, type); } if (map.containsKey("dynamicParam")) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializer.java index 604012c4870..ba987959a37 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializer.java @@ -20,14 +20,8 @@ import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.rel.externalize.RelJson; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.sql.SqlOperatorTable; -import org.apache.calcite.sql.fun.SqlLibrary; -import org.apache.calcite.sql.fun.SqlLibraryOperatorTableFactory; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.util.SqlOperatorTables; import org.apache.calcite.util.JsonBuilder; import org.opensearch.sql.calcite.CalcitePlanContext; -import org.opensearch.sql.expression.function.PPLBuiltinOperators; import org.opensearch.sql.opensearch.executor.OpenSearchExecutionEngine.OperatorTable; /** @@ -45,7 +39,6 @@ public class RelJsonSerializer { private static final ObjectMapper mapper = new ObjectMapper(); private static final TypeReference> TYPE_REF = new TypeReference<>() {}; - private static volatile SqlOperatorTable pplSqlOperatorTable; static { mapper.configure(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, true); @@ -55,27 +48,6 @@ public RelJsonSerializer(RelOptCluster cluster) { this.cluster = cluster; } - private static SqlOperatorTable getPplSqlOperatorTable() { - if (pplSqlOperatorTable == null) { - synchronized (RelJsonSerializer.class) { - if (pplSqlOperatorTable == null) { - pplSqlOperatorTable = - SqlOperatorTables.chain( - PPLBuiltinOperators.instance(), - SqlStdOperatorTable.instance(), - OperatorTable.instance(), - // Add a list of necessary SqlLibrary if needed - SqlLibraryOperatorTableFactory.INSTANCE.getOperatorTable( - SqlLibrary.MYSQL, - SqlLibrary.BIG_QUERY, - SqlLibrary.SPARK, - SqlLibrary.POSTGRESQL)); - } - } - } - return pplSqlOperatorTable; - } - /** * Serializes Calcite expressions and field types into a map object string. * @@ -127,7 +99,7 @@ public RexNode deserialize(String struct) { relJson = relJson .withInputTranslator(ExtendedRelJson::translateInput) - .withOperatorTable(getPplSqlOperatorTable()); + .withOperatorTable(OperatorTable.getChainedOperatorTable()); Map exprMap = mapper.readValue(exprStr, TYPE_REF); return relJson.toRex(cluster, exprMap); } catch (Exception e) { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java index 0c1aef9ddfa..a9e84d8adb5 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java @@ -831,7 +831,8 @@ void isNullOr_ScriptPushDown() throws ExpressionNotAnalyzableException { .add("a", builder.getTypeFactory().createSqlType(SqlTypeName.BIGINT)) .add("b", builder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR)) .build(); - // PPL IS_EMPTY is translated to OR(IS_NULL(arg), IS_EMPTY(arg)) + // PPL IS_EMPTY is translated to OR(IS_NULL(arg), EQUALS("")) (not IS_EMPTY because IS_EMPTY is + // only for collections) RexNode call = PPLFuncImpTable.INSTANCE.resolve(builder, BuiltinFunctionName.IS_EMPTY, field2); Hook.CURRENT_TIME.addThread((Consumer>) h -> h.set(0L)); QueryExpression expression = @@ -839,7 +840,33 @@ void isNullOr_ScriptPushDown() throws ExpressionNotAnalyzableException { assert (expression .builder() .toString() - .contains("\"lang\" : \"opensearch_compounded_script\"")); + .contains( + """ + "should" : [ + { + "bool" : { + "must_not" : [ + { + "exists" : { + "field" : "b", + "boost" : 1.0 + } + } + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } + }, + { + "term" : { + "b.keyword" : { + "value" : "", + "boost" : 1.0 + } + } + } + ]\ + """)); } @Test diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java index ab07cd9b5c1..b5a7b451275 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java @@ -45,6 +45,8 @@ import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.calcite.CalciteRelNodeVisitor; import org.opensearch.sql.calcite.SysLimit; +import org.opensearch.sql.calcite.validate.OpenSearchSparkSqlDialect; +import org.opensearch.sql.calcite.validate.converters.PplRelToSqlNodeConverter; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.exception.ExpressionEvaluationException; @@ -64,7 +66,7 @@ public CalcitePPLAbstractTest(CalciteAssert.SchemaSpec... schemaSpecs) { this.config = config(schemaSpecs); this.dataSourceService = mock(DataSourceService.class); this.planTransformer = new CalciteRelNodeVisitor(dataSourceService); - this.converter = new RelToSqlConverter(OpenSearchSparkSqlDialect.DEFAULT); + this.converter = new PplRelToSqlNodeConverter(OpenSearchSparkSqlDialect.DEFAULT); this.settings = mock(Settings.class); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregateFunctionTypeTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregateFunctionTypeTest.java index b363be9bee1..ef3bbf70e96 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregateFunctionTypeTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregateFunctionTypeTest.java @@ -5,6 +5,7 @@ package org.opensearch.sql.ppl.calcite; +import org.apache.calcite.rel.RelNode; import org.apache.calcite.test.CalciteAssert; import org.junit.Test; @@ -16,166 +17,273 @@ public CalcitePPLAggregateFunctionTypeTest() { @Test public void testAvgWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | stats avg(HIREDATE) as avg_name", - "Aggregation function AVG expects field type {[INTEGER]|[DOUBLE]}, but got [DATE]"); + String ppl = "source=EMP | stats avg(HIREDATE) as avg_name"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], avg_name=[AVG($0)])\n" + + " LogicalProject(HIREDATE=[$4])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "avg_name=1982-05-02\n"); + verifyPPLToSparkSQL(root, "SELECT AVG(`HIREDATE`) `avg_name`\nFROM `scott`.`EMP`"); } @Test public void testVarsampWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | stats var_samp(HIREDATE) as varsamp_name", - "Aggregation function VARSAMP expects field type {[INTEGER]|[DOUBLE]}, but got [DATE]"); + String ppl = "source=EMP | stats var_samp(HIREDATE) as varsamp_name"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], varsamp_name=[VAR_SAMP($0)])\n" + + " LogicalProject(HIREDATE=[$4])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "varsamp_name=8277-10-13\n"); + verifyPPLToSparkSQL(root, "SELECT VAR_SAMP(`HIREDATE`) `varsamp_name`\nFROM `scott`.`EMP`"); } @Test public void testVarpopWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | stats var_pop(HIREDATE) as varpop_name", - "Aggregation function VARPOP expects field type {[INTEGER]|[DOUBLE]}, but got [DATE]"); + String ppl = "source=EMP | stats var_pop(HIREDATE) as varpop_name"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], varpop_name=[VAR_POP($0)])\n" + + " LogicalProject(HIREDATE=[$4])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "varpop_name=3541-07-07\n"); + verifyPPLToSparkSQL(root, "SELECT VAR_POP(`HIREDATE`) `varpop_name`\nFROM `scott`.`EMP`"); } @Test public void testStddevSampWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | stats stddev_samp(HIREDATE) as stddev_name", - "Aggregation function STDDEV_SAMP expects field type {[INTEGER]|[DOUBLE]}, but got" - + " [DATE]"); + String ppl = "source=EMP | stats stddev_samp(HIREDATE) as stddev_name"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], stddev_name=[STDDEV_SAMP($0)])\n" + + " LogicalProject(HIREDATE=[$4])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "stddev_name=1983-06-23\n"); + verifyPPLToSparkSQL(root, "SELECT STDDEV_SAMP(`HIREDATE`) `stddev_name`\nFROM `scott`.`EMP`"); } @Test public void testStddevPopWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | stats stddev_pop(HIREDATE) as stddev_name", - "Aggregation function STDDEV_POP expects field type {[INTEGER]|[DOUBLE]}, but got" - + " [DATE]"); + String ppl = "source=EMP | stats stddev_pop(HIREDATE) as stddev_name"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], stddev_name=[STDDEV_POP($0)])\n" + + " LogicalProject(HIREDATE=[$4])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "stddev_name=1982-12-26\n"); + verifyPPLToSparkSQL(root, "SELECT STDDEV_POP(`HIREDATE`) `stddev_name`\nFROM `scott`.`EMP`"); } @Test public void testPercentileApproxWithWrongArgType() { // First argument should be numeric - verifyQueryThrowsException( - "source=EMP | stats percentile_approx(HIREDATE, 50) as percentile", - "Aggregation function PERCENTILE_APPROX expects field type and additional arguments" - + " {[INTEGER,INTEGER]|[INTEGER,DOUBLE]|[DOUBLE,INTEGER]|[DOUBLE,DOUBLE]|[INTEGER,INTEGER,INTEGER]|[INTEGER,INTEGER,DOUBLE]|[INTEGER,DOUBLE,INTEGER]|[INTEGER,DOUBLE,DOUBLE]|[DOUBLE,INTEGER,INTEGER]|[DOUBLE,INTEGER,DOUBLE]|[DOUBLE,DOUBLE,INTEGER]|[DOUBLE,DOUBLE,DOUBLE]}," - + " but got [DATE,INTEGER]"); + // This test verifies logical plan generation, but execution fails with ClassCastException + String ppl = "source=EMP | stats percentile_approx(HIREDATE, 50) as percentile"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], percentile=[percentile_approx($0, $1, $2)])\n" + + " LogicalProject(HIREDATE=[$4], $f1=[50], $f2=[FLAG(DATE)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + // Execution fails with: ClassCastException: Double cannot be cast to Integer } @Test public void testListFunctionWithArrayArgType() { // Test LIST function with array expression (which is not a supported scalar type) - verifyQueryThrowsException( - "source=EMP | stats list(array(ENAME, JOB)) as name_list", - "Aggregation function LIST expects field type" - + " {[BYTE]|[SHORT]|[INTEGER]|[LONG]|[FLOAT]|[DOUBLE]|[STRING]|[BOOLEAN]|[DATE]|[TIME]|[TIMESTAMP]|[IP]|[BINARY]}," - + " but got [ARRAY]"); + String ppl = "source=EMP | stats list(array(ENAME, JOB)) as name_list"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], name_list=[LIST($0)])\n" + + " LogicalProject($f2=[array($1, $2)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult( + root, + "name_list=[[SMITH, CLERK], [ALLEN, SALESMAN], [WARD, SALESMAN], [JONES, MANAGER], [MARTIN," + + " SALESMAN], [BLAKE, MANAGER], [CLARK, MANAGER], [SCOTT, ANALYST], [KING, PRESIDENT]," + + " [TURNER, SALESMAN], [ADAMS, CLERK], [JAMES, CLERK], [FORD, ANALYST], [MILLER," + + " CLERK]]\n"); + verifyPPLToSparkSQL( + root, "SELECT `LIST`(ARRAY(`ENAME`, `JOB`)) `name_list`\nFROM `scott`.`EMP`"); } @Test public void testCountWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats count(EMPNO, DEPTNO)", - "Aggregation function COUNT expects field type and additional arguments {[]|[ANY]}," - + " but got [SHORT,BYTE]"); + // COUNT with extra parameters throws IllegalArgumentException + Exception e = + org.junit.Assert.assertThrows( + IllegalArgumentException.class, + () -> getRelNode("source=EMP | stats count(EMPNO, DEPTNO)")); + verifyErrorMessageContains(e, "Field [DEPTNO] not found"); } @Test public void testAvgWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats avg(EMPNO, DEPTNO)", - "Aggregation function AVG expects field type and additional arguments {[INTEGER]|[DOUBLE]}," - + " but got [SHORT,BYTE]"); + // AVG with extra parameters throws IllegalArgumentException + Exception e = + org.junit.Assert.assertThrows( + IllegalArgumentException.class, + () -> getRelNode("source=EMP | stats avg(EMPNO, DEPTNO)")); + verifyErrorMessageContains(e, "Field [DEPTNO] not found"); } @Test public void testSumWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats sum(EMPNO, DEPTNO)", - "Aggregation function SUM expects field type and additional arguments {[INTEGER]|[DOUBLE]}," - + " but got [SHORT,BYTE]"); + String ppl = "source=EMP | stats sum(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], sum(EMPNO, DEPTNO)=[SUM($0, $1)])\n" + + " LogicalProject(EMPNO=[$0], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "sum(EMPNO, DEPTNO)=108172\n"); + verifyPPLToSparkSQL( + root, "SELECT SUM(`EMPNO`, `DEPTNO`) `sum(EMPNO, DEPTNO)`\nFROM `scott`.`EMP`"); } @Test public void testMinWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats min(EMPNO, DEPTNO)", - "Aggregation function MIN expects field type and additional arguments {[COMPARABLE_TYPE]}," - + " but got [SHORT,BYTE]"); + String ppl = "source=EMP | stats min(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], min(EMPNO, DEPTNO)=[MIN($0, $1)])\n" + + " LogicalProject(EMPNO=[$0], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "min(EMPNO, DEPTNO)=7369\n"); + verifyPPLToSparkSQL( + root, "SELECT MIN(`EMPNO`, `DEPTNO`) `min(EMPNO, DEPTNO)`\nFROM `scott`.`EMP`"); } @Test public void testMaxWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats max(EMPNO, DEPTNO)", - "Aggregation function MAX expects field type and additional arguments {[COMPARABLE_TYPE]}," - + " but got [SHORT,BYTE]"); + String ppl = "source=EMP | stats max(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], max(EMPNO, DEPTNO)=[MAX($0, $1)])\n" + + " LogicalProject(EMPNO=[$0], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "max(EMPNO, DEPTNO)=7934\n"); + verifyPPLToSparkSQL( + root, "SELECT MAX(`EMPNO`, `DEPTNO`) `max(EMPNO, DEPTNO)`\nFROM `scott`.`EMP`"); } @Test public void testVarSampWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats var_samp(EMPNO, DEPTNO)", - "Aggregation function VARSAMP expects field type and additional arguments" - + " {[INTEGER]|[DOUBLE]}, but got [SHORT,BYTE]"); + // This test verifies logical plan generation, but execution fails with AssertionError + String ppl = "source=EMP | stats var_samp(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], var_samp(EMPNO, DEPTNO)=[VAR_SAMP($0, $1)])\n" + + " LogicalProject(EMPNO=[$0], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + // Execution fails with: AssertionError [0, 1] in AggregateReduceFunctionsRule } @Test public void testVarPopWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats var_pop(EMPNO, DEPTNO)", - "Aggregation function VARPOP expects field type and additional arguments" - + " {[INTEGER]|[DOUBLE]}, but got [SHORT,BYTE]"); + // This test verifies logical plan generation, but execution fails with AssertionError + String ppl = "source=EMP | stats var_pop(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], var_pop(EMPNO, DEPTNO)=[VAR_POP($0, $1)])\n" + + " LogicalProject(EMPNO=[$0], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + // Execution fails with: AssertionError [0, 1] in AggregateReduceFunctionsRule } @Test public void testStddevSampWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats stddev_samp(EMPNO, DEPTNO)", - "Aggregation function STDDEV_SAMP expects field type and additional arguments" - + " {[INTEGER]|[DOUBLE]}, but got [SHORT,BYTE]"); + // This test verifies logical plan generation, but execution fails with AssertionError + String ppl = "source=EMP | stats stddev_samp(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], stddev_samp(EMPNO, DEPTNO)=[STDDEV_SAMP($0, $1)])\n" + + " LogicalProject(EMPNO=[$0], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + // Execution fails with: AssertionError [0, 1] in AggregateReduceFunctionsRule } @Test public void testStddevPopWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats stddev_pop(EMPNO, DEPTNO)", - "Aggregation function STDDEV_POP expects field type and additional arguments" - + " {[INTEGER]|[DOUBLE]}, but got [SHORT,BYTE]"); + // This test verifies logical plan generation, but execution fails with AssertionError + String ppl = "source=EMP | stats stddev_pop(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], stddev_pop(EMPNO, DEPTNO)=[STDDEV_POP($0, $1)])\n" + + " LogicalProject(EMPNO=[$0], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + // Execution fails with: AssertionError [0, 1] in AggregateReduceFunctionsRule } @Test public void testPercentileWithMissingParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats percentile(EMPNO)", - "Aggregation function PERCENTILE_APPROX expects field type" - + " {[INTEGER,INTEGER]|[INTEGER,DOUBLE]|[DOUBLE,INTEGER]|[DOUBLE,DOUBLE]|[INTEGER,INTEGER,INTEGER]|[INTEGER,INTEGER,DOUBLE]|[INTEGER,DOUBLE,INTEGER]|[INTEGER,DOUBLE,DOUBLE]|[DOUBLE,INTEGER,INTEGER]|[DOUBLE,INTEGER,DOUBLE]|[DOUBLE,DOUBLE,INTEGER]|[DOUBLE,DOUBLE,DOUBLE]}," - + " but got [SHORT]"); + // This test verifies logical plan generation, but execution fails with ClassCastException + String ppl = "source=EMP | stats percentile(EMPNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], percentile(EMPNO)=[percentile_approx($0, $1)])\n" + + " LogicalProject(EMPNO=[$0], $f1=[FLAG(SMALLINT)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + // Execution fails with: ClassCastException: SqlTypeName cannot be cast to Number } @Test public void testPercentileWithInvalidParameterTypesThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats percentile(EMPNO, 50, HIREDATE)", - "Aggregation function PERCENTILE_APPROX expects field type and additional arguments" - + " {[INTEGER,INTEGER]|[INTEGER,DOUBLE]|[DOUBLE,INTEGER]|[DOUBLE,DOUBLE]|[INTEGER,INTEGER,INTEGER]|[INTEGER,INTEGER,DOUBLE]|[INTEGER,DOUBLE,INTEGER]|[INTEGER,DOUBLE,DOUBLE]|[DOUBLE,INTEGER,INTEGER]|[DOUBLE,INTEGER,DOUBLE]|[DOUBLE,DOUBLE,INTEGER]|[DOUBLE,DOUBLE,DOUBLE]}," - + " but got [SHORT,INTEGER,DATE]"); + // This test verifies logical plan generation, but execution fails with ClassCastException + String ppl = "source=EMP | stats percentile(EMPNO, 50, HIREDATE)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], percentile(EMPNO, 50, HIREDATE)=[percentile_approx($0, $2," + + " $1, $3)])\n" + + " LogicalProject(EMPNO=[$0], HIREDATE=[$4], $f2=[50], $f3=[FLAG(SMALLINT)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + // Execution fails with: ClassCastException: Double cannot be cast to Short } @Test public void testEarliestWithTooManyParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats earliest(ENAME, HIREDATE, JOB)", - "Aggregation function EARLIEST expects field type and additional arguments" - + " {[ANY]|[ANY,ANY]}, but got" - + " [STRING,DATE,STRING]"); + String ppl = "source=EMP | stats earliest(ENAME, HIREDATE, JOB)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], earliest(ENAME, HIREDATE, JOB)=[ARG_MIN($0, $1, $2)])\n" + + " LogicalProject(ENAME=[$1], HIREDATE=[$4], JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "earliest(ENAME, HIREDATE, JOB)=SMITH\n"); + verifyPPLToSparkSQL( + root, + "SELECT MIN_BY(`ENAME`, `HIREDATE`, `JOB`) `earliest(ENAME, HIREDATE, JOB)`\n" + + "FROM `scott`.`EMP`"); } @Test public void testLatestWithTooManyParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats latest(ENAME, HIREDATE, JOB)", - "Aggregation function LATEST expects field type and additional arguments" - + " {[ANY]|[ANY,ANY]}, but got" - + " [STRING,DATE,STRING]"); + String ppl = "source=EMP | stats latest(ENAME, HIREDATE, JOB)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], latest(ENAME, HIREDATE, JOB)=[ARG_MAX($0, $1, $2)])\n" + + " LogicalProject(ENAME=[$1], HIREDATE=[$4], JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "latest(ENAME, HIREDATE, JOB)=ADAMS\n"); + verifyPPLToSparkSQL( + root, + "SELECT MAX_BY(`ENAME`, `HIREDATE`, `JOB`) `latest(ENAME, HIREDATE, JOB)`\n" + + "FROM `scott`.`EMP`"); } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java index 9258d7bf4db..1730c7a28a6 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java @@ -356,7 +356,9 @@ public void testAvgByFieldNonBucketNullable() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT AVG(`SAL`) `avg(SAL)`, `DEPTNO`\n" + "SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "AVG(`SAL`) `avg(SAL)`, `DEPTNO`\n" + "FROM `scott`.`EMP`\n" + "WHERE `DEPTNO` IS NOT NULL\n" + "GROUP BY `DEPTNO`"; @@ -957,7 +959,9 @@ public void testSortAggregationMetrics1() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT AVG(`SAL`) `avg`, `DEPTNO`\n" + "SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "AVG(`SAL`) `avg`, `DEPTNO`\n" + "FROM `scott`.`EMP`\n" + "WHERE `DEPTNO` IS NOT NULL\n" + "GROUP BY `DEPTNO`\n" @@ -982,7 +986,9 @@ public void testSortAggregationMetrics2() { verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT AVG(`SAL`) `avg`, SPAN(`HIREDATE`, 1, 'y') `hiredate_span`\n" + "SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "AVG(`SAL`) `avg`, SPAN(`HIREDATE`, 1, 'y') `hiredate_span`\n" + "FROM `scott`.`EMP`\n" + "WHERE `HIREDATE` IS NOT NULL\n" + "GROUP BY SPAN(`HIREDATE`, 1, 'y')\n" @@ -1033,7 +1039,9 @@ public void testHaving2() { String expectedSparkSql = "SELECT *\n" - + "FROM (SELECT AVG(`SAL`) `avg`, `DEPTNO`\n" + + "FROM (SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "AVG(`SAL`) `avg`, `DEPTNO`\n" + "FROM `scott`.`EMP`\n" + "WHERE `DEPTNO` IS NOT NULL\n" + "GROUP BY `DEPTNO`) `t2`\n" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java index 107fe7b9cb1..3bc60526961 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java @@ -80,7 +80,9 @@ public void testChartWithSingleGroupKey() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "`gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`\n" @@ -94,7 +96,9 @@ public void testChartWithOverSyntax() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "`gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`\n" @@ -164,7 +168,9 @@ public void testChartWithLimit() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "`gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`\n" @@ -192,7 +198,9 @@ public void testChartWithSpan() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT SPAN(`age`, 10, NULL) `age`, MAX(`balance`) `max(balance)`\n" + "SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "SPAN(`age`, 10, NULL) `age`, MAX(`balance`) `max(balance)`\n" + "FROM `scott`.`bank`\n" + "WHERE `age` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY SPAN(`age`, 10, NULL)\n" @@ -234,7 +242,9 @@ public void testChartWithUseOtherTrue() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "`gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`\n" @@ -248,7 +258,9 @@ public void testChartWithUseOtherFalse() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "`gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`\n" @@ -262,7 +274,9 @@ public void testChartWithOtherStr() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "`gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`\n" @@ -276,7 +290,9 @@ public void testChartWithNullStr() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "`gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`\n" @@ -290,7 +306,9 @@ public void testChartWithUseNull() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "`gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`\n" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEnhancedCoalesceTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEnhancedCoalesceTest.java index 56141eae584..c8f4bf80aff 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEnhancedCoalesceTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEnhancedCoalesceTest.java @@ -42,13 +42,12 @@ public void testCoalesceWithMixedTypes() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalSort(fetch=[3])\n" - + " LogicalProject(EMPNO=[$0], COMM=[$6], result=[COALESCE($6, $0," - + " 'fallback':VARCHAR)])\n" + + " LogicalProject(EMPNO=[$0], COMM=[$6], result=[COALESCE($6, $0)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `EMPNO`, `COMM`, COALESCE(`COMM`, `EMPNO`, 'fallback') `result`\n" + "SELECT `EMPNO`, `COMM`, COALESCE(`COMM`, `EMPNO`) `result`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 3"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -61,14 +60,12 @@ public void testCoalesceWithLiterals() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalSort(fetch=[1])\n" - + " LogicalProject(EMPNO=[$0], result=[COALESCE($6, 123, 'unknown':VARCHAR)])\n" + + " LogicalProject(EMPNO=[$0], result=[COALESCE($6, 123)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `EMPNO`, COALESCE(`COMM`, 123, 'unknown') `result`\n" - + "FROM `scott`.`EMP`\n" - + "LIMIT 1"; + "SELECT `EMPNO`, COALESCE(`COMM`, 123) `result`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -118,13 +115,13 @@ public void testCoalesceNested() { String expectedLogical = "LogicalSort(fetch=[2])\n" + " LogicalProject(EMPNO=[$0], COMM=[$6], SAL=[$5], result1=[COALESCE($6, 0)]," - + " result2=[COALESCE(COALESCE($6, 0), $5)])\n" + + " result2=[COALESCE($6, 0)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `EMPNO`, `COMM`, `SAL`, COALESCE(`COMM`, 0) `result1`, COALESCE(COALESCE(`COMM`," - + " 0), `SAL`) `result2`\n" + "SELECT `EMPNO`, `COMM`, `SAL`, COALESCE(`COMM`, 0) `result1`, COALESCE(`COMM`, 0)" + + " `result2`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 2"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -138,12 +135,12 @@ public void testCoalesceWithNonExistentField() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalSort(fetch=[2])\n" - + " LogicalProject(EMPNO=[$0], result=[COALESCE(null:VARCHAR, $1)])\n" + + " LogicalProject(EMPNO=[$0], result=[$1])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `EMPNO`, COALESCE(NULL, `ENAME`) `result`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 2"; + "SELECT `EMPNO`, `ENAME` `result`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 2"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -155,13 +152,12 @@ public void testCoalesceWithMultipleNonExistentFields() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalSort(fetch=[1])\n" - + " LogicalProject(EMPNO=[$0], result=[COALESCE(null:VARCHAR, null:VARCHAR, $1," - + " 'fallback':VARCHAR)])\n" + + " LogicalProject(EMPNO=[$0], result=[COALESCE($1, 'fallback':VARCHAR)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `EMPNO`, COALESCE(NULL, NULL, `ENAME`, 'fallback') `result`\n" + "SELECT `EMPNO`, COALESCE(`ENAME`, 'fallback') `result`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -175,15 +171,12 @@ public void testCoalesceWithAllNonExistentFields() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalSort(fetch=[1])\n" - + " LogicalProject(EMPNO=[$0], result=[COALESCE(null:VARCHAR, null:VARCHAR," - + " null:VARCHAR)])\n" + + " LogicalProject(EMPNO=[$0], result=[null:VARCHAR])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `EMPNO`, COALESCE(NULL, NULL, NULL) `result`\n" - + "FROM `scott`.`EMP`\n" - + "LIMIT 1"; + "SELECT `EMPNO`, CAST(NULL AS STRING) `result`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -193,12 +186,11 @@ public void testCoalesceWithEmptyString() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalSort(fetch=[1])\n" - + " LogicalProject(EMPNO=[$0], result=[COALESCE('':VARCHAR, $1)])\n" + + " LogicalProject(EMPNO=[$0], result=['':VARCHAR])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); - String expectedSparkSql = - "SELECT `EMPNO`, COALESCE('', `ENAME`) `result`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 1"; + String expectedSparkSql = "SELECT `EMPNO`, '' `result`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -208,12 +200,11 @@ public void testCoalesceWithSpaceString() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalSort(fetch=[1])\n" - + " LogicalProject(EMPNO=[$0], result=[COALESCE(' ', $1)])\n" + + " LogicalProject(EMPNO=[$0], result=[' '])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); - String expectedSparkSql = - "SELECT `EMPNO`, COALESCE(' ', `ENAME`) `result`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 1"; + String expectedSparkSql = "SELECT `EMPNO`, ' ' `result`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEventstatsTypeTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEventstatsTypeTest.java index 24bd9ac18d0..2444a6c4ba9 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEventstatsTypeTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEventstatsTypeTest.java @@ -5,6 +5,7 @@ package org.opensearch.sql.ppl.calcite; +import org.apache.calcite.rel.RelNode; import org.apache.calcite.test.CalciteAssert; import org.junit.Test; @@ -16,128 +17,507 @@ public CalcitePPLEventstatsTypeTest() { @Test public void testCountWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats count(EMPNO, DEPTNO)", - "Aggregation function COUNT expects field type and additional arguments {[]|[ANY]}," - + " but got [SHORT,BYTE]"); + // This test verifies logical plan generation for window functions + // Note: COUNT ignores the second parameter + String ppl = "source=EMP | eventstats count(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], count(EMPNO, DEPTNO)=[COUNT($0) OVER ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult( + root, + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7566; ENAME=JONES; JOB=MANAGER; MGR=7839; HIREDATE=1981-02-04; SAL=2975.00;" + + " COMM=null; DEPTNO=20; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7654; ENAME=MARTIN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-28; SAL=1250.00;" + + " COMM=1400.00; DEPTNO=30; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7698; ENAME=BLAKE; JOB=MANAGER; MGR=7839; HIREDATE=1981-01-05; SAL=2850.00;" + + " COMM=null; DEPTNO=30; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7782; ENAME=CLARK; JOB=MANAGER; MGR=7839; HIREDATE=1981-06-09; SAL=2450.00;" + + " COMM=null; DEPTNO=10; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7788; ENAME=SCOTT; JOB=ANALYST; MGR=7566; HIREDATE=1987-04-19; SAL=3000.00;" + + " COMM=null; DEPTNO=20; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7839; ENAME=KING; JOB=PRESIDENT; MGR=null; HIREDATE=1981-11-17; SAL=5000.00;" + + " COMM=null; DEPTNO=10; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7844; ENAME=TURNER; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-08; SAL=1500.00;" + + " COMM=0.00; DEPTNO=30; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7876; ENAME=ADAMS; JOB=CLERK; MGR=7788; HIREDATE=1987-05-23; SAL=1100.00;" + + " COMM=null; DEPTNO=20; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7900; ENAME=JAMES; JOB=CLERK; MGR=7698; HIREDATE=1981-12-03; SAL=950.00;" + + " COMM=null; DEPTNO=30; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7902; ENAME=FORD; JOB=ANALYST; MGR=7566; HIREDATE=1981-12-03; SAL=3000.00;" + + " COMM=null; DEPTNO=20; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7934; ENAME=MILLER; JOB=CLERK; MGR=7782; HIREDATE=1982-01-23; SAL=1300.00;" + + " COMM=null; DEPTNO=10; count(EMPNO, DEPTNO)=14\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, COUNT(`EMPNO`)" + + " OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) `count(EMPNO," + + " DEPTNO)`\n" + + "FROM `scott`.`EMP`"); } @Test public void testAvgWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats avg(EMPNO, DEPTNO)", - "Aggregation function AVG expects field type and additional arguments {[INTEGER]|[DOUBLE]}," - + " but got [SHORT,BYTE]"); + // This test verifies logical plan generation for window functions + // Note: AVG ignores the second parameter + String ppl = "source=EMP | eventstats avg(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], avg(EMPNO, DEPTNO)=[/(SUM($0) OVER (), CAST(COUNT($0) OVER" + + " ()):DOUBLE NOT NULL)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult( + root, + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7566; ENAME=JONES; JOB=MANAGER; MGR=7839; HIREDATE=1981-02-04; SAL=2975.00;" + + " COMM=null; DEPTNO=20; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7654; ENAME=MARTIN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-28; SAL=1250.00;" + + " COMM=1400.00; DEPTNO=30; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7698; ENAME=BLAKE; JOB=MANAGER; MGR=7839; HIREDATE=1981-01-05; SAL=2850.00;" + + " COMM=null; DEPTNO=30; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7782; ENAME=CLARK; JOB=MANAGER; MGR=7839; HIREDATE=1981-06-09; SAL=2450.00;" + + " COMM=null; DEPTNO=10; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7788; ENAME=SCOTT; JOB=ANALYST; MGR=7566; HIREDATE=1987-04-19; SAL=3000.00;" + + " COMM=null; DEPTNO=20; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7839; ENAME=KING; JOB=PRESIDENT; MGR=null; HIREDATE=1981-11-17; SAL=5000.00;" + + " COMM=null; DEPTNO=10; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7844; ENAME=TURNER; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-08; SAL=1500.00;" + + " COMM=0.00; DEPTNO=30; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7876; ENAME=ADAMS; JOB=CLERK; MGR=7788; HIREDATE=1987-05-23; SAL=1100.00;" + + " COMM=null; DEPTNO=20; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7900; ENAME=JAMES; JOB=CLERK; MGR=7698; HIREDATE=1981-12-03; SAL=950.00;" + + " COMM=null; DEPTNO=30; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7902; ENAME=FORD; JOB=ANALYST; MGR=7566; HIREDATE=1981-12-03; SAL=3000.00;" + + " COMM=null; DEPTNO=20; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7934; ENAME=MILLER; JOB=CLERK; MGR=7782; HIREDATE=1982-01-23; SAL=1300.00;" + + " COMM=null; DEPTNO=10; avg(EMPNO, DEPTNO)=7726.571428571428\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, (SUM(`EMPNO`)" + + " OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) /" + + " CAST(COUNT(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE) `avg(EMPNO, DEPTNO)`\n" + + "FROM `scott`.`EMP`"); } @Test public void testSumWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats sum(EMPNO, DEPTNO)", - "Aggregation function SUM expects field type and additional arguments {[INTEGER]|[DOUBLE]}," - + " but got [SHORT,BYTE]"); + // This test verifies logical plan generation for window functions + String ppl = "source=EMP | eventstats sum(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], sum(EMPNO, DEPTNO)=[SUM($0, $7) OVER ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult( + root, + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7566; ENAME=JONES; JOB=MANAGER; MGR=7839; HIREDATE=1981-02-04; SAL=2975.00;" + + " COMM=null; DEPTNO=20; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7654; ENAME=MARTIN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-28; SAL=1250.00;" + + " COMM=1400.00; DEPTNO=30; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7698; ENAME=BLAKE; JOB=MANAGER; MGR=7839; HIREDATE=1981-01-05; SAL=2850.00;" + + " COMM=null; DEPTNO=30; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7782; ENAME=CLARK; JOB=MANAGER; MGR=7839; HIREDATE=1981-06-09; SAL=2450.00;" + + " COMM=null; DEPTNO=10; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7788; ENAME=SCOTT; JOB=ANALYST; MGR=7566; HIREDATE=1987-04-19; SAL=3000.00;" + + " COMM=null; DEPTNO=20; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7839; ENAME=KING; JOB=PRESIDENT; MGR=null; HIREDATE=1981-11-17; SAL=5000.00;" + + " COMM=null; DEPTNO=10; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7844; ENAME=TURNER; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-08; SAL=1500.00;" + + " COMM=0.00; DEPTNO=30; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7876; ENAME=ADAMS; JOB=CLERK; MGR=7788; HIREDATE=1987-05-23; SAL=1100.00;" + + " COMM=null; DEPTNO=20; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7900; ENAME=JAMES; JOB=CLERK; MGR=7698; HIREDATE=1981-12-03; SAL=950.00;" + + " COMM=null; DEPTNO=30; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7902; ENAME=FORD; JOB=ANALYST; MGR=7566; HIREDATE=1981-12-03; SAL=3000.00;" + + " COMM=null; DEPTNO=20; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7934; ENAME=MILLER; JOB=CLERK; MGR=7782; HIREDATE=1982-01-23; SAL=1300.00;" + + " COMM=null; DEPTNO=10; sum(EMPNO, DEPTNO)=108172\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, SUM(`EMPNO`," + + " `DEPTNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)" + + " `sum(EMPNO, DEPTNO)`\n" + + "FROM `scott`.`EMP`"); } @Test public void testMinWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats min(EMPNO, DEPTNO)", - "Aggregation function MIN expects field type and additional arguments {[COMPARABLE_TYPE]}," - + " but got [SHORT,BYTE]"); + // This test verifies logical plan generation for window functions + String ppl = "source=EMP | eventstats min(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], min(EMPNO, DEPTNO)=[MIN($0, $7) OVER ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult( + root, + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7566; ENAME=JONES; JOB=MANAGER; MGR=7839; HIREDATE=1981-02-04; SAL=2975.00;" + + " COMM=null; DEPTNO=20; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7654; ENAME=MARTIN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-28; SAL=1250.00;" + + " COMM=1400.00; DEPTNO=30; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7698; ENAME=BLAKE; JOB=MANAGER; MGR=7839; HIREDATE=1981-01-05; SAL=2850.00;" + + " COMM=null; DEPTNO=30; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7782; ENAME=CLARK; JOB=MANAGER; MGR=7839; HIREDATE=1981-06-09; SAL=2450.00;" + + " COMM=null; DEPTNO=10; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7788; ENAME=SCOTT; JOB=ANALYST; MGR=7566; HIREDATE=1987-04-19; SAL=3000.00;" + + " COMM=null; DEPTNO=20; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7839; ENAME=KING; JOB=PRESIDENT; MGR=null; HIREDATE=1981-11-17; SAL=5000.00;" + + " COMM=null; DEPTNO=10; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7844; ENAME=TURNER; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-08; SAL=1500.00;" + + " COMM=0.00; DEPTNO=30; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7876; ENAME=ADAMS; JOB=CLERK; MGR=7788; HIREDATE=1987-05-23; SAL=1100.00;" + + " COMM=null; DEPTNO=20; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7900; ENAME=JAMES; JOB=CLERK; MGR=7698; HIREDATE=1981-12-03; SAL=950.00;" + + " COMM=null; DEPTNO=30; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7902; ENAME=FORD; JOB=ANALYST; MGR=7566; HIREDATE=1981-12-03; SAL=3000.00;" + + " COMM=null; DEPTNO=20; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7934; ENAME=MILLER; JOB=CLERK; MGR=7782; HIREDATE=1982-01-23; SAL=1300.00;" + + " COMM=null; DEPTNO=10; min(EMPNO, DEPTNO)=7369\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, MIN(`EMPNO`," + + " `DEPTNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)" + + " `min(EMPNO, DEPTNO)`\n" + + "FROM `scott`.`EMP`"); } @Test public void testMaxWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats max(EMPNO, DEPTNO)", - "Aggregation function MAX expects field type and additional arguments {[COMPARABLE_TYPE]}," - + " but got [SHORT,BYTE]"); + // This test verifies logical plan generation for window functions + String ppl = "source=EMP | eventstats max(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], max(EMPNO, DEPTNO)=[MAX($0, $7) OVER ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult( + root, + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7566; ENAME=JONES; JOB=MANAGER; MGR=7839; HIREDATE=1981-02-04; SAL=2975.00;" + + " COMM=null; DEPTNO=20; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7654; ENAME=MARTIN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-28; SAL=1250.00;" + + " COMM=1400.00; DEPTNO=30; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7698; ENAME=BLAKE; JOB=MANAGER; MGR=7839; HIREDATE=1981-01-05; SAL=2850.00;" + + " COMM=null; DEPTNO=30; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7782; ENAME=CLARK; JOB=MANAGER; MGR=7839; HIREDATE=1981-06-09; SAL=2450.00;" + + " COMM=null; DEPTNO=10; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7788; ENAME=SCOTT; JOB=ANALYST; MGR=7566; HIREDATE=1987-04-19; SAL=3000.00;" + + " COMM=null; DEPTNO=20; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7839; ENAME=KING; JOB=PRESIDENT; MGR=null; HIREDATE=1981-11-17; SAL=5000.00;" + + " COMM=null; DEPTNO=10; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7844; ENAME=TURNER; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-08; SAL=1500.00;" + + " COMM=0.00; DEPTNO=30; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7876; ENAME=ADAMS; JOB=CLERK; MGR=7788; HIREDATE=1987-05-23; SAL=1100.00;" + + " COMM=null; DEPTNO=20; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7900; ENAME=JAMES; JOB=CLERK; MGR=7698; HIREDATE=1981-12-03; SAL=950.00;" + + " COMM=null; DEPTNO=30; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7902; ENAME=FORD; JOB=ANALYST; MGR=7566; HIREDATE=1981-12-03; SAL=3000.00;" + + " COMM=null; DEPTNO=20; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7934; ENAME=MILLER; JOB=CLERK; MGR=7782; HIREDATE=1982-01-23; SAL=1300.00;" + + " COMM=null; DEPTNO=10; max(EMPNO, DEPTNO)=7934\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, MAX(`EMPNO`," + + " `DEPTNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)" + + " `max(EMPNO, DEPTNO)`\n" + + "FROM `scott`.`EMP`"); } @Test public void testVarSampWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats var_samp(EMPNO, DEPTNO)", - "Aggregation function VARSAMP expects field type and additional arguments" - + " {[INTEGER]|[DOUBLE]}, but got [SHORT,BYTE]"); + // This test verifies logical plan generation, but execution fails with ArithmeticException + String ppl = "source=EMP | eventstats var_samp(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], var_samp(EMPNO, DEPTNO)=[/(-(SUM(*($0, $0)) OVER ()," + + " /(*(SUM($0) OVER (), SUM($0) OVER ()), CAST(COUNT($0) OVER ()):DOUBLE NOT NULL))," + + " -(CAST(COUNT($0) OVER ()):DOUBLE NOT NULL, 1))])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, ((SUM(`EMPNO`" + + " * `EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) -" + + " (SUM(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) *" + + " (SUM(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) /" + + " CAST(COUNT(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE)) / (CAST(COUNT(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED" + + " PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE) - 1) `var_samp(EMPNO, DEPTNO)`\n" + + "FROM `scott`.`EMP`"); + // Execution fails with: ArithmeticException: Value out of range + Exception e = + org.junit.Assert.assertThrows(RuntimeException.class, () -> verifyResultCount(root, 0)); + verifyErrorMessageContains(e, "out of range"); } @Test public void testVarPopWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats var_pop(EMPNO, DEPTNO)", - "Aggregation function VARPOP expects field type and additional arguments" - + " {[INTEGER]|[DOUBLE]}, but got [SHORT,BYTE]"); + // This test verifies logical plan generation, but execution fails with ArithmeticException + String ppl = "source=EMP | eventstats var_pop(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], var_pop(EMPNO, DEPTNO)=[/(-(SUM(*($0, $0)) OVER ()," + + " /(*(SUM($0) OVER (), SUM($0) OVER ()), CAST(COUNT($0) OVER ()):DOUBLE NOT NULL))," + + " CAST(COUNT($0) OVER ()):DOUBLE NOT NULL)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, ((SUM(`EMPNO`" + + " * `EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) -" + + " (SUM(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) *" + + " (SUM(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) /" + + " CAST(COUNT(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE)) / CAST(COUNT(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED" + + " PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE) `var_pop(EMPNO, DEPTNO)`\n" + + "FROM `scott`.`EMP`"); + // Execution fails with: ArithmeticException: Value out of range + Exception e = + org.junit.Assert.assertThrows(RuntimeException.class, () -> verifyResultCount(root, 0)); + verifyErrorMessageContains(e, "out of range"); } @Test public void testStddevSampWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats stddev_samp(EMPNO, DEPTNO)", - "Aggregation function STDDEV_SAMP expects field type and additional arguments" - + " {[INTEGER]|[DOUBLE]}, but got [SHORT,BYTE]"); + // This test verifies logical plan generation, but execution fails with ArithmeticException + String ppl = "source=EMP | eventstats stddev_samp(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], stddev_samp(EMPNO, DEPTNO)=[POWER(/(-(SUM(*($0, $0)) OVER" + + " (), /(*(SUM($0) OVER (), SUM($0) OVER ()), CAST(COUNT($0) OVER ()):DOUBLE NOT" + + " NULL)), -(CAST(COUNT($0) OVER ()):DOUBLE NOT NULL, 1)), 0.5E0:DOUBLE)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " POWER(((SUM(`EMPNO` * `EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND" + + " UNBOUNDED FOLLOWING)) - (SUM(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND" + + " UNBOUNDED FOLLOWING)) * (SUM(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND" + + " UNBOUNDED FOLLOWING)) / CAST(COUNT(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING" + + " AND UNBOUNDED FOLLOWING) AS DOUBLE)) / (CAST(COUNT(`EMPNO`) OVER (RANGE BETWEEN" + + " UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE) - 1), 5E-1)" + + " `stddev_samp(EMPNO, DEPTNO)`\n" + + "FROM `scott`.`EMP`"); + // Execution fails with: ArithmeticException: Value out of range + Exception e = + org.junit.Assert.assertThrows(RuntimeException.class, () -> verifyResultCount(root, 0)); + verifyErrorMessageContains(e, "out of range"); } @Test public void testStddevPopWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats stddev_pop(EMPNO, DEPTNO)", - "Aggregation function STDDEV_POP expects field type and additional arguments" - + " {[INTEGER]|[DOUBLE]}, but got [SHORT,BYTE]"); + // This test verifies logical plan generation, but execution fails with ArithmeticException + String ppl = "source=EMP | eventstats stddev_pop(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], stddev_pop(EMPNO, DEPTNO)=[POWER(/(-(SUM(*($0, $0)) OVER" + + " (), /(*(SUM($0) OVER (), SUM($0) OVER ()), CAST(COUNT($0) OVER ()):DOUBLE NOT" + + " NULL)), CAST(COUNT($0) OVER ()):DOUBLE NOT NULL), 0.5E0:DOUBLE)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " POWER(((SUM(`EMPNO` * `EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND" + + " UNBOUNDED FOLLOWING)) - (SUM(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND" + + " UNBOUNDED FOLLOWING)) * (SUM(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND" + + " UNBOUNDED FOLLOWING)) / CAST(COUNT(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING" + + " AND UNBOUNDED FOLLOWING) AS DOUBLE)) / CAST(COUNT(`EMPNO`) OVER (RANGE BETWEEN" + + " UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE), 5E-1) `stddev_pop(EMPNO," + + " DEPTNO)`\n" + + "FROM `scott`.`EMP`"); + // Execution fails with: ArithmeticException: Value out of range + Exception e = + org.junit.Assert.assertThrows(RuntimeException.class, () -> verifyResultCount(root, 0)); + verifyErrorMessageContains(e, "out of range"); } @Test public void testEarliestWithTooManyParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats earliest(ENAME, HIREDATE, JOB)", - "Aggregation function EARLIEST expects field type and additional arguments" - + " {[ANY]|[ANY,ANY]}, but got" - + " [STRING,DATE,STRING]"); + // This test verifies logical plan generation for window functions + String ppl = "source=EMP | eventstats earliest(ENAME, HIREDATE, JOB)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], earliest(ENAME, HIREDATE, JOB)=[ARG_MIN($1, $4, $2) OVER" + + " ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult( + root, + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7566; ENAME=JONES; JOB=MANAGER; MGR=7839; HIREDATE=1981-02-04; SAL=2975.00;" + + " COMM=null; DEPTNO=20; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7654; ENAME=MARTIN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-28; SAL=1250.00;" + + " COMM=1400.00; DEPTNO=30; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7698; ENAME=BLAKE; JOB=MANAGER; MGR=7839; HIREDATE=1981-01-05; SAL=2850.00;" + + " COMM=null; DEPTNO=30; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7782; ENAME=CLARK; JOB=MANAGER; MGR=7839; HIREDATE=1981-06-09; SAL=2450.00;" + + " COMM=null; DEPTNO=10; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7788; ENAME=SCOTT; JOB=ANALYST; MGR=7566; HIREDATE=1987-04-19; SAL=3000.00;" + + " COMM=null; DEPTNO=20; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7839; ENAME=KING; JOB=PRESIDENT; MGR=null; HIREDATE=1981-11-17; SAL=5000.00;" + + " COMM=null; DEPTNO=10; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7844; ENAME=TURNER; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-08; SAL=1500.00;" + + " COMM=0.00; DEPTNO=30; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7876; ENAME=ADAMS; JOB=CLERK; MGR=7788; HIREDATE=1987-05-23; SAL=1100.00;" + + " COMM=null; DEPTNO=20; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7900; ENAME=JAMES; JOB=CLERK; MGR=7698; HIREDATE=1981-12-03; SAL=950.00;" + + " COMM=null; DEPTNO=30; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7902; ENAME=FORD; JOB=ANALYST; MGR=7566; HIREDATE=1981-12-03; SAL=3000.00;" + + " COMM=null; DEPTNO=20; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7934; ENAME=MILLER; JOB=CLERK; MGR=7782; HIREDATE=1982-01-23; SAL=1300.00;" + + " COMM=null; DEPTNO=10; earliest(ENAME, HIREDATE, JOB)=SMITH\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " MIN_BY(`ENAME`, `HIREDATE`, `JOB`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND" + + " UNBOUNDED FOLLOWING) `earliest(ENAME, HIREDATE, JOB)`\n" + + "FROM `scott`.`EMP`"); } @Test public void testLatestWithTooManyParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats latest(ENAME, HIREDATE, JOB)", - "Aggregation function LATEST expects field type and additional arguments" - + " {[ANY]|[ANY,ANY]}, but got" - + " [STRING,DATE,STRING]"); + // This test verifies logical plan generation for window functions + String ppl = "source=EMP | eventstats latest(ENAME, HIREDATE, JOB)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], latest(ENAME, HIREDATE, JOB)=[ARG_MAX($1, $4, $2) OVER" + + " ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult( + root, + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7566; ENAME=JONES; JOB=MANAGER; MGR=7839; HIREDATE=1981-02-04; SAL=2975.00;" + + " COMM=null; DEPTNO=20; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7654; ENAME=MARTIN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-28; SAL=1250.00;" + + " COMM=1400.00; DEPTNO=30; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7698; ENAME=BLAKE; JOB=MANAGER; MGR=7839; HIREDATE=1981-01-05; SAL=2850.00;" + + " COMM=null; DEPTNO=30; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7782; ENAME=CLARK; JOB=MANAGER; MGR=7839; HIREDATE=1981-06-09; SAL=2450.00;" + + " COMM=null; DEPTNO=10; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7788; ENAME=SCOTT; JOB=ANALYST; MGR=7566; HIREDATE=1987-04-19; SAL=3000.00;" + + " COMM=null; DEPTNO=20; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7839; ENAME=KING; JOB=PRESIDENT; MGR=null; HIREDATE=1981-11-17; SAL=5000.00;" + + " COMM=null; DEPTNO=10; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7844; ENAME=TURNER; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-08; SAL=1500.00;" + + " COMM=0.00; DEPTNO=30; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7876; ENAME=ADAMS; JOB=CLERK; MGR=7788; HIREDATE=1987-05-23; SAL=1100.00;" + + " COMM=null; DEPTNO=20; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7900; ENAME=JAMES; JOB=CLERK; MGR=7698; HIREDATE=1981-12-03; SAL=950.00;" + + " COMM=null; DEPTNO=30; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7902; ENAME=FORD; JOB=ANALYST; MGR=7566; HIREDATE=1981-12-03; SAL=3000.00;" + + " COMM=null; DEPTNO=20; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7934; ENAME=MILLER; JOB=CLERK; MGR=7782; HIREDATE=1982-01-23; SAL=1300.00;" + + " COMM=null; DEPTNO=10; latest(ENAME, HIREDATE, JOB)=ADAMS\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " MAX_BY(`ENAME`, `HIREDATE`, `JOB`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND" + + " UNBOUNDED FOLLOWING) `latest(ENAME, HIREDATE, JOB)`\n" + + "FROM `scott`.`EMP`"); } @Test public void testAvgWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | eventstats avg(HIREDATE) as avg_name", - "Aggregation function AVG expects field type {[INTEGER]|[DOUBLE]}, but got [DATE]"); + // AVG with DATE argument throws CalciteContextException during plan generation + String ppl = "source=EMP | eventstats avg(HIREDATE) as avg_name"; + Exception e = + org.junit.Assert.assertThrows( + org.apache.calcite.runtime.CalciteContextException.class, () -> getRelNode(ppl)); + verifyErrorMessageContains(e, "Cannot infer return type for /; operand types: [DATE, DOUBLE]"); } @Test public void testVarsampWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | eventstats var_samp(HIREDATE) as varsamp_name", - "Aggregation function VARSAMP expects field type {[INTEGER]|[DOUBLE]}, but got [DATE]"); + // VAR_SAMP with DATE argument throws CalciteContextException during plan generation + String ppl = "source=EMP | eventstats var_samp(HIREDATE) as varsamp_name"; + Exception e = + org.junit.Assert.assertThrows( + org.apache.calcite.runtime.CalciteContextException.class, () -> getRelNode(ppl)); + verifyErrorMessageContains(e, "Cannot infer return type for /; operand types: [DATE, DOUBLE]"); } @Test public void testVarpopWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | eventstats var_pop(HIREDATE) as varpop_name", - "Aggregation function VARPOP expects field type {[INTEGER]|[DOUBLE]}, but got [DATE]"); + // VAR_POP with DATE argument throws CalciteContextException during plan generation + String ppl = "source=EMP | eventstats var_pop(HIREDATE) as varpop_name"; + Exception e = + org.junit.Assert.assertThrows( + org.apache.calcite.runtime.CalciteContextException.class, () -> getRelNode(ppl)); + verifyErrorMessageContains(e, "Cannot infer return type for /; operand types: [DATE, DOUBLE]"); } @Test public void testStddevSampWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | eventstats stddev_samp(HIREDATE) as stddev_name", - "Aggregation function STDDEV_SAMP expects field type {[INTEGER]|[DOUBLE]}, but got" - + " [DATE]"); + // STDDEV_SAMP with DATE argument throws CalciteContextException during plan generation + String ppl = "source=EMP | eventstats stddev_samp(HIREDATE) as stddev_name"; + Exception e = + org.junit.Assert.assertThrows( + org.apache.calcite.runtime.CalciteContextException.class, () -> getRelNode(ppl)); + verifyErrorMessageContains(e, "Cannot infer return type for /; operand types: [DATE, DOUBLE]"); } @Test public void testStddevPopWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | eventstats stddev_pop(HIREDATE) as stddev_name", - "Aggregation function STDDEV_POP expects field type {[INTEGER]|[DOUBLE]}, but got" - + " [DATE]"); + // STDDEV_POP with DATE argument throws CalciteContextException during plan generation + String ppl = "source=EMP | eventstats stddev_pop(HIREDATE) as stddev_name"; + Exception e = + org.junit.Assert.assertThrows( + org.apache.calcite.runtime.CalciteContextException.class, () -> getRelNode(ppl)); + verifyErrorMessageContains(e, "Cannot infer return type for /; operand types: [DATE, DOUBLE]"); } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFunctionTypeTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFunctionTypeTest.java index 9513558952f..322305056b1 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFunctionTypeTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFunctionTypeTest.java @@ -6,13 +6,11 @@ package org.opensearch.sql.ppl.calcite; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; import org.apache.calcite.rel.RelNode; import org.apache.calcite.test.CalciteAssert; import org.junit.Assert; import org.junit.Test; -import org.opensearch.sql.exception.ExpressionEvaluationException; public class CalcitePPLFunctionTypeTest extends CalcitePPLAbstractTest { @@ -22,23 +20,23 @@ public CalcitePPLFunctionTypeTest() { @Test public void testLowerWithIntegerType() { - verifyQueryThrowsException( - "source=EMP | eval lower_name = lower(EMPNO) | fields lower_name", - "LOWER function expects {[STRING]}, but got [SHORT]"); + // Lower with IntegerType no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | eval lower_name = lower(EMPNO) | fields lower_name"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(lower_name=[LOWER($0)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"); } @Test public void testTimeDiffWithUdtInputType() { - String strPpl = - "source=EMP | eval time_diff = timediff('12:00:00', '12:00:06') | fields time_diff"; - String timePpl = - "source=EMP | eval time_diff = timediff(time('13:00:00'), time('12:00:06')) | fields" - + " time_diff"; - getRelNode(strPpl); - getRelNode(timePpl); - verifyQueryThrowsException( - "source=EMP | eval time_diff = timediff(12, '2009-12-10') | fields time_diff", - "TIMEDIFF function expects {[TIME,TIME]}, but got [INTEGER,STRING]"); + // TimeDiff with UdtInputType no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | eval time_diff = timediff(12, '2009-12-10') | fields time_diff"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(time_diff=[TIME_DIFF(12, '2009-12-10':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } @Test @@ -65,116 +63,148 @@ public void testCoalesceWithDifferentType() { @Test public void testSubstringWithWrongType() { - getRelNode("source=EMP | eval sub_name = substring(ENAME, 1, 3) | fields sub_name"); - getRelNode("source=EMP | eval sub_name = substring(ENAME, 1) | fields sub_name"); - verifyQueryThrowsException( - "source=EMP | eval sub_name = substring(ENAME, 1, '3') | fields sub_name", - "SUBSTRING function expects {[STRING,INTEGER]|[STRING,INTEGER,INTEGER]}, but got" - + " [STRING,INTEGER,STRING]"); + // Substring with wrong type no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | eval sub_name = substring(ENAME, 1, '3') | fields sub_name"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(sub_name=[SUBSTRING($1, 1, '3')])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } @Test public void testIfWithWrongType() { - getRelNode("source=EMP | eval if_name = if(EMPNO > 6, 'Jack', ENAME) | fields if_name"); - getRelNode("source=EMP | eval if_name = if(EMPNO > 6, EMPNO, DEPTNO) | fields if_name"); - verifyQueryThrowsException( - "source=EMP | eval if_name = if(EMPNO, 1, DEPTNO) | fields if_name", - "IF function expects {[BOOLEAN,ANY,ANY]}, but got [SHORT,INTEGER,BYTE]"); - verifyQueryThrowsException( - "source=EMP | eval if_name = if(EMPNO > 6, 'Jack', 1) | fields if_name", - "Cannot resolve function: IF, arguments: [BOOLEAN,STRING,INTEGER], caused by: Can't find" - + " leastRestrictive type for [VARCHAR, INTEGER]"); + // If with wrong type no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | eval if_name = if(EMPNO, 1, DEPTNO) | fields if_name"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(if_name=[CASE($0, 1, $7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } @Test public void testTimestampWithWrongArg() { - verifyQueryThrowsException( + // Timestamp with wrong argument no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | eval timestamp = timestamp('2020-08-26 13:49:00', 2009) | fields timestamp |" - + " head 1", - "TIMESTAMP function expects" - + " {[STRING]|[TIMESTAMP]|[DATE]|[TIME]|[STRING,STRING]|[TIMESTAMP,TIMESTAMP]|[TIMESTAMP,DATE]|[TIMESTAMP,TIME]|[DATE,TIMESTAMP]|[DATE,DATE]|[DATE,TIME]|[TIME,TIMESTAMP]|[TIME,DATE]|[TIME,TIME]|[STRING,TIMESTAMP]|[STRING,DATE]|[STRING,TIME]|[TIMESTAMP,STRING]|[DATE,STRING]|[TIME,STRING]}," - + " but got [STRING,INTEGER]"); + + " head 1"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalSort(fetch=[1])\n" + + " LogicalProject(timestamp=[TIMESTAMP('2020-08-26 13:49:00':VARCHAR, 2009)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } @Test public void testCurDateWithArg() { - verifyQueryThrowsException( - "source=EMP | eval curdate = CURDATE(1) | fields curdate | head 1", - "CURDATE function expects {[]}, but got [INTEGER]"); + // CurDate with Arg no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | eval curdate = CURDATE(1) | fields curdate | head 1"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalSort(fetch=[1])\n" + + " LogicalProject(curdate=[CURRENT_DATE(1)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } + // Test directly registered UDF: register(funcname, FuncImp) // Test directly registered UDF: register(funcname, FuncImp) @Test public void testLtrimWrongArg() { - verifyQueryThrowsException( - "source=EMP | where ltrim(EMPNO, DEPTNO) = 'Jim' | fields name, age", - "LTRIM function expects {[STRING]}, but got [SHORT,BYTE]"); + String ppl = "source=EMP | where ltrim(EMPNO, DEPTNO) = 'Jim' | fields name, age"; + Exception e = Assert.assertThrows(IllegalArgumentException.class, () -> getRelNode(ppl)); + verifyErrorMessageContains(e, "This function requires exactly 1 arguments"); } + // Test udf registered via sql library operator: registerOperator(REVERSE, + // SqlLibraryOperators.REVERSE); // Test udf registered via sql library operator: registerOperator(REVERSE, // SqlLibraryOperators.REVERSE); @Test public void testReverseWrongArgShouldThrow() { - verifyQueryThrowsException( - "source=EMP | where reverse(EMPNO) = '3202' | fields year", - "REVERSE function expects {[STRING]}, but got [SHORT]"); + String ppl = "source=EMP | where reverse(EMPNO) = '3202' | fields year"; + Throwable e = Assert.assertThrows(AssertionError.class, () -> getRelNode(ppl)); + verifyErrorMessageContains(e, "Was not expecting value 'SMALLINT'"); } + // test type checking on UDF with direct registration: register(funcname, FuncImp) // test type checking on UDF with direct registration: register(funcname, FuncImp) @Test public void testStrCmpWrongArgShouldThrow() { - verifyQueryThrowsException( - "source=EMP | where strcmp(10, 'Jane') = 0 | fields name, age", - "STRCMP function expects {[STRING,STRING]}, but got [INTEGER,STRING]"); + String ppl = "source=EMP | where strcmp(10, 'Jane') = 0 | fields name, age"; + Exception e = Assert.assertThrows(IllegalArgumentException.class, () -> getRelNode(ppl)); + verifyErrorMessageContains(e, "Field [name] not found"); } + // Test registered Sql Std Operator: registerOperator(funcName, SqlStdOperatorTable.OPERATOR) // Test registered Sql Std Operator: registerOperator(funcName, SqlStdOperatorTable.OPERATOR) @Test public void testLowerWrongArgShouldThrow() { - verifyQueryThrowsException( - "source=EMP | where lower(EMPNO) = 'hello' | fields name, age", - "LOWER function expects {[STRING]}, but got [SHORT]"); + String ppl = "source=EMP | where lower(EMPNO) = 'hello' | fields name, age"; + Exception e = Assert.assertThrows(IllegalArgumentException.class, () -> getRelNode(ppl)); + verifyErrorMessageContains(e, "Field [name] not found"); } @Test public void testSha2WrongArgShouldThrow() { - verifyQueryThrowsException( - "source=EMP | head 1 | eval sha256 = SHA2('hello', '256') | fields sha256", - "SHA2 function expects {[STRING,INTEGER]}, but got [STRING,STRING]"); + // Sha2WrongArg should throw no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | head 1 | eval sha256 = SHA2('hello', '256') | fields sha256"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(sha256=[SHA2('hello':VARCHAR, '256':VARCHAR)])\n" + + " LogicalSort(fetch=[1])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } // Test type checking on udf with direct registration: register(SQRT, funcImp) @Test public void testSqrtWithWrongArg() { - verifyQueryThrowsException( - "source=EMP | head 1 | eval sqrt_name = sqrt(HIREDATE) | fields sqrt_name", - "SQRT function expects {[INTEGER]|[DOUBLE]}, but got [DATE]"); + // Sqrt with wrong argument no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | head 1 | eval sqrt_name = sqrt(HIREDATE) | fields sqrt_name"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(sqrt_name=[POWER($4, 0.5E0:DOUBLE)])\n" + + " LogicalSort(fetch=[1])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } // Test UDF registered with PPL builtin operators: registerOperator(MOD, PPLBuiltinOperators.MOD); @Test public void testModWithWrongArg() { - verifyQueryThrowsException( - "source=EMP | eval z = mod(0.5, 1, 2) | fields z", - "MOD function expects" - + " {[INTEGER,INTEGER]|[INTEGER,DOUBLE]|[DOUBLE,INTEGER]|[DOUBLE,DOUBLE]}, but got" - + " [DOUBLE,INTEGER,INTEGER]"); + // Mod with wrong argument no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | eval z = mod(0.5, 1, 2) | fields z"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(z=[MOD(0.5:DECIMAL(2, 1), 1, 2)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } // Test UDF registered with sql std operators: registerOperator(PI, SqlStdOperatorTable.PI) @Test public void testPiWithArg() { - verifyQueryThrowsException( - "source=EMP | eval pi = pi(1) | fields pi", "PI function expects {[]}, but got [INTEGER]"); + // Pi with Arg no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | eval pi = pi(1) | fields pi"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, "LogicalProject(pi=[PI(1)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"); } // Test UDF registered with sql library operators: registerOperator(LOG2, // SqlLibraryOperators.LOG2) @Test public void testLog2WithWrongArgShouldThrow() { - verifyQueryThrowsException( - "source=EMP | eval log2 = log2(ENAME, JOB) | fields log2", - "LOG2 function expects {[INTEGER]|[DOUBLE]}, but got [STRING,STRING]"); + // Log2 with wrong argument should throw no longer throws exception, Calcite handles type + // coercion + String ppl = "source=EMP | eval log2 = log2(ENAME, JOB) | fields log2"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(log2=[LOG2($1, $2)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"); } @Test @@ -198,13 +228,13 @@ public void testStrftimeWithCorrectTypes() { @Test public void testStrftimeWithWrongFirstArgType() { - // First argument should be numeric/timestamp, not boolean + // First argument should be numeric/timestamp, but Calcite handles type coercion String ppl = "source=EMP | eval formatted = strftime(EMPNO > 5, '%Y-%m-%d') | fields formatted"; - Throwable t = Assert.assertThrows(ExpressionEvaluationException.class, () -> getRelNode(ppl)); - verifyErrorMessageContains( - t, - "STRFTIME function expects {[INTEGER,STRING]|[DOUBLE,STRING]|[TIMESTAMP,STRING]}, but got" - + " [BOOLEAN,STRING]"); + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(formatted=[STRFTIME(>($0, 5), '%Y-%m-%d':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } @Test @@ -215,9 +245,10 @@ public void testStrftimeAcceptsDateInput() { "source=EMP | eval formatted = strftime(date('2020-09-16'), '%Y-%m-%d') | fields formatted"; RelNode relNode = getRelNode(ppl); assertNotNull(relNode); - // The plan should show TIMESTAMP(DATE(...)) indicating auto-conversion - String planString = relNode.explain(); - assertTrue(planString.contains("STRFTIME") && planString.contains("TIMESTAMP")); + verifyLogical( + relNode, + "LogicalProject(formatted=[STRFTIME(DATE('2020-09-16':VARCHAR), '%Y-%m-%d':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } @Test @@ -254,50 +285,57 @@ public void testStrftimeWithDateReturningFunctions() { @Test public void testStrftimeWithWrongSecondArgType() { - // Second argument should be string, not numeric + // Second argument should be string, but Calcite handles type coercion String ppl = "source=EMP | eval formatted = strftime(1521467703, 123) | fields formatted"; - Throwable t = Assert.assertThrows(ExpressionEvaluationException.class, () -> getRelNode(ppl)); - verifyErrorMessageContains( - t, - "STRFTIME function expects {[INTEGER,STRING]|[DOUBLE,STRING]|[TIMESTAMP,STRING]}, but got" - + " [INTEGER,INTEGER]"); + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(formatted=[STRFTIME(1521467703, 123)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } @Test public void testStrftimeWithWrongNumberOfArgs() { - // strftime requires exactly 2 arguments + // strftime now accepts variable arguments, so these no longer throw exceptions String ppl1 = "source=EMP | eval formatted = strftime(1521467703) | fields formatted"; - Throwable t1 = Assert.assertThrows(ExpressionEvaluationException.class, () -> getRelNode(ppl1)); - verifyErrorMessageContains( - t1, - "STRFTIME function expects {[INTEGER,STRING]|[DOUBLE,STRING]|[TIMESTAMP,STRING]}, but got" - + " [INTEGER]"); + RelNode root1 = getRelNode(ppl1); + verifyLogical( + root1, + "LogicalProject(formatted=[STRFTIME(1521467703)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); String ppl2 = "source=EMP | eval formatted = strftime(1521467703, '%Y', 'extra') | fields formatted"; - Throwable t2 = Assert.assertThrows(ExpressionEvaluationException.class, () -> getRelNode(ppl2)); - verifyErrorMessageContains( - t2, - "STRFTIME function expects {[INTEGER,STRING]|[DOUBLE,STRING]|[TIMESTAMP,STRING]}, but got" - + " [INTEGER,STRING,STRING]"); + RelNode root2 = getRelNode(ppl2); + verifyLogical( + root2, + "LogicalProject(formatted=[STRFTIME(1521467703, '%Y':VARCHAR, 'extra':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } // Test VALUES function with array expression (which is not a supported scalar type) @Test public void testValuesFunctionWithArrayArgType() { - verifyQueryThrowsException( - "source=EMP | stats values(array(ENAME, JOB)) as unique_values", - "Aggregation function VALUES expects field type" - + " {[BYTE]|[SHORT]|[INTEGER]|[LONG]|[FLOAT]|[DOUBLE]|[STRING]|[BOOLEAN]|[DATE]|[TIME]|[TIMESTAMP]|[IP]|[BINARY]|[BYTE,INTEGER]" - + "|[SHORT,INTEGER]|[INTEGER,INTEGER]|[LONG,INTEGER]|[FLOAT,INTEGER]|[DOUBLE,INTEGER]|[STRING,INTEGER]|[BOOLEAN,INTEGER]|[DATE,INTEGER]|[TIME,INTEGER]|[TIMESTAMP,INTEGER]|[IP,INTEGER]|[BINARY,INTEGER]}," - + " but got [ARRAY]"); + // ValuesFunction with ArrayArgType no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | stats values(array(ENAME, JOB)) as unique_values"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], unique_values=[VALUES($0)])\n" + + " LogicalProject($f2=[array($1, $2)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } // mvjoin should reject non-string single values @Test public void testMvjoinRejectsNonStringValues() { - verifyQueryThrowsException( - "source=EMP | eval result = mvjoin(42, ',') | fields result | head 1", - "MVJOIN function expects {[ARRAY,STRING]}, but got [INTEGER,STRING]"); + // Mvjoin rejects non-stringValues no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | eval result = mvjoin(42, ',') | fields result | head 1"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalSort(fetch=[1])\n" + + " LogicalProject(result=[ARRAY_JOIN(42, ',')])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java index a8b6873b55b..c78dab28b38 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java @@ -200,9 +200,7 @@ public void testLeftSemi() { "" + "SELECT *\n" + "FROM `scott`.`EMP`\n" - + "WHERE EXISTS (SELECT 1\n" - + "FROM `scott`.`DEPT`\n" - + "WHERE `EMP`.`DEPTNO` = `DEPT`.`DEPTNO`)"; + + "LEFT SEMI JOIN `scott`.`DEPT` ON `EMP`.`DEPTNO` = `DEPT`.`DEPTNO`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -222,9 +220,7 @@ public void testLeftAnti() { "" + "SELECT *\n" + "FROM `scott`.`EMP`\n" - + "WHERE NOT EXISTS (SELECT 1\n" - + "FROM `scott`.`DEPT`\n" - + "WHERE `EMP`.`DEPTNO` = `DEPT`.`DEPTNO`)"; + + "LEFT ANTI JOIN `scott`.`DEPT` ON `EMP`.`DEPTNO` = `DEPT`.`DEPTNO`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -823,9 +819,7 @@ public void testSemiJoinWithFieldList() { String expectedSparkSql = "SELECT *\n" + "FROM `scott`.`EMP`\n" - + "WHERE EXISTS (SELECT 1\n" - + "FROM `scott`.`DEPT`\n" - + "WHERE `EMP`.`DEPTNO` = `DEPT`.`DEPTNO`)"; + + "LEFT SEMI JOIN `scott`.`DEPT` ON `EMP`.`DEPTNO` = `DEPT`.`DEPTNO`"; verifyPPLToSparkSQL(root, expectedSparkSql); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMathFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMathFunctionTest.java index 58e29c49a28..3b57aaa656f 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMathFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMathFunctionTest.java @@ -200,10 +200,9 @@ public void testLn() { public void testLog() { RelNode root = getRelNode("source=EMP | eval LOG = log(2) | fields LOG"); String expectedLogical = - "LogicalProject(LOG=[LOG(2, 2.718281828459045E0:DOUBLE)])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n"; + "LogicalProject(LOG=[LOG(2)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); - String expectedSparkSql = "SELECT LOG(2, 2.718281828459045E0) `LOG`\nFROM `scott`.`EMP`"; + String expectedSparkSql = "SELECT LOG(2) `LOG`\nFROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMultisearchTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMultisearchTest.java index 7185c85aa1d..bede1be7b87 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMultisearchTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMultisearchTest.java @@ -290,9 +290,9 @@ public void testMultisearchWithTimestampFiltering() { "LogicalSort(sort0=[$3], dir0=[DESC-nulls-last])\n" + " LogicalSort(sort0=[$3], dir0=[DESC])\n" + " LogicalUnion(all=[true])\n" - + " LogicalFilter(condition=[>($3, TIMESTAMP('2025-07-31 23:00:00':VARCHAR))])\n" + + " LogicalFilter(condition=[>($3, '2025-07-31 23:00:00':VARCHAR)])\n" + " LogicalTableScan(table=[[scott, TIME_DATA1]])\n" - + " LogicalFilter(condition=[>($3, TIMESTAMP('2025-07-31 23:00:00':VARCHAR))])\n" + + " LogicalFilter(condition=[>($3, '2025-07-31 23:00:00':VARCHAR)])\n" + " LogicalTableScan(table=[[scott, TIME_DATA2]])\n"; verifyLogical(root, expectedLogical); @@ -301,11 +301,11 @@ public void testMultisearchWithTimestampFiltering() { + "FROM (SELECT `timestamp`, `value`, `category`, `@timestamp`\n" + "FROM (SELECT *\n" + "FROM `scott`.`TIME_DATA1`\n" - + "WHERE `@timestamp` > TIMESTAMP('2025-07-31 23:00:00')\n" + + "WHERE `@timestamp` > '2025-07-31 23:00:00'\n" + "UNION ALL\n" + "SELECT *\n" + "FROM `scott`.`TIME_DATA2`\n" - + "WHERE `@timestamp` > TIMESTAMP('2025-07-31 23:00:00'))\n" + + "WHERE `@timestamp` > '2025-07-31 23:00:00')\n" + "ORDER BY `@timestamp` DESC NULLS FIRST) `t2`\n" + "ORDER BY `@timestamp` DESC"; verifyPPLToSparkSQL(root, expectedSparkSql); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java index 2fcee849317..dbc630c3677 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java @@ -193,7 +193,9 @@ public void testRareUseNullFalse() { String expectedSparkSql = "SELECT `DEPTNO`, `JOB`, `count`\n" - + "FROM (SELECT `DEPTNO`, `JOB`, COUNT(*) `count`, ROW_NUMBER() OVER (PARTITION BY" + + "FROM (SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "`DEPTNO`, `JOB`, COUNT(*) `count`, ROW_NUMBER() OVER (PARTITION BY" + " `DEPTNO` ORDER BY COUNT(*) NULLS LAST) `_row_number_rare_top_`\n" + "FROM `scott`.`EMP`\n" + "WHERE `DEPTNO` IS NOT NULL AND `JOB` IS NOT NULL\n" @@ -396,7 +398,9 @@ public void testTopUseNullFalse() { String expectedSparkSql = "SELECT `DEPTNO`, `JOB`, `count`\n" - + "FROM (SELECT `DEPTNO`, `JOB`, COUNT(*) `count`, ROW_NUMBER() OVER (PARTITION BY" + + "FROM (SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "`DEPTNO`, `JOB`, COUNT(*) `count`, ROW_NUMBER() OVER (PARTITION BY" + " `DEPTNO` ORDER BY COUNT(*) DESC NULLS FIRST) `_row_number_rare_top_`\n" + "FROM `scott`.`EMP`\n" + "WHERE `DEPTNO` IS NOT NULL AND `JOB` IS NOT NULL\n" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStreamstatsTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStreamstatsTest.java index 48c0e5cfa62..f05d0d8ff35 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStreamstatsTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStreamstatsTest.java @@ -25,7 +25,7 @@ public void testStreamstatsBy() { + " LogicalSort(sort0=[$8], dir0=[ASC])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[$8], max(SAL)=[MAX($5) OVER" - + " (PARTITION BY $7 ROWS UNBOUNDED PRECEDING)])\n" + + " (PARTITION BY $7 ORDER BY $0 NULLS LAST ROWS UNBOUNDED PRECEDING)])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[ROW_NUMBER() OVER ()])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; @@ -33,8 +33,8 @@ public void testStreamstatsBy() { String expectedSparkSql = "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, MAX(`SAL`)" - + " OVER (PARTITION BY `DEPTNO` ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)" - + " `max(SAL)`\n" + + " OVER (PARTITION BY `DEPTNO` ORDER BY `EMPNO` NULLS LAST ROWS BETWEEN UNBOUNDED" + + " PRECEDING AND CURRENT ROW) `max(SAL)`\n" + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + " ROW_NUMBER() OVER () `__stream_seq__`\n" + "FROM `scott`.`EMP`) `t`\n" @@ -52,8 +52,8 @@ public void testStreamstatsByNullBucket() { + " LogicalSort(sort0=[$8], dir0=[ASC])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[$8], max(SAL)=[CASE(IS NOT" - + " NULL($7), MAX($5) OVER (PARTITION BY $7 ROWS UNBOUNDED PRECEDING), null:DECIMAL(7," - + " 2))])\n" + + " NULL($7), MAX($5) OVER (PARTITION BY $7 ORDER BY $0 NULLS LAST ROWS UNBOUNDED" + + " PRECEDING), null:DECIMAL(7, 2))])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[ROW_NUMBER() OVER ()])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; @@ -61,8 +61,9 @@ public void testStreamstatsByNullBucket() { String expectedSparkSql = "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, CASE WHEN" - + " `DEPTNO` IS NOT NULL THEN MAX(`SAL`) OVER (PARTITION BY `DEPTNO` ROWS BETWEEN" - + " UNBOUNDED PRECEDING AND CURRENT ROW) ELSE NULL END `max(SAL)`\n" + + " `DEPTNO` IS NOT NULL THEN MAX(`SAL`) OVER (PARTITION BY `DEPTNO` ORDER BY `EMPNO`" + + " NULLS LAST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) ELSE NULL END" + + " `max(SAL)`\n" + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + " ROW_NUMBER() OVER () `__stream_seq__`\n" + "FROM `scott`.`EMP`) `t`\n" @@ -76,14 +77,15 @@ public void testStreamstatsCurrent() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," - + " COMM=[$6], DEPTNO=[$7], max(SAL)=[MAX($5) OVER (ROWS BETWEEN UNBOUNDED PRECEDING" - + " AND 1 PRECEDING)])\n" + + " COMM=[$6], DEPTNO=[$7], max(SAL)=[MAX($5) OVER (ORDER BY $0 NULLS LAST ROWS BETWEEN" + + " UNBOUNDED PRECEDING AND 1 PRECEDING)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, MAX(`SAL`)" - + " OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) `max(SAL)`\n" + + " OVER (ORDER BY `EMPNO` NULLS LAST ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING)" + + " `max(SAL)`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -117,14 +119,14 @@ public void testStreamstatsWindow() { + " `$cor0`.`SAL`, `$cor0`.`COMM`, `$cor0`.`DEPTNO`, `t3`.`max(SAL)`\n" + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + " ROW_NUMBER() OVER () `__stream_seq__`\n" - + "FROM `scott`.`EMP`) `$cor0`,\n" - + "LATERAL (SELECT MAX(`SAL`) `max(SAL)`\n" + + "FROM `scott`.`EMP`) `$cor0`\n" + + "LEFT JOIN LATERAL (SELECT MAX(`SAL`) `max(SAL)`\n" + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + " ROW_NUMBER() OVER () `__stream_seq__`\n" + "FROM `scott`.`EMP`) `t0`\n" + "WHERE `__stream_seq__` >= `$cor0`.`__stream_seq__` - 4 AND `__stream_seq__` <=" + " `$cor0`.`__stream_seq__` AND (`DEPTNO` = `$cor0`.`DEPTNO` OR `DEPTNO` IS NULL AND" - + " `$cor0`.`DEPTNO` IS NULL)) `t3`\n" + + " `$cor0`.`DEPTNO` IS NULL)) `t3` ON TRUE\n" + "ORDER BY `$cor0`.`__stream_seq__` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -139,7 +141,7 @@ public void testStreamstatsGlobal() { + " LogicalSort(sort0=[$8], dir0=[ASC])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[$8], max(SAL)=[MAX($5) OVER" - + " (PARTITION BY $7 ROWS 4 PRECEDING)])\n" + + " (PARTITION BY $7 ORDER BY $0 NULLS LAST ROWS 4 PRECEDING)])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[ROW_NUMBER() OVER ()])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; @@ -147,7 +149,8 @@ public void testStreamstatsGlobal() { String expectedSparkSql = "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, MAX(`SAL`)" - + " OVER (PARTITION BY `DEPTNO` ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) `max(SAL)`\n" + + " OVER (PARTITION BY `DEPTNO` ORDER BY `EMPNO` NULLS LAST ROWS BETWEEN 4 PRECEDING" + + " AND CURRENT ROW) `max(SAL)`\n" + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + " ROW_NUMBER() OVER () `__stream_seq__`\n" + "FROM `scott`.`EMP`) `t`\n" @@ -204,8 +207,8 @@ public void testStreamstatsReset() { + " ROW_NUMBER() OVER () `__stream_seq__`, CASE WHEN `SAL` > 100 THEN 1 ELSE 0 END" + " `__reset_before_flag__`, CASE WHEN `SAL` < 50 THEN 1 ELSE 0 END" + " `__reset_after_flag__`\n" - + "FROM `scott`.`EMP`) `t`) `$cor0`,\n" - + "LATERAL (SELECT AVG(`SAL`) `avg(SAL)`\n" + + "FROM `scott`.`EMP`) `t`) `$cor0`\n" + + "LEFT JOIN LATERAL (SELECT AVG(`SAL`) `avg(SAL)`\n" + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + " `__stream_seq__`, `__reset_before_flag__`, `__reset_after_flag__`," + " (SUM(`__reset_before_flag__`) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT" @@ -218,7 +221,7 @@ public void testStreamstatsReset() { + "FROM `scott`.`EMP`) `t1`) `t2`\n" + "WHERE `__stream_seq__` <= `$cor0`.`__stream_seq__` AND `__seg_id__` =" + " `$cor0`.`__seg_id__` AND (`DEPTNO` = `$cor0`.`DEPTNO` OR `DEPTNO` IS NULL AND" - + " `$cor0`.`DEPTNO` IS NULL)) `t5`\n" + + " `$cor0`.`DEPTNO` IS NULL)) `t5` ON TRUE\n" + "ORDER BY `$cor0`.`__stream_seq__` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimechartTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimechartTest.java index ca0ff70f0b7..cb896f9fc4f 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimechartTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimechartTest.java @@ -75,7 +75,9 @@ public void testTimechartBasic() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT SPAN(`@timestamp`, 1, 'm') `@timestamp`, COUNT(*) `count()`\n" + "SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "SPAN(`@timestamp`, 1, 'm') `@timestamp`, COUNT(*) `count()`\n" + "FROM `scott`.`events`\n" + "WHERE `@timestamp` IS NOT NULL\n" + "GROUP BY SPAN(`@timestamp`, 1, 'm')\n" @@ -90,7 +92,9 @@ public void testTimechartPerSecond() { "SELECT `@timestamp`, DIVIDE(`per_second(cpu_usage)` * 1.0000E3," + " TIMESTAMPDIFF('MILLISECOND', `@timestamp`, TIMESTAMPADD('MINUTE', 1," + " `@timestamp`))) `per_second(cpu_usage)`\n" - + "FROM (SELECT SPAN(`@timestamp`, 1, 'm') `@timestamp`, SUM(`cpu_usage`)" + + "FROM (SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "SPAN(`@timestamp`, 1, 'm') `@timestamp`, SUM(`cpu_usage`)" + " `per_second(cpu_usage)`\n" + "FROM `scott`.`events`\n" + "WHERE `@timestamp` IS NOT NULL AND `cpu_usage` IS NOT NULL\n" @@ -105,7 +109,9 @@ public void testTimechartPerMinute() { "SELECT `@timestamp`, DIVIDE(`per_minute(cpu_usage)` * 6.00000E4," + " TIMESTAMPDIFF('MILLISECOND', `@timestamp`, TIMESTAMPADD('MINUTE', 1," + " `@timestamp`))) `per_minute(cpu_usage)`\n" - + "FROM (SELECT SPAN(`@timestamp`, 1, 'm') `@timestamp`, SUM(`cpu_usage`)" + + "FROM (SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "SPAN(`@timestamp`, 1, 'm') `@timestamp`, SUM(`cpu_usage`)" + " `per_minute(cpu_usage)`\n" + "FROM `scott`.`events`\n" + "WHERE `@timestamp` IS NOT NULL AND `cpu_usage` IS NOT NULL\n" @@ -120,7 +126,9 @@ public void testTimechartPerHour() { "SELECT `@timestamp`, DIVIDE(`per_hour(cpu_usage)` * 3.6000000E6," + " TIMESTAMPDIFF('MILLISECOND', `@timestamp`, TIMESTAMPADD('MINUTE', 1," + " `@timestamp`))) `per_hour(cpu_usage)`\n" - + "FROM (SELECT SPAN(`@timestamp`, 1, 'm') `@timestamp`, SUM(`cpu_usage`)" + + "FROM (SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "SPAN(`@timestamp`, 1, 'm') `@timestamp`, SUM(`cpu_usage`)" + " `per_hour(cpu_usage)`\n" + "FROM `scott`.`events`\n" + "WHERE `@timestamp` IS NOT NULL AND `cpu_usage` IS NOT NULL\n" @@ -135,7 +143,9 @@ public void testTimechartPerDay() { "SELECT `@timestamp`, DIVIDE(`per_day(cpu_usage)` * 8.64E7," + " TIMESTAMPDIFF('MILLISECOND', `@timestamp`, TIMESTAMPADD('MINUTE', 1," + " `@timestamp`))) `per_day(cpu_usage)`\n" - + "FROM (SELECT SPAN(`@timestamp`, 1, 'm') `@timestamp`, SUM(`cpu_usage`)" + + "FROM (SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "SPAN(`@timestamp`, 1, 'm') `@timestamp`, SUM(`cpu_usage`)" + " `per_day(cpu_usage)`\n" + "FROM `scott`.`events`\n" + "WHERE `@timestamp` IS NOT NULL AND `cpu_usage` IS NOT NULL\n" @@ -149,7 +159,9 @@ public void testTimechartWithSpan() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT SPAN(`@timestamp`, 1, 'h') `@timestamp`, COUNT(*) `count()`\n" + "SELECT\n" + + "/*+ `stats_args`(`bucket_nullable` = 'false') */\n" + + "SPAN(`@timestamp`, 1, 'h') `@timestamp`, COUNT(*) `count()`\n" + "FROM `scott`.`events`\n" + "WHERE `@timestamp` IS NOT NULL\n" + "GROUP BY SPAN(`@timestamp`, 1, 'h')\n"