From 68aadceaaeb152e6a9254accb872ea6dff39b0aa Mon Sep 17 00:00:00 2001 From: Kirill Tkalenko Date: Sun, 7 Jun 2026 17:18:23 +0300 Subject: [PATCH 01/12] [CALCITE-759] Wip --- core/src/main/codegen/templates/Parser.jj | 20 +++++++++++++-- .../adapter/enumerable/EnumerableLimit.java | 25 ++++++++++++++++++- .../enumerable/EnumerableLimitSort.java | 3 ++- .../calcite/rel/metadata/RelMdUtil.java | 2 +- .../calcite/rel/rules/MeasureRules.java | 4 +-- .../calcite/rel/rules/PruneEmptyRules.java | 4 +-- .../rel/rules/SortJoinTransposeRule.java | 4 +-- .../rel/rules/SortRemoveRedundantRule.java | 3 +++ .../org/apache/calcite/sql/SqlDialect.java | 13 +++++++++- .../sql/validate/SqlValidatorImpl.java | 9 +++++++ .../org/apache/calcite/tools/RelBuilder.java | 9 +------ .../org/apache/calcite/test/JdbcTest.java | 10 ++++++++ .../calcite/test/SqlToRelConverterTest.java | 6 +++++ .../calcite/test/SqlToRelConverterTest.xml | 12 +++++++++ .../calcite/sql/parser/SqlParserTest.java | 8 ++++++ 15 files changed, 110 insertions(+), 22 deletions(-) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 2f784b34125b..09778ba05062 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -690,7 +690,7 @@ SqlNode ExprOrJoinOrOrderedQuery(ExprContext exprContext) : * * 
  *    [ OFFSET start { ROW | ROWS } ]
- *    [ FETCH { FIRST | NEXT } [ count ] { ROW | ROWS } ONLY ]
+ * [ FETCH { FIRST | NEXT } [ count | (expression) ] { ROW | ROWS } ONLY ] *
*/ SqlNode OrderedQueryOrExpr(ExprContext exprContext) : @@ -777,10 +777,26 @@ void FetchClause(SqlNode[] offsetFetch) : { // SQL:2008-style syntax. "OFFSET ... FETCH ...". // If you specify both LIMIT and FETCH, FETCH wins. - ( | ) offsetFetch[1] = UnsignedNumericLiteralOrParam() + ( | ) offsetFetch[1] = FetchCount() ( | ) } +/** + * Parses the row count of a FETCH clause. Expressions must be parenthesized. + */ +SqlNode FetchCount() : +{ + final SqlNode e; +} +{ + ( + e = UnsignedNumericLiteralOrParam() + | + e = Expression(ExprContext.ACCEPT_NON_QUERY) + ) + { return e; } +} + /** * Parses a LIMIT clause in an ORDER BY expression. */ diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java index 3c046de9228d..336af2329e5c 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java @@ -111,7 +111,7 @@ public static EnumerableLimit create(final RelNode input, @Nullable RexNode offs v = builder.append("fetch", Expressions.call(v, BuiltInMethod.TAKE.method, - getExpression(fetch))); + getExpressionForFetch(fetch, implementor, builder))); } builder.add(Expressions.return_(null, v)); @@ -133,4 +133,27 @@ static Expression getExpression(RexNode rexNode) { return Expressions.constant(RexLiteral.intValue(rexNode)); } } + + static Expression getExpressionForFetch(RexNode rexNode, + EnumerableRelImplementor implementor, BlockBuilder builder) { + if (rexNode instanceof RexDynamicParam) { + final RexDynamicParam param = (RexDynamicParam) rexNode; + return Expressions.convert_( + Expressions.call(DataContext.ROOT, + BuiltInMethod.DATA_CONTEXT_GET.method, + Expressions.constant("?" + param.getIndex())), + Integer.class); + } else { + if (rexNode instanceof RexLiteral) { + // TODO: Enumerable runtime only supports INT types for FETCH and OFFSET, + // not BIGINT types. See CALCITE-7156. + return Expressions.constant(RexLiteral.intValue(rexNode)); + } + final Expression expression = + RexToLixTranslator.forAggregation(implementor.getTypeFactory(), + builder, null, implementor.getConformance()) + .translate(rexNode); + return Expressions.convert_(expression, int.class); + } + } } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java index defc28f69a53..0b03d7e5c87a 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java @@ -31,6 +31,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import static org.apache.calcite.adapter.enumerable.EnumerableLimit.getExpression; +import static org.apache.calcite.adapter.enumerable.EnumerableLimit.getExpressionForFetch; /** * Implementation of {@link org.apache.calcite.rel.core.Sort} in @@ -100,7 +101,7 @@ public static EnumerableLimitSort create( if (this.fetch == null) { fetchVal = Expressions.constant(Integer.MAX_VALUE); } else { - fetchVal = getExpression(this.fetch); + fetchVal = getExpressionForFetch(this.fetch, implementor, builder); } final Expression offsetVal; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java index 1f6502243626..e86ee2d5690d 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java @@ -1044,7 +1044,7 @@ private static boolean alreadySmaller(RelMetadataQuery mq, RelNode input, return true; } final Double rowCount = mq.getMaxRowCount(input); - if (rowCount == null || offset instanceof RexDynamicParam || fetch instanceof RexDynamicParam) { + if (rowCount == null || offset instanceof RexDynamicParam || !(fetch instanceof RexLiteral)) { // Cannot be determined return false; } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/MeasureRules.java b/core/src/main/java/org/apache/calcite/rel/rules/MeasureRules.java index 037a4d605459..f69810a14d42 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/MeasureRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/MeasureRules.java @@ -30,7 +30,6 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexCorrelVariable; import org.apache.calcite.rex.RexInputRef; -import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexUtil; @@ -508,8 +507,7 @@ protected ProjectSortMeasureRule(ProjectSortMeasureRuleConfig config) { relBuilder.push(sort.getInput()) .projectPlus(map.keySet()) - .sortLimit(sort.offset == null ? 0 : RexLiteral.numberValue(sort.offset), - sort.fetch == null ? -1 : RexLiteral.numberValue(sort.fetch), + .sortLimit(sort.offset, sort.fetch, sort.getSortExps()) .project(newProjects); call.transformTo(relBuilder.build()); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java index 5d70c5e0dec4..1739ef491f0a 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java @@ -41,7 +41,6 @@ import org.apache.calcite.rel.logical.LogicalValues; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexDynamicParam; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.RelBuilder; @@ -519,8 +518,7 @@ public interface SortFetchZeroRuleConfig extends PruneEmptyRule.Config { return new RemoveEmptySingleRule(this) { @Override public boolean matches(final RelOptRuleCall call) { Sort sort = call.rel(0); - return sort.fetch != null - && !(sort.fetch instanceof RexDynamicParam) + return sort.fetch instanceof RexLiteral && RexLiteral.bigDecimalValue(sort.fetch).equals(BigDecimal.ZERO); } }; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java index 4310d6d65576..0a5ce60e6063 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java @@ -105,9 +105,9 @@ public SortJoinTransposeRule(Class sortClass, final Sort sort = call.rel(0); final Join join = call.rel(1); - // Do nothing if SORT contains dynamic parameters in offset or fetch + // The pushed fetch is calculated from literal offset and fetch values. if (sort.offset instanceof RexDynamicParam - || sort.fetch instanceof RexDynamicParam) { + || !(sort.fetch instanceof RexLiteral)) { return false; } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java index 9bcf026fc656..08563cdcdb86 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java @@ -133,6 +133,9 @@ protected SortRemoveRedundantRule(final SortRemoveRedundantRule.Config config) { private static Optional getRowCountThreshold(Sort sort) { if (RelOptUtil.isLimit(sort)) { assert sort.fetch != null; + if (!(sort.fetch instanceof RexLiteral)) { + return Optional.empty(); + } final BigDecimal fetch = RexLiteral.bigDecimalValue(sort.fetch); // We don't need to deal with fetch is 0. diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java index 869976f0d42a..3a2a2598c063 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java @@ -1081,7 +1081,18 @@ protected static void unparseFetchUsingAnsi(SqlWriter writer, @Nullable SqlNode writer.startList(SqlWriter.FrameTypeEnum.FETCH); writer.keyword("FETCH"); writer.keyword("NEXT"); - fetch.unparse(writer, -1, -1); + if (fetch instanceof SqlLiteral + || fetch instanceof SqlDynamicParam) { + fetch.unparse(writer, -1, -1); + } else { + final SqlWriter.Frame expressionFrame = writer.startList("(", ")"); + if (fetch instanceof SqlCall) { + writer.getDialect().unparseCall(writer, (SqlCall) fetch, 0, 0); + } else { + fetch.unparse(writer, 0, 0); + } + writer.endList(expressionFrame); + } writer.keyword("ROWS"); writer.keyword("ONLY"); writer.endList(fetchFrame); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 370ef5ecc39a..39d72b2ea91b 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -1745,6 +1745,14 @@ private void handleOffsetFetch(@Nullable SqlNode offset, @Nullable SqlNode fetch } } + private void validateFetchExpression(@Nullable SqlNode fetch) { + if (fetch == null || fetch instanceof SqlLiteral || fetch instanceof SqlDynamicParam) { + return; + } + validateNoAggs(aggFinder, fetch, "FETCH"); + validateExpr(fetch, getEmptyScope()); + } + /** * Performs expression rewrites which are always used unconditionally. These * rewrites massage the expression tree into a standard form so that the @@ -4442,6 +4450,7 @@ protected void validateSelect( validateWindowClause(select); validateQualifyClause(select); handleOffsetFetch(select.getOffset(), select.getFetch()); + validateFetchExpression(select.getFetch()); // Validate the SELECT clause late, because a select item might // depend on the GROUP BY list, or the window function might reference diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index cf9ccecdefef..cea08e929b79 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -3797,8 +3797,7 @@ public RelBuilder sortLimit(Number offset, Number fetch, * * @param offsetNode RexLiteral means number of rows to skip is deterministic, * RexDynamicParam means number of rows to skip is dynamic. - * @param fetchNode RexLiteral means maximum number of rows to fetch is deterministic, - * RexDynamicParam mean maximum number is dynamic. + * @param fetchNode Maximum number of rows to fetch * @param nodes Sort expressions */ public RelBuilder sortLimit(@Nullable RexNode offsetNode, @Nullable RexNode fetchNode, @@ -3808,12 +3807,6 @@ public RelBuilder sortLimit(@Nullable RexNode offsetNode, @Nullable RexNode fetc throw new IllegalArgumentException("OFFSET node must be RexLiteral or RexDynamicParam"); } } - if (fetchNode != null) { - if (!(fetchNode instanceof RexLiteral || fetchNode instanceof RexDynamicParam)) { - throw new IllegalArgumentException("FETCH node must be RexLiteral or RexDynamicParam"); - } - } - final Registrar registrar = new Registrar(fields(), ImmutableList.of()); final List fieldCollations = registrar.registerFieldCollations(nodes); diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index 78098912edfc..e9cf50c85cf0 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -3765,6 +3765,16 @@ public void checkOrderBy(final boolean desc, + "store_id=4; grocery_sqft=16844\n"); } + /** Tests FETCH with a parenthesized expression. */ + @Test void testFetchExpression() { + CalciteAssert.that() + .query("select * from (values (1), (2), (3), (4)) as t(x)\n" + + "fetch next (1 + abs(-2)) rows only") + .returns("X=1\n" + + "X=2\n" + + "X=3\n"); + } + /** Tests ORDER BY ... OFFSET ... FETCH. */ @Test void testOrderByOffsetFetch() { CalciteAssert.that() diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 4e8d17cd7a27..c807e5881fa1 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -1250,6 +1250,12 @@ public static void checkActualAndReferenceFiles() { sql(sql).ok(); } + @Test void testFetchWithExpression() { + final String sql = + "select empno from emp fetch next (1 + abs(-2)) rows only"; + sql(sql).ok(); + } + /** Test case for * [CALCITE-439] * SqlValidatorUtil.uniquify() may not terminate under some conditions. */ diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 5ee7180efa75..fa39cbab96fd 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -2426,6 +2426,18 @@ LogicalSort(fetch=[5]) LogicalSort(fetch=[?0]) LogicalProject(EMPNO=[$0]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 4ab6f776ba4a..026132709616 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -4029,6 +4029,14 @@ void checkPeriodPredicate(Checker checker) { + "FROM `FOO`\n" + "OFFSET ? ROWS\n" + "FETCH NEXT ? ROWS ONLY"); + // Arithmetic and scalar expressions are allowed within parentheses. + sql("select a from foo fetch next (1 + abs(-2)) rows only") + .ok("SELECT `A`\n" + + "FROM `FOO`\n" + + "FETCH NEXT (1 + ABS(-2)) ROWS ONLY"); + // Expressions without parentheses are not allowed. + sql("select a from foo fetch next 1 ^+^ 2 rows only") + .fails("(?s).*Encountered \"\\+\" at .*"); // missing ROWS after FETCH sql("select a from foo offset 1 fetch next 3 ^only^") .fails("(?s).*Encountered \"only\" at .*"); From 143b9b6506e31cd1c9b0af9b2952ae6a00ef2718 Mon Sep 17 00:00:00 2001 From: Kirill Tkalenko Date: Mon, 8 Jun 2026 13:11:04 +0300 Subject: [PATCH 02/12] [CALCITE-7592] Wip --- .../adapter/cassandra/CassandraRules.java | 17 ++- .../calcite/test/CassandraAdapterTest.java | 16 +++ .../adapter/enumerable/EnumerableLimit.java | 86 ++++++++++-- .../enumerable/EnumerableLimitSort.java | 1 + .../enumerable/EnumerableMergeUnionRule.java | 10 +- .../apache/calcite/interpreter/SortNode.java | 38 +++++- .../calcite/rel/rules/PruneEmptyRules.java | 2 +- .../rel/rules/SortJoinTransposeRule.java | 2 +- .../rel/rules/SortUnionTransposeRule.java | 10 +- .../java/org/apache/calcite/rex/RexUtil.java | 15 +++ .../calcite/runtime/CalciteResource.java | 9 ++ .../sql/validate/SqlValidatorImpl.java | 22 +++- .../calcite/sql2rel/RelDecorrelator.java | 2 +- .../runtime/CalciteResource.properties | 3 + .../org/apache/calcite/test/JdbcTest.java | 123 ++++++++++++++++++ .../apache/calcite/test/RelOptRulesTest.java | 32 +++++ .../apache/calcite/test/SqlValidatorTest.java | 12 ++ .../enumerable/EnumerableMergeUnionTest.java | 24 ++++ .../apache/calcite/test/RelOptRulesTest.xml | 76 +++++++++++ .../calcite/adapter/druid/DruidRules.java | 11 +- .../apache/calcite/test/DruidAdapter2IT.java | 14 ++ .../elasticsearch/ElasticsearchRules.java | 12 +- .../ElasticSearchAdapterTest.java | 18 +++ .../calcite/adapter/geode/rel/GeodeRules.java | 10 +- .../adapter/geode/rel/GeodeBookstoreTest.java | 12 ++ .../calcite/adapter/mongodb/MongoRules.java | 13 +- .../adapter/mongodb/MongoAdapterTest.java | 24 ++++ .../org/apache/calcite/test/ServerTest.java | 32 +++++ 28 files changed, 611 insertions(+), 35 deletions(-) diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java index db5d83960d0c..619ad13988ec 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java @@ -35,6 +35,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.sql.SqlKind; @@ -405,16 +406,26 @@ protected CassandraLimitRule(CassandraLimitRuleConfig config) { super(config); } - public RelNode convert(EnumerableLimit limit) { + public @Nullable RelNode convert(EnumerableLimit limit) { + final RexLiteral fetch = + limit.fetch == null + ? null + : EnumerableLimit.reduceFetchToLiteral(limit.getCluster(), limit.fetch); + if (limit.fetch != null && fetch == null) { + return null; + } final RelTraitSet traitSet = limit.getTraitSet().replace(CassandraRel.CONVENTION); return new CassandraLimit(limit.getCluster(), traitSet, - convert(limit.getInput(), CassandraRel.CONVENTION), limit.offset, limit.fetch); + convert(limit.getInput(), CassandraRel.CONVENTION), limit.offset, fetch); } @Override public void onMatch(RelOptRuleCall call) { EnumerableLimit limit = call.rel(0); - call.transformTo(convert(limit)); + final RelNode converted = convert(limit); + if (converted != null) { + call.transformTo(converted); + } } /** Deprecated in favor of CassandraLimitRuleConfig. */ diff --git a/cassandra/src/test/java/org/apache/calcite/test/CassandraAdapterTest.java b/cassandra/src/test/java/org/apache/calcite/test/CassandraAdapterTest.java index 411ffdbb65be..cd096fc0acb6 100644 --- a/cassandra/src/test/java/org/apache/calcite/test/CassandraAdapterTest.java +++ b/cassandra/src/test/java/org/apache/calcite/test/CassandraAdapterTest.java @@ -127,6 +127,22 @@ static void load(CqlSession session) { .explainContains("CassandraLimit(fetch=[8])\n"); } + @Test void testFetchExpression() { + CalciteAssert.that() + .with(TWISSANDRA) + .query("select \"tweet_id\" from \"userline\" " + + "where \"username\" = '!PUBLIC!' " + + "fetch next (1 + abs(-2)) rows only") + .returnsCount(3) + .explainContains("CassandraLimit(fetch=[3])\n"); + CalciteAssert.that() + .with(TWISSANDRA) + .query("select \"tweet_id\" from \"userline\" " + + "where \"username\" = '!PUBLIC!' " + + "fetch next (0 - 1) rows only") + .throws_("FETCH value -1 is out of range"); + } + @Test void testSortLimit() { CalciteAssert.that() .with(TWISSANDRA) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java index 336af2329e5c..8d5446f873cf 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java @@ -31,12 +31,19 @@ import org.apache.calcite.rel.metadata.RelMdDistribution; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rex.RexDynamicParam; +import org.apache.calcite.rex.RexExecutor; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.util.BuiltInMethod; +import org.apache.calcite.util.NumberUtil; +import org.apache.calcite.util.Util; import org.checkerframework.checker.nullness.qual.Nullable; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** Relational expression that applies a limit and/or offset to its input. */ @@ -54,6 +61,7 @@ public EnumerableLimit( @Nullable RexNode offset, @Nullable RexNode fetch) { super(cluster, traitSet, input); + validateLiteralFetch(fetch); this.offset = offset; this.fetch = fetch; assert getConvention() instanceof EnumerableConvention; @@ -138,22 +146,78 @@ static Expression getExpressionForFetch(RexNode rexNode, EnumerableRelImplementor implementor, BlockBuilder builder) { if (rexNode instanceof RexDynamicParam) { final RexDynamicParam param = (RexDynamicParam) rexNode; - return Expressions.convert_( - Expressions.call(DataContext.ROOT, - BuiltInMethod.DATA_CONTEXT_GET.method, - Expressions.constant("?" + param.getIndex())), - Integer.class); + return Expressions.call(EnumerableLimit.class, "toIntFetch", + Expressions.convert_( + Expressions.call(DataContext.ROOT, + BuiltInMethod.DATA_CONTEXT_GET.method, + Expressions.constant("?" + param.getIndex())), + Number.class)); + } else if (rexNode instanceof RexLiteral) { + return Expressions.constant( + toIntFetch(((RexLiteral) rexNode).getValueAs(Number.class))); } else { - if (rexNode instanceof RexLiteral) { - // TODO: Enumerable runtime only supports INT types for FETCH and OFFSET, - // not BIGINT types. See CALCITE-7156. - return Expressions.constant(RexLiteral.intValue(rexNode)); - } final Expression expression = RexToLixTranslator.forAggregation(implementor.getTypeFactory(), builder, null, implementor.getConformance()) .translate(rexNode); - return Expressions.convert_(expression, int.class); + return Expressions.call(EnumerableLimit.class, "toIntFetch", + Expressions.convert_(Expressions.box(expression), Number.class)); + } + } + + /** Converts a FETCH expression result to the range supported by Enumerable. */ + public static int toIntFetch(@Nullable Number value) { + if (value == null) { + throw new IllegalArgumentException("FETCH expression evaluated to NULL"); + } + final BigDecimal decimal = NumberUtil.toBigDecimal(value); + if (decimal == null) { + throw new IllegalArgumentException("FETCH value is not numeric: " + value); + } + final int result; + try { + result = decimal.intValueExact(); + } catch (ArithmeticException e) { + throw new IllegalArgumentException("FETCH value " + value + + " is out of range; expected a value between 0 and " + + Integer.MAX_VALUE, e); + } + if (result < 0) { + throw new IllegalArgumentException("FETCH value " + value + + " is out of range; expected a value between 0 and " + + Integer.MAX_VALUE); + } + return result; + } + + static void validateLiteralFetch(@Nullable RexNode fetch) { + if (fetch instanceof RexLiteral) { + toIntFetch(((RexLiteral) fetch).getValueAs(Number.class)); + } + } + + /** Reduces a constant FETCH expression to a validated integer literal. */ + public static @Nullable RexLiteral reduceFetchToLiteral( + RelOptCluster cluster, RexNode fetch) { + final RexLiteral literal; + if (fetch instanceof RexLiteral) { + literal = (RexLiteral) fetch; + } else { + if (!RexUtil.isConstant(fetch) || !RexUtil.isDeterministic(fetch)) { + return null; + } + final RexExecutor executor = + Util.first(cluster.getPlanner().getExecutor(), RexUtil.EXECUTOR); + final List reducedValues = new ArrayList<>(1); + executor.reduce(cluster.getRexBuilder(), + Collections.singletonList(fetch), reducedValues); + final RexNode reduced = reducedValues.get(0); + if (!(reduced instanceof RexLiteral)) { + return null; + } + literal = (RexLiteral) reduced; } + final int value = toIntFetch(literal.getValueAs(Number.class)); + return cluster.getRexBuilder().makeExactLiteral(BigDecimal.valueOf(value)); } } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java index 0b03d7e5c87a..fbd8bb6d42db 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java @@ -53,6 +53,7 @@ public EnumerableLimitSort( @Nullable RexNode offset, @Nullable RexNode fetch) { super(cluster, traitSet, input, collation, offset, fetch); + EnumerableLimit.validateLiteralFetch(fetch); assert this.getConvention() instanceof EnumerableConvention; assert this.getConvention() == input.getConvention(); } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java index 7d47e639b78e..88b007fd6c47 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java @@ -29,6 +29,7 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableBitSet; @@ -88,9 +89,14 @@ public EnumerableMergeUnionRule(Config config) { // Push down sort limit, if possible. RexNode inputFetch = null; if (sort.fetch != null) { - if (sort.offset == null) { + final boolean safeToRepeat = + RexUtil.isDeterministic(sort.fetch) + && !RexUtil.containsDynamicParam(sort.fetch); + if (sort.offset == null && safeToRepeat) { inputFetch = sort.fetch; - } else if (sort.fetch instanceof RexLiteral && sort.offset instanceof RexLiteral) { + } else if (safeToRepeat + && sort.fetch instanceof RexLiteral + && sort.offset instanceof RexLiteral) { inputFetch = call.builder().literal(RexLiteral.bigDecimalValue(sort.fetch) .add(RexLiteral.bigDecimalValue(sort.offset))); diff --git a/core/src/main/java/org/apache/calcite/interpreter/SortNode.java b/core/src/main/java/org/apache/calcite/interpreter/SortNode.java index 71d9f2b22e42..7a1eef1cb4b8 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/SortNode.java +++ b/core/src/main/java/org/apache/calcite/interpreter/SortNode.java @@ -16,14 +16,18 @@ */ package org.apache.calcite.interpreter; +import org.apache.calcite.adapter.enumerable.EnumerableLimit; import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.Util; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Ordering; +import org.checkerframework.checker.nullness.qual.Nullable; + import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -35,8 +39,18 @@ * {@link org.apache.calcite.rel.core.Sort}. */ public class SortNode extends AbstractSingleNode { + private final @Nullable Scalar fetchScalar; + private final @Nullable Context fetchContext; + public SortNode(Compiler compiler, Sort rel) { super(compiler, rel); + if (rel.fetch != null && !(rel.fetch instanceof RexLiteral)) { + this.fetchScalar = compiler.compile(ImmutableList.of(rel.fetch), null); + this.fetchContext = compiler.createContext(); + } else { + this.fetchScalar = null; + this.fetchContext = null; + } } private static int getValueAsInt(RexNode node) { @@ -44,15 +58,31 @@ private static int getValueAsInt(RexNode node) { () -> "getValueAs(Integer.class) for " + node); } + private int getFetch() { + if (rel.fetch == null) { + return -1; + } + final @Nullable Number value; + if (rel.fetch instanceof RexLiteral) { + value = ((RexLiteral) rel.fetch).getValueAs(Number.class); + } else { + final Object result = + requireNonNull(fetchScalar, "fetchScalar") + .execute(requireNonNull(fetchContext, "fetchContext")); + if (result != null && !(result instanceof Number)) { + throw new IllegalArgumentException("FETCH value is not numeric: " + result); + } + value = (Number) result; + } + return EnumerableLimit.toIntFetch(value); + } + @Override public void run() throws InterruptedException { final int offset = rel.offset == null ? 0 : getValueAsInt(rel.offset); - final int fetch = - rel.fetch == null - ? -1 - : getValueAsInt(rel.fetch); + final int fetch = getFetch(); // In pure limit mode. No sort required. Row row; loop: diff --git a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java index 1739ef491f0a..5cb60bc90652 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java @@ -519,7 +519,7 @@ public interface SortFetchZeroRuleConfig extends PruneEmptyRule.Config { @Override public boolean matches(final RelOptRuleCall call) { Sort sort = call.rel(0); return sort.fetch instanceof RexLiteral - && RexLiteral.bigDecimalValue(sort.fetch).equals(BigDecimal.ZERO); + && BigDecimal.ZERO.equals(RexLiteral.bigDecimalValue(sort.fetch)); } }; } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java index 0a5ce60e6063..df967e56aad6 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java @@ -107,7 +107,7 @@ public SortJoinTransposeRule(Class sortClass, // The pushed fetch is calculated from literal offset and fetch values. if (sort.offset instanceof RexDynamicParam - || !(sort.fetch instanceof RexLiteral)) { + || sort.fetch != null && !(sort.fetch instanceof RexLiteral)) { return false; } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java index 416825ee926d..6d6e252075a8 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java @@ -23,7 +23,7 @@ import org.apache.calcite.rel.core.Union; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.rex.RexDynamicParam; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.tools.RelBuilderFactory; import org.immutables.value.Value; @@ -67,13 +67,15 @@ public SortUnionTransposeRule( @Override public boolean matches(RelOptRuleCall call) { final Sort sort = call.rel(0); final Union union = call.rel(1); - // We only apply this rule if Union.all is true, Sort.offset is null and Sort.fetch is not - // a dynamic param. + // Re-evaluating a non-deterministic or parameterized FETCH in every + // branch can produce a different limit from the top Sort. // There is a flag indicating if this rule should be applied when // Sort.fetch is null. return union.all && sort.offset == null - && !(sort.fetch instanceof RexDynamicParam) + && (sort.fetch == null + || RexUtil.isDeterministic(sort.fetch) + && !RexUtil.containsDynamicParam(sort.fetch)) && (config.matchNullFetch() || sort.fetch != null); } diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java index 3604e98dfd5b..0f9263916e22 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java +++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java @@ -840,6 +840,21 @@ public static boolean isDeterministic(RexNode e) { } } + /** Returns whether an expression contains a dynamic parameter. */ + public static boolean containsDynamicParam(RexNode node) { + try { + node.accept( + new RexVisitorImpl(true) { + @Override public Void visitDynamicParam(RexDynamicParam dynamicParam) { + throw Util.FoundOne.NULL; + } + }); + return false; + } catch (Util.FoundOne e) { + return true; + } + } + public static List retainDeterministic(List list) { List conjunctions = new ArrayList<>(); for (RexNode x : list) { diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index a9cb02d065cb..9d77b69cd9b3 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -164,6 +164,15 @@ ExInstWithCause validatorContext(int a0, int a1, @BaseMessage("Values passed to {0} operator must have compatible types") ExInst incompatibleValueType(String a0); + @BaseMessage("FETCH expression must have an integral numeric type; actual type is ''{0}''") + ExInst fetchExpressionMustBeIntegral(String type); + + @BaseMessage("FETCH expression cannot reference table column ''{0}''") + ExInst fetchExpressionCannotReferenceColumn(String column); + + @BaseMessage("FETCH expression evaluated to NULL") + ExInst fetchExpressionEvaluatedToNull(); + @BaseMessage("Values in expression list must have compatible types") ExInst incompatibleTypesInList(); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 39d72b2ea91b..1f6a71ab8e5e 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -1746,11 +1746,29 @@ private void handleOffsetFetch(@Nullable SqlNode offset, @Nullable SqlNode fetch } private void validateFetchExpression(@Nullable SqlNode fetch) { - if (fetch == null || fetch instanceof SqlLiteral || fetch instanceof SqlDynamicParam) { + if (fetch == null || fetch instanceof SqlDynamicParam) { return; } + if (SqlUtil.isNullLiteral(fetch, true)) { + throw newValidationError(fetch, + RESOURCE.fetchExpressionEvaluatedToNull()); + } validateNoAggs(aggFinder, fetch, "FETCH"); - validateExpr(fetch, getEmptyScope()); + fetch.accept(new SqlBasicVisitor() { + @Override public Void visit(SqlIdentifier id) { + throw newValidationError(id, + RESOURCE.fetchExpressionCannotReferenceColumn(id.toString())); + } + }); + final SqlValidatorScope scope = getEmptyScope(); + inferUnknownTypes(typeFactory.createSqlType(SqlTypeName.INTEGER), scope, fetch); + validateExpr(fetch, scope); + final RelDataType type = getValidatedNodeType(fetch); + if (!SqlTypeUtil.isIntType(type) + && !(SqlTypeUtil.isDecimal(type) && type.getScale() == 0)) { + throw newValidationError(fetch, + RESOURCE.fetchExpressionMustBeIntegral(type.getFullTypeString())); + } } /** diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index 14d72f8dfe69..9c78aa235581 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -1137,7 +1137,7 @@ private static void shiftMapping(Map mapping, int startIndex, } protected @Nullable Frame decorrelateSortAsAggregate(Sort sort, final Frame frame) { - if (sort.offset != null || sort.fetch == null) { + if (sort.offset != null || !(sort.fetch instanceof RexLiteral)) { return null; } diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 056aeb7b0715..6cacd96bb195 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -61,6 +61,9 @@ ValidatorContext=From line {0,number,#}, column {1,number,#} to line {2,number,# CannotCastValue=Cast function cannot convert value of type {0} to type {1} UnknownDatatypeName=Unknown datatype name ''{0}'' IncompatibleValueType=Values passed to {0} operator must have compatible types +FetchExpressionMustBeIntegral=FETCH expression must have an integral numeric type; actual type is ''{0}'' +FetchExpressionCannotReferenceColumn=FETCH expression cannot reference table column ''{0}'' +FetchExpressionEvaluatedToNull=FETCH expression evaluated to NULL IncompatibleTypesInList=Values in expression list must have compatible types IncompatibleCharset=Cannot apply operation ''{0}'' to strings with different charsets ''{1}'' and ''{2}'' InvalidOrderByPos=ORDER BY is only allowed on top-level SELECT diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index e9cf50c85cf0..91895e33db6e 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -3775,6 +3775,57 @@ public void checkOrderBy(final boolean desc, + "X=3\n"); } + /** Tests FETCH expressions in bindable/interpreter convention. */ + @Test void testBindableFetchExpression() { + try (Hook.Closeable ignored = Hook.ENABLE_BINDABLE.addThread(Hook.propertyJ(true))) { + CalciteAssert.that() + .query("select * from (values (1), (2), (3), (4)) as t(x)\n" + + "fetch next (rand_integer(1) + 2) rows only") + .explainContains("BindableSort(fetch=[+(RAND_INTEGER(1), 2)])") + .returns("X=1\n" + + "X=2\n"); + } + } + + /** Tests scalar functions with positive and negative arguments in FETCH. */ + @Test void testFetchExpressionFunctionArguments() { + final CalciteAssert.AssertThat with = CalciteAssert.that(); + final String values = "select * from (values (1), (2), (3)) as t(x)\n"; + with.query(values + "fetch next (abs(2)) rows only") + .returns("X=1\n" + + "X=2\n"); + with.query(values + "fetch next (abs(-2)) rows only") + .returns("X=1\n" + + "X=2\n"); + } + + /** Tests invalid runtime values produced by FETCH expressions. */ + @Test void testFetchExpressionInvalidValue() { + final CalciteAssert.AssertThat with = CalciteAssert.that(); + final String values = "select * from (values (1), (2), (3)) as t(x)\n"; + with.query(values + "fetch next (0 - 1) rows only") + .throws_("FETCH value -1 is out of range; expected a value between 0 and " + + Integer.MAX_VALUE); + with.query(values + "fetch next (-1) rows only") + .throws_("FETCH value -1 is out of range; expected a value between 0 and " + + Integer.MAX_VALUE); + with.query(values + + "fetch next (cast(3000000000 as bigint) + 1) rows only") + .throws_("FETCH value 3000000001 is out of range; " + + "expected a value between 0 and " + Integer.MAX_VALUE); + with.query(values + + "fetch next (cast(3000000001 as bigint)) rows only") + .throws_("FETCH value 3000000001 is out of range; " + + "expected a value between 0 and " + Integer.MAX_VALUE); + with.query(values + + "fetch next (abs(cast(3000000001 as bigint))) rows only") + .throws_("FETCH value 3000000001 is out of range; " + + "expected a value between 0 and " + Integer.MAX_VALUE); + with.query(values + + "fetch next (cast(null as integer)) rows only") + .throws_("FETCH expression evaluated to NULL"); + } + /** Tests ORDER BY ... OFFSET ... FETCH. */ @Test void testOrderByOffsetFetch() { CalciteAssert.that() @@ -5987,6 +6038,78 @@ private CalciteAssert.AssertQuery withEmpDept(String sql) { Matchers.returnsUnordered("name=Eric")); } + /** Tests dynamic parameters in parenthesized FETCH expressions. */ + @Test void testPreparedFetchExpression() throws Exception { + CalciteAssert.that() + .doWithConnection(connection -> { + final String values = + "select * from (values (1), (2), (3), (4)) as t(x)\n"; + checkPreparedFetch(connection, values + "fetch next (?) rows only", + 2, "X=1\nX=2\n"); + checkPreparedFetch(connection, values + "fetch next (? + 1) rows only", + 2, "X=1\nX=2\nX=3\n"); + checkPreparedFetch(connection, + values + "fetch next (abs(cast(? as integer))) rows only", + 2, "X=1\nX=2\n"); + checkPreparedFetch(connection, + values + "fetch next (abs(cast(? as integer))) rows only", + -2, "X=1\nX=2\n"); + + checkPreparedFetchFails(connection, + values + "fetch next (?) rows only", -1, + "FETCH value -1 is out of range"); + checkPreparedFetchFails(connection, + values + "fetch next (? + 1) rows only", -2, + "FETCH value -1 is out of range"); + checkPreparedFetchFails(connection, + values + "fetch next (abs(cast(? as bigint))) rows only", + 3_000_000_001L, + "FETCH value 3000000001 is out of range"); + }); + } + + /** Tests dynamic parameters in bindable/interpreter FETCH expressions. */ + @Test void testBindablePreparedFetchExpression() throws Exception { + try (Hook.Closeable ignored = Hook.ENABLE_BINDABLE.addThread(Hook.propertyJ(true))) { + CalciteAssert.that() + .doWithConnection(connection -> { + final String values = + "select * from (values (1), (2), (3), (4)) as t(x)\n"; + checkPreparedFetch(connection, + values + "fetch next (? + 1) rows only", + 2, "X=1\nX=2\nX=3\n"); + }); + } + } + + private static void checkPreparedFetch(Connection connection, String sql, + int value, String expected) { + try (PreparedStatement p = connection.prepareStatement(sql)) { + p.setInt(1, value); + try (ResultSet r = p.executeQuery()) { + assertThat(CalciteAssert.toString(r), is(expected)); + } + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + } + + private static void checkPreparedFetchFails(Connection connection, String sql, + long value, String expectedMessage) { + try (PreparedStatement p = connection.prepareStatement(sql)) { + if (value >= Integer.MIN_VALUE && value <= Integer.MAX_VALUE) { + p.setInt(1, (int) value); + } else { + p.setLong(1, value); + } + final SQLException e = + assertThrows(SQLException.class, p::executeQuery); + assertThat(e.getMessage(), containsString(expectedMessage)); + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + } + private void checkPreparedOffsetFetch(final int offset, final int fetch, final Matcher matcher) throws Exception { CalciteAssert.hr() diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 715d9f4f7b8e..054468508520 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -1680,6 +1680,28 @@ private void checkSemiOrAntiJoinProjectTranspose(JoinRelType type) { .check(); } + @Test void testSortUnionTransposeWithNonDeterministicFetch() { + final String sql = "select a.name from dept a\n" + + "union all\n" + + "select b.name from dept b\n" + + "order by name fetch next (rand_integer(10)) rows only"; + sql(sql) + .withPreRule(CoreRules.PROJECT_SET_OP_TRANSPOSE) + .withRule(CoreRules.SORT_UNION_TRANSPOSE) + .checkUnchanged(); + } + + @Test void testSortUnionTransposeWithParameterizedFetchExpression() { + final String sql = "select a.name from dept a\n" + + "union all\n" + + "select b.name from dept b\n" + + "order by name fetch next (? + 1) rows only"; + sql(sql) + .withPreRule(CoreRules.PROJECT_SET_OP_TRANSPOSE) + .withRule(CoreRules.SORT_UNION_TRANSPOSE) + .checkUnchanged(); + } + @Test void testSortRemovalAllKeysConstant() { final String sql = "select count(*) as c\n" + "from sales.emp\n" @@ -9617,6 +9639,16 @@ private void checkSemiJoinRuleOnAntiJoin(RelOptRule rule) { .check(); } + @Test void testDecorrelateProjectWithFetchExpression() { + final String query = "SELECT name, " + + "(SELECT sal FROM emp where dept.deptno = emp.deptno order by sal " + + "fetch next (1 + 0) rows only) " + + "FROM dept"; + sql(query).withRule(CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE) + .withLateDecorrelate(true) + .check(); + } + /** Test case for [CALCITE-7289] * Select NULL subquery throwing exception. */ @Test void testNullSelect() { diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 5cd74f62f21a..051344955594 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -9998,6 +9998,18 @@ void testGroupExpressionEquivalenceParams() { .rewritesTo(expected); } + @Test void testFetchExpressionType() { + sql("select name from dept fetch next (^upper('x')^) rows only") + .fails("FETCH expression must have an integral numeric type; " + + "actual type is 'CHAR\\(1\\) NOT NULL'"); + sql("select name from dept fetch next (^'x'^) rows only") + .fails("FETCH expression must have an integral numeric type; " + + "actual type is 'CHAR\\(1\\) NOT NULL'"); + sql("select name from dept fetch next (^1.5^) rows only") + .fails("FETCH expression must have an integral numeric type; " + + "actual type is 'DECIMAL\\(2, 1\\) NOT NULL'"); + } + @Test void testRewriteWithOffsetWithoutOrderBy() { final String sql = "select name from dept offset 2"; final String expected = "SELECT `NAME`\n" diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java index 68bb56cf366d..1e0a76bea1e3 100644 --- a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java +++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java @@ -78,6 +78,30 @@ class EnumerableMergeUnionTest { "empid=45; name=Pascal"); } + @Test void mergeUnionDoesNotPushNonDeterministicFetch() { + tester(false, + new HrSchemaBig(), + "select * from (select empid, name from emps " + + "union all select empid, name from emps) " + + "order by empid fetch next (rand_integer(10)) rows only") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "fetch=[RAND_INTEGER(10)])\n" + + " EnumerableMergeUnion(all=[true])\n" + + " EnumerableSort(sort0=[$0], dir0=[ASC])\n"); + } + + @Test void mergeUnionDoesNotPushParameterizedFetchExpression() { + tester(false, + new HrSchemaBig(), + "select * from (select empid, name from emps " + + "union all select empid, name from emps) " + + "order by empid fetch next (? + 1) rows only") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "fetch=[+(?0, 1)])\n" + + " EnumerableMergeUnion(all=[true])\n" + + " EnumerableSort(sort0=[$0], dir0=[ASC])\n"); + } + @Test void mergeUnionAllOrderByName() { tester(false, new HrSchemaBig(), diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index c782bb913cc1..81e46426be63 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -2755,6 +2755,46 @@ LogicalProject(NAME=[$1]) LogicalFilter(condition=[<=($3, 1)]) LogicalProject(SAL=[$5], EXPR$1=[EXTRACT(FLAG(YEAR), $4)], DEPTNO=[$7], rn=[ROW_NUMBER() OVER (PARTITION BY $7 ORDER BY EXTRACT(FLAG(YEAR), $4) NULLS LAST, $5 DESC NULLS FIRST)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + + + + @@ -19581,6 +19621,42 @@ LogicalSort(sort0=[$0], dir0=[ASC], fetch=[0]) LogicalSort(sort0=[$0], dir0=[ASC], fetch=[0]) LogicalProject(NAME=[$1]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + + + + + + + + + diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java index ff64d44ed2c1..e187f3040573 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.adapter.druid; +import org.apache.calcite.adapter.enumerable.EnumerableLimit; import org.apache.calcite.config.CalciteConnectionConfig; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptPredicateList; @@ -850,6 +851,13 @@ protected DruidSortRule(DruidSortRuleConfig config) { @Override public void onMatch(RelOptRuleCall call) { final Sort sort = call.rel(0); final DruidQuery query = call.rel(1); + final RexLiteral fetch = + sort.fetch == null + ? null + : EnumerableLimit.reduceFetchToLiteral(sort.getCluster(), sort.fetch); + if (sort.fetch != null && fetch == null) { + return; + } if (!DruidQuery.isValidSignature(query.signature() + 'l')) { return; } @@ -865,7 +873,8 @@ protected DruidSortRule(DruidSortRuleConfig config) { } final RelNode newSort = sort - .copy(sort.getTraitSet(), ImmutableList.of(Util.last(query.rels))); + .copy(sort.getTraitSet(), Util.last(query.rels), sort.getCollation(), + sort.offset, fetch); call.transformTo(DruidQuery.extendQuery(query, newSort)); } diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java index eec575e331d3..7f8f2c119c83 100644 --- a/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java +++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java @@ -967,6 +967,20 @@ private void checkGroupBySingleSortLimit(boolean approx) { .explainContains(explain); } + @Test void testFetchExpression() { + final String sql = "select \"state_province\"\n" + + "from \"foodmart\"\n" + + "fetch next (1 + abs(-2)) rows only"; + sql(sql) + .returnsCount(3) + .explainContains("DruidQuery(table=[[foodmart, foodmart]], " + + "intervals=[[1900-01-09T00:00:00.000Z/" + + "2992-01-10T00:00:00.000Z]], projects=[[$30]], fetch=[3])"); + sql("select \"state_province\" from \"foodmart\" " + + "fetch next (0 - 1) rows only") + .throws_("FETCH value -1 is out of range"); + } + /** Tests that distinct-count is pushed down to Druid and evaluated using * "cardinality". The result is approximate, but gives the correct result in * this example when rounded down using FLOOR. */ diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java index 3c44992b0a48..ebb370aa76cb 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.adapter.elasticsearch; +import org.apache.calcite.adapter.enumerable.EnumerableLimit; import org.apache.calcite.adapter.enumerable.RexImpTable; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.adapter.java.JavaTypeFactory; @@ -217,12 +218,19 @@ protected ElasticsearchSortRule(Config config) { super(config); } - @Override public RelNode convert(RelNode relNode) { + @Override public @Nullable RelNode convert(RelNode relNode) { final Sort sort = (Sort) relNode; + final RexLiteral fetch = + sort.fetch == null + ? null + : EnumerableLimit.reduceFetchToLiteral(sort.getCluster(), sort.fetch); + if (sort.fetch != null && fetch == null) { + return null; + } final RelTraitSet traitSet = sort.getTraitSet().replace(out).replace(sort.getCollation()); return new ElasticsearchSort(relNode.getCluster(), traitSet, convert(sort.getInput(), traitSet.replace(RelCollations.EMPTY)), sort.getCollation(), - sort.offset, sort.fetch); + sort.offset, fetch); } } diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java index b548d511540b..f4e6a78a51d4 100644 --- a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java +++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java @@ -571,6 +571,24 @@ private static Consumer sortedResultSetChecker(String column, ElasticsearchChecker.elasticsearchChecker( "'_source':['state','id']", "size:3")); + calciteAssert() + .query("select state, id from zips\n" + + "fetch next (0 - 1) rows only") + .throws_("FETCH value -1 is out of range"); + } + + @Test void testFetchExpression() { + final String sql = "select state, id from zips\n" + + "fetch next (1 + abs(-2)) rows only"; + + calciteAssert() + .query(sql) + .returnsCount(3) + .explainContains("ElasticsearchSort(fetch=[3])") + .queryContains( + ElasticsearchChecker.elasticsearchChecker( + "'_source':['state','id']", + "size:3")); } @Test void limit2() { diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java index b8beb854e0f6..611201557fc3 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.adapter.geode.rel; +import org.apache.calcite.adapter.enumerable.EnumerableLimit; import org.apache.calcite.plan.Convention; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; @@ -221,6 +222,13 @@ protected GeodeSortLimitRule(GeodeSortLimitRuleConfig config) { @Override public void onMatch(RelOptRuleCall call) { final Sort sort = call.rel(0); + final RexLiteral fetch = + sort.fetch == null + ? null + : EnumerableLimit.reduceFetchToLiteral(sort.getCluster(), sort.fetch); + if (sort.fetch != null && fetch == null) { + return; + } final RelTraitSet traitSet = sort.getTraitSet() .replace(GeodeRel.CONVENTION) @@ -229,7 +237,7 @@ protected GeodeSortLimitRule(GeodeSortLimitRuleConfig config) { GeodeSort geodeSort = new GeodeSort(sort.getCluster(), traitSet, convert(call.getPlanner(), sort.getInput(), traitSet.replace(RelCollations.EMPTY)), - sort.getCollation(), sort.fetch); + sort.getCollation(), fetch); call.transformTo(geodeSort); } diff --git a/geode/src/test/java/org/apache/calcite/adapter/geode/rel/GeodeBookstoreTest.java b/geode/src/test/java/org/apache/calcite/adapter/geode/rel/GeodeBookstoreTest.java index 3604be2e6a13..b51b7770cb99 100644 --- a/geode/src/test/java/org/apache/calcite/adapter/geode/rel/GeodeBookstoreTest.java +++ b/geode/src/test/java/org/apache/calcite/adapter/geode/rel/GeodeBookstoreTest.java @@ -356,6 +356,18 @@ private CalciteAssert.AssertThat calciteAssert() { + " GeodeTableScan(table=[[geode, BookCustomer]])\n"); } + @Test void testFetchExpression() { + calciteAssert() + .query("select * from geode.BookCustomer " + + "fetch next (1 + abs(-2)) rows only") + .returnsCount(3) + .explainContains("GeodeSort(fetch=[3])"); + calciteAssert() + .query("select * from geode.BookCustomer " + + "fetch next (0 - 1) rows only") + .throws_("FETCH value -1 is out of range"); + } + @Test void testSelectWithNestedPdx2() { calciteAssert() .query("select primaryAddress from geode.BookCustomer limit 2") diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java index 48e46e610d55..63d394c9404d 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.adapter.mongodb; +import org.apache.calcite.adapter.enumerable.EnumerableLimit; import org.apache.calcite.adapter.enumerable.RexImpTable; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.adapter.java.JavaTypeFactory; @@ -46,6 +47,7 @@ import org.apache.calcite.util.Util; import org.apache.calcite.util.trace.CalciteTrace; +import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import java.util.AbstractList; @@ -262,14 +264,21 @@ private static class MongoSortRule extends MongoConverterRule { super(config); } - @Override public RelNode convert(RelNode rel) { + @Override public @Nullable RelNode convert(RelNode rel) { final Sort sort = (Sort) rel; + final RexLiteral fetch = + sort.fetch == null + ? null + : EnumerableLimit.reduceFetchToLiteral(sort.getCluster(), sort.fetch); + if (sort.fetch != null && fetch == null) { + return null; + } final RelTraitSet traitSet = sort.getTraitSet().replace(out) .replace(sort.getCollation()); return new MongoSort(rel.getCluster(), traitSet, convert(sort.getInput(), traitSet.replace(RelCollations.EMPTY)), - sort.getCollation(), sort.offset, sort.fetch); + sort.getCollation(), sort.offset, fetch); } } diff --git a/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java b/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java index 7c79718e7984..30f01c94c47f 100644 --- a/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java +++ b/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java @@ -215,6 +215,30 @@ private CalciteAssert.AssertThat assertModel(URL url) { mongoChecker( "{$limit: 3}", "{$project: {STATE: '$state', ID: '$_id'}}")); + assertModel(MODEL) + .query("select state, id from zips\n" + + "fetch next (0 - 1) rows only") + .throws_("FETCH value -1 is out of range"); + assertModel(MODEL) + .query("select state, id from zips\n" + + "fetch next (cast(null as integer)) rows only") + .throws_("FETCH expression evaluated to NULL"); + assertModel(MODEL) + .query("select state, id from zips\n" + + "fetch next (cast(3000000001 as bigint)) rows only") + .throws_("FETCH value 3000000001 is out of range"); + } + + @Test void testFetchExpression() { + assertModel(MODEL) + .query("select state, id from zips\n" + + "fetch next (1 + abs(-2)) rows only") + .returnsCount(3) + .explainContains("MongoSort(fetch=[3])") + .queryContains( + mongoChecker( + "{$limit: 3}", + "{$project: {STATE: '$state', ID: '$_id'}}")); } @Test void testJoin() { diff --git a/server/src/test/java/org/apache/calcite/test/ServerTest.java b/server/src/test/java/org/apache/calcite/test/ServerTest.java index 40e1430c8799..aca66b1c882d 100644 --- a/server/src/test/java/org/apache/calcite/test/ServerTest.java +++ b/server/src/test/java/org/apache/calcite/test/ServerTest.java @@ -43,6 +43,7 @@ import java.math.BigDecimal; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -299,6 +300,37 @@ static Connection connect() throws SQLException { } } + /** Tests that FETCH cannot reference a column of its input table. */ + @Test void testFetchExpressionCannotReferenceInputColumn() throws Exception { + try (Connection c = connect(); + Statement s = c.createStatement()) { + s.execute("create table person (id int not null, name varchar(20))"); + try (PreparedStatement p = + c.prepareStatement("insert into person (id, name) values (?, ?)")) { + p.setInt(1, 1); + p.setString(2, "foo"); + assertThat(p.executeUpdate(), is(1)); + } + + SQLException e = assertThrows(SQLException.class, + () -> s.executeQuery("select * from person " + + "fetch next id rows only")); + assertThat(e.getMessage(), containsString("Encountered \"id\"")); + + e = assertThrows(SQLException.class, + () -> s.executeQuery("select * from person " + + "fetch next (id) rows only")); + assertThat(e.getMessage(), + containsString("FETCH expression cannot reference table column 'ID'")); + + e = assertThrows(SQLException.class, + () -> s.executeQuery("select * from person " + + "fetch next (1 + id) rows only")); + assertThat(e.getMessage(), + containsString("FETCH expression cannot reference table column 'ID'")); + } + } + /** Test case for * [CALCITE-6022] * Support "CREATE TABLE ... LIKE" DDL in server module. */ From f03a3b954d9e308ea67ccfb86fa3333225fee1ad Mon Sep 17 00:00:00 2001 From: Kirill Tkalenko Date: Mon, 8 Jun 2026 16:33:56 +0300 Subject: [PATCH 03/12] [CALCITE-7592] Wip --- .../adapter/enumerable/EnumerableLimit.java | 25 +++++- .../enumerable/EnumerableMergeUnionRule.java | 3 +- .../calcite/adapter/jdbc/JdbcRules.java | 17 +++- .../calcite/rel/metadata/RelMdUtil.java | 10 +++ .../rel/rules/SortUnionTransposeRule.java | 7 +- .../java/org/apache/calcite/rex/RexUtil.java | 15 ---- .../sql/validate/SqlValidatorImpl.java | 2 +- .../org/apache/calcite/tools/RelBuilder.java | 77 +++++++++++++++++++ .../apache/calcite/test/JdbcAdapterTest.java | 46 +++++++++++ .../org/apache/calcite/test/JdbcTest.java | 28 +++++++ .../apache/calcite/test/RelBuilderTest.java | 70 +++++++++++++++++ .../apache/calcite/test/RelOptRulesTest.java | 4 +- .../apache/calcite/test/SqlValidatorTest.java | 3 + .../enumerable/EnumerableMergeUnionTest.java | 8 +- .../apache/calcite/test/RelOptRulesTest.xml | 24 ++++-- 15 files changed, 301 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java index 8d5446f873cf..81a682ed955c 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java @@ -35,6 +35,7 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.util.BuiltInMethod; import org.apache.calcite.util.NumberUtil; import org.apache.calcite.util.Util; @@ -192,7 +193,11 @@ public static int toIntFetch(@Nullable Number value) { static void validateLiteralFetch(@Nullable RexNode fetch) { if (fetch instanceof RexLiteral) { - toIntFetch(((RexLiteral) fetch).getValueAs(Number.class)); + final Number value = ((RexLiteral) fetch).getValueAs(Number.class); + final BigDecimal decimal = NumberUtil.toBigDecimal(value); + if (decimal != null && decimal.signum() < 0) { + toIntFetch(value); + } } } @@ -203,7 +208,9 @@ static void validateLiteralFetch(@Nullable RexNode fetch) { if (fetch instanceof RexLiteral) { literal = (RexLiteral) fetch; } else { - if (!RexUtil.isConstant(fetch) || !RexUtil.isDeterministic(fetch)) { + if (!RexUtil.isConstant(fetch) + || !RexUtil.isDeterministic(fetch) + || containsDynamicParam(fetch)) { return null; } final RexExecutor executor = @@ -220,4 +227,18 @@ static void validateLiteralFetch(@Nullable RexNode fetch) { final int value = toIntFetch(literal.getValueAs(Number.class)); return cluster.getRexBuilder().makeExactLiteral(BigDecimal.valueOf(value)); } + + private static boolean containsDynamicParam(RexNode node) { + try { + node.accept( + new RexVisitorImpl(true) { + @Override public Void visitDynamicParam(RexDynamicParam dynamicParam) { + throw Util.FoundOne.NULL; + } + }); + return false; + } catch (Util.FoundOne e) { + return true; + } + } } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java index 88b007fd6c47..f0b574a43d61 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java @@ -90,8 +90,7 @@ public EnumerableMergeUnionRule(Config config) { RexNode inputFetch = null; if (sort.fetch != null) { final boolean safeToRepeat = - RexUtil.isDeterministic(sort.fetch) - && !RexUtil.containsDynamicParam(sort.fetch); + RexUtil.isDeterministic(sort.fetch); if (sort.offset == null && safeToRepeat) { inputFetch = sort.fetch; } else if (safeToRepeat diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java index 9a35cf7cce2d..960cec24414e 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.adapter.jdbc; +import org.apache.calcite.adapter.enumerable.EnumerableLimit; import org.apache.calcite.linq4j.Queryable; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.plan.Contexts; @@ -54,6 +55,7 @@ import org.apache.calcite.rel.rel2sql.SqlImplementor; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexDynamicParam; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexProgram; @@ -767,7 +769,18 @@ protected JdbcSortRule(Config config) { * JDBC convention * @return A new JdbcSort */ - public RelNode convert(Sort sort, boolean convertInputTraits) { + public @Nullable RelNode convert(Sort sort, boolean convertInputTraits) { + final RexNode fetch; + if (sort.fetch == null + || sort.fetch instanceof RexLiteral + || sort.fetch instanceof RexDynamicParam) { + fetch = sort.fetch; + } else { + fetch = EnumerableLimit.reduceFetchToLiteral(sort.getCluster(), sort.fetch); + if (fetch == null) { + return null; + } + } final RelTraitSet traitSet = sort.getTraitSet().replace(out); final RelNode input; @@ -779,7 +792,7 @@ public RelNode convert(Sort sort, boolean convertInputTraits) { } return new JdbcSort(sort.getCluster(), traitSet, - input, sort.getCollation(), sort.offset, sort.fetch); + input, sort.getCollation(), sort.offset, fetch); } } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java index e86ee2d5690d..ffd9fb8e7c0f 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java @@ -25,6 +25,7 @@ import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Minus; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.core.Union; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; @@ -56,6 +57,7 @@ import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; @@ -1043,6 +1045,14 @@ private static boolean alreadySmaller(RelMetadataQuery mq, RelNode input, if (fetch == null) { return true; } + final RelNode strippedInput = input.stripped(); + if (strippedInput instanceof Sort) { + final Sort sort = (Sort) strippedInput; + if (Objects.equals(offset, sort.offset) + && Objects.equals(fetch, sort.fetch)) { + return true; + } + } final Double rowCount = mq.getMaxRowCount(input); if (rowCount == null || offset instanceof RexDynamicParam || !(fetch instanceof RexLiteral)) { // Cannot be determined diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java index 6d6e252075a8..93b6af657c43 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java @@ -67,15 +67,14 @@ public SortUnionTransposeRule( @Override public boolean matches(RelOptRuleCall call) { final Sort sort = call.rel(0); final Union union = call.rel(1); - // Re-evaluating a non-deterministic or parameterized FETCH in every - // branch can produce a different limit from the top Sort. + // Re-evaluating a non-deterministic FETCH in every branch can produce a + // different limit from the top Sort. // There is a flag indicating if this rule should be applied when // Sort.fetch is null. return union.all && sort.offset == null && (sort.fetch == null - || RexUtil.isDeterministic(sort.fetch) - && !RexUtil.containsDynamicParam(sort.fetch)) + || RexUtil.isDeterministic(sort.fetch)) && (config.matchNullFetch() || sort.fetch != null); } diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java index 0f9263916e22..3604e98dfd5b 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java +++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java @@ -840,21 +840,6 @@ public static boolean isDeterministic(RexNode e) { } } - /** Returns whether an expression contains a dynamic parameter. */ - public static boolean containsDynamicParam(RexNode node) { - try { - node.accept( - new RexVisitorImpl(true) { - @Override public Void visitDynamicParam(RexDynamicParam dynamicParam) { - throw Util.FoundOne.NULL; - } - }); - return false; - } catch (Util.FoundOne e) { - return true; - } - } - public static List retainDeterministic(List list) { List conjunctions = new ArrayList<>(); for (RexNode x : list) { diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 1f6a71ab8e5e..0144294eb63c 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -1753,7 +1753,7 @@ private void validateFetchExpression(@Nullable SqlNode fetch) { throw newValidationError(fetch, RESOURCE.fetchExpressionEvaluatedToNull()); } - validateNoAggs(aggFinder, fetch, "FETCH"); + validateNoAggs(aggOrOverFinder, fetch, "FETCH"); fetch.accept(new SqlBasicVisitor() { @Override public Void visit(SqlIdentifier id) { throw newValidationError(id, diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index cea08e929b79..92d987188aec 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -79,13 +79,22 @@ import org.apache.calcite.rex.RexExecutor; import org.apache.calcite.rex.RexFieldCollation; import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLambda; +import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexLocalRef; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexNodeAndFieldIndex; +import org.apache.calcite.rex.RexOver; +import org.apache.calcite.rex.RexPatternFieldRef; +import org.apache.calcite.rex.RexRangeRef; import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexSimplify; import org.apache.calcite.rex.RexSubQuery; +import org.apache.calcite.rex.RexTableInputRef; import org.apache.calcite.rex.RexUnknownAs; import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.rex.RexWindowBound; import org.apache.calcite.rex.RexWindowBounds; import org.apache.calcite.rex.RexWindowExclusion; @@ -108,6 +117,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.sql.type.TableFunctionReturnTypeInference; import org.apache.calcite.sql.validate.SqlValidatorUtil; import org.apache.calcite.sql2rel.SqlToRelConverter; @@ -3807,6 +3817,18 @@ public RelBuilder sortLimit(@Nullable RexNode offsetNode, @Nullable RexNode fetc throw new IllegalArgumentException("OFFSET node must be RexLiteral or RexDynamicParam"); } } + if (fetchNode != null && isInvalidFetchExpression(fetchNode)) { + throw new IllegalArgumentException( + "FETCH node must not reference input fields or contain window functions or subqueries"); + } + if (fetchNode != null + && !SqlTypeUtil.isIntType(fetchNode.getType()) + && !(SqlTypeUtil.isDecimal(fetchNode.getType()) + && fetchNode.getType().getScale() == 0)) { + throw new IllegalArgumentException( + "FETCH node must have an integral numeric type; actual type is " + + fetchNode.getType().getFullTypeString()); + } final Registrar registrar = new Registrar(fields(), ImmutableList.of()); final List fieldCollations = registrar.registerFieldCollations(nodes); @@ -3873,6 +3895,61 @@ public RelBuilder sortLimit(@Nullable RexNode offsetNode, @Nullable RexNode fetc return this; } + private static boolean isInvalidFetchExpression(RexNode node) { + try { + node.accept( + new RexVisitorImpl(true) { + @Override public Void visitInputRef(RexInputRef inputRef) { + throw Util.FoundOne.NULL; + } + + @Override public Void visitLocalRef(RexLocalRef localRef) { + throw Util.FoundOne.NULL; + } + + @Override public Void visitTableInputRef(RexTableInputRef ref) { + throw Util.FoundOne.NULL; + } + + @Override public Void visitPatternFieldRef(RexPatternFieldRef fieldRef) { + throw Util.FoundOne.NULL; + } + + @Override public Void visitCorrelVariable(RexCorrelVariable correlVariable) { + throw Util.FoundOne.NULL; + } + + @Override public Void visitRangeRef(RexRangeRef rangeRef) { + throw Util.FoundOne.NULL; + } + + @Override public Void visitOver(RexOver over) { + throw Util.FoundOne.NULL; + } + + @Override public Void visitSubQuery(RexSubQuery subQuery) { + throw Util.FoundOne.NULL; + } + + @Override public Void visitNodeAndFieldIndex( + RexNodeAndFieldIndex nodeAndFieldIndex) { + throw Util.FoundOne.NULL; + } + + @Override public Void visitLambda(RexLambda lambda) { + throw Util.FoundOne.NULL; + } + + @Override public Void visitLambdaRef(RexLambdaRef lambdaRef) { + throw Util.FoundOne.NULL; + } + }); + return false; + } catch (Util.FoundOne e) { + return true; + } + } + private static RelFieldCollation collation(RexNode node, RelFieldCollation.Direction direction, RelFieldCollation.@Nullable NullDirection nullDirection, diff --git a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java index df12fbe8a8d1..990a4aed2d43 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java @@ -1386,6 +1386,52 @@ private LockWrapper exclusiveCleanDb(Connection c) throws SQLException { .planHasSql("SELECT \"EMPNO\", \"ENAME\"\nFROM \"SCOTT\".\"EMP\"\nWHERE \"EMPNO\" = ?"); } + @Test void testFetchExpressionPushDown() { + CalciteAssert.model(JdbcTest.SCOTT_MODEL) + .query("select empno from scott.emp " + + "fetch next (1 + abs(-2)) rows only") + .explainContains("JdbcSort(fetch=[3])") + .returnsCount(3); + } + + @Test void testParameterizedFetchExpressionIsNotPushedDown() { + CalciteAssert.model(JdbcTest.SCOTT_MODEL) + .query("select empno from scott.emp " + + "fetch next (? + 1) rows only") + .consumesPreparedStatement(p -> p.setInt(1, 2)) + .explainContains("EnumerableLimit(fetch=[+(?0, 1)])\n" + + " JdbcToEnumerableConverter\n") + .returnsCount(3); + } + + @Test void testParameterizedFetchExpressionRepeatedExecution() throws Exception { + CalciteAssert.model(JdbcTest.SCOTT_MODEL) + .doWithConnection(connection -> { + final String sql = "select empno from scott.emp order by empno " + + "fetch next (? + 1) rows only"; + try (PreparedStatement p = connection.prepareStatement(sql)) { + p.setInt(1, 0); + try (ResultSet resultSet = p.executeQuery()) { + assertThat(rowCount(resultSet), is(1)); + } + p.setInt(1, 2); + try (ResultSet resultSet = p.executeQuery()) { + assertThat(rowCount(resultSet), is(3)); + } + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + }); + } + + private static int rowCount(ResultSet resultSet) throws SQLException { + int count = 0; + while (resultSet.next()) { + count++; + } + return count; + } + /** * Test case for * [CALCITE-4619] diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index 91895e33db6e..ec7fb7d8e290 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -6054,6 +6054,15 @@ private CalciteAssert.AssertQuery withEmpDept(String sql) { checkPreparedFetch(connection, values + "fetch next (abs(cast(? as integer))) rows only", -2, "X=1\nX=2\n"); + checkPreparedFetchRepeated(connection, + values + "fetch next (?) rows only", + new int[] {1, 3}, + new String[] {"X=1\n", "X=1\nX=2\nX=3\n"}); + checkPreparedFetchRepeated(connection, + values + "fetch next (? + 1) rows only", + new int[] {0, 2, 3}, + new String[] {"X=1\n", "X=1\nX=2\nX=3\n", + "X=1\nX=2\nX=3\nX=4\n"}); checkPreparedFetchFails(connection, values + "fetch next (?) rows only", -1, @@ -6078,6 +6087,11 @@ private CalciteAssert.AssertQuery withEmpDept(String sql) { checkPreparedFetch(connection, values + "fetch next (? + 1) rows only", 2, "X=1\nX=2\nX=3\n"); + checkPreparedFetchRepeated(connection, + values + "fetch next (? + 1) rows only", + new int[] {0, 2, 3}, + new String[] {"X=1\n", "X=1\nX=2\nX=3\n", + "X=1\nX=2\nX=3\nX=4\n"}); }); } } @@ -6094,6 +6108,20 @@ private static void checkPreparedFetch(Connection connection, String sql, } } + private static void checkPreparedFetchRepeated(Connection connection, String sql, + int[] values, String[] expected) { + try (PreparedStatement p = connection.prepareStatement(sql)) { + for (int i = 0; i < values.length; i++) { + p.setInt(1, values[i]); + try (ResultSet r = p.executeQuery()) { + assertThat(CalciteAssert.toString(r), is(expected[i])); + } + } + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + } + private static void checkPreparedFetchFails(Connection connection, String sql, long value, String expectedMessage) { try (PreparedStatement p = connection.prepareStatement(sql)) { diff --git a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java index f1a4f1c27162..a127e47bfd84 100644 --- a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java @@ -52,7 +52,10 @@ import org.apache.calcite.rex.RexCorrelVariable; import org.apache.calcite.rex.RexFieldCollation; import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexNodeAndFieldIndex; +import org.apache.calcite.rex.RexSubQuery; import org.apache.calcite.rex.RexWindowBounds; import org.apache.calcite.runtime.CalciteException; import org.apache.calcite.schema.SchemaPlus; @@ -5227,6 +5230,73 @@ private static RelNode buildCorrelateWithJoin(JoinRelType type, RelBuilder build assertThat(mq.getMaxRowCount(planAfter), is(Double.POSITIVE_INFINITY)); } + @Test void testFetchExpressionCannotReferenceInputField() { + final RelBuilder builder = RelBuilder.create(config().build()); + builder.scan("DEPT"); + final RexNode field = builder.field("DEPTNO"); + + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, field, ImmutableList.of())); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, + builder.call(SqlStdOperatorTable.PLUS, builder.literal(1), field), + ImmutableList.of())); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, + new RexNodeAndFieldIndex(0, 0, "DEPTNO", field.getType()), + ImmutableList.of())); + } + + @Test void testFetchExpressionMustHaveIntegralType() { + final RelBuilder builder = RelBuilder.create(config().build()); + builder.scan("DEPT"); + + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, builder.literal("x"), ImmutableList.of())); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, builder.literal(new BigDecimal("1.5")), + ImmutableList.of())); + } + + @Test void testFetchExpressionCannotContainWindowOrSubQuery() { + final RelBuilder builder = RelBuilder.create(config().build()); + final RelDataType intType = + builder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); + builder.scan("DEPT"); + final RexNode over = + builder.getRexBuilder().makeOver(intType, + SqlStdOperatorTable.ROW_NUMBER, ImmutableList.of(), + ImmutableList.of(), ImmutableList.of(), + RexWindowBounds.UNBOUNDED_PRECEDING, + RexWindowBounds.UNBOUNDED_FOLLOWING, + true, true, false, false, false); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, over, ImmutableList.of())); + + final RelBuilder subQueryBuilder = RelBuilder.create(config().build()); + final RexNode subQuery = + RexSubQuery.scalar(subQueryBuilder.values(new String[] {"N"}, 1).build()); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, subQuery, ImmutableList.of())); + } + + @Test void testFetchExpressionCannotContainLambda() { + final RelBuilder builder = RelBuilder.create(config().build()); + final RelDataType intType = + builder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); + builder.scan("DEPT"); + final RexLambdaRef lambdaRef = new RexLambdaRef(0, "x", intType); + final RexNode lambda = + builder.getRexBuilder().makeLambdaCall( + builder.call(SqlStdOperatorTable.PLUS, lambdaRef, builder.literal(1)), + ImmutableList.of(lambdaRef)); + + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, lambda, ImmutableList.of())); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, lambdaRef, ImmutableList.of())); + } + @Test void testAdoptConventionEnumerable() { final RelBuilder builder = RelBuilder.create(config().build()); RelNode root = builder diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 054468508520..b08e04f4df09 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -1691,7 +1691,7 @@ private void checkSemiOrAntiJoinProjectTranspose(JoinRelType type) { .checkUnchanged(); } - @Test void testSortUnionTransposeWithParameterizedFetchExpression() { + @Test void testSortUnionTransposePushesParameterizedFetchExpression() { final String sql = "select a.name from dept a\n" + "union all\n" + "select b.name from dept b\n" @@ -1699,7 +1699,7 @@ private void checkSemiOrAntiJoinProjectTranspose(JoinRelType type) { sql(sql) .withPreRule(CoreRules.PROJECT_SET_OP_TRANSPOSE) .withRule(CoreRules.SORT_UNION_TRANSPOSE) - .checkUnchanged(); + .check(); } @Test void testSortRemovalAllKeysConstant() { diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 051344955594..2422fb3860bb 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -10008,6 +10008,9 @@ void testGroupExpressionEquivalenceParams() { sql("select name from dept fetch next (^1.5^) rows only") .fails("FETCH expression must have an integral numeric type; " + "actual type is 'DECIMAL\\(2, 1\\) NOT NULL'"); + sql("select name from dept " + + "fetch next (^row_number() over ()^) rows only") + .fails("Windowed aggregate expression is illegal in FETCH clause"); } @Test void testRewriteWithOffsetWithoutOrderBy() { diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java index 1e0a76bea1e3..23afb8c24628 100644 --- a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java +++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java @@ -90,16 +90,16 @@ class EnumerableMergeUnionTest { + " EnumerableSort(sort0=[$0], dir0=[ASC])\n"); } - @Test void mergeUnionDoesNotPushParameterizedFetchExpression() { + @Test void mergeUnionPushesParameterizedFetchExpression() { tester(false, new HrSchemaBig(), "select * from (select empid, name from emps " + "union all select empid, name from emps) " + "order by empid fetch next (? + 1) rows only") - .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " - + "fetch=[+(?0, 1)])\n" + .explainContains("EnumerableLimit(fetch=[+(?0, 1)])\n" + " EnumerableMergeUnion(all=[true])\n" - + " EnumerableSort(sort0=[$0], dir0=[ASC])\n"); + + " EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "fetch=[+(?0, 1)])\n"); } @Test void mergeUnionAllOrderByName() { diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 81e46426be63..c47dc48c781a 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -19624,34 +19624,46 @@ LogicalSort(sort0=[$0], dir0=[ASC], fetch=[0]) ]]> - + +order by name fetch next (? + 1) rows only]]> + + + - + +order by name fetch next (rand_integer(10)) rows only]]> Date: Mon, 8 Jun 2026 20:25:09 +0300 Subject: [PATCH 04/12] [CALCITE-7592] Wip --- .../rel/rel2sql/RelToSqlConverter.java | 40 ++++++++++++++++++- .../org/apache/calcite/sql/SqlDialect.java | 6 +++ .../rel/rel2sql/RelToSqlConverterTest.java | 19 +++++++++ .../apache/calcite/test/RelMetadataTest.java | 2 +- .../apache/calcite/test/RelOptRulesTest.java | 8 ++-- .../apache/calcite/test/RelOptRulesTest.xml | 20 +++++----- .../org/apache/calcite/test/ServerTest.java | 15 ++++--- 7 files changed, 87 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java index 0404fa7bc51c..a5b66040b8a0 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java @@ -54,11 +54,15 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexDynamicParam; +import org.apache.calcite.rex.RexExecutor; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexLocalRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexProgram; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.sql.JoinConditionType; import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SqlAsofJoin; @@ -1191,7 +1195,7 @@ public Result visit(Sort e) { sqlSelect.setOffset(offset); } if (e.fetch != null) { - SqlNode fetch = builder.context.toSql(null, e.fetch); + SqlNode fetch = toSqlFetch(e, builder.context); sqlSelect.setFetch(fetch); } return result(sqlSelect, ImmutableList.of(Clause.ORDER_BY), e, null); @@ -1249,13 +1253,45 @@ public Result visit(Sort e) { * The builder must have been created with OFFSET and FETCH clauses. */ void offsetFetch(Sort e, Builder builder) { if (e.fetch != null) { - builder.setFetch(builder.context.toSql(null, e.fetch)); + builder.setFetch(toSqlFetch(e, builder.context)); } if (e.offset != null) { builder.setOffset(builder.context.toSql(null, e.offset)); } } + private static SqlNode toSqlFetch(Sort sort, Context context) { + final RexNode fetch = requireNonNull(sort.fetch, "fetch"); + if (fetch instanceof RexLiteral + || fetch instanceof RexDynamicParam + || !RexUtil.isConstant(fetch) + || !RexUtil.isDeterministic(fetch) + || containsDynamicParam(fetch)) { + return context.toSql(null, fetch); + } + final RexExecutor executor = + Util.first(sort.getCluster().getPlanner().getExecutor(), RexUtil.EXECUTOR); + final List reducedValues = new ArrayList<>(1); + executor.reduce(sort.getCluster().getRexBuilder(), + Collections.singletonList(fetch), reducedValues); + final RexNode reduced = reducedValues.get(0); + return context.toSql(null, reduced instanceof RexLiteral ? reduced : fetch); + } + + private static boolean containsDynamicParam(RexNode node) { + try { + node.accept( + new RexVisitorImpl(true) { + @Override public Void visitDynamicParam(RexDynamicParam dynamicParam) { + throw Util.FoundOne.NULL; + } + }); + return false; + } catch (Util.FoundOne e) { + return true; + } + } + public boolean hasTrickyRollup(Sort e, Aggregate aggregate) { return !dialect.supportsAggregateFunction(SqlKind.ROLLUP) && dialect.supportsGroupByWithRollup() diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java index 3a2a2598c063..4d559bc29eb1 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java @@ -1109,6 +1109,12 @@ protected static void unparseFetchUsingLimit(SqlWriter writer, @Nullable SqlNode protected static void unparseLimit(SqlWriter writer, @Nullable SqlNode fetch) { if (fetch != null) { + if (!(fetch instanceof SqlLiteral) + && !(fetch instanceof SqlDynamicParam)) { + throw new IllegalArgumentException( + "LIMIT dialect does not support FETCH expressions that cannot " + + "be reduced to a literal"); + } writer.newlineAndIndent(); final SqlWriter.Frame fetchFrame = writer.startList(SqlWriter.FrameTypeEnum.FETCH); diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 66afda8653b2..b54e316a4759 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -4831,6 +4831,25 @@ private SqlDialect nonOrdinalDialect() { .withSybase().ok(expectedSybase); } + @Test void testFetchExpressionWithLimitDialect() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "fetch next (1 + 2) rows only"; + final String expected = "SELECT `product_id`\n" + + "FROM `foodmart`.`product`\n" + + "LIMIT 3"; + sql(query).withMysql().ok(expected); + } + + @Test void testParameterizedFetchExpressionWithLimitDialect() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "fetch next (? + 1) rows only"; + sql(query).withMysql().throws_( + "LIMIT dialect does not support FETCH expressions that cannot " + + "be reduced to a literal"); + } + @Test void testSelectQueryComplex() { String query = "select count(*), \"units_per_case\" from \"product\" where \"cases_per_pallet\" > 100 " diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java index f438a0ad9423..34f64b814a7b 100644 --- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java @@ -1467,7 +1467,7 @@ void testColumnOriginsUnion() { @Test void testRowCountSortLimitBeyondLong() { final BigDecimal fetch = BigDecimal.valueOf(Long.MAX_VALUE).add(BigDecimal.ONE); final double fetchDouble = fetch.doubleValue(); - final String sql = "select * from emp order by ename limit " + fetchDouble; + final String sql = "select * from emp order by ename limit " + fetch.toPlainString(); final RelMetadataFixture fixture = sql(sql); fixture.assertThatRowCount(is(EMP_SIZE), is(0D), is(fetchDouble)); } diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index b08e04f4df09..a00e39ebdfe0 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -5921,11 +5921,9 @@ private void checkEmptyJoin(RelOptFixture f) { .checkUnchanged(); } - /** Test case for - * [CALCITE-6647] - * SortUnionTransposeRule should not push SORT past a UNION when SORT's fetch is DynamicParam - . */ - @Test void testSortWithDynamicParam() { + /** Verifies that SortUnionTransposeRule pushes a deterministic dynamic FETCH + * past a UNION only once. */ + @Test void testSortWithDynamicParamPushesOnce() { HepProgramBuilder builder = new HepProgramBuilder(); builder.addRuleClass(SortProjectTransposeRule.class); builder.addRuleClass(SortUnionTransposeRule.class); diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index c47dc48c781a..64745a910f51 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -19672,7 +19672,7 @@ LogicalSort(sort0=[$0], dir0=[ASC], fetch=[RAND_INTEGER(10)]) ]]> - + diff --git a/server/src/test/java/org/apache/calcite/test/ServerTest.java b/server/src/test/java/org/apache/calcite/test/ServerTest.java index aca66b1c882d..edc4e308f5be 100644 --- a/server/src/test/java/org/apache/calcite/test/ServerTest.java +++ b/server/src/test/java/org/apache/calcite/test/ServerTest.java @@ -312,19 +312,22 @@ static Connection connect() throws SQLException { assertThat(p.executeUpdate(), is(1)); } - SQLException e = assertThrows(SQLException.class, - () -> s.executeQuery("select * from person " + SQLException e = + assertThrows( + SQLException.class, () -> s.executeQuery("select * from person " + "fetch next id rows only")); assertThat(e.getMessage(), containsString("Encountered \"id\"")); - e = assertThrows(SQLException.class, - () -> s.executeQuery("select * from person " + e = + assertThrows( + SQLException.class, () -> s.executeQuery("select * from person " + "fetch next (id) rows only")); assertThat(e.getMessage(), containsString("FETCH expression cannot reference table column 'ID'")); - e = assertThrows(SQLException.class, - () -> s.executeQuery("select * from person " + e = + assertThrows( + SQLException.class, () -> s.executeQuery("select * from person " + "fetch next (1 + id) rows only")); assertThat(e.getMessage(), containsString("FETCH expression cannot reference table column 'ID'")); From 139fdc34ec4a25aea3dc06f517327a66eb37c5f9 Mon Sep 17 00:00:00 2001 From: Kirill Tkalenko Date: Mon, 8 Jun 2026 21:23:10 +0300 Subject: [PATCH 05/12] [CALCITE-7592] Wip --- .../adapter/cassandra/CassandraRules.java | 2 +- .../adapter/enumerable/EnumerableLimit.java | 73 +++++++++++++++++-- .../rel/rel2sql/RelToSqlConverter.java | 1 + .../java/org/apache/calcite/rex/RexUtil.java | 19 +++++ .../org/apache/calcite/sql/SqlDialect.java | 17 ++++- .../calcite/sql/dialect/SqliteSqlDialect.java | 2 +- .../sql/validate/SqlValidatorImpl.java | 3 + .../org/apache/calcite/tools/RelBuilder.java | 10 ++- .../rel/rel2sql/RelToSqlConverterTest.java | 23 ++++++ .../apache/calcite/test/JdbcAdapterTest.java | 8 ++ .../apache/calcite/test/RelBuilderTest.java | 7 +- .../calcite/adapter/druid/DruidRules.java | 2 +- .../elasticsearch/ElasticsearchRules.java | 2 +- .../adapter/elasticsearch/Scrolling.java | 26 ++++++- .../ElasticSearchAdapterTest.java | 5 ++ .../calcite/adapter/geode/rel/GeodeRules.java | 2 +- .../adapter/mongodb/MongoAdapterTest.java | 18 ++++- 17 files changed, 200 insertions(+), 20 deletions(-) diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java index 619ad13988ec..5ec5ea798480 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java @@ -410,7 +410,7 @@ protected CassandraLimitRule(CassandraLimitRuleConfig config) { final RexLiteral fetch = limit.fetch == null ? null - : EnumerableLimit.reduceFetchToLiteral(limit.getCluster(), limit.fetch); + : EnumerableLimit.reduceFetchToIntLiteral(limit.getCluster(), limit.fetch); if (limit.fetch != null && fetch == null) { return null; } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java index 81a682ed955c..80c6cfffb6a7 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java @@ -168,13 +168,7 @@ static Expression getExpressionForFetch(RexNode rexNode, /** Converts a FETCH expression result to the range supported by Enumerable. */ public static int toIntFetch(@Nullable Number value) { - if (value == null) { - throw new IllegalArgumentException("FETCH expression evaluated to NULL"); - } - final BigDecimal decimal = NumberUtil.toBigDecimal(value); - if (decimal == null) { - throw new IllegalArgumentException("FETCH value is not numeric: " + value); - } + final BigDecimal decimal = validateFetchValue(value); final int result; try { result = decimal.intValueExact(); @@ -191,6 +185,42 @@ public static int toIntFetch(@Nullable Number value) { return result; } + /** Converts a FETCH expression result to the range supported by a long. */ + public static long toLongFetch(@Nullable Number value) { + final BigDecimal decimal = validateFetchValue(value); + final long result; + try { + result = decimal.longValueExact(); + } catch (ArithmeticException e) { + throw new IllegalArgumentException("FETCH value " + value + + " is out of range; expected a value between 0 and " + + Long.MAX_VALUE, e); + } + if (result < 0) { + throw new IllegalArgumentException("FETCH value " + value + + " is out of range; expected a value between 0 and " + + Long.MAX_VALUE); + } + return result; + } + + private static BigDecimal validateFetchValue(@Nullable Number value) { + if (value == null) { + throw new IllegalArgumentException("FETCH expression evaluated to NULL"); + } + final BigDecimal decimal = NumberUtil.toBigDecimal(value); + if (decimal == null) { + throw new IllegalArgumentException("FETCH value is not numeric: " + value); + } + try { + decimal.toBigIntegerExact(); + } catch (ArithmeticException e) { + throw new IllegalArgumentException("FETCH value " + value + + " is not an integer", e); + } + return decimal; + } + static void validateLiteralFetch(@Nullable RexNode fetch) { if (fetch instanceof RexLiteral) { final Number value = ((RexLiteral) fetch).getValueAs(Number.class); @@ -201,7 +231,7 @@ static void validateLiteralFetch(@Nullable RexNode fetch) { } } - /** Reduces a constant FETCH expression to a validated integer literal. */ + /** Reduces a constant FETCH expression to a validated literal. */ public static @Nullable RexLiteral reduceFetchToLiteral( RelOptCluster cluster, RexNode fetch) { final RexLiteral literal; @@ -210,6 +240,7 @@ static void validateLiteralFetch(@Nullable RexNode fetch) { } else { if (!RexUtil.isConstant(fetch) || !RexUtil.isDeterministic(fetch) + || RexUtil.containsDynamicFunction(fetch) || containsDynamicParam(fetch)) { return null; } @@ -224,10 +255,36 @@ static void validateLiteralFetch(@Nullable RexNode fetch) { } literal = (RexLiteral) reduced; } + final Number value = literal.getValueAs(Number.class); + if (validateFetchValue(value).signum() < 0) { + throw new IllegalArgumentException( + "FETCH value " + value + " is out of range; expected a non-negative value"); + } + return literal; + } + + /** Reduces a constant FETCH expression to a validated integer literal. */ + public static @Nullable RexLiteral reduceFetchToIntLiteral( + RelOptCluster cluster, RexNode fetch) { + final RexLiteral literal = reduceFetchToLiteral(cluster, fetch); + if (literal == null) { + return null; + } final int value = toIntFetch(literal.getValueAs(Number.class)); return cluster.getRexBuilder().makeExactLiteral(BigDecimal.valueOf(value)); } + /** Reduces a constant FETCH expression to a validated long literal. */ + public static @Nullable RexLiteral reduceFetchToLongLiteral( + RelOptCluster cluster, RexNode fetch) { + final RexLiteral literal = reduceFetchToLiteral(cluster, fetch); + if (literal == null) { + return null; + } + final long value = toLongFetch(literal.getValueAs(Number.class)); + return cluster.getRexBuilder().makeExactLiteral(BigDecimal.valueOf(value)); + } + private static boolean containsDynamicParam(RexNode node) { try { node.accept( diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java index a5b66040b8a0..e9f67b3502a8 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java @@ -1266,6 +1266,7 @@ private static SqlNode toSqlFetch(Sort sort, Context context) { || fetch instanceof RexDynamicParam || !RexUtil.isConstant(fetch) || !RexUtil.isDeterministic(fetch) + || RexUtil.containsDynamicFunction(fetch) || containsDynamicParam(fetch)) { return context.toSql(null, fetch); } diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java index 3604e98dfd5b..03719a2bffc1 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java +++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java @@ -840,6 +840,25 @@ public static boolean isDeterministic(RexNode e) { } } + /** Returns whether an expression contains a dynamic function. */ + public static boolean containsDynamicFunction(RexNode e) { + try { + e.accept( + new RexVisitorImpl(true) { + @Override public Void visitCall(RexCall call) { + if (call.getOperator().isDynamicFunction()) { + throw Util.FoundOne.NULL; + } + return super.visitCall(call); + } + }); + return false; + } catch (Util.FoundOne ex) { + Util.swallow(ex, null); + return true; + } + } + public static List retainDeterministic(List list) { List conjunctions = new ArrayList<>(); for (RexNode x : list) { diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java index 4d559bc29eb1..663aee63b0db 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java @@ -1102,14 +1102,27 @@ protected static void unparseFetchUsingAnsi(SqlWriter writer, @Nullable SqlNode /** Unparses offset/fetch using "LIMIT fetch OFFSET offset" syntax. */ protected static void unparseFetchUsingLimit(SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch) { + unparseFetchUsingLimit(writer, offset, fetch, false); + } + + /** Unparses offset/fetch using "LIMIT fetch OFFSET offset" syntax, + * optionally allowing a scalar expression as fetch. */ + protected static void unparseFetchUsingLimit(SqlWriter writer, @Nullable SqlNode offset, + @Nullable SqlNode fetch, boolean allowExpression) { checkArgument(fetch != null || offset != null); - unparseLimit(writer, fetch); + unparseLimit(writer, fetch, allowExpression); unparseOffset(writer, offset); } protected static void unparseLimit(SqlWriter writer, @Nullable SqlNode fetch) { + unparseLimit(writer, fetch, false); + } + + private static void unparseLimit(SqlWriter writer, @Nullable SqlNode fetch, + boolean allowExpression) { if (fetch != null) { - if (!(fetch instanceof SqlLiteral) + if (!allowExpression + && !(fetch instanceof SqlLiteral) && !(fetch instanceof SqlDynamicParam)) { throw new IllegalArgumentException( "LIMIT dialect does not support FETCH expressions that cannot " diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java index 82376ae576ab..f31276413600 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java @@ -90,7 +90,7 @@ public SqliteSqlDialect(SqlDialect.Context context) { @Override public void unparseOffsetFetch(SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch) { - unparseFetchUsingLimit(writer, offset, fetch); + unparseFetchUsingLimit(writer, offset, fetch, true); } @Override public void unparseCall(SqlWriter writer, SqlCall call, diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 0144294eb63c..c1143ff1c7c4 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -1756,6 +1756,9 @@ private void validateFetchExpression(@Nullable SqlNode fetch) { validateNoAggs(aggOrOverFinder, fetch, "FETCH"); fetch.accept(new SqlBasicVisitor() { @Override public Void visit(SqlIdentifier id) { + if (makeNullaryCall(id) != null) { + return null; + } throw newValidationError(id, RESOURCE.fetchExpressionCannotReferenceColumn(id.toString())); } diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 92d987188aec..48d96afe1c10 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -3819,7 +3819,8 @@ public RelBuilder sortLimit(@Nullable RexNode offsetNode, @Nullable RexNode fetc } if (fetchNode != null && isInvalidFetchExpression(fetchNode)) { throw new IllegalArgumentException( - "FETCH node must not reference input fields or contain window functions or subqueries"); + "FETCH node must not reference input fields or contain aggregate functions, " + + "window functions, or subqueries"); } if (fetchNode != null && !SqlTypeUtil.isIntType(fetchNode.getType()) @@ -3899,6 +3900,13 @@ private static boolean isInvalidFetchExpression(RexNode node) { try { node.accept( new RexVisitorImpl(true) { + @Override public Void visitCall(RexCall call) { + if (call.getOperator().isAggregator()) { + throw Util.FoundOne.NULL; + } + return super.visitCall(call); + } + @Override public Void visitInputRef(RexInputRef inputRef) { throw Util.FoundOne.NULL; } diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index b54e316a4759..bb667b62c775 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -4850,6 +4850,29 @@ private SqlDialect nonOrdinalDialect() { + "be reduced to a literal"); } + @Test void testParameterizedFetchExpressionWithSQLite() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "fetch next (? + 1) rows only"; + final String expected = "SELECT \"product_id\"\n" + + "FROM \"foodmart\".\"product\"\n" + + "LIMIT ? + 1"; + sql(query).withSQLite().ok(expected); + } + + @Test void testDynamicFetchExpressionIsNotReduced() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "fetch next (extract(day from current_date)) rows only"; + final String expected = "SELECT \"product_id\"\n" + + "FROM \"foodmart\".\"product\"\n" + + "FETCH NEXT (EXTRACT(DAY FROM CURRENT_DATE)) ROWS ONLY"; + sql(query).ok(expected); + sql(query).withMysql().throws_( + "LIMIT dialect does not support FETCH expressions that cannot " + + "be reduced to a literal"); + } + @Test void testSelectQueryComplex() { String query = "select count(*), \"units_per_case\" from \"product\" where \"cases_per_pallet\" > 100 " diff --git a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java index 990a4aed2d43..942fabb670f9 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java @@ -1404,6 +1404,14 @@ private LockWrapper exclusiveCleanDb(Connection c) throws SQLException { .returnsCount(3); } + @Test void testDynamicFetchExpressionIsNotPushedDown() { + CalciteAssert.model(JdbcTest.SCOTT_MODEL) + .query("select empno from scott.emp " + + "fetch next (extract(day from current_date)) rows only") + .explainContains("EnumerableLimit(fetch=[EXTRACT(FLAG(DAY), CURRENT_DATE)])\n" + + " JdbcToEnumerableConverter\n"); + } + @Test void testParameterizedFetchExpressionRepeatedExecution() throws Exception { CalciteAssert.model(JdbcTest.SCOTT_MODEL) .doWithConnection(connection -> { diff --git a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java index a127e47bfd84..20e4c9c97df6 100644 --- a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java @@ -5258,11 +5258,16 @@ private static RelNode buildCorrelateWithJoin(JoinRelType type, RelBuilder build ImmutableList.of())); } - @Test void testFetchExpressionCannotContainWindowOrSubQuery() { + @Test void testFetchExpressionCannotContainAggregateWindowOrSubQuery() { final RelBuilder builder = RelBuilder.create(config().build()); final RelDataType intType = builder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); builder.scan("DEPT"); + final RexNode aggregate = + builder.call(SqlStdOperatorTable.SUM, builder.literal(1)); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, aggregate, ImmutableList.of())); + final RexNode over = builder.getRexBuilder().makeOver(intType, SqlStdOperatorTable.ROW_NUMBER, ImmutableList.of(), diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java index e187f3040573..0c67379e5407 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java @@ -854,7 +854,7 @@ protected DruidSortRule(DruidSortRuleConfig config) { final RexLiteral fetch = sort.fetch == null ? null - : EnumerableLimit.reduceFetchToLiteral(sort.getCluster(), sort.fetch); + : EnumerableLimit.reduceFetchToIntLiteral(sort.getCluster(), sort.fetch); if (sort.fetch != null && fetch == null) { return; } diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java index ebb370aa76cb..9c47cd8b3852 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java @@ -223,7 +223,7 @@ protected ElasticsearchSortRule(Config config) { final RexLiteral fetch = sort.fetch == null ? null - : EnumerableLimit.reduceFetchToLiteral(sort.getCluster(), sort.fetch); + : EnumerableLimit.reduceFetchToLongLiteral(sort.getCluster(), sort.fetch); if (sort.fetch != null && fetch == null) { return null; } diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/Scrolling.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/Scrolling.java index 64b1d34681df..2e2561c4349f 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/Scrolling.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/Scrolling.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.Iterator; +import java.util.NoSuchElementException; import java.util.function.Consumer; import static com.google.common.base.Preconditions.checkArgument; @@ -72,12 +73,35 @@ Iterator query(ObjectNode query) { Iterator result = flatten(iterator); // apply limit if (limit != Long.MAX_VALUE) { - result = Iterators.limit(result, (int) limit); + result = limit(result, limit); } return result; } + private static Iterator limit(Iterator iterator, long limit) { + checkArgument(limit >= 0, "limit: %s >= 0", limit); + return new Iterator() { + private long remaining = limit; + + @Override public boolean hasNext() { + return remaining > 0 && iterator.hasNext(); + } + + @Override public E next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + --remaining; + return iterator.next(); + } + + @Override public void remove() { + iterator.remove(); + } + }; + } + /** * Combines lazily multiple {@link ElasticsearchJson.Result} into a single iterator of * {@link ElasticsearchJson.SearchHit}. diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java index f4e6a78a51d4..c0d43105b6c3 100644 --- a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java +++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java @@ -589,6 +589,11 @@ private static Consumer sortedResultSetChecker(String column, ElasticsearchChecker.elasticsearchChecker( "'_source':['state','id']", "size:3")); + calciteAssert() + .query("select state, id from zips\n" + + "fetch next (cast(3000000000 as bigint) + 1) rows only") + .runs() + .explainContains("ElasticsearchSort(fetch=[3000000001:BIGINT])"); } @Test void limit2() { diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java index 611201557fc3..13a29a9e3b5d 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java @@ -225,7 +225,7 @@ protected GeodeSortLimitRule(GeodeSortLimitRuleConfig config) { final RexLiteral fetch = sort.fetch == null ? null - : EnumerableLimit.reduceFetchToLiteral(sort.getCluster(), sort.fetch); + : EnumerableLimit.reduceFetchToLongLiteral(sort.getCluster(), sort.fetch); if (sort.fetch != null && fetch == null) { return; } diff --git a/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java b/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java index 30f01c94c47f..e59a17fc5ac0 100644 --- a/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java +++ b/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java @@ -225,8 +225,13 @@ private CalciteAssert.AssertThat assertModel(URL url) { .throws_("FETCH expression evaluated to NULL"); assertModel(MODEL) .query("select state, id from zips\n" - + "fetch next (cast(3000000001 as bigint)) rows only") - .throws_("FETCH value 3000000001 is out of range"); + + "fetch next 3000000001 rows only") + .runs() + .explainContains("MongoSort(fetch=[3000000001:BIGINT])") + .queryContains( + mongoChecker( + "{$project: {STATE: '$state', ID: '$_id'}}", + "{$limit: 3000000001}")); } @Test void testFetchExpression() { @@ -239,6 +244,15 @@ private CalciteAssert.AssertThat assertModel(URL url) { mongoChecker( "{$limit: 3}", "{$project: {STATE: '$state', ID: '$_id'}}")); + assertModel(MODEL) + .query("select state, id from zips\n" + + "fetch next (cast(3000000000 as bigint) + 1) rows only") + .runs() + .explainContains("MongoSort(fetch=[3000000001:BIGINT])") + .queryContains( + mongoChecker( + "{$project: {STATE: '$state', ID: '$_id'}}", + "{$limit: 3000000001}")); } @Test void testJoin() { From c499982034385deea3b43f5f4135c68cfcb487b5 Mon Sep 17 00:00:00 2001 From: Kirill Tkalenko Date: Tue, 9 Jun 2026 09:50:14 +0300 Subject: [PATCH 06/12] [CALCITE-7592] Add Quidem test --- core/src/test/resources/sql/fetch.iq | 160 +++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 core/src/test/resources/sql/fetch.iq diff --git a/core/src/test/resources/sql/fetch.iq b/core/src/test/resources/sql/fetch.iq new file mode 100644 index 000000000000..b2dfbcd22a3f --- /dev/null +++ b/core/src/test/resources/sql/fetch.iq @@ -0,0 +1,160 @@ +# fetch.iq +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +!use post +!set outputformat mysql + +# FETCH accepts a parenthesized arithmetic expression. +select * +from (values (1), (2), (3), (4)) as t(x) +fetch next (1 + abs(-2)) rows only; ++---+ +| X | ++---+ +| 1 | +| 2 | +| 3 | ++---+ +(3 rows) + +!ok + +# FETCH accepts a parenthesized scalar expression. +select * +from (values (1), (2), (3), (4)) as t(x) +fetch next (abs(2)) rows only; ++---+ +| X | ++---+ +| 1 | +| 2 | ++---+ +(2 rows) + +!ok + +# FETCH expression cannot be negative. +select * +from (values (1), (2), (3)) as t(x) +fetch next (0 - 1) rows only; +FETCH value -1 is out of range +!error + +# FETCH expression cannot evaluate to NULL. +select * +from (values (1), (2), (3)) as t(x) +fetch next (cast(null as integer)) rows only; +FETCH expression evaluated to NULL +!error + +# FETCH expression must have an integral numeric type. +select * +from (values (1), (2), (3)) as t(x) +fetch next (1.5) rows only; +FETCH expression must have an integral numeric type +!error + +# FETCH expression cannot reference input columns. +select * +from (values (1), (2), (3)) as t(x) +fetch next (x) rows only; +FETCH expression cannot reference table column 'X' +!error + +# Expressions without parentheses are not allowed in FETCH. +select * +from (values (1), (2), (3)) as t(x) +fetch next 1 + 2 rows only; +Encountered "+" +!error + +# FETCH expression works with a table source. +select deptno, dname +from dept +order by deptno +fetch next (1 + 1) rows only; ++--------+-------------+ +| DEPTNO | DNAME | ++--------+-------------+ +| 10 | Sales | +| 20 | Marketing | ++--------+-------------+ +(2 rows) + +!ok + +# FETCH expression works together with OFFSET on a table source. +select deptno, dname +from dept +order by deptno +offset 1 rows +fetch next (1 + 1) rows only; ++--------+-------------+ +| DEPTNO | DNAME | ++--------+-------------+ +| 20 | Marketing | +| 30 | Engineering | ++--------+-------------+ +(2 rows) + +!ok + +# FETCH expression may contain a scalar function on a table source. +select deptno +from dept +order by deptno +fetch next (abs(-3)) rows only; ++--------+ +| DEPTNO | ++--------+ +| 10 | +| 20 | +| 30 | ++--------+ +(3 rows) + +!ok + +# FETCH expression cannot reference columns of a table source. +select deptno, dname +from dept +order by deptno +fetch next (deptno) rows only; +FETCH expression cannot reference table column 'DEPTNO' +!error + +# FETCH expression cannot reference columns even inside a larger expression. +select deptno, dname +from dept +order by deptno +fetch next (deptno + 1) rows only; +FETCH expression cannot reference table column 'DEPTNO' +!error + +# FETCH expression may be zero on a table source. +select deptno +from dept +order by deptno +fetch next (2 - 2) rows only; ++--------+ +| DEPTNO | ++--------+ ++--------+ +(0 rows) + +!ok From e83ec8ed36e1ff50d30265d611b16036fa7f02c2 Mon Sep 17 00:00:00 2001 From: Kirill Tkalenko Date: Mon, 15 Jun 2026 18:45:14 +0300 Subject: [PATCH 07/12] [CALCITE-7592] Add tests by review --- .../test/java/org/apache/calcite/test/JdbcTest.java | 3 +++ .../org/apache/calcite/sql/parser/SqlParserTest.java | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index ec7fb7d8e290..00a79286d855 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -6063,6 +6063,9 @@ private CalciteAssert.AssertQuery withEmpDept(String sql) { new int[] {0, 2, 3}, new String[] {"X=1\n", "X=1\nX=2\nX=3\n", "X=1\nX=2\nX=3\nX=4\n"}); + checkPreparedFetch(connection, + values + "fetch next (? + abs(2)) rows only", + 1, "X=1\nX=2\nX=3\n"); checkPreparedFetchFails(connection, values + "fetch next (?) rows only", -1, diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 026132709616..8e1c1cb31bce 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -4037,12 +4037,23 @@ void checkPeriodPredicate(Checker checker) { // Expressions without parentheses are not allowed. sql("select a from foo fetch next 1 ^+^ 2 rows only") .fails("(?s).*Encountered \"\\+\" at .*"); + sql("select a from foo fetch next ? ^+^ abs(2) rows only") + .fails("(?s).*Encountered \"\\+\" at .*"); // missing ROWS after FETCH sql("select a from foo offset 1 fetch next 3 ^only^") .fails("(?s).*Encountered \"only\" at .*"); // FETCH before OFFSET is illegal sql("select a from foo fetch next 3 rows only ^offset^ 1") .fails("(?s).*Encountered \"offset\" at .*"); + // Subqueries are not allowed in FETCH + sql("select a from foo fetch next ^select^ 2 rows only") + .fails("(?s).*Encountered \"select\" at .*"); + sql("select a from foo fetch next (^select^ 2) rows only") + .fails("(?s).*Encountered \"select\" at .*"); + sql("select a from foo fetch next (^select^ ?) rows only") + .fails("(?s).*Encountered \"select\" at .*"); + sql("select a from foo fetch next (^select^ max(a) from foo) rows only") + .fails("(?s).*Encountered \"select\" at .*"); } /** From 2739aa21bb230775e5666199e057204d5223f0cc Mon Sep 17 00:00:00 2001 From: Kirill Tkalenko Date: Mon, 15 Jun 2026 18:57:31 +0300 Subject: [PATCH 08/12] [CALCITE-7592] Little refactor for check dinamic parameters --- .../adapter/enumerable/EnumerableLimit.java | 16 +--------------- .../calcite/rel/rel2sql/RelToSqlConverter.java | 17 +---------------- .../java/org/apache/calcite/rex/RexUtil.java | 16 ++++++++++++++++ .../org/apache/calcite/rex/RexProgramTest.java | 12 ++++++++++++ 4 files changed, 30 insertions(+), 31 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java index 80c6cfffb6a7..5d5bb4548ac4 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java @@ -35,7 +35,6 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; -import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.util.BuiltInMethod; import org.apache.calcite.util.NumberUtil; import org.apache.calcite.util.Util; @@ -241,7 +240,7 @@ static void validateLiteralFetch(@Nullable RexNode fetch) { if (!RexUtil.isConstant(fetch) || !RexUtil.isDeterministic(fetch) || RexUtil.containsDynamicFunction(fetch) - || containsDynamicParam(fetch)) { + || RexUtil.containsDynamicParam(fetch)) { return null; } final RexExecutor executor = @@ -285,17 +284,4 @@ static void validateLiteralFetch(@Nullable RexNode fetch) { return cluster.getRexBuilder().makeExactLiteral(BigDecimal.valueOf(value)); } - private static boolean containsDynamicParam(RexNode node) { - try { - node.accept( - new RexVisitorImpl(true) { - @Override public Void visitDynamicParam(RexDynamicParam dynamicParam) { - throw Util.FoundOne.NULL; - } - }); - return false; - } catch (Util.FoundOne e) { - return true; - } - } } diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java index e9f67b3502a8..12cbd7831200 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java @@ -62,7 +62,6 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexProgram; import org.apache.calcite.rex.RexUtil; -import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.sql.JoinConditionType; import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SqlAsofJoin; @@ -1267,7 +1266,7 @@ private static SqlNode toSqlFetch(Sort sort, Context context) { || !RexUtil.isConstant(fetch) || !RexUtil.isDeterministic(fetch) || RexUtil.containsDynamicFunction(fetch) - || containsDynamicParam(fetch)) { + || RexUtil.containsDynamicParam(fetch)) { return context.toSql(null, fetch); } final RexExecutor executor = @@ -1279,20 +1278,6 @@ private static SqlNode toSqlFetch(Sort sort, Context context) { return context.toSql(null, reduced instanceof RexLiteral ? reduced : fetch); } - private static boolean containsDynamicParam(RexNode node) { - try { - node.accept( - new RexVisitorImpl(true) { - @Override public Void visitDynamicParam(RexDynamicParam dynamicParam) { - throw Util.FoundOne.NULL; - } - }); - return false; - } catch (Util.FoundOne e) { - return true; - } - } - public boolean hasTrickyRollup(Sort e, Aggregate aggregate) { return !dialect.supportsAggregateFunction(SqlKind.ROLLUP) && dialect.supportsGroupByWithRollup() diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java index 03719a2bffc1..841cbf2d7d42 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java +++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java @@ -859,6 +859,22 @@ public static boolean containsDynamicFunction(RexNode e) { } } + /** Returns whether an expression contains a dynamic parameter. */ + public static boolean containsDynamicParam(RexNode e) { + try { + e.accept( + new RexVisitorImpl(true) { + @Override public Void visitDynamicParam(RexDynamicParam dynamicParam) { + throw Util.FoundOne.NULL; + } + }); + return false; + } catch (Util.FoundOne ex) { + Util.swallow(ex, null); + return true; + } + } + public static List retainDeterministic(List list) { List conjunctions = new ArrayList<>(); for (RexNode x : list) { diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index 2bdc7a1d56c4..004ac0514faa 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -3564,6 +3564,18 @@ private void assertTypeAndToString( hasSize(0)); } + @Test void testContainsDynamicParam() { + final RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER); + final RexNode literal = rexBuilder.makeExactLiteral(BigDecimal.ONE, intType); + final RexNode dynamicParam = rexBuilder.makeDynamicParam(intType, 0); + final RexNode expression = + rexBuilder.makeCall(SqlStdOperatorTable.PLUS, literal, dynamicParam); + + assertThat(RexUtil.containsDynamicParam(literal), is(false)); + assertThat(RexUtil.containsDynamicParam(dynamicParam), is(true)); + assertThat(RexUtil.containsDynamicParam(expression), is(true)); + } + @Test void testConstantMap() { final RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER); final RelDataType bigintType = typeFactory.createSqlType(SqlTypeName.BIGINT); From 6f1741d8b277441a7d996440f0409423a5d29838 Mon Sep 17 00:00:00 2001 From: Kirill Tkalenko Date: Tue, 23 Jun 2026 12:41:53 +0300 Subject: [PATCH 09/12] [CALCITE-7592] Applied changes based on the review 1.0 --- .../adapter/cassandra/CassandraRules.java | 9 +- .../calcite/test/CassandraAdapterTest.java | 12 ++ .../adapter/enumerable/EnumerableLimit.java | 140 +++++++++--------- .../enumerable/EnumerableLimitSort.java | 15 +- .../apache/calcite/interpreter/SortNode.java | 22 ++- .../calcite/sql2rel/RelDecorrelator.java | 6 + .../sql2rel/TopDownGeneralDecorrelator.java | 4 + .../org/apache/calcite/test/JdbcTest.java | 63 +++++--- .../apache/calcite/test/RelOptRulesTest.java | 27 ++++ .../apache/calcite/test/RelOptRulesTest.xml | 89 +++++++++++ core/src/test/resources/sql/fetch.iq | 16 ++ .../calcite/adapter/druid/DruidRules.java | 9 +- .../apache/calcite/test/DruidAdapter2IT.java | 11 ++ .../elasticsearch/ElasticsearchRules.java | 9 +- .../elasticsearch/ElasticsearchSort.java | 2 +- .../ElasticSearchAdapterTest.java | 10 ++ .../calcite/adapter/geode/rel/GeodeRules.java | 9 +- .../calcite/adapter/geode/rel/GeodeSort.java | 2 +- .../adapter/geode/rel/GeodeBookstoreTest.java | 10 ++ .../calcite/adapter/mongodb/MongoRules.java | 7 + .../adapter/mongodb/MongoAdapterTest.java | 10 ++ 21 files changed, 372 insertions(+), 110 deletions(-) diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java index 5ec5ea798480..341f5c41c47a 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java @@ -410,10 +410,17 @@ protected CassandraLimitRule(CassandraLimitRuleConfig config) { final RexLiteral fetch = limit.fetch == null ? null - : EnumerableLimit.reduceFetchToIntLiteral(limit.getCluster(), limit.fetch); + : EnumerableLimit.reduceFetchToLiteral(limit.getCluster(), limit.fetch); if (limit.fetch != null && fetch == null) { return null; } + if (fetch != null) { + try { + RexLiteral.bigDecimalValue(fetch).intValueExact(); + } catch (ArithmeticException e) { + return null; + } + } final RelTraitSet traitSet = limit.getTraitSet().replace(CassandraRel.CONVENTION); return new CassandraLimit(limit.getCluster(), traitSet, diff --git a/cassandra/src/test/java/org/apache/calcite/test/CassandraAdapterTest.java b/cassandra/src/test/java/org/apache/calcite/test/CassandraAdapterTest.java index cd096fc0acb6..2536cde7a54d 100644 --- a/cassandra/src/test/java/org/apache/calcite/test/CassandraAdapterTest.java +++ b/cassandra/src/test/java/org/apache/calcite/test/CassandraAdapterTest.java @@ -143,6 +143,18 @@ static void load(CqlSession session) { .throws_("FETCH value -1 is out of range"); } + @Test void testFetchExpressionBeyondIntegerRange() { + CalciteAssert.that() + .with(TWISSANDRA) + .query("select \"tweet_id\" from \"userline\" " + + "where \"username\" = '!PUBLIC!' " + + "fetch next " + + "(cast(18446744073709551616 as decimal(20, 0))) rows only") + .returnsCount(146) + .explainContains("EnumerableLimit(fetch=[18446744073709551616:DECIMAL(20, 0)])\n" + + " CassandraToEnumerableConverter\n"); + } + @Test void testSortLimit() { CalciteAssert.that() .with(TWISSANDRA) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java index 5d5bb4548ac4..5a24137d86da 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java @@ -17,6 +17,11 @@ package org.apache.calcite.adapter.enumerable; import org.apache.calcite.DataContext; +import org.apache.calcite.linq4j.AbstractEnumerable; +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.EnumerableDefaults; +import org.apache.calcite.linq4j.Enumerator; +import org.apache.calcite.linq4j.function.Function1; import org.apache.calcite.linq4j.tree.BlockBuilder; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; @@ -42,8 +47,10 @@ import org.checkerframework.checker.nullness.qual.Nullable; import java.math.BigDecimal; +import java.math.BigInteger; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.List; /** Relational expression that applies a limit and/or offset to its input. */ @@ -118,7 +125,7 @@ public static EnumerableLimit create(final RelNode input, @Nullable RexNode offs if (fetch != null) { v = builder.append("fetch", - Expressions.call(v, BuiltInMethod.TAKE.method, + Expressions.call(EnumerableLimit.class, "take", v, getExpressionForFetch(fetch, implementor, builder))); } @@ -135,8 +142,8 @@ static Expression getExpression(RexNode rexNode) { Expressions.constant("?" + param.getIndex())), Integer.class); } else { - // TODO: Enumerable runtime only supports INT types for FETCH and OFFSET, not BIGINT types. - // Currently, using BIGINT types for execution will result in an error message. + // TODO: Enumerable runtime only supports INT types for OFFSET, not BIGINT types. + // Currently, using BIGINT types for OFFSET will result in an error message. // This issue needs to be fixed. For more information, see CALCITE-7156. return Expressions.constant(RexLiteral.intValue(rexNode)); } @@ -146,7 +153,7 @@ static Expression getExpressionForFetch(RexNode rexNode, EnumerableRelImplementor implementor, BlockBuilder builder) { if (rexNode instanceof RexDynamicParam) { final RexDynamicParam param = (RexDynamicParam) rexNode; - return Expressions.call(EnumerableLimit.class, "toIntFetch", + return Expressions.call(EnumerableLimit.class, "toFetchValue", Expressions.convert_( Expressions.call(DataContext.ROOT, BuiltInMethod.DATA_CONTEXT_GET.method, @@ -154,53 +161,83 @@ static Expression getExpressionForFetch(RexNode rexNode, Number.class)); } else if (rexNode instanceof RexLiteral) { return Expressions.constant( - toIntFetch(((RexLiteral) rexNode).getValueAs(Number.class))); + toFetchValue(((RexLiteral) rexNode).getValueAs(Number.class))); } else { final Expression expression = RexToLixTranslator.forAggregation(implementor.getTypeFactory(), builder, null, implementor.getConformance()) .translate(rexNode); - return Expressions.call(EnumerableLimit.class, "toIntFetch", + return Expressions.call(EnumerableLimit.class, "toFetchValue", Expressions.convert_(Expressions.box(expression), Number.class)); } } - /** Converts a FETCH expression result to the range supported by Enumerable. */ - public static int toIntFetch(@Nullable Number value) { + /** Converts a FETCH expression result to Calcite's canonical representation. */ + public static BigDecimal toFetchValue(@Nullable Number value) { final BigDecimal decimal = validateFetchValue(value); - final int result; - try { - result = decimal.intValueExact(); - } catch (ArithmeticException e) { - throw new IllegalArgumentException("FETCH value " + value - + " is out of range; expected a value between 0 and " - + Integer.MAX_VALUE, e); - } - if (result < 0) { + if (decimal.signum() < 0) { throw new IllegalArgumentException("FETCH value " + value - + " is out of range; expected a value between 0 and " - + Integer.MAX_VALUE); + + " is out of range; expected a non-negative value"); } - return result; + return decimal; } - /** Converts a FETCH expression result to the range supported by a long. */ - public static long toLongFetch(@Nullable Number value) { - final BigDecimal decimal = validateFetchValue(value); - final long result; - try { - result = decimal.longValueExact(); - } catch (ArithmeticException e) { - throw new IllegalArgumentException("FETCH value " + value - + " is out of range; expected a value between 0 and " - + Long.MAX_VALUE, e); + /** Applies a FETCH value without narrowing it to {@code int} or {@code long}. */ + public static Enumerable take(Enumerable source, BigDecimal fetch) { + final BigInteger count = fetch.toBigIntegerExact(); + return new AbstractEnumerable() { + @Override public Enumerator enumerator() { + final Enumerator input = source.enumerator(); + return new Enumerator() { + private BigInteger remaining = count; + private boolean done; + + @Override public T current() { + return input.current(); + } + + @Override public boolean moveNext() { + if (done) { + return false; + } + if (remaining.signum() == 0 || !input.moveNext()) { + done = true; + return false; + } + // Preserve take(int)'s eager evaluation of the current row. + input.current(); + remaining = remaining.subtract(BigInteger.ONE); + return true; + } + + @Override public void reset() { + input.reset(); + remaining = count; + done = false; + } + + @Override public void close() { + input.close(); + } + }; + } + }; + } + + /** Sorts and applies FETCH while preserving an arbitrary-precision value. */ + public static Enumerable orderBy(Enumerable source, + Function1 keySelector, @Nullable Comparator comparator, + int offset, @Nullable BigDecimal fetch) { + if (fetch != null + && fetch.compareTo(BigDecimal.valueOf(Integer.MAX_VALUE)) <= 0) { + return EnumerableDefaults.orderBy(source, keySelector, comparator, + offset, fetch.intValueExact()); } - if (result < 0) { - throw new IllegalArgumentException("FETCH value " + value - + " is out of range; expected a value between 0 and " - + Long.MAX_VALUE); + Enumerable result = EnumerableDefaults.orderBy(source, keySelector, comparator); + if (offset > 0) { + result = result.skip(offset); } - return result; + return fetch == null ? result : take(result, fetch); } private static BigDecimal validateFetchValue(@Nullable Number value) { @@ -223,10 +260,7 @@ private static BigDecimal validateFetchValue(@Nullable Number value) { static void validateLiteralFetch(@Nullable RexNode fetch) { if (fetch instanceof RexLiteral) { final Number value = ((RexLiteral) fetch).getValueAs(Number.class); - final BigDecimal decimal = NumberUtil.toBigDecimal(value); - if (decimal != null && decimal.signum() < 0) { - toIntFetch(value); - } + toFetchValue(value); } } @@ -254,34 +288,8 @@ static void validateLiteralFetch(@Nullable RexNode fetch) { } literal = (RexLiteral) reduced; } - final Number value = literal.getValueAs(Number.class); - if (validateFetchValue(value).signum() < 0) { - throw new IllegalArgumentException( - "FETCH value " + value + " is out of range; expected a non-negative value"); - } + toFetchValue(literal.getValueAs(Number.class)); return literal; } - /** Reduces a constant FETCH expression to a validated integer literal. */ - public static @Nullable RexLiteral reduceFetchToIntLiteral( - RelOptCluster cluster, RexNode fetch) { - final RexLiteral literal = reduceFetchToLiteral(cluster, fetch); - if (literal == null) { - return null; - } - final int value = toIntFetch(literal.getValueAs(Number.class)); - return cluster.getRexBuilder().makeExactLiteral(BigDecimal.valueOf(value)); - } - - /** Reduces a constant FETCH expression to a validated long literal. */ - public static @Nullable RexLiteral reduceFetchToLongLiteral( - RelOptCluster cluster, RexNode fetch) { - final RexLiteral literal = reduceFetchToLiteral(cluster, fetch); - if (literal == null) { - return null; - } - final long value = toLongFetch(literal.getValueAs(Number.class)); - return cluster.getRexBuilder().makeExactLiteral(BigDecimal.valueOf(value)); - } - } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java index fbd8bb6d42db..f48389f28c8e 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java @@ -25,11 +25,12 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.util.BuiltInMethod; import org.apache.calcite.util.Pair; import org.checkerframework.checker.nullness.qual.Nullable; +import java.math.BigDecimal; + import static org.apache.calcite.adapter.enumerable.EnumerableLimit.getExpression; import static org.apache.calcite.adapter.enumerable.EnumerableLimit.getExpressionForFetch; @@ -100,7 +101,7 @@ public static EnumerableLimitSort create( final Expression fetchVal; if (this.fetch == null) { - fetchVal = Expressions.constant(Integer.MAX_VALUE); + fetchVal = Expressions.constant(null, BigDecimal.class); } else { fetchVal = getExpressionForFetch(this.fetch, implementor, builder); } @@ -114,17 +115,13 @@ public static EnumerableLimitSort create( builder.add( Expressions.return_(null, - Expressions.call(BuiltInMethod.ORDER_BY_WITH_FETCH_AND_OFFSET.method, + Expressions.call(EnumerableLimit.class, "orderBy", Expressions.list(childExp, builder.append("keySelector", pair.left)) .appendIfNotNull( builder.appendIfNotNull("comparator", pair.right)) - .appendIfNotNull( - builder.appendIfNotNull("offset", - Expressions.constant(offsetVal))) - .appendIfNotNull( - builder.appendIfNotNull("fetch", - Expressions.constant(fetchVal)))))); + .append(builder.append("offset", offsetVal)) + .append(builder.append("fetch", fetchVal))))); return implementor.result(physType, builder.toBlock()); } } diff --git a/core/src/main/java/org/apache/calcite/interpreter/SortNode.java b/core/src/main/java/org/apache/calcite/interpreter/SortNode.java index 7a1eef1cb4b8..1425689d45ce 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/SortNode.java +++ b/core/src/main/java/org/apache/calcite/interpreter/SortNode.java @@ -28,6 +28,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -58,9 +59,9 @@ private static int getValueAsInt(RexNode node) { () -> "getValueAs(Integer.class) for " + node); } - private int getFetch() { + private @Nullable BigDecimal getFetch() { if (rel.fetch == null) { - return -1; + return null; } final @Nullable Number value; if (rel.fetch instanceof RexLiteral) { @@ -74,7 +75,7 @@ private int getFetch() { } value = (Number) result; } - return EnumerableLimit.toIntFetch(value); + return EnumerableLimit.toFetchValue(value); } @Override public void run() throws InterruptedException { @@ -82,7 +83,7 @@ private int getFetch() { rel.offset == null ? 0 : getValueAsInt(rel.offset); - final int fetch = getFetch(); + final @Nullable BigDecimal fetch = getFetch(); // In pure limit mode. No sort required. Row row; loop: @@ -93,9 +94,12 @@ private int getFetch() { break loop; } } - if (fetch >= 0) { - for (int i = 0; i < fetch && (row = source.receive()) != null; i++) { + if (fetch != null) { + BigDecimal fetched = BigDecimal.ZERO; + while (fetched.compareTo(fetch) < 0 + && (row = source.receive()) != null) { sink.send(row); + fetched = fetched.add(BigDecimal.ONE); } } else { while ((row = source.receive()) != null) { @@ -109,9 +113,11 @@ private int getFetch() { list.add(row); } list.sort(comparator()); - final int end = fetch < 0 || offset + fetch > list.size() + final int available = Math.max(list.size() - offset, 0); + final int end = fetch == null + || fetch.compareTo(BigDecimal.valueOf(available)) >= 0 ? list.size() - : offset + fetch; + : offset + fetch.intValueExact(); for (int i = offset; i < end; i++) { sink.send(list.get(i)); } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index 9c78aa235581..66f2cfb1f3dc 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -574,6 +574,12 @@ protected RexNode removeCorrelationExpr( // Its output does not change the input ordering, so there's no // need to call propagateExpr. + if (isCorVarDefined + && rel.fetch != null + && !RexUtil.isDeterministic(rel.fetch)) { + return null; + } + final RelNode oldInput = rel.getInput(); final Frame frame = getInvoke(oldInput, isCorVarDefined, rel, true); if (frame == null) { diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java index c3d2bd924961..0a2b2c610992 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -616,6 +616,10 @@ public RelNode unnestInternal(Aggregate aggregate, boolean allowEmptyOutputFromR } public RelNode unnestInternal(Sort sort, boolean allowEmptyOutputFromRewrite) { + if (sort.fetch != null && !RexUtil.isDeterministic(sort.fetch)) { + throw new UnsupportedOperationException( + "Cannot decorrelate Sort with non-deterministic FETCH"); + } RelNode newInput = unnest(sort.getInput(), allowEmptyOutputFromRewrite); UnnestedQuery inputInfo = requireNonNull(mapRelToUnnestedQuery.get(sort.getInput())); diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index 00a79286d855..221334546cae 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -3778,12 +3778,16 @@ public void checkOrderBy(final boolean desc, /** Tests FETCH expressions in bindable/interpreter convention. */ @Test void testBindableFetchExpression() { try (Hook.Closeable ignored = Hook.ENABLE_BINDABLE.addThread(Hook.propertyJ(true))) { - CalciteAssert.that() + final CalciteAssert.AssertThat with = CalciteAssert.that(); + with .query("select * from (values (1), (2), (3), (4)) as t(x)\n" + "fetch next (rand_integer(1) + 2) rows only") .explainContains("BindableSort(fetch=[+(RAND_INTEGER(1), 2)])") .returns("X=1\n" + "X=2\n"); + with.query("select * from (values (1), (2), (3), (4)) as t(x)\n" + + "fetch next (cast(9223372036854775808 as decimal(20, 0))) rows only") + .returns("X=1\nX=2\nX=3\nX=4\n"); } } @@ -3804,28 +3808,29 @@ public void checkOrderBy(final boolean desc, final CalciteAssert.AssertThat with = CalciteAssert.that(); final String values = "select * from (values (1), (2), (3)) as t(x)\n"; with.query(values + "fetch next (0 - 1) rows only") - .throws_("FETCH value -1 is out of range; expected a value between 0 and " - + Integer.MAX_VALUE); + .throws_("FETCH value -1 is out of range; expected a non-negative value"); with.query(values + "fetch next (-1) rows only") - .throws_("FETCH value -1 is out of range; expected a value between 0 and " - + Integer.MAX_VALUE); - with.query(values - + "fetch next (cast(3000000000 as bigint) + 1) rows only") - .throws_("FETCH value 3000000001 is out of range; " - + "expected a value between 0 and " + Integer.MAX_VALUE); - with.query(values - + "fetch next (cast(3000000001 as bigint)) rows only") - .throws_("FETCH value 3000000001 is out of range; " - + "expected a value between 0 and " + Integer.MAX_VALUE); - with.query(values - + "fetch next (abs(cast(3000000001 as bigint))) rows only") - .throws_("FETCH value 3000000001 is out of range; " - + "expected a value between 0 and " + Integer.MAX_VALUE); + .throws_("FETCH value -1 is out of range; expected a non-negative value"); with.query(values + "fetch next (cast(null as integer)) rows only") .throws_("FETCH expression evaluated to NULL"); } + /** Tests FETCH values beyond the range of BIGINT. */ + @Test void testFetchExpressionBeyondLong() { + final CalciteAssert.AssertThat with = CalciteAssert.that(); + final String values = "select * from (values (1), (2), (3), (4)) as t(x)\n"; + final String expected = "X=1\nX=2\nX=3\nX=4\n"; + with.query(values + "fetch next 9223372036854775808 rows only") + .returns(expected); + with.query(values + "fetch next " + + "(cast(9223372036854775808 as decimal(20, 0)) + 1) rows only") + .returns(expected); + with.query(values + "order by x fetch next " + + "(cast(9223372036854775808 as decimal(20, 0)) + 1) rows only") + .returns(expected); + } + /** Tests ORDER BY ... OFFSET ... FETCH. */ @Test void testOrderByOffsetFetch() { CalciteAssert.that() @@ -6066,6 +6071,10 @@ private CalciteAssert.AssertQuery withEmpDept(String sql) { checkPreparedFetch(connection, values + "fetch next (? + abs(2)) rows only", 1, "X=1\nX=2\nX=3\n"); + checkPreparedFetch(connection, + values + "fetch next (cast(? as decimal(20, 0))) rows only", + new BigDecimal("9223372036854775808"), + "X=1\nX=2\nX=3\nX=4\n"); checkPreparedFetchFails(connection, values + "fetch next (?) rows only", -1, @@ -6073,10 +6082,6 @@ private CalciteAssert.AssertQuery withEmpDept(String sql) { checkPreparedFetchFails(connection, values + "fetch next (? + 1) rows only", -2, "FETCH value -1 is out of range"); - checkPreparedFetchFails(connection, - values + "fetch next (abs(cast(? as bigint))) rows only", - 3_000_000_001L, - "FETCH value 3000000001 is out of range"); }); } @@ -6095,6 +6100,10 @@ private CalciteAssert.AssertQuery withEmpDept(String sql) { new int[] {0, 2, 3}, new String[] {"X=1\n", "X=1\nX=2\nX=3\n", "X=1\nX=2\nX=3\nX=4\n"}); + checkPreparedFetch(connection, + values + "fetch next (cast(? as decimal(20, 0))) rows only", + new BigDecimal("9223372036854775808"), + "X=1\nX=2\nX=3\nX=4\n"); }); } } @@ -6111,6 +6120,18 @@ private static void checkPreparedFetch(Connection connection, String sql, } } + private static void checkPreparedFetch(Connection connection, String sql, + BigDecimal value, String expected) { + try (PreparedStatement p = connection.prepareStatement(sql)) { + p.setBigDecimal(1, value); + try (ResultSet r = p.executeQuery()) { + assertThat(CalciteAssert.toString(r), is(expected)); + } + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + } + private static void checkPreparedFetchRepeated(Connection connection, String sql, int[] values, String[] expected) { try (PreparedStatement p = connection.prepareStatement(sql)) { diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index a00e39ebdfe0..711e8e9b8432 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -12010,6 +12010,33 @@ private void checkLoptOptimizeJoinRule(LoptOptimizeJoinRule rule) { .check(); } + @Test void testNondeterministicFetchPreventsDecorrelation() { + checkNondeterministicFetchPreventsDecorrelation(false); + } + + @Test void testNondeterministicFetchPreventsTopDownDecorrelation() { + checkNondeterministicFetchPreventsDecorrelation(true); + } + + private void checkNondeterministicFetchPreventsDecorrelation(boolean topDown) { + final String sql = "select t.deptno, e.ename\n" + + "from (select distinct deptno from emp) t,\n" + + "lateral (select ename from emp\n" + + " where emp.deptno = t.deptno\n" + + " order by sal\n" + + " fetch next (rand_integer(2) + 1) rows only) e"; + + final RelOptFixture fixture = sql(sql) + .withRule() // empty program + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(topDown); + if (topDown) { + fixture.check(); + } else { + fixture.checkUnchanged(); + } + } + @Test void testTopDownGeneralDecorrelateForFilterSome() { final String sql = "select empno from emp where " + "empno > SOME(select empno from emp_b where emp.ename = emp_b.ename)"; diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 64745a910f51..1d25ccf3b7e8 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -11024,6 +11024,95 @@ LogicalProject(USER=[USER]) LogicalAggregate(group=[{0}], EXPR$1=[SUM($1)]) LogicalProject(NAME=[$1], DEPTNO=[$0]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/src/test/resources/sql/fetch.iq b/core/src/test/resources/sql/fetch.iq index b2dfbcd22a3f..9e335f712a07 100644 --- a/core/src/test/resources/sql/fetch.iq +++ b/core/src/test/resources/sql/fetch.iq @@ -48,6 +48,22 @@ fetch next (abs(2)) rows only; !ok +# FETCH values are not restricted to the BIGINT range. +select * +from (values (1), (2), (3), (4)) as t(x) +fetch next (cast(9223372036854775808 as decimal(20, 0)) + 1) rows only; ++---+ +| X | ++---+ +| 1 | +| 2 | +| 3 | +| 4 | ++---+ +(4 rows) + +!ok + # FETCH expression cannot be negative. select * from (values (1), (2), (3)) as t(x) diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java index 0c67379e5407..a9f7ddb202d0 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java @@ -854,10 +854,17 @@ protected DruidSortRule(DruidSortRuleConfig config) { final RexLiteral fetch = sort.fetch == null ? null - : EnumerableLimit.reduceFetchToIntLiteral(sort.getCluster(), sort.fetch); + : EnumerableLimit.reduceFetchToLiteral(sort.getCluster(), sort.fetch); if (sort.fetch != null && fetch == null) { return; } + if (fetch != null) { + try { + RexLiteral.bigDecimalValue(fetch).intValueExact(); + } catch (ArithmeticException e) { + return; + } + } if (!DruidQuery.isValidSignature(query.signature() + 'l')) { return; } diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java index 7f8f2c119c83..f5f889926e05 100644 --- a/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java +++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java @@ -981,6 +981,17 @@ private void checkGroupBySingleSortLimit(boolean approx) { .throws_("FETCH value -1 is out of range"); } + @Test void testFetchExpressionBeyondIntegerRange() { + final String sql = "select \"state_province\"\n" + + "from \"foodmart\"\n" + + "fetch next " + + "(cast(18446744073709551616 as decimal(20, 0))) rows only"; + sql(sql) + .returnsCount(86837) + .explainContains("BindableSort(fetch=[18446744073709551616:DECIMAL(20, 0)])\n" + + " DruidQuery(table=[[foodmart, foodmart]], "); + } + /** Tests that distinct-count is pushed down to Druid and evaluated using * "cardinality". The result is approximate, but gives the correct result in * this example when rounded down using FLOOR. */ diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java index 9c47cd8b3852..8b7352e70950 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java @@ -223,10 +223,17 @@ protected ElasticsearchSortRule(Config config) { final RexLiteral fetch = sort.fetch == null ? null - : EnumerableLimit.reduceFetchToLongLiteral(sort.getCluster(), sort.fetch); + : EnumerableLimit.reduceFetchToLiteral(sort.getCluster(), sort.fetch); if (sort.fetch != null && fetch == null) { return null; } + if (fetch != null) { + try { + RexLiteral.bigDecimalValue(fetch).longValueExact(); + } catch (ArithmeticException e) { + return null; + } + } final RelTraitSet traitSet = sort.getTraitSet().replace(out).replace(sort.getCollation()); return new ElasticsearchSort(relNode.getCluster(), traitSet, convert(sort.getInput(), traitSet.replace(RelCollations.EMPTY)), sort.getCollation(), diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSort.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSort.java index e7af28627978..56e84659f99d 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSort.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSort.java @@ -78,7 +78,7 @@ public class ElasticsearchSort extends Sort implements ElasticsearchRel { } if (fetch != null) { - implementor.fetch(RexLiteral.numberValue(fetch).longValue()); + implementor.fetch(RexLiteral.bigDecimalValue(fetch).longValueExact()); } } diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java index c0d43105b6c3..52d9dfc4c38e 100644 --- a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java +++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java @@ -596,6 +596,16 @@ private static Consumer sortedResultSetChecker(String column, .explainContains("ElasticsearchSort(fetch=[3000000001:BIGINT])"); } + @Test void testFetchExpressionBeyondLongRange() { + calciteAssert() + .query("select state, id from zips\n" + + "fetch next " + + "(cast(9223372036854775808 as decimal(20, 0))) rows only") + .returnsCount(ZIPS_SIZE) + .explainContains("EnumerableLimit(fetch=[9223372036854775808:DECIMAL(19, 0)])\n" + + " ElasticsearchToEnumerableConverter\n"); + } + @Test void limit2() { final String sql = "select id from zips limit 5"; calciteAssert() diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java index 13a29a9e3b5d..5c9fa73e3f59 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java @@ -225,10 +225,17 @@ protected GeodeSortLimitRule(GeodeSortLimitRuleConfig config) { final RexLiteral fetch = sort.fetch == null ? null - : EnumerableLimit.reduceFetchToLongLiteral(sort.getCluster(), sort.fetch); + : EnumerableLimit.reduceFetchToLiteral(sort.getCluster(), sort.fetch); if (sort.fetch != null && fetch == null) { return; } + if (fetch != null) { + try { + RexLiteral.bigDecimalValue(fetch).longValueExact(); + } catch (ArithmeticException e) { + return; + } + } final RelTraitSet traitSet = sort.getTraitSet() .replace(GeodeRel.CONVENTION) diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSort.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSort.java index c9b33252d980..75b0d2d1aa98 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSort.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSort.java @@ -86,7 +86,7 @@ public class GeodeSort extends Sort implements GeodeRel { } if (fetch != null) { - geodeImplementContext.setLimit(RexLiteral.numberValue(fetch).longValue()); + geodeImplementContext.setLimit(RexLiteral.bigDecimalValue(fetch).longValueExact()); } } diff --git a/geode/src/test/java/org/apache/calcite/adapter/geode/rel/GeodeBookstoreTest.java b/geode/src/test/java/org/apache/calcite/adapter/geode/rel/GeodeBookstoreTest.java index b51b7770cb99..2bfbd58ff2ac 100644 --- a/geode/src/test/java/org/apache/calcite/adapter/geode/rel/GeodeBookstoreTest.java +++ b/geode/src/test/java/org/apache/calcite/adapter/geode/rel/GeodeBookstoreTest.java @@ -368,6 +368,16 @@ private CalciteAssert.AssertThat calciteAssert() { .throws_("FETCH value -1 is out of range"); } + @Test void testFetchExpressionBeyondLongRange() { + calciteAssert() + .query("select * from geode.BookCustomer " + + "fetch next " + + "(cast(9223372036854775808 as decimal(20, 0))) rows only") + .returnsCount(3) + .explainContains("EnumerableLimit(fetch=[9223372036854775808:DECIMAL(19, 0)])\n" + + " GeodeToEnumerableConverter\n"); + } + @Test void testSelectWithNestedPdx2() { calciteAssert() .query("select primaryAddress from geode.BookCustomer limit 2") diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java index 63d394c9404d..9c40de820cac 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java @@ -273,6 +273,13 @@ private static class MongoSortRule extends MongoConverterRule { if (sort.fetch != null && fetch == null) { return null; } + if (fetch != null) { + try { + RexLiteral.bigDecimalValue(fetch).longValueExact(); + } catch (ArithmeticException e) { + return null; + } + } final RelTraitSet traitSet = sort.getTraitSet().replace(out) .replace(sort.getCollation()); diff --git a/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java b/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java index e59a17fc5ac0..a745dd227b9e 100644 --- a/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java +++ b/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java @@ -255,6 +255,16 @@ private CalciteAssert.AssertThat assertModel(URL url) { "{$limit: 3000000001}")); } + @Test void testFetchExpressionBeyondLongRange() { + assertModel(MODEL) + .query("select state, id from zips\n" + + "fetch next " + + "(cast(9223372036854775808 as decimal(20, 0))) rows only") + .returnsCount(ZIPS_SIZE) + .explainContains("EnumerableLimit(fetch=[9223372036854775808:DECIMAL(19, 0)])\n" + + " MongoToEnumerableConverter\n"); + } + @Test void testJoin() { assertModel(MODEL) .query("select b.state, a.id from zips as a join zips as b on a.id=b.id where a.id='02401' " From 116aff9722215a9466d96ef787c2362c364edc48 Mon Sep 17 00:00:00 2001 From: Kirill Tkalenko Date: Tue, 23 Jun 2026 15:17:48 +0300 Subject: [PATCH 10/12] [CALCITE-7592] Wip --- .../adapter/cassandra/CassandraRules.java | 3 +- .../adapter/enumerable/EnumerableLimit.java | 60 +------------------ .../calcite/adapter/jdbc/JdbcRules.java | 3 +- .../java/org/apache/calcite/rex/RexUtil.java | 51 ++++++++++++++++ .../calcite/sql2rel/RelDecorrelator.java | 9 ++- .../sql2rel/TopDownGeneralDecorrelator.java | 4 +- .../org/apache/calcite/test/JdbcTest.java | 15 +++++ .../calcite/adapter/druid/DruidRules.java | 3 +- .../elasticsearch/ElasticsearchRules.java | 4 +- .../calcite/adapter/geode/rel/GeodeRules.java | 4 +- .../calcite/adapter/mongodb/MongoRules.java | 4 +- 11 files changed, 87 insertions(+), 73 deletions(-) diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java index 341f5c41c47a..71f7592191b4 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java @@ -37,6 +37,7 @@ import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.validate.SqlValidatorUtil; @@ -410,7 +411,7 @@ protected CassandraLimitRule(CassandraLimitRuleConfig config) { final RexLiteral fetch = limit.fetch == null ? null - : EnumerableLimit.reduceFetchToLiteral(limit.getCluster(), limit.fetch); + : RexUtil.reduceFetchToLiteral(limit.getCluster(), limit.fetch); if (limit.fetch != null && fetch == null) { return null; } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java index 5a24137d86da..55c22ae0560b 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java @@ -36,20 +36,15 @@ import org.apache.calcite.rel.metadata.RelMdDistribution; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rex.RexDynamicParam; -import org.apache.calcite.rex.RexExecutor; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.util.BuiltInMethod; -import org.apache.calcite.util.NumberUtil; -import org.apache.calcite.util.Util; import org.checkerframework.checker.nullness.qual.Nullable; import java.math.BigDecimal; import java.math.BigInteger; -import java.util.ArrayList; -import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -174,12 +169,7 @@ static Expression getExpressionForFetch(RexNode rexNode, /** Converts a FETCH expression result to Calcite's canonical representation. */ public static BigDecimal toFetchValue(@Nullable Number value) { - final BigDecimal decimal = validateFetchValue(value); - if (decimal.signum() < 0) { - throw new IllegalArgumentException("FETCH value " + value - + " is out of range; expected a non-negative value"); - } - return decimal; + return RexUtil.validateFetchValue(value); } /** Applies a FETCH value without narrowing it to {@code int} or {@code long}. */ @@ -229,7 +219,8 @@ public static Enumerable orderBy(Enumerable source, Function1 keySelector, @Nullable Comparator comparator, int offset, @Nullable BigDecimal fetch) { if (fetch != null - && fetch.compareTo(BigDecimal.valueOf(Integer.MAX_VALUE)) <= 0) { + && fetch.compareTo(BigDecimal.valueOf(Integer.MAX_VALUE)) <= 0 + && comparator != null) { return EnumerableDefaults.orderBy(source, keySelector, comparator, offset, fetch.intValueExact()); } @@ -240,23 +231,6 @@ public static Enumerable orderBy(Enumerable source, return fetch == null ? result : take(result, fetch); } - private static BigDecimal validateFetchValue(@Nullable Number value) { - if (value == null) { - throw new IllegalArgumentException("FETCH expression evaluated to NULL"); - } - final BigDecimal decimal = NumberUtil.toBigDecimal(value); - if (decimal == null) { - throw new IllegalArgumentException("FETCH value is not numeric: " + value); - } - try { - decimal.toBigIntegerExact(); - } catch (ArithmeticException e) { - throw new IllegalArgumentException("FETCH value " + value - + " is not an integer", e); - } - return decimal; - } - static void validateLiteralFetch(@Nullable RexNode fetch) { if (fetch instanceof RexLiteral) { final Number value = ((RexLiteral) fetch).getValueAs(Number.class); @@ -264,32 +238,4 @@ static void validateLiteralFetch(@Nullable RexNode fetch) { } } - /** Reduces a constant FETCH expression to a validated literal. */ - public static @Nullable RexLiteral reduceFetchToLiteral( - RelOptCluster cluster, RexNode fetch) { - final RexLiteral literal; - if (fetch instanceof RexLiteral) { - literal = (RexLiteral) fetch; - } else { - if (!RexUtil.isConstant(fetch) - || !RexUtil.isDeterministic(fetch) - || RexUtil.containsDynamicFunction(fetch) - || RexUtil.containsDynamicParam(fetch)) { - return null; - } - final RexExecutor executor = - Util.first(cluster.getPlanner().getExecutor(), RexUtil.EXECUTOR); - final List reducedValues = new ArrayList<>(1); - executor.reduce(cluster.getRexBuilder(), - Collections.singletonList(fetch), reducedValues); - final RexNode reduced = reducedValues.get(0); - if (!(reduced instanceof RexLiteral)) { - return null; - } - literal = (RexLiteral) reduced; - } - toFetchValue(literal.getValueAs(Number.class)); - return literal; - } - } diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java index 960cec24414e..1b080506677e 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java @@ -16,7 +16,6 @@ */ package org.apache.calcite.adapter.jdbc; -import org.apache.calcite.adapter.enumerable.EnumerableLimit; import org.apache.calcite.linq4j.Queryable; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.plan.Contexts; @@ -776,7 +775,7 @@ protected JdbcSortRule(Config config) { || sort.fetch instanceof RexDynamicParam) { fetch = sort.fetch; } else { - fetch = EnumerableLimit.reduceFetchToLiteral(sort.getCluster(), sort.fetch); + fetch = RexUtil.reduceFetchToLiteral(sort.getCluster(), sort.fetch); if (fetch == null) { return null; } diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java index 841cbf2d7d42..9c8e60c2cf3e 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java +++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java @@ -19,6 +19,7 @@ import org.apache.calcite.DataContexts; import org.apache.calcite.linq4j.function.Predicate1; import org.apache.calcite.plan.PlanTooComplexError; +import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptPredicateList; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelCollation; @@ -48,6 +49,7 @@ import org.apache.calcite.util.ControlFlowException; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.Litmus; +import org.apache.calcite.util.NumberUtil; import org.apache.calcite.util.Pair; import org.apache.calcite.util.RangeSets; import org.apache.calcite.util.Sarg; @@ -63,9 +65,11 @@ import org.apiguardian.api.API; import org.checkerframework.checker.nullness.qual.Nullable; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -875,6 +879,53 @@ public static boolean containsDynamicParam(RexNode e) { } } + /** Converts a FETCH expression result to its validated canonical representation. */ + public static BigDecimal validateFetchValue(@Nullable Number value) { + if (value == null) { + throw new IllegalArgumentException("FETCH expression evaluated to NULL"); + } + final BigDecimal decimal = NumberUtil.toBigDecimal(value); + try { + decimal.toBigIntegerExact(); + } catch (ArithmeticException e) { + throw new IllegalArgumentException("FETCH value " + value + + " is not an integer", e); + } + if (decimal.signum() < 0) { + throw new IllegalArgumentException("FETCH value " + value + + " is out of range; expected a non-negative value"); + } + return decimal; + } + + /** Reduces a constant FETCH expression to a validated literal. */ + public static @Nullable RexLiteral reduceFetchToLiteral( + RelOptCluster cluster, RexNode fetch) { + final RexLiteral literal; + if (fetch instanceof RexLiteral) { + literal = (RexLiteral) fetch; + } else { + if (!isConstant(fetch) + || !isDeterministic(fetch) + || containsDynamicFunction(fetch) + || containsDynamicParam(fetch)) { + return null; + } + final RexExecutor executor = + Util.first(cluster.getPlanner().getExecutor(), EXECUTOR); + final List reducedValues = new ArrayList<>(1); + executor.reduce(cluster.getRexBuilder(), + Collections.singletonList(fetch), reducedValues); + final RexNode reduced = reducedValues.get(0); + if (!(reduced instanceof RexLiteral)) { + return null; + } + literal = (RexLiteral) reduced; + } + validateFetchValue(literal.getValueAs(Number.class)); + return literal; + } + public static List retainDeterministic(List list) { List conjunctions = new ArrayList<>(); for (RexNode x : list) { diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index 66f2cfb1f3dc..2b75f0cdb324 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -574,9 +574,7 @@ protected RexNode removeCorrelationExpr( // Its output does not change the input ordering, so there's no // need to call propagateExpr. - if (isCorVarDefined - && rel.fetch != null - && !RexUtil.isDeterministic(rel.fetch)) { + if (isCorVarDefined && !canDecorrelateFetch(rel)) { return null; } @@ -1142,6 +1140,11 @@ private static void shiftMapping(Map mapping, int startIndex, return register(sort, result, mapOldToNewOutputs, corDefOutputs); } + static boolean canDecorrelateFetch(Sort sort) { + return sort.fetch == null + || RexUtil.reduceFetchToLiteral(sort.getCluster(), sort.fetch) != null; + } + protected @Nullable Frame decorrelateSortAsAggregate(Sort sort, final Frame frame) { if (sort.offset != null || !(sort.fetch instanceof RexLiteral)) { return null; diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java index 0a2b2c610992..c80e91d6de57 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -616,9 +616,9 @@ public RelNode unnestInternal(Aggregate aggregate, boolean allowEmptyOutputFromR } public RelNode unnestInternal(Sort sort, boolean allowEmptyOutputFromRewrite) { - if (sort.fetch != null && !RexUtil.isDeterministic(sort.fetch)) { + if (!RelDecorrelator.canDecorrelateFetch(sort)) { throw new UnsupportedOperationException( - "Cannot decorrelate Sort with non-deterministic FETCH"); + "Cannot decorrelate Sort with a runtime FETCH expression"); } RelNode newInput = unnest(sort.getInput(), allowEmptyOutputFromRewrite); UnnestedQuery inputInfo = diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index 221334546cae..43afce9ade4a 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -3816,6 +3816,21 @@ public void checkOrderBy(final boolean desc, .throws_("FETCH expression evaluated to NULL"); } + @Test void testCorrelatedFetchExpressionInvalidValue() { + final String sqlPrefix = "select d.\"name\", e.\"name\"\n" + + "from \"hr\".\"depts\" d,\n" + + "lateral (select \"name\" from \"hr\".\"emps\"\n" + + " where \"deptno\" = d.\"deptno\"\n"; + for (String fetch : new String[] {"(0 - 1)", "(-1)"}) { + for (boolean topDown : new boolean[] {false, true}) { + CalciteAssert.hr() + .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, topDown) + .query(sqlPrefix + " fetch next " + fetch + " rows only) e") + .throws_("FETCH value -1 is out of range"); + } + } + } + /** Tests FETCH values beyond the range of BIGINT. */ @Test void testFetchExpressionBeyondLong() { final CalciteAssert.AssertThat with = CalciteAssert.that(); diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java index a9f7ddb202d0..1c8f9ef98623 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java @@ -16,7 +16,6 @@ */ package org.apache.calcite.adapter.druid; -import org.apache.calcite.adapter.enumerable.EnumerableLimit; import org.apache.calcite.config.CalciteConnectionConfig; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptPredicateList; @@ -854,7 +853,7 @@ protected DruidSortRule(DruidSortRuleConfig config) { final RexLiteral fetch = sort.fetch == null ? null - : EnumerableLimit.reduceFetchToLiteral(sort.getCluster(), sort.fetch); + : RexUtil.reduceFetchToLiteral(sort.getCluster(), sort.fetch); if (sort.fetch != null && fetch == null) { return; } diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java index 8b7352e70950..259c7bc55090 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java @@ -16,7 +16,6 @@ */ package org.apache.calcite.adapter.elasticsearch; -import org.apache.calcite.adapter.enumerable.EnumerableLimit; import org.apache.calcite.adapter.enumerable.RexImpTable; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.adapter.java.JavaTypeFactory; @@ -39,6 +38,7 @@ import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.fun.SqlStdOperatorTable; @@ -223,7 +223,7 @@ protected ElasticsearchSortRule(Config config) { final RexLiteral fetch = sort.fetch == null ? null - : EnumerableLimit.reduceFetchToLiteral(sort.getCluster(), sort.fetch); + : RexUtil.reduceFetchToLiteral(sort.getCluster(), sort.fetch); if (sort.fetch != null && fetch == null) { return null; } diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java index 5c9fa73e3f59..f9788f27d63b 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java @@ -16,7 +16,6 @@ */ package org.apache.calcite.adapter.geode.rel; -import org.apache.calcite.adapter.enumerable.EnumerableLimit; import org.apache.calcite.plan.Convention; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; @@ -35,6 +34,7 @@ import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.fun.SqlStdOperatorTable; @@ -225,7 +225,7 @@ protected GeodeSortLimitRule(GeodeSortLimitRuleConfig config) { final RexLiteral fetch = sort.fetch == null ? null - : EnumerableLimit.reduceFetchToLiteral(sort.getCluster(), sort.fetch); + : RexUtil.reduceFetchToLiteral(sort.getCluster(), sort.fetch); if (sort.fetch != null && fetch == null) { return; } diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java index 9c40de820cac..ee55036564a1 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java @@ -16,7 +16,6 @@ */ package org.apache.calcite.adapter.mongodb; -import org.apache.calcite.adapter.enumerable.EnumerableLimit; import org.apache.calcite.adapter.enumerable.RexImpTable; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.adapter.java.JavaTypeFactory; @@ -37,6 +36,7 @@ import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; @@ -269,7 +269,7 @@ private static class MongoSortRule extends MongoConverterRule { final RexLiteral fetch = sort.fetch == null ? null - : EnumerableLimit.reduceFetchToLiteral(sort.getCluster(), sort.fetch); + : RexUtil.reduceFetchToLiteral(sort.getCluster(), sort.fetch); if (sort.fetch != null && fetch == null) { return null; } From e7ef7c062464976dfb65576e88b1c8c4f4cf6324 Mon Sep 17 00:00:00 2001 From: Kirill Tkalenko Date: Tue, 23 Jun 2026 15:43:54 +0300 Subject: [PATCH 11/12] [CALCITE-7592] Wip --- .../java/org/apache/calcite/test/CassandraAdapterTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cassandra/src/test/java/org/apache/calcite/test/CassandraAdapterTest.java b/cassandra/src/test/java/org/apache/calcite/test/CassandraAdapterTest.java index 2536cde7a54d..3d36224e6243 100644 --- a/cassandra/src/test/java/org/apache/calcite/test/CassandraAdapterTest.java +++ b/cassandra/src/test/java/org/apache/calcite/test/CassandraAdapterTest.java @@ -149,9 +149,9 @@ static void load(CqlSession session) { .query("select \"tweet_id\" from \"userline\" " + "where \"username\" = '!PUBLIC!' " + "fetch next " - + "(cast(18446744073709551616 as decimal(20, 0))) rows only") + + "(cast(3000000000 as bigint) + 1) rows only") .returnsCount(146) - .explainContains("EnumerableLimit(fetch=[18446744073709551616:DECIMAL(20, 0)])\n" + .explainContains("EnumerableLimit(fetch=[3000000001:BIGINT])\n" + " CassandraToEnumerableConverter\n"); } From 6a25f7bb5d19b2f944bb71c3172869bd3a4913cd Mon Sep 17 00:00:00 2001 From: Kirill Tkalenko Date: Tue, 23 Jun 2026 15:45:15 +0300 Subject: [PATCH 12/12] [CALCITE-7592] Wip --- .../test/java/org/apache/calcite/test/DruidAdapter2IT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java index f5f889926e05..1323fcb3552b 100644 --- a/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java +++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java @@ -985,10 +985,10 @@ private void checkGroupBySingleSortLimit(boolean approx) { final String sql = "select \"state_province\"\n" + "from \"foodmart\"\n" + "fetch next " - + "(cast(18446744073709551616 as decimal(20, 0))) rows only"; + + "(cast(3000000000 as bigint) + 1) rows only"; sql(sql) .returnsCount(86837) - .explainContains("BindableSort(fetch=[18446744073709551616:DECIMAL(20, 0)])\n" + .explainContains("BindableSort(fetch=[3000000001:BIGINT])\n" + " DruidQuery(table=[[foodmart, foodmart]], "); }