From 07663723ed168b94885ad0a02bad5927bb33b0e5 Mon Sep 17 00:00:00 2001 From: kikyo Date: Thu, 26 May 2022 13:23:57 +0800 Subject: [PATCH 01/18] plan translator --- .../org/apache/doris/nereids/PlanVisitor.java | 14 +++++++ .../doris/nereids/operators/OperatorType.java | 3 ++ .../plans/physical/PhysicalAggregation.java | 26 ++++++++++++ .../plans/physical/PhysicalHashJoin.java | 22 ++++++++++ .../plans/physical/PhysicalSort.java | 39 ++++++++++++++++++ .../doris/nereids/properties/Order.java | 19 +++++++++ .../expressions/FunctionCallExpression.java | 40 +++++++++++++++++++ .../trees/plans/PhysicalPlanTranslator.java | 15 +++++++ .../doris/nereids/trees/plans/Plan.java | 2 + .../trees/plans/physical/PhysicalPlan.java | 1 + 10 files changed, 181 insertions(+) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/PlanVisitor.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/properties/Order.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanVisitor.java new file mode 100644 index 00000000000000..f7cedbff5dd836 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanVisitor.java @@ -0,0 +1,14 @@ +package org.apache.doris.nereids; + +import org.apache.doris.nereids.trees.plans.Plan; + +public abstract class PlanVisitor { + abstract R visit(Plan plan, C context); + + public R visitPhysicalAggregationPlan(Plan plan, C context) { + return null; + } + + public R visitPhysicalOlapScanPlan(Plan plan, C context) {return null;} + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/OperatorType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/OperatorType.java index 6ba4ee6d13cd77..df97d1704406ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/OperatorType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/OperatorType.java @@ -33,6 +33,9 @@ public enum OperatorType { PHYSICAL_PROJECT, PHYSICAL_FILTER, PHYSICAL_BROADCAST_HASH_JOIN, + PHYSICAL_AGGREGATION, + PHYSICAL_SORT, + PHYSICAL_HASH_JOIN, // pattern ANY, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java new file mode 100644 index 00000000000000..6379c63d97a5fb --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java @@ -0,0 +1,26 @@ +package org.apache.doris.nereids.operators.plans.physical; + +import org.apache.doris.analysis.AggregateInfo; +import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; + +import java.util.List; + +public class PhysicalAggregation extends PhysicalUnaryOperator{ + + private List groupByExprList; + + private List aggExprList; + + private AggregateInfo.AggPhase aggPhase; + + private boolean needFinalize; + + private boolean usingStream; + + public PhysicalAggregation() { + super(OperatorType.PHYSICAL_AGGREGATION); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java new file mode 100644 index 00000000000000..993242848b56f7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java @@ -0,0 +1,22 @@ +package org.apache.doris.nereids.operators.plans.physical; + +import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.plans.JoinType; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; + +public class PhysicalHashJoin + extends PhysicalBinaryOperator + { + + private JoinType joinType; + + // TODO: temp define, use Predicate when it's ready + private ComparisonPredicate predicate; + + public PhysicalHashJoin() { + super(OperatorType.PHYSICAL_HASH_JOIN); + } + + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java new file mode 100644 index 00000000000000..b2041f02b8706c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java @@ -0,0 +1,39 @@ +package org.apache.doris.nereids.operators.plans.physical; + +import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.properties.Order; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; + +import java.util.List; + +public class PhysicalSort extends PhysicalUnaryOperator { + + private int offset; + + private int limit; + + private List orderList; + + // if true, the output of this node feeds an AnalyticNode + private boolean isAnalyticSort; + + public PhysicalSort() { + super(OperatorType.PHYSICAL_SORT); + } + + public int getOffset() { + return offset; + } + + public int getLimit() { + return limit; + } + + public List getOrderList() { + return orderList; + } + + public boolean isAnalyticSort() { + return isAnalyticSort; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/Order.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/Order.java new file mode 100644 index 00000000000000..53e7ce7f709a37 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/Order.java @@ -0,0 +1,19 @@ +package org.apache.doris.nereids.properties; + +import org.apache.doris.nereids.trees.expressions.Expression; + +public class Order { + + private Expression expr; + + private boolean isAsc; + + private boolean nullFirst; + + public Order(Expression expr, boolean isAsc, boolean nullFirst) { + this.expr = expr; + this.isAsc = isAsc; + this.nullFirst = nullFirst; + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java new file mode 100644 index 00000000000000..cff315fa08902e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java @@ -0,0 +1,40 @@ +package org.apache.doris.nereids.trees.expressions; + +import org.apache.doris.analysis.FunctionName; +import org.apache.doris.catalog.Function; +import org.apache.doris.nereids.trees.NodeType; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +// Temp define +public class FunctionCallExpression extends Expression { + + private FunctionName functionName; + + private List params; + + private Function fn; + + public FunctionCallExpression(FunctionName functionName, + Function fn, Expression... children) { + super(NodeType.EXPRESSION, children); + this.functionName = functionName; + this.params = Arrays.stream(children).collect(Collectors.toList()); + this.fn = fn; + } + + public FunctionName getFunctionName() { + return functionName; + } + + public List getParams() { + return params; + } + + public Function getFn() { + return fn; + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java new file mode 100644 index 00000000000000..4dee698fafb285 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java @@ -0,0 +1,15 @@ +package org.apache.doris.nereids.trees.plans; + +import org.apache.doris.nereids.operators.plans.physical.PhysicalOperator; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.planner.PlanFragment; + +@SuppressWarnings("rawtypes") +public class PhysicalPlanTranslator { + + public PlanFragment translatePlan(PhysicalPlan physicalPlan) { + + } + + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java index c359a93ed73c68..f0c21828a426d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java @@ -44,4 +44,6 @@ public interface Plan< @Override Plan child(int index); + + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPlan.java index ed03a950338ec2..9301c294e642f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPlan.java @@ -35,4 +35,5 @@ public interface PhysicalPlan< @Override Plan child(int index); + } From ab472d31df5ffed5800e2b6fb3dc623fb0f3f079 Mon Sep 17 00:00:00 2001 From: kikyo Date: Wed, 8 Jun 2022 10:49:03 +0800 Subject: [PATCH 02/18] fix code style --- .../apache/doris/analysis/AggregateInfo.java | 210 ++++++++---------- .../plans/physical/PhysicalAggregation.java | 5 +- .../plans/physical/PhysicalHashJoin.java | 3 - .../plans/physical/PhysicalOperator.java | 2 +- .../plans/physical/PhysicalScan.java | 3 +- .../plans/physical/PhysicalSort.java | 10 +- .../trees/plans/PhysicalPlanTranslator.java | 1 + .../doris/nereids/trees/plans/Plan.java | 1 - .../apache/doris/planner/ExchangeNode.java | 9 - .../apache/doris/planner/HashJoinNode.java | 69 +++--- 10 files changed, 127 insertions(+), 186 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java index 32d6f30ec7b726..197d71de55276d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java @@ -38,7 +38,7 @@ * Encapsulates all the information needed to compute the aggregate functions of a single * Select block, including a possible 2nd phase aggregation step for DISTINCT aggregate * functions and merge aggregation steps needed for distributed execution. - * + *

* The latter requires a tree structure of AggregateInfo objects which express the * original aggregate computations as well as the necessary merging aggregate * computations. @@ -46,25 +46,25 @@ * SELECT COUNT(DISTINCT a, b, ..) GROUP BY x, y, ... * into an equivalent query with a inline view: * SELECT COUNT(*) FROM (SELECT DISTINCT a, b, ..., x, y, ...) GROUP BY x, y, ... - * + *

* The tree structure looks as follows: * - for non-distinct aggregation: - * - aggInfo: contains the original aggregation functions and grouping exprs - * - aggInfo.mergeAggInfo: contains the merging aggregation functions (grouping - * exprs are identical) + * - aggInfo: contains the original aggregation functions and grouping exprs + * - aggInfo.mergeAggInfo: contains the merging aggregation functions (grouping + * exprs are identical) * - for distinct aggregation (for an explanation of the phases, see - * SelectStmt.createDistinctAggInfo()): - * - aggInfo: contains the phase 1 aggregate functions and grouping exprs - * - aggInfo.2ndPhaseDistinctAggInfo: contains the phase 2 aggregate functions and - * grouping exprs - * - aggInfo.mergeAggInfo: contains the merging aggregate functions for the phase 1 - * computation (grouping exprs are identical) - * - aggInfo.2ndPhaseDistinctAggInfo.mergeAggInfo: contains the merging aggregate - * functions for the phase 2 computation (grouping exprs are identical) - * + * SelectStmt.createDistinctAggInfo()): + * - aggInfo: contains the phase 1 aggregate functions and grouping exprs + * - aggInfo.2ndPhaseDistinctAggInfo: contains the phase 2 aggregate functions and + * grouping exprs + * - aggInfo.mergeAggInfo: contains the merging aggregate functions for the phase 1 + * computation (grouping exprs are identical) + * - aggInfo.2ndPhaseDistinctAggInfo.mergeAggInfo: contains the merging aggregate + * functions for the phase 2 computation (grouping exprs are identical) + *

* In general, merging aggregate computations are idempotent; in other words, * aggInfo.mergeAggInfo == aggInfo.mergeAggInfo.mergeAggInfo. - * + *

* TODO: move the merge construction logic from SelectStmt into AggregateInfo * TODO: Add query tests for aggregation with intermediate tuples with num_nodes=1. */ @@ -72,15 +72,14 @@ public final class AggregateInfo extends AggregateInfoBase { private final static Logger LOG = LogManager.getLogger(AggregateInfo.class); public enum AggPhase { - FIRST, - FIRST_MERGE, - SECOND, - SECOND_MERGE; + FIRST, FIRST_MERGE, SECOND, SECOND_MERGE; public boolean isMerge() { return this == FIRST_MERGE || this == SECOND_MERGE; } - }; + } + + ; // created by createMergeAggInfo() private AggregateInfo mergeAggInfo; @@ -101,8 +100,7 @@ public boolean isMerge() { // Map from slots of outputTupleSmap_ to the corresponding slot in // intermediateTupleSmap_. - protected ExprSubstitutionMap outputToIntermediateTupleSmap = - new ExprSubstitutionMap(); + protected ExprSubstitutionMap outputToIntermediateTupleSmap = new ExprSubstitutionMap(); // if set, a subset of groupingExprs_; set and used during planning private List partitionExprs; @@ -120,13 +118,12 @@ public boolean isMerge() { private ArrayList lastIdx = Lists.newArrayList(); // C'tor creates copies of groupingExprs and aggExprs. - private AggregateInfo(ArrayList groupingExprs, - ArrayList aggExprs, AggPhase aggPhase) { + private AggregateInfo(ArrayList groupingExprs, ArrayList aggExprs, AggPhase aggPhase) { this(groupingExprs, aggExprs, aggPhase, false); } - private AggregateInfo(ArrayList groupingExprs, - ArrayList aggExprs, AggPhase aggPhase, boolean isMultiDistinct) { + private AggregateInfo(ArrayList groupingExprs, ArrayList aggExprs, AggPhase aggPhase, + boolean isMultiDistinct) { super(groupingExprs, aggExprs); this.aggPhase = aggPhase; this.isMultiDistinct = isMultiDistinct; @@ -151,8 +148,7 @@ private AggregateInfo(AggregateInfo other) { Preconditions.checkState(other.intermediateTupleDesc == other.outputTupleDesc); intermediateTupleSmap = outputTupleSmap; } - partitionExprs = - (other.partitionExprs != null) ? Expr.cloneList(other.partitionExprs) : null; + partitionExprs = (other.partitionExprs != null) ? Expr.cloneList(other.partitionExprs) : null; } public List getPartitionExprs() { @@ -172,13 +168,10 @@ public void setPartitionExprs(List exprs) { * If an aggTupleDesc is created, also registers eq predicates between the * grouping exprs and their respective slots with 'analyzer'. */ - static public AggregateInfo create( - ArrayList groupingExprs, ArrayList aggExprs, - TupleDescriptor tupleDesc, Analyzer analyzer) - throws AnalysisException { + static public AggregateInfo create(ArrayList groupingExprs, ArrayList aggExprs, + TupleDescriptor tupleDesc, Analyzer analyzer) throws AnalysisException { Preconditions.checkState( - (groupingExprs != null && !groupingExprs.isEmpty()) - || (aggExprs != null && !aggExprs.isEmpty())); + (groupingExprs != null && !groupingExprs.isEmpty()) || (aggExprs != null && !aggExprs.isEmpty())); AggregateInfo result = new AggregateInfo(groupingExprs, aggExprs, AggPhase.FIRST); // collect agg exprs with DISTINCT clause @@ -227,6 +220,7 @@ static public AggregateInfo create( /** * estimate if functions contains multi distinct + * * @param distinctAggExprs * @return */ @@ -277,29 +271,27 @@ public static boolean estimateIfContainsMultiDistinct(List dis * - aggTupleDesc * - a complete secondPhaseDistinctAggInfo * - mergeAggInfo - * + *

* At the moment, we require that all distinct aggregate * functions be applied to the same set of exprs (ie, we can't do something * like SELECT COUNT(DISTINCT id), COUNT(DISTINCT address)). * Aggregation happens in two successive phases: * - the first phase aggregates by all grouping exprs plus all parameter exprs - * of DISTINCT aggregate functions - * + * of DISTINCT aggregate functions + *

* Example: - * SELECT a, COUNT(DISTINCT b, c), MIN(d), COUNT(*) FROM T GROUP BY a + * SELECT a, COUNT(DISTINCT b, c), MIN(d), COUNT(*) FROM T GROUP BY a * - 1st phase grouping exprs: a, b, c * - 1st phase agg exprs: MIN(d), COUNT(*) * - 2nd phase grouping exprs: a * - 2nd phase agg exprs: COUNT(*), MIN(), - * SUM() - * + * SUM() + *

* TODO: expand implementation to cover the general case; this will require * a different execution strategy */ - private void createDistinctAggInfo( - ArrayList origGroupingExprs, - ArrayList distinctAggExprs, Analyzer analyzer) - throws AnalysisException { + private void createDistinctAggInfo(ArrayList origGroupingExprs, ArrayList distinctAggExprs, + Analyzer analyzer) throws AnalysisException { Preconditions.checkState(!distinctAggExprs.isEmpty()); // make sure that all DISTINCT params are the same; // ignore top-level implicit casts in the comparison, we might have inserted @@ -367,8 +359,7 @@ public ExprSubstitutionMap getOutputToIntermediateSmap() { } public boolean hasAggregateExprs() { - return !aggregateExprs.isEmpty() - || (secondPhaseDistinctAggInfo != null + return !aggregateExprs.isEmpty() || (secondPhaseDistinctAggInfo != null && !secondPhaseDistinctAggInfo.getAggregateExprs().isEmpty()); } @@ -414,20 +405,20 @@ public void getRefdSlots(List ids) { * Substitute all the expressions (grouping expr, aggregate expr) and update our * substitution map according to the given substitution map: * - smap typically maps from tuple t1 to tuple t2 (example: the smap of an - * inline view maps the virtual table ref t1 into a base table ref t2) + * inline view maps the virtual table ref t1 into a base table ref t2) * - our grouping and aggregate exprs need to be substituted with the given - * smap so that they also reference t2 + * smap so that they also reference t2 * - aggTupleSMap needs to be recomputed to map exprs based on t2 - * onto our aggTupleDesc (ie, the left-hand side needs to be substituted with - * smap) + * onto our aggTupleDesc (ie, the left-hand side needs to be substituted with + * smap) * - mergeAggInfo: this is not affected, because - * * its grouping and aggregate exprs only reference aggTupleDesc_ - * * its smap is identical to aggTupleSMap_ + * * its grouping and aggregate exprs only reference aggTupleDesc_ + * * its smap is identical to aggTupleSMap_ * - 2ndPhaseDistinctAggInfo: - * * its grouping and aggregate exprs also only reference aggTupleDesc_ - * and are therefore not affected - * * its smap needs to be recomputed to map exprs based on t2 to its own - * aggTupleDesc + * * its grouping and aggregate exprs also only reference aggTupleDesc_ + * and are therefore not affected + * * its smap needs to be recomputed to map exprs based on t2 to its own + * aggTupleDesc */ public void substitute(ExprSubstitutionMap smap, Analyzer analyzer) { groupingExprs = Expr.substituteList(groupingExprs, smap, analyzer, true); @@ -454,14 +445,14 @@ public void substitute(ExprSubstitutionMap smap, Analyzer analyzer) { * Create the info for an aggregation node that merges its pre-aggregated inputs: * - pre-aggregation is computed by 'this' * - tuple desc and smap are the same as that of the input (we're materializing - * the same logical tuple) + * the same logical tuple) * - grouping exprs: slotrefs to the input's grouping slots * - aggregate exprs: aggregation of the input's aggregateExprs slots - * + *

* The returned AggregateInfo shares its descriptor and smap with the input info; * createAggTupleDesc() must not be called on it. */ - private void createMergeAggInfo(Analyzer analyzer) { + private void createMergeAggInfo(Analyzer analyzer) { Preconditions.checkState(mergeAggInfo == null); TupleDescriptor inputDesc = intermediateTupleDesc; // construct grouping exprs @@ -478,20 +469,17 @@ private void createMergeAggInfo(Analyzer analyzer) { List paramExprs = new ArrayList<>(); // TODO(zhannngchen), change intermediate argument to a list, and remove this // ad-hoc logic - if (inputExpr.fn.functionName().equals("max_by") - || inputExpr.fn.functionName().equals("min_by")) { + if (inputExpr.fn.functionName().equals("max_by") || inputExpr.fn.functionName().equals("min_by")) { paramExprs.addAll(inputExpr.getFnParams().exprs()); } else { paramExprs.add(new SlotRef(inputDesc.getSlots().get(i + getGroupingExprs().size()))); } - FunctionCallExpr aggExpr = FunctionCallExpr.createMergeAggCall( - inputExpr, paramExprs); + FunctionCallExpr aggExpr = FunctionCallExpr.createMergeAggCall(inputExpr, paramExprs); aggExpr.analyzeNoThrow(analyzer); aggExprs.add(aggExpr); } - AggPhase aggPhase = - (this.aggPhase == AggPhase.FIRST) ? AggPhase.FIRST_MERGE : AggPhase.SECOND_MERGE; + AggPhase aggPhase = (this.aggPhase == AggPhase.FIRST) ? AggPhase.FIRST_MERGE : AggPhase.SECOND_MERGE; mergeAggInfo = new AggregateInfo(groupingExprs, aggExprs, aggPhase, isMultiDistinct); mergeAggInfo.intermediateTupleDesc = intermediateTupleDesc; mergeAggInfo.outputTupleDesc = outputTupleDesc; @@ -508,8 +496,7 @@ private void createMergeAggInfo(Analyzer analyzer) { * Returns null if firstIdx is greater than lastIdx. * Returns a SlotRef to the last slot if there is only one slot in range. */ - private Expr createCountDistinctAggExprParam(int firstIdx, int lastIdx, - ArrayList slots) { + private Expr createCountDistinctAggExprParam(int firstIdx, int lastIdx, ArrayList slots) { if (firstIdx > lastIdx) { return null; } @@ -539,15 +526,14 @@ private Expr createCountDistinctAggExprParam(int firstIdx, int lastIdx, * - 'this' is the phase 1 aggregation * - grouping exprs are those of the original query (param origGroupingExprs) * - aggregate exprs for the DISTINCT agg fns: these are aggregating the grouping - * slots that were added to the original grouping slots in phase 1; - * count is mapped to count(*) and sum is mapped to sum + * slots that were added to the original grouping slots in phase 1; + * count is mapped to count(*) and sum is mapped to sum * - other aggregate exprs: same as the non-DISTINCT merge case - * (count is mapped to sum, everything else stays the same) - * + * (count is mapped to sum, everything else stays the same) + *

* This call also creates the tuple descriptor and smap for the returned AggregateInfo. */ - private void createSecondPhaseAggInfo( - ArrayList origGroupingExprs, + private void createSecondPhaseAggInfo(ArrayList origGroupingExprs, ArrayList distinctAggExprs, Analyzer analyzer) throws AnalysisException { Preconditions.checkState(secondPhaseDistinctAggInfo == null); Preconditions.checkState(!distinctAggExprs.isEmpty()); @@ -569,8 +555,7 @@ private void createSecondPhaseAggInfo( // column-value combinations if any of the distinct columns are NULL. // This behavior is consistent with MySQL. Expr ifExpr = createCountDistinctAggExprParam(origGroupingExprs.size(), - origGroupingExprs.size() + inputExpr.getChildren().size() - 1, - inputDesc.getSlots()); + origGroupingExprs.size() + inputExpr.getChildren().size() - 1, inputDesc.getSlots()); Preconditions.checkNotNull(ifExpr); ifExpr.analyzeNoThrow(analyzer); aggExpr = new FunctionCallExpr(FunctionSet.COUNT, Lists.newArrayList(ifExpr)); @@ -604,24 +589,21 @@ private void createSecondPhaseAggInfo( FunctionCallExpr inputExpr = aggregateExprs.get(i); Preconditions.checkState(inputExpr.isAggregateFunction()); // we're aggregating an output slot of the 1st agg phase - Expr aggExprParam = - new SlotRef(inputDesc.getSlots().get(i + getGroupingExprs().size())); - FunctionCallExpr aggExpr = FunctionCallExpr.createMergeAggCall( - inputExpr, Lists.newArrayList(aggExprParam)); + Expr aggExprParam = new SlotRef(inputDesc.getSlots().get(i + getGroupingExprs().size())); + FunctionCallExpr aggExpr = FunctionCallExpr.createMergeAggCall(inputExpr, Lists.newArrayList(aggExprParam)); secondPhaseAggExprs.add(aggExpr); } - Preconditions.checkState( - secondPhaseAggExprs.size() == aggregateExprs.size() + distinctAggExprs.size()); + Preconditions.checkState(secondPhaseAggExprs.size() == aggregateExprs.size() + distinctAggExprs.size()); for (FunctionCallExpr aggExpr : secondPhaseAggExprs) { aggExpr.analyzeNoThrow(analyzer); Preconditions.checkState(aggExpr.isAggregateFunction()); } - ArrayList substGroupingExprs = - Expr.substituteList(origGroupingExprs, intermediateTupleSmap, analyzer, false); - secondPhaseDistinctAggInfo = - new AggregateInfo(substGroupingExprs, secondPhaseAggExprs, AggPhase.SECOND, isMultiDistinct); + ArrayList substGroupingExprs = Expr.substituteList(origGroupingExprs, intermediateTupleSmap, analyzer, + false); + secondPhaseDistinctAggInfo = new AggregateInfo(substGroupingExprs, secondPhaseAggExprs, AggPhase.SECOND, + isMultiDistinct); secondPhaseDistinctAggInfo.createTupleDescs(analyzer); secondPhaseDistinctAggInfo.createSecondPhaseAggSMap(this, distinctAggExprs); secondPhaseDistinctAggInfo.createMergeAggInfo(analyzer); @@ -631,8 +613,7 @@ private void createSecondPhaseAggInfo( * Create smap to map original grouping and aggregate exprs onto output * of secondPhaseDistinctAggInfo. */ - private void createSecondPhaseAggSMap( - AggregateInfo inputAggInfo, ArrayList distinctAggExprs) { + private void createSecondPhaseAggSMap(AggregateInfo inputAggInfo, ArrayList distinctAggExprs) { outputTupleSmap.clear(); int slotIdx = 0; ArrayList slotDescs = outputTupleDesc.getSlots(); @@ -654,14 +635,13 @@ private void createSecondPhaseAggSMap( int numOrigGroupingExprs = inputAggInfo.getGroupingExprs().size() - numDistinctParams; Preconditions.checkState( - slotDescs.size() == numOrigGroupingExprs + distinctAggExprs.size() - + inputAggInfo.getAggregateExprs().size()); + slotDescs.size() == numOrigGroupingExprs + distinctAggExprs.size() + inputAggInfo.getAggregateExprs() + .size()); // original grouping exprs -> first m slots for (int i = 0; i < numOrigGroupingExprs; ++i, ++slotIdx) { Expr groupingExpr = inputAggInfo.getGroupingExprs().get(i); - outputTupleSmap.put( - groupingExpr.clone(), new SlotRef(slotDescs.get(slotIdx))); + outputTupleSmap.put(groupingExpr.clone(), new SlotRef(slotDescs.get(slotIdx))); } // distinct agg exprs -> next n slots @@ -687,31 +667,25 @@ public void createSmaps(Analyzer analyzer) { Preconditions.checkNotNull(outputTupleDesc); Preconditions.checkNotNull(intermediateTupleDesc); - List exprs = Lists.newArrayListWithCapacity( - groupingExprs.size() + aggregateExprs.size()); + List exprs = Lists.newArrayListWithCapacity(groupingExprs.size() + aggregateExprs.size()); exprs.addAll(groupingExprs); exprs.addAll(aggregateExprs); for (int i = 0; i < exprs.size(); ++i) { Expr expr = exprs.get(i); if (expr.isImplicitCast()) { - outputTupleSmap.put(expr.getChild(0).clone(), - new SlotRef(outputTupleDesc.getSlots().get(i))); + outputTupleSmap.put(expr.getChild(0).clone(), new SlotRef(outputTupleDesc.getSlots().get(i))); } else { - outputTupleSmap.put(expr.clone(), - new SlotRef(outputTupleDesc.getSlots().get(i))); + outputTupleSmap.put(expr.clone(), new SlotRef(outputTupleDesc.getSlots().get(i))); } if (!requiresIntermediateTuple()) { continue; } - intermediateTupleSmap.put(expr.clone(), - new SlotRef(intermediateTupleDesc.getSlots().get(i))); - outputToIntermediateTupleSmap.put( - new SlotRef(outputTupleDesc.getSlots().get(i)), + intermediateTupleSmap.put(expr.clone(), new SlotRef(intermediateTupleDesc.getSlots().get(i))); + outputToIntermediateTupleSmap.put(new SlotRef(outputTupleDesc.getSlots().get(i)), new SlotRef(intermediateTupleDesc.getSlots().get(i))); if (i < groupingExprs.size()) { - analyzer.createAuxEquivPredicate( - new SlotRef(outputTupleDesc.getSlots().get(i)), + analyzer.createAuxEquivPredicate(new SlotRef(outputTupleDesc.getSlots().get(i)), new SlotRef(intermediateTupleDesc.getSlots().get(i))); } } @@ -733,8 +707,8 @@ public void createSmaps(Analyzer analyzer) { * - The parameters of the sum function may involve the columns of a materialized view. * - The type of this column may happen to be inconsistent with the column type of the base table. * - In order to ensure the correctness of the result, - * the parameter type needs to be changed to the type of the materialized view column - * to ensure the correctness of the result. + * the parameter type needs to be changed to the type of the materialized view column + * to ensure the correctness of the result. * - Currently only the sum function will involve this problem. */ public void updateTypeOfAggregateExprs() { @@ -761,10 +735,10 @@ public void updateTypeOfAggregateExprs() { * Mark slots required for this aggregation as materialized: * - all grouping output slots as well as grouping exprs * - for non-distinct aggregation: the aggregate exprs of materialized aggregate slots; - * this assumes that the output slots corresponding to aggregate exprs have already - * been marked by the consumer of this select block + * this assumes that the output slots corresponding to aggregate exprs have already + * been marked by the consumer of this select block * - for distinct aggregation, we mark all aggregate output slots in order to keep - * things simple + * things simple * Also computes materializedAggregateExprs. * This call must be idempotent because it may be called more than once for Union stmt. */ @@ -786,10 +760,8 @@ public void materializeRequiredSlots(Analyzer analyzer, ExprSubstitutionMap smap boolean isDistinctAgg = isDistinctAgg(); for (int i = 0; i < aggregateExprsSize; ++i) { FunctionCallExpr functionCallExpr = aggregateExprs.get(i); - SlotDescriptor slotDesc = - outputTupleDesc.getSlots().get(groupExprsSize + i); - SlotDescriptor intermediateSlotDesc = - intermediateTupleDesc.getSlots().get(groupExprsSize + i); + SlotDescriptor slotDesc = outputTupleDesc.getSlots().get(groupExprsSize + i); + SlotDescriptor intermediateSlotDesc = intermediateTupleDesc.getSlots().get(groupExprsSize + i); if (isDistinctAgg || isMultiDistinct) { slotDesc.setIsMaterialized(true); intermediateSlotDesc.setIsMaterialized(true); @@ -828,17 +800,14 @@ public DataPartition getPartition() { public String debugString() { StringBuilder out = new StringBuilder(super.debugString()); - out.append(MoreObjects.toStringHelper(this) - .add("phase", aggPhase) + out.append(MoreObjects.toStringHelper(this).add("phase", aggPhase) .add("intermediate_smap", intermediateTupleSmap.debugString()) - .add("output_smap", outputTupleSmap.debugString()) - .toString()); + .add("output_smap", outputTupleSmap.debugString()).toString()); if (mergeAggInfo != this && mergeAggInfo != null) { out.append("\nmergeAggInfo:\n" + mergeAggInfo.debugString()); } if (secondPhaseDistinctAggInfo != null) { - out.append("\nsecondPhaseDistinctAggInfo:\n" - + secondPhaseDistinctAggInfo.debugString()); + out.append("\nsecondPhaseDistinctAggInfo:\n" + secondPhaseDistinctAggInfo.debugString()); } return out.toString(); } @@ -857,9 +826,8 @@ public List getInputPartitionExprs() { return partitionExprs != null ? partitionExprs : groupingExprs; } - public static AggregateInfo create( - ArrayList groupingExprs, ArrayList aggExprs, - TupleDescriptor tupleDesc, TupleDescriptor intermediateTupleDesc, AggPhase phase, Analyzer analyzer) { + public static AggregateInfo create(ArrayList groupingExprs, ArrayList aggExprs, + TupleDescriptor tupleDesc, TupleDescriptor intermediateTupleDesc, AggPhase phase, Analyzer analyzer) { AggregateInfo result = new AggregateInfo(groupingExprs, aggExprs, phase); result.outputTupleDesc = tupleDesc; result.intermediateTupleDesc = intermediateTupleDesc; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java index 86da85729b4d27..bbbaac370a93df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java @@ -7,7 +7,7 @@ import java.util.List; -public class PhysicalAggregation extends PhysicalUnaryOperator{ +public class PhysicalAggregation extends PhysicalUnaryOperator { private List groupByExprList; @@ -25,7 +25,8 @@ public PhysicalAggregation() { super(OperatorType.PHYSICAL_AGGREGATION); } - public PhysicalAggregation(OperatorType type, List groupByExprList, List aggExprList, AggregateInfo.AggPhase aggPhase) { + public PhysicalAggregation(OperatorType type, List groupByExprList, List aggExprList, + AggregateInfo.AggPhase aggPhase) { super(type); this.groupByExprList = groupByExprList; this.aggExprList = aggExprList; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java index 9ebcdc753f7121..8d23d4070f24fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java @@ -5,9 +5,6 @@ import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; -import org.apache.doris.nereids.util.Utils; - -import java.util.List; public class PhysicalHashJoin extends PhysicalBinaryOperator diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalOperator.java index 1704c6ecac84f4..dd5aa79a9f72dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalOperator.java @@ -28,7 +28,7 @@ * interface for all concrete physical operator. */ public interface PhysicalOperator> extends PlanOperator { - List computeOutputs(LogicalProperties logicalProperties, Plan... inputs); + List computeOutputs(LogicalProperties logicalProperties, Plan... inputs); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalScan.java index 708edfbe278a98..bb10c2e1fc0b09 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalScan.java @@ -26,8 +26,7 @@ /** * Abstract class for all physical scan operator. */ -public abstract class PhysicalScan> - extends PhysicalLeafOperator { +public abstract class PhysicalScan> extends PhysicalLeafOperator { protected final List qualifier; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java index c2199f62507233..395bf473bec4c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java @@ -31,17 +31,13 @@ public class PhysicalSort extends PhysicalUnaryOperator orderList; - // if true, the output of this node feeds an AnalyticNode - private boolean isAnalyticSort; - private boolean useTopN; - public PhysicalSort(int offset, int limit, List orderList, boolean isAnalyticSort, boolean useTopN) { + public PhysicalSort(int offset, int limit, List orderList, boolean useTopN) { super(OperatorType.PHYSICAL_SORT); this.offset = offset; this.limit = limit; this.orderList = orderList; - this.isAnalyticSort = isAnalyticSort; this.useTopN = useTopN; } @@ -61,10 +57,6 @@ public List getOrderList() { return orderList; } - public boolean isAnalyticSort() { - return isAnalyticSort; - } - public boolean isUseTopN() { return useTopN; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java index 4caed2d9d1fa09..94eacf2bc9716c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java @@ -193,6 +193,7 @@ public PlanFragment visitPhysicalSortPlan( return mergeFragment; } + // TODO: support broadcast join / co-locate / bucket shuffle join later @Override public PlanFragment visitPhysicalHashJoinPlan( PhysicalPlan physicalPlan, PlanContext context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java index f0c21828a426d0..77bf7ac625e4e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java @@ -45,5 +45,4 @@ public interface Plan< @Override Plan child(int index); - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java index 94fea6f0b5dc1c..6d1125407ec194 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java @@ -64,8 +64,6 @@ public class ExchangeNode extends PlanNode { // only if mergeInfo_ is non-null, i.e. this is a merging exchange node. private long offset; - private DataPartition dataPartition; - /** * Create ExchangeNode that consumes output of inputNode. * An ExchangeNode doesn't have an input node as a child, which is why we @@ -160,11 +158,4 @@ public int getNumInstances() { return numInstances; } - public DataPartition getDataPartition() { - return dataPartition; - } - - public void setDataPartition(DataPartition dataPartition) { - this.dataPartition = dataPartition; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java index f51ff5d5128693..1ba13a8030c13d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java @@ -83,8 +83,8 @@ public class HashJoinNode extends PlanNode { private List hashOutputSlotIds; - public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, TableRef innerRef, - List eqJoinConjuncts, List otherJoinConjuncts) { + public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, TableRef innerRef, List eqJoinConjuncts, + List otherJoinConjuncts) { super(id, "HASH JOIN"); Preconditions.checkArgument(eqJoinConjuncts != null && !eqJoinConjuncts.isEmpty()); Preconditions.checkArgument(otherJoinConjuncts != null); @@ -139,8 +139,8 @@ public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, TableRef inne } } - public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, JoinOperator joinOp, - List eqJoinConjuncts, List otherJoinConjuncts) { + public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, JoinOperator joinOp, List eqJoinConjuncts, + List otherJoinConjuncts) { super(id, "HASH JOIN"); Preconditions.checkArgument(eqJoinConjuncts != null && !eqJoinConjuncts.isEmpty()); Preconditions.checkArgument(otherJoinConjuncts != null); @@ -150,7 +150,7 @@ public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, JoinOperator // TODO: Support not vec exec engine cut unless tupleid in semi/anti join if (VectorizedUtil.isVectorized()) { if (joinOp.equals(JoinOperator.LEFT_ANTI_JOIN) || joinOp.equals(JoinOperator.LEFT_SEMI_JOIN) - || joinOp.equals(JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN)) { + || joinOp.equals(JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN)) { tupleIds.addAll(outer.getTupleIds()); } else if (joinOp.equals(JoinOperator.RIGHT_ANTI_JOIN) || joinOp.equals(JoinOperator.RIGHT_SEMI_JOIN)) { tupleIds.addAll(inner.getTupleIds()); @@ -229,7 +229,7 @@ public void setColocate(boolean colocate, String reason) { /** * Calculate the slots output after going through the hash table in the hash join node. * The most essential difference between 'hashOutputSlots' and 'outputSlots' is that - * it's output needs to contain other conjunct and conjunct columns. + * it's output needs to contain other conjunct and conjunct columns. * hash output slots = output slots + conjunct slots + other conjunct slots * For example: * select b.k1 from test.t1 a right join test.t1 b on a.k1=b.k1 and b.k2>1 where a.k2>1; @@ -238,6 +238,7 @@ public void setColocate(boolean colocate, String reason) { * conjuncts: b.k2>1 * hash output slots: a.k2, b.k2, b.k1 * eq conjuncts: a.k1=b.k1 + * * @param slotIdList */ private void initHashOutputSlotIds(List slotIdList) { @@ -257,8 +258,8 @@ public void initOutputSlotIds(Set requiredSlotIdSet, Analyzer analyzer) outputSlotIds = Lists.newArrayList(); for (TupleId tupleId : tupleIds) { for (SlotDescriptor slotDescriptor : analyzer.getTupleDesc(tupleId).getSlots()) { - if (slotDescriptor.isMaterialized() - && (requiredSlotIdSet == null || requiredSlotIdSet.contains(slotDescriptor.getId()))) { + if (slotDescriptor.isMaterialized() && (requiredSlotIdSet == null || requiredSlotIdSet.contains( + slotDescriptor.getId()))) { outputSlotIds.add(slotDescriptor.getId()); } } @@ -297,13 +298,11 @@ public void init(Analyzer analyzer) throws UserException { computeStats(analyzer); ExprSubstitutionMap combinedChildSmap = getCombinedChildWithoutTupleIsNullSmap(); - List newEqJoinConjuncts = - Expr.substituteList(eqJoinConjuncts, combinedChildSmap, analyzer, false); - eqJoinConjuncts = newEqJoinConjuncts.stream() - .map(entity -> (BinaryPredicate) entity).collect(Collectors.toList()); + List newEqJoinConjuncts = Expr.substituteList(eqJoinConjuncts, combinedChildSmap, analyzer, false); + eqJoinConjuncts = newEqJoinConjuncts.stream().map(entity -> (BinaryPredicate) entity) + .collect(Collectors.toList()); assignedConjuncts = analyzer.getAssignedConjuncts(); - otherJoinConjuncts = - Expr.substituteList(otherJoinConjuncts, combinedChildSmap, analyzer, false); + otherJoinConjuncts = Expr.substituteList(otherJoinConjuncts, combinedChildSmap, analyzer, false); } private void replaceOutputSmapForOuterJoin() { @@ -339,8 +338,7 @@ public static final class EqJoinConjunctScanSlots { private final SlotDescriptor lhs; private final SlotDescriptor rhs; - private EqJoinConjunctScanSlots(Expr eqJoinConjunct, SlotDescriptor lhs, - SlotDescriptor rhs) { + private EqJoinConjunctScanSlots(Expr eqJoinConjunct, SlotDescriptor lhs, SlotDescriptor rhs) { this.eqJoinConjunct = eqJoinConjunct; this.lhs = lhs; this.rhs = rhs; @@ -412,8 +410,7 @@ private static boolean hasNumRowsAndNdvStats(SlotDescriptor slotDesc) { */ public static Map, List> groupByJoinedTupleIds( List eqJoinConjunctSlots) { - Map, List> scanSlotsByJoinedTids = - new LinkedHashMap<>(); + Map, List> scanSlotsByJoinedTids = new LinkedHashMap<>(); for (EqJoinConjunctScanSlots slots : eqJoinConjunctSlots) { Pair tids = Pair.create(slots.lhsTid(), slots.rhsTid()); List scanSlots = scanSlotsByJoinedTids.get(tids); @@ -473,7 +470,8 @@ private long getJoinCardinality() { * - we adjust the NDVs from both sides to account for predicates that may * might have reduce the cardinality and NDVs */ - private long getGenericJoinCardinality(List eqJoinConjunctSlots, long lhsCard, long rhsCard) { + private long getGenericJoinCardinality(List eqJoinConjunctSlots, long lhsCard, + long rhsCard) { Preconditions.checkState(joinOp.isInnerJoin() || joinOp.isOuterJoin()); Preconditions.checkState(!eqJoinConjunctSlots.isEmpty()); Preconditions.checkState(lhsCard >= 0 && rhsCard >= 0); @@ -588,8 +586,8 @@ protected void computeOldCardinality() { // FK/PK join (which doesn't alter the cardinality of the left-hand side) cardinality = getChild(0).cardinality; } else { - cardinality = Math.round((double) getChild(0).cardinality * (double) getChild( - 1).cardinality / (double) maxNumDistinct); + cardinality = Math.round( + (double) getChild(0).cardinality * (double) getChild(1).cardinality / (double) maxNumDistinct); LOG.debug("lhs card: {}, rhs card: {}", getChild(0).cardinality, getChild(1).cardinality); } LOG.debug("stats HashJoin: cardinality {}", cardinality); @@ -642,8 +640,7 @@ private long getSemiJoinCardinality() { // Return -1 if the cardinality of the returned side is unknown. long cardinality; - if (joinOp == JoinOperator.RIGHT_SEMI_JOIN - || joinOp == JoinOperator.RIGHT_ANTI_JOIN) { + if (joinOp == JoinOperator.RIGHT_SEMI_JOIN || joinOp == JoinOperator.RIGHT_ANTI_JOIN) { if (getChild(1).cardinality == -1) { return -1; } @@ -697,8 +694,8 @@ private long getSemiJoinCardinality() { @Override protected String debugString() { - return MoreObjects.toStringHelper(this).add("eqJoinConjuncts", - eqJoinConjunctsDebugString()).addValue(super.debugString()).toString(); + return MoreObjects.toStringHelper(this).add("eqJoinConjuncts", eqJoinConjunctsDebugString()) + .addValue(super.debugString()).toString(); } private String eqJoinConjunctsDebugString() { @@ -750,23 +747,23 @@ protected void toThrift(TPlanNode msg) { @Override public String getNodeExplainString(String detailPrefix, TExplainLevel detailLevel) { - String distrModeStr = - (distrMode != DistributionMode.NONE) ? (" (" + distrMode.toString() + ")") : ""; - StringBuilder output = new StringBuilder() - .append(detailPrefix).append("join op: ").append(joinOp.toString()).append(distrModeStr).append("\n"); + String distrModeStr = (distrMode != DistributionMode.NONE) ? (" (" + distrMode.toString() + ")") : ""; + StringBuilder output = new StringBuilder().append(detailPrefix).append("join op: ").append(joinOp.toString()) + .append(distrModeStr).append("\n"); if (detailLevel == TExplainLevel.BRIEF) { return output.toString(); } - output.append(detailPrefix).append("hash predicates:\n") - .append(detailPrefix).append("colocate: ").append(isColocate).append(isColocate ? "" : ", reason: " + colocateReason).append("\n"); + output.append(detailPrefix).append("hash predicates:\n").append(detailPrefix).append("colocate: ") + .append(isColocate).append(isColocate ? "" : ", reason: " + colocateReason).append("\n"); for (BinaryPredicate eqJoinPredicate : eqJoinConjuncts) { output.append(detailPrefix).append("equal join conjunct: ").append(eqJoinPredicate.toSql()).append("\n"); } if (!otherJoinConjuncts.isEmpty()) { - output.append(detailPrefix).append("other join predicates: ").append(getExplainString(otherJoinConjuncts)).append("\n"); + output.append(detailPrefix).append("other join predicates: ").append(getExplainString(otherJoinConjuncts)) + .append("\n"); } if (!conjuncts.isEmpty()) { output.append(detailPrefix).append("other predicates: ").append(getExplainString(conjuncts)).append("\n"); @@ -775,8 +772,7 @@ public String getNodeExplainString(String detailPrefix, TExplainLevel detailLeve output.append(detailPrefix).append("runtime filters: "); output.append(getRuntimeFilterExplainString(true)); } - output.append(detailPrefix).append(String.format( - "cardinality=%s", cardinality)).append("\n"); + output.append(detailPrefix).append(String.format("cardinality=%s", cardinality)).append("\n"); // todo unify in plan node if (outputSlotIds != null) { output.append(detailPrefix).append("output slot ids: "); @@ -805,10 +801,7 @@ public boolean isShuffleJoin() { } public enum DistributionMode { - NONE("NONE"), - BROADCAST("BROADCAST"), - PARTITIONED("PARTITIONED"), - BUCKET_SHUFFLE("BUCKET_SHUFFLE"); + NONE("NONE"), BROADCAST("BROADCAST"), PARTITIONED("PARTITIONED"), BUCKET_SHUFFLE("BUCKET_SHUFFLE"); private final String description; From 7d81e06e3a5a13a4575d945e4e7698cd0d9a311a Mon Sep 17 00:00:00 2001 From: kikyo Date: Wed, 8 Jun 2022 16:01:04 +0800 Subject: [PATCH 03/18] 1. add new agg phase enum for nereids 2. remove the Analyzer from PlanContext.java 3. implement PlanTranslator::visitPhysicalFilter --- .../apache/doris/analysis/AggregateInfo.java | 3 +- .../doris/analysis/DescriptorTable.java | 7 +++ .../nereids/operators/plans/AggPhase.java | 49 +++++++++++++++++++ .../plans/physical/PhysicalAggregation.java | 26 ++++++++-- .../plans/physical/PhysicalHashJoin.java | 17 +++++++ .../expressions/ExpressionConverter.java | 3 +- .../doris/nereids/trees/expressions/Slot.java | 11 +++++ .../trees/plans/PhysicalPlanTranslator.java | 32 ++++++------ .../nereids/trees/plans/PlanContext.java | 19 ++++--- 9 files changed, 135 insertions(+), 32 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/AggPhase.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java index 197d71de55276d..b8cfbda9f87a7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java @@ -827,11 +827,10 @@ public List getInputPartitionExprs() { } public static AggregateInfo create(ArrayList groupingExprs, ArrayList aggExprs, - TupleDescriptor tupleDesc, TupleDescriptor intermediateTupleDesc, AggPhase phase, Analyzer analyzer) { + TupleDescriptor tupleDesc, TupleDescriptor intermediateTupleDesc, AggPhase phase) { AggregateInfo result = new AggregateInfo(groupingExprs, aggExprs, phase); result.outputTupleDesc = tupleDesc; result.intermediateTupleDesc = intermediateTupleDesc; - result.materializeRequiredSlots(analyzer, null); return result; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java index 526c6e4806941c..595b7d65f7a86b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java @@ -73,6 +73,13 @@ public SlotDescriptor addSlotDescriptor(TupleDescriptor d) { return result; } + public SlotDescriptor addSlotDescriptor(TupleDescriptor d, int id) { + SlotDescriptor result = new SlotDescriptor(new SlotId(id), d); + d.addSlot(result); + slotDescs.put(result.getId(), result); + return result; + } + /** * Create copy of src with new id. The returned descriptor has its mem layout * computed. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/AggPhase.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/AggPhase.java new file mode 100644 index 00000000000000..c3ef8422c8197e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/AggPhase.java @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.operators.plans; + +import org.apache.doris.analysis.AggregateInfo; + +public enum AggPhase { + FIRST("FIRST", AggregateInfo.AggPhase.FIRST), + FIRST_MERGE("FIRST_MERGE", AggregateInfo.AggPhase.FIRST_MERGE), + SECOND("SECOND", AggregateInfo.AggPhase.SECOND), + SECOND_MERGE("SECOND_MERGE", AggregateInfo.AggPhase.SECOND_MERGE); + + private final String name; + + private final AggregateInfo.AggPhase execAggPhase; + + AggPhase(String name, AggregateInfo.AggPhase execAggPhase) { + this.name = name; + this.execAggPhase = execAggPhase; + } + + public boolean isMerge() { + return this == FIRST_MERGE || this == SECOND_MERGE; + } + + public AggregateInfo.AggPhase toExec() { + return this.execAggPhase; + } + + @Override + public String toString() { + return name; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java index bbbaac370a93df..cc786b6f217db0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java @@ -1,7 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package org.apache.doris.nereids.operators.plans.physical; import org.apache.doris.analysis.AggregateInfo; import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.plans.AggPhase; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; @@ -15,7 +33,7 @@ public class PhysicalAggregation extends PhysicalUnaryOperator partitionExprList; - private AggregateInfo.AggPhase aggPhase; + private AggPhase aggPhase; private boolean needFinalize; @@ -26,7 +44,7 @@ public PhysicalAggregation() { } public PhysicalAggregation(OperatorType type, List groupByExprList, List aggExprList, - AggregateInfo.AggPhase aggPhase) { + AggPhase aggPhase) { super(type); this.groupByExprList = groupByExprList; this.aggExprList = aggExprList; @@ -41,7 +59,7 @@ public List getAggExprList() { return aggExprList; } - public AggregateInfo.AggPhase getAggPhase() { + public AggPhase getAggPhase() { return aggPhase; } @@ -69,7 +87,7 @@ public void setPartitionExprList(List partitionExprList) { this.partitionExprList = partitionExprList; } - public void setAggPhase(AggregateInfo.AggPhase aggPhase) { + public void setAggPhase(AggPhase aggPhase) { this.aggPhase = aggPhase; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java index 8d23d4070f24fa..eb51bd4acd8984 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package org.apache.doris.nereids.operators.plans.physical; import org.apache.doris.nereids.operators.OperatorType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExpressionConverter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExpressionConverter.java index 78df86e5b06d0e..15d6de9ecc2444 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExpressionConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExpressionConverter.java @@ -6,7 +6,8 @@ public class ExpressionConverter { public static ExpressionConverter converter = new ExpressionConverter(); - // TODO: implement this + // TODO: implement this, besides if expression is a slot, should set the slotId to + // converted the org.apache.doris.analysis.Expr public Expr convert(Expression expression) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Slot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Slot.java index 16d23ffd98e870..3143b807f001cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Slot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Slot.java @@ -25,6 +25,13 @@ public abstract class Slot> extends NamedExpression implements LeafExpression { + private int id; + + public Slot(NodeType type, int id, Expression... children) { + super(type, children); + this.id = id; + } + public Slot(NodeType type) { super(type); } @@ -33,4 +40,8 @@ public Slot(NodeType type) { public Slot toSlot() { return this; } + + public int getId() { + return id; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java index 94eacf2bc9716c..90f9ae89364064 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.operators.AbstractOperator; import org.apache.doris.nereids.operators.plans.JoinType; import org.apache.doris.nereids.operators.plans.physical.PhysicalAggregation; +import org.apache.doris.nereids.operators.plans.physical.PhysicalFilter; import org.apache.doris.nereids.operators.plans.physical.PhysicalHashJoin; import org.apache.doris.nereids.operators.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.operators.plans.physical.PhysicalOperator; @@ -83,7 +84,7 @@ public PlanFragment visitPhysicalAggregationPlan( List slotList = physicalPlan.getOutput(); TupleDescriptor outputTupleDesc = generateTupleDesc(slotList, context, null); PhysicalAggregation physicalAggregation = (PhysicalAggregation) physicalPlan.getOperator(); - AggregateInfo.AggPhase phase = physicalAggregation.getAggPhase(); + AggregateInfo.AggPhase phase = physicalAggregation.getAggPhase().toExec(); List groupByExpressionList = physicalAggregation.getGroupByExprList(); ArrayList execGroupingExpressions = groupByExpressionList.stream() @@ -103,7 +104,7 @@ public PlanFragment visitPhysicalAggregationPlan( switch (phase) { case FIRST: aggInfo = AggregateInfo.create(execGroupingExpressions, execAggExpressions, outputTupleDesc, - outputTupleDesc, AggregateInfo.AggPhase.FIRST, context.getAnalyzer()); + outputTupleDesc, AggregateInfo.AggPhase.FIRST); aggregationNode = new AggregationNode(context.nextNodeId(), inputPlanFragment.getPlanRoot(), aggInfo); aggregationNode.unsetNeedsFinalize(); aggregationNode.setUseStreamingPreagg(physicalAggregation.isUsingStream()); @@ -114,7 +115,7 @@ public PlanFragment visitPhysicalAggregationPlan( break; case FIRST_MERGE: aggInfo = AggregateInfo.create(execGroupingExpressions, execAggExpressions, outputTupleDesc, - outputTupleDesc, AggregateInfo.AggPhase.FIRST_MERGE, context.getAnalyzer()); + outputTupleDesc, AggregateInfo.AggPhase.FIRST_MERGE); aggregationNode = new AggregationNode(context.nextNodeId(), inputPlanFragment.getPlanRoot(), aggInfo); break; default: @@ -172,9 +173,6 @@ public PlanFragment visitPhysicalSortPlan( PlanFragment mergeFragment = createParentFragment(childFragment, DataPartition.UNPARTITIONED, context); ExchangeNode exchNode = (ExchangeNode) mergeFragment.getPlanRoot(); - exec(() -> { - exchNode.init(context.getAnalyzer()); - }); exchNode.unsetLimit(); if (physicalSort.hasLimit()) { exchNode.setLimit(limit); @@ -221,9 +219,6 @@ public PlanFragment visitPhysicalHashJoinPlan( crossJoinNode.addConjuncts(conjuncts); ExchangeNode exchangeNode = new ExchangeNode(context.nextNodeId(), rightFragment.getPlanRoot(), false); exchangeNode.setNumInstances(rightFragmentPlanRoot.getNumInstances()); - exec(() -> { - exchangeNode.init(context.getAnalyzer()); - }); exchangeNode.setFragment(leftFragment); leftFragmentPlanRoot.setChild(1, exchangeNode); rightFragment.setDestination(exchangeNode); @@ -246,10 +241,6 @@ public PlanFragment visitPhysicalHashJoinPlan( leftExch.setNumInstances(leftFragmentPlanRoot.getNumInstances()); ExchangeNode rightExch = new ExchangeNode(context.nextNodeId(), leftFragmentPlanRoot, false); rightExch.setNumInstances(rightFragmentPlanRoot.getNumInstances()); - exec(() -> { - leftExch.init(context.getAnalyzer()); - rightExch.init(context.getAnalyzer()); - }); hashJoinNode.setChild(0, leftFragmentPlanRoot); hashJoinNode.setChild(1, leftFragmentPlanRoot); hashJoinNode.setDistributionMode(HashJoinNode.DistributionMode.PARTITIONED); @@ -268,7 +259,14 @@ public PlanFragment visitPhysicalProject( @Override public PlanFragment visitPhysicalFilter( PhysicalPlan physicalPlan, PlanContext context) { - return visit((PhysicalPlan) physicalPlan.child(0), context); + PlanFragment inputFragment = visit( + (PhysicalPlan) physicalPlan.child(0), context); + PlanNode planNode = inputFragment.getPlanRoot(); + PhysicalFilter filter = (PhysicalFilter) physicalPlan.getOperator(); + Expression expression = filter.getPredicates(); + List expressionList = Utils.extractConjuncts(expression); + expressionList.stream().map(ExpressionConverter.converter::convert).forEach(planNode::addConjunct); + return inputFragment; } private TupleDescriptor generateTupleDesc(List slotList, PlanContext context, Table table) { @@ -276,9 +274,10 @@ private TupleDescriptor generateTupleDesc(List slotList, PlanContext conte tupleDescriptor.setTable(table); for (Slot slot : slotList) { SlotReference slotReference = (SlotReference) slot; - SlotDescriptor slotDescriptor = context.addSlotDesc(tupleDescriptor); + SlotDescriptor slotDescriptor = context.addSlotDesc(tupleDescriptor, slot.getId()); slotDescriptor.setColumn(slotReference.getColumn()); slotDescriptor.setType(slotReference.getDataType().toCatalogDataType()); + slotDescriptor.setIsMaterialized(true); } return tupleDescriptor; } @@ -287,9 +286,6 @@ private PlanFragment createParentFragment(PlanFragment childFragment, DataPartit PlanContext ctx) { ExchangeNode exchangeNode = new ExchangeNode(ctx.nextNodeId(), childFragment.getPlanRoot(), false); exchangeNode.setNumInstances(childFragment.getPlanRoot().getNumInstances()); - exec(() -> { - exchangeNode.init(ctx.getAnalyzer()); - }); PlanFragment parentFragment = new PlanFragment(ctx.nextFragmentId(), exchangeNode, parentPartition); childFragment.setDestination(exchangeNode); childFragment.setOutputPartition(parentPartition); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanContext.java index 39cc289cdb5b63..86974b6e4034c0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanContext.java @@ -1,6 +1,7 @@ package org.apache.doris.nereids.trees.plans; import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.DescriptorTable; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.common.IdGenerator; @@ -8,12 +9,15 @@ import org.apache.doris.planner.PlanFragmentId; import org.apache.doris.planner.PlanNodeId; +import com.clearspring.analytics.util.Lists; + import java.util.List; public class PlanContext { - private List planFragmentList; + private List planFragmentList = Lists.newArrayList(); + + private DescriptorTable descTable = new DescriptorTable(); - private Analyzer analyzer; private final IdGenerator fragmentIdGenerator = PlanFragmentId.createGenerator(); @@ -24,7 +28,7 @@ public List getPlanFragmentList() { } public TupleDescriptor generateTupleDesc() { - return analyzer.getDescTbl().createTupleDescriptor(); + return descTable.createTupleDescriptor(); } public PlanNodeId nextNodeId() { @@ -32,7 +36,11 @@ public PlanNodeId nextNodeId() { } public SlotDescriptor addSlotDesc(TupleDescriptor t) { - return analyzer.getDescTbl().addSlotDescriptor(t); + return descTable.addSlotDescriptor(t); + } + + public SlotDescriptor addSlotDesc(TupleDescriptor t, int id) { + return descTable.addSlotDescriptor(t, id); } public PlanFragmentId nextFragmentId() { @@ -43,7 +51,4 @@ public void addPlanFragment(PlanFragment planFragment) { this.planFragmentList.add(planFragment); } - public Analyzer getAnalyzer() { - return analyzer; - } } From d68cf9f9d0050fbe79d7aed10c83a7a570bba465 Mon Sep 17 00:00:00 2001 From: kikyo Date: Thu, 9 Jun 2022 00:33:19 +0800 Subject: [PATCH 04/18] remove useless import --- .../org/apache/doris/nereids/properties/PhysicalProperties.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/PhysicalProperties.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/PhysicalProperties.java index e139d8a009e474..abe767c40e1166 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/PhysicalProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/PhysicalProperties.java @@ -17,8 +17,6 @@ package org.apache.doris.nereids.properties; -import org.apache.doris.analysis.DistributionDesc; - /** * Physical properties used in cascades. */ From 67d37fb7c4ffb5df310332d7347eaf2a8e2104f7 Mon Sep 17 00:00:00 2001 From: kikyo Date: Thu, 9 Jun 2022 10:44:55 +0800 Subject: [PATCH 05/18] fix code style --- .../nereids/operators/AbstractOperator.java | 4 +++- .../plans/physical/PhysicalAggregation.java | 1 - .../properties/RandomDistributionDesc.java | 17 +++++++++++++++++ .../trees/expressions/ExpressionConverter.java | 17 +++++++++++++++++ .../expressions/FunctionCallExpression.java | 17 +++++++++++++++++ .../doris/nereids/trees/plans/PlanContext.java | 18 +++++++++++++++++- 6 files changed, 71 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/AbstractOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/AbstractOperator.java index 61564c3dac919f..71af9ec54d30aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/AbstractOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/AbstractOperator.java @@ -39,7 +39,9 @@ public OperatorType getType() { return type; } - public R accept(PlanOperatorVisitor visitor, PhysicalPlan physicalPlan, C context){ return null; } + public R accept(PlanOperatorVisitor visitor, PhysicalPlan physicalPlan, C context) { + return null; + } public long getLimited() { return limited; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java index cc786b6f217db0..d7e319e4544a56 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.operators.plans.physical; -import org.apache.doris.analysis.AggregateInfo; import org.apache.doris.nereids.operators.OperatorType; import org.apache.doris.nereids.operators.plans.AggPhase; import org.apache.doris.nereids.trees.expressions.Expression; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RandomDistributionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RandomDistributionDesc.java index 2212bdc0834466..5d73dfee091f91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RandomDistributionDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RandomDistributionDesc.java @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package org.apache.doris.nereids.properties; public class RandomDistributionDesc extends DistributionSpec { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExpressionConverter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExpressionConverter.java index 15d6de9ecc2444..0212accd4bd180 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExpressionConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExpressionConverter.java @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package org.apache.doris.nereids.trees.expressions; import org.apache.doris.analysis.Expr; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java index cff315fa08902e..2ff6df4bc45805 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package org.apache.doris.nereids.trees.expressions; import org.apache.doris.analysis.FunctionName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanContext.java index 86974b6e4034c0..959b58ab68efac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanContext.java @@ -1,6 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package org.apache.doris.nereids.trees.plans; -import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.DescriptorTable; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.TupleDescriptor; From 672364a925c3b98d5129c74b366c9ce0a437fb43 Mon Sep 17 00:00:00 2001 From: kikyo Date: Thu, 9 Jun 2022 10:53:04 +0800 Subject: [PATCH 06/18] fix code style --- .../plans/physical/PhysicalHashJoin.java | 4 +--- .../properties/HashDistributionSpec.java | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java index eb51bd4acd8984..c7e79dc01b8830 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java @@ -23,9 +23,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; -public class PhysicalHashJoin - extends PhysicalBinaryOperator - { +public class PhysicalHashJoin extends PhysicalBinaryOperator { private JoinType joinType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/HashDistributionSpec.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/HashDistributionSpec.java index e98a4d6a5fb443..f2fab7103f66f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/HashDistributionSpec.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/HashDistributionSpec.java @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package org.apache.doris.nereids.properties; import org.apache.doris.analysis.HashDistributionDesc; From 83f06b1be2a19644d4c676e46e3d4857d7b9420e Mon Sep 17 00:00:00 2001 From: kikyo Date: Thu, 9 Jun 2022 12:31:00 +0800 Subject: [PATCH 07/18] 1. set properties of physical nodes as final 2. modify the parameter of base visit method 3. rename the visitxxxNode parameter name to concrete plan 4. implement the accept method in the concrete physical node 5. modify the plan type of the parameter of Operator to base Plan --- .../doris/nereids/PlanOperatorVisitor.java | 17 +++--- .../nereids/operators/AbstractOperator.java | 17 +++--- .../doris/nereids/operators/Operator.java | 4 +- .../plans/physical/PhysicalAggregation.java | 51 ++++++----------- .../plans/physical/PhysicalFilter.java | 8 +++ .../plans/physical/PhysicalHashJoin.java | 21 ++++--- .../plans/physical/PhysicalOlapScan.java | 22 +++++--- .../plans/physical/PhysicalProject.java | 8 +++ .../plans/physical/PhysicalScan.java | 4 +- .../plans/physical/PhysicalSort.java | 20 ++++--- .../trees/plans/PhysicalPlanTranslator.java | 56 +++++++++---------- 11 files changed, 123 insertions(+), 105 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java index 92d862c6b55e0f..dd04f44569339a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java @@ -17,39 +17,42 @@ package org.apache.doris.nereids; +import org.apache.doris.nereids.operators.Operator; import org.apache.doris.nereids.operators.plans.physical.PhysicalOperator; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; @SuppressWarnings("rawtypes") public abstract class PlanOperatorVisitor { - public abstract R visit(PhysicalPlan physicalPlan, C context); - public R visitPhysicalAggregationPlan(PhysicalPlan physicalPlan, + public abstract R visit(Plan plan, C context); + + public R visitPhysicalAggregationPlan(PhysicalPlan aggPlan, C context) { return null; } - public R visitPhysicalOlapScanPlan(PhysicalPlan physicalPlan, + public R visitPhysicalOlapScanPlan(PhysicalPlan olapScanPlan, C context) { return null; } - public R visitPhysicalSortPlan(PhysicalPlan physicalPlan, + public R visitPhysicalSortPlan(PhysicalPlan sortPlan, C context) { return null; } - public R visitPhysicalHashJoinPlan(PhysicalPlan physicalPlan, + public R visitPhysicalHashJoinPlan(PhysicalPlan hashJoinPlan, C context) { return null; } - public R visitPhysicalProject(PhysicalPlan physicalPlan, + public R visitPhysicalProject(PhysicalPlan projectPlan, C context) { return null; } - public R visitPhysicalFilter(PhysicalPlan physicalPlan, + public R visitPhysicalFilter(PhysicalPlan filterPlan, C context) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/AbstractOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/AbstractOperator.java index 71af9ec54d30aa..47567581b14226 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/AbstractOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/AbstractOperator.java @@ -18,8 +18,7 @@ package org.apache.doris.nereids.operators; import org.apache.doris.nereids.PlanOperatorVisitor; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.nereids.trees.plans.Plan; import java.util.Objects; @@ -28,10 +27,16 @@ */ public abstract class AbstractOperator> implements Operator { protected final OperatorType type; - protected long limited; + protected final long limited; public AbstractOperator(OperatorType type) { this.type = Objects.requireNonNull(type, "type can not be null"); + this.limited = -1; + } + + public AbstractOperator(OperatorType type, long limited) { + this.type = type; + this.limited = limited; } @Override @@ -39,7 +44,7 @@ public OperatorType getType() { return type; } - public R accept(PlanOperatorVisitor visitor, PhysicalPlan physicalPlan, C context) { + public R accept(PlanOperatorVisitor visitor, Plan plan, C context) { return null; } @@ -47,8 +52,4 @@ public long getLimited() { return limited; } - public void setLimited(long limited) { - this.limited = limited; - } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/Operator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/Operator.java index b62fec37f413ed..abca2b79d2ba4d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/Operator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/Operator.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.operators; import org.apache.doris.nereids.PlanOperatorVisitor; -import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.trees.TreeNode; @@ -30,6 +30,6 @@ public interface Operator> { NODE_TYPE toTreeNode(GroupExpression groupExpression); - public R accept(PlanOperatorVisitor visitor, PhysicalPlan physicalPlan, C context); + public R accept(PlanOperatorVisitor visitor, Plan plan, C context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java index d7e319e4544a56..2fbdc6d44e80a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java @@ -17,37 +17,38 @@ package org.apache.doris.nereids.operators.plans.physical; +import org.apache.doris.nereids.PlanOperatorVisitor; import org.apache.doris.nereids.operators.OperatorType; import org.apache.doris.nereids.operators.plans.AggPhase; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import java.util.List; public class PhysicalAggregation extends PhysicalUnaryOperator { - private List groupByExprList; + private final List groupByExprList; - private List aggExprList; + private final List aggExprList; - private List partitionExprList; + private final List partitionExprList; - private AggPhase aggPhase; + private final AggPhase aggPhase; - private boolean needFinalize; + private final boolean needFinalize; - private boolean usingStream; - - public PhysicalAggregation() { - super(OperatorType.PHYSICAL_AGGREGATION); - } + private final boolean usingStream; public PhysicalAggregation(OperatorType type, List groupByExprList, List aggExprList, - AggPhase aggPhase) { - super(type); + List partitionExprList, AggPhase aggPhase, boolean needFinalize, boolean usingStream) { + super(OperatorType.PHYSICAL_AGGREGATION); this.groupByExprList = groupByExprList; this.aggExprList = aggExprList; + this.partitionExprList = partitionExprList; this.aggPhase = aggPhase; + this.needFinalize = needFinalize; + this.usingStream = usingStream; } public List getGroupByExprList() { @@ -70,31 +71,13 @@ public boolean isUsingStream() { return usingStream; } - public void setGroupByExprList(List groupByExprList) { - this.groupByExprList = groupByExprList; - } - - public void setAggExprList(List aggExprList) { - this.aggExprList = aggExprList; - } - public List getPartitionExprList() { return partitionExprList; } - public void setPartitionExprList(List partitionExprList) { - this.partitionExprList = partitionExprList; - } - - public void setAggPhase(AggPhase aggPhase) { - this.aggPhase = aggPhase; - } - - public void setNeedFinalize(boolean needFinalize) { - this.needFinalize = needFinalize; - } - - public void setUsingStream(boolean usingStream) { - this.usingStream = usingStream; + @Override + public R accept(PlanOperatorVisitor visitor, Plan plan, C context) { + return visitor.visitPhysicalAggregationPlan( + (PhysicalPlan) plan, context); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalFilter.java index e3a8229b90f82f..b501abb91ed075 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalFilter.java @@ -17,9 +17,11 @@ package org.apache.doris.nereids.operators.plans.physical; +import org.apache.doris.nereids.PlanOperatorVisitor; import org.apache.doris.nereids.operators.OperatorType; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import java.util.Objects; @@ -50,4 +52,10 @@ public String toString() { } return "Filter (" + cond + ")"; } + + @Override + public R accept(PlanOperatorVisitor visitor, Plan plan, C context) { + return visitor.visitPhysicalFilter((PhysicalPlan) plan, + context); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java index c7e79dc01b8830..4e7686a1448880 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java @@ -17,28 +17,26 @@ package org.apache.doris.nereids.operators.plans.physical; +import org.apache.doris.nereids.PlanOperatorVisitor; import org.apache.doris.nereids.operators.OperatorType; import org.apache.doris.nereids.operators.plans.JoinType; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; public class PhysicalHashJoin extends PhysicalBinaryOperator { - private JoinType joinType; + private final JoinType joinType; - private Expression predicate; + private final Expression predicate; - public PhysicalHashJoin(OperatorType type, JoinType joinType, ComparisonPredicate predicate) { - super(type); + public PhysicalHashJoin(JoinType joinType, ComparisonPredicate predicate) { + super(OperatorType.PHYSICAL_HASH_JOIN); this.joinType = joinType; this.predicate = predicate; } - public PhysicalHashJoin() { - super(OperatorType.PHYSICAL_HASH_JOIN); - } - public JoinType getJoinType() { return joinType; } @@ -46,4 +44,11 @@ public JoinType getJoinType() { public Expression getPredicate() { return predicate; } + + @Override + public R accept(PlanOperatorVisitor visitor, Plan plan, C context) { + return visitor.visitPhysicalHashJoinPlan( + (PhysicalPlan) plan, context); + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalOlapScan.java index 406f066ef8ce60..ac8a51d3865c04 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalOlapScan.java @@ -19,8 +19,10 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.Table; +import org.apache.doris.nereids.PlanOperatorVisitor; import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import com.clearspring.analytics.util.Lists; import org.apache.commons.lang3.StringUtils; @@ -35,7 +37,7 @@ public class PhysicalOlapScan extends PhysicalScan { private final List selectedTabletId; private final List selectedPartitionId; - private OlapTable olapTable; + private final OlapTable olapTable; /** * Constructor for PhysicalOlapScan. @@ -44,7 +46,8 @@ public class PhysicalOlapScan extends PhysicalScan { * @param qualifier table's name */ public PhysicalOlapScan(OlapTable olapTable, List qualifier) { - super(OperatorType.PHYSICAL_OLAP_SCAN, olapTable, qualifier); + super(OperatorType.PHYSICAL_OLAP_SCAN, qualifier); + this.olapTable = olapTable; this.selectedIndexId = olapTable.getBaseIndexId(); this.selectedTabletId = Lists.newArrayList(); this.selectedPartitionId = olapTable.getPartitionIds(); @@ -72,9 +75,14 @@ public OlapTable getTable() { @Override public String toString() { return "Scan Olap Table " + StringUtils.join(qualifier, ".") + "." + olapTable.getName() - + " (selected index id: " + selectedTabletId - + ", selected partition ids: " + selectedPartitionId - + ", selected tablet ids: " + selectedTabletId - + ")"; + + " (selected index id: " + selectedTabletId + ", selected partition ids: " + selectedPartitionId + + ", selected tablet ids: " + selectedTabletId + ")"; } + + @Override + public R accept(PlanOperatorVisitor visitor, Plan plan, C context) { + return visitor.visitPhysicalOlapScanPlan( + (PhysicalPlan) plan, context); + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalProject.java index bd74fc813b29dc..29437d9c1fd9c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalProject.java @@ -17,9 +17,11 @@ package org.apache.doris.nereids.operators.plans.physical; +import org.apache.doris.nereids.PlanOperatorVisitor; import org.apache.doris.nereids.operators.OperatorType; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.commons.lang3.StringUtils; @@ -47,4 +49,10 @@ public List getProjects() { public String toString() { return "Project (" + StringUtils.join(projects, ", ") + ")"; } + + @Override + public R accept(PlanOperatorVisitor visitor, Plan plan, C context) { + return visitor.visitPhysicalProject( + (PhysicalPlan) plan, context); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalScan.java index bb10c2e1fc0b09..10d434c7e2c33c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalScan.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.operators.plans.physical; -import org.apache.doris.catalog.Table; import org.apache.doris.nereids.operators.OperatorType; import java.util.List; @@ -35,10 +34,9 @@ public abstract class PhysicalScan> extends Phys * Constructor for PhysicalScan. * * @param type node type - * @param table scan table * @param qualifier table's name */ - public PhysicalScan(OperatorType type, Table table, List qualifier) { + public PhysicalScan(OperatorType type, List qualifier) { super(type); this.qualifier = Objects.requireNonNull(qualifier, "qualifier can not be null"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java index 395bf473bec4c6..b871d831b95890 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java @@ -17,21 +17,23 @@ package org.apache.doris.nereids.operators.plans.physical; +import org.apache.doris.nereids.PlanOperatorVisitor; import org.apache.doris.nereids.operators.OperatorType; import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import java.util.List; public class PhysicalSort extends PhysicalUnaryOperator { - private int offset; + private final int offset; - private int limit = -1; + private final int limit; - private List orderList; + private final List orderList; - private boolean useTopN; + private final boolean useTopN; public PhysicalSort(int offset, int limit, List orderList, boolean useTopN) { super(OperatorType.PHYSICAL_SORT); @@ -41,10 +43,6 @@ public PhysicalSort(int offset, int limit, List orderList, boolean use this.useTopN = useTopN; } - public PhysicalSort() { - super(OperatorType.PHYSICAL_SORT); - } - public int getOffset() { return offset; } @@ -64,4 +62,10 @@ public boolean isUseTopN() { public boolean hasLimit() { return limit > -1; } + + @Override + public R accept(PlanOperatorVisitor visitor, Plan plan, C context) { + return visitor.visitPhysicalSortPlan((PhysicalPlan) plan, + context); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java index 90f9ae89364064..f45afbad464db5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java @@ -27,6 +27,7 @@ import org.apache.doris.catalog.Table; import org.apache.doris.nereids.PlanOperatorVisitor; import org.apache.doris.nereids.operators.AbstractOperator; +import org.apache.doris.nereids.operators.Operator; import org.apache.doris.nereids.operators.plans.JoinType; import org.apache.doris.nereids.operators.plans.physical.PhysicalAggregation; import org.apache.doris.nereids.operators.plans.physical.PhysicalFilter; @@ -67,23 +68,22 @@ public void translatePlan(PhysicalPlan physicalPlan, - PlanContext context) { - PhysicalOperator operator = physicalPlan.getOperator(); - return operator.accept(this, physicalPlan, context); + public PlanFragment visit(Plan plan, PlanContext context) { + PhysicalOperator operator = (PhysicalOperator) plan.getOperator(); + return operator.accept(this, plan, context); } @Override public PlanFragment visitPhysicalAggregationPlan( - PhysicalPlan physicalPlan, PlanContext context) { + PhysicalPlan aggPlan, PlanContext context) { PlanFragment inputPlanFragment = visit( - (PhysicalPlan) physicalPlan.child(0), context); + (PhysicalPlan) aggPlan.child(0), context); AggregationNode aggregationNode = null; - List slotList = physicalPlan.getOutput(); + List slotList = aggPlan.getOutput(); TupleDescriptor outputTupleDesc = generateTupleDesc(slotList, context, null); - PhysicalAggregation physicalAggregation = (PhysicalAggregation) physicalPlan.getOperator(); + PhysicalAggregation physicalAggregation = (PhysicalAggregation) aggPlan.getOperator(); AggregateInfo.AggPhase phase = physicalAggregation.getAggPhase().toExec(); List groupByExpressionList = physicalAggregation.getGroupByExprList(); @@ -127,10 +127,10 @@ public PlanFragment visitPhysicalAggregationPlan( @Override public PlanFragment visitPhysicalOlapScanPlan( - PhysicalPlan physicalPlan, PlanContext context) { + PhysicalPlan olapScanPlan, PlanContext context) { // Create OlapScanNode - List slotList = physicalPlan.getOutput(); - PhysicalOlapScan physicalOlapScan = (PhysicalOlapScan) physicalPlan.getOperator(); + List slotList = olapScanPlan.getOutput(); + PhysicalOlapScan physicalOlapScan = (PhysicalOlapScan) olapScanPlan.getOperator(); OlapTable olapTable = physicalOlapScan.getTable(); TupleDescriptor tupleDescriptor = generateTupleDesc(slotList, context, olapTable); OlapScanNode olapScanNode = new OlapScanNode(context.nextNodeId(), tupleDescriptor, olapTable.getName()); @@ -141,11 +141,11 @@ public PlanFragment visitPhysicalOlapScanPlan( } @Override - public PlanFragment visitPhysicalSortPlan( - PhysicalPlan physicalPlan, PlanContext context) { + public PlanFragment visitPhysicalSortPlan(PhysicalPlan sortPlan, + PlanContext context) { PlanFragment childFragment = visit( - (PhysicalPlan) physicalPlan.child(0), context); - PhysicalSort physicalSort = (PhysicalSort) physicalPlan.getOperator(); + (PhysicalPlan) sortPlan.child(0), context); + PhysicalSort physicalSort = (PhysicalSort) sortPlan.getOperator(); if (!childFragment.isPartitioned()) { return childFragment; } @@ -163,7 +163,7 @@ public PlanFragment visitPhysicalSortPlan( nullsFirstParamList.add(k.isNullFirst()); }); - List outputList = physicalPlan.getOutput(); + List outputList = sortPlan.getOutput(); TupleDescriptor tupleDesc = generateTupleDesc(outputList, context, null); SortInfo sortInfo = new SortInfo(execOrderingExprList, ascOrderList, nullsFirstParamList, tupleDesc); @@ -194,15 +194,15 @@ public PlanFragment visitPhysicalSortPlan( // TODO: support broadcast join / co-locate / bucket shuffle join later @Override public PlanFragment visitPhysicalHashJoinPlan( - PhysicalPlan physicalPlan, PlanContext context) { + PhysicalPlan hashJoinPlan, PlanContext context) { PlanFragment leftFragment = visit( - (PhysicalPlan) physicalPlan.child(0), context); + (PhysicalPlan) hashJoinPlan.child(0), context); PlanFragment rightFragment = visit( - (PhysicalPlan) physicalPlan.child(0), context); - PhysicalHashJoin physicalHashJoin = (PhysicalHashJoin) physicalPlan.getOperator(); + (PhysicalPlan) hashJoinPlan.child(0), context); + PhysicalHashJoin physicalHashJoin = (PhysicalHashJoin) hashJoinPlan.getOperator(); Expression predicateExpr = physicalHashJoin.getPredicate(); - List eqExprList = Utils.getEqConjuncts(physicalPlan.child(0).getOutput(), - physicalPlan.child(1).getOutput(), predicateExpr); + List eqExprList = Utils.getEqConjuncts(hashJoinPlan.child(0).getOutput(), + hashJoinPlan.child(1).getOutput(), predicateExpr); JoinType joinType = physicalHashJoin.getJoinType(); PlanNode leftFragmentPlanRoot = leftFragment.getPlanRoot(); @@ -252,17 +252,17 @@ public PlanFragment visitPhysicalHashJoinPlan( @Override public PlanFragment visitPhysicalProject( - PhysicalPlan physicalPlan, PlanContext context) { - return visit((PhysicalPlan) physicalPlan.child(0), context); + PhysicalPlan projectPlan, PlanContext context) { + return visit((PhysicalPlan) projectPlan.child(0), context); } @Override - public PlanFragment visitPhysicalFilter( - PhysicalPlan physicalPlan, PlanContext context) { + public PlanFragment visitPhysicalFilter(PhysicalPlan filterPlan, + PlanContext context) { PlanFragment inputFragment = visit( - (PhysicalPlan) physicalPlan.child(0), context); + (PhysicalPlan) filterPlan.child(0), context); PlanNode planNode = inputFragment.getPlanRoot(); - PhysicalFilter filter = (PhysicalFilter) physicalPlan.getOperator(); + PhysicalFilter filter = (PhysicalFilter) filterPlan.getOperator(); Expression expression = filter.getPredicates(); List expressionList = Utils.extractConjuncts(expression); expressionList.stream().map(ExpressionConverter.converter::convert).forEach(planNode::addConjunct); From 96750f6b84cb78971c0cda5ccbceee4f69e2fca3 Mon Sep 17 00:00:00 2001 From: kikyo Date: Thu, 9 Jun 2022 15:32:45 +0800 Subject: [PATCH 08/18] add concrete operator type for PlanVisitor's method --- .../doris/nereids/PlanOperatorVisitor.java | 19 ++++++++++------- .../plans/physical/PhysicalAggregation.java | 2 +- .../plans/physical/PhysicalFilter.java | 2 +- .../plans/physical/PhysicalHashJoin.java | 2 +- .../plans/physical/PhysicalOlapScan.java | 2 +- .../plans/physical/PhysicalProject.java | 2 +- .../plans/physical/PhysicalSort.java | 2 +- .../trees/plans/PhysicalPlanTranslator.java | 21 ++++++++++--------- 8 files changed, 29 insertions(+), 23 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java index dd04f44569339a..f49cdeaaeef934 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java @@ -18,7 +18,12 @@ package org.apache.doris.nereids; import org.apache.doris.nereids.operators.Operator; -import org.apache.doris.nereids.operators.plans.physical.PhysicalOperator; +import org.apache.doris.nereids.operators.plans.physical.PhysicalAggregation; +import org.apache.doris.nereids.operators.plans.physical.PhysicalFilter; +import org.apache.doris.nereids.operators.plans.physical.PhysicalHashJoin; +import org.apache.doris.nereids.operators.plans.physical.PhysicalOlapScan; +import org.apache.doris.nereids.operators.plans.physical.PhysicalProject; +import org.apache.doris.nereids.operators.plans.physical.PhysicalSort; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; @@ -27,32 +32,32 @@ public abstract class PlanOperatorVisitor { public abstract R visit(Plan plan, C context); - public R visitPhysicalAggregationPlan(PhysicalPlan aggPlan, + public R visitPhysicalAggregationPlan(PhysicalPlan aggPlan, C context) { return null; } - public R visitPhysicalOlapScanPlan(PhysicalPlan olapScanPlan, + public R visitPhysicalOlapScanPlan(PhysicalPlan olapScanPlan, C context) { return null; } - public R visitPhysicalSortPlan(PhysicalPlan sortPlan, + public R visitPhysicalSortPlan(PhysicalPlan sortPlan, C context) { return null; } - public R visitPhysicalHashJoinPlan(PhysicalPlan hashJoinPlan, + public R visitPhysicalHashJoinPlan(PhysicalPlan hashJoinPlan, C context) { return null; } - public R visitPhysicalProject(PhysicalPlan projectPlan, + public R visitPhysicalProject(PhysicalPlan projectPlan, C context) { return null; } - public R visitPhysicalFilter(PhysicalPlan filterPlan, + public R visitPhysicalFilter(PhysicalPlan filterPlan, C context) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java index 2fbdc6d44e80a7..4d17d640dabd48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java @@ -78,6 +78,6 @@ public List getPartitionExprList() { @Override public R accept(PlanOperatorVisitor visitor, Plan plan, C context) { return visitor.visitPhysicalAggregationPlan( - (PhysicalPlan) plan, context); + (PhysicalPlan) plan, context); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalFilter.java index b501abb91ed075..f62b69e3adfb1d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalFilter.java @@ -55,7 +55,7 @@ public String toString() { @Override public R accept(PlanOperatorVisitor visitor, Plan plan, C context) { - return visitor.visitPhysicalFilter((PhysicalPlan) plan, + return visitor.visitPhysicalFilter((PhysicalPlan) plan, context); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java index 4e7686a1448880..0a9ec4f414beee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java @@ -48,7 +48,7 @@ public Expression getPredicate() { @Override public R accept(PlanOperatorVisitor visitor, Plan plan, C context) { return visitor.visitPhysicalHashJoinPlan( - (PhysicalPlan) plan, context); + (PhysicalPlan) plan, context); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalOlapScan.java index ac8a51d3865c04..c56ae50c8126fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalOlapScan.java @@ -82,7 +82,7 @@ public String toString() { @Override public R accept(PlanOperatorVisitor visitor, Plan plan, C context) { return visitor.visitPhysicalOlapScanPlan( - (PhysicalPlan) plan, context); + (PhysicalPlan) plan, context); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalProject.java index 29437d9c1fd9c3..d5046083bfb075 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalProject.java @@ -53,6 +53,6 @@ public String toString() { @Override public R accept(PlanOperatorVisitor visitor, Plan plan, C context) { return visitor.visitPhysicalProject( - (PhysicalPlan) plan, context); + (PhysicalPlan) plan, context); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java index b871d831b95890..64b4d199c8e0af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java @@ -65,7 +65,7 @@ public boolean hasLimit() { @Override public R accept(PlanOperatorVisitor visitor, Plan plan, C context) { - return visitor.visitPhysicalSortPlan((PhysicalPlan) plan, + return visitor.visitPhysicalSortPlan((PhysicalPlan) plan, context); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java index f45afbad464db5..dcefe9dbf1900a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java @@ -34,6 +34,7 @@ import org.apache.doris.nereids.operators.plans.physical.PhysicalHashJoin; import org.apache.doris.nereids.operators.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.operators.plans.physical.PhysicalOperator; +import org.apache.doris.nereids.operators.plans.physical.PhysicalProject; import org.apache.doris.nereids.operators.plans.physical.PhysicalSort; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.trees.expressions.Expression; @@ -75,7 +76,7 @@ public PlanFragment visit(Plan plan, PlanCon @Override public PlanFragment visitPhysicalAggregationPlan( - PhysicalPlan aggPlan, PlanContext context) { + PhysicalPlan aggPlan, PlanContext context) { PlanFragment inputPlanFragment = visit( (PhysicalPlan) aggPlan.child(0), context); @@ -127,10 +128,10 @@ public PlanFragment visitPhysicalAggregationPlan( @Override public PlanFragment visitPhysicalOlapScanPlan( - PhysicalPlan olapScanPlan, PlanContext context) { + PhysicalPlan olapScanPlan, PlanContext context) { // Create OlapScanNode List slotList = olapScanPlan.getOutput(); - PhysicalOlapScan physicalOlapScan = (PhysicalOlapScan) olapScanPlan.getOperator(); + PhysicalOlapScan physicalOlapScan = olapScanPlan.getOperator(); OlapTable olapTable = physicalOlapScan.getTable(); TupleDescriptor tupleDescriptor = generateTupleDesc(slotList, context, olapTable); OlapScanNode olapScanNode = new OlapScanNode(context.nextNodeId(), tupleDescriptor, olapTable.getName()); @@ -141,11 +142,11 @@ public PlanFragment visitPhysicalOlapScanPlan( } @Override - public PlanFragment visitPhysicalSortPlan(PhysicalPlan sortPlan, + public PlanFragment visitPhysicalSortPlan(PhysicalPlan sortPlan, PlanContext context) { PlanFragment childFragment = visit( (PhysicalPlan) sortPlan.child(0), context); - PhysicalSort physicalSort = (PhysicalSort) sortPlan.getOperator(); + PhysicalSort physicalSort = sortPlan.getOperator(); if (!childFragment.isPartitioned()) { return childFragment; } @@ -194,12 +195,12 @@ public PlanFragment visitPhysicalSortPlan(PhysicalPlan hashJoinPlan, PlanContext context) { + PhysicalPlan hashJoinPlan, PlanContext context) { PlanFragment leftFragment = visit( (PhysicalPlan) hashJoinPlan.child(0), context); PlanFragment rightFragment = visit( (PhysicalPlan) hashJoinPlan.child(0), context); - PhysicalHashJoin physicalHashJoin = (PhysicalHashJoin) hashJoinPlan.getOperator(); + PhysicalHashJoin physicalHashJoin = hashJoinPlan.getOperator(); Expression predicateExpr = physicalHashJoin.getPredicate(); List eqExprList = Utils.getEqConjuncts(hashJoinPlan.child(0).getOutput(), hashJoinPlan.child(1).getOutput(), predicateExpr); @@ -252,17 +253,17 @@ public PlanFragment visitPhysicalHashJoinPlan( @Override public PlanFragment visitPhysicalProject( - PhysicalPlan projectPlan, PlanContext context) { + PhysicalPlan projectPlan, PlanContext context) { return visit((PhysicalPlan) projectPlan.child(0), context); } @Override - public PlanFragment visitPhysicalFilter(PhysicalPlan filterPlan, + public PlanFragment visitPhysicalFilter(PhysicalPlan filterPlan, PlanContext context) { PlanFragment inputFragment = visit( (PhysicalPlan) filterPlan.child(0), context); PlanNode planNode = inputFragment.getPlanRoot(); - PhysicalFilter filter = (PhysicalFilter) filterPlan.getOperator(); + PhysicalFilter filter = filterPlan.getOperator(); Expression expression = filter.getPredicates(); List expressionList = Utils.extractConjuncts(expression); expressionList.stream().map(ExpressionConverter.converter::convert).forEach(planNode::addConjunct); From d0422416a514eac46286baf31f8301d82307b961 Mon Sep 17 00:00:00 2001 From: kikyo Date: Thu, 9 Jun 2022 20:27:14 +0800 Subject: [PATCH 09/18] add some comments --- .../java/org/apache/doris/nereids/PlanOperatorVisitor.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java index f49cdeaaeef934..0172af645512db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java @@ -27,6 +27,11 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +/** + * Base class for the processing of logical and physical plan + * @param Return type of each visit method + * @param Context type + */ @SuppressWarnings("rawtypes") public abstract class PlanOperatorVisitor { From c6895b5faa7bd0b805e816a999b7f92181bf44f1 Mon Sep 17 00:00:00 2001 From: kikyo Date: Fri, 10 Jun 2022 11:03:49 +0800 Subject: [PATCH 10/18] fix check style --- .../main/java/org/apache/doris/nereids/operators/Operator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/Operator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/Operator.java index abca2b79d2ba4d..e3ea0fa3e97f53 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/Operator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/Operator.java @@ -30,6 +30,6 @@ public interface Operator> { NODE_TYPE toTreeNode(GroupExpression groupExpression); - public R accept(PlanOperatorVisitor visitor, Plan plan, C context); + public R accept(PlanOperatorVisitor visitor, Plan plan, C context); } From 5b0f10fb2a05d8a314aa3bcf7561bcec52f19f29 Mon Sep 17 00:00:00 2001 From: kikyo Date: Fri, 10 Jun 2022 12:03:49 +0800 Subject: [PATCH 11/18] revert unnecessary changes to comments --- .../apache/doris/analysis/AggregateInfo.java | 211 ++++++++++-------- 1 file changed, 122 insertions(+), 89 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java index b8cfbda9f87a7b..03f369b23df1f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java @@ -38,7 +38,7 @@ * Encapsulates all the information needed to compute the aggregate functions of a single * Select block, including a possible 2nd phase aggregation step for DISTINCT aggregate * functions and merge aggregation steps needed for distributed execution. - *

+ * * The latter requires a tree structure of AggregateInfo objects which express the * original aggregate computations as well as the necessary merging aggregate * computations. @@ -46,25 +46,26 @@ * SELECT COUNT(DISTINCT a, b, ..) GROUP BY x, y, ... * into an equivalent query with a inline view: * SELECT COUNT(*) FROM (SELECT DISTINCT a, b, ..., x, y, ...) GROUP BY x, y, ... - *

+ * * The tree structure looks as follows: + * 

  * - for non-distinct aggregation:
- * - aggInfo: contains the original aggregation functions and grouping exprs
- * - aggInfo.mergeAggInfo: contains the merging aggregation functions (grouping
- * exprs are identical)
+ *   - aggInfo: contains the original aggregation functions and grouping exprs
+ *   - aggInfo.mergeAggInfo: contains the merging aggregation functions (grouping
+ *     exprs are identical)
  * - for distinct aggregation (for an explanation of the phases, see
- * SelectStmt.createDistinctAggInfo()):
- * - aggInfo: contains the phase 1 aggregate functions and grouping exprs
- * - aggInfo.2ndPhaseDistinctAggInfo: contains the phase 2 aggregate functions and
- * grouping exprs
- * - aggInfo.mergeAggInfo: contains the merging aggregate functions for the phase 1
- * computation (grouping exprs are identical)
- * - aggInfo.2ndPhaseDistinctAggInfo.mergeAggInfo: contains the merging aggregate
- * functions for the phase 2 computation (grouping exprs are identical)
- * 

+ *   SelectStmt.createDistinctAggInfo()):
+ *   - aggInfo: contains the phase 1 aggregate functions and grouping exprs
+ *   - aggInfo.2ndPhaseDistinctAggInfo: contains the phase 2 aggregate functions and
+ *     grouping exprs
+ *   - aggInfo.mergeAggInfo: contains the merging aggregate functions for the phase 1
+ *     computation (grouping exprs are identical)
+ *   - aggInfo.2ndPhaseDistinctAggInfo.mergeAggInfo: contains the merging aggregate
+ *     functions for the phase 2 computation (grouping exprs are identical)
+ *
* In general, merging aggregate computations are idempotent; in other words, * aggInfo.mergeAggInfo == aggInfo.mergeAggInfo.mergeAggInfo. - *

+ * * TODO: move the merge construction logic from SelectStmt into AggregateInfo * TODO: Add query tests for aggregation with intermediate tuples with num_nodes=1. */ @@ -72,14 +73,15 @@ public final class AggregateInfo extends AggregateInfoBase { private final static Logger LOG = LogManager.getLogger(AggregateInfo.class); public enum AggPhase { - FIRST, FIRST_MERGE, SECOND, SECOND_MERGE; + FIRST, + FIRST_MERGE, + SECOND, + SECOND_MERGE; public boolean isMerge() { return this == FIRST_MERGE || this == SECOND_MERGE; } - } - - ; + }; // created by createMergeAggInfo() private AggregateInfo mergeAggInfo; @@ -100,7 +102,8 @@ public boolean isMerge() { // Map from slots of outputTupleSmap_ to the corresponding slot in // intermediateTupleSmap_. - protected ExprSubstitutionMap outputToIntermediateTupleSmap = new ExprSubstitutionMap(); + protected ExprSubstitutionMap outputToIntermediateTupleSmap = + new ExprSubstitutionMap(); // if set, a subset of groupingExprs_; set and used during planning private List partitionExprs; @@ -118,12 +121,13 @@ public boolean isMerge() { private ArrayList lastIdx = Lists.newArrayList(); // C'tor creates copies of groupingExprs and aggExprs. - private AggregateInfo(ArrayList groupingExprs, ArrayList aggExprs, AggPhase aggPhase) { + private AggregateInfo(ArrayList groupingExprs, + ArrayList aggExprs, AggPhase aggPhase) { this(groupingExprs, aggExprs, aggPhase, false); } - private AggregateInfo(ArrayList groupingExprs, ArrayList aggExprs, AggPhase aggPhase, - boolean isMultiDistinct) { + private AggregateInfo(ArrayList groupingExprs, + ArrayList aggExprs, AggPhase aggPhase, boolean isMultiDistinct) { super(groupingExprs, aggExprs); this.aggPhase = aggPhase; this.isMultiDistinct = isMultiDistinct; @@ -148,7 +152,8 @@ private AggregateInfo(AggregateInfo other) { Preconditions.checkState(other.intermediateTupleDesc == other.outputTupleDesc); intermediateTupleSmap = outputTupleSmap; } - partitionExprs = (other.partitionExprs != null) ? Expr.cloneList(other.partitionExprs) : null; + partitionExprs = + (other.partitionExprs != null) ? Expr.cloneList(other.partitionExprs) : null; } public List getPartitionExprs() { @@ -168,10 +173,13 @@ public void setPartitionExprs(List exprs) { * If an aggTupleDesc is created, also registers eq predicates between the * grouping exprs and their respective slots with 'analyzer'. */ - static public AggregateInfo create(ArrayList groupingExprs, ArrayList aggExprs, - TupleDescriptor tupleDesc, Analyzer analyzer) throws AnalysisException { + static public AggregateInfo create( + ArrayList groupingExprs, ArrayList aggExprs, + TupleDescriptor tupleDesc, Analyzer analyzer) + throws AnalysisException { Preconditions.checkState( - (groupingExprs != null && !groupingExprs.isEmpty()) || (aggExprs != null && !aggExprs.isEmpty())); + (groupingExprs != null && !groupingExprs.isEmpty()) + || (aggExprs != null && !aggExprs.isEmpty())); AggregateInfo result = new AggregateInfo(groupingExprs, aggExprs, AggPhase.FIRST); // collect agg exprs with DISTINCT clause @@ -220,7 +228,6 @@ static public AggregateInfo create(ArrayList groupingExprs, ArrayList dis * - aggTupleDesc * - a complete secondPhaseDistinctAggInfo * - mergeAggInfo - *

+ * * At the moment, we require that all distinct aggregate * functions be applied to the same set of exprs (ie, we can't do something * like SELECT COUNT(DISTINCT id), COUNT(DISTINCT address)). * Aggregation happens in two successive phases: * - the first phase aggregates by all grouping exprs plus all parameter exprs - * of DISTINCT aggregate functions - *

+ * of DISTINCT aggregate functions + * * Example: - * SELECT a, COUNT(DISTINCT b, c), MIN(d), COUNT(*) FROM T GROUP BY a + * SELECT a, COUNT(DISTINCT b, c), MIN(d), COUNT(*) FROM T GROUP BY a * - 1st phase grouping exprs: a, b, c * - 1st phase agg exprs: MIN(d), COUNT(*) * - 2nd phase grouping exprs: a * - 2nd phase agg exprs: COUNT(*), MIN(), - * SUM() - *

+ * SUM() + * * TODO: expand implementation to cover the general case; this will require * a different execution strategy */ - private void createDistinctAggInfo(ArrayList origGroupingExprs, ArrayList distinctAggExprs, - Analyzer analyzer) throws AnalysisException { + private void createDistinctAggInfo( + ArrayList origGroupingExprs, + ArrayList distinctAggExprs, Analyzer analyzer) + throws AnalysisException { Preconditions.checkState(!distinctAggExprs.isEmpty()); // make sure that all DISTINCT params are the same; // ignore top-level implicit casts in the comparison, we might have inserted @@ -359,7 +368,8 @@ public ExprSubstitutionMap getOutputToIntermediateSmap() { } public boolean hasAggregateExprs() { - return !aggregateExprs.isEmpty() || (secondPhaseDistinctAggInfo != null + return !aggregateExprs.isEmpty() + || (secondPhaseDistinctAggInfo != null && !secondPhaseDistinctAggInfo.getAggregateExprs().isEmpty()); } @@ -405,20 +415,20 @@ public void getRefdSlots(List ids) { * Substitute all the expressions (grouping expr, aggregate expr) and update our * substitution map according to the given substitution map: * - smap typically maps from tuple t1 to tuple t2 (example: the smap of an - * inline view maps the virtual table ref t1 into a base table ref t2) + * inline view maps the virtual table ref t1 into a base table ref t2) * - our grouping and aggregate exprs need to be substituted with the given - * smap so that they also reference t2 + * smap so that they also reference t2 * - aggTupleSMap needs to be recomputed to map exprs based on t2 - * onto our aggTupleDesc (ie, the left-hand side needs to be substituted with - * smap) + * onto our aggTupleDesc (ie, the left-hand side needs to be substituted with + * smap) * - mergeAggInfo: this is not affected, because - * * its grouping and aggregate exprs only reference aggTupleDesc_ - * * its smap is identical to aggTupleSMap_ + * * its grouping and aggregate exprs only reference aggTupleDesc_ + * * its smap is identical to aggTupleSMap_ * - 2ndPhaseDistinctAggInfo: - * * its grouping and aggregate exprs also only reference aggTupleDesc_ - * and are therefore not affected - * * its smap needs to be recomputed to map exprs based on t2 to its own - * aggTupleDesc + * * its grouping and aggregate exprs also only reference aggTupleDesc_ + * and are therefore not affected + * * its smap needs to be recomputed to map exprs based on t2 to its own + * aggTupleDesc */ public void substitute(ExprSubstitutionMap smap, Analyzer analyzer) { groupingExprs = Expr.substituteList(groupingExprs, smap, analyzer, true); @@ -445,14 +455,14 @@ public void substitute(ExprSubstitutionMap smap, Analyzer analyzer) { * Create the info for an aggregation node that merges its pre-aggregated inputs: * - pre-aggregation is computed by 'this' * - tuple desc and smap are the same as that of the input (we're materializing - * the same logical tuple) + * the same logical tuple) * - grouping exprs: slotrefs to the input's grouping slots * - aggregate exprs: aggregation of the input's aggregateExprs slots - *

+ * * The returned AggregateInfo shares its descriptor and smap with the input info; * createAggTupleDesc() must not be called on it. */ - private void createMergeAggInfo(Analyzer analyzer) { + private void createMergeAggInfo(Analyzer analyzer) { Preconditions.checkState(mergeAggInfo == null); TupleDescriptor inputDesc = intermediateTupleDesc; // construct grouping exprs @@ -469,17 +479,20 @@ private void createMergeAggInfo(Analyzer analyzer) { List paramExprs = new ArrayList<>(); // TODO(zhannngchen), change intermediate argument to a list, and remove this // ad-hoc logic - if (inputExpr.fn.functionName().equals("max_by") || inputExpr.fn.functionName().equals("min_by")) { + if (inputExpr.fn.functionName().equals("max_by") + || inputExpr.fn.functionName().equals("min_by")) { paramExprs.addAll(inputExpr.getFnParams().exprs()); } else { paramExprs.add(new SlotRef(inputDesc.getSlots().get(i + getGroupingExprs().size()))); } - FunctionCallExpr aggExpr = FunctionCallExpr.createMergeAggCall(inputExpr, paramExprs); + FunctionCallExpr aggExpr = FunctionCallExpr.createMergeAggCall( + inputExpr, paramExprs); aggExpr.analyzeNoThrow(analyzer); aggExprs.add(aggExpr); } - AggPhase aggPhase = (this.aggPhase == AggPhase.FIRST) ? AggPhase.FIRST_MERGE : AggPhase.SECOND_MERGE; + AggPhase aggPhase = + (this.aggPhase == AggPhase.FIRST) ? AggPhase.FIRST_MERGE : AggPhase.SECOND_MERGE; mergeAggInfo = new AggregateInfo(groupingExprs, aggExprs, aggPhase, isMultiDistinct); mergeAggInfo.intermediateTupleDesc = intermediateTupleDesc; mergeAggInfo.outputTupleDesc = outputTupleDesc; @@ -496,7 +509,8 @@ private void createMergeAggInfo(Analyzer analyzer) { * Returns null if firstIdx is greater than lastIdx. * Returns a SlotRef to the last slot if there is only one slot in range. */ - private Expr createCountDistinctAggExprParam(int firstIdx, int lastIdx, ArrayList slots) { + private Expr createCountDistinctAggExprParam(int firstIdx, int lastIdx, + ArrayList slots) { if (firstIdx > lastIdx) { return null; } @@ -526,14 +540,15 @@ private Expr createCountDistinctAggExprParam(int firstIdx, int lastIdx, ArrayLis * - 'this' is the phase 1 aggregation * - grouping exprs are those of the original query (param origGroupingExprs) * - aggregate exprs for the DISTINCT agg fns: these are aggregating the grouping - * slots that were added to the original grouping slots in phase 1; - * count is mapped to count(*) and sum is mapped to sum + * slots that were added to the original grouping slots in phase 1; + * count is mapped to count(*) and sum is mapped to sum * - other aggregate exprs: same as the non-DISTINCT merge case - * (count is mapped to sum, everything else stays the same) - *

+ * (count is mapped to sum, everything else stays the same) + * * This call also creates the tuple descriptor and smap for the returned AggregateInfo. */ - private void createSecondPhaseAggInfo(ArrayList origGroupingExprs, + private void createSecondPhaseAggInfo( + ArrayList origGroupingExprs, ArrayList distinctAggExprs, Analyzer analyzer) throws AnalysisException { Preconditions.checkState(secondPhaseDistinctAggInfo == null); Preconditions.checkState(!distinctAggExprs.isEmpty()); @@ -555,7 +570,8 @@ private void createSecondPhaseAggInfo(ArrayList origGroupingExprs, // column-value combinations if any of the distinct columns are NULL. // This behavior is consistent with MySQL. Expr ifExpr = createCountDistinctAggExprParam(origGroupingExprs.size(), - origGroupingExprs.size() + inputExpr.getChildren().size() - 1, inputDesc.getSlots()); + origGroupingExprs.size() + inputExpr.getChildren().size() - 1, + inputDesc.getSlots()); Preconditions.checkNotNull(ifExpr); ifExpr.analyzeNoThrow(analyzer); aggExpr = new FunctionCallExpr(FunctionSet.COUNT, Lists.newArrayList(ifExpr)); @@ -589,21 +605,24 @@ private void createSecondPhaseAggInfo(ArrayList origGroupingExprs, FunctionCallExpr inputExpr = aggregateExprs.get(i); Preconditions.checkState(inputExpr.isAggregateFunction()); // we're aggregating an output slot of the 1st agg phase - Expr aggExprParam = new SlotRef(inputDesc.getSlots().get(i + getGroupingExprs().size())); - FunctionCallExpr aggExpr = FunctionCallExpr.createMergeAggCall(inputExpr, Lists.newArrayList(aggExprParam)); + Expr aggExprParam = + new SlotRef(inputDesc.getSlots().get(i + getGroupingExprs().size())); + FunctionCallExpr aggExpr = FunctionCallExpr.createMergeAggCall( + inputExpr, Lists.newArrayList(aggExprParam)); secondPhaseAggExprs.add(aggExpr); } - Preconditions.checkState(secondPhaseAggExprs.size() == aggregateExprs.size() + distinctAggExprs.size()); + Preconditions.checkState( + secondPhaseAggExprs.size() == aggregateExprs.size() + distinctAggExprs.size()); for (FunctionCallExpr aggExpr : secondPhaseAggExprs) { aggExpr.analyzeNoThrow(analyzer); Preconditions.checkState(aggExpr.isAggregateFunction()); } - ArrayList substGroupingExprs = Expr.substituteList(origGroupingExprs, intermediateTupleSmap, analyzer, - false); - secondPhaseDistinctAggInfo = new AggregateInfo(substGroupingExprs, secondPhaseAggExprs, AggPhase.SECOND, - isMultiDistinct); + ArrayList substGroupingExprs = + Expr.substituteList(origGroupingExprs, intermediateTupleSmap, analyzer, false); + secondPhaseDistinctAggInfo = + new AggregateInfo(substGroupingExprs, secondPhaseAggExprs, AggPhase.SECOND, isMultiDistinct); secondPhaseDistinctAggInfo.createTupleDescs(analyzer); secondPhaseDistinctAggInfo.createSecondPhaseAggSMap(this, distinctAggExprs); secondPhaseDistinctAggInfo.createMergeAggInfo(analyzer); @@ -613,7 +632,8 @@ private void createSecondPhaseAggInfo(ArrayList origGroupingExprs, * Create smap to map original grouping and aggregate exprs onto output * of secondPhaseDistinctAggInfo. */ - private void createSecondPhaseAggSMap(AggregateInfo inputAggInfo, ArrayList distinctAggExprs) { + private void createSecondPhaseAggSMap( + AggregateInfo inputAggInfo, ArrayList distinctAggExprs) { outputTupleSmap.clear(); int slotIdx = 0; ArrayList slotDescs = outputTupleDesc.getSlots(); @@ -635,13 +655,14 @@ private void createSecondPhaseAggSMap(AggregateInfo inputAggInfo, ArrayList first m slots for (int i = 0; i < numOrigGroupingExprs; ++i, ++slotIdx) { Expr groupingExpr = inputAggInfo.getGroupingExprs().get(i); - outputTupleSmap.put(groupingExpr.clone(), new SlotRef(slotDescs.get(slotIdx))); + outputTupleSmap.put( + groupingExpr.clone(), new SlotRef(slotDescs.get(slotIdx))); } // distinct agg exprs -> next n slots @@ -667,25 +688,31 @@ public void createSmaps(Analyzer analyzer) { Preconditions.checkNotNull(outputTupleDesc); Preconditions.checkNotNull(intermediateTupleDesc); - List exprs = Lists.newArrayListWithCapacity(groupingExprs.size() + aggregateExprs.size()); + List exprs = Lists.newArrayListWithCapacity( + groupingExprs.size() + aggregateExprs.size()); exprs.addAll(groupingExprs); exprs.addAll(aggregateExprs); for (int i = 0; i < exprs.size(); ++i) { Expr expr = exprs.get(i); if (expr.isImplicitCast()) { - outputTupleSmap.put(expr.getChild(0).clone(), new SlotRef(outputTupleDesc.getSlots().get(i))); + outputTupleSmap.put(expr.getChild(0).clone(), + new SlotRef(outputTupleDesc.getSlots().get(i))); } else { - outputTupleSmap.put(expr.clone(), new SlotRef(outputTupleDesc.getSlots().get(i))); + outputTupleSmap.put(expr.clone(), + new SlotRef(outputTupleDesc.getSlots().get(i))); } if (!requiresIntermediateTuple()) { continue; } - intermediateTupleSmap.put(expr.clone(), new SlotRef(intermediateTupleDesc.getSlots().get(i))); - outputToIntermediateTupleSmap.put(new SlotRef(outputTupleDesc.getSlots().get(i)), + intermediateTupleSmap.put(expr.clone(), + new SlotRef(intermediateTupleDesc.getSlots().get(i))); + outputToIntermediateTupleSmap.put( + new SlotRef(outputTupleDesc.getSlots().get(i)), new SlotRef(intermediateTupleDesc.getSlots().get(i))); if (i < groupingExprs.size()) { - analyzer.createAuxEquivPredicate(new SlotRef(outputTupleDesc.getSlots().get(i)), + analyzer.createAuxEquivPredicate( + new SlotRef(outputTupleDesc.getSlots().get(i)), new SlotRef(intermediateTupleDesc.getSlots().get(i))); } } @@ -707,8 +734,8 @@ public void createSmaps(Analyzer analyzer) { * - The parameters of the sum function may involve the columns of a materialized view. * - The type of this column may happen to be inconsistent with the column type of the base table. * - In order to ensure the correctness of the result, - * the parameter type needs to be changed to the type of the materialized view column - * to ensure the correctness of the result. + * the parameter type needs to be changed to the type of the materialized view column + * to ensure the correctness of the result. * - Currently only the sum function will involve this problem. */ public void updateTypeOfAggregateExprs() { @@ -735,10 +762,10 @@ public void updateTypeOfAggregateExprs() { * Mark slots required for this aggregation as materialized: * - all grouping output slots as well as grouping exprs * - for non-distinct aggregation: the aggregate exprs of materialized aggregate slots; - * this assumes that the output slots corresponding to aggregate exprs have already - * been marked by the consumer of this select block + * this assumes that the output slots corresponding to aggregate exprs have already + * been marked by the consumer of this select block * - for distinct aggregation, we mark all aggregate output slots in order to keep - * things simple + * things simple * Also computes materializedAggregateExprs. * This call must be idempotent because it may be called more than once for Union stmt. */ @@ -760,8 +787,10 @@ public void materializeRequiredSlots(Analyzer analyzer, ExprSubstitutionMap smap boolean isDistinctAgg = isDistinctAgg(); for (int i = 0; i < aggregateExprsSize; ++i) { FunctionCallExpr functionCallExpr = aggregateExprs.get(i); - SlotDescriptor slotDesc = outputTupleDesc.getSlots().get(groupExprsSize + i); - SlotDescriptor intermediateSlotDesc = intermediateTupleDesc.getSlots().get(groupExprsSize + i); + SlotDescriptor slotDesc = + outputTupleDesc.getSlots().get(groupExprsSize + i); + SlotDescriptor intermediateSlotDesc = + intermediateTupleDesc.getSlots().get(groupExprsSize + i); if (isDistinctAgg || isMultiDistinct) { slotDesc.setIsMaterialized(true); intermediateSlotDesc.setIsMaterialized(true); @@ -800,14 +829,17 @@ public DataPartition getPartition() { public String debugString() { StringBuilder out = new StringBuilder(super.debugString()); - out.append(MoreObjects.toStringHelper(this).add("phase", aggPhase) + out.append(MoreObjects.toStringHelper(this) + .add("phase", aggPhase) .add("intermediate_smap", intermediateTupleSmap.debugString()) - .add("output_smap", outputTupleSmap.debugString()).toString()); + .add("output_smap", outputTupleSmap.debugString()) + .toString()); if (mergeAggInfo != this && mergeAggInfo != null) { out.append("\nmergeAggInfo:\n" + mergeAggInfo.debugString()); } if (secondPhaseDistinctAggInfo != null) { - out.append("\nsecondPhaseDistinctAggInfo:\n" + secondPhaseDistinctAggInfo.debugString()); + out.append("\nsecondPhaseDistinctAggInfo:\n" + + secondPhaseDistinctAggInfo.debugString()); } return out.toString(); } @@ -826,8 +858,9 @@ public List getInputPartitionExprs() { return partitionExprs != null ? partitionExprs : groupingExprs; } - public static AggregateInfo create(ArrayList groupingExprs, ArrayList aggExprs, - TupleDescriptor tupleDesc, TupleDescriptor intermediateTupleDesc, AggPhase phase) { + public static AggregateInfo create( + ArrayList groupingExprs, ArrayList aggExprs, + TupleDescriptor tupleDesc, TupleDescriptor intermediateTupleDesc, AggPhase phase) { AggregateInfo result = new AggregateInfo(groupingExprs, aggExprs, phase); result.outputTupleDesc = tupleDesc; result.intermediateTupleDesc = intermediateTupleDesc; From f8011c949a6808c4dad65c828f1d4679e232caf4 Mon Sep 17 00:00:00 2001 From: kikyo Date: Mon, 13 Jun 2022 15:46:36 +0800 Subject: [PATCH 12/18] add comment to the modifications to old optimizer --- .../src/main/java/org/apache/doris/analysis/AggregateInfo.java | 1 + .../src/main/java/org/apache/doris/planner/AggregationNode.java | 2 ++ .../src/main/java/org/apache/doris/planner/HashJoinNode.java | 1 + 3 files changed, 4 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java index 03f369b23df1f3..ff40097eed8cdb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java @@ -858,6 +858,7 @@ public List getInputPartitionExprs() { return partitionExprs != null ? partitionExprs : groupingExprs; } + // Used by new optimizer public static AggregateInfo create( ArrayList groupingExprs, ArrayList aggExprs, TupleDescriptor tupleDesc, TupleDescriptor intermediateTupleDesc, AggPhase phase) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java index 36454ce0cae298..ac7c7fee96499e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java @@ -103,10 +103,12 @@ public void setIsPreagg(PlannerContext ctx) { && aggInfo.getGroupingExprs().size() > 0; } + // Used by new optimizer public void setNeedsFinalize(boolean needsFinalize) { this.needsFinalize = needsFinalize; } + // Used by new optimizer public void setUseStreamingPreagg(boolean useStreamingPreagg) { this.useStreamingPreagg = useStreamingPreagg; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java index 2648fdb0a96481..34d18a3b17f13a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java @@ -141,6 +141,7 @@ public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, TableRef inne } } + // Used by new optimizer public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, JoinOperator joinOp, List eqJoinConjuncts, List otherJoinConjuncts) { super(id, "HASH JOIN", NodeType.HASH_JOIN_NODE); From 4d821f05ab7456f6af48c58f7a8167dcaee63731 Mon Sep 17 00:00:00 2001 From: kikyo Date: Tue, 14 Jun 2022 15:28:04 +0800 Subject: [PATCH 13/18] fix code style --- .../java/org/apache/doris/analysis/AggregateInfo.java | 8 +++++--- .../java/org/apache/doris/analysis/DescriptorTable.java | 3 +++ .../src/main/java/org/apache/doris/analysis/SortInfo.java | 3 +++ .../org/apache/doris/nereids/PlanOperatorVisitor.java | 6 +++--- .../apache/doris/nereids/operators/plans/AggPhase.java | 4 ++++ .../operators/plans/physical/PhysicalAggregation.java | 3 +++ .../operators/plans/physical/PhysicalHashJoin.java | 3 +++ .../nereids/operators/plans/physical/PhysicalSort.java | 3 +++ .../apache/doris/nereids/properties/DistributionSpec.java | 3 +++ .../doris/nereids/properties/HashDistributionSpec.java | 3 +++ .../org/apache/doris/nereids/properties/OrderKey.java | 3 +++ .../doris/nereids/properties/RandomDistributionDesc.java | 3 +++ .../nereids/trees/expressions/ExpressionConverter.java | 3 +++ .../nereids/trees/expressions/FunctionCallExpression.java | 4 +++- .../main/java/org/apache/doris/planner/HashJoinNode.java | 4 +++- 15 files changed, 48 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java index ff40097eed8cdb..9e32dae255b5a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java @@ -858,10 +858,12 @@ public List getInputPartitionExprs() { return partitionExprs != null ? partitionExprs : groupingExprs; } - // Used by new optimizer + /** + * Used by new optimizer. + */ public static AggregateInfo create( - ArrayList groupingExprs, ArrayList aggExprs, - TupleDescriptor tupleDesc, TupleDescriptor intermediateTupleDesc, AggPhase phase) { + ArrayList groupingExprs, ArrayList aggExprs, + TupleDescriptor tupleDesc, TupleDescriptor intermediateTupleDesc, AggPhase phase) { AggregateInfo result = new AggregateInfo(groupingExprs, aggExprs, phase); result.outputTupleDesc = tupleDesc; result.intermediateTupleDesc = intermediateTupleDesc; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java index 595b7d65f7a86b..f733e3a9a619a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java @@ -73,6 +73,9 @@ public SlotDescriptor addSlotDescriptor(TupleDescriptor d) { return result; } + /** + * Used by new optimizer. + */ public SlotDescriptor addSlotDescriptor(TupleDescriptor d, int id) { SlotDescriptor result = new SlotDescriptor(new SlotId(id), d); d.addSlot(result); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java index 1aa57ba858ec80..fe287e3900454a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java @@ -73,6 +73,9 @@ public SortInfo(List orderingExprs, List isAscOrder, materializedOrderingExprs = Lists.newArrayList(); } + /** + * Used by new optimizer. + */ public SortInfo(List orderingExprs, List isAscOrder, List nullsFirstParams, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java index 0172af645512db..36218799fef447 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java @@ -28,9 +28,9 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; /** - * Base class for the processing of logical and physical plan - * @param Return type of each visit method - * @param Context type + * Base class for the processing of logical and physical plan. + * @param Return type of each visit method. + * @param Context type. */ @SuppressWarnings("rawtypes") public abstract class PlanOperatorVisitor { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/AggPhase.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/AggPhase.java index c3ef8422c8197e..c365b06d89138f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/AggPhase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/AggPhase.java @@ -19,6 +19,10 @@ import org.apache.doris.analysis.AggregateInfo; +/** + * Represents different phase of agg and map it to the + * enum of agg phase definition of stale optimizer. + */ public enum AggPhase { FIRST("FIRST", AggregateInfo.AggPhase.FIRST), FIRST_MERGE("FIRST_MERGE", AggregateInfo.AggPhase.FIRST_MERGE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java index 4d17d640dabd48..38d153709622f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java @@ -26,6 +26,9 @@ import java.util.List; +/** + * Physical aggregation plan operator. + */ public class PhysicalAggregation extends PhysicalUnaryOperator { private final List groupByExprList; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java index 0a9ec4f414beee..1e88ad74216b3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java @@ -25,6 +25,9 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +/** + * Physical hash join plan operator + */ public class PhysicalHashJoin extends PhysicalBinaryOperator { private final JoinType joinType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java index 64b4d199c8e0af..cd763a74975ef3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java @@ -25,6 +25,9 @@ import java.util.List; +/** + * Physical sort plan operator. + */ public class PhysicalSort extends PhysicalUnaryOperator { private final int offset; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpec.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpec.java index 392e65071a1a97..4122de6901fd44 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpec.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpec.java @@ -19,6 +19,9 @@ import org.apache.doris.planner.DataPartition; +/** + * Base class for data distribution. + */ public class DistributionSpec { private DataPartition dataPartition; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/HashDistributionSpec.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/HashDistributionSpec.java index f2fab7103f66f0..9cdf6248aa2e6c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/HashDistributionSpec.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/HashDistributionSpec.java @@ -19,6 +19,9 @@ import org.apache.doris.analysis.HashDistributionDesc; +/** + * Describe hash distribution. + */ public class HashDistributionSpec extends DistributionSpec { public enum ShuffleType { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java index b11537a749c6f9..4169ea5f6fcf1d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java @@ -19,6 +19,9 @@ import org.apache.doris.nereids.trees.expressions.Expression; +/** + * Represents the order key of a statement. + */ public class OrderKey { private Expression expr; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RandomDistributionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RandomDistributionDesc.java index 5d73dfee091f91..f54ed12a19c4e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RandomDistributionDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RandomDistributionDesc.java @@ -17,6 +17,9 @@ package org.apache.doris.nereids.properties; +/** + * Describe random distribution. + */ public class RandomDistributionDesc extends DistributionSpec { } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExpressionConverter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExpressionConverter.java index 0212accd4bd180..40016180f86cdd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExpressionConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExpressionConverter.java @@ -19,6 +19,9 @@ import org.apache.doris.analysis.Expr; +/** + * Used to convert expression of new optimizer to stale expr. + */ public class ExpressionConverter { public static ExpressionConverter converter = new ExpressionConverter(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java index 2ff6df4bc45805..46933a7d8badec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java @@ -25,7 +25,9 @@ import java.util.List; import java.util.stream.Collectors; -// Temp define +/** + * Temp definition of FunctionCallExpression. + */ public class FunctionCallExpression extends Expression { private FunctionName functionName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java index 34d18a3b17f13a..3ddcefa18d742a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java @@ -141,7 +141,9 @@ public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, TableRef inne } } - // Used by new optimizer + /** + * This constructor is used by new optimizer. + */ public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, JoinOperator joinOp, List eqJoinConjuncts, List otherJoinConjuncts) { super(id, "HASH JOIN", NodeType.HASH_JOIN_NODE); From 95ca9ba40efaa8786c8ec3c668c3432df4157d7f Mon Sep 17 00:00:00 2001 From: kikyo Date: Tue, 14 Jun 2022 15:43:22 +0800 Subject: [PATCH 14/18] remove redundant code --- .../src/main/java/org/apache/doris/planner/HashJoinNode.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java index fdc667fd4a5078..39d597451bb9b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java @@ -762,9 +762,6 @@ public String getNodeExplainString(String detailPrefix, TExplainLevel detailLeve return output.toString(); } - output.append(detailPrefix).append("hash predicates:\n").append(detailPrefix).append("colocate: ") - .append(isColocate).append(isColocate ? "" : ", reason: " + colocateReason).append("\n"); - for (BinaryPredicate eqJoinPredicate : eqJoinConjuncts) { output.append(detailPrefix).append("equal join conjunct: ").append(eqJoinPredicate.toSql()).append("\n"); } From e9618882ac28208b1a376f35ad13e5395445ba59 Mon Sep 17 00:00:00 2001 From: kikyo Date: Tue, 14 Jun 2022 17:13:17 +0800 Subject: [PATCH 15/18] add some comment --- .../doris/nereids/PlanOperatorVisitor.java | 1 + .../nereids/operators/AbstractOperator.java | 8 ++++++++ .../doris/nereids/operators/Operator.java | 2 +- .../plans/physical/PhysicalAggregation.java | 19 ++++++++++--------- .../plans/physical/PhysicalHashJoin.java | 11 ++++++++--- .../plans/physical/PhysicalSort.java | 3 +++ .../properties/HashDistributionSpec.java | 3 +++ .../doris/nereids/properties/OrderKey.java | 5 +++++ .../expressions/FunctionCallExpression.java | 3 +++ .../trees/expressions/SlotReference.java | 1 - .../trees/plans/PhysicalPlanTranslator.java | 17 ++++++++++++++++- .../nereids/trees/plans/PlanContext.java | 3 +++ 12 files changed, 61 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java index 36218799fef447..063f92aae7aeaa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanOperatorVisitor.java @@ -29,6 +29,7 @@ /** * Base class for the processing of logical and physical plan. + * * @param Return type of each visit method. * @param Context type. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/AbstractOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/AbstractOperator.java index 47567581b14226..137317cbeb1eb2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/AbstractOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/AbstractOperator.java @@ -44,6 +44,14 @@ public OperatorType getType() { return type; } + /** + * Child operator should overwrite this method. + * for example: + * + * visitor.visitPhysicalOlapScanPlan( + * (PhysicalPlan) plan, context); + * + */ public R accept(PlanOperatorVisitor visitor, Plan plan, C context) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/Operator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/Operator.java index e3ea0fa3e97f53..efecb3a2013517 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/Operator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/Operator.java @@ -18,9 +18,9 @@ package org.apache.doris.nereids.operators; import org.apache.doris.nereids.PlanOperatorVisitor; -import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.trees.TreeNode; +import org.apache.doris.nereids.trees.plans.Plan; /** * interface for all concrete operator. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java index 38d153709622f4..feb72bd8377a7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java @@ -39,18 +39,23 @@ public class PhysicalAggregation extends PhysicalUnaryOperator groupByExprList, List aggExprList, - List partitionExprList, AggPhase aggPhase, boolean needFinalize, boolean usingStream) { + /** + * Constructor of PhysicalAggNode. + * + * @param groupByExprList group by expr list. + * @param aggExprList agg expr list. + * @param partitionExprList partition expr list, used for analytic agg. + * @param usingStream whether it's stream agg. + */ + public PhysicalAggregation(List groupByExprList, List aggExprList, + List partitionExprList, AggPhase aggPhase, boolean usingStream) { super(OperatorType.PHYSICAL_AGGREGATION); this.groupByExprList = groupByExprList; this.aggExprList = aggExprList; this.partitionExprList = partitionExprList; this.aggPhase = aggPhase; - this.needFinalize = needFinalize; this.usingStream = usingStream; } @@ -66,10 +71,6 @@ public AggPhase getAggPhase() { return aggPhase; } - public boolean isNeedFinalize() { - return needFinalize; - } - public boolean isUsingStream() { return usingStream; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java index 1e88ad74216b3e..41785dbab8a441 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java @@ -20,13 +20,12 @@ import org.apache.doris.nereids.PlanOperatorVisitor; import org.apache.doris.nereids.operators.OperatorType; import org.apache.doris.nereids.operators.plans.JoinType; -import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; /** - * Physical hash join plan operator + * Physical hash join plan operator. */ public class PhysicalHashJoin extends PhysicalBinaryOperator { @@ -34,7 +33,13 @@ public class PhysicalHashJoin extends PhysicalBinaryOperator orderList, boolean useTopN) { super(OperatorType.PHYSICAL_SORT); this.offset = offset; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/HashDistributionSpec.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/HashDistributionSpec.java index 9cdf6248aa2e6c..ade20758ffcb99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/HashDistributionSpec.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/HashDistributionSpec.java @@ -24,6 +24,9 @@ */ public class HashDistributionSpec extends DistributionSpec { + /** + * Enums for concrete shuffle type. + */ public enum ShuffleType { COLOCATE, BUCKET, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java index 4169ea5f6fcf1d..b01862dff64446 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java @@ -30,6 +30,11 @@ public class OrderKey { private boolean nullFirst; + /** + * Constructor of OrderKey. + * + * @param nullFirst True if "NULLS FIRST", false if "NULLS LAST", null if not specified. + */ public OrderKey(Expression expr, boolean isAsc, boolean nullFirst) { this.expr = expr; this.isAsc = isAsc; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java index 46933a7d8badec..88e4bc6ba59cab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java @@ -36,6 +36,9 @@ public class FunctionCallExpression extends Expression { private Function fn; + /** + * Constructor of FunctionCallExpression. + */ public FunctionCallExpression(FunctionName functionName, Function fn, Expression... children) { super(NodeType.EXPRESSION, children); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java index 1568ffe62cc061..acecad51ad6538 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.trees.expressions; import org.apache.doris.catalog.Column; -import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.trees.NodeType; import org.apache.doris.nereids.types.DataType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java index dcefe9dbf1900a..3c4384b67148b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PhysicalPlanTranslator.java @@ -60,6 +60,9 @@ import java.util.List; import java.util.stream.Collectors; +/** + * Used to translate to physical plan generated by new optimizer to the plan fragments. + */ @SuppressWarnings("rawtypes") public class PhysicalPlanTranslator extends PlanOperatorVisitor { @@ -74,6 +77,11 @@ public PlanFragment visit(Plan plan, PlanCon return operator.accept(this, plan, context); } + /** + * Translate in following steps: + * 1. + * + */ @Override public PlanFragment visitPhysicalAggregationPlan( PhysicalPlan aggPlan, PlanContext context) { @@ -151,7 +159,6 @@ public PlanFragment visitPhysicalSortPlan(PhysicalPlan execOrderingExprList = Lists.newArrayList(); List ascOrderList = Lists.newArrayList(); @@ -178,6 +185,7 @@ public PlanFragment visitPhysicalSortPlan(PhysicalPlan planFragmentList = Lists.newArrayList(); From a32e7829588a71ea12620be8817987617b912cd7 Mon Sep 17 00:00:00 2001 From: kikyo Date: Tue, 14 Jun 2022 17:29:32 +0800 Subject: [PATCH 16/18] add some comment --- .../apache/doris/analysis/AggregateInfo.java | 23 +++++++++---------- .../apache/doris/planner/HashJoinNode.java | 4 +++- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java index 9e32dae255b5a6..298837d11ea2ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java @@ -225,6 +225,17 @@ static public AggregateInfo create( return result; } + /** + * Used by new optimizer. + */ + public static AggregateInfo create( + ArrayList groupingExprs, ArrayList aggExprs, + TupleDescriptor tupleDesc, TupleDescriptor intermediateTupleDesc, AggPhase phase) { + AggregateInfo result = new AggregateInfo(groupingExprs, aggExprs, phase); + result.outputTupleDesc = tupleDesc; + result.intermediateTupleDesc = intermediateTupleDesc; + return result; + } /** * estimate if functions contains multi distinct @@ -858,16 +869,4 @@ public List getInputPartitionExprs() { return partitionExprs != null ? partitionExprs : groupingExprs; } - /** - * Used by new optimizer. - */ - public static AggregateInfo create( - ArrayList groupingExprs, ArrayList aggExprs, - TupleDescriptor tupleDesc, TupleDescriptor intermediateTupleDesc, AggPhase phase) { - AggregateInfo result = new AggregateInfo(groupingExprs, aggExprs, phase); - result.outputTupleDesc = tupleDesc; - result.intermediateTupleDesc = intermediateTupleDesc; - return result; - } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java index 39d597451bb9b4..0bc86e724a37bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java @@ -84,7 +84,9 @@ public class HashJoinNode extends PlanNode { private List hashOutputSlotIds; - + /** + * Constructor of HashJoinNode. + */ public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, TableRef innerRef, List eqJoinConjuncts, List otherJoinConjuncts) { super(id, "HASH JOIN", NodeType.HASH_JOIN_NODE); From 96e70c070b2a24289af516edd8cca2cc2b0c82aa Mon Sep 17 00:00:00 2001 From: kikyo Date: Thu, 9 Jun 2022 15:22:50 +0800 Subject: [PATCH 17/18] abstract the statistics objects --- .../java/org/apache/doris/analysis/Expr.java | 3 +- .../nereids/operators/plans/PlanOperator.java | 1 + .../nereids/trees/expressions/Expression.java | 25 ++++++++++- .../nereids/trees/plans/AbstractPlan.java | 37 +++++++++++++++++ .../nereids/trees/plans/PlaceHolderPlan.java | 34 +++++++++++++++ .../doris/nereids/trees/plans/Plan.java | 3 +- .../plans/physical/AbstractPhysicalPlan.java | 6 +++ .../trees/plans/physical/PhysicalPlan.java | 1 + .../org/apache/doris/planner/PlanNode.java | 8 +++- .../doris/statistics/AggStatsDerive.java | 3 +- .../doris/statistics/BaseStatsDerive.java | 13 +++--- .../apache/doris/statistics/ExprStats.java | 33 +++++++++++++++ .../doris/statistics/HashJoinStatsDerive.java | 2 +- .../doris/statistics/OlapScanStatsDerive.java | 3 +- .../apache/doris/statistics/PlanStats.java | 41 +++++++++++++++++++ .../statistics/StatsRecursiveDerive.java | 4 +- 16 files changed, 199 insertions(+), 18 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/statistics/ExprStats.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/statistics/PlanStats.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java index 53875392547c69..383373fb1055b2 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java @@ -31,6 +31,7 @@ import org.apache.doris.common.TreeNode; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.VectorizedUtil; +import org.apache.doris.statistics.ExprStats; import org.apache.doris.thrift.TExpr; import org.apache.doris.thrift.TExprNode; import org.apache.doris.thrift.TExprOpcode; @@ -60,7 +61,7 @@ /** * Root of the expr node hierarchy. */ -abstract public class Expr extends TreeNode implements ParseNode, Cloneable, Writable { +abstract public class Expr extends TreeNode implements ParseNode, Cloneable, Writable, ExprStats { private static final Logger LOG = LogManager.getLogger(Expr.class); // Name of the function that needs to be implemented by every Expr that diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/PlanOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/PlanOperator.java index 43adb3a4bfaa69..df138d142dda0c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/PlanOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/PlanOperator.java @@ -23,4 +23,5 @@ * interface for all concrete plan operator. */ public interface PlanOperator> extends Operator { + long getLimit(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java index af3b59687cb3df..3f68b73197a0f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.NodeType; import org.apache.doris.nereids.trees.TreeNode; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.statistics.ExprStats; import java.util.List; @@ -29,7 +30,9 @@ * Abstract class for all Expression in Nereids. */ public abstract class Expression> - extends AbstractTreeNode { + extends AbstractTreeNode implements ExprStats { + + protected double selectivity; public Expression(NodeType type, Expression... children) { super(type, children); @@ -61,4 +64,24 @@ public Expression child(int index) { public EXPR_TYPE newChildren(List children) { throw new RuntimeException(); } + + @Override + public boolean hasSelectivity() { + return false; + } + + @Override + public double getSelectivity() { + return 0; + } + + @Override + public void setSelectivity() { + selectivity = -1; + } + + @Override + public long getNumDistinctValues() { + return 0; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java index 581cd5bab0c1e2..6963b4c0432f54 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java @@ -21,10 +21,15 @@ import org.apache.doris.nereids.operators.plans.PlanOperator; import org.apache.doris.nereids.trees.AbstractTreeNode; import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.planner.PlanNode; +import org.apache.doris.statistics.ExprStats; +import org.apache.doris.statistics.PlanStats; +import org.apache.doris.statistics.StatsDeriveResult; import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; @@ -40,6 +45,8 @@ public abstract class AbstractPlan< extends AbstractTreeNode implements Plan { public final OP_TYPE operator; + protected StatsDeriveResult statsDeriveResult; + protected long limit; public AbstractPlan(NodeType type, OP_TYPE operator, Plan... children) { super(type, children); @@ -101,4 +108,34 @@ private void treeString(List lines, int depth, List lastChildre treeString(lines, depth + 1, newLasts, children.get(i)); } } + + @Override + public List getChildrenStats() { + return children(); + } + + @Override + public StatsDeriveResult getStatsDeriveResult() { + return statsDeriveResult; + } + + @Override + public PlanNode.NodeType getNodeType() { + return null; + } + + @Override + public void setStatsDeriveResult(StatsDeriveResult result) { + this.statsDeriveResult = result; + } + + @Override + public long getLimit() { + return getOperator().getLimit(); + } + + @Override + public List getConjuncts() { + return Collections.emptyList(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlaceHolderPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlaceHolderPlan.java index 014001e04c9a9c..d482225567de4f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlaceHolderPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlaceHolderPlan.java @@ -23,6 +23,10 @@ import org.apache.doris.nereids.trees.NodeType; import org.apache.doris.nereids.trees.TreeNode; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.planner.PlanNode; +import org.apache.doris.statistics.ExprStats; +import org.apache.doris.statistics.PlanStats; +import org.apache.doris.statistics.StatsDeriveResult; import java.util.List; @@ -77,4 +81,34 @@ public List children() { public Plan child(int index) { throw new RuntimeException(); } + + @Override + public List getChildrenStats() { + return null; + } + + @Override + public StatsDeriveResult getStatsDeriveResult() { + return null; + } + + @Override + public PlanNode.NodeType getNodeType() { + return null; + } + + @Override + public void setStatsDeriveResult(StatsDeriveResult result) { + } + + @Override + public long getLimit() { + return 0; + } + + @Override + public List getConjuncts() { + return null; + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java index 77bf7ac625e4e1..e5afe5c251e68d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.TreeNode; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.statistics.PlanStats; import java.util.List; @@ -29,7 +30,7 @@ */ public interface Plan< PLAN_TYPE extends Plan, - OP_TYPE extends PlanOperator> extends TreeNode { + OP_TYPE extends PlanOperator> extends TreeNode, PlanStats { OP_TYPE getOperator(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java index 81362b557c295a..daaf9dabbf4729 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java @@ -25,7 +25,12 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.planner.PlanNode; +import org.apache.doris.statistics.ExprStats; +import org.apache.doris.statistics.PlanStats; +import org.apache.doris.statistics.StatsDeriveResult; +import java.util.Collections; import java.util.List; import java.util.Objects; @@ -82,4 +87,5 @@ public LogicalProperties getLogicalProperties() { public PhysicalProperties getPhysicalProperties() { return physicalProperties; } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPlan.java index 9301c294e642f7..a3363e7cd83c26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPlan.java @@ -19,6 +19,7 @@ import org.apache.doris.nereids.operators.plans.physical.PhysicalOperator; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.statistics.PlanStats; import java.util.List; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java index 140e6d32c60c6f..5ec1b6dd3bac20 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java @@ -36,6 +36,7 @@ import org.apache.doris.common.TreeNode; import org.apache.doris.common.UserException; import org.apache.doris.common.util.VectorizedUtil; +import org.apache.doris.statistics.PlanStats; import org.apache.doris.statistics.StatsDeriveResult; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TFunctionBinaryType; @@ -71,7 +72,7 @@ * this node, ie, they only reference tuples materialized by this node or one of * its children (= are bound by tupleIds). */ -abstract public class PlanNode extends TreeNode { +abstract public class PlanNode extends TreeNode implements PlanStats { private final static Logger LOG = LogManager.getLogger(PlanNode.class); protected String planNodeName; @@ -353,6 +354,11 @@ public List getConjuncts() { return conjuncts; } + @Override + public List getChildrenStats() { + return children; + } + void initCompoundPredicate(Expr expr) { if (expr instanceof CompoundPredicate) { CompoundPredicate compoundPredicate = (CompoundPredicate) expr; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AggStatsDerive.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AggStatsDerive.java index 6d626ce21a20e0..365c8e8f39b2c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AggStatsDerive.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AggStatsDerive.java @@ -20,7 +20,6 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.common.UserException; import org.apache.doris.planner.AggregationNode; -import org.apache.doris.planner.PlanNode; import com.google.common.base.Preconditions; import org.apache.logging.log4j.LogManager; @@ -37,7 +36,7 @@ public class AggStatsDerive extends BaseStatsDerive { List groupingExprs = new ArrayList<>(); @Override - public void init(PlanNode node) throws UserException { + public void init(PlanStats node) throws UserException { Preconditions.checkState(node instanceof AggregationNode); super.init(node); groupingExprs.addAll(((AggregationNode) node).getAggInfo().getGroupingExprs()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseStatsDerive.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseStatsDerive.java index d2f047c8779d4e..5e1dd4984f4f80 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseStatsDerive.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseStatsDerive.java @@ -20,7 +20,6 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.SlotId; import org.apache.doris.common.UserException; -import org.apache.doris.planner.PlanNode; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -42,14 +41,14 @@ public class BaseStatsDerive { protected long rowCount = -1; protected long limit = -1; - protected List conjuncts = Lists.newArrayList(); + protected List conjuncts = Lists.newArrayList(); protected List childrenStatsResult = Lists.newArrayList(); - protected void init(PlanNode node) throws UserException { + protected void init(PlanStats node) throws UserException { limit = node.getLimit(); conjuncts.addAll(node.getConjuncts()); - for (PlanNode childNode : node.getChildren()) { + for (PlanStats childNode : node.getChildrenStats()) { StatsDeriveResult result = childNode.getStatsDeriveResult(); if (result == null) { throw new UserException( @@ -87,7 +86,7 @@ private void applySelectivity() { } protected double computeSelectivity() { - for (Expr expr : conjuncts) { + for (ExprStats expr : conjuncts) { expr.setSelectivity(); } return computeCombinedSelectivity(conjuncts); @@ -110,10 +109,10 @@ protected double computeSelectivity() { * * additional selectivity into the final result. *

*/ - protected double computeCombinedSelectivity(List conjuncts) { + protected double computeCombinedSelectivity(List conjuncts) { // Collect all estimated selectivities. List selectivities = new ArrayList<>(); - for (Expr e : conjuncts) { + for (ExprStats e : conjuncts) { if (e.hasSelectivity()) { selectivities.add(e.getSelectivity()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ExprStats.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ExprStats.java new file mode 100644 index 00000000000000..bc7effe452fabb --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ExprStats.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics; + +/** + * Used to abstract a common expression interface for statistics deduction to fit both optimizers. + */ +public interface ExprStats { + + boolean hasSelectivity(); + + double getSelectivity(); + + void setSelectivity(); + + long getNumDistinctValues(); + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HashJoinStatsDerive.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HashJoinStatsDerive.java index 649fd1cc7e3aa9..552133c2e8c864 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HashJoinStatsDerive.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HashJoinStatsDerive.java @@ -42,7 +42,7 @@ public class HashJoinStatsDerive extends BaseStatsDerive { private List eqJoinConjuncts = Lists.newArrayList(); @Override - public void init(PlanNode node) throws UserException { + public void init(PlanStats node) throws UserException { Preconditions.checkState(node instanceof HashJoinNode); super.init(node); joinOp = ((HashJoinNode) node).getJoinOp(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapScanStatsDerive.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapScanStatsDerive.java index efa1e1aafb3725..4e12cdc81477f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapScanStatsDerive.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapScanStatsDerive.java @@ -23,7 +23,6 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.planner.OlapScanNode; -import org.apache.doris.planner.PlanNode; import com.google.common.base.Preconditions; @@ -45,7 +44,7 @@ public class OlapScanStatsDerive extends BaseStatsDerive { private Map> slotIdToTableIdAndColumnName; @Override - public void init(PlanNode node) throws UserException { + public void init(PlanStats node) throws UserException { Preconditions.checkState(node instanceof OlapScanNode); super.init(node); buildStructure((OlapScanNode) node); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/PlanStats.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/PlanStats.java new file mode 100644 index 00000000000000..4c9f23ea702440 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/PlanStats.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics; + +import org.apache.doris.planner.PlanNode.NodeType; + +import java.util.List; + +/** + * Used to abstract a common operator interface for statistics deduction to fit both optimizers. + */ +public interface PlanStats { + + List getChildrenStats(); + + StatsDeriveResult getStatsDeriveResult(); + + NodeType getNodeType(); + + void setStatsDeriveResult(StatsDeriveResult result); + + long getLimit(); + + List getConjuncts(); + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsRecursiveDerive.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsRecursiveDerive.java index e6159a594de6ba..11941aa6e162d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsRecursiveDerive.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsRecursiveDerive.java @@ -38,11 +38,11 @@ private static class Inner { * This parameter is an input and output parameter, * which will store the derivation result of statistical information in the corresponding node */ - public void statsRecursiveDerive(PlanNode node) throws UserException { + public void statsRecursiveDerive(PlanStats node) throws UserException { if (node.getStatsDeriveResult() != null) { return; } - for (PlanNode childNode : node.getChildren()) { + for (PlanStats childNode : node.getChildrenStats()) { if (childNode.getStatsDeriveResult() == null) { statsRecursiveDerive(childNode); } From 0afa829a485429dde159295fbe63defe95b01399 Mon Sep 17 00:00:00 2001 From: kikyo Date: Thu, 16 Jun 2022 17:01:21 +0800 Subject: [PATCH 18/18] Refactor some code: 1. rename the org.apache.doris.nereids.trees.NodeType to org.apache.doris.nereids.trees.Operator 2. Move PlanNode.NodeType to org.apache.doris.statistics.NodeType 3. Abstract the code of statistics --- .../doris/nereids/analyzer/UnboundAlias.java | 4 +-- .../nereids/analyzer/UnboundRelation.java | 6 ++-- .../doris/nereids/analyzer/UnboundSlot.java | 4 +-- .../doris/nereids/analyzer/UnboundStar.java | 4 +-- .../nereids/operators/AbstractOperator.java | 8 ++--- .../doris/nereids/operators/Operator.java | 2 +- .../{OperatorType.java => PlanType.java} | 2 +- .../plans/logical/LogicalBinaryOperator.java | 4 +-- .../plans/logical/LogicalFilter.java | 4 +-- .../operators/plans/logical/LogicalJoin.java | 4 +-- .../plans/logical/LogicalLeafOperator.java | 4 +-- .../plans/logical/LogicalProject.java | 4 +-- .../plans/logical/LogicalRelation.java | 4 +-- .../plans/logical/LogicalUnaryOperator.java | 4 +-- .../plans/physical/PhysicalAggregation.java | 4 +-- .../physical/PhysicalBinaryOperator.java | 4 +-- .../physical/PhysicalBroadcastHashJoin.java | 5 ++- .../plans/physical/PhysicalFilter.java | 4 +-- .../plans/physical/PhysicalHashJoin.java | 4 +-- .../plans/physical/PhysicalLeafOperator.java | 4 +-- .../plans/physical/PhysicalOlapScan.java | 4 +-- .../plans/physical/PhysicalProject.java | 4 +-- .../plans/physical/PhysicalScan.java | 4 +-- .../plans/physical/PhysicalSort.java | 4 +-- .../plans/physical/PhysicalUnaryOperator.java | 4 +-- .../doris/nereids/pattern/GroupMatching.java | 6 ++-- .../apache/doris/nereids/pattern/Pattern.java | 36 +++++++++---------- .../doris/nereids/pattern/TypePattern.java | 6 ++-- .../rules/NormalizeExpressionRule.java | 4 +-- .../rewrite/rules/SimplifyNotExprRule.java | 4 +-- .../doris/nereids/trees/AbstractTreeNode.java | 8 ++--- .../{NodeType.java => OperatorType.java} | 2 +- .../apache/doris/nereids/trees/TreeNode.java | 2 +- .../nereids/trees/expressions/Alias.java | 4 +-- .../expressions/ComparisonPredicate.java | 4 +-- .../nereids/trees/expressions/EqualTo.java | 4 +-- .../nereids/trees/expressions/ExprId.java | 4 ++- .../nereids/trees/expressions/Expression.java | 5 ++- .../expressions/FunctionCallExpression.java | 4 +-- .../trees/expressions/GreaterThan.java | 4 +-- .../trees/expressions/GreaterThanEqual.java | 4 +-- .../nereids/trees/expressions/LessThan.java | 4 +-- .../trees/expressions/LessThanEqual.java | 4 +-- .../nereids/trees/expressions/Literal.java | 6 ++-- .../trees/expressions/NamedExpression.java | 4 +-- .../doris/nereids/trees/expressions/Not.java | 4 +-- .../trees/expressions/NullSafeEqual.java | 4 +-- .../doris/nereids/trees/expressions/Slot.java | 6 ++-- .../trees/expressions/SlotReference.java | 4 +-- .../nereids/trees/plans/AbstractPlan.java | 11 +++--- .../nereids/trees/plans/PlaceHolderPlan.java | 10 +++--- .../plans/logical/AbstractLogicalPlan.java | 6 ++-- .../plans/logical/LogicalBinaryPlan.java | 6 ++-- .../trees/plans/logical/LogicalLeafPlan.java | 6 ++-- .../trees/plans/logical/LogicalUnaryPlan.java | 6 ++-- .../plans/physical/AbstractPhysicalPlan.java | 11 ++---- .../plans/physical/PhysicalBinaryPlan.java | 6 ++-- .../plans/physical/PhysicalLeafPlan.java | 6 ++-- .../plans/physical/PhysicalUnaryPlan.java | 6 ++-- .../apache/doris/planner/AggregationNode.java | 1 + .../doris/planner/AnalyticEvalNode.java | 1 + .../doris/planner/AssertNumRowsNode.java | 1 + .../apache/doris/planner/BrokerScanNode.java | 1 + .../apache/doris/planner/CrossJoinNode.java | 1 + .../apache/doris/planner/EmptySetNode.java | 1 + .../org/apache/doris/planner/EsScanNode.java | 1 + .../org/apache/doris/planner/ExceptNode.java | 1 + .../apache/doris/planner/ExchangeNode.java | 1 + .../apache/doris/planner/HashJoinNode.java | 1 + .../apache/doris/planner/HiveScanNode.java | 1 + .../apache/doris/planner/IcebergScanNode.java | 1 + .../apache/doris/planner/LoadScanNode.java | 1 + .../apache/doris/planner/MysqlScanNode.java | 1 + .../apache/doris/planner/OdbcScanNode.java | 1 + .../apache/doris/planner/OlapScanNode.java | 1 + .../org/apache/doris/planner/PlanNode.java | 30 +--------------- .../org/apache/doris/planner/RepeatNode.java | 1 + .../org/apache/doris/planner/ScanNode.java | 1 + .../apache/doris/planner/SchemaScanNode.java | 1 + .../org/apache/doris/planner/SelectNode.java | 1 + .../doris/planner/SetOperationNode.java | 1 + .../org/apache/doris/planner/SortNode.java | 1 + .../doris/planner/StreamLoadScanNode.java | 1 + .../doris/planner/TableFunctionNode.java | 1 + .../org/apache/doris/planner/UnionNode.java | 1 + .../planner/external/ExternalScanNode.java | 1 + .../doris/statistics/BaseStatsDerive.java | 9 ++--- .../doris/statistics/DeriveFactory.java | 4 +-- .../org/apache/doris/statistics/NodeType.java | 30 ++++++++++++++++ .../doris/statistics/OlapScanStatsDerive.java | 10 +++--- .../apache/doris/statistics/PlanStats.java | 2 -- .../statistics/SlotStatsDeriveResult.java | 35 ++++++++++++++++++ .../doris/statistics/StatsDeriveResult.java | 12 +++---- .../nereids/jobs/RewriteTopDownJobTest.java | 6 ++-- .../pattern/GroupExpressionMatchingTest.java | 24 ++++++------- 95 files changed, 278 insertions(+), 224 deletions(-) rename fe/fe-core/src/main/java/org/apache/doris/nereids/operators/{OperatorType.java => PlanType.java} (98%) rename fe/fe-core/src/main/java/org/apache/doris/nereids/trees/{NodeType.java => OperatorType.java} (97%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/statistics/NodeType.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/statistics/SlotStatsDeriveResult.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundAlias.java index d7140292f45472..ecb8363ca2c6af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundAlias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundAlias.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.analyzer; import org.apache.doris.nereids.exceptions.UnboundException; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -34,7 +34,7 @@ public class UnboundAlias implements UnaryExpression { public UnboundAlias(CHILD_TYPE child) { - super(NodeType.UNBOUND_ALIAS, child); + super(OperatorType.UNBOUND_ALIAS, child); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java index 550894126ddd85..86a0ce648d9457 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java @@ -19,7 +19,7 @@ import org.apache.doris.nereids.analyzer.identifier.TableIdentifier; import org.apache.doris.nereids.exceptions.UnboundException; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.operators.plans.logical.LogicalLeafOperator; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.util.Utils; @@ -36,7 +36,7 @@ public class UnboundRelation extends LogicalLeafOperator { private final List nameParts; public UnboundRelation(List nameParts) { - super(OperatorType.LOGICAL_UNBOUND_RELATION); + super(PlanType.LOGICAL_UNBOUND_RELATION); this.nameParts = nameParts; } @@ -46,7 +46,7 @@ public UnboundRelation(List nameParts) { * @param identifier relation identifier */ public UnboundRelation(TableIdentifier identifier) { - super(OperatorType.LOGICAL_UNBOUND_RELATION); + super(PlanType.LOGICAL_UNBOUND_RELATION); this.nameParts = Lists.newArrayList(); if (identifier.getDatabaseName().isPresent()) { nameParts.add(identifier.getDatabaseName().get()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java index 5d1d4d7cbbe4c2..1f926be0426677 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java @@ -17,7 +17,7 @@ package org.apache.doris.nereids.analyzer; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.util.Utils; @@ -32,7 +32,7 @@ public class UnboundSlot extends Slot { private final List nameParts; public UnboundSlot(List nameParts) { - super(NodeType.UNBOUND_SLOT); + super(OperatorType.UNBOUND_SLOT); this.nameParts = nameParts; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java index 11b8a54d3d0104..aeae8a243817c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java @@ -17,7 +17,7 @@ package org.apache.doris.nereids.analyzer; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import org.apache.doris.nereids.trees.expressions.LeafExpression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.util.Utils; @@ -33,7 +33,7 @@ public class UnboundStar extends NamedExpression implements LeafExpression { private final List target; public UnboundStar(List target) { - super(NodeType.UNBOUND_STAR); + super(OperatorType.UNBOUND_STAR); this.target = target; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/AbstractOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/AbstractOperator.java index 6363fbcae440b4..0588c028edaeac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/AbstractOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/AbstractOperator.java @@ -26,21 +26,21 @@ * Abstract class for all concrete operator. */ public abstract class AbstractOperator implements Operator { - protected final OperatorType type; + protected final PlanType type; protected final long limited; - public AbstractOperator(OperatorType type) { + public AbstractOperator(PlanType type) { this.type = Objects.requireNonNull(type, "type can not be null"); this.limited = -1; } - public AbstractOperator(OperatorType type, long limited) { + public AbstractOperator(PlanType type, long limited) { this.type = type; this.limited = limited; } @Override - public OperatorType getType() { + public PlanType getType() { return type; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/Operator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/Operator.java index b391ff24a5b480..79188561653165 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/Operator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/Operator.java @@ -26,7 +26,7 @@ * interface for all concrete operator. */ public interface Operator { - OperatorType getType(); + PlanType getType(); > NODE_TYPE toTreeNode(GroupExpression groupExpression); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/OperatorType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/PlanType.java similarity index 98% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/operators/OperatorType.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/operators/PlanType.java index f2ddc067167c1d..8aa34b5e887494 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/OperatorType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/PlanType.java @@ -27,7 +27,7 @@ * 4. MULTI_FIXED: the leaf node of pattern tree, which can be matched by multiple operators, * but these operators cannot be used in rules */ -public enum OperatorType { +public enum PlanType { // logical plan LOGICAL_UNBOUND_RELATION, LOGICAL_BOUND_RELATION, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalBinaryOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalBinaryOperator.java index 228ee0835883bd..5bb0bb9f6a8967 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalBinaryOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalBinaryOperator.java @@ -19,7 +19,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.operators.AbstractOperator; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.operators.plans.BinaryPlanOperator; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Slot; @@ -35,7 +35,7 @@ public abstract class LogicalBinaryOperator extends AbstractOperator implements LogicalOperator, BinaryPlanOperator { - public LogicalBinaryOperator(OperatorType type) { + public LogicalBinaryOperator(PlanType type) { super(type); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalFilter.java index c7d579898cb62e..37f1d63ba21f8d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalFilter.java @@ -17,7 +17,7 @@ package org.apache.doris.nereids.operators.plans.logical; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; @@ -33,7 +33,7 @@ public class LogicalFilter extends LogicalUnaryOperator { private final Expression predicates; public LogicalFilter(Expression predicates) { - super(OperatorType.LOGICAL_FILTER); + super(PlanType.LOGICAL_FILTER); this.predicates = Objects.requireNonNull(predicates, "predicates can not be null"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalJoin.java index b1672627bdd756..373fbd40bf3867 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalJoin.java @@ -17,7 +17,7 @@ package org.apache.doris.nereids.operators.plans.logical; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.operators.plans.JoinType; import org.apache.doris.nereids.rules.exploration.JoinReorderContext; import org.apache.doris.nereids.trees.expressions.Expression; @@ -57,7 +57,7 @@ public LogicalJoin(JoinType joinType) { * @param onClause on clause for join node */ public LogicalJoin(JoinType joinType, Optional onClause) { - super(OperatorType.LOGICAL_JOIN); + super(PlanType.LOGICAL_JOIN); this.joinType = Objects.requireNonNull(joinType, "joinType can not be null"); this.onClause = Objects.requireNonNull(onClause, "onClause can not be null"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalLeafOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalLeafOperator.java index 44c72f06e6f60c..135e1d78950b33 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalLeafOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalLeafOperator.java @@ -19,7 +19,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.operators.AbstractOperator; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.operators.plans.LeafPlanOperator; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; @@ -33,7 +33,7 @@ public abstract class LogicalLeafOperator extends AbstractOperator implements LogicalOperator, LeafPlanOperator { - public LogicalLeafOperator(OperatorType type) { + public LogicalLeafOperator(PlanType type) { super(type); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalProject.java index 555454244373ff..c583042fcbae08 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalProject.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.operators.plans.logical; import org.apache.doris.nereids.exceptions.UnboundException; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; @@ -42,7 +42,7 @@ public class LogicalProject extends LogicalUnaryOperator { * @param projects project list */ public LogicalProject(List projects) { - super(OperatorType.LOGICAL_PROJECT); + super(PlanType.LOGICAL_PROJECT); this.projects = Objects.requireNonNull(projects, "projects can not be null"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalRelation.java index 6f01817b82b83b..9083ee155448e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalRelation.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.operators.plans.logical; import org.apache.doris.catalog.Table; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; @@ -43,7 +43,7 @@ public class LogicalRelation extends LogicalLeafOperator { * @param qualifier qualified relation name */ public LogicalRelation(Table table, List qualifier) { - super(OperatorType.LOGICAL_BOUND_RELATION); + super(PlanType.LOGICAL_BOUND_RELATION); this.table = Objects.requireNonNull(table, "table can not be null"); this.qualifier = Objects.requireNonNull(qualifier, "qualifier can not be null"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalUnaryOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalUnaryOperator.java index b04f2ae8f3af91..e8607d634a2175 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalUnaryOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/logical/LogicalUnaryOperator.java @@ -19,7 +19,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.operators.AbstractOperator; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.operators.plans.UnaryPlanOperator; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Slot; @@ -35,7 +35,7 @@ public abstract class LogicalUnaryOperator extends AbstractOperator implements LogicalOperator, UnaryPlanOperator { - public LogicalUnaryOperator(OperatorType type) { + public LogicalUnaryOperator(PlanType type) { super(type); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java index 324043591711d9..7d6136e2c74ae2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalAggregation.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.operators.plans.physical; import org.apache.doris.nereids.PlanOperatorVisitor; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.operators.plans.AggPhase; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; @@ -51,7 +51,7 @@ public class PhysicalAggregation extends PhysicalUnaryOperator { */ public PhysicalAggregation(List groupByExprList, List aggExprList, List partitionExprList, AggPhase aggPhase, boolean usingStream) { - super(OperatorType.PHYSICAL_AGGREGATION); + super(PlanType.PHYSICAL_AGGREGATION); this.groupByExprList = groupByExprList; this.aggExprList = aggExprList; this.partitionExprList = partitionExprList; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalBinaryOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalBinaryOperator.java index 813cb3d4453374..8c2522c9bc3084 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalBinaryOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalBinaryOperator.java @@ -19,7 +19,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.operators.AbstractOperator; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.operators.plans.BinaryPlanOperator; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Slot; @@ -35,7 +35,7 @@ public abstract class PhysicalBinaryOperator extends AbstractOperator implements PhysicalOperator, BinaryPlanOperator { - public PhysicalBinaryOperator(OperatorType type) { + public PhysicalBinaryOperator(PlanType type) { super(type); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalBroadcastHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalBroadcastHashJoin.java index 3ccbf70657bc6b..a4026b5208c611 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalBroadcastHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalBroadcastHashJoin.java @@ -17,10 +17,9 @@ package org.apache.doris.nereids.operators.plans.physical; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.operators.plans.JoinType; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.plans.Plan; import java.util.Objects; import java.util.Optional; @@ -49,7 +48,7 @@ public PhysicalBroadcastHashJoin(JoinType joinType) { * @param onClause on clause expression */ public PhysicalBroadcastHashJoin(JoinType joinType, Optional onClause) { - super(OperatorType.PHYSICAL_BROADCAST_HASH_JOIN); + super(PlanType.PHYSICAL_BROADCAST_HASH_JOIN); this.joinType = Objects.requireNonNull(joinType, "joinType can not be null"); this.onClause = Objects.requireNonNull(onClause, "onClause can not be null"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalFilter.java index aa54b67caaa02d..8ecc6397251a60 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalFilter.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.operators.plans.physical; import org.apache.doris.nereids.PlanOperatorVisitor; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.PhysicalUnaryPlan; @@ -33,7 +33,7 @@ public class PhysicalFilter extends PhysicalUnaryOperator { private final Expression predicates; public PhysicalFilter(Expression predicates) { - super(OperatorType.PHYSICAL_FILTER); + super(PlanType.PHYSICAL_FILTER); this.predicates = Objects.requireNonNull(predicates, "predicates can not be null"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java index 94eb6f91476d63..2fc2fc64a1b6af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalHashJoin.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.operators.plans.physical; import org.apache.doris.nereids.PlanOperatorVisitor; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.operators.plans.JoinType; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; @@ -40,7 +40,7 @@ public class PhysicalHashJoin extends PhysicalBinaryOperator { * @param predicate join condition. */ public PhysicalHashJoin(JoinType joinType, Expression predicate) { - super(OperatorType.PHYSICAL_HASH_JOIN); + super(PlanType.PHYSICAL_HASH_JOIN); this.joinType = joinType; this.predicate = predicate; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalLeafOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalLeafOperator.java index 6cf64ede7cb6ab..2095c91c8f65e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalLeafOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalLeafOperator.java @@ -19,7 +19,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.operators.AbstractOperator; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.operators.plans.LeafPlanOperator; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Slot; @@ -34,7 +34,7 @@ public abstract class PhysicalLeafOperator extends AbstractOperator implements PhysicalOperator, LeafPlanOperator { - public PhysicalLeafOperator(OperatorType type) { + public PhysicalLeafOperator(PlanType type) { super(type); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalOlapScan.java index b79e3531069b89..3374234093230a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalOlapScan.java @@ -20,7 +20,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.nereids.PlanOperatorVisitor; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.PhysicalLeafPlan; @@ -46,7 +46,7 @@ public class PhysicalOlapScan extends PhysicalScan { * @param qualifier table's name */ public PhysicalOlapScan(OlapTable olapTable, List qualifier) { - super(OperatorType.PHYSICAL_OLAP_SCAN, qualifier); + super(PlanType.PHYSICAL_OLAP_SCAN, qualifier); this.olapTable = olapTable; this.selectedIndexId = olapTable.getBaseIndexId(); this.selectedTabletId = Lists.newArrayList(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalProject.java index 5de5fc8c019d65..e86aed0c76f58e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalProject.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.operators.plans.physical; import org.apache.doris.nereids.PlanOperatorVisitor; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.PhysicalUnaryPlan; @@ -36,7 +36,7 @@ public class PhysicalProject extends PhysicalUnaryOperator { private final List projects; public PhysicalProject(List projects) { - super(OperatorType.PHYSICAL_PROJECT); + super(PlanType.PHYSICAL_PROJECT); this.projects = Objects.requireNonNull(projects, "projects can not be null"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalScan.java index 63ffef9b0f3687..d2bbe66edf412a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalScan.java @@ -17,7 +17,7 @@ package org.apache.doris.nereids.operators.plans.physical; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import java.util.List; import java.util.Objects; @@ -36,7 +36,7 @@ public abstract class PhysicalScan extends PhysicalLeafOperator { * @param type node type * @param qualifier table's name */ - public PhysicalScan(OperatorType type, List qualifier) { + public PhysicalScan(PlanType type, List qualifier) { super(type); this.qualifier = Objects.requireNonNull(qualifier, "qualifier can not be null"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java index dc1ab43b70c62a..0812b4ade847d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalSort.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.operators.plans.physical; import org.apache.doris.nereids.PlanOperatorVisitor; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.PhysicalUnaryPlan; @@ -42,7 +42,7 @@ public class PhysicalSort extends PhysicalUnaryOperator { * Constructor of PhysicalHashJoinNode. */ public PhysicalSort(int offset, int limit, List orderList, boolean useTopN) { - super(OperatorType.PHYSICAL_SORT); + super(PlanType.PHYSICAL_SORT); this.offset = offset; this.limit = limit; this.orderList = orderList; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalUnaryOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalUnaryOperator.java index daecac819171af..3d475f221d5e12 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalUnaryOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalUnaryOperator.java @@ -19,7 +19,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.operators.AbstractOperator; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.operators.plans.UnaryPlanOperator; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Slot; @@ -35,7 +35,7 @@ public abstract class PhysicalUnaryOperator extends AbstractOperator implements PhysicalOperator, UnaryPlanOperator { - public PhysicalUnaryOperator(OperatorType type) { + public PhysicalUnaryOperator(PlanType type) { super(type); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java index 4771300ad942bb..6c1a50cd232c0f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java @@ -19,7 +19,7 @@ import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.trees.TreeNode; import com.google.common.collect.Lists; @@ -89,8 +89,8 @@ public NODE_TYPE next() { throw new NoSuchElementException(); } - if (OperatorType.FIXED == pattern.getOperatorType() - || OperatorType.MULTI_FIXED == pattern.getOperatorType()) { + if (PlanType.FIXED == pattern.getOperatorType() + || PlanType.MULTI_FIXED == pattern.getOperatorType()) { iteratorIndex = iterator.size(); return iterator.get(0).next(); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Pattern.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Pattern.java index 781e10aa979d9a..ddc91db8a355cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Pattern.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Pattern.java @@ -19,9 +19,9 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.operators.Operator; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.trees.AbstractTreeNode; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import org.apache.doris.nereids.trees.TreeNode; import com.google.common.collect.ImmutableList; @@ -36,13 +36,13 @@ */ public class Pattern> extends AbstractTreeNode> { - public static final Pattern ANY = new Pattern(OperatorType.ANY); - public static final Pattern MULTI = new Pattern(OperatorType.MULTI); - public static final Pattern FIXED = new Pattern(OperatorType.FIXED); - public static final Pattern MULTI_FIXED = new Pattern(OperatorType.MULTI_FIXED); + public static final Pattern ANY = new Pattern(PlanType.ANY); + public static final Pattern MULTI = new Pattern(PlanType.MULTI); + public static final Pattern FIXED = new Pattern(PlanType.FIXED); + public static final Pattern MULTI_FIXED = new Pattern(PlanType.MULTI_FIXED); protected final List> predicates; - protected final OperatorType operatorType; + protected final PlanType operatorType; /** * Constructor for Pattern. @@ -50,8 +50,8 @@ public class Pattern> predicates, Pattern... children) { - super(NodeType.PATTERN, children); + public Pattern(PlanType operatorType, List> predicates, Pattern... children) { + super(OperatorType.PATTERN, children); this.operatorType = operatorType; this.predicates = ImmutableList.copyOf(predicates); } @@ -74,20 +74,20 @@ public Pattern(OperatorType operatorType, List> predicates, Patt * * @return node type in pattern */ - public OperatorType getOperatorType() { + public PlanType getOperatorType() { return operatorType; } public boolean isFixed() { - return operatorType == OperatorType.FIXED; + return operatorType == PlanType.FIXED; } public boolean isAny() { - return operatorType == OperatorType.ANY; + return operatorType == PlanType.ANY; } public boolean isMulti() { - return operatorType == OperatorType.MULTI; + return operatorType == PlanType.MULTI; } /** @@ -100,8 +100,8 @@ public boolean matchOperator(Operator operator) { if (operator == null) { return false; } - if (operatorType == OperatorType.MULTI || operatorType == OperatorType.ANY - || operatorType == OperatorType.MULTI_FIXED || operatorType == OperatorType.FIXED) { + if (operatorType == PlanType.MULTI || operatorType == PlanType.ANY + || operatorType == PlanType.MULTI_FIXED || operatorType == PlanType.FIXED) { return true; } return getOperatorType().equals(operator.getType()); @@ -122,7 +122,7 @@ public boolean matchRoot(TYPE root) { return false; } - if (operatorType == OperatorType.MULTI || operatorType == OperatorType.ANY) { + if (operatorType == PlanType.MULTI || operatorType == PlanType.ANY) { return true; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/TypePattern.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/TypePattern.java index bbb93e65c9ca31..ec77b6ac43e0d1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/TypePattern.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/TypePattern.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.pattern; import org.apache.doris.nereids.operators.Operator; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.trees.TreeNode; import java.util.List; @@ -31,12 +31,12 @@ public class TypePattern type; public TypePattern(Class clazz, Pattern... children) { - super(OperatorType.NORMAL_PATTERN, children); + super(PlanType.NORMAL_PATTERN, children); this.type = Objects.requireNonNull(clazz, "class can not be null"); } public TypePattern(Class clazz, List> predicates, Pattern... children) { - super(OperatorType.NORMAL_PATTERN, predicates, children); + super(PlanType.NORMAL_PATTERN, predicates, children); this.type = Objects.requireNonNull(clazz, "class can not be null"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/NormalizeExpressionRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/NormalizeExpressionRule.java index 8da9c1e5f96e5a..f0b9e7a2b62cc3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/NormalizeExpressionRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/NormalizeExpressionRule.java @@ -20,7 +20,7 @@ import org.apache.doris.nereids.rules.expression.rewrite.AbstractExpressionRewriteRule; import org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewriteContext; import org.apache.doris.nereids.rules.expression.rewrite.RewriteHelper; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; @@ -42,7 +42,7 @@ public class NormalizeExpressionRule extends AbstractExpressionRewriteRule { public Expression visitComparisonPredicate(ComparisonPredicate expr, ExpressionRewriteContext context) { if (RewriteHelper.isConstant(expr.left()) && !RewriteHelper.isConstant(expr.right())) { - NodeType exprType = expr.getType(); + OperatorType exprType = expr.getType(); switch (exprType) { case EQUAL_TO: return new EqualTo(expr.right(), expr.left()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/SimplifyNotExprRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/SimplifyNotExprRule.java index f8025e8f39002e..6a55308c7428a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/SimplifyNotExprRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/SimplifyNotExprRule.java @@ -19,7 +19,7 @@ import org.apache.doris.nereids.rules.expression.rewrite.AbstractExpressionRewriteRule; import org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewriteContext; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.GreaterThan; @@ -54,7 +54,7 @@ public Expression visitNot(Not expr, ExpressionRewriteContext context) { ComparisonPredicate cp = (ComparisonPredicate) expr.child(); Expression left = rewrite(cp.left(), context); Expression right = rewrite(cp.right(), context); - NodeType type = cp.getType(); + OperatorType type = cp.getType(); switch (type) { case GREATER_THAN: return new LessThanEqual(left, right); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java index 16cbaf9f440e5b..2bd375d1cb6c00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java @@ -35,14 +35,14 @@ public abstract class AbstractTreeNode> implements TreeNode { - protected final NodeType type; + protected final OperatorType type; protected final List children; // TODO: Maybe we should use a GroupPlan to avoid TreeNode hold the GroupExpression. // https://github.com/apache/incubator-doris/pull/9807#discussion_r884829067 protected final Optional groupExpression; - public AbstractTreeNode(NodeType type, NODE_TYPE... children) { + public AbstractTreeNode(OperatorType type, NODE_TYPE... children) { this(type, null, children); } @@ -53,7 +53,7 @@ public AbstractTreeNode(NodeType type, NODE_TYPE... children) { * @param groupExpression group expression related to the operator of this node * @param children children of this node */ - public AbstractTreeNode(NodeType type, GroupExpression groupExpression, NODE_TYPE... children) { + public AbstractTreeNode(OperatorType type, GroupExpression groupExpression, NODE_TYPE... children) { this.type = type; this.children = ImmutableList.copyOf(children); this.groupExpression = Optional.ofNullable(groupExpression); @@ -80,7 +80,7 @@ public List children() { } @Override - public NodeType getType() { + public OperatorType getType() { return type; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/NodeType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/OperatorType.java similarity index 97% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/trees/NodeType.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/trees/OperatorType.java index 1f22f3020f1fae..a5b376baaed08f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/NodeType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/OperatorType.java @@ -20,7 +20,7 @@ /** * Types for all TreeNode in Nereids, include Plan and Expression. */ -public enum NodeType { +public enum OperatorType { // plan LOGICAL, PHYSICAL, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java index 12a8597782367b..df2865bbb623d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java @@ -36,7 +36,7 @@ public interface TreeNode> { // cache GroupExpression for fast exit from Memo.copyIn. Optional getGroupExpression(); - NodeType getType(); + OperatorType getType(); List children(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java index 425c3b0bc9a1a1..45ddf03ee5c075 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.trees.expressions; import org.apache.doris.nereids.exceptions.UnboundException; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import com.google.common.collect.Lists; @@ -41,7 +41,7 @@ public class Alias extends NamedExpression * @param name alias name */ public Alias(CHILD_TYPE child, String name) { - super(NodeType.ALIAS, child); + super(OperatorType.ALIAS, child); exprId = NamedExpressionUtil.newExprId(); this.name = name; qualifier = Lists.newArrayList(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ComparisonPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ComparisonPredicate.java index 6f5ae18fe2941d..807bb80d7aa98e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ComparisonPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ComparisonPredicate.java @@ -19,7 +19,7 @@ import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.rules.expression.rewrite.ExpressionVisitor; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.DataType; @@ -39,7 +39,7 @@ public class ComparisonPredicate { public EqualTo(LEFT_CHILD_TYPE left, RIGHT_CHILD_TYPE right) { - super(NodeType.EQUAL_TO, left, right); + super(OperatorType.EQUAL_TO, left, right); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExprId.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExprId.java index 9be4db50b9d2ce..f14843965c31f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExprId.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExprId.java @@ -17,13 +17,15 @@ package org.apache.doris.nereids.trees.expressions; +import org.apache.doris.common.Id; + import java.util.Objects; import java.util.UUID; /** * UUID for Expression in Nereids. */ -public class ExprId { +public class ExprId extends Id { private final long id; private final UUID jvmId; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java index e6832ba0a7b35d..1f65363f9a4bcb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java @@ -20,9 +20,8 @@ import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.rules.expression.rewrite.ExpressionVisitor; import org.apache.doris.nereids.trees.AbstractTreeNode; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import org.apache.doris.nereids.types.DataType; -import org.apache.doris.statistics.ExprStats; import java.util.List; @@ -31,7 +30,7 @@ */ public abstract class Expression extends AbstractTreeNode { - public Expression(NodeType type, Expression... children) { + public Expression(OperatorType type, Expression... children) { super(type, children); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java index fd172cd42908ae..36ca3f0e2369c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/FunctionCallExpression.java @@ -19,7 +19,7 @@ import org.apache.doris.analysis.FunctionName; import org.apache.doris.catalog.Function; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import java.util.Arrays; import java.util.List; @@ -41,7 +41,7 @@ public class FunctionCallExpression extends Expression { */ public FunctionCallExpression(FunctionName functionName, Function fn, Expression... children) { - super(NodeType.EXPRESSION, children); + super(OperatorType.EXPRESSION, children); this.functionName = functionName; this.params = Arrays.stream(children).collect(Collectors.toList()); this.fn = fn; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThan.java index 7e0e291366c326..b76a19bcbd41fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThan.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.trees.expressions; import org.apache.doris.nereids.exceptions.UnboundException; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import com.google.common.base.Preconditions; @@ -36,7 +36,7 @@ public class GreaterThan extends Expression implements UnaryExpression { public Not(CHILD_TYPE child) { - super(NodeType.NOT, child); + super(OperatorType.NOT, child); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java index 61488318b9f607..0e363120842f0e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.trees.expressions; import org.apache.doris.nereids.exceptions.UnboundException; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; /** * Null safe equal expression: a <=> b. @@ -33,7 +33,7 @@ public class NullSafeEqual qualifier) { - super(NodeType.SLOT_REFERENCE); + super(OperatorType.SLOT_REFERENCE); this.exprId = exprId; this.name = name; this.dataType = dataType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java index a440ddb1f92865..aa8d532c634851 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java @@ -20,10 +20,9 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.operators.plans.PlanOperator; import org.apache.doris.nereids.trees.AbstractTreeNode; -import org.apache.doris.nereids.trees.NodeType; -import org.apache.doris.planner.PlanNode; +import org.apache.doris.nereids.trees.OperatorType; import org.apache.doris.statistics.ExprStats; -import org.apache.doris.statistics.PlanStats; +import org.apache.doris.statistics.NodeType; import org.apache.doris.statistics.StatsDeriveResult; import org.apache.commons.lang3.StringUtils; @@ -43,12 +42,12 @@ public abstract class AbstractPlan protected StatsDeriveResult statsDeriveResult; protected long limit; - public AbstractPlan(NodeType type, OP_TYPE operator, Plan... children) { + public AbstractPlan(OperatorType type, OP_TYPE operator, Plan... children) { super(type, children); this.operator = Objects.requireNonNull(operator, "operator can not be null"); } - public AbstractPlan(NodeType type, OP_TYPE operator, GroupExpression groupExpression, Plan... children) { + public AbstractPlan(OperatorType type, OP_TYPE operator, GroupExpression groupExpression, Plan... children) { super(type, groupExpression, children); this.operator = Objects.requireNonNull(operator, "operator can not be null"); } @@ -105,7 +104,7 @@ public StatsDeriveResult getStatsDeriveResult() { } @Override - public PlanNode.NodeType getNodeType() { + public NodeType getNodeType() { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlaceHolderPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlaceHolderPlan.java index 8a18610b0c9d2b..51f5aeb9bf3369 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlaceHolderPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlaceHolderPlan.java @@ -20,10 +20,10 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.operators.plans.LeafPlanOperator; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.planner.PlanNode; import org.apache.doris.statistics.ExprStats; +import org.apache.doris.statistics.NodeType; import org.apache.doris.statistics.PlanStats; import org.apache.doris.statistics.StatsDeriveResult; @@ -43,8 +43,8 @@ public Optional getGroupExpression() { } @Override - public NodeType getType() { - return NodeType.FIXED; + public OperatorType getType() { + return OperatorType.FIXED; } @Override @@ -93,7 +93,7 @@ public StatsDeriveResult getStatsDeriveResult() { } @Override - public PlanNode.NodeType getNodeType() { + public NodeType getNodeType() { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java index 780dcbb59c5025..042d233a48664e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java @@ -20,7 +20,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.operators.plans.logical.LogicalOperator; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.Plan; @@ -36,12 +36,12 @@ public abstract class AbstractLogicalPlan protected final LogicalProperties logicalProperties; - public AbstractLogicalPlan(NodeType type, OP_TYPE operator, Plan... children) { + public AbstractLogicalPlan(OperatorType type, OP_TYPE operator, Plan... children) { super(type, operator, children); this.logicalProperties = new LogicalProperties(Collections.emptyList()); } - public AbstractLogicalPlan(NodeType type, OP_TYPE operator, + public AbstractLogicalPlan(OperatorType type, OP_TYPE operator, GroupExpression groupExpression, LogicalProperties logicalProperties, Plan... children) { super(type, operator, groupExpression, children); this.logicalProperties = logicalProperties; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalBinaryPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalBinaryPlan.java index 8d860b9dd7e7d5..caf40be2904c24 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalBinaryPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalBinaryPlan.java @@ -20,7 +20,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.operators.plans.logical.LogicalBinaryOperator; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import org.apache.doris.nereids.trees.plans.BinaryPlan; import org.apache.doris.nereids.trees.plans.Plan; @@ -39,12 +39,12 @@ public class LogicalBinaryPlan< implements BinaryPlan { public LogicalBinaryPlan(OP_TYPE operator, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { - super(NodeType.LOGICAL, operator, leftChild, rightChild); + super(OperatorType.LOGICAL, operator, leftChild, rightChild); } public LogicalBinaryPlan(OP_TYPE operator, GroupExpression groupExpression, LogicalProperties logicalProperties, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { - super(NodeType.LOGICAL, operator, groupExpression, logicalProperties, leftChild, rightChild); + super(OperatorType.LOGICAL, operator, groupExpression, logicalProperties, leftChild, rightChild); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLeafPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLeafPlan.java index 012424125e3ba3..4ddcbc3be5ead9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLeafPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLeafPlan.java @@ -20,7 +20,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.operators.plans.logical.LogicalLeafOperator; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import org.apache.doris.nereids.trees.plans.LeafPlan; import org.apache.doris.nereids.trees.plans.Plan; @@ -36,11 +36,11 @@ public class LogicalLeafPlan implements LeafPlan { public LogicalLeafPlan(OP_TYPE operator) { - super(NodeType.LOGICAL, operator); + super(OperatorType.LOGICAL, operator); } public LogicalLeafPlan(OP_TYPE operator, GroupExpression groupExpression, LogicalProperties logicalProperties) { - super(NodeType.LOGICAL, operator, groupExpression, logicalProperties); + super(OperatorType.LOGICAL, operator, groupExpression, logicalProperties); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnaryPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnaryPlan.java index b44db32de20007..8222b428543890 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnaryPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnaryPlan.java @@ -20,7 +20,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.operators.plans.logical.LogicalUnaryOperator; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.UnaryPlan; @@ -36,12 +36,12 @@ public class LogicalUnaryPlan { public LogicalUnaryPlan(OP_TYPE operator, CHILD_TYPE child) { - super(NodeType.LOGICAL, operator, child); + super(OperatorType.LOGICAL, operator, child); } public LogicalUnaryPlan(OP_TYPE operator, GroupExpression groupExpression, LogicalProperties logicalProperties, CHILD_TYPE child) { - super(NodeType.LOGICAL, operator, groupExpression, logicalProperties, child); + super(OperatorType.LOGICAL, operator, groupExpression, logicalProperties, child); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java index 461c25e3a3eec9..d0675e5b12d448 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java @@ -21,16 +21,11 @@ import org.apache.doris.nereids.operators.plans.physical.PhysicalOperator; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.planner.PlanNode; -import org.apache.doris.statistics.ExprStats; -import org.apache.doris.statistics.PlanStats; -import org.apache.doris.statistics.StatsDeriveResult; -import java.util.Collections; import java.util.List; import java.util.Objects; @@ -47,7 +42,7 @@ public abstract class AbstractPhysicalPlan /** * create physical plan by op, logicalProperties and children. */ - public AbstractPhysicalPlan(NodeType type, OP_TYPE operator, + public AbstractPhysicalPlan(OperatorType type, OP_TYPE operator, LogicalProperties logicalProperties, Plan... children) { super(type, operator, children); this.logicalProperties = Objects.requireNonNull(logicalProperties, "logicalProperties can not be null"); @@ -64,7 +59,7 @@ public AbstractPhysicalPlan(NodeType type, OP_TYPE operator, * @param logicalProperties logical properties of this plan * @param children children of this plan */ - public AbstractPhysicalPlan(NodeType type, OP_TYPE operator, GroupExpression groupExpression, + public AbstractPhysicalPlan(OperatorType type, OP_TYPE operator, GroupExpression groupExpression, LogicalProperties logicalProperties, Plan... children) { super(type, operator, groupExpression, children); this.logicalProperties = Objects.requireNonNull(logicalProperties, "logicalProperties can not be null"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalBinaryPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalBinaryPlan.java index 9b95552216d78f..72911722d601db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalBinaryPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalBinaryPlan.java @@ -20,7 +20,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.operators.plans.physical.PhysicalBinaryOperator; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import org.apache.doris.nereids.trees.plans.BinaryPlan; import org.apache.doris.nereids.trees.plans.Plan; @@ -40,12 +40,12 @@ public class PhysicalBinaryPlan< public PhysicalBinaryPlan(OP_TYPE operator, LogicalProperties logicalProperties, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { - super(NodeType.PHYSICAL, operator, logicalProperties, leftChild, rightChild); + super(OperatorType.PHYSICAL, operator, logicalProperties, leftChild, rightChild); } public PhysicalBinaryPlan(OP_TYPE operator, GroupExpression groupExpression, LogicalProperties logicalProperties, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { - super(NodeType.PHYSICAL, operator, groupExpression, logicalProperties, leftChild, rightChild); + super(OperatorType.PHYSICAL, operator, groupExpression, logicalProperties, leftChild, rightChild); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLeafPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLeafPlan.java index 0408641b797557..4796be7eee69ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLeafPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLeafPlan.java @@ -20,7 +20,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.operators.plans.physical.PhysicalLeafOperator; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import org.apache.doris.nereids.trees.plans.LeafPlan; import org.apache.doris.nereids.trees.plans.Plan; @@ -36,11 +36,11 @@ public class PhysicalLeafPlan implements LeafPlan { public PhysicalLeafPlan(OP_TYPE operator, LogicalProperties logicalProperties) { - super(NodeType.PHYSICAL, operator, logicalProperties); + super(OperatorType.PHYSICAL, operator, logicalProperties); } public PhysicalLeafPlan(OP_TYPE operator, GroupExpression groupExpression, LogicalProperties logicalProperties) { - super(NodeType.PHYSICAL, operator, groupExpression, logicalProperties); + super(OperatorType.PHYSICAL, operator, groupExpression, logicalProperties); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnaryPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnaryPlan.java index caadb0c177a64d..6772f9d044f44c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnaryPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnaryPlan.java @@ -20,7 +20,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.operators.plans.physical.PhysicalUnaryOperator; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.NodeType; +import org.apache.doris.nereids.trees.OperatorType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.UnaryPlan; @@ -36,12 +36,12 @@ public class PhysicalUnaryPlan { public PhysicalUnaryPlan(OP_TYPE operator, LogicalProperties logicalProperties, CHILD_TYPE child) { - super(NodeType.PHYSICAL, operator, logicalProperties, child); + super(OperatorType.PHYSICAL, operator, logicalProperties, child); } public PhysicalUnaryPlan(OP_TYPE operator, GroupExpression groupExpression, LogicalProperties logicalProperties, CHILD_TYPE child) { - super(NodeType.PHYSICAL, operator, groupExpression, logicalProperties, child); + super(OperatorType.PHYSICAL, operator, groupExpression, logicalProperties, child); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java index ac7c7fee96499e..bb2140181e7ab7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java @@ -28,6 +28,7 @@ import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.UserException; import org.apache.doris.common.util.VectorizedUtil; +import org.apache.doris.statistics.NodeType; import org.apache.doris.statistics.StatsRecursiveDerive; import org.apache.doris.thrift.TAggregationNode; import org.apache.doris.thrift.TExplainLevel; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java index a439ed3f393e70..2f181b97a2776a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java @@ -27,6 +27,7 @@ import org.apache.doris.analysis.OrderByElement; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.common.UserException; +import org.apache.doris.statistics.NodeType; import org.apache.doris.statistics.StatsRecursiveDerive; import org.apache.doris.thrift.TAnalyticNode; import org.apache.doris.thrift.TExplainLevel; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java index 6b8ff8f8ef9907..849512f0e94182 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.AssertNumRowsElement; import org.apache.doris.common.UserException; +import org.apache.doris.statistics.NodeType; import org.apache.doris.statistics.StatsRecursiveDerive; import org.apache.doris.thrift.TAssertNumRowsNode; import org.apache.doris.thrift.TExplainLevel; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java index 07ec26cae3d87e..c9f5d34cdca4e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java @@ -45,6 +45,7 @@ import org.apache.doris.mysql.privilege.UserProperty; import org.apache.doris.qe.ConnectContext; import org.apache.doris.resource.Tag; +import org.apache.doris.statistics.NodeType; import org.apache.doris.system.Backend; import org.apache.doris.system.BeSelectionPolicy; import org.apache.doris.task.LoadTaskInfo; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/CrossJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/CrossJoinNode.java index 81accd9810835a..7c3d366f8acc43 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/CrossJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/CrossJoinNode.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.TableRef; import org.apache.doris.common.UserException; +import org.apache.doris.statistics.NodeType; import org.apache.doris.statistics.StatsRecursiveDerive; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TPlanNode; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/EmptySetNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/EmptySetNode.java index 07856fed1babe2..4d0896689be88f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/EmptySetNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/EmptySetNode.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.TupleId; import org.apache.doris.common.UserException; +import org.apache.doris.statistics.NodeType; import org.apache.doris.statistics.StatsRecursiveDerive; import org.apache.doris.thrift.TPlanNode; import org.apache.doris.thrift.TPlanNodeType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java index cad5eeefd05da7..06d88d8d2449ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java @@ -30,6 +30,7 @@ import org.apache.doris.external.elasticsearch.EsShardPartitions; import org.apache.doris.external.elasticsearch.EsShardRouting; import org.apache.doris.external.elasticsearch.EsTablePartitions; +import org.apache.doris.statistics.NodeType; import org.apache.doris.system.Backend; import org.apache.doris.thrift.TEsScanNode; import org.apache.doris.thrift.TEsScanRange; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ExceptNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ExceptNode.java index 84911cad5bd404..8fa0cf36c79f8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ExceptNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ExceptNode.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.TupleId; +import org.apache.doris.statistics.NodeType; import org.apache.doris.thrift.TPlanNode; import org.apache.doris.thrift.TPlanNodeType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java index 0ce4dd5ba83a76..89e4b4232407a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java @@ -26,6 +26,7 @@ import org.apache.doris.analysis.TupleId; import org.apache.doris.common.UserException; import org.apache.doris.common.util.VectorizedUtil; +import org.apache.doris.statistics.NodeType; import org.apache.doris.statistics.StatsRecursiveDerive; import org.apache.doris.thrift.TExchangeNode; import org.apache.doris.thrift.TPlanNode; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java index 0bc86e724a37bf..95a24c45698000 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java @@ -38,6 +38,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.util.VectorizedUtil; +import org.apache.doris.statistics.NodeType; import org.apache.doris.statistics.StatsRecursiveDerive; import org.apache.doris.thrift.TEqJoinCondition; import org.apache.doris.thrift.TExplainLevel; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HiveScanNode.java index 4fda0515e7bef1..926356397bf407 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HiveScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HiveScanNode.java @@ -28,6 +28,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; import org.apache.doris.load.BrokerFileGroup; +import org.apache.doris.statistics.NodeType; import org.apache.doris.thrift.TBrokerFileStatus; import org.apache.doris.thrift.TExplainLevel; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergScanNode.java index 3439631f0d6e94..f5732a42e3e55f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergScanNode.java @@ -26,6 +26,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.external.iceberg.util.IcebergUtils; import org.apache.doris.load.BrokerFileGroup; +import org.apache.doris.statistics.NodeType; import org.apache.doris.thrift.TBrokerFileStatus; import org.apache.doris.thrift.TExplainLevel; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/LoadScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/LoadScanNode.java index 980876c6309ecd..17d6562e93cdec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/LoadScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/LoadScanNode.java @@ -38,6 +38,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.rewrite.ExprRewriter; +import org.apache.doris.statistics.NodeType; import org.apache.doris.thrift.TBrokerScanNode; import org.apache.doris.thrift.TBrokerScanRangeParams; import org.apache.doris.thrift.TPlanNode; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/MysqlScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/MysqlScanNode.java index 82ea85da8e2b99..2f441e93523dd3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/MysqlScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MysqlScanNode.java @@ -26,6 +26,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.MysqlTable; import org.apache.doris.common.UserException; +import org.apache.doris.statistics.NodeType; import org.apache.doris.statistics.StatsRecursiveDerive; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TMySQLScanNode; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java index cacbc48ad5f2f7..668b3727d61380 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java @@ -27,6 +27,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.OdbcTable; import org.apache.doris.common.UserException; +import org.apache.doris.statistics.NodeType; import org.apache.doris.statistics.StatsRecursiveDerive; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TOdbcScanNode; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index e2e6ec0d418a15..f88a52ce8fdff7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -53,6 +53,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.resource.Tag; +import org.apache.doris.statistics.NodeType; import org.apache.doris.statistics.StatsRecursiveDerive; import org.apache.doris.system.Backend; import org.apache.doris.thrift.TExplainLevel; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java index 5ec1b6dd3bac20..c9f2363e7d9906 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java @@ -36,6 +36,7 @@ import org.apache.doris.common.TreeNode; import org.apache.doris.common.UserException; import org.apache.doris.common.util.VectorizedUtil; +import org.apache.doris.statistics.NodeType; import org.apache.doris.statistics.PlanStats; import org.apache.doris.statistics.StatsDeriveResult; import org.apache.doris.thrift.TExplainLevel; @@ -181,35 +182,6 @@ protected PlanNode(PlanNodeId id, PlanNode node, String planNodeName, NodeType n this.nodeType = nodeType; } - public enum NodeType { - DEFAULT, - AGG_NODE, - ANALYTIC_EVAL_NODE, - ASSERT_NUM_ROWS_NODE, - BROKER_SCAN_NODE, - CROSS_JOIN_NODE, - EMPTY_SET_NODE, - ES_SCAN_NODE, - EXCEPT_NODE, - EXCHANGE_NODE, - HASH_JOIN_NODE, - HIVE_SCAN_NODE, - ICEBERG_SCAN_NODE, - INTERSECT_NODE, - LOAD_SCAN_NODE, - MYSQL_SCAN_NODE, - ODBC_SCAN_NODE, - OLAP_SCAN_NODE, - REPEAT_NODE, - SELECT_NODE, - SET_OPERATION_NODE, - SCHEMA_SCAN_NODE, - SORT_NODE, - STREAM_LOAD_SCAN_NODE, - TABLE_FUNCTION_NODE, - UNION_NODE, - } - public String getPlanNodeName() { return planNodeName; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java index 82e3c82f9750a1..6a32ec7672223f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java @@ -30,6 +30,7 @@ import org.apache.doris.analysis.TupleId; import org.apache.doris.analysis.VirtualSlotRef; import org.apache.doris.common.UserException; +import org.apache.doris.statistics.NodeType; import org.apache.doris.statistics.StatsRecursiveDerive; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TPlanNode; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index e2392ddc7a864f..0c6adb6a3ceb42 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -35,6 +35,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.UserException; +import org.apache.doris.statistics.NodeType; import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.thrift.TScanRangeLocations; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java index 6bea955cd5bff2..ece5e3bdf85ce5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java @@ -25,6 +25,7 @@ import org.apache.doris.common.util.Util; import org.apache.doris.qe.ConnectContext; import org.apache.doris.service.FrontendOptions; +import org.apache.doris.statistics.NodeType; import org.apache.doris.thrift.TPlanNode; import org.apache.doris.thrift.TPlanNodeType; import org.apache.doris.thrift.TScanRangeLocations; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java index b56880c8892f3e..a9cd3983f49b4f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java @@ -23,6 +23,7 @@ import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.Expr; import org.apache.doris.common.UserException; +import org.apache.doris.statistics.NodeType; import org.apache.doris.statistics.StatsRecursiveDerive; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TPlanNode; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java index 257ff517ccb60a..b3a1effea53411 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java @@ -26,6 +26,7 @@ import org.apache.doris.common.CheckedMath; import org.apache.doris.common.UserException; import org.apache.doris.common.util.VectorizedUtil; +import org.apache.doris.statistics.NodeType; import org.apache.doris.thrift.TExceptNode; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TExpr; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java index 0e223c4f1ab413..35a02e168e4d94 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java @@ -29,6 +29,7 @@ import org.apache.doris.analysis.SortInfo; import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.UserException; +import org.apache.doris.statistics.NodeType; import org.apache.doris.statistics.StatsRecursiveDerive; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TPlanNode; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java index 0049eeaa0ebdce..b07cb04bc4c5a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java @@ -30,6 +30,7 @@ import org.apache.doris.common.util.VectorizedUtil; import org.apache.doris.load.Load; import org.apache.doris.load.loadv2.LoadTask; +import org.apache.doris.statistics.NodeType; import org.apache.doris.task.LoadTaskInfo; import org.apache.doris.thrift.TBrokerRangeDesc; import org.apache.doris.thrift.TBrokerScanRange; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java index ed61dd85eb6c42..ec9940e36f4362 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java @@ -26,6 +26,7 @@ import org.apache.doris.analysis.TupleId; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; +import org.apache.doris.statistics.NodeType; import org.apache.doris.statistics.StatsRecursiveDerive; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TPlanNode; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/UnionNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/UnionNode.java index 1fcf125dd36b5c..f6d8cd8bc0268c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/UnionNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/UnionNode.java @@ -22,6 +22,7 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.TupleId; +import org.apache.doris.statistics.NodeType; import org.apache.doris.thrift.TPlanNode; import org.apache.doris.thrift.TPlanNodeType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalScanNode.java index 1d4f4b983f4f2d..c12e2913cb0ed4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalScanNode.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanNode; +import org.apache.doris.statistics.NodeType; import org.apache.doris.thrift.TPlanNode; import org.apache.doris.thrift.TScanRangeLocations; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseStatsDerive.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseStatsDerive.java index 5e1dd4984f4f80..98fcbd455f357c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseStatsDerive.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseStatsDerive.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.SlotId; +import org.apache.doris.common.Id; import org.apache.doris.common.UserException; import com.google.common.base.Preconditions; @@ -154,16 +155,16 @@ protected long deriveRowCount() { } - protected HashMap deriveColumnToDataSize() { - HashMap columnToDataSize = new HashMap<>(); + protected HashMap deriveColumnToDataSize() { + HashMap columnToDataSize = new HashMap<>(); for (StatsDeriveResult child : childrenStatsResult) { columnToDataSize.putAll(child.getColumnToDataSize()); } return columnToDataSize; } - protected HashMap deriveColumnToNdv() { - HashMap columnToNdv = new HashMap<>(); + protected HashMap deriveColumnToNdv() { + HashMap columnToNdv = new HashMap<>(); for (StatsDeriveResult child : childrenStatsResult) { columnToNdv.putAll(child.getColumnToNdv()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/DeriveFactory.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/DeriveFactory.java index 7d694ee6812dac..e92e6d79be208b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/DeriveFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/DeriveFactory.java @@ -17,11 +17,9 @@ package org.apache.doris.statistics; -import org.apache.doris.planner.PlanNode; - public class DeriveFactory { - public BaseStatsDerive getStatsDerive(PlanNode.NodeType nodeType) { + public BaseStatsDerive getStatsDerive(NodeType nodeType) { switch (nodeType) { case AGG_NODE: return new AggStatsDerive(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/NodeType.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/NodeType.java new file mode 100644 index 00000000000000..28face6dc639e6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/NodeType.java @@ -0,0 +1,30 @@ +package org.apache.doris.statistics; + +public enum NodeType { + DEFAULT, + AGG_NODE, + ANALYTIC_EVAL_NODE, + ASSERT_NUM_ROWS_NODE, + BROKER_SCAN_NODE, + CROSS_JOIN_NODE, + EMPTY_SET_NODE, + ES_SCAN_NODE, + EXCEPT_NODE, + EXCHANGE_NODE, + HASH_JOIN_NODE, + HIVE_SCAN_NODE, + ICEBERG_SCAN_NODE, + INTERSECT_NODE, + LOAD_SCAN_NODE, + MYSQL_SCAN_NODE, + ODBC_SCAN_NODE, + OLAP_SCAN_NODE, + REPEAT_NODE, + SELECT_NODE, + SET_OPERATION_NODE, + SCHEMA_SCAN_NODE, + SORT_NODE, + STREAM_LOAD_SCAN_NODE, + TABLE_FUNCTION_NODE, + UNION_NODE, +} \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapScanStatsDerive.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapScanStatsDerive.java index 4e12cdc81477f1..3be8fffcba05b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapScanStatsDerive.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapScanStatsDerive.java @@ -18,8 +18,8 @@ package org.apache.doris.statistics; import org.apache.doris.analysis.SlotDescriptor; -import org.apache.doris.analysis.SlotId; import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.Id; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.planner.OlapScanNode; @@ -39,9 +39,9 @@ public class OlapScanStatsDerive extends BaseStatsDerive { // The rowCount here is the number of rows. private long inputRowCount = -1; - private Map slotIdToDataSize; - private Map slotIdToNdv; - private Map> slotIdToTableIdAndColumnName; + private Map slotIdToDataSize; + private Map slotIdToNdv; + private Map> slotIdToTableIdAndColumnName; @Override public void init(PlanStats node) throws UserException { @@ -60,7 +60,7 @@ public StatsDeriveResult deriveStats() { * - So only an inaccurate cardinality can be calculated here. */ rowCount = inputRowCount; - for (Map.Entry> pairEntry : slotIdToTableIdAndColumnName.entrySet()) { + for (Map.Entry> pairEntry : slotIdToTableIdAndColumnName.entrySet()) { Pair ndvAndDataSize = getNdvAndDataSizeFromStatistics(pairEntry.getValue()); long ndv = ndvAndDataSize.first; float dataSize = ndvAndDataSize.second; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/PlanStats.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/PlanStats.java index 4c9f23ea702440..be2a4dde5aa5fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/PlanStats.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/PlanStats.java @@ -17,8 +17,6 @@ package org.apache.doris.statistics; -import org.apache.doris.planner.PlanNode.NodeType; - import java.util.List; /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/SlotStatsDeriveResult.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/SlotStatsDeriveResult.java new file mode 100644 index 00000000000000..eb31a1ab4aeaa3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/SlotStatsDeriveResult.java @@ -0,0 +1,35 @@ +package org.apache.doris.statistics; + +import org.apache.doris.nereids.trees.expressions.Literal; + +public class SlotStatsDeriveResult { + + // number of distinct value + private long ndv; + private Literal max; + private Literal min; + + public long getNdv() { + return ndv; + } + + public void setNdv(long ndv) { + this.ndv = ndv; + } + + public Literal getMax() { + return max; + } + + public void setMax(Literal max) { + this.max = max; + } + + public Literal getMin() { + return min; + } + + public void setMin(Literal min) { + this.min = min; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsDeriveResult.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsDeriveResult.java index 2bf0a53f81ab26..1888b3dd45f613 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsDeriveResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsDeriveResult.java @@ -17,7 +17,7 @@ package org.apache.doris.statistics; -import org.apache.doris.analysis.SlotId; +import org.apache.doris.common.Id; import com.google.common.collect.Maps; @@ -28,12 +28,12 @@ public class StatsDeriveResult { private long rowCount = -1; // The data size of the corresponding column in the operator // The actual key is slotId - private final Map columnToDataSize = Maps.newHashMap(); + private final Map columnToDataSize = Maps.newHashMap(); // The ndv of the corresponding column in the operator // The actual key is slotId - private final Map columnToNdv = Maps.newHashMap(); + private final Map columnToNdv = Maps.newHashMap(); - public StatsDeriveResult(long rowCount, Map columnToDataSize, Map columnToNdv) { + public StatsDeriveResult(long rowCount, Map columnToDataSize, Map columnToNdv) { this.rowCount = rowCount; this.columnToDataSize.putAll(columnToDataSize); this.columnToNdv.putAll(columnToNdv); @@ -47,11 +47,11 @@ public long getRowCount() { return rowCount; } - public Map getColumnToNdv() { + public Map getColumnToNdv() { return columnToNdv; } - public Map getColumnToDataSize() { + public Map getColumnToDataSize() { return columnToDataSize; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java index b5e5fba18f50a8..c1914704b2d5f2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java @@ -26,7 +26,7 @@ import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.memo.Memo; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.operators.plans.logical.LogicalProject; import org.apache.doris.nereids.operators.plans.logical.LogicalRelation; import org.apache.doris.nereids.properties.PhysicalProperties; @@ -72,10 +72,10 @@ public void testSimplestScene() throws AnalysisException { Assertions.assertEquals(1, rootGroup.getLogicalExpressions().size()); GroupExpression rootGroupExpression = rootGroup.getLogicalExpression(); Assertions.assertEquals(1, rootGroupExpression.children().size()); - Assertions.assertEquals(OperatorType.LOGICAL_PROJECT, rootGroupExpression.getOperator().getType()); + Assertions.assertEquals(PlanType.LOGICAL_PROJECT, rootGroupExpression.getOperator().getType()); Group leafGroup = rootGroupExpression.child(0); Assertions.assertEquals(1, leafGroup.getLogicalExpressions().size()); GroupExpression leafGroupExpression = leafGroup.getLogicalExpression(); - Assertions.assertEquals(OperatorType.LOGICAL_BOUND_RELATION, leafGroupExpression.getOperator().getType()); + Assertions.assertEquals(PlanType.LOGICAL_BOUND_RELATION, leafGroupExpression.getOperator().getType()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java index 4a3bbed0c4191e..ded3e956030ba7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java @@ -19,7 +19,7 @@ import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.memo.Memo; -import org.apache.doris.nereids.operators.OperatorType; +import org.apache.doris.nereids.operators.PlanType; import org.apache.doris.nereids.operators.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.Plans; @@ -34,7 +34,7 @@ public class GroupExpressionMatchingTest implements Plans { @Test public void testLeafNode() { - Pattern pattern = new Pattern<>(OperatorType.LOGICAL_UNBOUND_RELATION); + Pattern pattern = new Pattern<>(PlanType.LOGICAL_UNBOUND_RELATION); UnboundRelation unboundRelation = new UnboundRelation(Lists.newArrayList("test")); Plan plan = plan(unboundRelation); @@ -47,14 +47,14 @@ public void testLeafNode() { Assertions.assertTrue(iterator.hasNext()); Plan actual = iterator.next(); - Assertions.assertEquals(OperatorType.LOGICAL_UNBOUND_RELATION, actual.getOperator().getType()); + Assertions.assertEquals(PlanType.LOGICAL_UNBOUND_RELATION, actual.getOperator().getType()); Assertions.assertFalse(iterator.hasNext()); } @Test public void testDepth2() { - Pattern pattern = new Pattern<>(OperatorType.LOGICAL_PROJECT, - new Pattern<>(OperatorType.LOGICAL_UNBOUND_RELATION)); + Pattern pattern = new Pattern<>(PlanType.LOGICAL_PROJECT, + new Pattern<>(PlanType.LOGICAL_UNBOUND_RELATION)); UnboundRelation unboundRelation = new UnboundRelation(Lists.newArrayList("test")); Plan leaf = plan(unboundRelation); @@ -74,20 +74,20 @@ public void testDepth2() { Assertions.assertTrue(iterator.hasNext()); Plan actual; actual = iterator.next(); - Assertions.assertEquals(OperatorType.LOGICAL_PROJECT, actual.getOperator().getType()); + Assertions.assertEquals(PlanType.LOGICAL_PROJECT, actual.getOperator().getType()); Assertions.assertEquals(1, actual.arity()); - Assertions.assertEquals(OperatorType.LOGICAL_UNBOUND_RELATION, actual.child(0).getOperator().getType()); + Assertions.assertEquals(PlanType.LOGICAL_UNBOUND_RELATION, actual.child(0).getOperator().getType()); Assertions.assertTrue(iterator.hasNext()); actual = iterator.next(); - Assertions.assertEquals(OperatorType.LOGICAL_PROJECT, actual.getOperator().getType()); + Assertions.assertEquals(PlanType.LOGICAL_PROJECT, actual.getOperator().getType()); Assertions.assertEquals(1, actual.arity()); - Assertions.assertEquals(OperatorType.LOGICAL_UNBOUND_RELATION, actual.child(0).getOperator().getType()); + Assertions.assertEquals(PlanType.LOGICAL_UNBOUND_RELATION, actual.child(0).getOperator().getType()); Assertions.assertFalse(iterator.hasNext()); } @Test public void testDepth2WithFixed() { - Pattern pattern = new Pattern<>(OperatorType.LOGICAL_PROJECT, new Pattern<>(OperatorType.FIXED)); + Pattern pattern = new Pattern<>(PlanType.LOGICAL_PROJECT, new Pattern<>(PlanType.FIXED)); UnboundRelation unboundRelation = new UnboundRelation(Lists.newArrayList("test")); Plan leaf = plan(unboundRelation); @@ -107,9 +107,9 @@ public void testDepth2WithFixed() { Assertions.assertTrue(iterator.hasNext()); Plan actual; actual = iterator.next(); - Assertions.assertEquals(OperatorType.LOGICAL_PROJECT, actual.getOperator().getType()); + Assertions.assertEquals(PlanType.LOGICAL_PROJECT, actual.getOperator().getType()); Assertions.assertEquals(1, actual.arity()); - Assertions.assertEquals(OperatorType.LOGICAL_UNBOUND_RELATION, actual.child(0).getOperator().getType()); + Assertions.assertEquals(PlanType.LOGICAL_UNBOUND_RELATION, actual.child(0).getOperator().getType()); Assertions.assertFalse(iterator.hasNext()); } }