From e2e91d5a22eb3e37cc7bc189b80135fb9beae0cb Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Wed, 1 Apr 2026 19:20:48 +0800 Subject: [PATCH 01/10] First step by opus-4.6 --- .../query/recent/IoTExplainJsonFormatIT.java | 251 ++++++++++++ .../queryengine/common/MPPQueryContext.java | 10 + .../operator/ExplainAnalyzeOperator.java | 37 ++ ...ableModelStatementMemorySourceVisitor.java | 65 ++- .../plan/planner/TableOperatorGenerator.java | 7 +- .../plan/node/PlanGraphJsonPrinter.java | 220 +++++++++++ .../analyzer/StatementAnalyzer.java | 2 + .../planner/TableLogicalPlanner.java | 3 +- .../planner/node/ExplainAnalyzeNode.java | 39 +- .../plan/relational/sql/ast/Explain.java | 22 +- .../relational/sql/ast/ExplainAnalyze.java | 24 +- .../sql/ast/ExplainOutputFormat.java | 26 ++ .../relational/sql/parser/AstBuilder.java | 35 +- .../FragmentInstanceStatisticsJsonDrawer.java | 371 ++++++++++++++++++ .../node/PlanGraphJsonPrinterTest.java | 120 ++++++ .../relational/sql/ExplainFormatTest.java | 127 ++++++ .../relational/grammar/sql/RelationalSql.g4 | 4 +- 17 files changed, 1345 insertions(+), 18 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/PlanGraphJsonPrinterTest.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ExplainFormatTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java new file mode 100644 index 0000000000000..94ed692ab6410 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java @@ -0,0 +1,251 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Locale; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class}) +public class IoTExplainJsonFormatIT { + private static final String DATABASE_NAME = "testdb_json"; + + @BeforeClass + public static void setUp() { + Locale.setDefault(Locale.ENGLISH); + + EnvFactory.getEnv().getConfig().getCommonConfig().setPartitionInterval(1000); + EnvFactory.getEnv().initClusterEnvironment(); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE IF NOT EXISTS " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE TABLE IF NOT EXISTS testtb(deviceid STRING TAG, voltage FLOAT FIELD)"); + statement.execute("INSERT INTO testtb VALUES(1000, 'd1', 100.0)"); + statement.execute("INSERT INTO testtb VALUES(2000, 'd1', 200.0)"); + statement.execute("INSERT INTO testtb VALUES(1000, 'd2', 300.0)"); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @AfterClass + public static void tearDown() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("DROP DATABASE IF EXISTS " + DATABASE_NAME); + } catch (Exception e) { + // ignore + } + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testExplainJsonFormat() { + String sql = "EXPLAIN (FORMAT JSON) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue("Should have at least one row", resultSet.next()); + String jsonStr = resultSet.getString(1); + Assert.assertNotNull(jsonStr); + + // Verify it's valid JSON + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + Assert.assertTrue("JSON should have 'name' field", root.has("name")); + Assert.assertTrue("JSON should have 'id' field", root.has("id")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainDefaultFormatIsNotJson() { + String sql = "EXPLAIN SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue("Should have at least one row", resultSet.next()); + String firstLine = resultSet.getString(1); + // Default format (GRAPHVIZ) produces box-drawing characters, not JSON + Assert.assertFalse( + "Default EXPLAIN should not produce JSON", firstLine.trim().startsWith("{")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainAnalyzeJsonFormat() { + String sql = "EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + // JSON format should produce a single row with full JSON + Assert.assertTrue("Should have at least one row", resultSet.next()); + String jsonStr = resultSet.getString(1); + Assert.assertNotNull(jsonStr); + + // Verify it's valid JSON + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + Assert.assertTrue("JSON should have 'planStatistics' field", root.has("planStatistics")); + Assert.assertTrue( + "JSON should have 'fragmentInstances' field", root.has("fragmentInstances")); + Assert.assertTrue( + "JSON should have 'fragmentInstancesCount' field", root.has("fragmentInstancesCount")); + + // Plan statistics should contain known keys + JsonObject planStats = root.getAsJsonObject("planStatistics"); + Assert.assertTrue(planStats.has("analyzeCostMs")); + Assert.assertTrue(planStats.has("logicalPlanCostMs")); + Assert.assertTrue(planStats.has("distributionPlanCostMs")); + Assert.assertTrue(planStats.has("dispatchCostMs")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainAnalyzeVerboseJsonFormat() { + String sql = "EXPLAIN ANALYZE VERBOSE (FORMAT JSON) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue("Should have at least one row", resultSet.next()); + String jsonStr = resultSet.getString(1); + Assert.assertNotNull(jsonStr); + + // Verify it's valid JSON + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + Assert.assertTrue(root.has("planStatistics")); + Assert.assertTrue(root.has("fragmentInstances")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainAnalyzeDefaultIsText() { + String sql = "EXPLAIN ANALYZE SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue("Should have at least one row", resultSet.next()); + String firstLine = resultSet.getString(1); + // Default format (TEXT) should not start with '{' + Assert.assertFalse( + "Default EXPLAIN ANALYZE should not produce JSON", firstLine.trim().startsWith("{")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainGraphvizFormatExplicit() { + String sql = "EXPLAIN (FORMAT GRAPHVIZ) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue("Should have at least one row", resultSet.next()); + String firstLine = resultSet.getString(1); + // GRAPHVIZ format produces box-drawing characters, not JSON + Assert.assertFalse( + "GRAPHVIZ EXPLAIN should not produce JSON", firstLine.trim().startsWith("{")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainAnalyzeTextFormatExplicit() { + String sql = "EXPLAIN ANALYZE (FORMAT TEXT) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue("Should have at least one row", resultSet.next()); + String firstLine = resultSet.getString(1); + Assert.assertFalse( + "TEXT EXPLAIN ANALYZE should not produce JSON", firstLine.trim().startsWith("{")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test(expected = SQLException.class) + public void testExplainInvalidFormat() throws SQLException { + String sql = "EXPLAIN (FORMAT XML) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + statement.executeQuery(sql); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index 88bd1998f683c..43b214f226cc3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -34,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.db.queryengine.plan.planner.memory.NotThreadSafeMemoryReservationManager; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.NodeRef; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; @@ -108,6 +109,7 @@ public enum ExplainType { // - EXPLAIN: Show the logical and physical query plan without execution // - EXPLAIN_ANALYZE: Execute the query and collect detailed execution statistics private ExplainType explainType = ExplainType.NONE; + private ExplainOutputFormat explainOutputFormat = null; private boolean verbose = false; private QueryPlanStatistics queryPlanStatistics = null; @@ -346,6 +348,14 @@ public ExplainType getExplainType() { return explainType; } + public void setExplainOutputFormat(ExplainOutputFormat explainOutputFormat) { + this.explainOutputFormat = explainOutputFormat; + } + + public ExplainOutputFormat getExplainOutputFormat() { + return explainOutputFormat; + } + public boolean isExplainAnalyze() { return explainType == ExplainType.EXPLAIN_ANALYZE; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java index b4f1a5c7261bf..52af0bb745583 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java @@ -33,8 +33,10 @@ import org.apache.iotdb.db.queryengine.plan.execution.QueryExecution; import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.NodeRef; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; import org.apache.iotdb.db.queryengine.statistics.FragmentInstanceStatisticsDrawer; +import org.apache.iotdb.db.queryengine.statistics.FragmentInstanceStatisticsJsonDrawer; import org.apache.iotdb.db.queryengine.statistics.QueryStatisticsFetcher; import org.apache.iotdb.db.queryengine.statistics.StatisticLine; import org.apache.iotdb.db.utils.SetThreadName; @@ -78,9 +80,12 @@ public class ExplainAnalyzeOperator implements ProcessOperator { private final boolean verbose; private boolean outputResult = false; private final List instances; + private final ExplainOutputFormat outputFormat; private final FragmentInstanceStatisticsDrawer fragmentInstanceStatisticsDrawer = new FragmentInstanceStatisticsDrawer(); + private final FragmentInstanceStatisticsJsonDrawer fragmentInstanceStatisticsJsonDrawer = + new FragmentInstanceStatisticsJsonDrawer(); private final ScheduledFuture logRecordTask; private final IClientManager clientManager; @@ -92,9 +97,20 @@ public ExplainAnalyzeOperator( long queryId, boolean verbose, long timeout) { + this(operatorContext, child, queryId, verbose, timeout, ExplainOutputFormat.TEXT); + } + + public ExplainAnalyzeOperator( + OperatorContext operatorContext, + Operator child, + long queryId, + boolean verbose, + long timeout, + ExplainOutputFormat outputFormat) { this.operatorContext = operatorContext; this.child = child; this.verbose = verbose; + this.outputFormat = outputFormat; Coordinator coordinator = Coordinator.getInstance(); this.clientManager = coordinator.getInternalServiceClientManager(); @@ -103,6 +119,7 @@ public ExplainAnalyzeOperator( this.instances = queryExecution.getDistributedPlan().getInstances(); mppQueryContext = queryExecution.getContext(); fragmentInstanceStatisticsDrawer.renderPlanStatistics(mppQueryContext); + fragmentInstanceStatisticsJsonDrawer.renderPlanStatistics(mppQueryContext); // The time interval guarantees the result of EXPLAIN ANALYZE will be printed at least three // times. @@ -130,6 +147,7 @@ public TsBlock next() throws Exception { } fragmentInstanceStatisticsDrawer.renderDispatchCost(mppQueryContext); + fragmentInstanceStatisticsJsonDrawer.renderDispatchCost(mppQueryContext); // fetch statics from all fragment instances TsBlock result = buildResult(); @@ -182,6 +200,9 @@ private void logIntermediateResultIfTimeout() { } private TsBlock buildResult() throws FragmentInstanceFetchException { + if (outputFormat == ExplainOutputFormat.JSON) { + return buildJsonResult(); + } Map, Pair>> cteAnalyzeResults = mppQueryContext.getCteExplainResults(); List mainAnalyzeResult = buildFragmentInstanceStatistics(instances, verbose); @@ -199,6 +220,22 @@ private TsBlock buildResult() throws FragmentInstanceFetchException { return builder.build(); } + private TsBlock buildJsonResult() throws FragmentInstanceFetchException { + Map allStatistics = + QueryStatisticsFetcher.fetchAllStatistics(instances, clientManager); + String jsonResult = + fragmentInstanceStatisticsJsonDrawer.renderFragmentInstancesAsJson( + instances, allStatistics, verbose); + + TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.TEXT)); + TimeColumnBuilder timeColumnBuilder = builder.getTimeColumnBuilder(); + ColumnBuilder columnBuilder = builder.getColumnBuilder(0); + timeColumnBuilder.writeLong(0); + columnBuilder.writeBinary(new Binary(jsonResult.getBytes())); + builder.declarePosition(); + return builder.build(); + } + private List mergeAnalyzeResults( Map, Pair>> cteAnalyzeResults, List mainAnalyzeResult) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java index 932d941979223..c6a7692395a37 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.queryengine.plan.Coordinator; import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner; import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanGraphJsonPrinter; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanGraphPrinter; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.NodeRef; @@ -35,6 +36,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CountDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; @@ -103,15 +105,26 @@ public StatementMemorySource visitExplain( Coordinator.getInstance().getDataNodeLocationSupplier()) .generateDistributedPlanWithOptimize(planContext); - List mainExplainResult = - outputNodeWithExchange.accept( - new PlanGraphPrinter(), - new PlanGraphPrinter.GraphContext( - context.getQueryContext().getTypeProvider().getTemplatedInfo())); + List mainExplainResult; + if (node.getOutputFormat() == ExplainOutputFormat.JSON) { + mainExplainResult = PlanGraphJsonPrinter.getJsonLines(outputNodeWithExchange); + } else { + mainExplainResult = + outputNodeWithExchange.accept( + new PlanGraphPrinter(), + new PlanGraphPrinter.GraphContext( + context.getQueryContext().getTypeProvider().getTemplatedInfo())); + } Map, Pair>> cteExplainResults = context.getQueryContext().getCteExplainResults(); - List lines = mergeExplainResults(cteExplainResults, mainExplainResult); + List lines; + if (node.getOutputFormat() == ExplainOutputFormat.JSON) { + // For JSON format, we merge CTE results into the JSON output + lines = mergeExplainResultsJson(cteExplainResults, mainExplainResult); + } else { + lines = mergeExplainResults(cteExplainResults, mainExplainResult); + } return getStatementMemorySource(header, lines); } @@ -149,4 +162,44 @@ private List mergeExplainResults( return analyzeResult; } + + private List mergeExplainResultsJson( + Map, Pair>> cteExplainResults, + List mainExplainResult) { + if (cteExplainResults.isEmpty()) { + return mainExplainResult; + } + + // For JSON format with CTEs, wrap everything in a combined JSON object + StringBuilder sb = new StringBuilder(); + sb.append("{\n"); + sb.append(" \"cteQueries\": [\n"); + int cteIndex = 0; + int cteSize = cteExplainResults.size(); + for (Map.Entry, Pair>> entry : + cteExplainResults.entrySet()) { + sb.append(" {\n"); + sb.append(" \"name\": \"").append(entry.getKey().getNode().getName()).append("\",\n"); + sb.append(" \"plan\": "); + // Each CTE's plan is already a JSON string + for (String line : entry.getValue().getRight()) { + sb.append(line); + } + sb.append("\n }"); + if (++cteIndex < cteSize) { + sb.append(","); + } + sb.append("\n"); + } + sb.append(" ],\n"); + sb.append(" \"mainQuery\": "); + for (String line : mainExplainResult) { + sb.append(line); + } + sb.append("\n}"); + + List result = new ArrayList<>(); + result.add(sb.toString()); + return result; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index c31a2061f49ec..fe96177ee5f2c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -3472,7 +3472,12 @@ public Operator visitExplainAnalyze(ExplainAnalyzeNode node, LocalExecutionPlanC node.getPlanNodeId(), ExplainAnalyzeOperator.class.getSimpleName()); return new ExplainAnalyzeOperator( - operatorContext, operator, node.getQueryId(), node.isVerbose(), node.getTimeout()); + operatorContext, + operator, + node.getQueryId(), + node.isVerbose(), + node.getTimeout(), + node.getOutputFormat()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java new file mode 100644 index 0000000000000..e35c23d466059 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExplainAnalyzeNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + +import java.util.ArrayList; +import java.util.List; + +/** + * Converts a plan tree into a JSON representation. Each plan node becomes a JSON object with: - + * "name": the node type with its plan node ID - "id": the plan node ID - "properties": key-value + * pairs - "children": array of child nodes + */ +public class PlanGraphJsonPrinter { + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + + private static final String REGION_NOT_ASSIGNED = "Not Assigned"; + + public static String toPrettyJson(PlanNode node) { + JsonObject root = buildJsonNode(node); + return GSON.toJson(root); + } + + private static JsonObject buildJsonNode(PlanNode node) { + JsonObject jsonNode = new JsonObject(); + String simpleName = node.getClass().getSimpleName(); + String nodeId = node.getPlanNodeId().getId(); + + jsonNode.addProperty("name", simpleName + "-" + nodeId); + jsonNode.addProperty("id", nodeId); + + JsonObject properties = buildProperties(node); + if (properties.size() > 0) { + jsonNode.add("properties", properties); + } + + List children = node.getChildren(); + if (children != null && !children.isEmpty()) { + JsonArray childrenArray = new JsonArray(); + for (PlanNode child : children) { + childrenArray.add(buildJsonNode(child)); + } + jsonNode.add("children", childrenArray); + } + + return jsonNode; + } + + private static JsonObject buildProperties(PlanNode node) { + JsonObject properties = new JsonObject(); + + if (node instanceof OutputNode) { + OutputNode n = (OutputNode) node; + properties.addProperty("OutputColumns", String.valueOf(n.getOutputColumnNames())); + properties.addProperty("OutputSymbols", String.valueOf(n.getOutputSymbols())); + } else if (node instanceof ExplainAnalyzeNode) { + ExplainAnalyzeNode n = (ExplainAnalyzeNode) node; + properties.addProperty("ChildPermittedOutputs", String.valueOf(n.getChildPermittedOutputs())); + } else if (node instanceof TableScanNode) { + buildTableScanProperties(properties, (TableScanNode) node); + } else if (node instanceof ExchangeNode) { + // No extra properties needed + } else if (node + instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode) { + buildAggregationProperties( + properties, + (org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode) node); + } else if (node + instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode) { + org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode n = + (org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode) node; + properties.addProperty("Predicate", String.valueOf(n.getPredicate())); + } else if (node + instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode) { + org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode n = + (org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode) node; + properties.addProperty("OutputSymbols", String.valueOf(n.getOutputSymbols())); + properties.addProperty("Expressions", String.valueOf(n.getAssignments().getMap().values())); + } else if (node + instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode) { + org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode n = + (org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode) node; + properties.addProperty("Count", String.valueOf(n.getCount())); + } else if (node + instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode) { + org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode n = + (org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode) node; + properties.addProperty("Count", String.valueOf(n.getCount())); + } else if (node + instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode) { + org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode n = + (org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode) node; + properties.addProperty("OrderBy", String.valueOf(n.getOrderingScheme())); + } else if (node + instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode) { + org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode n = + (org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode) node; + properties.addProperty("OrderBy", String.valueOf(n.getOrderingScheme())); + } else if (node + instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode) { + org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode n = + (org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode) node; + properties.addProperty("JoinType", String.valueOf(n.getJoinType())); + properties.addProperty("Criteria", String.valueOf(n.getCriteria())); + properties.addProperty("OutputSymbols", String.valueOf(n.getOutputSymbols())); + } else if (node + instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.UnionNode) { + org.apache.iotdb.db.queryengine.plan.relational.planner.node.UnionNode n = + (org.apache.iotdb.db.queryengine.plan.relational.planner.node.UnionNode) node; + properties.addProperty("OutputSymbols", String.valueOf(n.getOutputSymbols())); + } + + return properties; + } + + private static void buildTableScanProperties(JsonObject properties, TableScanNode node) { + properties.addProperty("QualifiedTableName", node.getQualifiedObjectName().toString()); + properties.addProperty("OutputSymbols", String.valueOf(node.getOutputSymbols())); + + if (node instanceof DeviceTableScanNode) { + DeviceTableScanNode deviceNode = (DeviceTableScanNode) node; + properties.addProperty("DeviceNumber", String.valueOf(deviceNode.getDeviceEntries().size())); + properties.addProperty("ScanOrder", String.valueOf(deviceNode.getScanOrder())); + if (deviceNode.getTimePredicate().isPresent()) { + properties.addProperty( + "TimePredicate", String.valueOf(deviceNode.getTimePredicate().get())); + } + } + + if (node.getPushDownPredicate() != null) { + properties.addProperty("PushDownPredicate", String.valueOf(node.getPushDownPredicate())); + } + properties.addProperty("PushDownOffset", String.valueOf(node.getPushDownOffset())); + properties.addProperty("PushDownLimit", String.valueOf(node.getPushDownLimit())); + + if (node instanceof DeviceTableScanNode) { + properties.addProperty( + "PushDownLimitToEachDevice", + String.valueOf(((DeviceTableScanNode) node).isPushLimitToEachDevice())); + } + + properties.addProperty( + "RegionId", + node.getRegionReplicaSet() == null || node.getRegionReplicaSet().getRegionId() == null + ? REGION_NOT_ASSIGNED + : String.valueOf(node.getRegionReplicaSet().getRegionId().getId())); + + if (node instanceof TreeDeviceViewScanNode) { + TreeDeviceViewScanNode treeNode = (TreeDeviceViewScanNode) node; + properties.addProperty("TreeDB", treeNode.getTreeDBName()); + properties.addProperty( + "MeasurementToColumnName", String.valueOf(treeNode.getMeasurementColumnNameMap())); + } + } + + private static void buildAggregationProperties( + JsonObject properties, + org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode node) { + properties.addProperty("OutputSymbols", String.valueOf(node.getOutputSymbols())); + + JsonArray aggregators = new JsonArray(); + int i = 0; + for (org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Aggregation + aggregation : node.getAggregations().values()) { + JsonObject agg = new JsonObject(); + agg.addProperty("index", i++); + agg.addProperty("function", aggregation.getResolvedFunction().toString()); + if (aggregation.hasMask()) { + agg.addProperty("mask", String.valueOf(aggregation.getMask().get())); + } + if (aggregation.isDistinct()) { + agg.addProperty("distinct", true); + } + aggregators.add(agg); + } + properties.add("Aggregators", aggregators); + + properties.addProperty("GroupingKeys", String.valueOf(node.getGroupingKeys())); + if (node.isStreamable()) { + properties.addProperty("Streamable", true); + properties.addProperty("PreGroupedSymbols", String.valueOf(node.getPreGroupedSymbols())); + } + properties.addProperty("Step", String.valueOf(node.getStep())); + } + + public static List getJsonLines(PlanNode node) { + List lines = new ArrayList<>(); + lines.add(toPrettyJson(node)); + return lines; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index cff29c0d83a1a..eb4e1241d1400 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -847,6 +847,7 @@ protected Scope visitLoadTsFile(final LoadTsFile node, final Optional sco @Override protected Scope visitExplain(Explain node, Optional context) { queryContext.setExplainType(ExplainType.EXPLAIN); + queryContext.setExplainOutputFormat(node.getOutputFormat()); analysis.setFinishQueryAfterAnalyze(); return visitQuery((Query) node.getStatement(), context); } @@ -863,6 +864,7 @@ protected Scope visitCopyTo(CopyTo node, Optional context) { protected Scope visitExplainAnalyze(ExplainAnalyze node, Optional context) { queryContext.setExplainType(ExplainType.EXPLAIN_ANALYZE); queryContext.setVerbose(node.isVerbose()); + queryContext.setExplainOutputFormat(node.getOutputFormat()); return visitQuery((Query) node.getStatement(), context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java index c33cb05250bd1..538890b15682a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java @@ -625,7 +625,8 @@ private RelationPlan planExplainAnalyze(final ExplainAnalyze statement, final An queryContext.getTimeOut(), symbol, // recording permittedOutputs of ExplainAnalyzeNode's child - getChildPermittedOutputs(analysis, statement.getStatement(), originalQueryPlan)); + getChildPermittedOutputs(analysis, statement.getStatement(), originalQueryPlan), + statement.getOutputFormat()); return new RelationPlan( newRoot, originalQueryPlan.getScope(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ExplainAnalyzeNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ExplainAnalyzeNode.java index 23aa8baad2dfd..d50d07521ec23 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ExplainAnalyzeNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ExplainAnalyzeNode.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; import java.io.DataOutputStream; import java.io.IOException; @@ -38,6 +39,7 @@ public class ExplainAnalyzeNode extends SingleChildProcessNode { private final long timeout; private final Symbol outputSymbol; private final List childPermittedOutputs; + private final ExplainOutputFormat outputFormat; public ExplainAnalyzeNode( PlanNodeId id, @@ -47,18 +49,46 @@ public ExplainAnalyzeNode( long timeout, Symbol outputSymbol, List childPermittedOutputs) { + this( + id, + child, + verbose, + queryId, + timeout, + outputSymbol, + childPermittedOutputs, + ExplainOutputFormat.TEXT); + } + + public ExplainAnalyzeNode( + PlanNodeId id, + PlanNode child, + boolean verbose, + long queryId, + long timeout, + Symbol outputSymbol, + List childPermittedOutputs, + ExplainOutputFormat outputFormat) { super(id, child); this.verbose = verbose; this.timeout = timeout; this.queryId = queryId; this.outputSymbol = outputSymbol; this.childPermittedOutputs = childPermittedOutputs; + this.outputFormat = outputFormat; } @Override public PlanNode clone() { return new ExplainAnalyzeNode( - getPlanNodeId(), child, verbose, queryId, timeout, outputSymbol, childPermittedOutputs); + getPlanNodeId(), + child, + verbose, + queryId, + timeout, + outputSymbol, + childPermittedOutputs, + outputFormat); } @Override @@ -89,7 +119,8 @@ public PlanNode replaceChildren(List newChildren) { queryId, timeout, outputSymbol, - childPermittedOutputs); + childPermittedOutputs, + outputFormat); } // ExplainAnalyze should be at the same region as Coordinator all the time. Therefore, there will @@ -116,6 +147,10 @@ public long getTimeout() { return timeout; } + public ExplainOutputFormat getOutputFormat() { + return outputFormat; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Explain.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Explain.java index f3a50175269e8..6e51e9bae607d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Explain.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Explain.java @@ -33,21 +33,34 @@ public class Explain extends Statement { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(Explain.class); private final Statement statement; + private final ExplainOutputFormat outputFormat; public Explain(Statement statement) { super(null); this.statement = requireNonNull(statement, "statement is null"); + this.outputFormat = ExplainOutputFormat.GRAPHVIZ; } public Explain(NodeLocation location, Statement statement) { super(requireNonNull(location, "location is null")); this.statement = requireNonNull(statement, "statement is null"); + this.outputFormat = ExplainOutputFormat.GRAPHVIZ; + } + + public Explain(NodeLocation location, Statement statement, ExplainOutputFormat outputFormat) { + super(requireNonNull(location, "location is null")); + this.statement = requireNonNull(statement, "statement is null"); + this.outputFormat = requireNonNull(outputFormat, "outputFormat is null"); } public Statement getStatement() { return statement; } + public ExplainOutputFormat getOutputFormat() { + return outputFormat; + } + @Override public R accept(AstVisitor visitor, C context) { return visitor.visitExplain(this, context); @@ -60,7 +73,7 @@ public List getChildren() { @Override public int hashCode() { - return Objects.hash(statement); + return Objects.hash(statement, outputFormat); } @Override @@ -72,12 +85,15 @@ public boolean equals(Object obj) { return false; } Explain o = (Explain) obj; - return Objects.equals(statement, o.statement); + return Objects.equals(statement, o.statement) && outputFormat == o.outputFormat; } @Override public String toString() { - return toStringHelper(this).add("statement", statement).toString(); + return toStringHelper(this) + .add("statement", statement) + .add("outputFormat", outputFormat) + .toString(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java index cd3df879fc55e..dc7798967e4d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java @@ -35,17 +35,31 @@ public class ExplainAnalyze extends Statement { private final Statement statement; private final boolean verbose; + private final ExplainOutputFormat outputFormat; public ExplainAnalyze(Statement statement, boolean verbose) { super(null); this.statement = requireNonNull(statement, "statement is null"); this.verbose = verbose; + this.outputFormat = ExplainOutputFormat.TEXT; } public ExplainAnalyze(NodeLocation location, boolean verbose, Statement statement) { super(requireNonNull(location, "location is null")); this.statement = requireNonNull(statement, "statement is null"); this.verbose = verbose; + this.outputFormat = ExplainOutputFormat.TEXT; + } + + public ExplainAnalyze( + NodeLocation location, + boolean verbose, + Statement statement, + ExplainOutputFormat outputFormat) { + super(requireNonNull(location, "location is null")); + this.statement = requireNonNull(statement, "statement is null"); + this.verbose = verbose; + this.outputFormat = requireNonNull(outputFormat, "outputFormat is null"); } public Statement getStatement() { @@ -56,6 +70,10 @@ public boolean isVerbose() { return verbose; } + public ExplainOutputFormat getOutputFormat() { + return outputFormat; + } + @Override public R accept(AstVisitor visitor, C context) { return visitor.visitExplainAnalyze(this, context); @@ -85,7 +103,11 @@ public boolean equals(Object obj) { @Override public String toString() { - return toStringHelper(this).add("statement", statement).add("verbose", verbose).toString(); + return toStringHelper(this) + .add("statement", statement) + .add("verbose", verbose) + .add("outputFormat", outputFormat) + .toString(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java new file mode 100644 index 0000000000000..9dba3649a7f7f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +public enum ExplainOutputFormat { + GRAPHVIZ, + TEXT, + JSON +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index a31b48c24b4a5..c825d6b92cb26 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -98,6 +98,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExtendRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; @@ -1804,7 +1805,22 @@ public Node visitExplain(RelationalSqlParser.ExplainContext ctx) { } else { innerStatement = (Statement) visit(ctx.executeImmediateStatement()); } - return new Explain(getLocation(ctx), innerStatement); + ExplainOutputFormat format = ExplainOutputFormat.GRAPHVIZ; + if (ctx.identifier() != null) { + String formatStr = ((Identifier) visit(ctx.identifier())).getValue().toUpperCase(); + switch (formatStr) { + case "GRAPHVIZ": + format = ExplainOutputFormat.GRAPHVIZ; + break; + case "JSON": + format = ExplainOutputFormat.JSON; + break; + default: + throw new SemanticException( + "Invalid EXPLAIN format: " + formatStr + ". Supported formats: GRAPHVIZ, JSON"); + } + } + return new Explain(getLocation(ctx), innerStatement, format); } @Override @@ -1817,7 +1833,22 @@ public Node visitExplainAnalyze(RelationalSqlParser.ExplainAnalyzeContext ctx) { } else { innerStatement = (Statement) visit(ctx.executeImmediateStatement()); } - return new ExplainAnalyze(getLocation(ctx), ctx.VERBOSE() != null, innerStatement); + ExplainOutputFormat format = ExplainOutputFormat.TEXT; + if (ctx.identifier() != null) { + String formatStr = ((Identifier) visit(ctx.identifier())).getValue().toUpperCase(); + switch (formatStr) { + case "TEXT": + format = ExplainOutputFormat.TEXT; + break; + case "JSON": + format = ExplainOutputFormat.JSON; + break; + default: + throw new SemanticException( + "Invalid EXPLAIN ANALYZE format: " + formatStr + ". Supported formats: TEXT, JSON"); + } + } + return new ExplainAnalyze(getLocation(ctx), ctx.VERBOSE() != null, innerStatement, format); } // ********************** author expressions ******************** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java new file mode 100644 index 0000000000000..3c6ff7bfb41b2 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java @@ -0,0 +1,371 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.statistics; + +import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.mpp.rpc.thrift.TFetchFragmentInstanceStatisticsResp; +import org.apache.iotdb.mpp.rpc.thrift.TOperatorStatistics; +import org.apache.iotdb.mpp.rpc.thrift.TQueryStatistics; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Produces JSON output for EXPLAIN ANALYZE results, mirroring the same data as {@link + * FragmentInstanceStatisticsDrawer} but in JSON format. + */ +public class FragmentInstanceStatisticsJsonDrawer { + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + private static final double NS_TO_MS_FACTOR = 1.0 / 1000000; + private static final double EPSILON = 1e-10; + + private final JsonObject planStatistics = new JsonObject(); + + public void renderPlanStatistics(MPPQueryContext context) { + planStatistics.addProperty( + "analyzeCostMs", formatMs(context.getAnalyzeCost() * NS_TO_MS_FACTOR)); + planStatistics.addProperty( + "fetchPartitionCostMs", formatMs(context.getFetchPartitionCost() * NS_TO_MS_FACTOR)); + planStatistics.addProperty( + "fetchSchemaCostMs", formatMs(context.getFetchSchemaCost() * NS_TO_MS_FACTOR)); + planStatistics.addProperty( + "logicalPlanCostMs", formatMs(context.getLogicalPlanCost() * NS_TO_MS_FACTOR)); + planStatistics.addProperty( + "logicalOptimizationCostMs", + formatMs(context.getLogicalOptimizationCost() * NS_TO_MS_FACTOR)); + planStatistics.addProperty( + "distributionPlanCostMs", formatMs(context.getDistributionPlanCost() * NS_TO_MS_FACTOR)); + } + + public void renderDispatchCost(MPPQueryContext context) { + planStatistics.addProperty( + "dispatchCostMs", formatMs(context.getDispatchCost() * NS_TO_MS_FACTOR)); + } + + public String renderFragmentInstancesAsJson( + List instancesToBeRendered, + Map allStatistics, + boolean verbose) { + + JsonObject root = new JsonObject(); + root.add("planStatistics", planStatistics); + + List validInstances = + instancesToBeRendered.stream() + .filter( + instance -> { + TFetchFragmentInstanceStatisticsResp statistics = + allStatistics.get(instance.getId()); + return statistics != null && statistics.getDataRegion() != null; + }) + .collect(Collectors.toList()); + + root.addProperty("fragmentInstancesCount", validInstances.size()); + + JsonArray fragmentInstancesArray = new JsonArray(); + for (FragmentInstance instance : validInstances) { + TFetchFragmentInstanceStatisticsResp statistics = allStatistics.get(instance.getId()); + JsonObject fiJson = new JsonObject(); + + fiJson.addProperty("id", instance.getId().toString()); + fiJson.addProperty("ip", statistics.getIp()); + fiJson.addProperty("dataRegion", statistics.getDataRegion()); + fiJson.addProperty("state", statistics.getState()); + fiJson.addProperty( + "totalWallTimeMs", statistics.getEndTimeInMS() - statistics.getStartTimeInMS()); + fiJson.addProperty( + "initDataQuerySourceCostMs", + formatMs(statistics.getInitDataQuerySourceCost() * NS_TO_MS_FACTOR)); + + if (statistics.isSetInitDataQuerySourceRetryCount() + && statistics.getInitDataQuerySourceRetryCount() > 0) { + fiJson.addProperty( + "initDataQuerySourceRetryCount", statistics.getInitDataQuerySourceRetryCount()); + } + + fiJson.addProperty("seqFileUnclosed", statistics.getSeqUnclosedNum()); + fiJson.addProperty("seqFileClosed", statistics.getSeqClosednNum()); + fiJson.addProperty("unseqFileUnclosed", statistics.getUnseqUnclosedNum()); + fiJson.addProperty("unseqFileClosed", statistics.getUnseqClosedNum()); + fiJson.addProperty( + "readyQueuedTimeMs", formatMs(statistics.getReadyQueuedTime() * NS_TO_MS_FACTOR)); + fiJson.addProperty( + "blockQueuedTimeMs", formatMs(statistics.getBlockQueuedTime() * NS_TO_MS_FACTOR)); + + // Query statistics + JsonObject queryStats = renderQueryStatisticsJson(statistics.getQueryStatistics(), verbose); + fiJson.add("queryStatistics", queryStats); + + // Operators + PlanNode planNodeTree = instance.getFragment().getPlanNodeTree(); + JsonObject operatorTree = + renderOperatorJson(planNodeTree, statistics.getOperatorStatisticsMap()); + if (operatorTree != null) { + fiJson.add("operators", operatorTree); + } + + fragmentInstancesArray.add(fiJson); + } + + root.add("fragmentInstances", fragmentInstancesArray); + return GSON.toJson(root); + } + + private JsonObject renderQueryStatisticsJson(TQueryStatistics qs, boolean verbose) { + JsonObject stats = new JsonObject(); + + if (verbose) { + stats.addProperty("loadBloomFilterFromCacheCount", qs.loadBloomFilterFromCacheCount); + stats.addProperty("loadBloomFilterFromDiskCount", qs.loadBloomFilterFromDiskCount); + stats.addProperty("loadBloomFilterActualIOSize", qs.loadBloomFilterActualIOSize); + stats.addProperty( + "loadBloomFilterTimeMs", formatMs(qs.loadBloomFilterTime * NS_TO_MS_FACTOR)); + + addIfNonZero( + stats, "loadTimeSeriesMetadataDiskSeqCount", qs.loadTimeSeriesMetadataDiskSeqCount); + addIfNonZero( + stats, "loadTimeSeriesMetadataDiskUnSeqCount", qs.loadTimeSeriesMetadataDiskUnSeqCount); + addIfNonZero( + stats, "loadTimeSeriesMetadataMemSeqCount", qs.loadTimeSeriesMetadataMemSeqCount); + addIfNonZero( + stats, "loadTimeSeriesMetadataMemUnSeqCount", qs.loadTimeSeriesMetadataMemUnSeqCount); + addIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedDiskSeqCount", + qs.loadTimeSeriesMetadataAlignedDiskSeqCount); + addIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedDiskUnSeqCount", + qs.loadTimeSeriesMetadataAlignedDiskUnSeqCount); + addIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedMemSeqCount", + qs.loadTimeSeriesMetadataAlignedMemSeqCount); + addIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedMemUnSeqCount", + qs.loadTimeSeriesMetadataAlignedMemUnSeqCount); + + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataDiskSeqTimeMs", + qs.loadTimeSeriesMetadataDiskSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataDiskUnSeqTimeMs", + qs.loadTimeSeriesMetadataDiskUnSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataMemSeqTimeMs", + qs.loadTimeSeriesMetadataMemSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataMemUnSeqTimeMs", + qs.loadTimeSeriesMetadataMemUnSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedDiskSeqTimeMs", + qs.loadTimeSeriesMetadataAlignedDiskSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedDiskUnSeqTimeMs", + qs.loadTimeSeriesMetadataAlignedDiskUnSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedMemSeqTimeMs", + qs.loadTimeSeriesMetadataAlignedMemSeqTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "loadTimeSeriesMetadataAlignedMemUnSeqTimeMs", + qs.loadTimeSeriesMetadataAlignedMemUnSeqTime * NS_TO_MS_FACTOR); + + stats.addProperty( + "loadTimeSeriesMetadataFromCacheCount", qs.loadTimeSeriesMetadataFromCacheCount); + stats.addProperty( + "loadTimeSeriesMetadataFromDiskCount", qs.loadTimeSeriesMetadataFromDiskCount); + stats.addProperty( + "loadTimeSeriesMetadataActualIOSize", qs.loadTimeSeriesMetadataActualIOSize); + + addIfNonZero( + stats, + "alignedTimeSeriesMetadataModificationCount", + qs.getAlignedTimeSeriesMetadataModificationCount()); + addMsIfNonZero( + stats, + "alignedTimeSeriesMetadataModificationTimeMs", + qs.getAlignedTimeSeriesMetadataModificationTime() * NS_TO_MS_FACTOR); + addIfNonZero( + stats, + "nonAlignedTimeSeriesMetadataModificationCount", + qs.getNonAlignedTimeSeriesMetadataModificationCount()); + addMsIfNonZero( + stats, + "nonAlignedTimeSeriesMetadataModificationTimeMs", + qs.getNonAlignedTimeSeriesMetadataModificationTime() * NS_TO_MS_FACTOR); + + addIfNonZero( + stats, + "constructNonAlignedChunkReadersDiskCount", + qs.constructNonAlignedChunkReadersDiskCount); + addIfNonZero( + stats, + "constructNonAlignedChunkReadersMemCount", + qs.constructNonAlignedChunkReadersMemCount); + addIfNonZero( + stats, "constructAlignedChunkReadersDiskCount", qs.constructAlignedChunkReadersDiskCount); + addIfNonZero( + stats, "constructAlignedChunkReadersMemCount", qs.constructAlignedChunkReadersMemCount); + addMsIfNonZero( + stats, + "constructNonAlignedChunkReadersDiskTimeMs", + qs.constructNonAlignedChunkReadersDiskTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "constructNonAlignedChunkReadersMemTimeMs", + qs.constructNonAlignedChunkReadersMemTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "constructAlignedChunkReadersDiskTimeMs", + qs.constructAlignedChunkReadersDiskTime * NS_TO_MS_FACTOR); + addMsIfNonZero( + stats, + "constructAlignedChunkReadersMemTimeMs", + qs.constructAlignedChunkReadersMemTime * NS_TO_MS_FACTOR); + + stats.addProperty("loadChunkFromCacheCount", qs.loadChunkFromCacheCount); + stats.addProperty("loadChunkFromDiskCount", qs.loadChunkFromDiskCount); + stats.addProperty("loadChunkActualIOSize", qs.loadChunkActualIOSize); + + addIfNonZero( + stats, "pageReadersDecodeAlignedDiskCount", qs.pageReadersDecodeAlignedDiskCount); + addMsIfNonZero( + stats, + "pageReadersDecodeAlignedDiskTimeMs", + qs.pageReadersDecodeAlignedDiskTime * NS_TO_MS_FACTOR); + addIfNonZero(stats, "pageReadersDecodeAlignedMemCount", qs.pageReadersDecodeAlignedMemCount); + addMsIfNonZero( + stats, + "pageReadersDecodeAlignedMemTimeMs", + qs.pageReadersDecodeAlignedMemTime * NS_TO_MS_FACTOR); + addIfNonZero( + stats, "pageReadersDecodeNonAlignedDiskCount", qs.pageReadersDecodeNonAlignedDiskCount); + addMsIfNonZero( + stats, + "pageReadersDecodeNonAlignedDiskTimeMs", + qs.pageReadersDecodeNonAlignedDiskTime * NS_TO_MS_FACTOR); + addIfNonZero( + stats, "pageReadersDecodeNonAlignedMemCount", qs.pageReadersDecodeNonAlignedMemCount); + addMsIfNonZero( + stats, + "pageReadersDecodeNonAlignedMemTimeMs", + qs.pageReadersDecodeNonAlignedMemTime * NS_TO_MS_FACTOR); + addIfNonZero(stats, "pageReaderMaxUsedMemorySize", qs.pageReaderMaxUsedMemorySize); + addIfNonZero(stats, "chunkWithMetadataErrorsCount", qs.chunkWithMetadataErrorsCount); + } + + stats.addProperty("timeSeriesIndexFilteredRows", qs.timeSeriesIndexFilteredRows); + stats.addProperty("chunkIndexFilteredRows", qs.chunkIndexFilteredRows); + stats.addProperty("pageIndexFilteredRows", qs.pageIndexFilteredRows); + + if (verbose) { + stats.addProperty("rowScanFilteredRows", qs.rowScanFilteredRows); + } + + return stats; + } + + private JsonObject renderOperatorJson( + PlanNode planNodeTree, Map operatorStatistics) { + if (planNodeTree == null) { + return null; + } + + JsonObject operatorJson = new JsonObject(); + TOperatorStatistics opStats = operatorStatistics.get(planNodeTree.getPlanNodeId().toString()); + + operatorJson.addProperty("planNodeId", planNodeTree.getPlanNodeId().toString()); + operatorJson.addProperty("nodeType", planNodeTree.getClass().getSimpleName()); + + if (opStats != null) { + operatorJson.addProperty("operatorType", opStats.getOperatorType()); + if (opStats.isSetCount()) { + operatorJson.addProperty("count", opStats.getCount()); + } + operatorJson.addProperty( + "cpuTimeMs", formatMs(opStats.getTotalExecutionTimeInNanos() * NS_TO_MS_FACTOR)); + operatorJson.addProperty("outputRows", opStats.getOutputRows()); + operatorJson.addProperty("hasNextCalledCount", opStats.hasNextCalledCount); + operatorJson.addProperty("nextCalledCount", opStats.nextCalledCount); + if (opStats.getMemoryUsage() != 0) { + operatorJson.addProperty("estimatedMemorySize", opStats.getMemoryUsage()); + } + + if (opStats.getSpecifiedInfoSize() != 0) { + JsonObject specifiedInfo = new JsonObject(); + for (Map.Entry entry : opStats.getSpecifiedInfo().entrySet()) { + specifiedInfo.addProperty(entry.getKey(), entry.getValue()); + } + operatorJson.add("specifiedInfo", specifiedInfo); + } + } + + List children = planNodeTree.getChildren(); + if (children != null && !children.isEmpty()) { + JsonArray childrenArray = new JsonArray(); + for (PlanNode child : children) { + JsonObject childJson = renderOperatorJson(child, operatorStatistics); + if (childJson != null) { + childrenArray.add(childJson); + } + } + if (childrenArray.size() > 0) { + operatorJson.add("children", childrenArray); + } + } + + return operatorJson; + } + + private static double formatMs(double ms) { + return Math.round(ms * 1000.0) / 1000.0; + } + + private static void addIfNonZero(JsonObject obj, String key, long value) { + if (value != 0) { + obj.addProperty(key, value); + } + } + + private static void addMsIfNonZero(JsonObject obj, String key, double value) { + if (Math.abs(value) > EPSILON) { + obj.addProperty(key, formatMs(value)); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/PlanGraphJsonPrinterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/PlanGraphJsonPrinterTest.java new file mode 100644 index 0000000000000..a600957b25fa3 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/PlanGraphJsonPrinterTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.node; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanGraphJsonPrinter; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class PlanGraphJsonPrinterTest { + + @Test + public void testSimplePlanToJson() { + // Build a simple plan: Output -> Limit -> Filter + LimitNode placeholder = new LimitNode(new PlanNodeId("placeholder"), null, 0, Optional.empty()); + FilterNode filterNode = + new FilterNode( + new PlanNodeId("3"), + placeholder, + new ComparisonExpression( + ComparisonExpression.Operator.EQUAL, + new SymbolReference("s1"), + new GenericLiteral("INT32", "1"))); + + LimitNode limitNode = new LimitNode(new PlanNodeId("2"), filterNode, 10, Optional.empty()); + + List outputSymbols = Arrays.asList(new Symbol("s1"), new Symbol("s2")); + OutputNode outputNode = + new OutputNode(new PlanNodeId("1"), limitNode, Arrays.asList("s1", "s2"), outputSymbols); + + String json = PlanGraphJsonPrinter.toPrettyJson(outputNode); + + assertNotNull(json); + assertTrue(json.length() > 0); + + // Parse the JSON and verify structure + JsonObject root = JsonParser.parseString(json).getAsJsonObject(); + assertEquals("OutputNode-1", root.get("name").getAsString()); + assertEquals("1", root.get("id").getAsString()); + assertTrue(root.has("properties")); + assertTrue(root.has("children")); + + // Verify child (LimitNode) + JsonObject limitJson = root.getAsJsonArray("children").get(0).getAsJsonObject(); + assertEquals("LimitNode-2", limitJson.get("name").getAsString()); + + // Verify grandchild (FilterNode) + JsonObject filterJson = limitJson.getAsJsonArray("children").get(0).getAsJsonObject(); + assertEquals("FilterNode-3", filterJson.get("name").getAsString()); + } + + @Test + public void testGetJsonLinesReturnsSingleElement() { + LimitNode limitNode = + new LimitNode( + new PlanNodeId("1"), + new LimitNode(new PlanNodeId("placeholder"), null, 0, Optional.empty()), + 5, + Optional.empty()); + + List lines = PlanGraphJsonPrinter.getJsonLines(limitNode); + assertEquals(1, lines.size()); + + // The single element should be valid JSON + JsonObject root = JsonParser.parseString(lines.get(0)).getAsJsonObject(); + assertNotNull(root.get("name")); + } + + @Test + public void testJsonIsValidFormat() { + LimitNode limitNode = + new LimitNode( + new PlanNodeId("1"), + new LimitNode(new PlanNodeId("placeholder"), null, 0, Optional.empty()), + 42, + Optional.empty()); + + String json = PlanGraphJsonPrinter.toPrettyJson(limitNode); + + // Should parse without errors + JsonObject root = JsonParser.parseString(json).getAsJsonObject(); + assertEquals("LimitNode-1", root.get("name").getAsString()); + assertTrue(root.has("properties")); + assertEquals("42", root.getAsJsonObject("properties").get("Count").getAsString()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ExplainFormatTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ExplainFormatTest.java new file mode 100644 index 0000000000000..56e204b38df9d --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ExplainFormatTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql; + +import org.apache.iotdb.db.protocol.session.IClientSession; +import org.apache.iotdb.db.protocol.session.InternalClientSession; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; +import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; + +import org.junit.Before; +import org.junit.Test; + +import java.time.ZoneId; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ExplainFormatTest { + + private SqlParser sqlParser; + private IClientSession clientSession; + + @Before + public void setUp() { + sqlParser = new SqlParser(); + clientSession = new InternalClientSession("testClient"); + clientSession.setDatabaseName("testdb"); + } + + private Statement parseSQL(String sql) { + return sqlParser.createStatement(sql, ZoneId.systemDefault(), clientSession); + } + + @Test + public void testExplainDefaultFormat() { + Statement stmt = parseSQL("EXPLAIN SELECT * FROM table1"); + assertTrue(stmt instanceof Explain); + assertEquals(ExplainOutputFormat.GRAPHVIZ, ((Explain) stmt).getOutputFormat()); + } + + @Test + public void testExplainGraphvizFormat() { + Statement stmt = parseSQL("EXPLAIN (FORMAT GRAPHVIZ) SELECT * FROM table1"); + assertTrue(stmt instanceof Explain); + assertEquals(ExplainOutputFormat.GRAPHVIZ, ((Explain) stmt).getOutputFormat()); + } + + @Test + public void testExplainJsonFormat() { + Statement stmt = parseSQL("EXPLAIN (FORMAT JSON) SELECT * FROM table1"); + assertTrue(stmt instanceof Explain); + assertEquals(ExplainOutputFormat.JSON, ((Explain) stmt).getOutputFormat()); + } + + @Test + public void testExplainJsonFormatCaseInsensitive() { + Statement stmt = parseSQL("EXPLAIN (FORMAT json) SELECT * FROM table1"); + assertTrue(stmt instanceof Explain); + assertEquals(ExplainOutputFormat.JSON, ((Explain) stmt).getOutputFormat()); + } + + @Test(expected = Exception.class) + public void testExplainInvalidFormat() { + parseSQL("EXPLAIN (FORMAT XML) SELECT * FROM table1"); + } + + @Test + public void testExplainAnalyzeDefaultFormat() { + Statement stmt = parseSQL("EXPLAIN ANALYZE SELECT * FROM table1"); + assertTrue(stmt instanceof ExplainAnalyze); + assertEquals(ExplainOutputFormat.TEXT, ((ExplainAnalyze) stmt).getOutputFormat()); + } + + @Test + public void testExplainAnalyzeTextFormat() { + Statement stmt = parseSQL("EXPLAIN ANALYZE (FORMAT TEXT) SELECT * FROM table1"); + assertTrue(stmt instanceof ExplainAnalyze); + assertEquals(ExplainOutputFormat.TEXT, ((ExplainAnalyze) stmt).getOutputFormat()); + } + + @Test + public void testExplainAnalyzeJsonFormat() { + Statement stmt = parseSQL("EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM table1"); + assertTrue(stmt instanceof ExplainAnalyze); + assertEquals(ExplainOutputFormat.JSON, ((ExplainAnalyze) stmt).getOutputFormat()); + } + + @Test + public void testExplainAnalyzeVerboseJsonFormat() { + Statement stmt = parseSQL("EXPLAIN ANALYZE VERBOSE (FORMAT JSON) SELECT * FROM table1"); + assertTrue(stmt instanceof ExplainAnalyze); + ExplainAnalyze ea = (ExplainAnalyze) stmt; + assertEquals(ExplainOutputFormat.JSON, ea.getOutputFormat()); + assertTrue(ea.isVerbose()); + } + + @Test(expected = Exception.class) + public void testExplainAnalyzeInvalidFormat() { + parseSQL("EXPLAIN ANALYZE (FORMAT GRAPHVIZ) SELECT * FROM table1"); + } + + @Test(expected = Exception.class) + public void testExplainTextFormatInvalid() { + // TEXT is not valid for EXPLAIN (only GRAPHVIZ and JSON) + parseSQL("EXPLAIN (FORMAT TEXT) SELECT * FROM table1"); + } +} diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 9b09d6ebad043..c7f1cf4b9bcfc 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -932,8 +932,8 @@ copyToStatementOption // ------------------------------------------- Query Statement --------------------------------------------------------- queryStatement : query #statementDefault - | EXPLAIN (query | executeStatement | executeImmediateStatement) #explain - | EXPLAIN ANALYZE VERBOSE? (query | executeStatement | executeImmediateStatement) #explainAnalyze + | EXPLAIN ('(' FORMAT identifier ')')? (query | executeStatement | executeImmediateStatement) #explain + | EXPLAIN ANALYZE VERBOSE? ('(' FORMAT identifier ')')? (query | executeStatement | executeImmediateStatement) #explainAnalyze ; query From 196df02de9d6eea8a923d3ed67a3f15872defdb3 Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Thu, 2 Apr 2026 12:14:20 +0800 Subject: [PATCH 02/10] Address review feedback for EXPLAIN FORMAT JSON feature - Fix ExplainAnalyzeOperator to only instantiate the needed drawer (TEXT or JSON), avoiding wasted work - Replace hand-concatenated JSON in mergeExplainResultsJson with Gson to prevent injection from unescaped CTE names - Add proper imports in PlanGraphJsonPrinter, replace FQN with simple class names - Use JsonArray for list-type properties instead of String.valueOf() - Fix ExplainAnalyze.equals()/hashCode() to include outputFormat and verbose - Add Javadoc to ExplainOutputFormat documenting valid formats per statement - Default MPPQueryContext.explainOutputFormat to TEXT instead of null - Mark old 5-arg ExplainAnalyzeOperator constructor @Deprecated - Improve testExplainInvalidFormat to assert on error message content - Add common pitfalls section to CLAUDE.md Co-Authored-By: Claude Opus 4.6 (1M context) --- CLAUDE.md | 137 ++++++++++++++++++ .../query/recent/IoTExplainJsonFormatIT.java | 10 +- .../queryengine/common/MPPQueryContext.java | 2 +- .../operator/ExplainAnalyzeOperator.java | 26 +++- ...ableModelStatementMemorySourceVisitor.java | 43 ++---- .../plan/node/PlanGraphJsonPrinter.java | 116 ++++++++------- .../relational/sql/ast/ExplainAnalyze.java | 6 +- .../sql/ast/ExplainOutputFormat.java | 9 ++ .../FragmentInstanceStatisticsJsonDrawer.java | 1 + 9 files changed, 254 insertions(+), 96 deletions(-) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000000..421c68ad8ba54 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,137 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Apache IoTDB is a time series database for IoT data. It uses a distributed architecture with ConfigNodes (metadata/coordination) and DataNodes (storage/query). Data is stored in TsFile columnar format (separate repo: https://github.com/apache/tsfile). Current version is 2.0.7-SNAPSHOT. + +## Build Commands + +```bash +# Full build (skip tests) +mvn clean package -pl distribution -am -DskipTests + +# Build a specific module (e.g., datanode) +mvn clean package -pl iotdb-core/datanode -am -DskipTests + +# Run unit tests for a specific module +mvn clean test -pl iotdb-core/datanode + +# Run a single test class +mvn clean test -pl iotdb-core/datanode -Dtest=ClassName + +# Run a single test method +mvn clean test -pl iotdb-core/datanode -Dtest=ClassName#methodName + +# Format code (requires JDK 17+; auto-skipped on JDK <17) +mvn spotless:apply + +# Format code in integration-test module +mvn spotless:apply -P with-integration-tests + +# Check formatting without applying +mvn spotless:check +``` + +## Integration Tests + +Integration tests live in `integration-test/` (not included in default build). They require the `with-integration-tests` profile: + +```bash +# Build template-node first (needed once, or after code changes) +mvn clean package -DskipTests -pl integration-test -am -P with-integration-tests + +# Run tree-model ITs (simple: 1 ConfigNode + 1 DataNode) +mvn clean verify -DskipUTs -pl integration-test -am -P with-integration-tests + +# Run tree-model ITs (cluster: 1 ConfigNode + 3 DataNodes) +mvn clean verify -DskipUTs -pl integration-test -am -PClusterIT -P with-integration-tests + +# Run table-model ITs (simple) +mvn clean verify -DskipUTs -pl integration-test -am -PTableSimpleIT -P with-integration-tests + +# Run table-model ITs (cluster) +mvn clean verify -DskipUTs -pl integration-test -am -PTableClusterIT -P with-integration-tests +``` + +To run integration tests from IntelliJ: enable the `with-integration-tests` profile in Maven sidebar, then run test cases directly. + +## Code Style + +- **Spotless** with Google Java Format (GOOGLE style). Import order: `org.apache.iotdb`, blank, `javax`, `java`, static. +- **Checkstyle** is also configured (see `checkstyle.xml` at project root). +- Java source/target level is 1.8 (compiled with `maven.compiler.release=8` on JDK 9+). + +## Architecture + +### Node Types + +- **ConfigNode** (`iotdb-core/confignode`): Manages cluster metadata, schema regions, data regions, partition tables. Coordinates via Ratis consensus. +- **DataNode** (`iotdb-core/datanode`): Handles data storage, query execution, and client connections. The main server component. +- **AINode** (`iotdb-core/ainode`): Python-based node for AI/ML inference tasks. + +### Dual Data Model + +IoTDB supports two data models operating on the same storage: +- **Tree model**: Traditional IoT hierarchy (e.g., `root.ln.wf01.wt01.temperature`). SQL uses path-based addressing. +- **Table model** (relational): SQL table semantics. Grammar lives in `iotdb-core/relational-grammar/`. Query plan code under `queryengine/plan/relational/`. + +### Key DataNode Subsystems (`iotdb-core/datanode`) + +- **queryengine**: SQL parsing, planning, optimization, and execution. + - `plan/parser/` - ANTLR-based SQL parser + - `plan/statement/` - AST statement nodes + - `plan/planner/` - Logical and physical planning (tree model: `TreeModelPlanner`, table model: under `plan/relational/`) + - `plan/optimization/` - Query optimization rules + - `execution/operator/` - Physical operators (volcano-style iterator model) + - `execution/exchange/` - Inter-node data exchange + - `execution/fragment/` - Distributed query fragment management +- **storageengine**: Write path, memtable, flush, WAL, compaction, TsFile management. + - `dataregion/` - DataRegion lifecycle, memtable, flush, compaction + - `dataregion/wal/` - Write-ahead log + - `buffer/` - Memory buffer management +- **schemaengine**: Schema (timeseries metadata) management. +- **pipe**: Data sync/replication framework (source -> processor -> sink pipeline). +- **consensus**: DataNode-side consensus integration. +- **subscription**: Client subscription service for streaming data changes. + +### Consensus (`iotdb-core/consensus`) + +Pluggable consensus protocols: Simple (single-node), Ratis (Raft-based), IoT Consensus (optimized for IoT writes). Factory pattern via `ConsensusFactory`. + +### Protocol Layer (`iotdb-protocol/`) + +Thrift IDL definitions for RPC between nodes. Generated sources are produced automatically during build. Sub-modules: `thrift-commons`, `thrift-confignode`, `thrift-datanode`, `thrift-consensus`, `thrift-ainode`. + +### Client Libraries (`iotdb-client/`) + +- `session/` - Java Session API (primary client interface) +- `jdbc/` - JDBC driver +- `cli/` - Command-line client +- `client-cpp/`, `client-go/`, `client-py/` - Multi-language clients +- `service-rpc/` - Shared Thrift service definitions + +### API Layer (`iotdb-api/`) + +Extension point interfaces: `udf-api` (user-defined functions), `trigger-api` (event triggers), `pipe-api` (data sync plugins), `external-api`, `external-service-api`. + +## IDE Setup + +After `mvn package`, right-click the root project in IntelliJ and choose "Maven -> Reload Project" to add generated source roots (Thrift and ANTLR). + +Generated source directories that need to be on the source path: +- `**/target/generated-sources/thrift` +- `**/target/generated-sources/antlr4` + +## Common Pitfalls + +### Build + +- **Missing Thrift compiler**: The local machine may not have the `thrift` binary installed. Running `mvn clean package -pl -am -DskipTests` will fail at the `iotdb-thrift` module. **Workaround**: To verify your changes compile, use `mvn compile -pl ` (without `-am` or `clean`) to leverage existing target caches. +- **Pre-existing compilation errors in unrelated modules**: The datanode module may have pre-existing compile errors in other subsystems (e.g., pipe, copyto) that cause `mvn clean test -pl iotdb-core/datanode -Dtest=XxxTest` to fail during compilation. **Workaround**: First run `mvn compile -pl iotdb-core/datanode` to confirm your changed files compile successfully. If the errors are in files you did not modify, they are pre-existing and do not affect your changes. + +### Code Style + +- **Always run `mvn spotless:apply` after editing Java files**: Spotless runs `spotless:check` automatically during the `compile` phase. Format violations cause an immediate BUILD FAILURE. Make it a habit to run `mvn spotless:apply -pl ` right after editing, not at the end. For files under `integration-test/`, add `-P with-integration-tests`. +- **Gson version compatibility**: `JsonObject.isEmpty()` / `JsonArray.isEmpty()` may not be available in the Gson version used by this project. Use `size() > 0` instead and add a comment explaining why. diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java index 94ed692ab6410..e023d6ce38653 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java @@ -239,13 +239,19 @@ public void testExplainAnalyzeTextFormatExplicit() { } } - @Test(expected = SQLException.class) - public void testExplainInvalidFormat() throws SQLException { + @Test + public void testExplainInvalidFormat() { String sql = "EXPLAIN (FORMAT XML) SELECT * FROM testtb"; try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = conn.createStatement()) { statement.execute("USE " + DATABASE_NAME); statement.executeQuery(sql); + fail("Expected SQLException for invalid format"); + } catch (SQLException e) { + Assert.assertTrue( + "Error message should mention the invalid format", + e.getMessage().toUpperCase().contains("FORMAT") + || e.getMessage().toUpperCase().contains("XML")); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index 43b214f226cc3..8f248c2936361 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -109,7 +109,7 @@ public enum ExplainType { // - EXPLAIN: Show the logical and physical query plan without execution // - EXPLAIN_ANALYZE: Execute the query and collect detailed execution statistics private ExplainType explainType = ExplainType.NONE; - private ExplainOutputFormat explainOutputFormat = null; + private ExplainOutputFormat explainOutputFormat = ExplainOutputFormat.TEXT; private boolean verbose = false; private QueryPlanStatistics queryPlanStatistics = null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java index 52af0bb745583..9676c94844a8a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java @@ -82,15 +82,14 @@ public class ExplainAnalyzeOperator implements ProcessOperator { private final List instances; private final ExplainOutputFormat outputFormat; - private final FragmentInstanceStatisticsDrawer fragmentInstanceStatisticsDrawer = - new FragmentInstanceStatisticsDrawer(); - private final FragmentInstanceStatisticsJsonDrawer fragmentInstanceStatisticsJsonDrawer = - new FragmentInstanceStatisticsJsonDrawer(); + private final FragmentInstanceStatisticsDrawer fragmentInstanceStatisticsDrawer; + private final FragmentInstanceStatisticsJsonDrawer fragmentInstanceStatisticsJsonDrawer; private final ScheduledFuture logRecordTask; private final IClientManager clientManager; private final MPPQueryContext mppQueryContext; + @Deprecated public ExplainAnalyzeOperator( OperatorContext operatorContext, Operator child, @@ -118,8 +117,16 @@ public ExplainAnalyzeOperator( QueryExecution queryExecution = (QueryExecution) coordinator.getQueryExecution(queryId); this.instances = queryExecution.getDistributedPlan().getInstances(); mppQueryContext = queryExecution.getContext(); - fragmentInstanceStatisticsDrawer.renderPlanStatistics(mppQueryContext); - fragmentInstanceStatisticsJsonDrawer.renderPlanStatistics(mppQueryContext); + + if (outputFormat == ExplainOutputFormat.JSON) { + this.fragmentInstanceStatisticsDrawer = null; + this.fragmentInstanceStatisticsJsonDrawer = new FragmentInstanceStatisticsJsonDrawer(); + fragmentInstanceStatisticsJsonDrawer.renderPlanStatistics(mppQueryContext); + } else { + this.fragmentInstanceStatisticsDrawer = new FragmentInstanceStatisticsDrawer(); + this.fragmentInstanceStatisticsJsonDrawer = null; + fragmentInstanceStatisticsDrawer.renderPlanStatistics(mppQueryContext); + } // The time interval guarantees the result of EXPLAIN ANALYZE will be printed at least three // times. @@ -146,8 +153,11 @@ public TsBlock next() throws Exception { return null; } - fragmentInstanceStatisticsDrawer.renderDispatchCost(mppQueryContext); - fragmentInstanceStatisticsJsonDrawer.renderDispatchCost(mppQueryContext); + if (outputFormat == ExplainOutputFormat.JSON) { + fragmentInstanceStatisticsJsonDrawer.renderDispatchCost(mppQueryContext); + } else { + fragmentInstanceStatisticsDrawer.renderDispatchCost(mppQueryContext); + } // fetch statics from all fragment instances TsBlock result = buildResult(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java index c6a7692395a37..2633805074d07 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java @@ -41,6 +41,11 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.utils.Pair; @@ -60,6 +65,8 @@ public class TableModelStatementMemorySourceVisitor extends AstVisitor { + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + @Override public StatementMemorySource visitNode( final Node node, final TableModelStatementMemorySourceContext context) { @@ -170,36 +177,18 @@ private List mergeExplainResultsJson( return mainExplainResult; } - // For JSON format with CTEs, wrap everything in a combined JSON object - StringBuilder sb = new StringBuilder(); - sb.append("{\n"); - sb.append(" \"cteQueries\": [\n"); - int cteIndex = 0; - int cteSize = cteExplainResults.size(); + JsonObject wrapper = new JsonObject(); + JsonArray cteArray = new JsonArray(); for (Map.Entry, Pair>> entry : cteExplainResults.entrySet()) { - sb.append(" {\n"); - sb.append(" \"name\": \"").append(entry.getKey().getNode().getName()).append("\",\n"); - sb.append(" \"plan\": "); - // Each CTE's plan is already a JSON string - for (String line : entry.getValue().getRight()) { - sb.append(line); - } - sb.append("\n }"); - if (++cteIndex < cteSize) { - sb.append(","); - } - sb.append("\n"); - } - sb.append(" ],\n"); - sb.append(" \"mainQuery\": "); - for (String line : mainExplainResult) { - sb.append(line); + JsonObject cte = new JsonObject(); + cte.addProperty("name", entry.getKey().getNode().getName()); + cte.add("plan", JsonParser.parseString(entry.getValue().getRight().get(0))); + cteArray.add(cte); } - sb.append("\n}"); + wrapper.add("cteQueries", cteArray); + wrapper.add("mainQuery", JsonParser.parseString(mainExplainResult.get(0))); - List result = new ArrayList<>(); - result.add(sb.toString()); - return result; + return Collections.singletonList(GSON.toJson(wrapper)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java index e35c23d466059..97e7d88328f58 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphJsonPrinter.java @@ -19,12 +19,21 @@ package org.apache.iotdb.db.queryengine.plan.planner.plan.node; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExplainAnalyzeNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.UnionNode; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -59,6 +68,7 @@ private static JsonObject buildJsonNode(PlanNode node) { jsonNode.addProperty("id", nodeId); JsonObject properties = buildProperties(node); + // JsonObject.isEmpty() is not available in all Gson versions if (properties.size() > 0) { jsonNode.add("properties", properties); } @@ -80,71 +90,68 @@ private static JsonObject buildProperties(PlanNode node) { if (node instanceof OutputNode) { OutputNode n = (OutputNode) node; - properties.addProperty("OutputColumns", String.valueOf(n.getOutputColumnNames())); - properties.addProperty("OutputSymbols", String.valueOf(n.getOutputSymbols())); + properties.add("OutputColumns", toJsonArray(n.getOutputColumnNames())); + properties.add("OutputSymbols", toJsonArray(n.getOutputSymbols())); } else if (node instanceof ExplainAnalyzeNode) { ExplainAnalyzeNode n = (ExplainAnalyzeNode) node; - properties.addProperty("ChildPermittedOutputs", String.valueOf(n.getChildPermittedOutputs())); + properties.add("ChildPermittedOutputs", toJsonArray(n.getChildPermittedOutputs())); } else if (node instanceof TableScanNode) { buildTableScanProperties(properties, (TableScanNode) node); } else if (node instanceof ExchangeNode) { // No extra properties needed - } else if (node - instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode) { - buildAggregationProperties( - properties, - (org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode) node); - } else if (node - instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode) { - org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode n = - (org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode) node; + } else if (node instanceof AggregationNode) { + buildAggregationProperties(properties, (AggregationNode) node); + } else if (node instanceof FilterNode) { + FilterNode n = (FilterNode) node; properties.addProperty("Predicate", String.valueOf(n.getPredicate())); - } else if (node - instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode) { - org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode n = - (org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode) node; - properties.addProperty("OutputSymbols", String.valueOf(n.getOutputSymbols())); - properties.addProperty("Expressions", String.valueOf(n.getAssignments().getMap().values())); - } else if (node - instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode) { - org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode n = - (org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode) node; - properties.addProperty("Count", String.valueOf(n.getCount())); - } else if (node - instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode) { - org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode n = - (org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode) node; - properties.addProperty("Count", String.valueOf(n.getCount())); - } else if (node - instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode) { - org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode n = - (org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode) node; + } else if (node instanceof ProjectNode) { + ProjectNode n = (ProjectNode) node; + properties.add("OutputSymbols", toJsonArray(n.getOutputSymbols())); + properties.add("Expressions", toJsonArray(n.getAssignments().getMap().values())); + } else if (node instanceof LimitNode) { + LimitNode n = (LimitNode) node; + properties.addProperty("Count", n.getCount()); + } else if (node instanceof OffsetNode) { + OffsetNode n = (OffsetNode) node; + properties.addProperty("Count", n.getCount()); + } else if (node instanceof SortNode) { + SortNode n = (SortNode) node; properties.addProperty("OrderBy", String.valueOf(n.getOrderingScheme())); - } else if (node - instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode) { - org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode n = - (org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode) node; + } else if (node instanceof MergeSortNode) { + MergeSortNode n = (MergeSortNode) node; properties.addProperty("OrderBy", String.valueOf(n.getOrderingScheme())); - } else if (node - instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode) { - org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode n = - (org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode) node; + } else if (node instanceof JoinNode) { + JoinNode n = (JoinNode) node; properties.addProperty("JoinType", String.valueOf(n.getJoinType())); - properties.addProperty("Criteria", String.valueOf(n.getCriteria())); - properties.addProperty("OutputSymbols", String.valueOf(n.getOutputSymbols())); - } else if (node - instanceof org.apache.iotdb.db.queryengine.plan.relational.planner.node.UnionNode) { - org.apache.iotdb.db.queryengine.plan.relational.planner.node.UnionNode n = - (org.apache.iotdb.db.queryengine.plan.relational.planner.node.UnionNode) node; - properties.addProperty("OutputSymbols", String.valueOf(n.getOutputSymbols())); + properties.add("Criteria", toJsonArray(n.getCriteria())); + properties.add("OutputSymbols", toJsonArray(n.getOutputSymbols())); + } else if (node instanceof UnionNode) { + UnionNode n = (UnionNode) node; + properties.add("OutputSymbols", toJsonArray(n.getOutputSymbols())); } return properties; } + private static JsonArray toJsonArray(java.util.Collection items) { + JsonArray array = new JsonArray(); + for (T item : items) { + array.add(String.valueOf(item)); + } + return array; + } + + private static JsonArray toJsonArray(List items) { + JsonArray array = new JsonArray(); + for (T item : items) { + array.add(String.valueOf(item)); + } + return array; + } + private static void buildTableScanProperties(JsonObject properties, TableScanNode node) { properties.addProperty("QualifiedTableName", node.getQualifiedObjectName().toString()); - properties.addProperty("OutputSymbols", String.valueOf(node.getOutputSymbols())); + properties.add("OutputSymbols", toJsonArray(node.getOutputSymbols())); if (node instanceof DeviceTableScanNode) { DeviceTableScanNode deviceNode = (DeviceTableScanNode) node; @@ -182,15 +189,12 @@ private static void buildTableScanProperties(JsonObject properties, TableScanNod } } - private static void buildAggregationProperties( - JsonObject properties, - org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode node) { - properties.addProperty("OutputSymbols", String.valueOf(node.getOutputSymbols())); + private static void buildAggregationProperties(JsonObject properties, AggregationNode node) { + properties.add("OutputSymbols", toJsonArray(node.getOutputSymbols())); JsonArray aggregators = new JsonArray(); int i = 0; - for (org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Aggregation - aggregation : node.getAggregations().values()) { + for (AggregationNode.Aggregation aggregation : node.getAggregations().values()) { JsonObject agg = new JsonObject(); agg.addProperty("index", i++); agg.addProperty("function", aggregation.getResolvedFunction().toString()); @@ -204,10 +208,10 @@ private static void buildAggregationProperties( } properties.add("Aggregators", aggregators); - properties.addProperty("GroupingKeys", String.valueOf(node.getGroupingKeys())); + properties.add("GroupingKeys", toJsonArray(node.getGroupingKeys())); if (node.isStreamable()) { properties.addProperty("Streamable", true); - properties.addProperty("PreGroupedSymbols", String.valueOf(node.getPreGroupedSymbols())); + properties.add("PreGroupedSymbols", toJsonArray(node.getPreGroupedSymbols())); } properties.addProperty("Step", String.valueOf(node.getStep())); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java index dc7798967e4d8..b824705df1839 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainAnalyze.java @@ -86,7 +86,7 @@ public List getChildren() { @Override public int hashCode() { - return Objects.hash(statement, verbose); + return Objects.hash(statement, verbose, outputFormat); } @Override @@ -98,7 +98,9 @@ public boolean equals(Object obj) { return false; } ExplainAnalyze o = (ExplainAnalyze) obj; - return Objects.equals(statement, o.statement); + return Objects.equals(statement, o.statement) + && verbose == o.verbose + && outputFormat == o.outputFormat; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java index 9dba3649a7f7f..d7d1fc08bb395 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExplainOutputFormat.java @@ -19,6 +19,15 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; +/** + * Output format for EXPLAIN and EXPLAIN ANALYZE statements. + * + *
    + *
  • {@link #GRAPHVIZ} - Box-drawing plan visualization. Valid for EXPLAIN only (default). + *
  • {@link #TEXT} - Text-based output. Valid for EXPLAIN ANALYZE only (default). + *
  • {@link #JSON} - Structured JSON output. Valid for both EXPLAIN and EXPLAIN ANALYZE. + *
+ */ public enum ExplainOutputFormat { GRAPHVIZ, TEXT, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java index 3c6ff7bfb41b2..2b00d6191fe78 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java @@ -345,6 +345,7 @@ private JsonObject renderOperatorJson( childrenArray.add(childJson); } } + // JsonArray.isEmpty() is not available in all Gson versions if (childrenArray.size() > 0) { operatorJson.add("children", childrenArray); } From 936694831fed04e680196b28caf4a7b5b1f56a5d Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Thu, 2 Apr 2026 12:20:33 +0800 Subject: [PATCH 03/10] Remove CLAUDE.md from tracking and add it to .gitignore Co-Authored-By: Claude Opus 4.6 (1M context) --- .gitignore | 4 ++ CLAUDE.md | 137 ----------------------------------------------------- 2 files changed, 4 insertions(+), 137 deletions(-) delete mode 100644 CLAUDE.md diff --git a/.gitignore b/.gitignore index 2c19b1b3a2cd5..8698059eb1fb9 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,7 @@ iotdb-core/tsfile/src/main/antlr4/org/apache/tsfile/parser/gen/ # Relational Grammar ANTLR iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/.antlr/ + +# Claude Code +CLAUDE.md +.claude/ diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 421c68ad8ba54..0000000000000 --- a/CLAUDE.md +++ /dev/null @@ -1,137 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Project Overview - -Apache IoTDB is a time series database for IoT data. It uses a distributed architecture with ConfigNodes (metadata/coordination) and DataNodes (storage/query). Data is stored in TsFile columnar format (separate repo: https://github.com/apache/tsfile). Current version is 2.0.7-SNAPSHOT. - -## Build Commands - -```bash -# Full build (skip tests) -mvn clean package -pl distribution -am -DskipTests - -# Build a specific module (e.g., datanode) -mvn clean package -pl iotdb-core/datanode -am -DskipTests - -# Run unit tests for a specific module -mvn clean test -pl iotdb-core/datanode - -# Run a single test class -mvn clean test -pl iotdb-core/datanode -Dtest=ClassName - -# Run a single test method -mvn clean test -pl iotdb-core/datanode -Dtest=ClassName#methodName - -# Format code (requires JDK 17+; auto-skipped on JDK <17) -mvn spotless:apply - -# Format code in integration-test module -mvn spotless:apply -P with-integration-tests - -# Check formatting without applying -mvn spotless:check -``` - -## Integration Tests - -Integration tests live in `integration-test/` (not included in default build). They require the `with-integration-tests` profile: - -```bash -# Build template-node first (needed once, or after code changes) -mvn clean package -DskipTests -pl integration-test -am -P with-integration-tests - -# Run tree-model ITs (simple: 1 ConfigNode + 1 DataNode) -mvn clean verify -DskipUTs -pl integration-test -am -P with-integration-tests - -# Run tree-model ITs (cluster: 1 ConfigNode + 3 DataNodes) -mvn clean verify -DskipUTs -pl integration-test -am -PClusterIT -P with-integration-tests - -# Run table-model ITs (simple) -mvn clean verify -DskipUTs -pl integration-test -am -PTableSimpleIT -P with-integration-tests - -# Run table-model ITs (cluster) -mvn clean verify -DskipUTs -pl integration-test -am -PTableClusterIT -P with-integration-tests -``` - -To run integration tests from IntelliJ: enable the `with-integration-tests` profile in Maven sidebar, then run test cases directly. - -## Code Style - -- **Spotless** with Google Java Format (GOOGLE style). Import order: `org.apache.iotdb`, blank, `javax`, `java`, static. -- **Checkstyle** is also configured (see `checkstyle.xml` at project root). -- Java source/target level is 1.8 (compiled with `maven.compiler.release=8` on JDK 9+). - -## Architecture - -### Node Types - -- **ConfigNode** (`iotdb-core/confignode`): Manages cluster metadata, schema regions, data regions, partition tables. Coordinates via Ratis consensus. -- **DataNode** (`iotdb-core/datanode`): Handles data storage, query execution, and client connections. The main server component. -- **AINode** (`iotdb-core/ainode`): Python-based node for AI/ML inference tasks. - -### Dual Data Model - -IoTDB supports two data models operating on the same storage: -- **Tree model**: Traditional IoT hierarchy (e.g., `root.ln.wf01.wt01.temperature`). SQL uses path-based addressing. -- **Table model** (relational): SQL table semantics. Grammar lives in `iotdb-core/relational-grammar/`. Query plan code under `queryengine/plan/relational/`. - -### Key DataNode Subsystems (`iotdb-core/datanode`) - -- **queryengine**: SQL parsing, planning, optimization, and execution. - - `plan/parser/` - ANTLR-based SQL parser - - `plan/statement/` - AST statement nodes - - `plan/planner/` - Logical and physical planning (tree model: `TreeModelPlanner`, table model: under `plan/relational/`) - - `plan/optimization/` - Query optimization rules - - `execution/operator/` - Physical operators (volcano-style iterator model) - - `execution/exchange/` - Inter-node data exchange - - `execution/fragment/` - Distributed query fragment management -- **storageengine**: Write path, memtable, flush, WAL, compaction, TsFile management. - - `dataregion/` - DataRegion lifecycle, memtable, flush, compaction - - `dataregion/wal/` - Write-ahead log - - `buffer/` - Memory buffer management -- **schemaengine**: Schema (timeseries metadata) management. -- **pipe**: Data sync/replication framework (source -> processor -> sink pipeline). -- **consensus**: DataNode-side consensus integration. -- **subscription**: Client subscription service for streaming data changes. - -### Consensus (`iotdb-core/consensus`) - -Pluggable consensus protocols: Simple (single-node), Ratis (Raft-based), IoT Consensus (optimized for IoT writes). Factory pattern via `ConsensusFactory`. - -### Protocol Layer (`iotdb-protocol/`) - -Thrift IDL definitions for RPC between nodes. Generated sources are produced automatically during build. Sub-modules: `thrift-commons`, `thrift-confignode`, `thrift-datanode`, `thrift-consensus`, `thrift-ainode`. - -### Client Libraries (`iotdb-client/`) - -- `session/` - Java Session API (primary client interface) -- `jdbc/` - JDBC driver -- `cli/` - Command-line client -- `client-cpp/`, `client-go/`, `client-py/` - Multi-language clients -- `service-rpc/` - Shared Thrift service definitions - -### API Layer (`iotdb-api/`) - -Extension point interfaces: `udf-api` (user-defined functions), `trigger-api` (event triggers), `pipe-api` (data sync plugins), `external-api`, `external-service-api`. - -## IDE Setup - -After `mvn package`, right-click the root project in IntelliJ and choose "Maven -> Reload Project" to add generated source roots (Thrift and ANTLR). - -Generated source directories that need to be on the source path: -- `**/target/generated-sources/thrift` -- `**/target/generated-sources/antlr4` - -## Common Pitfalls - -### Build - -- **Missing Thrift compiler**: The local machine may not have the `thrift` binary installed. Running `mvn clean package -pl -am -DskipTests` will fail at the `iotdb-thrift` module. **Workaround**: To verify your changes compile, use `mvn compile -pl ` (without `-am` or `clean`) to leverage existing target caches. -- **Pre-existing compilation errors in unrelated modules**: The datanode module may have pre-existing compile errors in other subsystems (e.g., pipe, copyto) that cause `mvn clean test -pl iotdb-core/datanode -Dtest=XxxTest` to fail during compilation. **Workaround**: First run `mvn compile -pl iotdb-core/datanode` to confirm your changed files compile successfully. If the errors are in files you did not modify, they are pre-existing and do not affect your changes. - -### Code Style - -- **Always run `mvn spotless:apply` after editing Java files**: Spotless runs `spotless:check` automatically during the `compile` phase. Format violations cause an immediate BUILD FAILURE. Make it a habit to run `mvn spotless:apply -pl ` right after editing, not at the end. For files under `integration-test/`, add `-P with-integration-tests`. -- **Gson version compatibility**: `JsonObject.isEmpty()` / `JsonArray.isEmpty()` may not be available in the Gson version used by this project. Use `size() > 0` instead and add a comment explaining why. From 14d31578f49aca7c57d14836dff8f03583783843 Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Thu, 2 Apr 2026 16:06:15 +0800 Subject: [PATCH 04/10] add IT --- .../query/recent/IoTExplainJsonFormatIT.java | 142 +++++++++++++++++- ...ableModelStatementMemorySourceVisitor.java | 2 +- 2 files changed, 142 insertions(+), 2 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java index e023d6ce38653..0f3d031312414 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java @@ -26,6 +26,7 @@ import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; import org.apache.iotdb.itbase.env.BaseEnv; +import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import org.junit.AfterClass; @@ -52,7 +53,12 @@ public class IoTExplainJsonFormatIT { public static void setUp() { Locale.setDefault(Locale.ENGLISH); - EnvFactory.getEnv().getConfig().getCommonConfig().setPartitionInterval(1000); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setPartitionInterval(1000) + .setMemtableSizeThreshold(10000) + .setMaxRowsInCteBuffer(100); EnvFactory.getEnv().initClusterEnvironment(); try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -61,9 +67,12 @@ public static void setUp() { statement.execute("USE " + DATABASE_NAME); statement.execute( "CREATE TABLE IF NOT EXISTS testtb(deviceid STRING TAG, voltage FLOAT FIELD)"); + // Insert data across multiple time partitions (partitionInterval=1000) statement.execute("INSERT INTO testtb VALUES(1000, 'd1', 100.0)"); statement.execute("INSERT INTO testtb VALUES(2000, 'd1', 200.0)"); + statement.execute("INSERT INTO testtb VALUES(3000, 'd1', 150.0)"); statement.execute("INSERT INTO testtb VALUES(1000, 'd2', 300.0)"); + statement.execute("INSERT INTO testtb VALUES(2000, 'd2', 250.0)"); } catch (Exception e) { fail(e.getMessage()); } @@ -254,4 +263,135 @@ public void testExplainInvalidFormat() { || e.getMessage().toUpperCase().contains("XML")); } } + + @Test + public void testExplainAnalyzeJsonMultipleFragmentInstances() { + String sql = "EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM testtb"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue(resultSet.next()); + String jsonStr = resultSet.getString(1); + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + + // Verify fragmentInstancesCount matches the size of fragmentInstances array + int declaredCount = root.get("fragmentInstancesCount").getAsInt(); + JsonArray fragmentInstances = root.getAsJsonArray("fragmentInstances"); + Assert.assertNotNull("fragmentInstances array should be present", fragmentInstances); + Assert.assertEquals( + "fragmentInstancesCount should match fragmentInstances array size", + declaredCount, + fragmentInstances.size()); + Assert.assertTrue( + "Should have at least 2 fragment instances for multi-partition data", declaredCount >= 2); + + // Verify each fragment instance has required fields + for (int i = 0; i < fragmentInstances.size(); i++) { + JsonObject fi = fragmentInstances.get(i).getAsJsonObject(); + Assert.assertTrue("Fragment instance should have 'id'", fi.has("id")); + Assert.assertTrue("Fragment instance should have 'state'", fi.has("state")); + Assert.assertTrue("Fragment instance should have 'dataRegion'", fi.has("dataRegion")); + } + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainJsonWithCte() { + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE TABLE IF NOT EXISTS cte_tb(deviceid STRING TAG, voltage FLOAT FIELD)"); + statement.execute("INSERT INTO cte_tb VALUES(1000, 'd1', 50.0)"); + + String sql = + "EXPLAIN (FORMAT JSON) WITH cte1 AS MATERIALIZED (SELECT * FROM cte_tb) " + + "SELECT * FROM testtb WHERE testtb.deviceid IN (SELECT deviceid FROM cte1)"; + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue(resultSet.next()); + String jsonStr = resultSet.getString(1); + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + + // When CTEs are present, the JSON should have cteQueries and mainQuery + Assert.assertTrue("JSON with CTE should have 'cteQueries' field", root.has("cteQueries")); + Assert.assertTrue("JSON with CTE should have 'mainQuery' field", root.has("mainQuery")); + + JsonArray cteQueries = root.getAsJsonArray("cteQueries"); + Assert.assertEquals("Should have exactly 1 CTE query", 1, cteQueries.size()); + + JsonObject cte = cteQueries.get(0).getAsJsonObject(); + Assert.assertTrue("CTE should have 'name' field", cte.has("name")); + Assert.assertEquals("cte1", cte.get("name").getAsString()); + Assert.assertTrue("CTE should have 'plan' field", cte.has("plan")); + + // The main query plan should be a JSON object with 'name' field (plan node) + JsonObject mainQuery = root.getAsJsonObject("mainQuery"); + Assert.assertTrue("Main query plan should have 'name' field", mainQuery.has("name")); + + statement.execute("DROP TABLE IF EXISTS cte_tb"); + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainJsonWithScalarSubquery() { + String sql = + "EXPLAIN (FORMAT JSON) SELECT * FROM testtb " + + "WHERE voltage > (SELECT avg(voltage) FROM testtb)"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue(resultSet.next()); + String jsonStr = resultSet.getString(1); + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + + // Verify it's a valid plan tree with children (subquery creates a more complex plan) + Assert.assertTrue("JSON should have 'name' field", root.has("name")); + Assert.assertTrue("JSON should have 'id' field", root.has("id")); + Assert.assertTrue("Plan with scalar subquery should have 'children'", root.has("children")); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testExplainAnalyzeJsonWithScalarSubquery() { + String sql = + "EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM testtb " + + "WHERE voltage > (SELECT avg(voltage) FROM testtb)"; + try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = conn.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sql); + + Assert.assertTrue(resultSet.next()); + String jsonStr = resultSet.getString(1); + JsonObject root = JsonParser.parseString(jsonStr).getAsJsonObject(); + + Assert.assertTrue(root.has("planStatistics")); + Assert.assertTrue(root.has("fragmentInstances")); + Assert.assertTrue(root.has("fragmentInstancesCount")); + + int declaredCount = root.get("fragmentInstancesCount").getAsInt(); + JsonArray fragmentInstances = root.getAsJsonArray("fragmentInstances"); + Assert.assertEquals(declaredCount, fragmentInstances.size()); + + resultSet.close(); + } catch (SQLException e) { + fail(e.getMessage()); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java index 2633805074d07..35ec7bcb1c4a7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java @@ -182,7 +182,7 @@ private List mergeExplainResultsJson( for (Map.Entry, Pair>> entry : cteExplainResults.entrySet()) { JsonObject cte = new JsonObject(); - cte.addProperty("name", entry.getKey().getNode().getName()); + cte.addProperty("name", entry.getKey().getNode().getName().toString()); cte.add("plan", JsonParser.parseString(entry.getValue().getRight().get(0))); cteArray.add(cte); } From b76978c6dced1f2b724b7b2aa9da2c2907cab0ec Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Thu, 2 Apr 2026 16:23:02 +0800 Subject: [PATCH 05/10] ignore some interp files of antlr in rat check --- iotdb-core/relational-grammar/pom.xml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/iotdb-core/relational-grammar/pom.xml b/iotdb-core/relational-grammar/pom.xml index f4fb18bee2c83..289972ce30fb6 100644 --- a/iotdb-core/relational-grammar/pom.xml +++ b/iotdb-core/relational-grammar/pom.xml @@ -96,8 +96,10 @@ true - **/SqlLexer.java - **/SqlLexer.interp + **/RelationalSqlParser.java + **/RelationalSqlLexer.java + **/RelationalSqlLexer.interp + **/RelationalSql.interp From a586954dde7d49b73c7a499ffb1572e98718cc9a Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Thu, 2 Apr 2026 18:45:06 +0800 Subject: [PATCH 06/10] add comments for expected json format for it --- .../query/recent/IoTExplainJsonFormatIT.java | 293 ++++++++++++++++++ .../UnaliasSymbolReferences.java | 3 +- 2 files changed, 295 insertions(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java index 0f3d031312414..4cbfd02bf5d7d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTExplainJsonFormatIT.java @@ -91,6 +91,54 @@ public static void tearDown() { @Test public void testExplainJsonFormat() { + // Expected output (single row, single JSON object representing the distributed plan tree): + // { + // "name": "OutputNode-", + // "id": "", + // "properties": { + // "OutputColumns": ["time", "deviceid", "voltage"], + // "OutputSymbols": ["time", "deviceid", "voltage"] + // }, + // "children": [ + // { + // "name": "CollectNode-", + // "id": "", + // "children": [ + // { + // "name": "ExchangeNode-", + // "id": "", + // "children": [ + // { + // "name": "DeviceTableScanNode-", + // "id": "", + // "properties": { + // "QualifiedTableName": "testdb_json.testtb", + // "OutputSymbols": ["time", "deviceid", "voltage"], + // "DeviceNumber": "1", + // "ScanOrder": "ASC", + // "PushDownOffset": "0", + // "PushDownLimit": "0", + // "PushDownLimitToEachDevice": "false", + // "RegionId": "" + // } + // } + // ] + // }, + // { + // "name": "ExchangeNode-", + // "id": "", + // "children": [ + // { + // "name": "DeviceTableScanNode-", + // "id": "", + // "properties": { ... } + // } + // ] + // } + // ] + // } + // ] + // } String sql = "EXPLAIN (FORMAT JSON) SELECT * FROM testtb"; try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = conn.createStatement()) { @@ -134,6 +182,76 @@ public void testExplainDefaultFormatIsNotJson() { @Test public void testExplainAnalyzeJsonFormat() { + // Expected output (single row, single JSON object with plan statistics + fragment instances): + // { + // "planStatistics": { + // "analyzeCostMs": , + // "fetchPartitionCostMs": , + // "fetchSchemaCostMs": , + // "logicalPlanCostMs": , + // "logicalOptimizationCostMs": , + // "distributionPlanCostMs": , + // "dispatchCostMs": + // }, + // "fragmentInstancesCount": 3, + // "fragmentInstances": [ + // { + // "id": "..", + // "ip": "127.0.0.1:", + // "dataRegion": "virtual_data_region", + // "state": "FINISHED", + // "totalWallTimeMs": , + // "initDataQuerySourceCostMs": , + // "seqFileUnclosed": 0, "seqFileClosed": 0, + // "unseqFileUnclosed": 0, "unseqFileClosed": 0, + // "readyQueuedTimeMs": , + // "blockQueuedTimeMs": , + // "queryStatistics": { + // "timeSeriesIndexFilteredRows": 0, + // "chunkIndexFilteredRows": 0, + // "pageIndexFilteredRows": 0 + // }, + // "operators": { + // "planNodeId": "", + // "nodeType": "IdentitySinkNode", + // "operatorType": "IdentitySinkOperator", + // "cpuTimeMs": , + // "outputRows": 5, + // "hasNextCalledCount": 5, + // "nextCalledCount": 4, + // "estimatedMemorySize": 1024, + // "specifiedInfo": { "DownStreamPlanNodeId": "" }, + // "children": [ + // { + // "planNodeId": "", + // "nodeType": "CollectNode", + // "operatorType": "CollectOperator", + // ... + // "children": [ + // { "planNodeId": "", "nodeType": "ExchangeNode", ... }, + // { "planNodeId": "", "nodeType": "ExchangeNode", ... } + // ] + // } + // ] + // } + // }, + // { + // "id": "...", "dataRegion": "4", "state": "FINISHED", + // ... + // "operators": { + // "nodeType": "IdentitySinkNode", ... + // "children": [ + // { "nodeType": "DeviceTableScanNode", "operatorType": "TableScanOperator", ... } + // ] + // } + // }, + // { + // "id": "...", "dataRegion": "3", "state": "FINISHED", + // ... + // "operators": { ... } + // } + // ] + // } String sql = "EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM testtb"; try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = conn.createStatement()) { @@ -168,6 +286,47 @@ public void testExplainAnalyzeJsonFormat() { @Test public void testExplainAnalyzeVerboseJsonFormat() { + // Expected output (same structure as testExplainAnalyzeJsonFormat, but queryStatistics + // includes verbose fields like bloom filter, metadata, chunk reader, and page decoder stats): + // { + // "planStatistics": { ... }, + // "fragmentInstancesCount": 3, + // "fragmentInstances": [ + // { + // "id": "...", "dataRegion": "virtual_data_region", ... + // "queryStatistics": { + // "loadBloomFilterFromCacheCount": 0, + // "loadBloomFilterFromDiskCount": 0, + // "loadBloomFilterActualIOSize": 0, + // "loadBloomFilterTimeMs": 0.0, + // "loadTimeSeriesMetadataFromCacheCount": 0, + // "loadTimeSeriesMetadataFromDiskCount": 0, + // "loadTimeSeriesMetadataActualIOSize": 0, + // "loadChunkFromCacheCount": 0, + // "loadChunkFromDiskCount": 0, + // "loadChunkActualIOSize": 0, + // "timeSeriesIndexFilteredRows": 0, + // "chunkIndexFilteredRows": 0, + // "pageIndexFilteredRows": 0, + // "rowScanFilteredRows": 0 + // }, + // "operators": { ... } + // }, + // { + // "id": "...", "dataRegion": "4", ... + // "queryStatistics": { + // ... (same as above, plus non-zero fields like:) + // "loadTimeSeriesMetadataAlignedMemSeqCount": 2, + // "loadTimeSeriesMetadataAlignedMemSeqTimeMs": , + // "pageReadersDecodeAlignedMemCount": 2, + // "pageReadersDecodeAlignedMemTimeMs": , + // ... + // }, + // "operators": { ... } + // }, + // { "id": "...", "dataRegion": "3", ... } + // ] + // } String sql = "EXPLAIN ANALYZE VERBOSE (FORMAT JSON) SELECT * FROM testtb"; try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = conn.createStatement()) { @@ -266,6 +425,17 @@ public void testExplainInvalidFormat() { @Test public void testExplainAnalyzeJsonMultipleFragmentInstances() { + // Expected output (same structure as testExplainAnalyzeJsonFormat): + // { + // "planStatistics": { ... }, + // "fragmentInstancesCount": 3, // >= 2 due to multi-partition data + // "fragmentInstances": [ + // { "id": "...", "dataRegion": "virtual_data_region", "state": "FINISHED", ... }, + // { "id": "...", "dataRegion": "4", "state": "FINISHED", ... }, + // { "id": "...", "dataRegion": "3", "state": "FINISHED", ... } + // ] + // } + // Each fragment instance must have "id", "state", and "dataRegion" fields. String sql = "EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM testtb"; try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = conn.createStatement()) { @@ -303,6 +473,42 @@ public void testExplainAnalyzeJsonMultipleFragmentInstances() { @Test public void testExplainJsonWithCte() { + // Expected output (when CTE is present, the JSON wraps cteQueries + mainQuery): + // { + // "cteQueries": [ + // { + // "name": "cte1", + // "plan": + // } + // ], + // "mainQuery": { + // "name": "OutputNode-", + // "id": "", + // "properties": { + // "OutputColumns": ["time", "deviceid", "voltage"], + // "OutputSymbols": ["time", "deviceid", "voltage"] + // }, + // "children": [ + // { + // "name": "ProjectNode-", ... + // "children": [ + // { + // "name": "FilterNode-", ... + // "children": [ + // { + // "name": "SemiJoinNode-", ... + // "children": [ + // { "name": "ExchangeNode-", ... }, // main table scan branch + // { "name": "ExchangeNode-", ... } // CTE scan branch + // ] + // } + // ] + // } + // ] + // } + // ] + // } + // } try (Connection conn = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = conn.createStatement()) { statement.execute("USE " + DATABASE_NAME); @@ -344,6 +550,43 @@ public void testExplainJsonWithCte() { @Test public void testExplainJsonWithScalarSubquery() { + // Expected output (scalar subquery is inlined during optimization, so the plan is a simple + // tree with the constant predicate pushed down to DeviceTableScanNode): + // { + // "name": "OutputNode-", + // "id": "", + // "properties": { + // "OutputColumns": ["time", "deviceid", "voltage"], + // "OutputSymbols": ["time", "deviceid", "voltage"] + // }, + // "children": [ + // { + // "name": "CollectNode-", + // "id": "", + // "children": [ + // { + // "name": "ExchangeNode-", ... + // "children": [ + // { + // "name": "DeviceTableScanNode-", + // "properties": { + // "QualifiedTableName": "testdb_json.testtb", + // "PushDownPredicate": "(\"voltage\" > 2E2)", + // ... + // } + // } + // ] + // }, + // { + // "name": "ExchangeNode-", ... + // "children": [ + // { "name": "DeviceTableScanNode-", ... } + // ] + // } + // ] + // } + // ] + // } String sql = "EXPLAIN (FORMAT JSON) SELECT * FROM testtb " + "WHERE voltage > (SELECT avg(voltage) FROM testtb)"; @@ -369,6 +612,56 @@ public void testExplainJsonWithScalarSubquery() { @Test public void testExplainAnalyzeJsonWithScalarSubquery() { + // Expected output (same EXPLAIN ANALYZE JSON structure, but the scalar subquery is resolved + // at planning time, so the executed plan only scans with a constant predicate): + // { + // "planStatistics": { + // "analyzeCostMs": , + // "fetchPartitionCostMs": , + // "fetchSchemaCostMs": , + // "logicalPlanCostMs": , // higher than simple query due to subquery planning + // "logicalOptimizationCostMs": , + // "distributionPlanCostMs": , + // "dispatchCostMs": + // }, + // "fragmentInstancesCount": 3, + // "fragmentInstances": [ + // { + // "id": "...", "dataRegion": "virtual_data_region", "state": "FINISHED", + // ... + // "operators": { + // "nodeType": "IdentitySinkNode", ... + // "children": [ + // { + // "nodeType": "CollectNode", ... + // "children": [ + // { "nodeType": "ExchangeNode", "outputRows": 2, ... }, + // { "nodeType": "ExchangeNode", "outputRows": 0, ... } + // ] + // } + // ] + // } + // }, + // { + // "dataRegion": "4", ... + // "operators": { + // "nodeType": "IdentitySinkNode", ... + // "children": [ + // { "nodeType": "DeviceTableScanNode", "operatorType": "TableScanOperator", ... } + // ] + // } + // }, + // { + // "dataRegion": "3", ... + // "operators": { + // "nodeType": "IdentitySinkNode", ... + // "children": [ + // { "nodeType": "DeviceTableScanNode", "outputRows": 0, ... } + // ] + // } + // } + // ] + // } String sql = "EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM testtb " + "WHERE voltage > (SELECT avg(voltage) FROM testtb)"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index 24adb746df4e1..4b7ab3f3ab43d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -385,7 +385,8 @@ public PlanAndMappings visitExplainAnalyze(ExplainAnalyzeNode node, UnaliasConte node.getQueryId(), node.getTimeout(), node.getOutputSymbols().get(0), - newChildPermittedOutputs), + newChildPermittedOutputs, + node.getOutputFormat()), mapping); } From d76f8e0138c5e866c1cc7720047cef37799432bd Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Fri, 3 Apr 2026 18:50:04 +0800 Subject: [PATCH 07/10] Fix testExplain IT to match updated EXPLAIN grammar with FORMAT clause The grammar change added optional '(' FORMAT identifier ')' to EXPLAIN and EXPLAIN ANALYZE rules, so '(' is now a valid expected token in parser error messages. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java index b67fe732f16b4..174b2d4711ae8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java @@ -460,7 +460,7 @@ public void testExplain() throws SQLException { e.getMessage(), e.getMessage() .contains( - "700: line 1:9: mismatched input 'INSERT'. Expecting: 'ANALYZE', 'EXECUTE', ")); + "700: line 1:9: mismatched input 'INSERT'. Expecting: '(', 'ANALYZE', 'EXECUTE', ")); } try { @@ -472,7 +472,7 @@ public void testExplain() throws SQLException { e.getMessage(), e.getMessage() .contains( - "700: line 1:17: mismatched input 'INSERT'. Expecting: 'EXECUTE', 'VERBOSE', ")); + "700: line 1:17: mismatched input 'INSERT'. Expecting: '(', 'EXECUTE', 'VERBOSE', ")); } } From c8cc5d781491feaed358c29eb492648e912ab3bf Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Fri, 3 Apr 2026 19:40:26 +0800 Subject: [PATCH 08/10] add more exclude files in rat check --- iotdb-core/relational-grammar/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iotdb-core/relational-grammar/pom.xml b/iotdb-core/relational-grammar/pom.xml index 289972ce30fb6..76acbf3355057 100644 --- a/iotdb-core/relational-grammar/pom.xml +++ b/iotdb-core/relational-grammar/pom.xml @@ -100,6 +100,8 @@ **/RelationalSqlLexer.java **/RelationalSqlLexer.interp **/RelationalSql.interp + **/RelationalSqlBaseListener.java + **/RelationalSqlListener.java From 7c6620157d6c15c7fc19a7992392304422be8abd Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Sat, 4 Apr 2026 11:59:38 +0800 Subject: [PATCH 09/10] Remove '|' borders from JSON content in CLI EXPLAIN FORMAT JSON output When using EXPLAIN (FORMAT JSON) or EXPLAIN ANALYZE (FORMAT JSON) in the CLI, the JSON content was wrapped in table borders (|), making it hard to copy for visualization. Now the header retains its table border formatting while JSON content lines are printed without '|' borders. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../iotdb/cli/it/ExplainJsonCliOutputIT.java | 293 ++++++++++++++++++ .../org/apache/iotdb/cli/AbstractCli.java | 46 ++- 2 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/cli/it/ExplainJsonCliOutputIT.java diff --git a/integration-test/src/test/java/org/apache/iotdb/cli/it/ExplainJsonCliOutputIT.java b/integration-test/src/test/java/org/apache/iotdb/cli/it/ExplainJsonCliOutputIT.java new file mode 100644 index 0000000000000..bed93ec6a3bf3 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/cli/it/ExplainJsonCliOutputIT.java @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.cli.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import com.google.gson.JsonParser; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Tests that EXPLAIN (FORMAT JSON) and EXPLAIN ANALYZE (FORMAT JSON) output raw JSON in CLI without + * table borders (no '|' or '+---' formatting), so users can directly copy the JSON for + * visualization. + */ +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class}) +public class ExplainJsonCliOutputIT extends AbstractScriptIT { + + private static String ip; + private static String port; + private static String sbinPath; + private static String libPath; + private static String homePath; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + ip = EnvFactory.getEnv().getIP(); + port = EnvFactory.getEnv().getPort(); + sbinPath = EnvFactory.getEnv().getSbinPath(); + libPath = EnvFactory.getEnv().getLibPath(); + homePath = + libPath.substring(0, libPath.lastIndexOf(File.separator + "lib" + File.separator + "*")); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void test() throws IOException { + String os = System.getProperty("os.name").toLowerCase(); + if (os.startsWith("windows")) { + testOnWindows(); + } else { + testOnUnix(); + } + } + + @Override + protected void testOnWindows() throws IOException { + // Setup test data + ProcessBuilder setupBuilder = + new ProcessBuilder( + "cmd.exe", + "/c", + sbinPath + File.separator + "windows" + File.separator + "start-cli.bat", + "-h", + ip, + "-p", + port, + "-sql_dialect", + "table", + "-e", + "\"CREATE DATABASE IF NOT EXISTS test_cli_json;" + + " USE test_cli_json;" + + " CREATE TABLE IF NOT EXISTS t1(id STRING TAG, v FLOAT FIELD);" + + " INSERT INTO t1 VALUES(1000, 'd1', 1.0)\"", + "&", + "exit", + "%^errorlevel%"); + setupBuilder.environment().put("IOTDB_HOME", homePath); + testOutput(setupBuilder, new String[] {"Msg: The statement is executed successfully."}, 0); + + // Test EXPLAIN (FORMAT JSON) output has no table borders + ProcessBuilder explainBuilder = + new ProcessBuilder( + "cmd.exe", + "/c", + sbinPath + File.separator + "windows" + File.separator + "start-cli.bat", + "-h", + ip, + "-p", + port, + "-sql_dialect", + "table", + "-e", + "\"USE test_cli_json; EXPLAIN (FORMAT JSON) SELECT * FROM t1\"", + "&", + "exit", + "%^errorlevel%"); + explainBuilder.environment().put("IOTDB_HOME", homePath); + assertRawJsonOutput(explainBuilder, "distribution plan"); + + // Test EXPLAIN ANALYZE (FORMAT JSON) output has no table borders + ProcessBuilder analyzeBuilder = + new ProcessBuilder( + "cmd.exe", + "/c", + sbinPath + File.separator + "windows" + File.separator + "start-cli.bat", + "-h", + ip, + "-p", + port, + "-sql_dialect", + "table", + "-e", + "\"USE test_cli_json; EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM t1\"", + "&", + "exit", + "%^errorlevel%"); + analyzeBuilder.environment().put("IOTDB_HOME", homePath); + assertRawJsonOutput(analyzeBuilder, "Explain Analyze"); + } + + @Override + protected void testOnUnix() throws IOException { + // Setup test data + ProcessBuilder setupBuilder = + new ProcessBuilder( + "bash", + sbinPath + File.separator + "start-cli.sh", + "-h", + ip, + "-p", + port, + "-sql_dialect", + "table", + "-e", + "\"CREATE DATABASE IF NOT EXISTS test_cli_json;" + + " USE test_cli_json;" + + " CREATE TABLE IF NOT EXISTS t1(id STRING TAG, v FLOAT FIELD);" + + " INSERT INTO t1 VALUES(1000, 'd1', 1.0)\""); + setupBuilder.environment().put("IOTDB_HOME", homePath); + testOutput(setupBuilder, new String[] {"Msg: The statement is executed successfully."}, 0); + + // Test EXPLAIN (FORMAT JSON) output has no table borders + ProcessBuilder explainBuilder = + new ProcessBuilder( + "bash", + sbinPath + File.separator + "start-cli.sh", + "-h", + ip, + "-p", + port, + "-sql_dialect", + "table", + "-e", + "\"USE test_cli_json; EXPLAIN (FORMAT JSON) SELECT * FROM t1\""); + explainBuilder.environment().put("IOTDB_HOME", homePath); + assertRawJsonOutput(explainBuilder, "distribution plan"); + + // Test EXPLAIN ANALYZE (FORMAT JSON) output has no table borders + ProcessBuilder analyzeBuilder = + new ProcessBuilder( + "bash", + sbinPath + File.separator + "start-cli.sh", + "-h", + ip, + "-p", + port, + "-sql_dialect", + "table", + "-e", + "\"USE test_cli_json; EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM t1\""); + analyzeBuilder.environment().put("IOTDB_HOME", homePath); + assertRawJsonOutput(analyzeBuilder, "Explain Analyze"); + } + + /** + * Collects all output lines from a process, then verifies: 1. The column header is printed before + * JSON content 2. No JSON content line has table border characters ('|' prefix or '+---' + * separator) 3. The combined JSON content is valid JSON + */ + private void assertRawJsonOutput(ProcessBuilder builder, String expectedHeader) + throws IOException { + builder.redirectErrorStream(true); + Process p = builder.start(); + BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); + String line; + List outputList = new ArrayList<>(); + while (true) { + line = r.readLine(); + if (line == null) { + break; + } else { + outputList.add(line); + } + } + r.close(); + p.destroy(); + + System.out.println("Process output:"); + for (String s : outputList) { + System.out.println(s); + } + + // Find the JSON content region: from the first line starting with '{' to the corresponding '}' + int jsonStart = -1; + int jsonEnd = -1; + for (int i = 0; i < outputList.size(); i++) { + String trimmed = outputList.get(i).trim(); + if (jsonStart == -1 && trimmed.startsWith("{")) { + jsonStart = i; + } + // The last line that is just '}' marks the end of JSON + if (jsonStart != -1 && trimmed.equals("}")) { + jsonEnd = i; + } + } + + assertTrue("Should find JSON start '{'", jsonStart >= 0); + assertTrue("Should find JSON end '}'", jsonEnd >= jsonStart); + + // Verify the column header with table border appears before JSON content + // Expected format: +---+ |header| +---+ {json...} +---+ + assertTrue("Header border should appear before JSON content", jsonStart >= 3); + assertTrue( + "Header border line should be present", + outputList.get(jsonStart - 3).trim().matches("\\+[-+]+\\+")); + assertTrue( + "Column header '" + expectedHeader + "' should be present", + outputList.get(jsonStart - 2).contains(expectedHeader)); + assertTrue( + "Header border line should be present", + outputList.get(jsonStart - 1).trim().matches("\\+[-+]+\\+")); + + // Verify JSON content lines do not have '|' borders + for (int i = jsonStart; i <= jsonEnd; i++) { + String s = outputList.get(i).trim(); + assertFalse("JSON line should not start with '|', but found: " + s, s.startsWith("|")); + } + + // Concatenate JSON lines and verify it's valid JSON + StringBuilder jsonBuilder = new StringBuilder(); + for (int i = jsonStart; i <= jsonEnd; i++) { + jsonBuilder.append(outputList.get(i)); + } + String jsonStr = jsonBuilder.toString(); + try { + JsonParser.parseString(jsonStr).getAsJsonObject(); + } catch (Exception e) { + fail("Output should be valid JSON, but got parse error: " + e.getMessage()); + } + + // Verify process exit code + while (p.isAlive()) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + e.printStackTrace(); + fail(); + } + } + assertEquals(0, p.exitValue()); + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java index 688158e78b251..3c4a4a030350f 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java @@ -565,7 +565,11 @@ private static int executeQuery(CliContext ctx, IoTDBConnection connection, Stri List maxSizeList = new ArrayList<>(columnLength); List> lists = cacheResult(ctx, resultSet, maxSizeList, columnLength, resultSetMetaData, zoneId); - output(ctx, lists, maxSizeList); + if (isJsonExplainResult(lists)) { + outputRawJson(ctx, lists, maxSizeList); + } else { + output(ctx, lists, maxSizeList); + } ctx.getPrinter().println(String.format("It costs %.3fs", costTime / 1000.0)); while (!isReachEnd) { if (continuePrint) { @@ -815,6 +819,46 @@ private static List> cacheTracingInfo(ResultSet resultSet, List> lists) { + if (lists.size() != 1 || lists.get(0).size() < 2) { + return false; + } + String columnName = lists.get(0).get(0); + if (!COLUMN_DISTRIBUTION_PLAN.equalsIgnoreCase(columnName) + && !COLUMN_EXPLAIN_ANALYZE.equalsIgnoreCase(columnName)) { + return false; + } + String value = lists.get(0).get(1).trim(); + return value.startsWith("{") || value.startsWith("["); + } + + private static void outputRawJson( + CliContext ctx, List> lists, List maxSizeList) { + // Use header text length for border width instead of the full content length + String header = lists.get(0).get(0); + int headerLen = header.length() + ctx.getPrinter().computeHANCount(header); + List headerSizeList = new ArrayList<>(1); + headerSizeList.add(headerLen); + // Print header with table border + ctx.getPrinter().printBlockLine(headerSizeList); + ctx.getPrinter().printRow(lists, 0, headerSizeList); + ctx.getPrinter().printBlockLine(headerSizeList); + // Print JSON content without '|' borders + for (int i = 1; i < lists.get(0).size(); i++) { + ctx.getPrinter().println(lists.get(0).get(i)); + } + ctx.getPrinter().printBlockLine(headerSizeList); + if (isReachEnd) { + lineCount += lists.get(0).size() - 1; + ctx.getPrinter().printCount(lineCount); + } else { + lineCount += maxPrintRowCount; + } + } + private static void output(CliContext ctx, List> lists, List maxSizeList) { ctx.getPrinter().printBlockLine(maxSizeList); ctx.getPrinter().printRow(lists, 0, maxSizeList); From adb686bd8901b8a99ecb3f1e4d1d08f405ef20ce Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Mon, 6 Apr 2026 15:24:00 +0800 Subject: [PATCH 10/10] Fix Windows CI failure in ExplainJsonCliOutputIT by removing inner quotes from -e args Java ProcessBuilder escapes internal " with \" but cmd.exe doesn't recognize \" as an escape - it treats every " as a quote toggle. This caused the SQL argument to be split into broken tokens with extra \ chars, producing malformed SQL that the server couldn't parse. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../org/apache/iotdb/cli/it/ExplainJsonCliOutputIT.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/cli/it/ExplainJsonCliOutputIT.java b/integration-test/src/test/java/org/apache/iotdb/cli/it/ExplainJsonCliOutputIT.java index bed93ec6a3bf3..5295779932f3e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/cli/it/ExplainJsonCliOutputIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/cli/it/ExplainJsonCliOutputIT.java @@ -98,10 +98,10 @@ protected void testOnWindows() throws IOException { "-sql_dialect", "table", "-e", - "\"CREATE DATABASE IF NOT EXISTS test_cli_json;" + "CREATE DATABASE IF NOT EXISTS test_cli_json;" + " USE test_cli_json;" + " CREATE TABLE IF NOT EXISTS t1(id STRING TAG, v FLOAT FIELD);" - + " INSERT INTO t1 VALUES(1000, 'd1', 1.0)\"", + + " INSERT INTO t1 VALUES(1000, 'd1', 1.0)", "&", "exit", "%^errorlevel%"); @@ -121,7 +121,7 @@ protected void testOnWindows() throws IOException { "-sql_dialect", "table", "-e", - "\"USE test_cli_json; EXPLAIN (FORMAT JSON) SELECT * FROM t1\"", + "USE test_cli_json; EXPLAIN (FORMAT JSON) SELECT * FROM t1", "&", "exit", "%^errorlevel%"); @@ -141,7 +141,7 @@ protected void testOnWindows() throws IOException { "-sql_dialect", "table", "-e", - "\"USE test_cli_json; EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM t1\"", + "USE test_cli_json; EXPLAIN ANALYZE (FORMAT JSON) SELECT * FROM t1", "&", "exit", "%^errorlevel%");