diff --git a/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out b/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out index 23b2d41d3dd0..fe032354407b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out @@ -35,22 +35,22 @@ Stage-0 Stage-1 Map 1 vectorized File Output Operator [FS_23] - Map Join Operator [MAPJOIN_22] (rows=2 width=530) + Map Join Operator [MAPJOIN_22] (rows=4 width=912) BucketMapJoin:true,Conds:SEL_21._col1, _col2=RS_19._col1, _col2(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] <-Map 2 [CUSTOM_EDGE] vectorized MULTICAST [RS_19] PartitionCols:_col2, _col1 - Select Operator [SEL_18] (rows=2 width=265) + Select Operator [SEL_18] (rows=2 width=456) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_17] (rows=2 width=265) + Filter Operator [FIL_17] (rows=2 width=456) predicate:(id is not null and part is not null) - TableScan [TS_3] (rows=2 width=265) + TableScan [TS_3] (rows=2 width=456) default@tbl,tbl2,Tbl:COMPLETE,Col:COMPLETE,Output:["foid","part","id"] - <-Select Operator [SEL_21] (rows=2 width=265) + <-Select Operator [SEL_21] (rows=2 width=456) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_20] (rows=2 width=265) + Filter Operator [FIL_20] (rows=2 width=456) predicate:(id is not null and part is not null) - TableScan [TS_0] (rows=2 width=265) + TableScan [TS_0] (rows=2 width=456) default@tbl,tbl,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:100,Grouping Partition Columns:["id","part"],Output:["foid","part","id"] PREHOOK: query: SELECT * FROM tbl JOIN tbl tbl2 ON tbl.id = tbl2.id AND tbl.part = tbl2.part diff --git a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out index a531c87a685c..ad4c13e4fbfb 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out @@ -67,12 +67,12 @@ POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tbl_ice Warning: Shuffle Join MERGEJOIN[61][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product Warning: Shuffle Join MERGEJOIN[63][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product -Warning: Shuffle Join MERGEJOIN[220][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product -Warning: Shuffle Join MERGEJOIN[222][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[218][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 9' is a cross product -Warning: Shuffle Join MERGEJOIN[224][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 11' is a cross product -Warning: Shuffle Join MERGEJOIN[216][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 14' is a cross product -Warning: Shuffle Join MERGEJOIN[226][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 16' is a cross product +Warning: Shuffle Join MERGEJOIN[218][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[220][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product +Warning: Shuffle Join MERGEJOIN[216][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 8' is a cross product +Warning: Shuffle Join MERGEJOIN[222][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 10' is a cross product +Warning: Shuffle Join MERGEJOIN[214][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 13' is a cross product +Warning: Shuffle Join MERGEJOIN[224][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 15' is a cross product PREHOOK: query: explain delete from tbl_ice where a in (select a from tbl_ice where a <= 5) or c in (select c from tbl_ice where c > 800) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice @@ -92,49 +92,48 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 24 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (XPROD_EDGE), Reducer 19 (XPROD_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE) - Reducer 14 <- Map 1 (XPROD_EDGE), Reducer 23 (XPROD_EDGE) - Reducer 15 <- Reducer 14 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) - Reducer 16 <- Reducer 15 (XPROD_EDGE), Reducer 19 (XPROD_EDGE) - Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) - Reducer 18 <- Reducer 17 (SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 19 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) - Reducer 20 <- Map 1 (SIMPLE_EDGE) - Reducer 21 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 23 <- Map 22 (CUSTOM_SIMPLE_EDGE) - Reducer 24 <- Map 22 (SIMPLE_EDGE) - Reducer 25 <- Map 22 (CUSTOM_SIMPLE_EDGE) - Reducer 26 <- Map 22 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (XPROD_EDGE), Reducer 25 (XPROD_EDGE) - Reducer 4 <- Reducer 20 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 21 (XPROD_EDGE), Reducer 4 (XPROD_EDGE) - Reducer 6 <- Reducer 13 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 8 <- Union 7 (SIMPLE_EDGE) - Reducer 9 <- Map 1 (XPROD_EDGE), Reducer 26 (XPROD_EDGE) + Reducer 10 <- Reducer 18 (XPROD_EDGE), Reducer 9 (XPROD_EDGE) + Reducer 11 <- Reducer 10 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) + Reducer 12 <- Reducer 11 (SIMPLE_EDGE) + Reducer 13 <- Map 1 (XPROD_EDGE), Reducer 22 (XPROD_EDGE) + Reducer 14 <- Reducer 13 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) + Reducer 15 <- Reducer 14 (XPROD_EDGE), Reducer 18 (XPROD_EDGE) + Reducer 16 <- Reducer 15 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) + Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Union 7 (CONTAINS) + Reducer 18 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 19 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) + Reducer 20 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 22 <- Map 21 (CUSTOM_SIMPLE_EDGE) + Reducer 23 <- Map 21 (SIMPLE_EDGE) + Reducer 24 <- Map 21 (CUSTOM_SIMPLE_EDGE) + Reducer 25 <- Map 21 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (XPROD_EDGE), Reducer 24 (XPROD_EDGE) + Reducer 4 <- Reducer 19 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 20 (XPROD_EDGE), Reducer 4 (XPROD_EDGE) + Reducer 6 <- Reducer 12 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE), Union 7 (CONTAINS) + Reducer 8 <- Map 1 (XPROD_EDGE), Reducer 25 (XPROD_EDGE) + Reducer 9 <- Reducer 23 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2916 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 3480 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 2916 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 3480 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) @@ -148,11 +147,11 @@ STAGE PLANS: Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 2868 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 3432 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 2868 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 3432 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) Filter Operator predicate: (c > 800) (type: boolean) @@ -194,81 +193,66 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: vectorized - Map 22 + Map 21 Map Operator Tree: TableScan alias: tbl_ice filterExpr: (a <= 5) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (a <= 5) (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.6666666 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Group By Operator keys: a (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.6666666 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(a) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.6666666 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Select Operator - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.6666666 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Execution mode: vectorized Reducer 10 - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col1, _col2, _col3, _col5 - Statistics: Num rows: 6 Data size: 1353 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 6 Data size: 1353 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean) - Reducer 11 Reduce Operator Tree: Merge Join Operator condition map: @@ -277,15 +261,15 @@ STAGE PLANS: 0 1 outputColumnNames: _col1, _col2, _col3, _col5, _col6 - Statistics: Num rows: 6 Data size: 1407 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1248 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 1407 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1248 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean), _col6 (type: bigint) - Reducer 12 + Reducer 11 Reduce Operator Tree: Merge Join Operator condition map: @@ -294,23 +278,23 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col2, _col3, _col5, _col6, _col8 - Statistics: Num rows: 6 Data size: 1547 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1248 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (((_col3 <> 0L) and _col5 is not null) or ((_col6 <> 0L) and _col8 is not null)) (type: boolean) - Statistics: Num rows: 6 Data size: 1547 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1248 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col2 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 6 Data size: 1547 Basic stats: COMPLETE Column stats: NONE - Reducer 13 + Statistics: Num rows: 6 Data size: 1248 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 12 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col2 - Statistics: Num rows: 6 Data size: 1547 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -330,27 +314,27 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 6 Data size: 1547 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 3 Data size: 773 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 773 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 773 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 3 Data size: 773 Basic stats: COMPLETE Column stats: NONE - Reducer 14 + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 13 Reduce Operator Tree: Merge Join Operator condition map: @@ -359,15 +343,15 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2946 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3480 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 2946 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3480 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint) - Reducer 15 + Reducer 14 Reduce Operator Tree: Merge Join Operator condition map: @@ -376,13 +360,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 6 Data size: 3240 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3504 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 3240 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3504 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) - Reducer 16 + Reducer 15 Reduce Operator Tree: Merge Join Operator condition map: @@ -391,15 +375,15 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 6 Data size: 3294 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3552 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 3294 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3552 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) - Reducer 17 + Reducer 16 Reduce Operator Tree: Merge Join Operator condition map: @@ -408,24 +392,24 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3576 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3576 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3576 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) - Reducer 18 + Reducer 17 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3432 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -444,22 +428,23 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3432 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 3 Data size: 1811 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 1716 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 1811 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 5265 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - Reducer 19 + Statistics: Num rows: 3 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 3480 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.tbl_ice + Reducer 18 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -477,22 +462,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) - Reducer 2 - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 6 Data size: 3207 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 6 Data size: 3207 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) - Reducer 20 + Reducer 19 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -525,7 +495,22 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) - Reducer 21 + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + Statistics: Num rows: 6 Data size: 3504 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 6 Data size: 3504 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + Reducer 20 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -538,77 +523,77 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 23 + Reducer 22 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 24 + Reducer 23 Execution mode: vectorized Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) - Reducer 25 + Reducer 24 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 26 + Reducer 25 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Reducer 3 Reduce Operator Tree: @@ -619,17 +604,17 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 - Statistics: Num rows: 6 Data size: 3333 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3600 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 - Statistics: Num rows: 6 Data size: 3333 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3600 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 3333 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3600 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean) Reducer 4 Reduce Operator Tree: @@ -640,11 +625,11 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13 - Statistics: Num rows: 6 Data size: 3666 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3624 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 3666 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3624 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col13 (type: boolean) Reducer 5 Reduce Operator Tree: @@ -655,24 +640,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13, _col14, _col15 - Statistics: Num rows: 6 Data size: 3768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3720 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col14 (type: bigint), _col15 (type: bigint), _col13 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col13, _col15 - Statistics: Num rows: 6 Data size: 3768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3720 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null) or (_col15 is not null and (_col12 <> 0L)) or ((_col2 is null or (_col13 < _col12)) and null and (_col12 <> 0L) and _col15 is null)) is null or (((_col8 = 0L) or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) and ((_col12 = 0L) or (_col15 is null and (_col13 >= _col12) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 5 Data size: 3140 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3720 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 3140 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3480 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 5 Data size: 3140 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 3480 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) Reducer 6 Reduce Operator Tree: @@ -683,34 +668,20 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 3454 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 3454 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 5265 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 3 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 3480 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.tbl_ice Reducer 8 - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 8 Data size: 5265 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.tbl_ice - Reducer 9 Reduce Operator Tree: Merge Join Operator condition map: @@ -719,14 +690,29 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 1230 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 1230 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint) + Reducer 9 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col1, _col2, _col3, _col5 + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean) Union 7 Vertex: Union 7 @@ -749,12 +735,12 @@ STAGE PLANS: Warning: Shuffle Join MERGEJOIN[61][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product Warning: Shuffle Join MERGEJOIN[63][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product -Warning: Shuffle Join MERGEJOIN[220][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product -Warning: Shuffle Join MERGEJOIN[222][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[218][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 9' is a cross product -Warning: Shuffle Join MERGEJOIN[224][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 11' is a cross product -Warning: Shuffle Join MERGEJOIN[216][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 14' is a cross product -Warning: Shuffle Join MERGEJOIN[226][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 16' is a cross product +Warning: Shuffle Join MERGEJOIN[218][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[220][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product +Warning: Shuffle Join MERGEJOIN[216][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 8' is a cross product +Warning: Shuffle Join MERGEJOIN[222][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 10' is a cross product +Warning: Shuffle Join MERGEJOIN[214][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 13' is a cross product +Warning: Shuffle Join MERGEJOIN[224][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 15' is a cross product PREHOOK: query: delete from tbl_ice where a in (select a from tbl_ice where a <= 5) or c in (select c from tbl_ice where c > 800) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice @@ -795,7 +781,7 @@ POSTHOOK: query: insert into tbl_ice_other values (10, 'ten'), (333, 'hundred') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tbl_ice_other -Warning: Shuffle Join MERGEJOIN[177][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[175][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain delete from tbl_ice where a in (select t1.a from tbl_ice t1 join tbl_ice_other t2 on t1.a = t2.a) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice @@ -817,78 +803,77 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (CUSTOM_SIMPLE_EDGE) - Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (XPROD_EDGE), Reducer 2 (XPROD_EDGE) - Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 6 <- Union 5 (SIMPLE_EDGE) - Reducer 7 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 9 <- Map 1 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) + Reducer 10 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 11 <- Reducer 10 (SIMPLE_EDGE) + Reducer 12 <- Reducer 8 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (XPROD_EDGE), Reducer 9 (XPROD_EDGE) + Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 6 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 7 <- Reducer 6 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 8 <- Map 1 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) + Reducer 9 <- Reducer 8 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 17434 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1160 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1160 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1144 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1144 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized - Map 14 + Map 13 Map Operator Tree: TableScan alias: t2 @@ -909,19 +894,6 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Reducer 10 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), count(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 11 Reduce Operator Tree: Merge Join Operator condition map: @@ -930,20 +902,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reducer 12 + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 11 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -963,44 +935,44 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reducer 13 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 12 Execution mode: vectorized Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reducer 2 Reduce Operator Tree: @@ -1011,11 +983,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1168 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1168 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) Reducer 3 Reduce Operator Tree: @@ -1026,24 +998,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 - Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1200 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 - Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1200 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) - Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1200 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1160 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1160 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) Reducer 4 Reduce Operator Tree: @@ -1054,34 +1026,20 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 3 Data size: 712 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 1160 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.tbl_ice Reducer 6 - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 3 Data size: 712 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.tbl_ice - Reducer 7 Reduce Operator Tree: Merge Join Operator condition map: @@ -1090,21 +1048,21 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1144 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1144 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) - Reducer 8 + Reducer 7 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1144 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1123,22 +1081,23 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1144 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 3 Data size: 712 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - Reducer 9 + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 1160 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.tbl_ice + Reducer 8 Reduce Operator Tree: Merge Join Operator condition map: @@ -1147,54 +1106,67 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count(), count(_col0) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 9 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), count(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: bigint), _col1 (type: bigint) Union 5 Vertex: Union 5 @@ -1215,7 +1187,7 @@ STAGE PLANS: Stats Work Basic Stats Work: -Warning: Shuffle Join MERGEJOIN[177][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[175][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: delete from tbl_ice where a in (select t1.a from tbl_ice t1 join tbl_ice_other t2 on t1.a = t2.a) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice @@ -1259,7 +1231,7 @@ POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tbl_standard_other POSTHOOK: Lineage: tbl_standard_other.a SCRIPT [] POSTHOOK: Lineage: tbl_standard_other.b SCRIPT [] -Warning: Shuffle Join MERGEJOIN[177][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[175][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain delete from tbl_ice where a in (select t1.a from tbl_ice t1 join tbl_ice_other t2 on t1.a = t2.a) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice @@ -1281,78 +1253,77 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 9 (CUSTOM_SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (XPROD_EDGE), Reducer 2 (XPROD_EDGE) - Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 6 <- Union 5 (SIMPLE_EDGE) - Reducer 7 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 9 <- Map 1 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) + Reducer 10 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 11 <- Reducer 10 (SIMPLE_EDGE) + Reducer 12 <- Reducer 8 (CUSTOM_SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 3 <- Reducer 12 (XPROD_EDGE), Reducer 2 (XPROD_EDGE) + Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 6 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 7 <- Reducer 6 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 8 <- Map 1 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) + Reducer 9 <- Reducer 8 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8752 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized - Map 14 + Map 13 Map Operator Tree: TableScan alias: t2 @@ -1373,25 +1344,6 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Reducer 10 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), true (type: boolean) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: boolean) - Reducer 11 Reduce Operator Tree: Merge Join Operator condition map: @@ -1400,20 +1352,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reducer 12 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 11 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1433,38 +1385,38 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reducer 13 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 12 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 2 Reduce Operator Tree: @@ -1475,11 +1427,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 584 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 584 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) Reducer 3 Reduce Operator Tree: @@ -1490,24 +1442,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 - Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 - Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) - Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) Reducer 4 Reduce Operator Tree: @@ -1518,34 +1470,20 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 2 Data size: 254 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 1160 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.tbl_ice Reducer 6 - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 2 Data size: 254 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.tbl_ice - Reducer 7 Reduce Operator Tree: Merge Join Operator condition map: @@ -1554,21 +1492,21 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) - Reducer 8 + Reducer 7 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1587,22 +1525,23 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 2 Data size: 254 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - Reducer 9 + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 1160 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.tbl_ice + Reducer 8 Reduce Operator Tree: Merge Join Operator condition map: @@ -1611,62 +1550,81 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col1 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count(), count(_col1) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Select Operator expressions: _col1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 9 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: boolean) Union 5 Vertex: Union 5 @@ -1687,7 +1645,7 @@ STAGE PLANS: Stats Work Basic Stats Work: -Warning: Shuffle Join MERGEJOIN[155][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[153][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: delete from tbl_ice where a in (select t1.a from tbl_ice t1 join tbl_standard_other t2 on t1.a = t2.a) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice diff --git a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out index 563564b4013a..6206ce08d56c 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out @@ -306,17 +306,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Map 4 Map Operator Tree: @@ -435,17 +435,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Map 4 Map Operator Tree: @@ -566,17 +566,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Execution mode: vectorized Map 5 @@ -660,13 +660,13 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -757,17 +757,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Execution mode: vectorized Map 5 @@ -819,13 +819,13 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -922,17 +922,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Map 4 Map Operator Tree: @@ -1067,17 +1067,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Map 4 Map Operator Tree: @@ -1191,17 +1191,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Map 4 Map Operator Tree: @@ -1320,17 +1320,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Map 4 Map Operator Tree: @@ -1442,17 +1442,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: hr is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: hr (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Map 4 Map Operator Tree: @@ -1571,17 +1571,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: hr is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: hr (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Map 4 Map Operator Tree: @@ -1684,17 +1684,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Map 4 Map Operator Tree: @@ -1834,20 +1834,20 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ((UDFToDouble(hr) = 11.0D) and ds is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (UDFToDouble(hr) = 11.0D) (type: boolean) - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Execution mode: vectorized Map 5 @@ -1931,13 +1931,13 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -2028,20 +2028,20 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ((UDFToDouble(hr) = 13.0D) and ds is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (UDFToDouble(hr) = 13.0D) (type: boolean) - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Execution mode: vectorized Map 5 @@ -2125,13 +2125,13 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -2217,27 +2217,27 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Map 4 Map Operator Tree: TableScan alias: srcpart_iceberg - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(ds) minReductionHashAggr: 0.99 @@ -2420,27 +2420,27 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Map 4 Map Operator Tree: TableScan alias: srcpart_iceberg - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(ds) minReductionHashAggr: 0.99 @@ -2473,19 +2473,19 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reducer 3 Execution mode: vectorized Reduce Operator Tree: @@ -2493,10 +2493,10 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2627,35 +2627,35 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: ds (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Map 6 Map Operator Tree: TableScan alias: srcpart_iceberg - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(ds) minReductionHashAggr: 0.99 @@ -2686,13 +2686,13 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -2702,10 +2702,10 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2717,13 +2717,13 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE Reducer 7 Execution mode: vectorized Reduce Operator Tree: @@ -2853,11 +2853,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 @@ -2993,11 +2993,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 @@ -3007,7 +3007,7 @@ STAGE PLANS: outputColumnNames: _col1 input vertices: 1 Map 3 - Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 @@ -3177,11 +3177,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 @@ -3327,11 +3327,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 @@ -3450,11 +3450,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: hr is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: hr (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 @@ -3590,14 +3590,14 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ((UDFToDouble(hr) = 11.0D) and ds is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (UDFToDouble(hr) = 11.0D) (type: boolean) - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 @@ -3607,7 +3607,7 @@ STAGE PLANS: outputColumnNames: _col1 input vertices: 1 Map 3 - Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 @@ -3770,14 +3770,14 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ((UDFToDouble(hr) = 13.0D) and ds is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (UDFToDouble(hr) = 13.0D) (type: boolean) - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 @@ -3787,7 +3787,7 @@ STAGE PLANS: outputColumnNames: _col1 input vertices: 1 Map 3 - Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 @@ -3946,11 +3946,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Left Semi Join 0 to 1 @@ -3960,29 +3960,29 @@ STAGE PLANS: outputColumnNames: _col0 input vertices: 1 Union 5 - Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Map 3 Map Operator Tree: TableScan alias: srcpart_iceberg - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(ds) minReductionHashAggr: 0.99 @@ -4013,10 +4013,10 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4172,11 +4172,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out index e1e5d28cf61b..c027eac831b5 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out @@ -948,35 +948,35 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2796 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), country (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2796 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2796 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2796 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) keys: country (type: string), state (type: string) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.6666666 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2464 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2464 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) Execution mode: vectorized Reducer 2 @@ -988,7 +988,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2796 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1002,14 +1002,14 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2056 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2956 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2956 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1071,35 +1071,35 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 12 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 3384 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), 'USA' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 12 Data size: 3252 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 4428 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 12 Data size: 3252 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 4428 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 12 Data size: 3252 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 4428 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) keys: country (type: string), state (type: string) - minReductionHashAggr: 0.9166667 + minReductionHashAggr: 0.8333333 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2270 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2270 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) Execution mode: vectorized Reducer 2 @@ -1111,7 +1111,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 12 Data size: 3252 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 4428 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1125,14 +1125,14 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 1862 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2956 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2956 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1194,35 +1194,35 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 24 Data size: 4440 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 24 Data size: 6768 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), country (type: string), 'CA' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 24 Data size: 6504 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 24 Data size: 8832 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 24 Data size: 6504 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 24 Data size: 8832 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 24 Data size: 6504 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 24 Data size: 8832 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) keys: country (type: string), state (type: string) minReductionHashAggr: 0.9583333 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1134 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1134 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) Execution mode: vectorized Reducer 2 @@ -1234,7 +1234,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 24 Data size: 6504 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 24 Data size: 8832 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1248,7 +1248,7 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 @@ -1317,35 +1317,35 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 48 Data size: 8880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 48 Data size: 13536 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), country (type: string), 'TX' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 48 Data size: 13008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 48 Data size: 17664 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 48 Data size: 13008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 48 Data size: 17664 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 48 Data size: 13008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 48 Data size: 17664 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) keys: country (type: string), state (type: string) minReductionHashAggr: 0.9791667 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1134 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1134 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) Execution mode: vectorized Reducer 2 @@ -1357,7 +1357,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 48 Data size: 13008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 48 Data size: 17664 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1371,7 +1371,7 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 @@ -1442,35 +1442,35 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 96 Data size: 17664 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 96 Data size: 27072 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), 'India' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 96 Data size: 26208 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 96 Data size: 35616 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 96 Data size: 26208 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 96 Data size: 35616 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 96 Data size: 26208 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 96 Data size: 35616 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) keys: country (type: string), state (type: string) - minReductionHashAggr: 0.9895833 + minReductionHashAggr: 0.9791667 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2274 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2274 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) Execution mode: vectorized Reducer 2 @@ -1482,7 +1482,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 96 Data size: 26208 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 96 Data size: 35616 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1496,14 +1496,14 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 835 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 1866 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2956 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2956 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1567,35 +1567,35 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 192 Data size: 34560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 192 Data size: 53376 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), 54 (type: int), 'India' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 192 Data size: 52416 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 192 Data size: 71232 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 192 Data size: 52416 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 192 Data size: 71232 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 192 Data size: 52416 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 192 Data size: 71232 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) keys: country (type: string), state (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.9895833 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2274 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2274 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) Execution mode: vectorized Reducer 2 @@ -1607,7 +1607,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 192 Data size: 52416 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 192 Data size: 71232 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1621,14 +1621,14 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 835 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 1866 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2956 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2956 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out index 9212bb8dc3c8..572b486c6627 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out @@ -504,35 +504,35 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1864 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), country (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1864 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1864 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) keys: country (type: string), state (type: string) - minReductionHashAggr: 0.75 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2464 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2464 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) Execution mode: vectorized Reducer 2 @@ -544,7 +544,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1864 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -558,14 +558,14 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2056 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2956 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2956 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -627,35 +627,35 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), 'USA' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1476 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1476 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1476 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) keys: country (type: string), state (type: string) - minReductionHashAggr: 0.75 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2270 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2270 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) Execution mode: vectorized Reducer 2 @@ -667,7 +667,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1476 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -681,14 +681,14 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 1862 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2956 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2956 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -750,35 +750,35 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 4 Data size: 740 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), country (type: string), 'CA' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1472 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1472 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1472 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) keys: country (type: string), state (type: string) minReductionHashAggr: 0.75 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1134 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1134 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) Execution mode: vectorized Reducer 2 @@ -790,7 +790,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1472 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -804,7 +804,7 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 @@ -873,35 +873,35 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 6 Data size: 1110 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1692 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), country (type: string), 'TX' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) keys: country (type: string), state (type: string) minReductionHashAggr: 0.8333333 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1134 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1134 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) Execution mode: vectorized Reducer 2 @@ -913,7 +913,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -927,7 +927,7 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 @@ -998,35 +998,35 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 2820 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), 'India' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3710 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3710 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3710 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) keys: country (type: string), state (type: string) - minReductionHashAggr: 0.9 + minReductionHashAggr: 0.8 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2274 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2274 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) Execution mode: vectorized Reducer 2 @@ -1038,7 +1038,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3710 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1052,14 +1052,14 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 835 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 1866 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2956 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2956 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1123,35 +1123,35 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 20 Data size: 3600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 5560 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), 54 (type: int), 'India' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7420 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7420 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7420 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) keys: country (type: string), state (type: string) - minReductionHashAggr: 0.95 + minReductionHashAggr: 0.9 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2274 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2274 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) Execution mode: vectorized Reducer 2 @@ -1163,7 +1163,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7420 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1177,14 +1177,14 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 835 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 1866 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2956 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 2956 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out index b87b1a62286b..53aefe0814ae 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out @@ -293,22 +293,22 @@ STAGE PLANS: alias: tbl_ice_pp_key As of version: s1 filterExpr: (a > 2) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (a > 2) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Execution mode: vectorized Reducer 2 @@ -325,7 +325,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 99 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 107 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 198 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -337,14 +337,14 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'asof_s1' (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 99 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 107 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 198 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out index fae1fb8c26df..54cd43981568 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out @@ -109,14 +109,14 @@ STAGE PLANS: TableScan alias: ice_01 filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 710 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string), ds (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 710 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 710 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out index ea652fe65e98..a1202117e971 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out @@ -88,14 +88,14 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a = 22) (type: boolean) - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 22 (type: int), b (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 1440 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 960 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 1440 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 960 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -283,14 +283,14 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a = 226784902765739L) (type: boolean) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 226784902765739L (type: bigint), b (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 1452 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 1452 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -501,17 +501,17 @@ STAGE PLANS: TableScan alias: test_ice_str filterExpr: (b = 'ddd') (type: boolean) - Statistics: Num rows: 7 Data size: 665 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 7 Data size: 1344 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (b = 'ddd') (type: boolean) - Statistics: Num rows: 7 Data size: 665 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: bigint), 'ddd' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 7 Data size: 3381 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 3381 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -742,17 +742,17 @@ STAGE PLANS: TableScan alias: test_ice_date filterExpr: (b = DATE'2022-02-07') (type: boolean) - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 4438 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (b = DATE'2022-02-07') (type: boolean) - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 4438 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: bigint), DATE'2022-02-07' (type: date) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 4438 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 4438 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -933,14 +933,14 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a = 1.156748927566759E11D) (type: boolean) - Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 1.156748927566759E11D (type: double), b (type: date) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 1356 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 904 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 1356 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 904 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1112,17 +1112,17 @@ STAGE PLANS: TableScan alias: test_ice_double_date filterExpr: ((a = 1.156748927566759E11D) and (b = DATE'2022-02-07')) (type: boolean) - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: ((a = 1.156748927566759E11D) and (b = DATE'2022-02-07')) (type: boolean) - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 1.156748927566759E11D (type: double), DATE'2022-02-07' (type: date) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 904 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 904 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out index 7ce866f2b03c..a98e6ae8a70c 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out @@ -259,9 +259,9 @@ Stage-3 <-Map 2 [CUSTOM_SIMPLE_EDGE] vectorized, llap File Output Operator [FS_32] table:{"name:":"default.bucketmapjoin_tmp_result"} - Select Operator [SEL_31] (rows=785 width=366) + Select Operator [SEL_31] (rows=238000 width=366) Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_30] (rows=785 width=186) + Map Join Operator [MAPJOIN_30] (rows=238000 width=186) BucketMapJoin:true,Conds:RS_27._col0=SEL_29._col0(Inner),Output:["_col0","_col1","_col3"] <-Map 1 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_27] @@ -281,7 +281,7 @@ Stage-3 PARTITION_ONLY_SHUFFLE [RS_35] Group By Operator [GBY_34] (rows=1 width=704) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(key))","avg(COALESCE(length(key),0))","count(1)","count(key)","compute_bit_vector_hll(key)","max(length(value1))","avg(COALESCE(length(value1),0))","count(value1)","compute_bit_vector_hll(value1)","max(length(value2))","avg(COALESCE(length(value2),0))","count(value2)","compute_bit_vector_hll(value2)"] - Select Operator [SEL_33] (rows=785 width=366) + Select Operator [SEL_33] (rows=238000 width=366) Output:["key","value1","value2"] Please refer to the previous Select Operator [SEL_31] @@ -423,9 +423,9 @@ Stage-3 <-Map 2 [CUSTOM_SIMPLE_EDGE] vectorized, llap File Output Operator [FS_32] table:{"name:":"default.bucketmapjoin_tmp_result"} - Select Operator [SEL_31] (rows=809 width=366) + Select Operator [SEL_31] (rows=124712 width=366) Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_30] (rows=809 width=186) + Map Join Operator [MAPJOIN_30] (rows=124712 width=186) BucketMapJoin:true,Conds:RS_27._col0=SEL_29._col0(Inner),Output:["_col0","_col1","_col3"] <-Map 1 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_27] @@ -445,7 +445,7 @@ Stage-3 PARTITION_ONLY_SHUFFLE [RS_35] Group By Operator [GBY_34] (rows=1 width=704) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(key))","avg(COALESCE(length(key),0))","count(1)","count(key)","compute_bit_vector_hll(key)","max(length(value1))","avg(COALESCE(length(value1),0))","count(value1)","compute_bit_vector_hll(value1)","max(length(value2))","avg(COALESCE(length(value2),0))","count(value2)","compute_bit_vector_hll(value2)"] - Select Operator [SEL_33] (rows=809 width=366) + Select Operator [SEL_33] (rows=124712 width=366) Output:["key","value1","value2"] Please refer to the previous Select Operator [SEL_31] diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out index f21a3c84da24..0c413c28c6a1 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out @@ -129,22 +129,22 @@ Stage-0 PARTITION_ONLY_SHUFFLE [RS_28] Group By Operator [GBY_27] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_26] (rows=372 width=8) + Map Join Operator [MAPJOIN_26] (rows=56644 width=8) BucketMapJoin:true,Conds:SEL_25._col0=RS_23._col0(Inner) <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_23] PartitionCols:_col0 Select Operator [SEL_22] (rows=238 width=4) Output:["_col0"] - Filter Operator [FIL_21] (rows=238 width=89) + Filter Operator [FIL_21] (rows=238 width=188) predicate:((part = '1') and key is not null) - TableScan [TS_3] (rows=238 width=89) + TableScan [TS_3] (rows=238 width=188) default@srcbucket_mapjoin_part_2_n4,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] <-Select Operator [SEL_25] (rows=238 width=4) Output:["_col0"] - Filter Operator [FIL_24] (rows=238 width=89) + Filter Operator [FIL_24] (rows=238 width=188) predicate:((part = '1') and key is not null) - TableScan [TS_0] (rows=238 width=89) + TableScan [TS_0] (rows=238 width=188) default@srcbucket_mapjoin_part_1_n1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) @@ -210,15 +210,15 @@ Stage-0 PartitionCols:_col0 Select Operator [SEL_27] (rows=238 width=4) Output:["_col0"] - Filter Operator [FIL_26] (rows=238 width=89) + Filter Operator [FIL_26] (rows=238 width=188) predicate:((part = '1') and key is not null) - TableScan [TS_3] (rows=238 width=89) + TableScan [TS_3] (rows=238 width=188) default@srcbucket_mapjoin_part_2_n4,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] <-Select Operator [SEL_30] (rows=238 width=4) Output:["_col0"] - Filter Operator [FIL_29] (rows=238 width=89) + Filter Operator [FIL_29] (rows=238 width=188) predicate:((part = '1') and key is not null) - TableScan [TS_0] (rows=238 width=89) + TableScan [TS_0] (rows=238 width=188) default@srcbucket_mapjoin_part_1_n1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out index dd93fcf31851..8964093434f5 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out @@ -242,22 +242,22 @@ Stage-0 PARTITION_ONLY_SHUFFLE [RS_28] Group By Operator [GBY_27] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_26] (rows=1797 width=8) + Map Join Operator [MAPJOIN_26] (rows=544644 width=8) Conds:SEL_25._col0=RS_23._col0(Inner) <-Map 3 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_23] PartitionCols:_col0 Select Operator [SEL_22] (rows=738 width=4) Output:["_col0"] - Filter Operator [FIL_21] (rows=738 width=89) + Filter Operator [FIL_21] (rows=738 width=188) predicate:(part is not null and key is not null) - TableScan [TS_3] (rows=738 width=89) + TableScan [TS_3] (rows=738 width=188) default@srcbucket_mapjoin_part_2_n6,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] <-Select Operator [SEL_25] (rows=738 width=4) Output:["_col0"] - Filter Operator [FIL_24] (rows=738 width=89) + Filter Operator [FIL_24] (rows=738 width=188) predicate:(part is not null and key is not null) - TableScan [TS_0] (rows=738 width=89) + TableScan [TS_0] (rows=738 width=188) default@srcbucket_mapjoin_part_1_n2,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) @@ -309,28 +309,28 @@ Stage-0 PARTITION_ONLY_SHUFFLE [RS_37] Group By Operator [GBY_36] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_35] (rows=1797 width=8) + Map Join Operator [MAPJOIN_35] (rows=272322 width=8) Conds:SEL_34._col0, _col1=RS_29._col0, _col1(Inner) <-Map 3 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_29] PartitionCols:_col0, _col1 - Select Operator [SEL_28] (rows=738 width=89) + Select Operator [SEL_28] (rows=738 width=188) Output:["_col0","_col1"] - Filter Operator [FIL_27] (rows=738 width=89) + Filter Operator [FIL_27] (rows=738 width=188) predicate:(part is not null and key is not null) - TableScan [TS_3] (rows=738 width=89) + TableScan [TS_3] (rows=738 width=188) default@srcbucket_mapjoin_part_2_n6,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] - Dynamic Partitioning Event Operator [EVENT_32] (rows=2 width=85) - Group By Operator [GBY_31] (rows=2 width=85) + Dynamic Partitioning Event Operator [EVENT_32] (rows=2 width=184) + Group By Operator [GBY_31] (rows=2 width=184) Output:["_col0"],keys:_col0 - Select Operator [SEL_30] (rows=738 width=85) + Select Operator [SEL_30] (rows=738 width=184) Output:["_col0"] Please refer to the previous Select Operator [SEL_28] - <-Select Operator [SEL_34] (rows=738 width=89) + <-Select Operator [SEL_34] (rows=738 width=188) Output:["_col0","_col1"] - Filter Operator [FIL_33] (rows=738 width=89) + Filter Operator [FIL_33] (rows=738 width=188) predicate:(part is not null and key is not null) - TableScan [TS_0] (rows=738 width=89) + TableScan [TS_0] (rows=738 width=188) default@srcbucket_mapjoin_part_1_n2,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out index 8ff5e300aa46..c4f0814c49a5 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out @@ -188,15 +188,15 @@ Stage-0 PartitionCols:_col0 Select Operator [SEL_27] (rows=238 width=4) Output:["_col0"] - Filter Operator [FIL_26] (rows=238 width=89) + Filter Operator [FIL_26] (rows=238 width=188) predicate:((part = '1') and key is not null) - TableScan [TS_3] (rows=238 width=89) + TableScan [TS_3] (rows=238 width=188) default@srcbucket_mapjoin_part_2_n0,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] <-Select Operator [SEL_30] (rows=238 width=4) Output:["_col0"] - Filter Operator [FIL_29] (rows=238 width=89) + Filter Operator [FIL_29] (rows=238 width=188) predicate:((part = '1') and key is not null) - TableScan [TS_0] (rows=238 width=89) + TableScan [TS_0] (rows=238 width=188) default@srcbucket_mapjoin_part_1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) @@ -248,22 +248,22 @@ Stage-0 PARTITION_ONLY_SHUFFLE [RS_28] Group By Operator [GBY_27] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_26] (rows=372 width=8) + Map Join Operator [MAPJOIN_26] (rows=56644 width=8) BucketMapJoin:true,Conds:SEL_25._col0=RS_23._col0(Inner) <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_23] PartitionCols:_col0 Select Operator [SEL_22] (rows=238 width=4) Output:["_col0"] - Filter Operator [FIL_21] (rows=238 width=89) + Filter Operator [FIL_21] (rows=238 width=188) predicate:((part = '1') and key is not null) - TableScan [TS_3] (rows=238 width=89) + TableScan [TS_3] (rows=238 width=188) default@srcbucket_mapjoin_part_3,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] <-Select Operator [SEL_25] (rows=238 width=4) Output:["_col0"] - Filter Operator [FIL_24] (rows=238 width=89) + Filter Operator [FIL_24] (rows=238 width=188) predicate:((part = '1') and key is not null) - TableScan [TS_0] (rows=238 width=89) + TableScan [TS_0] (rows=238 width=188) default@srcbucket_mapjoin_part_1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out index 024fde7f1576..921f1431d8c9 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out @@ -149,9 +149,9 @@ Stage-3 <-Map 2 [CUSTOM_SIMPLE_EDGE] vectorized, llap File Output Operator [FS_32] table:{"name:":"default.bucketmapjoin_tmp_result_n3"} - Select Operator [SEL_31] (rows=809 width=366) + Select Operator [SEL_31] (rows=124712 width=366) Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_30] (rows=809 width=186) + Map Join Operator [MAPJOIN_30] (rows=124712 width=186) BucketMapJoin:true,Conds:RS_27._col0=SEL_29._col0(Inner),Output:["_col0","_col1","_col3"] <-Map 1 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_27] @@ -171,7 +171,7 @@ Stage-3 PARTITION_ONLY_SHUFFLE [RS_35] Group By Operator [GBY_34] (rows=1 width=704) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(key))","avg(COALESCE(length(key),0))","count(1)","count(key)","compute_bit_vector_hll(key)","max(length(value1))","avg(COALESCE(length(value1),0))","count(value1)","compute_bit_vector_hll(value1)","max(length(value2))","avg(COALESCE(length(value2),0))","count(value2)","compute_bit_vector_hll(value2)"] - Select Operator [SEL_33] (rows=809 width=366) + Select Operator [SEL_33] (rows=124712 width=366) Output:["key","value1","value2"] Please refer to the previous Select Operator [SEL_31] diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out index 7b486db67697..9c87e5b1a7cf 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out @@ -50,15 +50,15 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_46] - Limit [LIM_45] (rows=20 width=447) + Limit [LIM_45] (rows=20 width=637) Number of rows:20 - Select Operator [SEL_44] (rows=791 width=447) + Select Operator [SEL_44] (rows=791 width=637) Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_43] - Top N Key Operator [TNK_42] (rows=791 width=447) + Top N Key Operator [TNK_42] (rows=791 width=637) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_41] (rows=791 width=447) + Map Join Operator [MAPJOIN_41] (rows=791 width=637) BucketMapJoin:true,Conds:SEL_40._col0, _col1=RS_38._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_38] @@ -69,11 +69,11 @@ Stage-0 predicate:(key is not null and value is not null) TableScan [TS_3] (rows=500 width=178) default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_40] (rows=500 width=269) + <-Select Operator [SEL_40] (rows=500 width=459) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_39] (rows=500 width=269) + Filter Operator [FIL_39] (rows=500 width=459) predicate:(key1 is not null and key2 is not null) - TableScan [TS_0] (rows=500 width=269) + TableScan [TS_0] (rows=500 width=459) default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"] PREHOOK: query: SELECT * @@ -246,15 +246,15 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_36] - Limit [LIM_35] (rows=20 width=447) + Limit [LIM_35] (rows=20 width=637) Number of rows:20 - Select Operator [SEL_34] (rows=791 width=447) + Select Operator [SEL_34] (rows=791 width=637) Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_33] - Top N Key Operator [TNK_32] (rows=791 width=447) + Top N Key Operator [TNK_32] (rows=791 width=637) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_31] (rows=791 width=447) + Map Join Operator [MAPJOIN_31] (rows=791 width=637) BucketMapJoin:true,Conds:SEL_30._col0=RS_28._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_28] @@ -265,11 +265,11 @@ Stage-0 predicate:key is not null TableScan [TS_3] (rows=500 width=178) default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_30] (rows=500 width=269) + <-Select Operator [SEL_30] (rows=500 width=459) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_29] (rows=500 width=269) + Filter Operator [FIL_29] (rows=500 width=459) predicate:key1 is not null - TableScan [TS_0] (rows=500 width=269) + TableScan [TS_0] (rows=500 width=459) default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"] PREHOOK: query: SELECT * @@ -444,15 +444,15 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_36] - Limit [LIM_35] (rows=20 width=447) + Limit [LIM_35] (rows=20 width=637) Number of rows:20 - Select Operator [SEL_34] (rows=791 width=447) + Select Operator [SEL_34] (rows=791 width=637) Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_33] - Top N Key Operator [TNK_32] (rows=791 width=447) + Top N Key Operator [TNK_32] (rows=791 width=637) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_31] (rows=791 width=447) + Map Join Operator [MAPJOIN_31] (rows=791 width=637) BucketMapJoin:true,Conds:SEL_30._col0=RS_28._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_28] @@ -463,11 +463,11 @@ Stage-0 predicate:key is not null TableScan [TS_3] (rows=500 width=178) default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_30] (rows=500 width=269) + <-Select Operator [SEL_30] (rows=500 width=459) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_29] (rows=500 width=269) + Filter Operator [FIL_29] (rows=500 width=459) predicate:((key2 <> 'val_0') and (key2 <> 'val_100') and key1 is not null) - TableScan [TS_0] (rows=500 width=269) + TableScan [TS_0] (rows=500 width=459) default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"] PREHOOK: query: SELECT * @@ -542,15 +542,15 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_36] - Limit [LIM_35] (rows=20 width=447) + Limit [LIM_35] (rows=20 width=637) Number of rows:20 - Select Operator [SEL_34] (rows=814 width=447) + Select Operator [SEL_34] (rows=814 width=637) Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_33] - Top N Key Operator [TNK_32] (rows=814 width=447) + Top N Key Operator [TNK_32] (rows=814 width=637) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_31] (rows=814 width=447) + Map Join Operator [MAPJOIN_31] (rows=814 width=637) Conds:SEL_30._col1=RS_28._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 3 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_28] @@ -561,11 +561,11 @@ Stage-0 predicate:value is not null TableScan [TS_3] (rows=500 width=178) default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_30] (rows=500 width=269) + <-Select Operator [SEL_30] (rows=500 width=459) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_29] (rows=500 width=269) + Filter Operator [FIL_29] (rows=500 width=459) predicate:key2 is not null - TableScan [TS_0] (rows=500 width=269) + TableScan [TS_0] (rows=500 width=459) default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2","value"] PREHOOK: query: SELECT * @@ -638,15 +638,15 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_51] - Limit [LIM_50] (rows=20 width=447) + Limit [LIM_50] (rows=20 width=637) Number of rows:20 - Select Operator [SEL_49] (rows=791 width=447) + Select Operator [SEL_49] (rows=791 width=637) Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_48] - Top N Key Operator [TNK_47] (rows=791 width=447) + Top N Key Operator [TNK_47] (rows=791 width=637) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_46] (rows=791 width=447) + Map Join Operator [MAPJOIN_46] (rows=791 width=637) BucketMapJoin:true,Conds:SEL_45._col0, _col1, _col2=RS_43._col0, _col1, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_43] @@ -657,11 +657,11 @@ Stage-0 predicate:(key is not null and value is not null) TableScan [TS_3] (rows=500 width=178) default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_45] (rows=500 width=269) + <-Select Operator [SEL_45] (rows=500 width=459) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_44] (rows=500 width=269) + Filter Operator [FIL_44] (rows=500 width=459) predicate:(key1 is not null and key2 is not null and value is not null) - TableScan [TS_0] (rows=500 width=269) + TableScan [TS_0] (rows=500 width=459) default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"] PREHOOK: query: SELECT * @@ -734,15 +734,15 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_41] - Limit [LIM_40] (rows=20 width=447) + Limit [LIM_40] (rows=20 width=637) Number of rows:20 - Select Operator [SEL_39] (rows=814 width=447) + Select Operator [SEL_39] (rows=814 width=637) Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_38] - Top N Key Operator [TNK_37] (rows=814 width=447) + Top N Key Operator [TNK_37] (rows=814 width=637) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_36] (rows=814 width=447) + Map Join Operator [MAPJOIN_36] (rows=814 width=637) Conds:SEL_35._col2=RS_33._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 3 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_33] @@ -753,11 +753,11 @@ Stage-0 predicate:value is not null TableScan [TS_3] (rows=500 width=178) default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_35] (rows=500 width=269) + <-Select Operator [SEL_35] (rows=500 width=459) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_34] (rows=500 width=269) + Filter Operator [FIL_34] (rows=500 width=459) predicate:value is not null - TableScan [TS_0] (rows=500 width=269) + TableScan [TS_0] (rows=500 width=459) default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2","value"] PREHOOK: query: SELECT * diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out index 148f2d89ee20..78ab0217b492 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out @@ -232,11 +232,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_32] - Select Operator [SEL_31] (rows=16 width=92) + Select Operator [SEL_31] (rows=16 width=188) Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_30] - Map Join Operator [MAPJOIN_29] (rows=16 width=92) + Map Join Operator [MAPJOIN_29] (rows=16 width=188) BucketMapJoin:true,Conds:SEL_28._col0=RS_26._col0(Inner),Output:["_col0","_col1"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_26] @@ -247,11 +247,11 @@ Stage-0 predicate:key2 is not null TableScan [TS_3] (rows=6 width=72) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key2"] - <-Select Operator [SEL_28] (rows=16 width=92) + <-Select Operator [SEL_28] (rows=16 width=188) Output:["_col0","_col1"] - Filter Operator [FIL_27] (rows=16 width=92) + Filter Operator [FIL_27] (rows=16 width=188) predicate:key2 is not null - TableScan [TS_0] (rows=16 width=92) + TableScan [TS_0] (rows=16 width=188) default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:8,Grouping Partition Columns:["key2"],Output:["key2","id"] PREHOOK: query: SELECT a.key2, a.id @@ -400,11 +400,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_42] - Select Operator [SEL_41] (rows=12 width=96) + Select Operator [SEL_41] (rows=12 width=192) Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_40] - Map Join Operator [MAPJOIN_39] (rows=12 width=96) + Map Join Operator [MAPJOIN_39] (rows=12 width=192) BucketMapJoin:true,Conds:SEL_38._col0, _col1=RS_36._col0, _col1(Inner),Output:["_col0","_col1","_col2"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_36] @@ -415,11 +415,11 @@ Stage-0 predicate:(key1 is not null and key2 is not null) TableScan [TS_3] (rows=6 width=75) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2"] - <-Select Operator [SEL_38] (rows=12 width=96) + <-Select Operator [SEL_38] (rows=12 width=192) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_37] (rows=12 width=96) + Filter Operator [FIL_37] (rows=12 width=192) predicate:(key1 is not null and key2 is not null) - TableScan [TS_0] (rows=12 width=96) + TableScan [TS_0] (rows=12 width=192) default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:32,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","id"] PREHOOK: query: SELECT a.key1, a.key2, a.id @@ -480,11 +480,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_47] - Select Operator [SEL_46] (rows=9 width=78) + Select Operator [SEL_46] (rows=12 width=83) Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_45] - Map Join Operator [MAPJOIN_44] (rows=9 width=78) + Map Join Operator [MAPJOIN_44] (rows=12 width=83) BucketMapJoin:true,Conds:SEL_43._col0, _col1=RS_41._col0, _col1(Inner),Output:["_col0","_col1","_col2"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_41] @@ -560,11 +560,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_52] - Select Operator [SEL_51] (rows=5 width=150) + Select Operator [SEL_51] (rows=8 width=260) Output:["_col0","_col1","_col2","_col3"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_50] - Map Join Operator [MAPJOIN_49] (rows=5 width=150) + Map Join Operator [MAPJOIN_49] (rows=8 width=260) BucketMapJoin:true,Conds:SEL_48._col0, _col1, _col2=RS_46._col0, _col1, _col2(Inner),Output:["_col0","_col1","_col2","_col3"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_46] @@ -575,11 +575,11 @@ Stage-0 predicate:(key1 is not null and key2 is not null and value is not null) TableScan [TS_3] (rows=6 width=150) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2","value"] - <-Select Operator [SEL_48] (rows=8 width=164) + <-Select Operator [SEL_48] (rows=8 width=260) Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_47] (rows=8 width=164) + Filter Operator [FIL_47] (rows=8 width=260) predicate:(value is not null and key1 is not null and key2 is not null) - TableScan [TS_0] (rows=12 width=164) + TableScan [TS_0] (rows=12 width=260) default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:32,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value","id"] PREHOOK: query: SELECT a.key1, a.key2, a.value, a.id diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out index 8e77ed6fd0a9..0562bdeafb74 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out @@ -965,14 +965,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_orc - Statistics: Num rows: 9 Data size: 1592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1608 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 1592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1608 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 1592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1608 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out index b6e253358ee6..3af4acc84c37 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out @@ -73,61 +73,15 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 10 <- Reducer 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 11 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 5 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 5 <- Map 11 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 7 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE) + Reducer 7 <- Map 11 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) - Reducer 9 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE) + Reducer 9 <- Map 11 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 - Map Operator Tree: - TableScan - alias: src - filterExpr: ((a <= 100) or a is not null) (type: boolean) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (a <= 100) (type: boolean) - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: a is not null (type: boolean) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized - Map 11 Map Operator Tree: TableScan alias: target_ice @@ -148,38 +102,38 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 792 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int) Execution mode: vectorized - Map 12 + Map 11 Map Operator Tree: TableScan alias: target_ice - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 2320 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 2320 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 2288 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col4 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 2288 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col5 (type: string), _col6 (type: int) Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) @@ -196,13 +150,58 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 752 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: string) Execution mode: vectorized + Map 4 + Map Operator Tree: + TableScan + alias: src + filterExpr: ((a <= 100) or a is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (a <= 100) (type: boolean) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized Reducer 10 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int) outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 2288 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -221,57 +220,44 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 2288 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 950 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 1144 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 3870 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 2 Data size: 1160 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 3292 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.target_ice Reducer 2 Reduce Operator Tree: Merge Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: int) - 1 _col5 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 + 0 _col5 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 792 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), _col4 (type: bigint), _col5 (type: string), _col6 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), 'Merged' (type: string), (_col6 + 10) (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 2 Data size: 972 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 3870 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - Reducer 4 - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 8 Data size: 3870 Basic stats: COMPLETE Column stats: PARTIAL - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.target_ice + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 3292 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.target_ice Reducer 5 Reduce Operator Tree: Merge Join Operator @@ -281,20 +267,20 @@ STAGE PLANS: 0 _col5 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 2336 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator - predicate: ((_col5 <> _col8) or _col5 is null or _col8 is null) (type: boolean) - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: PARTIAL + predicate: (_col5 is null or (_col5 <> _col8) or _col8 is null) (type: boolean) + Statistics: Num rows: 4 Data size: 2336 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 2320 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col2 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 2320 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) Reducer 6 Reduce Operator Tree: @@ -305,14 +291,15 @@ STAGE PLANS: 0 _col2 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 3870 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 2 Data size: 1160 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 3292 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.target_ice Reducer 7 Reduce Operator Tree: Merge Join Operator @@ -384,13 +371,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col4 (type: int) outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 2288 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col3 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 2288 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) Union 3 Vertex: Union 3 @@ -462,28 +449,27 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: target_ice - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 1176 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 1176 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string) Execution mode: vectorized - Map 4 + Map 3 Map Operator Tree: TableScan alias: src @@ -509,36 +495,22 @@ STAGE PLANS: 0 _col5 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 1464 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: _col5 is null (type: boolean) - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 488 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int), _col7 (type: string), _col8 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - Reducer 3 - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.target_ice + Statistics: Num rows: 1 Data size: 484 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 484 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.target_ice Stage: Stage-2 Dependency Collection diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out index 693513fa2dfc..cd29e8bfe0f3 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out @@ -96,20 +96,20 @@ STAGE PLANS: TableScan alias: target_ice filterExpr: a is not null (type: boolean) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 2320 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 2320 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) Execution mode: vectorized Reducer 2 @@ -121,21 +121,21 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 10 Data size: 3375 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 2704 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col7 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col10 (type: int), _col9 (type: string), _col8 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 10 Data size: 3375 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 2704 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: ((_col10 = _col1) and (_col10 > 100)) (type: boolean) - Statistics: Num rows: 4 Data size: 1833 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 1352 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1449 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 1160 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 1449 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 1160 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -143,14 +143,14 @@ STAGE PLANS: name: default.target_ice Filter Operator predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 579 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 1352 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 483 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 1160 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 483 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 1160 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -158,52 +158,52 @@ STAGE PLANS: name: default.target_ice Filter Operator predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 579 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 1352 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col10 (type: int), 'Merged' (type: string), (_col8 + 10) (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col0, 16) (type: int), iceberg_truncate(_col1, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col0, 16) (type: int), iceberg_truncate(_col1, 3) (type: string) - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) Filter Operator predicate: _col10 is null (type: boolean) - Statistics: Num rows: 6 Data size: 2025 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 676 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: int), _col0 (type: string), _col4 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col0, 16) (type: int), iceberg_truncate(_col1, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col0, 16) (type: int), iceberg_truncate(_col1, 3) (type: string) - Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) Filter Operator predicate: (_col10 = _col1) (type: boolean) - Statistics: Num rows: 5 Data size: 1929 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 1352 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col2 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: int) outputColumnNames: _col2, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 1929 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 1352 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() keys: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 4 Data size: 644 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 424 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) null sort order: zzzz sort order: ++++ Map-reduce partition columns: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) - Statistics: Num rows: 4 Data size: 644 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 424 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col4 (type: bigint) Reducer 3 Execution mode: vectorized @@ -214,7 +214,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -229,7 +229,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -243,7 +243,7 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 4 Data size: 644 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 424 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (_col4 > 1L) (type: boolean) Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: PARTIAL diff --git a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out index 0ff1c86f2107..1e0163071486 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out @@ -284,30 +284,30 @@ Stage-3 <-Reducer 2 [SIMPLE_EDGE] vectorized SHUFFLE [RS_21] PartitionCols:_col4, _col5 - Select Operator [SEL_20] (rows=11 width=420) + Select Operator [SEL_20] (rows=11 width=1###) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Limit [LIM_19] (rows=11 width=420) + Limit [LIM_19] (rows=11 width=1###) Number of rows:100000 <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized PARTITION_ONLY_SHUFFLE [RS_18] - Select Operator [SEL_17] (rows=11 width=420) + Select Operator [SEL_17] (rows=11 width=1###) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Limit [LIM_16] (rows=11 width=420) + Limit [LIM_16] (rows=11 width=1###) Number of rows:100000 - TableScan [TS_0] (rows=11 width=420) + TableScan [TS_0] (rows=11 width=1###) default@llap_orders,llap_orders,Tbl:COMPLETE,Col:NONE,Output:["orderid","quantity","itemid","tradets","p1","p2"] Reducer 4 vectorized File Output Operator [FS_29] - Select Operator [SEL_28] (rows=5 width=420) + Select Operator [SEL_28] (rows=5 width=1###) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36"] - Group By Operator [GBY_27] (rows=5 width=420) + Group By Operator [GBY_27] (rows=5 width=1###) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)","min(VALUE._col13)","max(VALUE._col14)","count(VALUE._col15)","compute_bit_vector_hll(VALUE._col16)","max(VALUE._col17)","avg(VALUE._col18)","count(VALUE._col19)","compute_bit_vector_hll(VALUE._col20)","max(VALUE._col21)","avg(VALUE._col22)","count(VALUE._col23)","compute_bit_vector_hll(VALUE._col24)"],keys:KEY._col0, KEY._col1 <-Reducer 2 [SIMPLE_EDGE] vectorized SHUFFLE [RS_24] PartitionCols:_col0, _col1 - Group By Operator [GBY_23] (rows=11 width=420) + Group By Operator [GBY_23] (rows=11 width=1###) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26"],aggregations:["min(orderid)","max(orderid)","count(1)","count(orderid)","compute_bit_vector_hll(orderid)","min(quantity)","max(quantity)","count(quantity)","compute_bit_vector_hll(quantity)","min(itemid)","max(itemid)","count(itemid)","compute_bit_vector_hll(itemid)","min(tradets)","max(tradets)","count(tradets)","compute_bit_vector_hll(tradets)","max(length(p1))","avg(COALESCE(length(p1),0))","count(p1)","compute_bit_vector_hll(p1)","max(length(p2))","avg(COALESCE(length(p2),0))","count(p2)","compute_bit_vector_hll(p2)"],keys:p1, p2 - Select Operator [SEL_22] (rows=11 width=420) + Select Operator [SEL_22] (rows=11 width=1###) Output:["orderid","quantity","itemid","tradets","p1","p2"] Please refer to the previous Select Operator [SEL_20] diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out index 7dac902dc9ae..69475e9e82d8 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out @@ -51,10 +51,10 @@ STAGE PLANS: TableScan alias: tbl_ice filterExpr: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), 'Changed' (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 @@ -64,7 +64,7 @@ STAGE PLANS: null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 5 Data size: 2417 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 2617 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) Execution mode: vectorized Map 4 @@ -72,20 +72,20 @@ STAGE PLANS: TableScan alias: tbl_ice filterExpr: (((b) IN ('four', 'one') or (a = 22)) is null or ((b <> 'four') and (b <> 'one') and (a <> 22))) (type: boolean) - Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: ((((b) IN ('four', 'one') or (a = 22)) is null or ((b <> 'four') and (b <> 'one') and (a <> 22))) and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) Execution mode: vectorized Map 6 @@ -93,26 +93,26 @@ STAGE PLANS: TableScan alias: tbl_ice filterExpr: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: FILE__PATH (type: string) null sort order: a sort order: + Map-reduce partition columns: FILE__PATH (type: string) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), PARTITION__PROJECTION (type: string) Filter Operator predicate: (((b) IN ('four', 'one') or (a = 22)) and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: FILE__PATH (type: string) null sort order: a sort order: + Map-reduce partition columns: FILE__PATH (type: string) - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Reducer 3 Execution mode: vectorized @@ -123,7 +123,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 5 Data size: 2417 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 2617 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -138,17 +138,17 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 5 Data size: 2417 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 2617 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) Reducer 7 Execution mode: vectorized @@ -156,7 +156,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col7 - Statistics: Num rows: 3 Data size: 1428 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 1716 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -175,20 +175,20 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 3 Data size: 1428 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 1716 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 476 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 484 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 5 Data size: 2417 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 2617 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) Reducer 8 Execution mode: vectorized @@ -347,74 +347,74 @@ STAGE PLANS: TableScan alias: tbl_ice filterExpr: (a <= 5) (type: boolean) - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (a <= 5) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: a (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Group By Operator aggregations: count(), count(a) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: vectorized Map 26 Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 9 Data size: 891 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 4311 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 5148 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4311 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 5148 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 9 Data size: 891 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4383 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 5220 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 4383 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 5220 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) @@ -483,11 +483,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 4049 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3672 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4049 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3672 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) Reducer 11 Reduce Operator Tree: @@ -498,13 +498,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 9 Data size: 4130 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3744 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 9 Data size: 4130 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3744 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) Reducer 12 Reduce Operator Tree: @@ -515,20 +515,20 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 9 Data size: 4543 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3780 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 4543 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3780 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'Changed again' (type: string), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4543 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 4437 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 19 Data size: 11116 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 9077 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) Reducer 13 Reduce Operator Tree: @@ -539,11 +539,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 4821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5256 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5256 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) Reducer 14 Reduce Operator Tree: @@ -554,17 +554,17 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 - Statistics: Num rows: 9 Data size: 5010 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5400 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 - Statistics: Num rows: 9 Data size: 5010 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5400 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 5010 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5400 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean) Reducer 15 Reduce Operator Tree: @@ -575,11 +575,11 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13 - Statistics: Num rows: 9 Data size: 5511 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5436 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 5511 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5436 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col13 (type: boolean) Reducer 16 Reduce Operator Tree: @@ -590,24 +590,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13, _col14, _col15 - Statistics: Num rows: 9 Data size: 5664 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5580 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col14 (type: bigint), _col15 (type: bigint), _col13 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col13, _col15 - Statistics: Num rows: 9 Data size: 5664 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5580 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null) or (_col15 is not null and (_col12 <> 0L)) or ((_col2 is null or (_col13 < _col12)) and null and (_col12 <> 0L) and _col15 is null)) is null or (((_col8 = 0L) or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) and ((_col12 = 0L) or (_col15 is null and (_col13 >= _col12) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 6 Data size: 3776 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5580 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 3776 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5220 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 3776 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5220 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) Reducer 17 Reduce Operator Tree: @@ -618,11 +618,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 4870 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5256 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4870 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5256 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) Reducer 18 Reduce Operator Tree: @@ -633,13 +633,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 9 Data size: 4951 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5328 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 4951 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5328 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) Reducer 19 Reduce Operator Tree: @@ -650,16 +650,16 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5364 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5364 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5364 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) Reducer 2 Execution mode: vectorized @@ -668,38 +668,38 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Reducer 20 Execution mode: vectorized @@ -707,7 +707,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5148 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -726,20 +726,20 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5148 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 2420 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 2288 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 2420 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 2320 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 19 Data size: 11116 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 9077 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) Reducer 21 Execution mode: vectorized @@ -748,21 +748,21 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Reducer 22 Reduce Operator Tree: @@ -773,13 +773,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 9 Data size: 1845 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1800 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1845 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1800 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint) Reducer 23 Reduce Operator Tree: @@ -790,13 +790,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4428 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5220 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 4428 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5220 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint) Reducer 24 Reduce Operator Tree: @@ -807,13 +807,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 3681 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3636 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 3681 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3636 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint) Reducer 25 Execution mode: vectorized @@ -822,11 +822,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 27 Execution mode: vectorized @@ -913,11 +913,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1, _col2, _col3, _col5 - Statistics: Num rows: 9 Data size: 2029 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1800 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 2029 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1800 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean) Reducer 4 Reduce Operator Tree: @@ -928,13 +928,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col1, _col2, _col3, _col5, _col6 - Statistics: Num rows: 9 Data size: 2110 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1872 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 9 Data size: 2110 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1872 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean), _col6 (type: bigint) Reducer 5 Reduce Operator Tree: @@ -945,23 +945,23 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col2, _col3, _col5, _col6, _col8 - Statistics: Num rows: 9 Data size: 2321 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1872 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (((_col3 <> 0L) and _col5 is not null) or ((_col6 <> 0L) and _col8 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 2321 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1872 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col2 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 9 Data size: 2321 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1872 Basic stats: COMPLETE Column stats: PARTIAL Reducer 6 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col2 - Statistics: Num rows: 9 Data size: 2321 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -981,26 +981,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 2321 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 1031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 1031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 1031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 1031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Reducer 7 Reduce Operator Tree: Merge Join Operator @@ -1010,17 +1010,17 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 4153 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 2320 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 4153 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 2320 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 19 Data size: 11116 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 9077 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) Reducer 9 Execution mode: vectorized @@ -1031,7 +1031,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 19 Data size: 11116 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 9077 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1172,69 +1172,69 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 65358 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5220 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5220 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3564 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3564 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5148 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5148 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized Reducer 10 @@ -1244,11 +1244,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 11 Reduce Operator Tree: @@ -1259,20 +1259,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Reducer 12 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1292,26 +1292,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Reducer 13 Reduce Operator Tree: Merge Join Operator @@ -1321,13 +1321,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5148 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5148 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) Reducer 14 Execution mode: vectorized @@ -1335,7 +1335,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5148 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1354,20 +1354,20 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5148 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 2288 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 2320 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 9095 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) Reducer 2 Reduce Operator Tree: @@ -1378,60 +1378,60 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8888889 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(_col0) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8888889 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8888889 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8888889 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1441,17 +1441,17 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3564 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'Changed forever' (type: string), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 4455 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 9095 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) Reducer 5 Execution mode: vectorized @@ -1462,7 +1462,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 9095 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1475,17 +1475,17 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Reducer 7 Reduce Operator Tree: @@ -1496,11 +1496,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5256 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5256 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) Reducer 8 Reduce Operator Tree: @@ -1511,24 +1511,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5400 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5400 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5400 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5220 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5220 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) Reducer 9 Reduce Operator Tree: @@ -1539,17 +1539,17 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 2320 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 2320 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 9095 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) Union 4 Vertex: Union 4 @@ -1683,69 +1683,69 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 66198 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5220 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5220 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3564 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3564 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5148 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5148 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized Reducer 10 @@ -1755,11 +1755,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 11 Reduce Operator Tree: @@ -1770,20 +1770,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Reducer 12 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1803,26 +1803,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Reducer 13 Reduce Operator Tree: Merge Join Operator @@ -1832,13 +1832,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5148 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5148 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) Reducer 14 Execution mode: vectorized @@ -1846,7 +1846,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5148 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1865,20 +1865,20 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5148 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 2288 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 2320 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 9068 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) Reducer 2 Reduce Operator Tree: @@ -1889,60 +1889,60 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8888889 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(_col0) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8888889 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8888889 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8888889 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1952,17 +1952,17 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3564 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'The last one' (type: string), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 4428 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 9068 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) Reducer 5 Execution mode: vectorized @@ -1973,7 +1973,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 9068 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1986,17 +1986,17 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Reducer 7 Reduce Operator Tree: @@ -2007,11 +2007,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5256 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5256 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) Reducer 8 Reduce Operator Tree: @@ -2022,24 +2022,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5400 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5400 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5400 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5220 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 5220 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) Reducer 9 Reduce Operator Tree: @@ -2050,17 +2050,17 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 2320 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 2320 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 9068 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) Union 4 Vertex: Union 4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_orc.q.out index 66273d7216bb..0a245752e303 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_orc.q.out @@ -53,8 +53,8 @@ POSTHOOK: query: insert into tbl_ice values (444, 'hola', 800), (555, 'schola', POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tbl_ice -Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[68][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[64][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product PREHOOK: query: update tbl_ice set b='Changed again' where a in (select a from tbl_ice where a <= 5) or c in (select c from tbl_ice where c > 800) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_parquet.q.out index 3e6f66ad196b..fe5345d98e33 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_parquet.q.out @@ -53,8 +53,8 @@ POSTHOOK: query: insert into tbl_ice values (444, 'hola', 800), (555, 'schola', POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tbl_ice -Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[68][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[64][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product PREHOOK: query: update tbl_ice set b='Changed again' where a in (select a from tbl_ice where a <= 5) or c in (select c from tbl_ice where c > 800) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out index 6965a3604c45..7ed3c1e9ded1 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out @@ -325,17 +325,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 5 (CUSTOM_EDGE) + Map 1 <- Map 3 (CUSTOM_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s - Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL TableScan Vectorization: native: true vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk2:int, 3:ss_customer_sk2:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__PROJECTION:string] @@ -347,7 +345,7 @@ STAGE PLANS: native: true projectedOutputColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 32, 33] selectExpressions: LongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: DoubleColDivideDoubleScalar(col 28:double, val 1000.0)(children: CastLongToDouble(col 2:int) -> 28:double) -> 29:double) -> 30:bigint) -> 31:bigint) -> 32:boolean, DecimalColLessDecimalScalar(col 14:decimal(7,2), val 0) -> 33:boolean - Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Left Outer Join 0 to 1 @@ -371,8 +369,8 @@ STAGE PLANS: hashTableImplementationType: OPTIMIZED outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50, _col51 input vertices: - 1 Map 5 - Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + 1 Map 3 + Statistics: Num rows: 7 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col40 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col33 (type: int), _col39 (type: int), _col24 (type: int), _col21 (type: decimal(7,2)), _col26 (type: string), _col7 (type: int), _col3 (type: int), _col10 (type: decimal(7,2)), _col49 (type: decimal(7,2)), _col38 (type: int), _col28 (type: string), _col6 (type: int), _col50 (type: decimal(7,2)), _col11 (type: decimal(7,2)), _col34 (type: int), _col17 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col45 (type: decimal(7,2)), _col47 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col8 (type: int), _col35 (type: int), _col31 (type: int), _col41 (type: decimal(7,2)), _col36 (type: int), _col4 (type: int), _col48 (type: decimal(7,2)), _col5 (type: int), _col13 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col44 (type: decimal(7,2)), _col43 (type: decimal(7,2)), _col1 (type: int), _col16 (type: decimal(7,2)), _col29 (type: int), _col2 (type: int), _col15 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col25 (type: bigint), _col9 (type: int), _col30 (type: int), _col42 (type: decimal(7,2)), _col0 (type: int), _col27 (type: bigint), _col51 (type: decimal(7,2)), _col32 (type: int), _col37 (type: int), _col46 (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49 @@ -380,14 +378,14 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [50, 20, 43, 49, 34, 22, 36, 8, 4, 11, 59, 48, 38, 7, 60, 12, 44, 18, 15, 55, 57, 21, 9, 45, 41, 51, 46, 5, 58, 6, 14, 13, 54, 53, 2, 17, 39, 3, 16, 19, 35, 10, 40, 52, 1, 37, 61, 42, 47, 56] - Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL Filter Operator Filter Vectorization: className: VectorFilterOperator native: true predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 41:int, col 2:int), FilterLongColEqualLongColumn(col 42:int, col 3:int), FilterLongColEqualLongScalar(col 39:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 53:decimal(7,2))) predicate: ((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null) (type: boolean) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col4 (type: int), _col40 (type: bigint), _col6 (type: string), _col45 (type: bigint), _col12 (type: string), 2451181 (type: int), _col42 (type: int), _col24 (type: int), _col47 (type: int), _col2 (type: int), _col16 (type: int), _col23 (type: int), _col26 (type: int), _col48 (type: int), _col11 (type: int), _col3 (type: int), _col0 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col43 (type: decimal(7,2)), null (type: decimal(7,2)), _col32 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col49 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col28 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col46 (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27 @@ -396,13 +394,13 @@ STAGE PLANS: native: true projectedOutputColumnNums: [34, 35, 36, 37, 38, 30, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 62, 54, 55, 56, 57, 58, 59, 60, 61] selectExpressions: ConstantVectorExpression(val 2451181) -> 30:int, ConstantVectorExpression(val null) -> 62:decimal(7,2) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -414,7 +412,7 @@ STAGE PLANS: native: true predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 41:int, col 2:int), FilterLongColEqualLongColumn(col 42:int, col 3:int), FilterLongColEqualLongScalar(col 39:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 53:decimal(7,2))) predicate: ((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null) (type: boolean) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: 2451181 (type: int), _col42 (type: int), _col24 (type: int), _col47 (type: int), _col2 (type: int), _col16 (type: int), _col23 (type: int), _col26 (type: int), _col48 (type: int), _col11 (type: int), _col3 (type: int), _col0 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col43 (type: decimal(7,2)), 0 (type: decimal(7,2)), _col32 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col49 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col28 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col46 (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22 @@ -423,28 +421,25 @@ STAGE PLANS: native: true projectedOutputColumnNums: [31, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 63, 54, 55, 56, 57, 58, 59, 60, 61] selectExpressions: ConstantVectorExpression(val 2451181) -> 31:int, ConstantVectorExpression(val 0) -> 63:decimal(7,2) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkMultiKeyOperator - keyColumns: 42:int, 65:int - keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 65:int - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 31:int, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:decimal(7,2), 51:decimal(7,2), 52:decimal(7,2), 63:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)) + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.store_sales Filter Operator Filter Vectorization: className: VectorFilterOperator native: true predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 41:int), SelectColumnIsNull(col 42:int), SelectColumnIsNull(col 39:int)) predicate: (_col24 is null and _col47 is null and _col36 is null) (type: boolean) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: 2451181 (type: int), _col44 (type: int), _col34 (type: int), _col37 (type: int), _col8 (type: int), _col27 (type: int), _col29 (type: int), _col13 (type: int), _col7 (type: int), _col22 (type: int), _col41 (type: int), _col9 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col31 (type: decimal(7,2)), _col30 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col38 (type: decimal(7,2)), _col35 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col39 (type: decimal(7,2)), _col1 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col5 (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22 @@ -453,28 +448,25 @@ STAGE PLANS: native: true projectedOutputColumnNums: [64, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] selectExpressions: ConstantVectorExpression(val 2451181) -> 64:int - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkMultiKeyOperator - keyColumns: 3:int, 66:int - keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 66:int - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 64:int, 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)) + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.store_sales Filter Operator Filter Vectorization: className: VectorFilterOperator native: true predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 41:int, col 2:int), FilterLongColEqualLongColumn(col 42:int, col 3:int), FilterLongColEqualLongScalar(col 39:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0)) predicate: ((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0)) (type: boolean) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col4 (type: int), _col6 (type: string), _col40 (type: bigint), _col45 (type: bigint) outputColumnNames: _col4, _col6, _col40, _col45 @@ -482,7 +474,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [34, 36, 35, 37] - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() Group By Vectorization: @@ -497,7 +489,7 @@ STAGE PLANS: minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) null sort order: zzzz @@ -509,7 +501,7 @@ STAGE PLANS: native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true valueColumns: 4:bigint - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col4 (type: bigint) Execution mode: vectorized Map Vectorization: @@ -519,20 +511,20 @@ STAGE PLANS: featureSupportInUse: [] inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 23 includeColumns: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] dataColumns: ss_sold_date_sk:int, ss_sold_time_sk:int, ss_item_sk2:int, ss_customer_sk2:int, ss_cdemo_sk:int, ss_hdemo_sk:int, ss_addr_sk:int, ss_store_sk:int, ss_promo_sk:int, ss_ticket_number:int, ss_quantity:int, ss_wholesale_cost:decimal(7,2), ss_list_price:decimal(7,2), ss_sales_price:decimal(7,2), ss_ext_discount_amt:decimal(7,2), ss_ext_sales_price:decimal(7,2), ss_ext_wholesale_cost:decimal(7,2), ss_ext_list_price:decimal(7,2), ss_ext_tax:decimal(7,2), ss_coupon_amt:decimal(7,2), ss_net_paid:decimal(7,2), ss_net_paid_inc_tax:decimal(7,2), ss_net_profit:decimal(7,2) partitionColumnCount: 0 - scratchColumnTypeNames: [double, double, bigint, bigint, bigint, bigint, bigint, bigint, string, bigint, string, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), bigint, bigint, bigint] - Map 5 + scratchColumnTypeNames: [double, double, bigint, bigint, bigint, bigint, bigint, bigint, string, bigint, string, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), bigint] + Map 3 Map Operator Tree: TableScan alias: store_sales filterExpr: ((ss_sold_date_sk = 2451181) and ss_item_sk is not null and ss_customer_sk is not null) (type: boolean) - Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL TableScan Vectorization: native: true vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk:int, 3:ss_customer_sk:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__PROJECTION:string] @@ -542,7 +534,7 @@ STAGE PLANS: native: true predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongScalar(col 0:int, val 2451181), SelectColumnIsNotNull(col 2:int), SelectColumnIsNotNull(col 3:int)) predicate: ((ss_sold_date_sk = 2451181) and ss_item_sk is not null and ss_customer_sk is not null) (type: boolean) - Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 2451181 (type: int), ss_sold_time_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_cdemo_sk (type: int), ss_hdemo_sk (type: int), ss_addr_sk (type: int), ss_store_sk (type: int), ss_promo_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_list_price (type: decimal(7,2)), ss_sales_price (type: decimal(7,2)), ss_ext_discount_amt (type: decimal(7,2)), ss_ext_sales_price (type: decimal(7,2)), ss_ext_wholesale_cost (type: decimal(7,2)), ss_ext_list_price (type: decimal(7,2)), ss_ext_tax (type: decimal(7,2)), ss_coupon_amt (type: decimal(7,2)), ss_net_paid (type: decimal(7,2)), ss_net_paid_inc_tax (type: decimal(7,2)), ss_net_profit (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27 @@ -551,7 +543,7 @@ STAGE PLANS: native: true projectedOutputColumnNums: [23, 24, 25, 26, 27, 28, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] selectExpressions: ConstantVectorExpression(val 2451181) -> 28:int - Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col8 (type: int), _col7 (type: int) null sort order: zz @@ -564,7 +556,7 @@ STAGE PLANS: nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true partitionColumns: 2:int valueColumns: 23:int, 24:bigint, 25:string, 26:bigint, 27:string, 28:int, 1:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) - Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: int), _col12 (type: int), _col13 (type: int), _col14 (type: int), _col15 (type: int), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)), _col23 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col26 (type: decimal(7,2)), _col27 (type: decimal(7,2)) Execution mode: vectorized Map Vectorization: @@ -584,76 +576,6 @@ STAGE PLANS: partitionColumnCount: 0 scratchColumnTypeNames: [bigint] Reducer 2 - Execution mode: vectorized - Reduce Vectorization: - enabled: true - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true - reduceColumnNullOrder: zz - reduceColumnSortOrder: ++ - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 25 - dataColumns: KEY._col3:int, KEY.iceberg_bucket(_col2, 3):int, VALUE._col0:int, VALUE._col1:int, VALUE._col2:int, VALUE._col3:int, VALUE._col4:int, VALUE._col5:int, VALUE._col6:int, VALUE._col7:int, VALUE._col8:int, VALUE._col9:int, VALUE._col10:int, VALUE._col11:decimal(7,2)/DECIMAL_64, VALUE._col12:decimal(7,2)/DECIMAL_64, VALUE._col13:decimal(7,2)/DECIMAL_64, VALUE._col14:decimal(7,2)/DECIMAL_64, VALUE._col15:decimal(7,2)/DECIMAL_64, VALUE._col16:decimal(7,2)/DECIMAL_64, VALUE._col17:decimal(7,2)/DECIMAL_64, VALUE._col18:decimal(7,2)/DECIMAL_64, VALUE._col19:decimal(7,2)/DECIMAL_64, VALUE._col20:decimal(7,2)/DECIMAL_64, VALUE._col21:decimal(7,2)/DECIMAL_64, VALUE._col22:decimal(7,2)/DECIMAL_64 - partitionColumnCount: 0 - scratchColumnTypeNames: [] - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: int), VALUE._col5 (type: int), VALUE._col6 (type: int), VALUE._col7 (type: int), VALUE._col8 (type: int), VALUE._col9 (type: int), VALUE._col10 (type: int), VALUE._col11 (type: decimal(7,2)), VALUE._col12 (type: decimal(7,2)), VALUE._col13 (type: decimal(7,2)), VALUE._col14 (type: decimal(7,2)), VALUE._col15 (type: decimal(7,2)), VALUE._col16 (type: decimal(7,2)), VALUE._col17 (type: decimal(7,2)), VALUE._col18 (type: decimal(7,2)), VALUE._col19 (type: decimal(7,2)), VALUE._col20 (type: decimal(7,2)), VALUE._col21 (type: decimal(7,2)), VALUE._col22 (type: decimal(7,2)), KEY._col3 (type: int), KEY.iceberg_bucket(_col2, 3) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col3, iceberg_bucket(_col2, 3) - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 0, 1] - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.store_sales - Reducer 3 - Execution mode: vectorized - Reduce Vectorization: - enabled: true - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true - reduceColumnNullOrder: zz - reduceColumnSortOrder: ++ - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 25 - dataColumns: KEY._col3:int, KEY.iceberg_bucket(_col2, 3):int, VALUE._col0:int, VALUE._col1:int, VALUE._col2:int, VALUE._col3:int, VALUE._col4:int, VALUE._col5:int, VALUE._col6:int, VALUE._col7:int, VALUE._col8:int, VALUE._col9:int, VALUE._col10:int, VALUE._col11:decimal(7,2)/DECIMAL_64, VALUE._col12:decimal(7,2)/DECIMAL_64, VALUE._col13:decimal(7,2)/DECIMAL_64, VALUE._col14:decimal(7,2)/DECIMAL_64, VALUE._col15:decimal(7,2)/DECIMAL_64, VALUE._col16:decimal(7,2)/DECIMAL_64, VALUE._col17:decimal(7,2)/DECIMAL_64, VALUE._col18:decimal(7,2)/DECIMAL_64, VALUE._col19:decimal(7,2)/DECIMAL_64, VALUE._col20:decimal(7,2)/DECIMAL_64, VALUE._col21:decimal(7,2)/DECIMAL_64, VALUE._col22:decimal(7,2)/DECIMAL_64 - partitionColumnCount: 0 - scratchColumnTypeNames: [] - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: int), VALUE._col5 (type: int), VALUE._col6 (type: int), VALUE._col7 (type: int), VALUE._col8 (type: int), VALUE._col9 (type: int), VALUE._col10 (type: int), VALUE._col11 (type: decimal(7,2)), VALUE._col12 (type: decimal(7,2)), VALUE._col13 (type: decimal(7,2)), VALUE._col14 (type: decimal(7,2)), VALUE._col15 (type: decimal(7,2)), VALUE._col16 (type: decimal(7,2)), VALUE._col17 (type: decimal(7,2)), VALUE._col18 (type: decimal(7,2)), VALUE._col19 (type: decimal(7,2)), VALUE._col20 (type: decimal(7,2)), VALUE._col21 (type: decimal(7,2)), VALUE._col22 (type: decimal(7,2)), KEY._col3 (type: int), KEY.iceberg_bucket(_col2, 3) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col3, iceberg_bucket(_col2, 3) - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 0, 1] - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.store_sales - Reducer 4 Execution mode: vectorized Reduce Vectorization: enabled: true @@ -682,14 +604,14 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL Filter Operator Filter Vectorization: className: VectorFilterOperator native: true predicateExpression: FilterLongColGreaterLongScalar(col 4:bigint, val 1) predicate: (_col4 > 1L) (type: boolean) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: cardinality_violation(_col0,_col1,_col2,_col3) (type: int) outputColumnNames: _col0 @@ -698,13 +620,13 @@ STAGE PLANS: native: true projectedOutputColumnNums: [5] selectExpressions: VectorUDFAdaptor(cardinality_violation(_col0,_col1,_col2,_col3)) -> 5:int - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -885,10 +807,8 @@ POSTHOOK: Output: default@merge_tmp_table POSTHOOK: Output: default@store_sales POSTHOOK: Output: default@store_sales Vertex dependency in root stage -Map 1 <- Map 5 (CUSTOM_EDGE) +Map 1 <- Map 3 (CUSTOM_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) -Reducer 4 <- Map 1 (SIMPLE_EDGE) Stage-6 Stats Work{} @@ -899,66 +819,61 @@ Stage-6 Dependency Collection{} Stage-4 Reducer 2 vectorized - File Output Operator [FS_59] - table:{"name:":"default.store_sales"} - Select Operator [SEL_58] - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col3","iceberg_bucket(_col2, 3)"] - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_54] - PartitionCols:_col3, iceberg_bucket(_col2, 3) - Select Operator [SEL_50] (rows=1 width=#Masked#) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] - Filter Operator [FIL_46] (rows=1 width=#Masked#) - predicate:((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null) - Select Operator [SEL_44] (rows=5 width=#Masked#) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49"] - Map Join Operator [MAPJOIN_43] (rows=5 width=#Masked#) - BucketMapJoin:true,Conds:SEL_42._col2, _col1=RS_41._col8, _col7(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51"] - <-Map 5 [CUSTOM_EDGE] vectorized - MULTICAST [RS_41] - PartitionCols:_col7 - Select Operator [SEL_40] (rows=2 width=#Masked#) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27"] - Filter Operator [FIL_39] (rows=2 width=#Masked#) - predicate:((ss_sold_date_sk = 2451181) and ss_item_sk is not null and ss_customer_sk is not null) - TableScan [TS_2] (rows=2 width=#Masked#) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_sold_time_sk","ss_item_sk","ss_customer_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_list_price","ss_sales_price","ss_ext_discount_amt","ss_ext_sales_price","ss_ext_wholesale_cost","ss_ext_list_price","ss_ext_tax","ss_coupon_amt","ss_net_paid","ss_net_paid_inc_tax","ss_net_profit"] - <-Select Operator [SEL_42] (rows=5 width=#Masked#) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - TableScan [TS_0] (rows=5 width=#Masked#) - default@ssv,s,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:3,Grouping Partition Columns:["ss_item_sk2"],Output:["ss_sold_time_sk","ss_item_sk2","ss_customer_sk2","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_list_price","ss_sales_price","ss_ext_discount_amt","ss_ext_sales_price","ss_ext_wholesale_cost","ss_ext_list_price","ss_ext_tax","ss_coupon_amt","ss_net_paid","ss_net_paid_inc_tax","ss_net_profit"] - Reducer 3 vectorized - File Output Operator [FS_61] - table:{"name:":"default.store_sales"} - Select Operator [SEL_60] - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col3","iceberg_bucket(_col2, 3)"] - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_55] - PartitionCols:_col3, iceberg_bucket(_col2, 3) - Select Operator [SEL_51] (rows=1 width=#Masked#) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] - Filter Operator [FIL_47] (rows=1 width=#Masked#) - predicate:(_col24 is null and _col47 is null and _col36 is null) - Please refer to the previous Select Operator [SEL_44] - Reducer 4 vectorized - File Output Operator [FS_65] + File Output Operator [FS_57] table:{"name:":"default.merge_tmp_table"} - Select Operator [SEL_64] (rows=1 width=#Masked#) + Select Operator [SEL_56] (rows=1 width=#Masked#) Output:["_col0"] - Filter Operator [FIL_63] (rows=1 width=#Masked#) + Filter Operator [FIL_55] (rows=1 width=#Masked#) predicate:(_col4 > 1L) - Group By Operator [GBY_62] (rows=1 width=#Masked#) + Group By Operator [GBY_54] (rows=1 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_57] + File Output Operator [FS_49] + table:{"name:":"default.store_sales"} + Select Operator [SEL_45] (rows=1 width=#Masked#) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27"] + Filter Operator [FIL_41] (rows=1 width=#Masked#) + predicate:((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null) + Select Operator [SEL_40] (rows=7 width=#Masked#) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49"] + Map Join Operator [MAPJOIN_39] (rows=7 width=#Masked#) + BucketMapJoin:true,Conds:SEL_38._col2, _col1=RS_37._col8, _col7(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51"] + <-Map 3 [CUSTOM_EDGE] vectorized + MULTICAST [RS_37] + PartitionCols:_col7 + Select Operator [SEL_36] (rows=2 width=#Masked#) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27"] + Filter Operator [FIL_35] (rows=2 width=#Masked#) + predicate:((ss_sold_date_sk = 2451181) and ss_item_sk is not null and ss_customer_sk is not null) + TableScan [TS_2] (rows=2 width=#Masked#) + default@store_sales,store_sales,Tbl:COMPLETE,Col:PARTIAL,Output:["ss_sold_date_sk","ss_sold_time_sk","ss_item_sk","ss_customer_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_list_price","ss_sales_price","ss_ext_discount_amt","ss_ext_sales_price","ss_ext_wholesale_cost","ss_ext_list_price","ss_ext_tax","ss_coupon_amt","ss_net_paid","ss_net_paid_inc_tax","ss_net_profit"] + <-Select Operator [SEL_38] (rows=5 width=#Masked#) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] + TableScan [TS_0] (rows=5 width=#Masked#) + default@ssv,s,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:3,Grouping Partition Columns:["ss_item_sk2"],Output:["ss_sold_time_sk","ss_item_sk2","ss_customer_sk2","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_list_price","ss_sales_price","ss_ext_discount_amt","ss_ext_sales_price","ss_ext_wholesale_cost","ss_ext_list_price","ss_ext_tax","ss_coupon_amt","ss_net_paid","ss_net_paid_inc_tax","ss_net_profit"] + File Output Operator [FS_50] + table:{"name:":"default.store_sales"} + Select Operator [SEL_46] (rows=1 width=#Masked#) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] + Filter Operator [FIL_42] (rows=1 width=#Masked#) + predicate:((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null) + Please refer to the previous Select Operator [SEL_40] + File Output Operator [FS_51] + table:{"name:":"default.store_sales"} + Select Operator [SEL_47] (rows=3 width=#Masked#) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] + Filter Operator [FIL_43] (rows=3 width=#Masked#) + predicate:(_col24 is null and _col47 is null and _col36 is null) + Please refer to the previous Select Operator [SEL_40] + SHUFFLE [RS_53] PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_56] (rows=1 width=#Masked#) + Group By Operator [GBY_52] (rows=1 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["count()"],keys:_col4, _col40, _col6, _col45 - Select Operator [SEL_52] (rows=1 width=#Masked#) + Select Operator [SEL_48] (rows=1 width=#Masked#) Output:["_col4","_col6","_col40","_col45"] - Filter Operator [FIL_48] (rows=1 width=#Masked#) + Filter Operator [FIL_44] (rows=1 width=#Masked#) predicate:((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0)) - Please refer to the previous Select Operator [SEL_44] + Please refer to the previous Select Operator [SEL_40] Stage-7 Stats Work{} Stage-3 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java index b1b8003e21fd..9d918447a80b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java @@ -69,8 +69,10 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FunctionInfo; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveMultiJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; @@ -836,17 +838,14 @@ public Void visitInputRef(RexInputRef inputRef) { return deterministicFuncWithSingleInputRef; } - public static ImmutableMap getColInfoMap(List hiveCols, - int startIndx) { - Builder bldr = ImmutableMap. builder(); + public static ImmutableMap getColInfoMap(List hiveCols, Table table) { + Builder builder = ImmutableMap.builder(); - int indx = startIndx; - for (T ci : hiveCols) { - bldr.put(indx, ci); - indx++; + for (ColumnInfo ci : hiveCols) { + builder.put(table.getColumnIndexByName(ci.getInternalName()), ci); } - return bldr.build(); + return builder.build(); } public static ImmutableSet shiftVColsSet(Set hiveVCols, int shift) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java index f44d89671955..5f26b492b521 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java @@ -128,9 +128,9 @@ public RelOptHiveTable(RelOptSchema calciteSchema, RelDataTypeFactory typeFactor this.hiveTblMetadata = hiveTblMetadata; this.hiveColStatsMap = new HashMap<>(); this.hiveNonPartitionCols = ImmutableList.copyOf(hiveNonPartitionCols); - this.hiveNonPartitionColsMap = HiveCalciteUtil.getColInfoMap(hiveNonPartitionCols, 0); + this.hiveNonPartitionColsMap = HiveCalciteUtil.getColInfoMap(hiveNonPartitionCols, hiveTblMetadata); this.hivePartitionCols = ImmutableList.copyOf(hivePartitionCols); - this.hivePartitionColsMap = HiveCalciteUtil.getColInfoMap(hivePartitionCols, hiveNonPartitionColsMap.size()); + this.hivePartitionColsMap = HiveCalciteUtil.getColInfoMap(hivePartitionCols, hiveTblMetadata); this.noOfNonVirtualCols = hiveNonPartitionCols.size() + hivePartitionCols.size(); this.hiveVirtualCols = ImmutableList.copyOf(hiveVirtualCols); this.hiveConf = hconf; @@ -666,6 +666,8 @@ private void updateColStats(Set projIndxLst, boolean allowMissingStats) LOG.error(logMsg); throw new RuntimeException(logMsg); } + } else { + colStatsCached.updateState(State.COMPLETE); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewUtils.java index 0fe21dfdae1a..14fa09b7518a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewUtils.java @@ -548,6 +548,9 @@ public static RelOptMaterialization createCTEMaterialization(String viewName, Re throw new CalciteCteException("Failed to create temporary location", e); } Table hiveTable = new Table(metaTable); + hiveTable.setFields(columns.stream() + .map(columnInfo -> new FieldSchema(columnInfo.getInternalName(), columnInfo.getTypeName(), null)) + .toList()); hiveTable.setMaterializedTable(true); RelOptHiveTable optTable = new RelOptHiveTable(null, cluster.getTypeFactory(), fullName, body.getRowType(), hiveTable, columns, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 25489f477f55..aafb4d504467 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -27,6 +27,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Multimap; +import java.util.LinkedHashSet; import java.util.Map.Entry; import java.util.Optional; import java.util.function.Function; @@ -2951,7 +2952,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc // NOTE: Table logical schema = Non Partition Cols + Partition Cols + // Virtual Cols - // 3.1 Add Column info for non partion cols (Object Inspector fields) + // 3.1 Add Column info for cols (Object Inspector fields) final Deserializer deserializer = tabMetaData.getDeserializer(); StructObjectInspector rowObjectInspector = (StructObjectInspector) deserializer .getObjectInspector(); @@ -2959,42 +2960,49 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc deserializer.handleJobLevelConfiguration(conf); List fields = rowObjectInspector.getAllStructFieldRefs(); - ColumnInfo colInfo; - String colName; - ArrayList cInfoLst = new ArrayList<>(); - final NotNullConstraint nnc = tabMetaData.getNotNullConstraint(); final PrimaryKeyInfo pkc = tabMetaData.getPrimaryKeyInfo(); + int allColCount = tabMetaData.getAllCols().size(); + List colInfoList = new ArrayList<>(Collections.nCopies(allColCount, null)); + Set partColNames = new HashSet<>(tabMetaData.getPartColNames()); + List nonPartitionColumns = new ArrayList<>(fields.size()); + for (StructField structField : fields) { - colName = structField.getFieldName(); - colInfo = new ColumnInfo( + String colName = structField.getFieldName(); + if (partColNames.contains(colName)) { + continue; + } + + ColumnInfo colInfo = new ColumnInfo( structField.getFieldName(), TypeInfoUtils.getTypeInfoFromObjectInspector(structField.getFieldObjectInspector()), isNullable(colName, nnc, pkc), tableAlias, false); colInfo.setSkewedCol(isSkewedCol(tableAlias, qb, colName)); - rr.put(tableAlias, colName, colInfo); - cInfoLst.add(colInfo); + colInfoList.set(tabMetaData.getColumnIndexByName(colName), colInfo); + nonPartitionColumns.add(colInfo); } - // TODO: Fix this - ArrayList nonPartitionColumns = new ArrayList(cInfoLst); - ArrayList partitionColumns = new ArrayList(); // 3.2 Add column info corresponding to partition columns + // Normally, the column names in a schema should be unique, but in the case of Iceberg v1 tables, + // updating the partition spec doesn't remove the existing partition keys, so we can end up with a + // partition spec containing multiple columns with the same name. + Set partitionColumnSet = LinkedHashSet.newLinkedHashSet(tabMetaData.getPartCols().size()); + for (FieldSchema partCol : tabMetaData.getPartCols()) { - if (tabMetaData.hasNonNativePartitionSupport()) { - break; - } - colName = partCol.getName(); - colInfo = new ColumnInfo(colName, - TypeInfoFactory.getPrimitiveTypeInfo(partCol.getType()), - isNullable(colName, nnc, pkc), tableAlias, true); - rr.put(tableAlias, colName, colInfo); - cInfoLst.add(colInfo); - partitionColumns.add(colInfo); + String colName = partCol.getName(); + ColumnInfo colInfo = new ColumnInfo(colName, + TypeInfoFactory.getPrimitiveTypeInfo(partCol.getType()), + isNullable(colName, nnc, pkc), tableAlias, true); + colInfoList.set(tabMetaData.getColumnIndexByName(colName), colInfo); + partitionColumnSet.add(colInfo); } - final TableType tableType = obtainTableType(tabMetaData); + List partitionColumns = List.copyOf(partitionColumnSet); + + for (ColumnInfo colInfo : colInfoList) { + rr.put(tableAlias, colInfo.getInternalName(), colInfo); + } // 3.3 Add column info corresponding to virtual columns List virtualCols = tabMetaData.getVirtualColumns(); @@ -3007,6 +3015,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc ); // 4. Build operator + final TableType tableType = obtainTableType(tabMetaData); Map tabPropsFromQuery = qb.getTabPropsForAlias(tableAlias); HiveTableScan.HiveTableScanTrait tableScanTrait = HiveTableScan.HiveTableScanTrait.from(tabPropsFromQuery); RelOptHiveTable optTable;