[opt](build) Enable unity builds for the remaining BE targets

be/CMakeLists.txt

@@ -409,7 +409,14 @@ if (COMPILER_CLANG)
                         -Wthread-safety)
     add_compile_options(-Wno-gnu-statement-expression
                         -Wno-implicit-float-conversion
-                        -Wno-sign-conversion)
+                        -Wno-sign-conversion
+                        # libstdc++-15 puts `#pragma GCC unroll` on loops in its own
+                        # headers (bits/stl_algobase.h); when a sanitizer/coverage
+                        # build compiles at low optimization the unroller punts and
+                        # clang's -Wpass-failed turns that missed *hint* into an
+                        # -Werror failure. The diagnostic carries no correctness
+                        # signal, so drop it.
+                        -Wno-pass-failed)
     if (USE_LIBCPP)
         add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-stdlib=libc++>)
         add_definitions(-DUSE_LIBCPP)
@@ -981,6 +988,24 @@ function(pch_reuse target)
     endif()
 endfunction(pch_reuse target)
 
+# Every unity opt-out goes through this helper. set_source_files_properties()
+# silently ignores paths that do not exist, so a skip entry going stale after
+# a rename or move would quietly re-join its unity batch -- slower builds, or
+# new file-scope clashes, with no diagnostic anywhere. Fail the configure
+# instead. Entries under GENSRC_DIR are exempt: they are emitted by
+# add_custom_command at build time (wkt_lex.l.cpp, the generated
+# thrift/protobuf sources) and legitimately do not exist on a fresh configure.
+function(doris_skip_unity_inclusion)
+    foreach(entry IN LISTS ARGN)
+        string(FIND "${entry}" "${GENSRC_DIR}" gensrc_prefix_pos)
+        if(NOT gensrc_prefix_pos EQUAL 0 AND NOT EXISTS "${entry}")
+            message(FATAL_ERROR
+                "unity skip entry does not exist (renamed or moved?): ${entry}")
+        endif()
+    endforeach()
+    set_source_files_properties(${ARGN} PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
+endfunction()
+
 
 add_subdirectory(${SRC_DIR}/agent)
 add_subdirectory(${SRC_DIR}/common)

be/src/agent/CMakeLists.txt

@@ -29,4 +29,9 @@ endif()
 
 add_library(Agent STATIC ${AGENT_SOURCES})
 
-pch_reuse(Agent)
+pch_reuse(Agent)
+
+# Unity build: nine homogeneous task-worker glue TUs sharing the agent header
+# closure; batch 0 merges them into a single unity TU.
+set_target_properties(Agent PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD}
+                                       UNITY_BUILD_BATCH_SIZE 0)

be/src/agent/cgroup_cpu_ctl.h

@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#pragma once
+
 #include <fcntl.h>
 #include <sys/stat.h>
 #include <sys/types.h>

be/src/cloud/CMakeLists.txt

@@ -22,3 +22,15 @@ file(GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS *.cpp)
 add_library(Cloud STATIC ${SRC_FILES})
 
 pch_reuse(Cloud)
+
+# Unity build: homogeneous cloud glue TUs sharing the CloudStorageEngine /
+# meta-service RPC header closure. Batch 12 bounds jumbo-TU size and memory.
+# The two http action TUs stay individual: sibling TUs' file-scope
+# `using namespace ErrorCode` makes the unscoped HttpStatus enum constants
+# (OK, NOT_FOUND, ...) shadow ErrorCode variables under -Wshadow -Werror.
+set(CLOUD_UNITY_SKIP
+    ${CMAKE_CURRENT_SOURCE_DIR}/cloud_compaction_action.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/injection_point_action.cpp)
+doris_skip_unity_inclusion(${CLOUD_UNITY_SKIP})
+set_target_properties(Cloud PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD}
+                                       UNITY_BUILD_BATCH_SIZE 12)

be/src/cloud/cloud_internal_service.cpp

@@ -925,12 +925,7 @@ bvar::Adder<uint64_t> g_file_cache_warm_up_rowset_wait_for_compaction_num(
 bvar::Adder<uint64_t> g_file_cache_warm_up_rowset_wait_for_compaction_timeout_num(
         "file_cache_warm_up_rowset_wait_for_compaction_timeout_num");
 
-// Per-job windowed metrics for target BE
-// bvar::Window enforces MAX_SECONDS_LIMIT = 3600, so the longest window is 1h.
-static constexpr int WINDOW_5M = 300;
-static constexpr int WINDOW_30M = 1800;
-static constexpr int WINDOW_1H = 3600;
-
+// Per-job windowed metrics for target BE (window spans shared via bvar_windowed_adder.h)
 MBvarWindowedAdder g_warmup_ed_finish_segment_num("warmup_ed_finish_segment_num", {"job_id"},
                                                   {WINDOW_5M, WINDOW_30M, WINDOW_1H}, false);
 MBvarWindowedAdder g_warmup_ed_finish_segment_size("warmup_ed_finish_segment_size", {"job_id"},

be/src/cloud/cloud_warm_up_manager.cpp

@@ -118,12 +118,7 @@ bvar::Adder<int64_t> g_file_cache_warm_up_job_num("file_cache_warm_up_job_num");
 bvar::LatencyRecorder g_file_cache_warm_up_rowset_wait_for_compaction_latency(
         "file_cache_warm_up_rowset_wait_for_compaction_latency");
 
-// Per-job windowed metrics for source BE
-// bvar::Window enforces MAX_SECONDS_LIMIT = 3600, so the longest window is 1h.
-static constexpr int WINDOW_5M = 300;
-static constexpr int WINDOW_30M = 1800;
-static constexpr int WINDOW_1H = 3600;
-
+// Per-job windowed metrics for source BE (window spans shared via bvar_windowed_adder.h)
 MBvarWindowedAdder g_warmup_ed_requested_segment_num("warmup_ed_requested_segment_num", {"job_id"},
                                                      {WINDOW_5M, WINDOW_30M, WINDOW_1H}, false);
 MBvarWindowedAdder g_warmup_ed_requested_segment_size("warmup_ed_requested_segment_size",

be/src/common/CMakeLists.txt

@@ -23,6 +23,22 @@ add_library(Common STATIC ${SRC_FILES})
 
 pch_reuse(Common)
 
+# Unity build for the small common TUs. Seven files leak file-scope macros
+# (SET_FIELD/UPDATE_FIELD, MEMORY_SANITIZER, DW_* tables, the DEFINE_*_METRIC
+# families, USE_PHDR_CACHE/UNW_LOCAL_ONLY) and stay individual -- that set
+# also keeps the three same-name metric-hook statics (_s_hook_name) apart.
+set(COMMON_UNITY_SKIP
+    ${CMAKE_CURRENT_SOURCE_DIR}/config.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/demangle.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/dwarf.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/metrics/doris_metrics.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/metrics/jvm_metrics.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/metrics/system_metrics.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/phdr_cache.cpp)
+doris_skip_unity_inclusion(${COMMON_UNITY_SKIP})
+set_target_properties(Common PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD}
+                                        UNITY_BUILD_BATCH_SIZE 0)
+
 # Generate env_config.h according to env_config.h.in
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/env_config.h.in ${GENSRC_DIR}/common/env_config.h)
 target_include_directories(Common PUBLIC ${GENSRC_DIR}/common/)

be/src/core/CMakeLists.txt

@@ -32,3 +32,41 @@ pch_reuse(Core)
 # instantiations below without definitions. Opt this single TU out of the PCH.
 set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/wide_integer_from_double.cpp
                             PROPERTIES SKIP_PRECOMPILE_HEADERS ON)
+
+# Unity build: column/data_type/serde glue TUs share the core header closure.
+# Opt-outs:
+# - column/column_vector.cpp: the heaviest template-instantiation TU in the
+#   target (~40s alone); it would dominate any batch it joins
+# - three macro-leak files (WRITE_INTEGRAL_COLUMN_TO_ORC,
+#   DECLARE_DECIMAL_COMPARISON, DELARE_DATE_ADD_INTERVAL)
+# - wide_integer_from_double.cpp: macro-leak, and it is the PCH-skipped
+#   explicit-instantiation TU above -- it must keep its own compile
+# - data_type_serde.cpp: holds member-level explicit instantiations of serde
+#   members whose class-level explicit instantiations live in the per-type
+#   serde TUs; merging both forms into one TU is a duplicate explicit
+#   instantiation
+# - the value/variant/ family (9s of slot time in total): copied file-local
+#   helpers with diverging signatures (require_bytes, write_unsigned, ...) --
+#   not worth renaming for the gain
+# - two files that tests compile a second time by #including the .cpp
+#   (column/column_variant.cpp, data_type/convert_field_to_type.cpp): the
+#   test object must shadow a never-pulled archive member, but a unity batch
+#   is pulled in for its siblings and the linker sees a duplicate definition
+set(CORE_UNITY_SKIP
+    ${CMAKE_CURRENT_SOURCE_DIR}/column/column_vector.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/column/column_variant.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/data_type/convert_field_to_type.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/data_type_serde/data_type_number_serde.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/data_type_serde/data_type_serde.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/field.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/value/vdatetime_value.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/value/variant/variant_batch_builder.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/value/variant/variant_canonical.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/value/variant/variant_field.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/value/variant/variant_metadata.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/value/variant/variant_scalar.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/value/variant/variant_value.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/wide_integer_from_double.cpp)
+doris_skip_unity_inclusion(${CORE_UNITY_SKIP})
+set_target_properties(Core PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD}
+                                      UNITY_BUILD_BATCH_SIZE 12)

be/src/core/data_type_serde/complex_type_deserialize_util.h

@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#pragma once
+
 #include "core/data_type_serde/data_type_serde.h"
 #include "core/string_ref.h"
 

be/src/core/value/vdatetime_value.h

@@ -219,7 +219,9 @@ static constexpr uint32_t DATEV2_YEAR_WIDTH = 23;
 static constexpr uint32_t DATETIMEV2_YEAR_WIDTH = 18;
 static constexpr uint32_t DATETIMEV2_MONTH_WIDTH = 4;
 
-static RE2 time_zone_offset_format_reg(R"(^[+-]{1}\d{2}\:\d{2}$)");
+// One shared instance program-wide (was `static`, i.e. one copy constructed
+// per including TU). Visiting is thread-safe.
+inline RE2 time_zone_offset_format_reg(R"(^[+-]{1}\d{2}\:\d{2}$)");
 
 uint8_t mysql_week_mode(uint32_t mode);
 

be/src/exec/CMakeLists.txt

@@ -52,7 +52,7 @@ set(EXEC_UNITY_SKIP
     ${CMAKE_CURRENT_SOURCE_DIR}/operator/partitioned_aggregation_sink_operator.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/operator/scan_operator.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/sink/writer/vfile_result_writer.cpp)
-set_source_files_properties(${EXEC_UNITY_SKIP} PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
+doris_skip_unity_inclusion(${EXEC_UNITY_SKIP})
 set_target_properties(Exec PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD}
                                       UNITY_BUILD_BATCH_SIZE 12)
 

be/src/exprs/CMakeLists.txt

@@ -97,7 +97,7 @@ set(EXPRS_UNITY_SKIP
     ${CMAKE_CURRENT_SOURCE_DIR}/math_functions.cpp
     ${GENSRC_DIR}/geo/wkt_lex.l.cpp
     ${GENSRC_DIR}/geo/wkt_yacc.y.cpp)
-set_source_files_properties(${EXPRS_UNITY_SKIP} PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
+doris_skip_unity_inclusion(${EXPRS_UNITY_SKIP})
 set_target_properties(Exprs PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD}
                                        UNITY_BUILD_BATCH_SIZE 8)
 

be/src/format/CMakeLists.txt

@@ -29,3 +29,18 @@ list(APPEND SRC_FILES ${FORMAT_V2_SRC_FILES})
 add_library(Format STATIC ${SRC_FILES})
 
 pch_reuse(Format)
+
+# Unity build scoped to format_v2/: the v2 readers are the maintained line and
+# share one reader/expr header closure. The format/ (v1) tree is slated for
+# removal in a future version, so it is not worth the v1/v2 twin-symbol
+# deduplication a joint unity would require: every v1 file stays individual
+# until the tree is deleted. Within v2, adbc_reader.cpp stays individual
+# because its file-scope RETURN_IF_ADBC_ERROR macro must not leak into unity
+# siblings. Batch 8 bounds jumbo-TU size and memory.
+set(FORMAT_UNITY_SKIP ${SRC_FILES})
+list(FILTER FORMAT_UNITY_SKIP EXCLUDE REGEX ".*/format_v2/.*")
+list(APPEND FORMAT_UNITY_SKIP
+    ${CMAKE_CURRENT_SOURCE_DIR}/../format_v2/table/adbc_reader.cpp)
+doris_skip_unity_inclusion(${FORMAT_UNITY_SKIP})
+set_target_properties(Format PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD}
+                                        UNITY_BUILD_BATCH_SIZE 8)

be/src/format_v2/column_mapper.cpp

@@ -349,10 +349,6 @@ static VExprSPtr create_file_slot_ref(const VSlotRef& slot_ref,
     return ref;
 }
 
-static bool is_cast_expr(const VExprSPtr& expr) {
-    return dynamic_cast<const Cast*>(expr.get()) != nullptr;
-}
-
 static bool is_binary_comparison_predicate(const VExprSPtr& expr) {
     if (expr == nullptr || expr->get_num_children() != 2 ||
         (expr->node_type() != TExprNodeType::BINARY_PRED &&

be/src/format_v2/column_mapper_nested.cpp

@@ -37,10 +37,6 @@ namespace doris::format {
 
 namespace {
 
-static bool is_cast_expr(const VExprSPtr& expr) {
-    return dynamic_cast<const Cast*>(expr.get()) != nullptr;
-}
-
 static bool is_signed_integer_type(PrimitiveType type) {
     switch (type) {
     case TYPE_TINYINT:

be/src/format_v2/expr/cast.h

@@ -68,4 +68,9 @@ class Cast final : public VExpr {
     std::string _expr_name;
     FunctionBasePtr _function;
 };
+
+inline bool is_cast_expr(const VExprSPtr& expr) {
+    return dynamic_cast<const Cast*>(expr.get()) != nullptr;
+}
+
 } // namespace doris::format

be/src/format_v2/parquet/parquet_scan.cpp

@@ -49,6 +49,7 @@
 #include "format_v2/parquet/reader/native/column_chunk_reader.h"
 #include "format_v2/parquet/reader/native_column_reader.h"
 #include "format_v2/parquet/reader/row_position_column_reader.h"
+#include "format_v2/parquet/selection_vector.h" // count_range_rows
 #include "runtime/runtime_state.h"
 #include "util/defer_op.h"
 #include "util/time.h"
@@ -823,14 +824,6 @@ Status execute_batch_filters(const format::FileScanRequest& request, int64_t bat
 }
 
 namespace {
-int64_t count_range_rows(const std::vector<RowRange>& ranges) {
-    int64_t rows = 0;
-    for (const auto& range : ranges) {
-        rows += range.length;
-    }
-    return rows;
-}
-
 void append_intersection(const RowRange& left, const RowRange& right,
                          std::vector<RowRange>& result) {
     const int64_t start = std::max(left.start, right.start);

be/src/format_v2/parquet/parquet_statistics.cpp

@@ -42,6 +42,7 @@
 #include "format_v2/parquet/parquet_file_context.h"
 #include "format_v2/parquet/reader/native/block_split_bloom_filter.h"
 #include "format_v2/parquet/reader/native_column_reader.h"
+#include "format_v2/parquet/selection_vector.h" // count_range_rows
 #include "format_v2/timestamp_statistics.h"
 #include "runtime/runtime_profile.h"
 #include "storage/index/bloom_filter/bloom_filter.h"
@@ -989,14 +990,6 @@ std::vector<RowRange> intersect_ranges(const std::vector<RowRange>& left,
     return result;
 }
 
-int64_t count_range_rows(const std::vector<RowRange>& ranges) {
-    int64_t rows = 0;
-    for (const auto& range : ranges) {
-        rows += range.length;
-    }
-    return rows;
-}
-
 void append_row_range(const RowRange& range, std::vector<RowRange>* ranges) {
     if (range.length == 0) {
         return;

be/src/format_v2/parquet/selection_vector.h

@@ -31,6 +31,14 @@ struct RowRange {
     int64_t length = 0;
 };
 
+inline int64_t count_range_rows(const std::vector<RowRange>& ranges) {
+    int64_t rows = 0;
+    for (const auto& range : ranges) {
+        rows += range.length;
+    }
+    return rows;
+}
+
 struct ParquetPageSkipPlan {
     int leaf_column_id = -1;
     // Page ordinal is the data-page ordinal in the column chunk. It intentionally excludes

be/src/format_v2/table/schema_history_util.cpp

@@ -27,13 +27,6 @@
 namespace doris::format {
 namespace {
 
-const schema::external::TField* get_field_ptr(const schema::external::TFieldPtr& field_ptr) {
-    if (!field_ptr.__isset.field_ptr || field_ptr.field_ptr == nullptr) {
-        return nullptr;
-    }
-    return field_ptr.field_ptr.get();
-}
-
 const schema::external::TField* find_child_field_by_name(
         const std::vector<schema::external::TFieldPtr>& fields, const std::string& name) {
     for (const auto& field_ptr : fields) {

be/src/format_v2/table/schema_history_util.h

@@ -27,6 +27,14 @@
 
 namespace doris::format {
 
+// Unwrap a thrift TFieldPtr, returning nullptr when the pointer is unset.
+inline const schema::external::TField* get_field_ptr(const schema::external::TFieldPtr& field_ptr) {
+    if (!field_ptr.__isset.field_ptr || field_ptr.field_ptr == nullptr) {
+        return nullptr;
+    }
+    return field_ptr.field_ptr.get();
+}
+
 const schema::external::TSchema* find_history_schema(const TFileScanRangeParams* params,
                                                      int64_t schema_id);
 

be/src/format_v2/table_reader.cpp

@@ -50,6 +50,7 @@
 #include "format_v2/native/native_reader.h"
 #include "format_v2/orc/orc_reader.h"
 #include "format_v2/parquet/parquet_reader.h"
+#include "format_v2/table/schema_history_util.h" // get_field_ptr
 #include "runtime/file_scan_profile.h"
 #include "storage/segment/condition_cache.h"
 #include "util/debug_points.h"
@@ -140,13 +141,6 @@ std::string partition_values_debug_string(const std::map<std::string, Field>& pa
     return out.str();
 }
 
-const schema::external::TField* get_field_ptr(const schema::external::TFieldPtr& field_ptr) {
-    if (!field_ptr.__isset.field_ptr || field_ptr.field_ptr == nullptr) {
-        return nullptr;
-    }
-    return field_ptr.field_ptr.get();
-}
-
 const schema::external::TField* find_external_field_by_id(
         const schema::external::TStructField* root, int32_t field_id) {
     if (root == nullptr || !root->__isset.fields) {