diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 0741d0a17dd551..b2f1b8129ce89c 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -409,7 +409,14 @@ if (COMPILER_CLANG) -Wthread-safety) add_compile_options(-Wno-gnu-statement-expression -Wno-implicit-float-conversion - -Wno-sign-conversion) + -Wno-sign-conversion + # libstdc++-15 puts `#pragma GCC unroll` on loops in its own + # headers (bits/stl_algobase.h); when a sanitizer/coverage + # build compiles at low optimization the unroller punts and + # clang's -Wpass-failed turns that missed *hint* into an + # -Werror failure. The diagnostic carries no correctness + # signal, so drop it. + -Wno-pass-failed) if (USE_LIBCPP) add_compile_options($<$:-stdlib=libc++>) add_definitions(-DUSE_LIBCPP) @@ -981,6 +988,24 @@ function(pch_reuse target) endif() endfunction(pch_reuse target) +# Every unity opt-out goes through this helper. set_source_files_properties() +# silently ignores paths that do not exist, so a skip entry going stale after +# a rename or move would quietly re-join its unity batch -- slower builds, or +# new file-scope clashes, with no diagnostic anywhere. Fail the configure +# instead. Entries under GENSRC_DIR are exempt: they are emitted by +# add_custom_command at build time (wkt_lex.l.cpp, the generated +# thrift/protobuf sources) and legitimately do not exist on a fresh configure. +function(doris_skip_unity_inclusion) + foreach(entry IN LISTS ARGN) + string(FIND "${entry}" "${GENSRC_DIR}" gensrc_prefix_pos) + if(NOT gensrc_prefix_pos EQUAL 0 AND NOT EXISTS "${entry}") + message(FATAL_ERROR + "unity skip entry does not exist (renamed or moved?): ${entry}") + endif() + endforeach() + set_source_files_properties(${ARGN} PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) +endfunction() + add_subdirectory(${SRC_DIR}/agent) add_subdirectory(${SRC_DIR}/common) diff --git a/be/src/agent/CMakeLists.txt b/be/src/agent/CMakeLists.txt index c680c41d6b209c..423b9c5e1fbbdf 100644 --- a/be/src/agent/CMakeLists.txt +++ b/be/src/agent/CMakeLists.txt @@ -29,4 +29,9 @@ endif() add_library(Agent STATIC ${AGENT_SOURCES}) -pch_reuse(Agent) \ No newline at end of file +pch_reuse(Agent) + +# Unity build: nine homogeneous task-worker glue TUs sharing the agent header +# closure; batch 0 merges them into a single unity TU. +set_target_properties(Agent PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD} + UNITY_BUILD_BATCH_SIZE 0) \ No newline at end of file diff --git a/be/src/agent/cgroup_cpu_ctl.h b/be/src/agent/cgroup_cpu_ctl.h index 262be57b499dee..3e1933fb4d6492 100644 --- a/be/src/agent/cgroup_cpu_ctl.h +++ b/be/src/agent/cgroup_cpu_ctl.h @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#pragma once + #include #include #include diff --git a/be/src/cloud/CMakeLists.txt b/be/src/cloud/CMakeLists.txt index dbe8160c2f8d8c..e81a097b1ff676 100644 --- a/be/src/cloud/CMakeLists.txt +++ b/be/src/cloud/CMakeLists.txt @@ -22,3 +22,15 @@ file(GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS *.cpp) add_library(Cloud STATIC ${SRC_FILES}) pch_reuse(Cloud) + +# Unity build: homogeneous cloud glue TUs sharing the CloudStorageEngine / +# meta-service RPC header closure. Batch 12 bounds jumbo-TU size and memory. +# The two http action TUs stay individual: sibling TUs' file-scope +# `using namespace ErrorCode` makes the unscoped HttpStatus enum constants +# (OK, NOT_FOUND, ...) shadow ErrorCode variables under -Wshadow -Werror. +set(CLOUD_UNITY_SKIP + ${CMAKE_CURRENT_SOURCE_DIR}/cloud_compaction_action.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/injection_point_action.cpp) +doris_skip_unity_inclusion(${CLOUD_UNITY_SKIP}) +set_target_properties(Cloud PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD} + UNITY_BUILD_BATCH_SIZE 12) diff --git a/be/src/cloud/cloud_internal_service.cpp b/be/src/cloud/cloud_internal_service.cpp index 7dca1d1517f680..c903b49961fc15 100644 --- a/be/src/cloud/cloud_internal_service.cpp +++ b/be/src/cloud/cloud_internal_service.cpp @@ -925,12 +925,7 @@ bvar::Adder g_file_cache_warm_up_rowset_wait_for_compaction_num( bvar::Adder g_file_cache_warm_up_rowset_wait_for_compaction_timeout_num( "file_cache_warm_up_rowset_wait_for_compaction_timeout_num"); -// Per-job windowed metrics for target BE -// bvar::Window enforces MAX_SECONDS_LIMIT = 3600, so the longest window is 1h. -static constexpr int WINDOW_5M = 300; -static constexpr int WINDOW_30M = 1800; -static constexpr int WINDOW_1H = 3600; - +// Per-job windowed metrics for target BE (window spans shared via bvar_windowed_adder.h) MBvarWindowedAdder g_warmup_ed_finish_segment_num("warmup_ed_finish_segment_num", {"job_id"}, {WINDOW_5M, WINDOW_30M, WINDOW_1H}, false); MBvarWindowedAdder g_warmup_ed_finish_segment_size("warmup_ed_finish_segment_size", {"job_id"}, diff --git a/be/src/cloud/cloud_warm_up_manager.cpp b/be/src/cloud/cloud_warm_up_manager.cpp index 4590d96fb4ca80..68db44f0b1cdf6 100644 --- a/be/src/cloud/cloud_warm_up_manager.cpp +++ b/be/src/cloud/cloud_warm_up_manager.cpp @@ -118,12 +118,7 @@ bvar::Adder g_file_cache_warm_up_job_num("file_cache_warm_up_job_num"); bvar::LatencyRecorder g_file_cache_warm_up_rowset_wait_for_compaction_latency( "file_cache_warm_up_rowset_wait_for_compaction_latency"); -// Per-job windowed metrics for source BE -// bvar::Window enforces MAX_SECONDS_LIMIT = 3600, so the longest window is 1h. -static constexpr int WINDOW_5M = 300; -static constexpr int WINDOW_30M = 1800; -static constexpr int WINDOW_1H = 3600; - +// Per-job windowed metrics for source BE (window spans shared via bvar_windowed_adder.h) MBvarWindowedAdder g_warmup_ed_requested_segment_num("warmup_ed_requested_segment_num", {"job_id"}, {WINDOW_5M, WINDOW_30M, WINDOW_1H}, false); MBvarWindowedAdder g_warmup_ed_requested_segment_size("warmup_ed_requested_segment_size", diff --git a/be/src/common/CMakeLists.txt b/be/src/common/CMakeLists.txt index b1e51e8f30312e..949af9c9723f82 100644 --- a/be/src/common/CMakeLists.txt +++ b/be/src/common/CMakeLists.txt @@ -23,6 +23,22 @@ add_library(Common STATIC ${SRC_FILES}) pch_reuse(Common) +# Unity build for the small common TUs. Seven files leak file-scope macros +# (SET_FIELD/UPDATE_FIELD, MEMORY_SANITIZER, DW_* tables, the DEFINE_*_METRIC +# families, USE_PHDR_CACHE/UNW_LOCAL_ONLY) and stay individual -- that set +# also keeps the three same-name metric-hook statics (_s_hook_name) apart. +set(COMMON_UNITY_SKIP + ${CMAKE_CURRENT_SOURCE_DIR}/config.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/demangle.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/dwarf.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/metrics/doris_metrics.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/metrics/jvm_metrics.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/metrics/system_metrics.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/phdr_cache.cpp) +doris_skip_unity_inclusion(${COMMON_UNITY_SKIP}) +set_target_properties(Common PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD} + UNITY_BUILD_BATCH_SIZE 0) + # Generate env_config.h according to env_config.h.in configure_file(${CMAKE_CURRENT_SOURCE_DIR}/env_config.h.in ${GENSRC_DIR}/common/env_config.h) target_include_directories(Common PUBLIC ${GENSRC_DIR}/common/) diff --git a/be/src/core/CMakeLists.txt b/be/src/core/CMakeLists.txt index 348a375eef0c1d..d5faec6c40cfa2 100644 --- a/be/src/core/CMakeLists.txt +++ b/be/src/core/CMakeLists.txt @@ -32,3 +32,41 @@ pch_reuse(Core) # instantiations below without definitions. Opt this single TU out of the PCH. set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/wide_integer_from_double.cpp PROPERTIES SKIP_PRECOMPILE_HEADERS ON) + +# Unity build: column/data_type/serde glue TUs share the core header closure. +# Opt-outs: +# - column/column_vector.cpp: the heaviest template-instantiation TU in the +# target (~40s alone); it would dominate any batch it joins +# - three macro-leak files (WRITE_INTEGRAL_COLUMN_TO_ORC, +# DECLARE_DECIMAL_COMPARISON, DELARE_DATE_ADD_INTERVAL) +# - wide_integer_from_double.cpp: macro-leak, and it is the PCH-skipped +# explicit-instantiation TU above -- it must keep its own compile +# - data_type_serde.cpp: holds member-level explicit instantiations of serde +# members whose class-level explicit instantiations live in the per-type +# serde TUs; merging both forms into one TU is a duplicate explicit +# instantiation +# - the value/variant/ family (9s of slot time in total): copied file-local +# helpers with diverging signatures (require_bytes, write_unsigned, ...) -- +# not worth renaming for the gain +# - two files that tests compile a second time by #including the .cpp +# (column/column_variant.cpp, data_type/convert_field_to_type.cpp): the +# test object must shadow a never-pulled archive member, but a unity batch +# is pulled in for its siblings and the linker sees a duplicate definition +set(CORE_UNITY_SKIP + ${CMAKE_CURRENT_SOURCE_DIR}/column/column_vector.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/column/column_variant.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/data_type/convert_field_to_type.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/data_type_serde/data_type_number_serde.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/data_type_serde/data_type_serde.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/field.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/value/vdatetime_value.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/value/variant/variant_batch_builder.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/value/variant/variant_canonical.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/value/variant/variant_field.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/value/variant/variant_metadata.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/value/variant/variant_scalar.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/value/variant/variant_value.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/wide_integer_from_double.cpp) +doris_skip_unity_inclusion(${CORE_UNITY_SKIP}) +set_target_properties(Core PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD} + UNITY_BUILD_BATCH_SIZE 12) diff --git a/be/src/core/data_type_serde/complex_type_deserialize_util.h b/be/src/core/data_type_serde/complex_type_deserialize_util.h index ff76d64dcd6542..b1320b4b073fc3 100644 --- a/be/src/core/data_type_serde/complex_type_deserialize_util.h +++ b/be/src/core/data_type_serde/complex_type_deserialize_util.h @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#pragma once + #include "core/data_type_serde/data_type_serde.h" #include "core/string_ref.h" diff --git a/be/src/core/value/vdatetime_value.h b/be/src/core/value/vdatetime_value.h index 7419b02175575b..da0bba19bcb1c3 100644 --- a/be/src/core/value/vdatetime_value.h +++ b/be/src/core/value/vdatetime_value.h @@ -219,7 +219,9 @@ static constexpr uint32_t DATEV2_YEAR_WIDTH = 23; static constexpr uint32_t DATETIMEV2_YEAR_WIDTH = 18; static constexpr uint32_t DATETIMEV2_MONTH_WIDTH = 4; -static RE2 time_zone_offset_format_reg(R"(^[+-]{1}\d{2}\:\d{2}$)"); +// One shared instance program-wide (was `static`, i.e. one copy constructed +// per including TU). Visiting is thread-safe. +inline RE2 time_zone_offset_format_reg(R"(^[+-]{1}\d{2}\:\d{2}$)"); uint8_t mysql_week_mode(uint32_t mode); diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt index 7ec053789111a1..7ccbd60b5ea37b 100644 --- a/be/src/exec/CMakeLists.txt +++ b/be/src/exec/CMakeLists.txt @@ -52,7 +52,7 @@ set(EXEC_UNITY_SKIP ${CMAKE_CURRENT_SOURCE_DIR}/operator/partitioned_aggregation_sink_operator.cpp ${CMAKE_CURRENT_SOURCE_DIR}/operator/scan_operator.cpp ${CMAKE_CURRENT_SOURCE_DIR}/sink/writer/vfile_result_writer.cpp) -set_source_files_properties(${EXEC_UNITY_SKIP} PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) +doris_skip_unity_inclusion(${EXEC_UNITY_SKIP}) set_target_properties(Exec PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD} UNITY_BUILD_BATCH_SIZE 12) diff --git a/be/src/exprs/CMakeLists.txt b/be/src/exprs/CMakeLists.txt index b7bf6916a901ae..1940ba6d8e4657 100644 --- a/be/src/exprs/CMakeLists.txt +++ b/be/src/exprs/CMakeLists.txt @@ -97,7 +97,7 @@ set(EXPRS_UNITY_SKIP ${CMAKE_CURRENT_SOURCE_DIR}/math_functions.cpp ${GENSRC_DIR}/geo/wkt_lex.l.cpp ${GENSRC_DIR}/geo/wkt_yacc.y.cpp) -set_source_files_properties(${EXPRS_UNITY_SKIP} PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) +doris_skip_unity_inclusion(${EXPRS_UNITY_SKIP}) set_target_properties(Exprs PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD} UNITY_BUILD_BATCH_SIZE 8) diff --git a/be/src/format/CMakeLists.txt b/be/src/format/CMakeLists.txt index f73ef2dbfaab7c..94b7c6bb6d1b4c 100644 --- a/be/src/format/CMakeLists.txt +++ b/be/src/format/CMakeLists.txt @@ -29,3 +29,18 @@ list(APPEND SRC_FILES ${FORMAT_V2_SRC_FILES}) add_library(Format STATIC ${SRC_FILES}) pch_reuse(Format) + +# Unity build scoped to format_v2/: the v2 readers are the maintained line and +# share one reader/expr header closure. The format/ (v1) tree is slated for +# removal in a future version, so it is not worth the v1/v2 twin-symbol +# deduplication a joint unity would require: every v1 file stays individual +# until the tree is deleted. Within v2, adbc_reader.cpp stays individual +# because its file-scope RETURN_IF_ADBC_ERROR macro must not leak into unity +# siblings. Batch 8 bounds jumbo-TU size and memory. +set(FORMAT_UNITY_SKIP ${SRC_FILES}) +list(FILTER FORMAT_UNITY_SKIP EXCLUDE REGEX ".*/format_v2/.*") +list(APPEND FORMAT_UNITY_SKIP + ${CMAKE_CURRENT_SOURCE_DIR}/../format_v2/table/adbc_reader.cpp) +doris_skip_unity_inclusion(${FORMAT_UNITY_SKIP}) +set_target_properties(Format PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD} + UNITY_BUILD_BATCH_SIZE 8) diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 9f433cc853f61a..e98a0cde9a1e9f 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -349,10 +349,6 @@ static VExprSPtr create_file_slot_ref(const VSlotRef& slot_ref, return ref; } -static bool is_cast_expr(const VExprSPtr& expr) { - return dynamic_cast(expr.get()) != nullptr; -} - static bool is_binary_comparison_predicate(const VExprSPtr& expr) { if (expr == nullptr || expr->get_num_children() != 2 || (expr->node_type() != TExprNodeType::BINARY_PRED && diff --git a/be/src/format_v2/column_mapper_nested.cpp b/be/src/format_v2/column_mapper_nested.cpp index fefcdf26f12e1d..9d1980c357984f 100644 --- a/be/src/format_v2/column_mapper_nested.cpp +++ b/be/src/format_v2/column_mapper_nested.cpp @@ -37,10 +37,6 @@ namespace doris::format { namespace { -static bool is_cast_expr(const VExprSPtr& expr) { - return dynamic_cast(expr.get()) != nullptr; -} - static bool is_signed_integer_type(PrimitiveType type) { switch (type) { case TYPE_TINYINT: diff --git a/be/src/format_v2/expr/cast.h b/be/src/format_v2/expr/cast.h index 22604455e50099..18b3bea676c4fc 100644 --- a/be/src/format_v2/expr/cast.h +++ b/be/src/format_v2/expr/cast.h @@ -68,4 +68,9 @@ class Cast final : public VExpr { std::string _expr_name; FunctionBasePtr _function; }; + +inline bool is_cast_expr(const VExprSPtr& expr) { + return dynamic_cast(expr.get()) != nullptr; +} + } // namespace doris::format diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index a9154021999a4f..115ea2e373d23f 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -49,6 +49,7 @@ #include "format_v2/parquet/reader/native/column_chunk_reader.h" #include "format_v2/parquet/reader/native_column_reader.h" #include "format_v2/parquet/reader/row_position_column_reader.h" +#include "format_v2/parquet/selection_vector.h" // count_range_rows #include "runtime/runtime_state.h" #include "util/defer_op.h" #include "util/time.h" @@ -823,14 +824,6 @@ Status execute_batch_filters(const format::FileScanRequest& request, int64_t bat } namespace { -int64_t count_range_rows(const std::vector& ranges) { - int64_t rows = 0; - for (const auto& range : ranges) { - rows += range.length; - } - return rows; -} - void append_intersection(const RowRange& left, const RowRange& right, std::vector& result) { const int64_t start = std::max(left.start, right.start); diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp index e6f42c59ce61ea..b67536fd02c225 100644 --- a/be/src/format_v2/parquet/parquet_statistics.cpp +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -42,6 +42,7 @@ #include "format_v2/parquet/parquet_file_context.h" #include "format_v2/parquet/reader/native/block_split_bloom_filter.h" #include "format_v2/parquet/reader/native_column_reader.h" +#include "format_v2/parquet/selection_vector.h" // count_range_rows #include "format_v2/timestamp_statistics.h" #include "runtime/runtime_profile.h" #include "storage/index/bloom_filter/bloom_filter.h" @@ -989,14 +990,6 @@ std::vector intersect_ranges(const std::vector& left, return result; } -int64_t count_range_rows(const std::vector& ranges) { - int64_t rows = 0; - for (const auto& range : ranges) { - rows += range.length; - } - return rows; -} - void append_row_range(const RowRange& range, std::vector* ranges) { if (range.length == 0) { return; diff --git a/be/src/format_v2/parquet/selection_vector.h b/be/src/format_v2/parquet/selection_vector.h index 99c854d8b4e896..a9c8c040e9302e 100644 --- a/be/src/format_v2/parquet/selection_vector.h +++ b/be/src/format_v2/parquet/selection_vector.h @@ -31,6 +31,14 @@ struct RowRange { int64_t length = 0; }; +inline int64_t count_range_rows(const std::vector& ranges) { + int64_t rows = 0; + for (const auto& range : ranges) { + rows += range.length; + } + return rows; +} + struct ParquetPageSkipPlan { int leaf_column_id = -1; // Page ordinal is the data-page ordinal in the column chunk. It intentionally excludes diff --git a/be/src/format_v2/table/schema_history_util.cpp b/be/src/format_v2/table/schema_history_util.cpp index 96a466fa772837..a3dbad7c1e36ee 100644 --- a/be/src/format_v2/table/schema_history_util.cpp +++ b/be/src/format_v2/table/schema_history_util.cpp @@ -27,13 +27,6 @@ namespace doris::format { namespace { -const schema::external::TField* get_field_ptr(const schema::external::TFieldPtr& field_ptr) { - if (!field_ptr.__isset.field_ptr || field_ptr.field_ptr == nullptr) { - return nullptr; - } - return field_ptr.field_ptr.get(); -} - const schema::external::TField* find_child_field_by_name( const std::vector& fields, const std::string& name) { for (const auto& field_ptr : fields) { diff --git a/be/src/format_v2/table/schema_history_util.h b/be/src/format_v2/table/schema_history_util.h index 3c4a80b5d4c975..49704c5fb98468 100644 --- a/be/src/format_v2/table/schema_history_util.h +++ b/be/src/format_v2/table/schema_history_util.h @@ -27,6 +27,14 @@ namespace doris::format { +// Unwrap a thrift TFieldPtr, returning nullptr when the pointer is unset. +inline const schema::external::TField* get_field_ptr(const schema::external::TFieldPtr& field_ptr) { + if (!field_ptr.__isset.field_ptr || field_ptr.field_ptr == nullptr) { + return nullptr; + } + return field_ptr.field_ptr.get(); +} + const schema::external::TSchema* find_history_schema(const TFileScanRangeParams* params, int64_t schema_id); diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 40ffe6cb86eff0..b531ff9cf2322a 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -50,6 +50,7 @@ #include "format_v2/native/native_reader.h" #include "format_v2/orc/orc_reader.h" #include "format_v2/parquet/parquet_reader.h" +#include "format_v2/table/schema_history_util.h" // get_field_ptr #include "runtime/file_scan_profile.h" #include "storage/segment/condition_cache.h" #include "util/debug_points.h" @@ -140,13 +141,6 @@ std::string partition_values_debug_string(const std::map& pa return out.str(); } -const schema::external::TField* get_field_ptr(const schema::external::TFieldPtr& field_ptr) { - if (!field_ptr.__isset.field_ptr || field_ptr.field_ptr == nullptr) { - return nullptr; - } - return field_ptr.field_ptr.get(); -} - const schema::external::TField* find_external_field_by_id( const schema::external::TStructField* root, int32_t field_id) { if (root == nullptr || !root->__isset.fields) { diff --git a/be/src/gen_cpp/CMakeLists.txt b/be/src/gen_cpp/CMakeLists.txt index c4463f75520cb0..abbe3877079d9f 100644 --- a/be/src/gen_cpp/CMakeLists.txt +++ b/be/src/gen_cpp/CMakeLists.txt @@ -27,10 +27,26 @@ list(FILTER SRC_FILES EXCLUDE REGEX "_server\\.skeleton\\.cpp$") add_compile_options(-Wno-return-type) -# keep everything in one library, the object files reference +# keep everything in one library, the object files reference # each other add_library(DorisGen STATIC ${SRC_FILES}) +# Unity build for the thrift-generated .cpp half (pre-scanned clean: no +# file-scope macros or duplicate statics). Generated sources cannot be edited, +# so anything that clashes can only be opted out, never fixed: +# - every protobuf .cc stays individual: protoc emits the same file-scope +# statics (schemas[], file_default_instances[]) in each one, a structural +# clash for any merged pair +# - the two heaviest thrift TUs stay individual so no batch inherits them +set(GEN_UNITY_SKIP ${SRC_FILES}) +list(FILTER GEN_UNITY_SKIP INCLUDE REGEX "\\.cc$") +list(APPEND GEN_UNITY_SKIP + ${GEN_CPP_DIR}/FrontendService_types.cpp + ${GEN_CPP_DIR}/PlanNodes_types.cpp) +doris_skip_unity_inclusion(${GEN_UNITY_SKIP}) +set_target_properties(DorisGen PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD} + UNITY_BUILD_BATCH_SIZE 8) + if (COMPILER_CLANG) target_compile_options(DorisGen PRIVATE -Wno-implicit-int-conversion diff --git a/be/src/io/CMakeLists.txt b/be/src/io/CMakeLists.txt index 56c2eeb94a3819..3026c82e11fff1 100644 --- a/be/src/io/CMakeLists.txt +++ b/be/src/io/CMakeLists.txt @@ -34,11 +34,38 @@ if(ENABLE_TDE) endif() list(REMOVE_ITEM IO_FILES "${CMAKE_CURRENT_SOURCE_DIR}/fs/benchmark/fs_benchmark_tool.cpp") +# Standalone tool with its own main(); must not be an archive member of libIO.a, +# or unity builds merge it into an object that the doris_be link actually pulls in. +list(REMOVE_ITEM IO_FILES "${CMAKE_CURRENT_SOURCE_DIR}/cache/file_cache_lru_tool.cpp") add_library(IO STATIC ${IO_FILES}) pch_reuse(IO) +# Unity build for the io glue (small fs/cache TUs, parse-cost dominated). +# Opt-outs: +# - three filesystem TUs whose CHECK_*_CLIENT file-scope macros must not +# leak into unity siblings +# - http_file_system.cpp: sibling TUs' file-scope `using namespace ErrorCode` +# makes the unscoped HttpStatus enum constants shadow ErrorCode variables +# under -Wshadow -Werror (same pattern as the cloud http actions) +# - benchmark sources and ENABLE_TDE module swaps stay individual +set(IO_UNITY_SKIP + ${CMAKE_CURRENT_SOURCE_DIR}/fs/broker_file_system.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/fs/hdfs_file_system.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/fs/s3_file_system.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/fs/http_file_system.cpp) +file(GLOB_RECURSE IO_UNITY_BENCH_SRCS CONFIGURE_DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/fs/benchmark/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tools/*.cpp) +list(APPEND IO_UNITY_SKIP ${IO_UNITY_BENCH_SRCS}) +if (ENABLE_TDE) + list(APPEND IO_UNITY_SKIP ${EXTRA_SOURCES}) +endif() +doris_skip_unity_inclusion(${IO_UNITY_SKIP}) +set_target_properties(IO PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD} + UNITY_BUILD_BATCH_SIZE 16) + if (${BUILD_FS_BENCHMARK} STREQUAL "ON") add_executable(fs_benchmark_tool fs/benchmark/fs_benchmark_tool.cpp diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index b44679a94fa150..c4e7ef47853047 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -66,10 +66,6 @@ namespace { constexpr std::array LRU_LOG_REPLAY_TYPES = { FileCacheType::TTL, FileCacheType::INDEX, FileCacheType::NORMAL, FileCacheType::DISPOSABLE}; -size_t file_cache_type_index(FileCacheType type) { - return static_cast(type); -} - } // namespace // Insert a block pointer into one shard while swallowing allocation failures. diff --git a/be/src/io/cache/file_cache_common.h b/be/src/io/cache/file_cache_common.h index 8b52af8d161b9a..3f8ca5e1ff0161 100644 --- a/be/src/io/cache/file_cache_common.h +++ b/be/src/io/cache/file_cache_common.h @@ -43,6 +43,11 @@ enum FileCacheType { DISPOSABLE = 0, TTL = 3, }; + +inline size_t file_cache_type_index(FileCacheType type) { + return static_cast(type); +} + std::string cache_type_to_surfix(FileCacheType type); FileCacheType surfix_to_cache_type(const std::string& str); diff --git a/be/src/io/cache/lru_queue_recorder.cpp b/be/src/io/cache/lru_queue_recorder.cpp index 314a444e232e9e..82edc5f6070003 100644 --- a/be/src/io/cache/lru_queue_recorder.cpp +++ b/be/src/io/cache/lru_queue_recorder.cpp @@ -24,13 +24,7 @@ namespace doris::io { -namespace { - -size_t file_cache_type_index(FileCacheType type) { - return static_cast(type); -} - -} // namespace +namespace {} // namespace void LRUQueueRecorder::record_queue_event(FileCacheType type, CacheLRULogType log_type, const UInt128Wrapper hash, const size_t offset, diff --git a/be/src/load/CMakeLists.txt b/be/src/load/CMakeLists.txt index 5c2e970f6d9d8d..5faf360734fdef 100644 --- a/be/src/load/CMakeLists.txt +++ b/be/src/load/CMakeLists.txt @@ -25,3 +25,13 @@ file(GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS *.cpp) add_library(Load STATIC ${SRC_FILES}) pch_reuse(Load) + +# Unity build: load channel/delta-writer/group-commit glue sharing one header +# closure. Two files leak file-scope macros (UNKNOWN_ID_FOR_TEST, +# HANDLE_ERROR/HANDLE_MULTI_TABLE_ERROR) and stay individual. +set(LOAD_UNITY_SKIP + ${CMAKE_CURRENT_SOURCE_DIR}/channel/load_stream.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/routine_load/routine_load_task_executor.cpp) +doris_skip_unity_inclusion(${LOAD_UNITY_SKIP}) +set_target_properties(Load PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD} + UNITY_BUILD_BATCH_SIZE 12) diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt index 7d54f9bd76bbff..0d2f3b567d24f6 100644 --- a/be/src/runtime/CMakeLists.txt +++ b/be/src/runtime/CMakeLists.txt @@ -34,3 +34,17 @@ add_library(Runtime STATIC ) pch_reuse(Runtime) + +# Unity build: runtime glue TUs share the exec-env/runtime-state header +# closure. memory/jemalloc_hook.cpp (a Linux-only member, absent on macOS) +# stays individual: it defines malloc-replacement hooks and leaks +# __THROW/ALIAS macros. descriptors.cpp stays individual: its test compiles +# it a second time by #including the .cpp, and the test object must shadow a +# never-pulled archive member (a unity batch would get pulled in for its +# siblings and hand the linker a duplicate definition). +set(RUNTIME_UNITY_SKIP + ${CMAKE_CURRENT_SOURCE_DIR}/memory/jemalloc_hook.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/descriptors.cpp) +doris_skip_unity_inclusion(${RUNTIME_UNITY_SKIP}) +set_target_properties(Runtime PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD} + UNITY_BUILD_BATCH_SIZE 12) diff --git a/be/src/runtime/runtime_profile.h b/be/src/runtime/runtime_profile.h index 54f2e80c89e32d..662a92db11db2e 100644 --- a/be/src/runtime/runtime_profile.h +++ b/be/src/runtime/runtime_profile.h @@ -746,9 +746,6 @@ class RuntimeProfile { int64_t _metadata; bool _is_set_metadata = false; - bool _is_sink = false; - bool _is_set_sink = false; - // The timestamp when the profile was modified, make sure the update is up to date. time_t _timestamp; diff --git a/be/src/service/CMakeLists.txt b/be/src/service/CMakeLists.txt index 28985a2e031a7f..f07b896d4a45f0 100644 --- a/be/src/service/CMakeLists.txt +++ b/be/src/service/CMakeLists.txt @@ -37,16 +37,20 @@ add_library(Service STATIC ${SRC_FILES}) pch_reuse(Service) # Unity build scoped to the http glue: ~60 small handlers re-parsing the same -# header closure once per file. Non-http sources (service entry points, -# arrow_flight) are heterogeneous heavy TUs that gain nothing from merging. -# http_parser.cpp and be_thread_stack_action.cpp stay individual because their -# file-scope macros (CR/LF, UNW_LOCAL_ONLY) must not leak into unity siblings. +# header closure once per file, merged into one unity TU (batch 0). Widening +# to the whole target at batch 8 was tried and measured slower: splitting the +# single http jumbo into 8-file batches repays the shared-closure parse per +# batch, and the heterogeneous non-http TUs gain too little to cover it +# (service segment 104s -> 132s slot time). So non-http sources stay +# individual. http_parser.cpp and be_thread_stack_action.cpp stay individual +# because their file-scope macros (CR/LF, UNW_LOCAL_ONLY) must not leak into +# unity siblings. set(SERVICE_UNITY_SKIP ${SRC_FILES}) list(FILTER SERVICE_UNITY_SKIP EXCLUDE REGEX ".*/service/http/.*") list(APPEND SERVICE_UNITY_SKIP ${CMAKE_CURRENT_SOURCE_DIR}/http/http_parser.cpp ${CMAKE_CURRENT_SOURCE_DIR}/http/action/be_thread_stack_action.cpp) -set_source_files_properties(${SERVICE_UNITY_SKIP} PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) +doris_skip_unity_inclusion(${SERVICE_UNITY_SKIP}) set_target_properties(Service PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD} UNITY_BUILD_BATCH_SIZE 0) diff --git a/be/src/storage/CMakeLists.txt b/be/src/storage/CMakeLists.txt index 888b16ebe32fc7..21e82183f58dff 100644 --- a/be/src/storage/CMakeLists.txt +++ b/be/src/storage/CMakeLists.txt @@ -46,19 +46,36 @@ endif() pch_reuse(Storage) -# Unity build scoped to storage/index/: 115 homogeneous reader/writer/iterator -# TUs sharing one (CLucene-heavy) header closure. The rest of Storage keeps -# compiling individually. Batch 32 bounds jumbo-TU size and memory. Four index -# files stay individual because their file-scope macros (CL_MAX_PATH and -# friends, IS_CHINESE_CHAR, APPLY_FOR_PRIMITITYPE) must not leak into unity -# siblings. -set(STORAGE_UNITY_SKIP ${SRC_FILES}) -list(FILTER STORAGE_UNITY_SKIP EXCLUDE REGEX ".*/storage/index/.*") -list(APPEND STORAGE_UNITY_SKIP +# Unity build for the whole Storage target: segment/rowset/tablet/compaction/ +# index glue TUs re-parse the same storage header closure once per file, so +# batching them cuts the repeated frontend work. Batch 12 bounds jumbo-TU size +# and memory. Opted out: +# - seven files whose file-scope macros (CL_MAX_PATH and friends, +# IS_CHINESE_CHAR, APPLY_FOR_PRIMITITYPE, RETURN_IF_NOT_EOF_AND_OK, +# FORMAT_*_ADD_JSON_NODE, RETURN_IF_ERROR_) must not leak into unity siblings +# - the three heaviest template-instantiation TUs (predicate creators) which +# would dominate any batch they join +# - compaction/collection_statistics.cpp: its test compiles it a second time +# by #including the .cpp; the test object must shadow a never-pulled archive +# member, but a unity batch is pulled in for its siblings and the linker +# sees a duplicate definition +set(STORAGE_UNITY_SKIP ${CMAKE_CURRENT_SOURCE_DIR}/index/inverted/inverted_index_compound_reader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/index/inverted/inverted_index_fs_directory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/index/inverted/tokenizer/basic/basic_tokenizer.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/index/zone_map/zone_map_index.cpp) -set_source_files_properties(${STORAGE_UNITY_SKIP} PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) + ${CMAKE_CURRENT_SOURCE_DIR}/index/zone_map/zone_map_index.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/iterator/vcollect_iterator.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tablet/tablet.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/task/engine_clone_task.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/predicate/predicate_creator_comparison.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/predicate/predicate_creator_in_list_in.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/predicate/predicate_creator_in_list_not_in.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/compaction/collection_statistics.cpp) +if (ENABLE_VARIANT_NESTED_GROUP) + # Out-of-tree module sources swapped into this target: unity hygiene + # unaudited, keep them individual. + list(APPEND STORAGE_UNITY_SKIP ${VARIANT_NESTED_GROUP_SOURCES}) +endif() +doris_skip_unity_inclusion(${STORAGE_UNITY_SKIP}) set_target_properties(Storage PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD} - UNITY_BUILD_BATCH_SIZE 32) + UNITY_BUILD_BATCH_SIZE 12) diff --git a/be/src/storage/index/ann/CMakeLists.txt b/be/src/storage/index/ann/CMakeLists.txt index a30bfce9773d43..d671cdbf024678 100644 --- a/be/src/storage/index/ann/CMakeLists.txt +++ b/be/src/storage/index/ann/CMakeLists.txt @@ -30,3 +30,8 @@ target_link_libraries(ann_index PUBLIC faiss OpenMP::OpenMP_CXX) # Some header files from faiss are used by doris, they will break compile check. target_compile_options(ann_index PRIVATE -Wno-shadow-field) + +# Unity build: ten homogeneous TUs sharing the faiss header closure; batch 0 +# merges them into a single unity TU. +set_target_properties(ann_index PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD} + UNITY_BUILD_BATCH_SIZE 0) diff --git a/be/src/storage/merger.cpp b/be/src/storage/merger.cpp index d986b367288853..c8abc92e7f7dc3 100644 --- a/be/src/storage/merger.cpp +++ b/be/src/storage/merger.cpp @@ -121,8 +121,8 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet, ReaderType reader_type, auto tablet_state = tablet->tablet_state(); if (tablet_state != TABLET_RUNNING && tablet_state != TABLET_NOTREADY) { tablet->clear_cache(); - return Status::Error("tablet {} is not used any more", - tablet->tablet_id()); + return Status::Error("tablet {} is not used any more", + tablet->tablet_id()); } // Read one block from block reader @@ -144,8 +144,8 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet, ReaderType reader_type, block.clear_column_data(); } if (ExecEnv::GetInstance()->storage_engine().stopped()) { - return Status::Error("tablet {} failed to do compaction, engine stopped", - tablet->tablet_id()); + return Status::Error( + "tablet {} failed to do compaction, engine stopped", tablet->tablet_id()); } if (stats_output != nullptr) { @@ -312,8 +312,8 @@ Status Merger::vertical_compact_one_group( auto tablet_state = tablet->tablet_state(); if (tablet_state != TABLET_RUNNING && tablet_state != TABLET_NOTREADY) { tablet->clear_cache(); - return Status::Error("tablet {} is not used any more", - tablet->tablet_id()); + return Status::Error("tablet {} is not used any more", + tablet->tablet_id()); } // Read one block from block reader RETURN_NOT_OK_STATUS_WITH_WARN(reader.next_block_with_aggregation(&block, &eof), @@ -335,8 +335,8 @@ Status Merger::vertical_compact_one_group( block.clear_column_data(); } if (ExecEnv::GetInstance()->storage_engine().stopped()) { - return Status::Error("tablet {} failed to do compaction, engine stopped", - tablet->tablet_id()); + return Status::Error( + "tablet {} failed to do compaction, engine stopped", tablet->tablet_id()); } if (stats_output != nullptr) { @@ -391,8 +391,8 @@ Status Merger::vertical_compact_one_group( block.clear_column_data(); } if (ExecEnv::GetInstance()->storage_engine().stopped()) { - return Status::Error("tablet {} failed to do compaction, engine stopped", - tablet_id); + return Status::Error( + "tablet {} failed to do compaction, engine stopped", tablet_id); } if (stats_output != nullptr) { diff --git a/be/src/storage/predicate/column_predicate.h b/be/src/storage/predicate/column_predicate.h index 2bcc8359464ce9..1d55531eb5ed60 100644 --- a/be/src/storage/predicate/column_predicate.h +++ b/be/src/storage/predicate/column_predicate.h @@ -349,7 +349,8 @@ class ColumnPredicate : public std::enable_shared_from_this { if (_predicate_input_rows_counter == nullptr || _predicate_filtered_rows_counter == nullptr || _predicate_always_true_rows_counter == nullptr) { - throw Exception(INTERNAL_ERROR, "Predicate profile counters are not initialized"); + throw Exception(ErrorCode::INTERNAL_ERROR, + "Predicate profile counters are not initialized"); } COUNTER_UPDATE(_predicate_input_rows_counter, input_rows); COUNTER_UPDATE(_predicate_filtered_rows_counter, filter_rows); @@ -397,7 +398,7 @@ class ColumnPredicate : public std::enable_shared_from_this { protected: virtual bool _can_ignore() const { return _runtime_filter_id != -1; } virtual uint16_t _evaluate_inner(const IColumn& column, uint16_t* sel, uint16_t size) const { - throw Exception(INTERNAL_ERROR, "Not Implemented _evaluate_inner"); + throw Exception(ErrorCode::INTERNAL_ERROR, "Not Implemented _evaluate_inner"); } void reset_judge_selectivity() const { _rf_selectivity.reset_judge_selectivity(); } diff --git a/be/src/storage/segment/common.h b/be/src/storage/segment/common.h index 166aa9fca5812f..a35fef0a97e0f0 100644 --- a/be/src/storage/segment/common.h +++ b/be/src/storage/segment/common.h @@ -30,5 +30,10 @@ namespace segment_v2 { using rowid_t = uint32_t; using ordinal_t = uint64_t; +// Magic footer bytes identifying a V2 segment file, written by the segment +// writers and checked when parsing the footer. +inline constexpr const char* k_segment_magic = "D0R1"; +inline constexpr uint32_t k_segment_magic_length = 4; + } // namespace segment_v2 } // namespace doris diff --git a/be/src/storage/segment/external_col_meta_util.cpp b/be/src/storage/segment/external_col_meta_util.cpp index 87bc31f1079c33..9bc4bfeba1352d 100644 --- a/be/src/storage/segment/external_col_meta_util.cpp +++ b/be/src/storage/segment/external_col_meta_util.cpp @@ -29,8 +29,8 @@ namespace doris::segment_v2 { namespace { -io::IOContext create_index_io_context(const io::IOContext* source_io_ctx, - OlapReaderStatistics* stats) { +io::IOContext create_ext_meta_io_context(const io::IOContext* source_io_ctx, + OlapReaderStatistics* stats) { io::IOContext io_ctx; if (source_io_ctx != nullptr) { io_ctx = *source_io_ctx; @@ -145,7 +145,7 @@ Status ExternalColMetaUtil::read_col_meta(const io::FileReaderSPtr& file_reader, std::string buf; buf.resize(static_cast(size)); size_t meta_read = 0; - io::IOContext io_ctx = create_index_io_context(source_io_ctx, stats); + io::IOContext io_ctx = create_ext_meta_io_context(source_io_ctx, stats); RETURN_IF_ERROR(file_reader->read_at(pos, Slice(buf.data(), buf.size()), &meta_read, &io_ctx)); if (meta_read != size) { return Status::Corruption("short read ColumnMetaPB: expect={}, actual={}", size, meta_read); diff --git a/be/src/storage/segment/segment.cpp b/be/src/storage/segment/segment.cpp index 9e58a76e0a16e1..be80f7ae530510 100644 --- a/be/src/storage/segment/segment.cpp +++ b/be/src/storage/segment/segment.cpp @@ -73,11 +73,11 @@ #include "storage/segment/column_meta_accessor.h" #include "storage/segment/column_reader.h" #include "storage/segment/column_reader_cache.h" +#include "storage/segment/common.h" // k_segment_magic #include "storage/segment/empty_segment_iterator.h" #include "storage/segment/page_io.h" #include "storage/segment/page_pointer.h" #include "storage/segment/segment_iterator.h" -#include "storage/segment/segment_writer.h" // k_segment_magic_length #include "storage/segment/stream_reader.h" #include "storage/segment/variant/variant_column_reader.h" #include "storage/tablet/tablet_schema.h" diff --git a/be/src/storage/segment/segment_writer.cpp b/be/src/storage/segment/segment_writer.cpp index ed2b403c75fee1..2bdefe39ef775e 100644 --- a/be/src/storage/segment/segment_writer.cpp +++ b/be/src/storage/segment/segment_writer.cpp @@ -62,6 +62,7 @@ #include "storage/rowset/rowset_writer_context.h" // RowsetWriterContext #include "storage/rowset/segment_creator.h" #include "storage/segment/column_writer.h" // ColumnWriter +#include "storage/segment/common.h" // k_segment_magic #include "storage/segment/encoding_info.h" #include "storage/segment/external_col_meta_util.h" #include "storage/segment/page_io.h" @@ -80,9 +81,6 @@ namespace segment_v2 { using namespace ErrorCode; -const char* k_segment_magic = "D0R1"; -const uint32_t k_segment_magic_length = 4; - inline std::string segment_mem_tracker_name(uint32_t segment_id) { return "SegmentWriter:Segment-" + std::to_string(segment_id); } diff --git a/be/src/storage/segment/segment_writer.h b/be/src/storage/segment/segment_writer.h index be22adcefaa9af..ba5d2f9af4462e 100644 --- a/be/src/storage/segment/segment_writer.h +++ b/be/src/storage/segment/segment_writer.h @@ -60,9 +60,6 @@ class FileWriter; namespace segment_v2 { -extern const char* k_segment_magic; -extern const uint32_t k_segment_magic_length; - class VariantStatsCaculator; struct SegmentWriterOptions { diff --git a/be/src/storage/segment/variant/v2/variant_path_builder.cpp b/be/src/storage/segment/variant/v2/variant_path_builder.cpp index c7542669fe40fc..d908963ef74bda 100644 --- a/be/src/storage/segment/variant/v2/variant_path_builder.cpp +++ b/be/src/storage/segment/variant/v2/variant_path_builder.cpp @@ -849,10 +849,6 @@ size_t dotted_path_depth(const PathInData& path) { return path.get_parts().size(); } -size_t path_allocated_bytes(const PathInData& path) { - return path.get_path().capacity() + path.get_parts().capacity() * sizeof(PathInData::Part); -} - size_t recursive_null_count(const IColumn& column) { if (const auto* nullable = check_and_get_column(column)) { size_t count = 0; diff --git a/be/src/storage/segment/variant/v2/variant_shredder.cpp b/be/src/storage/segment/variant/v2/variant_shredder.cpp index d61b51aed9610e..99c312a83269d9 100644 --- a/be/src/storage/segment/variant/v2/variant_shredder.cpp +++ b/be/src/storage/segment/variant/v2/variant_shredder.cpp @@ -35,6 +35,7 @@ #include "exec/common/variant_util.h" #include "exprs/function/parse/variant_jsonb_parse.h" #include "storage/tablet/tablet_schema.h" +#include "util/json/path_in_data.h" #include "util/jsonb_writer.h" namespace doris::segment_v2 { @@ -52,10 +53,6 @@ PathInData normalize_doc_publication_path(const PathInData& path) { return PathInData(path.get_path(), path.get_is_typed()); } -size_t path_allocated_bytes(const PathInData& path) { - return path.get_path().capacity() + path.get_parts().capacity() * sizeof(PathInData::Part); -} - } // namespace struct VariantShredder::Impl { diff --git a/be/src/storage/segment/vertical_segment_writer.cpp b/be/src/storage/segment/vertical_segment_writer.cpp index abbdb857bc4815..319597a04183bf 100644 --- a/be/src/storage/segment/vertical_segment_writer.cpp +++ b/be/src/storage/segment/vertical_segment_writer.cpp @@ -68,6 +68,7 @@ #include "storage/rowset/rowset_writer_context.h" // RowsetWriterContext #include "storage/rowset/segment_creator.h" #include "storage/segment/column_writer.h" // ColumnWriter +#include "storage/segment/common.h" // k_segment_magic #include "storage/segment/encoding_info.h" #include "storage/segment/external_col_meta_util.h" #include "storage/segment/historical_row_retriever.h" @@ -88,9 +89,6 @@ namespace doris::segment_v2 { using namespace ErrorCode; -static constexpr const char* k_segment_magic = "D0R1"; -static constexpr uint32_t k_segment_magic_length = 4; - inline std::string vertical_segment_writer_mem_tracker_name(uint32_t segment_id) { return "VerticalSegmentWriter:Segment-" + std::to_string(segment_id); } diff --git a/be/src/udf/CMakeLists.txt b/be/src/udf/CMakeLists.txt index 0d57e85f5b193d..45e28f93cb40d4 100755 --- a/be/src/udf/CMakeLists.txt +++ b/be/src/udf/CMakeLists.txt @@ -26,3 +26,8 @@ file(GLOB PYTHON_UDF_SOURCES "python/*.cpp") # can have dependencies on our other libs. The second version is shipped as part # of the UDF sdk, which can't use other libs. add_library(Udf STATIC ${PYTHON_UDF_SOURCES}) + +# Unity build: eight homogeneous python-UDF glue TUs sharing the Python/arrow +# header closure; batch 0 merges them into a single unity TU. +set_target_properties(Udf PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD} + UNITY_BUILD_BATCH_SIZE 0) diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt index f23c3ad865c89b..8941a7ed57fe8e 100644 --- a/be/src/util/CMakeLists.txt +++ b/be/src/util/CMakeLists.txt @@ -41,3 +41,22 @@ add_library(Util STATIC ) pch_reuse(Util) + +# Unity build for the small util TUs (parse-overhead dominated). Ten files +# leak file-scope macros and stay individual; hash/city.cc keeps its generic +# k0..k4 static hash constants away from sibling scopes. perf_counters.cpp +# is a Linux-only member (macOS builds its _mac variant, which is clean). +set(UTIL_UNITY_SKIP + ${CMAKE_CURRENT_SOURCE_DIR}/bfd_parser.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/hash/city.cc + ${CMAKE_CURRENT_SOURCE_DIR}/hash/murmur_hash3.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/network_util.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/os_info.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/os_util.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/perf_counters.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/simd/parquet_kernels.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/thrift_util.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/zlib.cpp) +doris_skip_unity_inclusion(${UTIL_UNITY_SKIP}) +set_target_properties(Util PROPERTIES UNITY_BUILD ${DORIS_UNITY_BUILD} + UNITY_BUILD_BATCH_SIZE 24) diff --git a/be/src/util/bvar_windowed_adder.h b/be/src/util/bvar_windowed_adder.h index c4e9245b7e3246..f249dc3a198291 100644 --- a/be/src/util/bvar_windowed_adder.h +++ b/be/src/util/bvar_windowed_adder.h @@ -33,6 +33,12 @@ namespace doris { +// Shared window spans (seconds) for multi-window metrics. +// bvar::Window enforces MAX_SECONDS_LIMIT = 3600, so the longest window is 1h. +inline constexpr int WINDOW_5M = 300; +inline constexpr int WINDOW_30M = 1800; +inline constexpr int WINDOW_1H = 3600; + /** * Multi-dimension windowed adder. * diff --git a/be/src/util/json/path_in_data.h b/be/src/util/json/path_in_data.h index 593904dd931462..2e549cb714a83c 100644 --- a/be/src/util/json/path_in_data.h +++ b/be/src/util/json/path_in_data.h @@ -135,6 +135,12 @@ class PathInData { bool is_typed = false; }; +// Heap bytes retained beyond sizeof(PathInData): the dotted-path string buffer +// plus the parts vector. +inline size_t path_allocated_bytes(const PathInData& path) { + return path.get_path().capacity() + path.get_parts().capacity() * sizeof(PathInData::Part); +} + class PathInDataBuilder { public: const PathInData::Parts& get_parts() const { return parts; } diff --git a/be/src/util/sm3.cpp b/be/src/util/sm3.cpp index 2d4349abc840ec..448f2365e5f12d 100644 --- a/be/src/util/sm3.cpp +++ b/be/src/util/sm3.cpp @@ -46,11 +46,11 @@ void SM3Digest::digest() { char hex_buf[2 * SM3_DIGEST_LENGTH]; - static char dig_vec_lower[] = "0123456789abcdef"; + static constexpr char sm3_dig_vec_lower[] = "0123456789abcdef"; char* to = hex_buf; for (int i = 0; i < SM3_DIGEST_LENGTH; ++i) { - *to++ = dig_vec_lower[buf[i] >> 4]; - *to++ = dig_vec_lower[buf[i] & 0x0F]; + *to++ = sm3_dig_vec_lower[buf[i] >> 4]; + *to++ = sm3_dig_vec_lower[buf[i] & 0x0F]; } _hex.assign(hex_buf, 2 * SM3_DIGEST_LENGTH); } diff --git a/be/src/util/timezone_utils.cpp b/be/src/util/timezone_utils.cpp index b575ed7a29f8c3..358d8b962cc5fd 100644 --- a/be/src/util/timezone_utils.cpp +++ b/be/src/util/timezone_utils.cpp @@ -51,7 +51,7 @@ namespace doris { using ZoneList = std::unordered_map; -RE2 time_zone_offset_format_reg(R"(^[+-]{1}\d{2}\:\d{2}$)"); // visiting is thread-safe +static RE2 tz_offset_format_reg(R"(^[+-]{1}\d{2}\:\d{2}$)"); // visiting is thread-safe // for ut, make it never nullptr. std::unique_ptr lower_zone_cache_ = std::make_unique(); @@ -277,8 +277,8 @@ bool TimezoneUtils::parse_tz_offset_string(const std::string& timezone, cctz::ti } re2::StringPiece value; - if (time_zone_offset_format_reg.Match(normalized, 0, normalized.size(), RE2::UNANCHORED, &value, - 1)) [[likely]] { + if (tz_offset_format_reg.Match(normalized, 0, normalized.size(), RE2::UNANCHORED, &value, 1)) + [[likely]] { const bool positive = value[0] != '-'; const int hour = std::stoi(value.substr(1, 2).as_string()); const int minute = std::stoi(value.substr(4, 2).as_string()); diff --git a/be/test/io/cache/fs_file_cache_storage_leak_cleaner_test.cpp b/be/test/io/cache/fs_file_cache_storage_leak_cleaner_test.cpp index a4b5035ad457ca..0ab9953cb54309 100644 --- a/be/test/io/cache/fs_file_cache_storage_leak_cleaner_test.cpp +++ b/be/test/io/cache/fs_file_cache_storage_leak_cleaner_test.cpp @@ -131,8 +131,10 @@ class FSFileCacheLeakCleanerTest : public BlockFileCacheTest { BlockMetaKey mkey(0, hash, offset); BlockMeta meta(FileCacheType::NORMAL, 16, 0); storage._meta_store->put(mkey, meta); - // Wait for async write to complete for test stability - for (int i = 0; i < 100 && storage._meta_store->get_write_queue_size() > 0; ++i) { + // The async worker dequeues an operation before it lands in rocksdb, + // so an empty write queue does not mean the entry is readable yet. + // Poll the store itself: leak-scan decisions count rocksdb contents. + for (int i = 0; i < 100 && !storage._meta_store->get(mkey).has_value(); ++i) { std::this_thread::sleep_for(std::chrono::milliseconds(50)); } }