From 58395849ccec6f2b1e575939ec16e593610222c6 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 1 Aug 2026 16:15:23 +0800 Subject: [PATCH 01/29] [fix](iceberg) Harden write lifecycle and cleanup Problem Summary: Iceberg writes could bypass cleanup, miss overwrite conflicts, undercount partition sorter memory, hide close errors, exceed report limits, or leave incomplete remote files. Partition dispatch also allocated full-block filters for every partition. Solution: Unify failure handling, preserve the durable commit point, validate overwrite conflicts, account for all active sort writers, propagate close failures, bound commit metadata, abort incomplete uploads, add orphan cleanup, and use compact row permutations. --- .../spill_iceberg_table_sink_operator.cpp | 53 +++--- .../pipeline/pipeline_fragment_context.cpp | 10 +- be/src/exec/sink/viceberg_delete_sink.cpp | 8 +- .../iceberg/viceberg_partition_writer.cpp | 33 +++- .../writer/iceberg/viceberg_sort_writer.cpp | 91 +++------- .../writer/iceberg/viceberg_table_writer.cpp | 66 +++---- .../writer/iceberg/viceberg_table_writer.h | 20 +-- be/src/io/fs/file_writer.h | 3 + be/src/io/fs/obj_storage_client.h | 4 + .../io/fs/rate_limited_obj_storage_client.cpp | 9 + .../io/fs/rate_limited_obj_storage_client.h | 1 + be/src/io/fs/s3_file_writer.cpp | 43 ++++- be/src/io/fs/s3_file_writer.h | 3 +- be/src/io/fs/s3_obj_storage_client.cpp | 19 ++ be/src/io/fs/s3_obj_storage_client.h | 1 + be/src/runtime/runtime_state.cpp | 24 +++ be/src/runtime/runtime_state.h | 10 +- .../iceberg/iceberg_partition_writer_test.cpp | 37 +++- .../iceberg/iceberg_table_writer_test.cpp | 100 +++++++++++ be/test/io/fs/s3_file_writer_test.cpp | 15 ++ .../runtime_state_block_budget_test.cpp | 16 ++ .../iceberg/IcebergConnectorTransaction.java | 23 ++- .../iceberg/IcebergProcedureOps.java | 12 +- .../action/IcebergExecuteActionFactory.java | 8 +- .../IcebergRemoveOrphanFilesAction.java | 163 ++++++++++++++++++ .../IcebergConnectorTransactionTest.java | 45 +++++ .../IcebergExecuteActionFactoryTest.java | 37 ++-- .../insert/AbstractInsertExecutor.java | 13 +- .../BaseExternalTableInsertExecutor.java | 19 +- .../insert/OlapInsertExecutorTest.java | 46 +++++ .../PluginDrivenInsertExecutorTest.java | 12 ++ 31 files changed, 752 insertions(+), 192 deletions(-) create mode 100644 be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp create mode 100644 fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java diff --git a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp index cf7c8e6e1a1538..083d7b86c87c0f 100644 --- a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp +++ b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp @@ -55,26 +55,26 @@ size_t SpillIcebergTableSinkLocalState::get_reserve_mem_size(RuntimeState* state if (!_writer) { return 0; } - auto current_writer = _writer->current_writer(); - auto* sort_writer = dynamic_cast(current_writer.get()); - if (!sort_writer) { - return 0; + size_t reserve_size = 0; + for (const auto& writer : *_writer->active_writers()) { + if (auto* sort_writer = dynamic_cast(writer.get())) { + reserve_size += sort_writer->get_reserve_mem_size(state, eos); + } } - - return sort_writer->get_reserve_mem_size(state, eos); + return reserve_size; } size_t SpillIcebergTableSinkLocalState::get_revocable_mem_size(RuntimeState* state) const { if (!_writer) { return 0; } - auto current_writer = _writer->current_writer(); - auto* sort_writer = dynamic_cast(current_writer.get()); - if (!sort_writer) { - return 0; + size_t revocable_size = 0; + for (const auto& writer : *_writer->active_writers()) { + if (auto* sort_writer = dynamic_cast(writer.get())) { + revocable_size += sort_writer->data_size(); + } } - - return sort_writer->data_size(); + return revocable_size; } Status SpillIcebergTableSinkLocalState::revoke_memory(RuntimeState* state) { @@ -82,20 +82,23 @@ Status SpillIcebergTableSinkLocalState::revoke_memory(RuntimeState* state) { if (!_writer) { return Status::OK(); } - auto current_writer = _writer->current_writer(); - auto* sort_writer = dynamic_cast(current_writer.get()); - if (!sort_writer) { - return Status::OK(); + std::shared_ptr largest_writer; + size_t largest_size = 0; + for (const auto& writer : *_writer->active_writers()) { + if (auto* sort_writer = dynamic_cast(writer.get())) { + size_t size = sort_writer->data_size(); + if (size > largest_size) { + largest_size = size; + largest_writer = writer; + } + } } - - auto exception_catch_func = [current_writer, sort_writer]() { - auto status = [&]() { - RETURN_IF_CATCH_EXCEPTION({ return sort_writer->trigger_spill(); }); - }(); - return status; - }; - - return exception_catch_func(); + if (largest_writer != nullptr) { + // Repeated revocation drains the largest partition first without launching O(P) spill jobs at once. + auto* sort_writer = dynamic_cast(largest_writer.get()); + RETURN_IF_CATCH_EXCEPTION({ RETURN_IF_ERROR(sort_writer->trigger_spill()); }); + } + return Status::OK(); } SpillIcebergTableSinkOperatorX::SpillIcebergTableSinkOperatorX( diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index 84840e98f1f799..4f2c6dc00483e6 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -2515,16 +2515,14 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r } } } - if (auto icd = req.runtime_state->iceberg_commit_datas(); !icd.empty()) { + req.runtime_state->append_iceberg_commit_datas(¶ms.iceberg_commit_datas); + if (!params.iceberg_commit_datas.empty()) { params.__isset.iceberg_commit_datas = true; - params.iceberg_commit_datas.insert(params.iceberg_commit_datas.end(), icd.begin(), - icd.end()); } else if (!req.runtime_states.empty()) { for (auto* rs : req.runtime_states) { - if (auto rs_icd = rs->iceberg_commit_datas(); !rs_icd.empty()) { + rs->append_iceberg_commit_datas(¶ms.iceberg_commit_datas); + if (!params.iceberg_commit_datas.empty()) { params.__isset.iceberg_commit_datas = true; - params.iceberg_commit_datas.insert(params.iceberg_commit_datas.end(), - rs_icd.begin(), rs_icd.end()); } } } diff --git a/be/src/exec/sink/viceberg_delete_sink.cpp b/be/src/exec/sink/viceberg_delete_sink.cpp index 172fdd28177c62..a3dcb1df22f657 100644 --- a/be/src/exec/sink/viceberg_delete_sink.cpp +++ b/be/src/exec/sink/viceberg_delete_sink.cpp @@ -283,8 +283,12 @@ Status VIcebergDeleteSink::close(Status close_status) { _delete_file_count); if (_state != nullptr) { - for (const auto& commit_data : _commit_data_list) { - _state->add_iceberg_commit_datas(commit_data); + for (auto& commit_data : _commit_data_list) { + Status report_status = _state->add_iceberg_commit_datas(std::move(commit_data)); + if (!report_status.ok()) { + _cleanup_created_files(); + return report_status; + } } } diff --git a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp index ba7644daec751f..7faaebe9525e88 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp @@ -69,6 +69,7 @@ Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profil io::FileWriterOptions file_writer_options = {.used_by_s3_committer = false}; RETURN_IF_ERROR(_fs->create_file(file_description.path, &_file_writer, &file_writer_options)); + Status open_status; switch (_file_format_type) { case TFileFormatType::FORMAT_PARQUET: { TParquetCompressionType::type parquet_compression_type; @@ -92,9 +93,13 @@ Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profil break; } default: { - return Status::InternalError("Unsupported compress type {} with parquet", - to_string(_compress_type)); + open_status = Status::InternalError("Unsupported compress type {} with parquet", + to_string(_compress_type)); + break; + } } + if (!open_status.ok()) { + break; } ParquetFileOptions parquet_options = {.compression_type = parquet_compression_type, .parquet_version = TParquetVersion::PARQUET_1_0, @@ -103,19 +108,28 @@ Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profil _file_format_transformer = std::make_unique( state, _file_writer.get(), _write_output_expr_ctxs, _write_column_names, false, parquet_options, _iceberg_schema_json, &_schema); - return _file_format_transformer->open(); + open_status = _file_format_transformer->open(); + break; } case TFileFormatType::FORMAT_ORC: { _file_format_transformer = std::make_unique( state, _file_writer.get(), _write_output_expr_ctxs, "", _write_column_names, false, _compress_type, &_schema, _fs); - return _file_format_transformer->open(); + open_status = _file_format_transformer->open(); + break; } default: { - return Status::InternalError("Unsupported file format type {}", - to_string(_file_format_type)); + open_status = Status::InternalError("Unsupported file format type {}", + to_string(_file_format_type)); + break; } } + if (!open_status.ok()) { + // A transformer failure happens after object creation, so abort multipart state before deleting the path. + WARN_IF_ERROR(_file_writer->abort(), "failed to abort Iceberg file after open error"); + WARN_IF_ERROR(_fs->delete_file(_path), "failed to delete Iceberg file after open error"); + } + return open_status; } Status VIcebergPartitionWriter::close(const Status& status) { @@ -147,7 +161,12 @@ Status VIcebergPartitionWriter::close(const Status& status) { } return commit_status; } - _state->add_iceberg_commit_datas(commit_data); + Status report_status = _state->add_iceberg_commit_datas(std::move(commit_data)); + if (!report_status.ok()) { + // A closed object that cannot be reported can never be committed, so remove it immediately. + WARN_IF_ERROR(_fs->delete_file(_path), "failed to delete unreportable Iceberg file"); + return report_status; + } if (_closed_file_callback) { _closed_file_callback(_fs, _path); } diff --git a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp index 6081166777fc28..168199f1d00849 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp @@ -102,80 +102,35 @@ Status VIcebergSortWriter::close(const Status& status) { } Status VIcebergSortWriter::_close_locked(const Status& status) { - // Track the actual internal status of operations performed during close. - // This is important because if intermediate operations (like do_sort()) fail, - // we need to propagate the actual error status to the underlying partition writer's - // close() call, rather than the original status parameter which could be OK. - Status internal_status = Status::OK(); - // Track the close status of the underlying partition writer. - // If _iceberg_partition_writer->close() fails (e.g., Parquet file flush error), - // we must propagate this error to the caller to avoid silent data loss. - Status close_status = Status::OK(); - - // Defer ensures the underlying partition writer is always closed and - // spill streams are cleaned up, regardless of whether intermediate operations succeed. - // Uses internal_status to propagate any errors that occurred during close operations. - Defer defer {[&]() { - // If any intermediate operation failed, pass that error to the partition writer; - // otherwise, pass the original status from the caller. - close_status = - _iceberg_partition_writer->close(internal_status.ok() ? status : internal_status); - if (!close_status.ok()) { - LOG(WARNING) << fmt::format("_iceberg_partition_writer close failed, reason: {}", - close_status.to_string()); - } - _cleanup_spill_streams(); - }}; - - // If the original status is already an error or the query is cancelled, - // skip all close operations and propagate the original error - if (!status.ok() || _runtime_state->is_cancelled()) { - return status; - } - - // If sorter was never initialized (e.g., no data was written), nothing to do - if (_sorter == nullptr) { - return Status::OK(); - } - - // Check if there is any remaining data in the sorter (either unsorted or already sorted blocks) - if (!_sorter->merge_sort_state()->unsorted_block()->empty() || - !_sorter->merge_sort_state()->get_sorted_block().empty()) { - if (_sorted_spill_files.empty()) { - // No spill has occurred, all data is in memory. - // Sort the remaining data, prepare for reading, and write to file. - internal_status = _sorter->do_sort(); - if (!internal_status.ok()) { - return internal_status; - } - internal_status = _sorter->prepare_for_read(false); - if (!internal_status.ok()) { - return internal_status; + Status internal_status = status; + if (status.ok() && !_runtime_state->is_cancelled()) { + internal_status = Status::OK(); + if (_sorter != nullptr && (!_sorter->merge_sort_state()->unsorted_block()->empty() || + !_sorter->merge_sort_state()->get_sorted_block().empty())) { + if (_sorted_spill_files.empty()) { + internal_status = _sorter->do_sort(); + if (internal_status.ok()) { + internal_status = _sorter->prepare_for_read(false); + } + if (internal_status.ok()) { + internal_status = _write_sorted_data(); + } + } else { + internal_status = _do_spill(); } - internal_status = _write_sorted_data(); - return internal_status; } - - // Some data has already been spilled to disk. - // Spill the remaining in-memory data to a new spill stream. - internal_status = _do_spill(); - if (!internal_status.ok()) { - return internal_status; + if (internal_status.ok() && !_sorted_spill_files.empty()) { + internal_status = _combine_files_output(); } } - // Merge all spilled streams using multi-way merge sort and output final sorted data to files - if (!_sorted_spill_files.empty()) { - internal_status = _combine_files_output(); - if (!internal_status.ok()) { - return internal_status; - } + // Form the return value only after the underlying close runs; a deferred assignment is too late. + Status close_status = + _iceberg_partition_writer->close(internal_status.ok() ? status : internal_status); + _cleanup_spill_streams(); + if (!internal_status.ok()) { + return internal_status; } - - // Return close_status if internal operations succeeded but the underlying - // partition writer's close() failed (e.g., file flush error). - // This prevents silent data loss where the caller thinks the write succeeded - // but the file was not properly closed. return close_status; } diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp index 3519252d7ca940..52157c5a80ab27 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp @@ -49,6 +49,7 @@ VIcebergTableWriter::VIcebergTableWriter(const TDataSink& t_sink, std::shared_ptr fin_dep) : AsyncResultWriter(output_expr_ctxs, dep, fin_dep), _t_sink(t_sink) { DCHECK(_t_sink.__isset.iceberg_table_sink); + _active_writers.store(std::make_shared()); } Status VIcebergTableWriter::open(RuntimeState* state, RuntimeProfile* profile) { @@ -329,7 +330,8 @@ Status VIcebergTableWriter::_process_row_lineage_columns(Block& block) { Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { RETURN_IF_ERROR(_process_row_lineage_columns(output_block)); - std::unordered_map, IColumn::Filter> writer_positions; + std::unordered_map, IColumn::Permutation> + writer_positions; _row_count += output_block.rows(); // Case 1: Full static partition - all data goes to a single partition @@ -346,6 +348,7 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { } _partitions_to_writers.insert({_static_partition_path, writer}); RETURN_IF_ERROR(writer->open(_state, _operator_profile, _row_desc)); + _publish_active_writers(); } else { if (writer_iter->second->written_len() > _target_file_size_bytes) { std::string file_name(writer_iter->second->file_name()); @@ -355,6 +358,7 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { RETURN_IF_ERROR(writer_iter->second->close(Status::OK())); } _partitions_to_writers.erase(writer_iter); + _publish_active_writers(); try { writer = _create_partition_writer(nullptr, -1, &file_name, file_name_index + 1); @@ -363,6 +367,7 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { } _partitions_to_writers.insert({_static_partition_path, writer}); RETURN_IF_ERROR(writer->open(_state, _operator_profile, _row_desc)); + _publish_active_writers(); } else { writer = writer_iter->second; } @@ -371,7 +376,6 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { SCOPED_RAW_TIMER(&_partition_writers_write_ns); output_block.erase(_non_write_columns_indices); RETURN_IF_ERROR(writer->write(output_block)); - _current_writer.store(writer); return Status::OK(); } @@ -389,6 +393,7 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { } _partitions_to_writers.insert({"", writer}); RETURN_IF_ERROR(writer->open(_state, _operator_profile, _row_desc)); + _publish_active_writers(); } else { if (writer_iter->second->written_len() > _target_file_size_bytes) { std::string file_name(writer_iter->second->file_name()); @@ -398,6 +403,7 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { RETURN_IF_ERROR(writer_iter->second->close(Status::OK())); } _partitions_to_writers.erase(writer_iter); + _publish_active_writers(); try { writer = _create_partition_writer(nullptr, -1, &file_name, file_name_index + 1); @@ -406,6 +412,7 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { } _partitions_to_writers.insert({"", writer}); RETURN_IF_ERROR(writer->open(_state, _operator_profile, _row_desc)); + _publish_active_writers(); } else { writer = writer_iter->second; } @@ -414,7 +421,6 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { SCOPED_RAW_TIMER(&_partition_writers_write_ns); output_block.erase(_non_write_columns_indices); RETURN_IF_ERROR(writer->write(output_block)); - _current_writer.store(writer); return Status::OK(); } @@ -466,10 +472,8 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { auto writer = _create_partition_writer(&transformed_block, position, file_name, file_name_index); RETURN_IF_ERROR(writer->open(_state, _operator_profile, _row_desc)); - IColumn::Filter filter(output_block.rows(), 0); - filter[position] = 1; - writer_positions.insert({writer, std::move(filter)}); _partitions_to_writers.insert({partition_name, writer}); + _publish_active_writers(); writer_ptr = writer; } catch (doris::Exception& e) { return e.to_status(); @@ -478,8 +482,8 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { }; auto writer_iter = _partitions_to_writers.find(partition_name); + std::shared_ptr writer; if (writer_iter == _partitions_to_writers.end()) { - std::shared_ptr writer; if (_partitions_to_writers.size() + 1 > config::table_sink_partition_write_max_partition_nums_per_writer) { return Status::InternalError( @@ -488,7 +492,6 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { } RETURN_IF_ERROR(create_and_open_writer(partition_name, i, nullptr, 0, writer)); } else { - std::shared_ptr writer; if (writer_iter->second->written_len() > _target_file_size_bytes) { std::string file_name(writer_iter->second->file_name()); int file_name_index = writer_iter->second->file_name_index(); @@ -498,53 +501,53 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { } writer_positions.erase(writer_iter->second); _partitions_to_writers.erase(writer_iter); + _publish_active_writers(); RETURN_IF_ERROR(create_and_open_writer(partition_name, i, &file_name, file_name_index + 1, writer)); } else { writer = writer_iter->second; } - auto writer_pos_iter = writer_positions.find(writer); - if (writer_pos_iter == writer_positions.end()) { - IColumn::Filter filter(output_block.rows(), 0); - filter[i] = 1; - writer_positions.insert({writer, std::move(filter)}); - } else { - writer_pos_iter->second[i] = 1; - } + } + auto writer_pos_iter = writer_positions.find(writer); + if (writer_pos_iter == writer_positions.end()) { + IColumn::Permutation rows {static_cast(i)}; + writer_positions.insert({writer, std::move(rows)}); + } else { + writer_pos_iter->second.push_back(static_cast(i)); } } } SCOPED_RAW_TIMER(&_partition_writers_write_ns); output_block.erase(_non_write_columns_indices); for (auto it = writer_positions.begin(); it != writer_positions.end(); ++it) { - Block filtered_block; - RETURN_IF_ERROR(_filter_block(output_block, &it->second, &filtered_block)); - RETURN_IF_ERROR(it->first->write(filtered_block)); - _current_writer.store(it->first); + Block selected_block; + RETURN_IF_ERROR(_select_block(output_block, it->second, &selected_block)); + RETURN_IF_ERROR(it->first->write(selected_block)); } return Status::OK(); } -Status VIcebergTableWriter::_filter_block(doris::Block& block, const IColumn::Filter* filter, +Status VIcebergTableWriter::_select_block(doris::Block& block, const IColumn::Permutation& rows, doris::Block* output_block) { const ColumnsWithTypeAndName& columns_with_type_and_name = block.get_columns_with_type_and_name(); ColumnsWithTypeAndName result_columns; + result_columns.reserve(columns_with_type_and_name.size()); for (const auto& col : columns_with_type_and_name) { - result_columns.emplace_back(col.column->clone_resized(col.column->size()), col.type, - col.name); + // Across all partitions the permutations contain exactly one entry per input row, avoiding O(P*C*R). + result_columns.emplace_back(col.column->permute(rows, rows.size()), col.type, col.name); } *output_block = {std::move(result_columns)}; + return Status::OK(); +} - std::vector columns_to_filter; - int column_to_keep = output_block->columns(); - columns_to_filter.resize(column_to_keep); - for (uint32_t i = 0; i < column_to_keep; ++i) { - columns_to_filter[i] = i; +void VIcebergTableWriter::_publish_active_writers() { + auto snapshot = std::make_shared(); + snapshot->reserve(_partitions_to_writers.size()); + for (const auto& entry : _partitions_to_writers) { + snapshot->push_back(entry.second); } - - Block::filter_block_internal(output_block, columns_to_filter, *filter); - return Status::OK(); + _active_writers.store(std::move(snapshot)); } Status VIcebergTableWriter::close(Status status) { @@ -564,6 +567,7 @@ Status VIcebergTableWriter::close(Status status) { } } _partitions_to_writers.clear(); + _publish_active_writers(); } if (status.ok()) { SCOPED_TIMER(_operator_profile->total_time_counter()); diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h index 20d9562c35ee52..ce3c920986388e 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h @@ -73,23 +73,16 @@ class VIcebergTableWriter final : public AsyncResultWriter { TIcebergWriteType::type write_type() const { return _write_type; } - // Getter for the current partition writer. - // Used by SpillIcebergTableSinkLocalState to access the current writer for - // memory management operations (get_reserve_mem_size, revocable_mem_size, etc.). - // Returns a snapshot by value: the async writer thread updates _current_writer - // concurrently with the spill/revoke path, so callers must hold their own copy - // while operating on it instead of dereferencing the underlying member directly. - std::shared_ptr current_writer() const { return _current_writer.load(); } + using ActiveWriterSnapshot = std::vector>; + std::shared_ptr active_writers() const { return _active_writers.load(); } private: FRIEND_TEST(VIcebergTableWriterTest, RejectMissingPartitionSource); FRIEND_TEST(VIcebergTableWriterTest, ResolvesNestedPartitionSource); + friend class VIcebergTableWriterTest; - // The currently active partition writer (may be VIcebergPartitionWriter or VIcebergSortWriter). - // Updated during write() to track which writer received the most recent data. - // Wrapped in atomic_shared_ptr because revoke_memory / get_revocable_mem_size run on - // a different thread than the async writer that assigns to it. - doris::atomic_shared_ptr _current_writer; + // The spill thread needs a stable view of every partition sorter, while the async writer owns the map. + doris::atomic_shared_ptr _active_writers; class IcebergPartitionColumn { public: IcebergPartitionColumn(const iceberg::PartitionField& field, @@ -150,8 +143,9 @@ class VIcebergTableWriter final : public AsyncResultWriter { std::string _compute_file_name(); - Status _filter_block(doris::Block& block, const IColumn::Filter* filter, + Status _select_block(doris::Block& block, const IColumn::Permutation& rows, doris::Block* output_block); + void _publish_active_writers(); Status _write_prepared_block(Block& output_block); Status _process_row_lineage_columns(Block& block); diff --git a/be/src/io/fs/file_writer.h b/be/src/io/fs/file_writer.h index 9402fdef18303c..08754ec3689a0f 100644 --- a/be/src/io/fs/file_writer.h +++ b/be/src/io/fs/file_writer.h @@ -73,6 +73,9 @@ class FileWriter { // If there is no data appended, an empty file will be persisted. virtual Status close(bool non_block = false) = 0; + // Abandon an unpublished file. Remote writers should cancel multipart state instead of completing it. + virtual Status abort() { return close(); } + // Non-blocking probe for a previous close(true). // OK means close finished successfully. NeedSendAgain means close is still running. // Other errors mean close finished with error or the writer does not support this API. diff --git a/be/src/io/fs/obj_storage_client.h b/be/src/io/fs/obj_storage_client.h index fa239ca3282e2a..5db9071e64538c 100644 --- a/be/src/io/fs/obj_storage_client.h +++ b/be/src/io/fs/obj_storage_client.h @@ -106,6 +106,10 @@ class ObjStorageClient { virtual ObjectStorageResponse complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) = 0; + virtual ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions&) { + return {.status = {.code = ErrorCode::NOT_IMPLEMENTED_ERROR, + .msg = "abort multipart upload is not supported"}}; + } // According to the passed bucket and key, it will access whether the corresponding file exists in the object storage. // If it exists, it will return the corresponding file size virtual ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) = 0; diff --git a/be/src/io/fs/rate_limited_obj_storage_client.cpp b/be/src/io/fs/rate_limited_obj_storage_client.cpp index 1b8730847162df..73cf80d73c1630 100644 --- a/be/src/io/fs/rate_limited_obj_storage_client.cpp +++ b/be/src/io/fs/rate_limited_obj_storage_client.cpp @@ -73,6 +73,15 @@ ObjectStorageResponse RateLimitedObjStorageClient::complete_multipart_upload( return _inner->complete_multipart_upload(opts, completed_parts); } +ObjectStorageResponse RateLimitedObjStorageClient::abort_multipart_upload( + const ObjectStoragePathOptions& opts) { + S3RateLimitGuard guard(S3RateLimitType::PUT, 0); + if (!guard.ok()) { + return rate_limited_response(S3RateLimitType::PUT, guard.reject_reason()); + } + return _inner->abort_multipart_upload(opts); +} + ObjectStorageHeadResponse RateLimitedObjStorageClient::head_object( const ObjectStoragePathOptions& opts) { S3RateLimitGuard guard(S3RateLimitType::GET, 0); diff --git a/be/src/io/fs/rate_limited_obj_storage_client.h b/be/src/io/fs/rate_limited_obj_storage_client.h index 00725d7edcb299..dc6fb1503c375d 100644 --- a/be/src/io/fs/rate_limited_obj_storage_client.h +++ b/be/src/io/fs/rate_limited_obj_storage_client.h @@ -50,6 +50,7 @@ class RateLimitedObjStorageClient final : public ObjStorageClient { ObjectStorageResponse complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) override; + ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions& opts) override; ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override; ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, size_t offset, size_t bytes_read, diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index f8b836607a14a6..0ebb616a0e8cb5 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -78,6 +78,8 @@ S3FileWriter::~S3FileWriter() { // For thread safety std::ignore = _async_close_pack->future.get(); _async_close_pack = nullptr; + } else if (state() == State::OPENED) { + WARN_IF_ERROR(abort(), "failed to abort unfinished S3 writer"); } else { // Consider one situation where the file writer is destructed after it submit at least one async task // without calling close(), then there exists one occasion where the async task is executed right after @@ -85,13 +87,42 @@ S3FileWriter::~S3FileWriter() { _wait_until_finish(fmt::format("wait s3 file {} upload to be finished", _obj_storage_path_opts.path.native())); } - // We won't do S3 abort operation in BE, we let s3 service do it own. if (state() == State::OPENED && !_failed) { s3_bytes_written_total << _bytes_appended; } s3_file_being_written << -1; } +Status S3FileWriter::abort() { + if (state() == State::CLOSED) { + return Status::OK(); + } + if (state() == State::ASYNC_CLOSING) { + return Status::InternalError("cannot abort an asynchronously closing S3 writer"); + } + RETURN_IF_ERROR(_abort_impl()); + _state = State::CLOSED; + return Status::OK(); +} + +Status S3FileWriter::_abort_impl() { + _wait_until_finish( + fmt::format("wait s3 file {} before abort", _obj_storage_path_opts.path.native())); + _pending_buf.reset(); + if (_obj_storage_path_opts.upload_id.has_value()) { + const auto& client = _obj_client->get(); + if (client == nullptr) { + return Status::InternalError("invalid obj storage client"); + } + auto response = client->abort_multipart_upload(_obj_storage_path_opts); + if (response.status.code != ErrorCode::OK) { + return {response.status.code, std::move(response.status.msg)}; + } + } + // Once abort returns, no destructor or retry may complete the abandoned upload. + return Status::OK(); +} + Status S3FileWriter::_create_multi_upload_request() { LOG(INFO) << "create_multi_upload_request " << _obj_storage_path_opts.path.native(); const auto& client = _obj_client->get(); @@ -162,6 +193,10 @@ Status S3FileWriter::close(bool non_block) { s3_file_writer_async_close_queuing << -1; s3_file_writer_async_close_processing << 1; _st = _close_impl(); + if (!_st.ok()) { + // A failed completion must not leave server-side multipart state behind. + WARN_IF_ERROR(_abort_impl(), "failed to abort incomplete S3 upload"); + } _async_close_pack->promise.set_value(_st); s3_file_writer_async_close_processing << -1; }); @@ -172,12 +207,18 @@ Status S3FileWriter::close(bool non_block) { << _obj_storage_path_opts.path.native() << ", fallback to sync close, status=" << submit_status; _st = _close_impl(); + if (!_st.ok()) { + WARN_IF_ERROR(_abort_impl(), "failed to abort incomplete S3 upload"); + } _async_close_pack->promise.set_value(_st); return _st; } return Status::OK(); } _st = _close_impl(); + if (!_st.ok()) { + WARN_IF_ERROR(_abort_impl(), "failed to abort incomplete S3 upload"); + } _state = State::CLOSED; if (!non_block && _st.ok()) { _record_close_latency(); diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h index 5a8075e03cf404..bfebae917b1386 100644 --- a/be/src/io/fs/s3_file_writer.h +++ b/be/src/io/fs/s3_file_writer.h @@ -71,11 +71,12 @@ class S3FileWriter final : public FileWriter { } Status close(bool non_block = false) override; + Status abort() override; Status try_finish_close() override; private: + Status _abort_impl(); Status _close_impl(); - Status _abort(); [[nodiscard]] std::string _dump_completed_part() const; void _wait_until_finish(std::string_view task_name); Status _complete(); diff --git a/be/src/io/fs/s3_obj_storage_client.cpp b/be/src/io/fs/s3_obj_storage_client.cpp index 0c0b0370f8097f..54ba6b687e9790 100644 --- a/be/src/io/fs/s3_obj_storage_client.cpp +++ b/be/src/io/fs/s3_obj_storage_client.cpp @@ -275,6 +275,25 @@ ObjectStorageResponse S3ObjStorageClient::complete_multipart_upload( return ObjectStorageResponse::OK(); } +ObjectStorageResponse S3ObjStorageClient::abort_multipart_upload( + const ObjectStoragePathOptions& opts) { + AbortMultipartUploadRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key).WithUploadId(*opts.upload_id); + auto outcome = SYNC_POINT_HOOK_RETURN_VALUE(_client->AbortMultipartUpload(request), + "s3_file_writer::abort_multi_part", + std::cref(request).get()); + if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); + auto status = s3fs_error(outcome.GetError(), + fmt::format("failed to AbortMultipartUpload: {}, upload_id={}", + opts.path.native(), *opts.upload_id)); + return {convert_to_obj_response(std::move(status)), + static_cast(outcome.GetError().GetResponseCode()), + outcome.GetError().GetRequestId()}; + } + return ObjectStorageResponse::OK(); +} + ObjectStorageHeadResponse S3ObjStorageClient::head_object(const ObjectStoragePathOptions& opts) { Aws::S3::Model::HeadObjectRequest request; request.WithBucket(opts.bucket).WithKey(opts.key); diff --git a/be/src/io/fs/s3_obj_storage_client.h b/be/src/io/fs/s3_obj_storage_client.h index 45294226594d81..10bcf6b2e9495b 100644 --- a/be/src/io/fs/s3_obj_storage_client.h +++ b/be/src/io/fs/s3_obj_storage_client.h @@ -43,6 +43,7 @@ class S3ObjStorageClient final : public ObjStorageClient { ObjectStorageResponse complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) override; + ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions& opts) override; ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override; ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, size_t offset, size_t bytes_read, diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 5dfd027d42d4ce..7b3cae3d86d5f2 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -52,12 +52,36 @@ #include "runtime/thread_context.h" #include "storage/id_manager.h" #include "storage/storage_engine.h" +#include "util/thrift_util.h" #include "util/timezone_utils.h" #include "util/uid_util.h" namespace doris { using namespace ErrorCode; +Status RuntimeState::add_iceberg_commit_datas(TIcebergCommitData iceberg_commit_data) { + ThriftSerializer serializer(false, 256); + uint32_t serialized_size = 0; + uint8_t* buffer = nullptr; + RETURN_IF_ERROR(serializer.serialize(&iceberg_commit_data, &serialized_size, &buffer)); + + constexpr size_t report_envelope_headroom = 1024 * 1024; + const size_t thrift_limit = static_cast(std::max(config::thrift_max_message_size, 0)); + const size_t commit_data_limit = + thrift_limit > report_envelope_headroom ? thrift_limit - report_envelope_headroom : 0; + std::lock_guard lock(_iceberg_commit_datas_mutex); + // Reject metadata while its file can still be removed; a later oversized report would strand every output. + if (_iceberg_commit_datas_serialized_bytes + serialized_size + sizeof(uint32_t) > + commit_data_limit) { + return Status::InternalError( + "Iceberg commit metadata exceeds the Thrift report limit; reduce output file " + "count"); + } + _iceberg_commit_datas_serialized_bytes += serialized_size + sizeof(uint32_t); + _iceberg_commit_datas.emplace_back(std::move(iceberg_commit_data)); + return Status::OK(); +} + RuntimeState::RuntimeState(const TPlanFragmentExecParams& fragment_exec_params, const TQueryOptions& query_options, const TQueryGlobals& query_globals, ExecEnv* exec_env, QueryContext* ctx, diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index cf844dd8c45dcb..d440e3b45b9636 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -525,15 +525,12 @@ class RuntimeState { _hive_partition_updates.emplace_back(hive_partition_update); } - std::vector iceberg_commit_datas() const { + void append_iceberg_commit_datas(std::vector* output) const { std::lock_guard lock(_iceberg_commit_datas_mutex); - return _iceberg_commit_datas; + output->insert(output->end(), _iceberg_commit_datas.begin(), _iceberg_commit_datas.end()); } - void add_iceberg_commit_datas(const TIcebergCommitData& iceberg_commit_data) { - std::lock_guard lock(_iceberg_commit_datas_mutex); - _iceberg_commit_datas.emplace_back(iceberg_commit_data); - } + Status add_iceberg_commit_datas(TIcebergCommitData iceberg_commit_data); std::vector mc_commit_datas() const { std::lock_guard lock(_mc_commit_datas_mutex); @@ -978,6 +975,7 @@ class RuntimeState { mutable std::mutex _iceberg_commit_datas_mutex; std::vector _iceberg_commit_datas; + size_t _iceberg_commit_datas_serialized_bytes = 0; mutable std::mutex _mc_commit_datas_mutex; std::vector _mc_commit_datas; diff --git a/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp b/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp index d453177cf25044..18af0af7cb23bd 100644 --- a/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp +++ b/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp @@ -20,6 +20,8 @@ #include #include "exec/sink/writer/iceberg/viceberg_partition_writer.h" +#include "exec/sink/writer/iceberg/viceberg_sort_writer.h" +#include "testutil/mock/mock_runtime_state.h" namespace doris { @@ -27,13 +29,18 @@ namespace { class FakeFileFormatTransformer final : public VFileFormatTransformer { public: - explicit FakeFileFormatTransformer(const VExprContextSPtrs& output_exprs) - : VFileFormatTransformer(nullptr, output_exprs, false) {} + explicit FakeFileFormatTransformer(const VExprContextSPtrs& output_exprs, + Status close_status = Status::OK()) + : VFileFormatTransformer(nullptr, output_exprs, false), + _close_status(std::move(close_status)) {} Status open() override { return Status::OK(); } Status write(const Block&) override { return Status::OK(); } - Status close() override { return Status::OK(); } + Status close() override { return _close_status; } int64_t written_len() override { return 64; } + +private: + Status _close_status; }; TDataSink make_table_sink(std::optional collect_column_stats) { @@ -64,9 +71,10 @@ class VIcebergPartitionWriterTest : public testing::Test { } static void install_fake_transformer(VIcebergPartitionWriter* writer, - const VExprContextSPtrs& output_exprs) { + const VExprContextSPtrs& output_exprs, + Status close_status = Status::OK()) { writer->_file_format_transformer = - std::make_unique(output_exprs); + std::make_unique(output_exprs, std::move(close_status)); } static Status build_commit_data(VIcebergPartitionWriter* writer, @@ -104,4 +112,23 @@ TEST_F(VIcebergPartitionWriterTest, MissingPolicyKeepsCollectionEnabledForRollin EXPECT_TRUE(collect_column_stats(*writer)); } +TEST_F(VIcebergPartitionWriterTest, SortWriterPropagatesUnderlyingCloseFailure) { + VExprContextSPtrs output_exprs; + iceberg::Schema schema(std::vector {}); + std::string schema_json; + std::map hadoop_conf; + auto partition_writer = std::shared_ptr( + make_writer(make_table_sink(false), output_exprs, schema, &schema_json, hadoop_conf)); + install_fake_transformer(partition_writer.get(), output_exprs, + Status::IOError("injected close failure")); + VIcebergSortWriter sort_writer(partition_writer, TSortInfo(), 1024); + MockRuntimeState state; + sort_writer._runtime_state = &state; + + Status status = sort_writer.close(Status::OK()); + + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("injected close failure"), std::string::npos); +} + } // namespace doris diff --git a/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp b/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp new file mode 100644 index 00000000000000..264c9895990a4b --- /dev/null +++ b/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "exec/sink/writer/iceberg/viceberg_table_writer.h" +#include "exec/sink/writer/iceberg/vpartition_writer_base.h" + +namespace doris { + +namespace { + +class FakePartitionWriter final : public IPartitionWriterBase { +public: + Status open(RuntimeState*, RuntimeProfile*, const RowDescriptor*) override { + return Status::OK(); + } + Status write(Block&) override { return Status::OK(); } + Status close(const Status&) override { return Status::OK(); } + const std::string& file_name() const override { return _name; } + int file_name_index() const override { return 0; } + size_t written_len() const override { return 0; } + +private: + std::string _name = "fake"; +}; + +TDataSink make_sink() { + TDataSink sink; + sink.__set_type(TDataSinkType::ICEBERG_TABLE_SINK); + sink.__set_iceberg_table_sink(TIcebergTableSink()); + return sink; +} + +} // namespace + +class VIcebergTableWriterTest : public testing::Test { +protected: + static Status select_block(VIcebergTableWriter* writer, Block& input, + const IColumn::Permutation& rows, Block* selected) { + return writer->_select_block(input, rows, selected); + } + + static void add_writer(VIcebergTableWriter* writer, std::string partition) { + writer->_partitions_to_writers.emplace(std::move(partition), + std::make_shared()); + } + + static void publish_active_writers(VIcebergTableWriter* writer) { + writer->_publish_active_writers(); + } +}; + +TEST_F(VIcebergTableWriterTest, SelectBlockUsesRowPermutation) { + VIcebergTableWriter writer(make_sink(), {}, nullptr, nullptr); + auto values = ColumnInt32::create(); + values->insert_value(10); + values->insert_value(20); + values->insert_value(30); + Block input; + input.insert({std::move(values), std::make_shared(), "value"}); + IColumn::Permutation rows {2, 0}; + Block selected; + + ASSERT_TRUE(select_block(&writer, input, rows, &selected).ok()); + + const auto& result = assert_cast(*selected.get_by_position(0).column); + ASSERT_EQ(result.size(), 2); + EXPECT_EQ(result.get_element(0), 30); + EXPECT_EQ(result.get_element(1), 10); +} + +TEST_F(VIcebergTableWriterTest, ActiveWriterSnapshotContainsEveryOpenPartition) { + VIcebergTableWriter writer(make_sink(), {}, nullptr, nullptr); + add_writer(&writer, "p=1"); + add_writer(&writer, "p=2"); + + publish_active_writers(&writer); + + ASSERT_NE(writer.active_writers(), nullptr); + EXPECT_EQ(writer.active_writers()->size(), 2); +} + +} // namespace doris diff --git a/be/test/io/fs/s3_file_writer_test.cpp b/be/test/io/fs/s3_file_writer_test.cpp index 3937d6e38561fe..a3033c9f1c07e4 100644 --- a/be/test/io/fs/s3_file_writer_test.cpp +++ b/be/test/io/fs/s3_file_writer_test.cpp @@ -316,6 +316,21 @@ class S3FileWriterTest : public testing::Test { } }; +TEST_F(S3FileWriterTest, abort_cleans_up_multipart_upload) { + mock_client = std::make_shared(); + doris::io::FileWriterOptions options; + + io::FileWriterPtr writer; + ASSERT_TRUE(s3_fs->create_file("abort_multipart", &writer, &options).ok()); + std::string data(config::s3_write_buffer_size, 'a'); + ASSERT_TRUE(writer->append(Slice(data)).ok()); + ASSERT_FALSE(static_cast(writer.get())->upload_id().empty()); + + ASSERT_TRUE(writer->abort().ok()); + EXPECT_EQ(writer->state(), io::FileWriter::State::CLOSED); + EXPECT_TRUE(mock_client->contents().empty()); +} + TEST_F(S3FileWriterTest, multi_part_io_error) { mock_client = std::make_shared(); doris::io::FileWriterOptions state; diff --git a/be/test/runtime/runtime_state_block_budget_test.cpp b/be/test/runtime/runtime_state_block_budget_test.cpp index 22ebc5ebf8a0ee..b2b1e141508101 100644 --- a/be/test/runtime/runtime_state_block_budget_test.cpp +++ b/be/test/runtime/runtime_state_block_budget_test.cpp @@ -24,6 +24,22 @@ namespace doris { +TEST(RuntimeStateIcebergCommitDataTest, RejectsMetadataBeforeItCanExceedTheThriftLimit) { + RuntimeState state; + const int32_t saved_limit = config::thrift_max_message_size; + config::thrift_max_message_size = 128; + TIcebergCommitData commit_data; + commit_data.__set_file_path(std::string(256, 'x')); + + Status status = state.add_iceberg_commit_datas(commit_data); + + config::thrift_max_message_size = saved_limit; + EXPECT_FALSE(status.ok()); + std::vector collected; + state.append_iceberg_commit_datas(&collected); + EXPECT_TRUE(collected.empty()); +} + // --------------------------------------------------------------------------- // RuntimeState::batch_size() // --------------------------------------------------------------------------- diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java index 21b301a3416490..f21dfb99359082 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java @@ -154,8 +154,8 @@ public class IcebergConnectorTransaction implements ConnectorTransaction, Rewrit private Map staticPartitionValues = Collections.emptyMap(); private String branchName; private IcebergWriteSchemaContext writeSchemaContext; - // The current snapshot pinned at begin time for a DELETE/MERGE (null for INSERT/OVERWRITE). Consumed by - // the commit validation suite (validateFromSnapshot). + // The snapshot pinned at begin time for DELETE/MERGE and OVERWRITE (null for INSERT). Consumed by the + // commit validation suite (validateFromSnapshot). private Long baseSnapshotId; // Session zone for human-readable TIMESTAMP partition value parsing (DV-T04-f). private ZoneId zone = ZoneOffset.UTC; @@ -430,7 +430,6 @@ private void applyBeginGuards(IcebergWriteContext ctx, String tableName) { } } else { // INSERT / OVERWRITE (append path). - this.baseSnapshotId = null; if (ctx.getBranchName().isPresent()) { this.branchName = ctx.getBranchName().get(); SnapshotRef branchRef = table.refs().get(branchName); @@ -440,8 +439,10 @@ private void applyBeginGuards(IcebergWriteContext ctx, String tableName) { throw new IllegalArgumentException(branchName + " is a tag, not a branch. Tags cannot be targets for producing snapshots"); } + this.baseSnapshotId = op == WriteOperation.OVERWRITE ? branchRef.snapshotId() : null; } else { this.branchName = null; + this.baseSnapshotId = op == WriteOperation.OVERWRITE ? getSnapshotIdIfPresent(table) : null; } } } @@ -705,6 +706,8 @@ private void commitReplaceTxn(List pendingResults) { if (branchName != null) { overwriteFiles = overwriteFiles.toBranch(branchName); } + // Clearing a table must fail if any data or delete landed after the statement's base snapshot. + overwriteFiles = validateOverwrite(overwriteFiles, Expressions.alwaysTrue()); TableScan overwriteScan = table.newScan(); if (branchName != null) { overwriteScan = overwriteScan.useRef(branchName); @@ -724,6 +727,11 @@ private void commitReplaceTxn(List pendingResults) { if (branchName != null) { appendPartitionOp = appendPartitionOp.toBranch(branchName); } + // Partition replacement must preserve concurrent files instead of deleting or reviving them silently. + if (baseSnapshotId != null) { + appendPartitionOp = appendPartitionOp.validateFromSnapshot(baseSnapshotId); + } + appendPartitionOp = appendPartitionOp.validateNoConflictingData().validateNoConflictingDeletes(); for (WriteResult result : pendingResults) { Preconditions.checkState(result.referencedDataFiles().length == 0, "Should have no referenced data files."); @@ -749,6 +757,7 @@ private void commitStaticPartitionOverwrite(List pendingResults) { overwriteFiles = overwriteFiles.toBranch(branchName); } overwriteFiles = overwriteFiles.overwriteByRowFilter(partitionFilter); + overwriteFiles = validateOverwrite(overwriteFiles, partitionFilter); for (WriteResult result : pendingResults) { Preconditions.checkState(result.referencedDataFiles().length == 0, @@ -758,6 +767,14 @@ private void commitStaticPartitionOverwrite(List pendingResults) { overwriteFiles.commit(); } + private OverwriteFiles validateOverwrite(OverwriteFiles overwriteFiles, Expression conflictFilter) { + overwriteFiles = overwriteFiles.conflictDetectionFilter(conflictFilter); + if (baseSnapshotId != null) { + overwriteFiles = overwriteFiles.validateFromSnapshot(baseSnapshotId); + } + return overwriteFiles.validateNoConflictingData().validateNoConflictingDeletes(); + } + /** * Build an iceberg {@link Expression} from the static partition key-value pairs. Identity partitions * require the SOURCE column name (not the partition field name) in the expression. diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergProcedureOps.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergProcedureOps.java index 536babad0e517c..a75beb97ad12a5 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergProcedureOps.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergProcedureOps.java @@ -43,8 +43,7 @@ import java.util.stream.Collectors; /** - * Executes iceberg's {@code ALTER TABLE EXECUTE} procedures (the 9 legacy - * {@code datasource/iceberg/action/*} actions) behind the {@link ConnectorProcedureOps} SPI. + * Executes iceberg's {@code ALTER TABLE EXECUTE} procedures behind the {@link ConnectorProcedureOps} SPI. * *

Mirrors {@link IcebergWritePlanProvider}: a fresh instance per call over the lazily-built live * catalog, threading the same {@code properties} / {@link IcebergCatalogOps} / {@link ConnectorContext} @@ -52,14 +51,11 @@ * runs in the connector; argument validation is connector-local (the engine cannot reach * {@code org.apache.doris.common.NamedArguments} across the import gate).

* - *

T03 dispatch skeleton. {@link #getSupportedProcedures()} exports the factory's name list and + *

{@link #getSupportedProcedures()} exports the factory's name list and * {@link #execute} routes through {@link IcebergExecuteActionFactory} → {@link BaseIcebergAction}: validate * arguments, load the SDK table inside {@code context.executeAuthenticated}, run the body and wrap the - * single row. The 9 procedure bodies (the factory's switch cases) are ported in T04 (the 8 pure-SDK - * procedures) / T05–T06 ({@code rewrite_data_files}); until then a known name reaches the factory's faithful - * "Unsupported Iceberg procedure" rejection. Inert pre-cutover regardless: iceberg tables are not - * {@code PluginDrivenExternalTable} until P6.6, so {@code ExecuteActionCommand} still routes them to the - * legacy fe-core actions and never reaches this class.

+ * single row. {@code rewrite_data_files} is planned as a distributed INSERT-SELECT operation and therefore + * bypasses the single-call action factory.

*/ public class IcebergProcedureOps implements ConnectorProcedureOps { diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergExecuteActionFactory.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergExecuteActionFactory.java index 37d949c1d7abcf..7b785c9ac31674 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergExecuteActionFactory.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergExecuteActionFactory.java @@ -36,7 +36,7 @@ * {@code Optional} / nereids {@code Expression}. * *

T03 scaffolding. The {@code createAction} switch carries only the faithful default rejection; - * the 9 procedure cases (their bodies) are ported in T04 ({@code rewrite_data_files} in T05/T06). The + * the procedure cases (their bodies) are ported in T04 ({@code rewrite_data_files} in T05/T06). The * {@link #getSupportedActions()} registry — exported to {@code getSupportedProcedures()} and embedded in * the rejection message — is complete and final. */ @@ -52,6 +52,7 @@ public class IcebergExecuteActionFactory { public static final String REWRITE_DATA_FILES = "rewrite_data_files"; public static final String PUBLISH_CHANGES = "publish_changes"; public static final String REWRITE_MANIFESTS = "rewrite_manifests"; + public static final String REMOVE_ORPHAN_FILES = "remove_orphan_files"; /** * Create an iceberg procedure body for {@code actionType}. @@ -83,6 +84,8 @@ public static BaseIcebergAction createAction(String actionType, Map properties, List partitionNames, + ConnectorPredicate whereCondition) { + super("remove_orphan_files", properties, partitionNames, whereCondition); + } + + @Override + protected void registerIcebergArguments() { + namedArguments.registerRequiredArgument(OLDER_THAN, "Creation time cutoff in milliseconds", + ArgumentParsers.nonNegativeLong(OLDER_THAN)); + namedArguments.registerOptionalArgument(LOCATION, "Prefix within the table location", + null, ArgumentParsers.nonEmptyString(LOCATION)); + namedArguments.registerOptionalArgument(DRY_RUN, "Only count orphan files", true, + ArgumentParsers.booleanValue(DRY_RUN)); + } + + @Override + protected void validateIcebergAction() { + validateNoPartitions(); + validateNoWhereCondition(); + String location = namedArguments.getString(LOCATION); + if (location != null) { + try { + normalizeLocation(location); + } catch (IllegalArgumentException e) { + throw new DorisConnectorException("Invalid location URI: " + location, e); + } + } + } + + @Override + protected List executeAction(Table table, ConnectorSession session) { + if (!(table.io() instanceof SupportsPrefixOperations)) { + throw new DorisConnectorException("remove_orphan_files requires FileIO prefix listing support"); + } + String tableLocation = normalizeLocation(table.location()); + String scanLocation = namedArguments.getString(LOCATION); + scanLocation = scanLocation == null ? tableLocation : normalizeLocation(scanLocation); + // Normalize dot segments before the containment check so local FileIO paths cannot escape the table root. + if (!scanLocation.equals(tableLocation) && !scanLocation.startsWith(tableLocation + "/")) { + throw new DorisConnectorException("location must be within the Iceberg table location"); + } + + try { + Set reachable = collectReachableFiles(table); + long orphanCount = 0; + long deletedCount = 0; + long olderThan = namedArguments.getLong(OLDER_THAN); + boolean dryRun = namedArguments.getBoolean(DRY_RUN); + // Object stores use raw prefix matching, so the separator prevents "table_backup" siblings + // from being treated as children of "table". + String listingPrefix = scanLocation.endsWith("/") ? scanLocation : scanLocation + "/"; + for (FileInfo file : ((SupportsPrefixOperations) table.io()).listPrefix(listingPrefix)) { + if (file.createdAtMillis() < olderThan && !reachable.contains(file.location())) { + orphanCount++; + if (!dryRun) { + table.io().deleteFile(file.location()); + deletedCount++; + } + } + } + return Lists.newArrayList(String.valueOf(orphanCount), String.valueOf(deletedCount)); + } catch (Exception e) { + throw new DorisConnectorException("Failed to remove orphan files: " + e.getMessage(), e); + } + } + + private Set collectReachableFiles(Table table) throws IOException { + Set reachable = new HashSet<>(ReachableFileUtil.metadataFileLocations(table, true)); + Set scannedDeleteManifests = new HashSet<>(); + reachable.addAll(ReachableFileUtil.manifestListLocations(table)); + reachable.addAll(ReachableFileUtil.statisticsFilesLocations(table)); + for (Snapshot snapshot : table.snapshots()) { + for (ManifestFile manifest : snapshot.allManifests(table.io())) { + reachable.add(manifest.path()); + } + for (ManifestFile manifest : snapshot.deleteManifests(table.io())) { + if (!scannedDeleteManifests.add(manifest.path())) { + continue; + } + // A retained delete file may not apply to any current data task, so read delete manifests directly. + try (ManifestReader deletes = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + deletes.forEach(delete -> reachable.add(delete.location())); + } + } + try (CloseableIterable tasks = table.newScan() + .useSnapshot(snapshot.snapshotId()).planFiles()) { + for (FileScanTask task : tasks) { + reachable.add(task.file().location()); + task.deletes().forEach(delete -> reachable.add(delete.location())); + } + } + } + return reachable; + } + + private String normalizeLocation(String location) { + String normalized = URI.create(location).normalize().toString(); + return normalized.length() > 1 && normalized.endsWith("/") + ? normalized.substring(0, normalized.length() - 1) : normalized; + } + + @Override + protected List getResultSchema() { + return Lists.newArrayList( + new ConnectorColumn("orphan_file_count", ConnectorType.of("BIGINT"), + "Number of old unreachable files", false, null), + new ConnectorColumn("deleted_file_count", ConnectorType.of("BIGINT"), + "Number of files deleted", false, null)); + } +} diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java index cdc1a341f40cc1..1fa84febaac271 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java @@ -592,6 +592,28 @@ public void overwriteDynamicReplacesPartitions() { Assertions.assertEquals("1", snap.summary().get("added-data-files")); } + @Test + public void overwriteDynamicRejectsConcurrentDataInReplacedPartition() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t1"); + PartitionSpec spec = PartitionSpec.builderFor(PART_SCHEMA).identity("region").build(); + Table table = catalog.createTable(id, PART_SCHEMA, spec, props("write.format.default", "parquet")); + table.newAppend().appendFile(partitionedDataFile(spec, + "s3://b/db1/t1/region=us/seed.parquet", 1L, "region=us")).commit(); + + IcebergConnectorTransaction txn = txnFor(opsReturning(catalog.loadTable(id)), new RecordingConnectorContext()); + txn.beginWrite(SESSION, "db1", "t1", overwriteCtx()); + Table concurrent = catalog.loadTable(id); + concurrent.newAppend().appendFile(partitionedDataFile(spec, + "s3://b/db1/t1/region=us/concurrent.parquet", 1L, "region=us")).commit(); + txn.addCommitData(commitBytes( + dataFileItem("s3://b/db1/t1/region=us/replacement.parquet", 2L, 1024L, + Collections.singletonList("us")))); + + Assertions.assertThrows(DorisConnectorException.class, txn::commit, + "dynamic overwrite must not silently replace data committed after its base snapshot"); + } + @Test public void overwriteEmptyUnpartitionedClearsTable() { InMemoryCatalog catalog = freshCatalog(); @@ -682,6 +704,29 @@ public void overwriteStaticPartitionRejectsUnmatchedPartitionField() { "a stale static overwrite must fail at begin before it can degrade to an always-true filter"); } + @Test + public void overwriteStaticRejectsConcurrentDataInTargetPartition() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t1"); + PartitionSpec spec = PartitionSpec.builderFor(PART_SCHEMA).identity("region").build(); + Table table = catalog.createTable(id, PART_SCHEMA, spec, props("write.format.default", "parquet")); + table.newAppend().appendFile(partitionedDataFile(spec, + "s3://b/db1/t1/region=us/seed.parquet", 1L, "region=us")).commit(); + + IcebergConnectorTransaction txn = txnFor(opsReturning(catalog.loadTable(id)), new RecordingConnectorContext()); + txn.beginWrite(SESSION, "db1", "t1", + overwriteStaticCtx(table, Collections.singletonMap("region", "us"))); + Table concurrent = catalog.loadTable(id); + concurrent.newAppend().appendFile(partitionedDataFile(spec, + "s3://b/db1/t1/region=us/concurrent.parquet", 1L, "region=us")).commit(); + txn.addCommitData(commitBytes( + dataFileItem("s3://b/db1/t1/region=us/replacement.parquet", 2L, 1024L, + Collections.singletonList("us")))); + + Assertions.assertThrows(DorisConnectorException.class, txn::commit, + "static overwrite must reject concurrent data matching its target partition filter"); + } + @Test public void deleteWritesRowDeltaDeleteFiles() { InMemoryCatalog catalog = freshCatalog(); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergExecuteActionFactoryTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergExecuteActionFactoryTest.java index d1978fe7741fd7..76d4f34b57e2d1 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergExecuteActionFactoryTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergExecuteActionFactoryTest.java @@ -19,24 +19,23 @@ import org.apache.doris.connector.spi.DorisConnectorException; +import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.Collections; /** - * Pins the connector port of legacy {@code IcebergExecuteActionFactory} (the name registry + dispatch). + * Pins the Iceberg procedure name registry and dispatch. * *

WHY this matters: the supported-name list is exported to {@code getSupportedProcedures()} and - * embedded in the unknown-procedure error, so its membership and order must match legacy byte-for-byte - * (T08 byte-parity). The {@code table} parameter is dropped (it was always dead in legacy). The 9 switch - * cases are added in T04 (the procedure bodies); T03 fixes the registry + the faithful unknown-procedure - * rejection.

+ * embedded in the unknown-procedure error, so membership, ordering, and executable action mappings must stay + * synchronized.

*/ public class IcebergExecuteActionFactoryTest { @Test - public void getSupportedActionsReturnsNineNamesInLegacyOrder() { + public void getSupportedActionsIncludesOrphanCleanup() { Assertions.assertArrayEquals( new String[] { "rollback_to_snapshot", @@ -48,6 +47,7 @@ public void getSupportedActionsReturnsNineNamesInLegacyOrder() { "rewrite_data_files", "publish_changes", "rewrite_manifests", + "remove_orphan_files", }, IcebergExecuteActionFactory.getSupportedActions()); } @@ -60,15 +60,32 @@ public void createActionRejectsUnknownProcedureWithLegacyMessage() { Assertions.assertEquals( "Unsupported Iceberg procedure: no_such_proc. Supported procedures: rollback_to_snapshot, " + "rollback_to_timestamp, set_current_snapshot, cherrypick_snapshot, fast_forward, " - + "expire_snapshots, rewrite_data_files, publish_changes, rewrite_manifests", + + "expire_snapshots, rewrite_data_files, publish_changes, rewrite_manifests, " + + "remove_orphan_files", e.getMessage()); } + @Test + public void createRemoveOrphanFilesAction() { + BaseIcebergAction action = IcebergExecuteActionFactory.createAction( + "remove_orphan_files", Collections.singletonMap("older_than", "1"), + Collections.emptyList(), null); + Assertions.assertInstanceOf(IcebergRemoveOrphanFilesAction.class, action); + } + + @Test + public void removeOrphanFilesRejectsInvalidLocationUri() { + BaseIcebergAction action = IcebergExecuteActionFactory.createAction( + "remove_orphan_files", ImmutableMap.of("older_than", "1", "location", "://"), + Collections.emptyList(), null); + Assertions.assertThrows(DorisConnectorException.class, action::validate); + } + /** * CANARY for the dormant {@code rewrite_data_files} gap: it is advertised in {@link - * IcebergExecuteActionFactory#getSupportedActions()} (9 names) but has NO {@code createAction} switch - * case yet (8 cases), so it falls through to the faithful unknown-procedure rejection. This pins that - * dormant state and goes RED exactly when the T05/T06 body is wired in. + * IcebergExecuteActionFactory#getSupportedActions()} but has NO {@code createAction} switch + * case because it is dispatched through the distributed rewrite planner, so it falls through to the + * unknown-procedure rejection in this single-call factory. */ @Test public void rewriteDataFilesIsAdvertisedButNotYetExecutable() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java index bdcb6e29448e1e..381b38e9a8bd52 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java @@ -140,6 +140,10 @@ public void unregisterListener(InsertExecutorListener listener) { listeners.remove(listener); } + protected void handleAfterCompleteFailure(Exception e) throws Exception { + throw e; + } + public Coordinator getCoordinator() { return coordinator; } @@ -259,8 +263,9 @@ private void checkStrictModeAndFilterRatio() throws Exception { * execute insert txn for insert into select command. */ public void executeSingleInsert(StmtExecutor executor) throws Exception { - beforeExec(); try { + // Pre-execution work may register external resources, so it must share the transaction cleanup scope. + beforeExec(); executor.updateProfile(false); execImpl(executor); checkStrictModeAndFilterRatio(); @@ -269,7 +274,11 @@ public void executeSingleInsert(StmtExecutor executor) throws Exception { } onComplete(); for (InsertExecutorListener listener : listeners) { - listener.afterComplete(this, executor, jobId); + try { + listener.afterComplete(this, executor, jobId); + } catch (Exception e) { + handleAfterCompleteFailure(e); + } } } catch (Throwable t) { onFail(t); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java index c9e5f60a71a2c3..c24931a1a55ba9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java @@ -126,8 +126,13 @@ protected void onComplete() throws UserException { txnStatus = TransactionStatus.COMMITTED; long t2 = System.currentTimeMillis(); - // Handle post-commit operations (e.g., cache refresh) - doAfterCommit(); + try { + doAfterCommit(); + } catch (Exception e) { + // Cache refresh cannot undo a durable remote commit, so it must not make clients retry the write. + LOG.warn("Post-commit refresh failed for table {}. Data was committed successfully.", + table.getName(), e); + } long t3 = System.currentTimeMillis(); LOG.info("Transaction commit breakdown: doBeforeCommit={}ms, commit={}ms, doAfterCommit={}ms, total={}ms", t1 - t0, t2 - t1, t3 - t2, t3 - t0); @@ -149,6 +154,16 @@ protected void doAfterCommit() throws DdlException { true); } + @Override + protected void handleAfterCompleteFailure(Exception e) throws Exception { + if (txnStatus != TransactionStatus.COMMITTED) { + super.handleAfterCompleteFailure(e); + return; + } + // A post-commit listener cannot undo remote data, so failing the statement would invite duplicate retries. + LOG.warn("Post-commit listener failed for table {}. Data was committed successfully.", table.getName(), e); + } + @Override protected void finalizeSink(PlanFragment fragment, DataSink sink, PhysicalSink physicalSink) { try { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutorTest.java index 6c8537c5d62bba..96fb43ffbae48e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutorTest.java @@ -181,6 +181,33 @@ void testOnFailAbortsUncommittedTransaction() throws Exception { } } + @Test + void testBeforeExecFailureUsesTheNormalAbortAndCleanupPath() throws Exception { + ConnectContext ctx = createExecutorContext(); + Coordinator coordinator = createCoordinator(); + GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); + TransactionState txnState = Mockito.mock(TransactionState.class); + LoadManager loadManager = Mockito.mock(LoadManager.class); + Env currentEnv = createCurrentEnv(loadManager); + StmtExecutor stmtExecutor = createStmtExecutor(); + + try (MockedStatic envFactoryMock = Mockito.mockStatic(EnvFactory.class); + MockedStatic envMock = Mockito.mockStatic(Env.class)) { + prepareFactoryMocks(envFactoryMock, envMock, coordinator, txnMgr, txnState, currentEnv); + ctx.setEnv(currentEnv); + + OlapInsertExecutor executor = createExecutorWithBeforeExecFailure(ctx); + executor.txnId = 10004L; + + Assertions.assertDoesNotThrow(() -> executor.executeSingleInsert(stmtExecutor)); + Assertions.assertEquals(MysqlStateType.ERR, ctx.getState().getStateType()); + Assertions.assertTrue(ctx.getState().getErrorMessage().contains("beforeExec failure")); + Mockito.verify(txnMgr).abortTransaction(1L, 10004L, "beforeExec failure"); + Mockito.verify(coordinator).close(); + Mockito.verify(stmtExecutor).updateProfile(true); + } + } + // Build a fresh context per case so insertResult and QueryState do not leak between tests. private ConnectContext createExecutorContext() { ConnectContext ctx = new ConnectContext(); @@ -256,6 +283,25 @@ private OlapInsertExecutor createExecutor(ConnectContext ctx) { Optional.empty(), false, 0L); } + private OlapInsertExecutor createExecutorWithBeforeExecFailure(ConnectContext ctx) { + Database database = Mockito.mock(Database.class); + Mockito.when(database.getFullName()).thenReturn("test_db"); + Mockito.when(database.getId()).thenReturn(1L); + + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(table.getDatabase()).thenReturn(database); + Mockito.when(table.getName()).thenReturn("test_tbl"); + Mockito.when(table.getId()).thenReturn(2L); + + return new OlapInsertExecutor(ctx, table, "label_test", Mockito.mock(NereidsPlanner.class), + Optional.empty(), false, 0L) { + @Override + protected void beforeExec() { + throw new RuntimeException("beforeExec failure"); + } + }; + } + // Redirect coordinator creation and transaction access to mocks so the test stays deterministic. private void prepareFactoryMocks(MockedStatic envFactoryMock, MockedStatic envMock, Coordinator coordinator, GlobalTransactionMgrIface txnMgr, TransactionState txnState, Env currentEnv) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/PluginDrivenInsertExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/PluginDrivenInsertExecutorTest.java index 86569395c902c9..75fd22c20ce220 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/PluginDrivenInsertExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/PluginDrivenInsertExecutorTest.java @@ -33,6 +33,7 @@ import org.apache.doris.planner.PluginDrivenTableSink; import org.apache.doris.thrift.TDataSink; import org.apache.doris.transaction.PluginDrivenTransactionManager; +import org.apache.doris.transaction.TransactionStatus; import org.apache.doris.transaction.TransactionType; import org.junit.jupiter.api.Assertions; @@ -215,6 +216,17 @@ public void doBeforeCommitKeepsCoordinatorRowCountWhenTransactionReportsNoCount( "a -1 (no count) transaction must leave the coordinator-counted loadedRows untouched"); } + @Test + public void postCommitListenerFailureDoesNotTurnACommittedWriteIntoAnError() { + PluginDrivenInsertExecutor exec = newUnconstructedExecutor(); + Deencapsulation.setField(exec, "txnStatus", TransactionStatus.COMMITTED); + Deencapsulation.setField(exec, "table", Mockito.mock(PluginDrivenExternalTable.class)); + + Assertions.assertDoesNotThrow(() -> Deencapsulation.invoke(exec, + "handleAfterCompleteFailure", new RuntimeException("listener failure")), + "a listener cannot roll back or fail a connector write after its remote commit is durable"); + } + /** * Creates a {@link PluginDrivenInsertExecutor} without running its constructor. See the class * javadoc: the constructor builds a Coordinator that needs a live planner/EnvFactory. From f29923204c6b8b3d7fea6b5da9e3c2ca535418a2 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 1 Aug 2026 23:33:48 +0800 Subject: [PATCH 02/29] [fix](iceberg) Address write safety review findings ### What problem does this PR solve? Issue Number: None Related PR: #66348 Problem Summary: Parallel Iceberg sink tasks could exceed the shared report limit, partition writers could over-reserve memory or outlive a temporary snapshot, failed multipart uploads could skip mandatory cleanup, and orphan-file deletion lacked Iceberg safety fences. Share the report budget, retain writer snapshots, bound reservations, make abort cleanup provider-aware, and enforce safe canonical orphan detection. ### Release note Improve Iceberg write failure cleanup and orphan-file deletion safety. ### Check List (For Author) - Test: Unit Test - BE ASAN focused tests with Azure support enabled - Full fe-connector-iceberg unit test suite - Behavior changed: Yes, unsafe orphan cutoffs and GC-disabled deletion are rejected, and failed multipart uploads are cleaned up reliably. - Does this need documentation: No --- .../spill_iceberg_table_sink_operator.cpp | 28 +++- .../spill_iceberg_table_sink_operator.h | 5 +- .../pipeline/pipeline_fragment_context.cpp | 2 + be/src/io/fs/azure_obj_storage_client.cpp | 54 ++++++-- be/src/io/fs/azure_obj_storage_client.h | 1 + be/src/io/fs/obj_storage_client.h | 4 +- .../io/fs/rate_limited_obj_storage_client.cpp | 5 +- be/src/io/fs/s3_file_writer.cpp | 4 +- be/src/io/fs/s3_file_writer.h | 1 + be/src/runtime/runtime_state.cpp | 9 +- be/src/runtime/runtime_state.h | 19 ++- ...spill_iceberg_table_sink_operator_test.cpp | 30 +++++ .../iceberg/iceberg_table_writer_test.cpp | 44 +++++++ .../io/fs/azure_obj_storage_client_test.cpp | 20 +++ .../rate_limited_obj_storage_client_test.cpp | 23 ++++ be/test/io/fs/s3_file_writer_test.cpp | 23 ++++ .../runtime_state_block_budget_test.cpp | 19 +++ .../IcebergRemoveOrphanFilesAction.java | 108 +++++++++++++++- .../iceberg/IcebergProcedureOpsTest.java | 6 +- .../IcebergRemoveOrphanFilesActionTest.java | 121 ++++++++++++++++++ 20 files changed, 495 insertions(+), 31 deletions(-) create mode 100644 be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp create mode 100644 fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java diff --git a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp index 083d7b86c87c0f..a974e575916b7f 100644 --- a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp +++ b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp @@ -17,6 +17,8 @@ #include "exec/operator/spill_iceberg_table_sink_operator.h" +#include + #include "common/status.h" #include "exec/operator/iceberg_table_sink_operator.h" #include "exec/sink/writer/iceberg/viceberg_sort_writer.h" @@ -24,6 +26,13 @@ namespace doris { +size_t bounded_iceberg_reserve_size(const std::vector& per_partition_reservations) { + return per_partition_reservations.empty() + ? 0 + : *std::max_element(per_partition_reservations.begin(), + per_partition_reservations.end()); +} + SpillIcebergTableSinkLocalState::SpillIcebergTableSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state) : Base(parent, state) {} @@ -55,13 +64,16 @@ size_t SpillIcebergTableSinkLocalState::get_reserve_mem_size(RuntimeState* state if (!_writer) { return 0; } - size_t reserve_size = 0; - for (const auto& writer : *_writer->active_writers()) { + std::vector per_partition_reservations; + auto active_writers = _writer->active_writers(); + per_partition_reservations.reserve(active_writers->size()); + for (const auto& writer : *active_writers) { if (auto* sort_writer = dynamic_cast(writer.get())) { - reserve_size += sort_writer->get_reserve_mem_size(state, eos); + per_partition_reservations.push_back(sort_writer->get_reserve_mem_size(state, eos)); } } - return reserve_size; + // One input block is partitioned among writers and consumed serially, so their full-batch estimates overlap. + return bounded_iceberg_reserve_size(per_partition_reservations); } size_t SpillIcebergTableSinkLocalState::get_revocable_mem_size(RuntimeState* state) const { @@ -69,7 +81,9 @@ size_t SpillIcebergTableSinkLocalState::get_revocable_mem_size(RuntimeState* sta return 0; } size_t revocable_size = 0; - for (const auto& writer : *_writer->active_writers()) { + // Retain the published container while the async writer may replace the current snapshot. + auto active_writers = _writer->active_writers(); + for (const auto& writer : *active_writers) { if (auto* sort_writer = dynamic_cast(writer.get())) { revocable_size += sort_writer->data_size(); } @@ -84,7 +98,9 @@ Status SpillIcebergTableSinkLocalState::revoke_memory(RuntimeState* state) { } std::shared_ptr largest_writer; size_t largest_size = 0; - for (const auto& writer : *_writer->active_writers()) { + // Retain the published container while the async writer may replace the current snapshot. + auto active_writers = _writer->active_writers(); + for (const auto& writer : *active_writers) { if (auto* sort_writer = dynamic_cast(writer.get())) { size_t size = sort_writer->data_size(); if (size > largest_size) { diff --git a/be/src/exec/operator/spill_iceberg_table_sink_operator.h b/be/src/exec/operator/spill_iceberg_table_sink_operator.h index 6da926ae20fb91..60eefd1bfb5242 100644 --- a/be/src/exec/operator/spill_iceberg_table_sink_operator.h +++ b/be/src/exec/operator/spill_iceberg_table_sink_operator.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include "exec/operator/operator.h" #include "exec/sink/writer/iceberg/viceberg_table_writer.h" @@ -27,6 +28,8 @@ namespace doris { class SpillIcebergTableSinkLocalState; class SpillIcebergTableSinkOperatorX; +size_t bounded_iceberg_reserve_size(const std::vector& per_partition_reservations); + class SpillIcebergTableSinkLocalState final : public AsyncWriterSink { public: @@ -87,4 +90,4 @@ class SpillIcebergTableSinkOperatorX final ObjectPool* _pool = nullptr; }; -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index 4f2c6dc00483e6..8ccaca43dbf3c0 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -468,6 +468,8 @@ Status PipelineFragmentContext::_build_pipeline_tasks_for_instance( _params.query_options, _query_ctx->query_globals, _exec_env, _query_ctx.get()); { // Initialize runtime state for this task + task_runtime_state->set_iceberg_commit_data_budget( + _runtime_state->iceberg_commit_data_budget()); task_runtime_state->set_query_mem_tracker(_query_ctx->query_mem_tracker()); task_runtime_state->set_task_execution_context(shared_from_this()); diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp index 9702c87b3b304b..5bc0c8102e47b2 100644 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ b/be/src/io/fs/azure_obj_storage_client.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include "common/exception.h" @@ -46,8 +47,8 @@ #include "cpp/obj_retry_strategy.h" #include "io/fs/obj_storage_client.h" #include "util/bvar_helper.h" -#include "util/coding.h" #include "util/s3_util.h" +#include "util/uuid_generator.h" using namespace Azure::Storage::Blobs; @@ -64,10 +65,13 @@ std::string to_lower_ascii(std::string_view input) { return lowered; } -auto base64_encode_part_num(int part_num) { - uint8_t buf[4]; - doris::encode_fixed32_le(buf, static_cast(part_num)); - return Aws::Utils::HashingUtils::Base64Encode({buf, sizeof(buf)}); +std::string azure_block_id(const doris::io::ObjectStoragePathOptions& opts, int part_num) { + DCHECK(opts.upload_id.has_value()); + // Azure requires every block ID for one blob to have the same decoded length. + std::string raw_id = fmt::format("{}:{:010}", *opts.upload_id, part_num); + Aws::Utils::ByteBuffer bytes(reinterpret_cast(raw_id.data()), + raw_id.size()); + return Aws::Utils::HashingUtils::Base64Encode(bytes); } // Rate limiting is applied by RateLimitedObjStorageClient, the decorator that @@ -194,11 +198,13 @@ struct AzureBatchDeleter { std::vector> deferred_resps; }; -// Azure would do nothing ObjectStorageUploadResponse AzureObjStorageClient::create_multipart_upload( const ObjectStoragePathOptions& opts) { + std::stringstream upload_id; + upload_id << UUIDGenerator::instance()->next_uuid(); return ObjectStorageUploadResponse { .resp = ObjectStorageResponse::OK(), + .upload_id = upload_id.str(), }; } @@ -223,7 +229,7 @@ ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStora reinterpret_cast(stream.data()), stream.size()); // The blockId must be base64 encoded SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); - client.StageBlock(base64_encode_part_num(part_num), memory_body); + client.StageBlock(azure_block_id(opts, part_num), memory_body); }, opts, _tls_debug_context); return ObjectStorageUploadResponse { @@ -238,7 +244,7 @@ ObjectStorageResponse AzureObjStorageClient::complete_multipart_upload( std::vector string_block_ids; std::ranges::transform( completed_parts, std::back_inserter(string_block_ids), - [](const ObjectCompleteMultiPart& i) { return base64_encode_part_num(i.part_num); }); + [&opts](const ObjectCompleteMultiPart& i) { return azure_block_id(opts, i.part_num); }); return do_azure_client_call( [&]() { SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); @@ -247,6 +253,38 @@ ObjectStorageResponse AzureObjStorageClient::complete_multipart_upload( opts, _tls_debug_context); } +ObjectStorageResponse AzureObjStorageClient::abort_multipart_upload( + const ObjectStoragePathOptions& opts) { + auto client = _client->GetBlockBlobClient(opts.key); + auto response = do_azure_client_call( + [&]() { + GetBlockListOptions get_options; + get_options.ListType = Models::BlockListType::All; + auto block_list = client.GetBlockList(get_options); + if (block_list.Value.CommittedBlocks.empty()) { + DeleteBlobOptions delete_options; + // The ETag fence, when present, protects a concurrently committed blob. + delete_options.AccessConditions.IfMatch = block_list.Value.ETag; + client.Delete(delete_options); + return; + } + std::vector committed_ids; + committed_ids.reserve(block_list.Value.CommittedBlocks.size()); + std::ranges::transform(block_list.Value.CommittedBlocks, + std::back_inserter(committed_ids), + [](const Models::BlobBlock& block) { return block.Name; }); + CommitBlockListOptions commit_options; + commit_options.AccessConditions.IfMatch = block_list.Value.ETag; + // Recommitting only the old IDs discards this writer's unique staged blocks. + client.CommitBlockList(committed_ids, commit_options); + }, + opts, _tls_debug_context); + // Azure creates no server-side object until the first block is staged, so absence is clean. + return response.http_code == static_cast(Azure::Core::Http::HttpStatusCode::NotFound) + ? ObjectStorageResponse::OK() + : response; +} + ObjectStorageHeadResponse AzureObjStorageClient::head_object(const ObjectStoragePathOptions& opts) { Models::BlobProperties properties {}; auto resp = do_azure_client_call( diff --git a/be/src/io/fs/azure_obj_storage_client.h b/be/src/io/fs/azure_obj_storage_client.h index 7d1cecc502e44d..58c8a24481de69 100644 --- a/be/src/io/fs/azure_obj_storage_client.h +++ b/be/src/io/fs/azure_obj_storage_client.h @@ -49,6 +49,7 @@ class AzureObjStorageClient final : public ObjStorageClient { ObjectStorageResponse complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) override; + ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions& opts) override; ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override; ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, size_t offset, size_t bytes_read, diff --git a/be/src/io/fs/obj_storage_client.h b/be/src/io/fs/obj_storage_client.h index 5db9071e64538c..db326a931719f9 100644 --- a/be/src/io/fs/obj_storage_client.h +++ b/be/src/io/fs/obj_storage_client.h @@ -44,7 +44,7 @@ struct ObjectStoragePathOptions { std::string bucket = std::string(); // blob container in azure std::string key = std::string(); // blob name in azure std::string prefix = std::string(); // for batch delete and recursive delete - std::optional upload_id = std::nullopt; // only used for S3 upload + std::optional upload_id = std::nullopt; // provider-specific upload token }; struct ObjectCompleteMultiPart { @@ -86,7 +86,7 @@ struct ObjectStorageHeadResponse : ObjectStorageResponse { class ObjStorageClient { public: virtual ~ObjStorageClient() = default; - // Create a multi-part upload request. On AWS-compatible systems, it will return an upload ID, but not on Azure. + // Create a multi-part upload request. The returned provider token identifies this upload's parts. // The input parameters should include the bucket and key for the object storage. virtual ObjectStorageUploadResponse create_multipart_upload( const ObjectStoragePathOptions& opts) = 0; diff --git a/be/src/io/fs/rate_limited_obj_storage_client.cpp b/be/src/io/fs/rate_limited_obj_storage_client.cpp index 73cf80d73c1630..218c39cb4b19ec 100644 --- a/be/src/io/fs/rate_limited_obj_storage_client.cpp +++ b/be/src/io/fs/rate_limited_obj_storage_client.cpp @@ -75,10 +75,7 @@ ObjectStorageResponse RateLimitedObjStorageClient::complete_multipart_upload( ObjectStorageResponse RateLimitedObjStorageClient::abort_multipart_upload( const ObjectStoragePathOptions& opts) { - S3RateLimitGuard guard(S3RateLimitType::PUT, 0); - if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::PUT, guard.reject_reason()); - } + // Cleanup must reach the provider even when a hard PUT limit caused the upload failure. return _inner->abort_multipart_upload(opts); } diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index 0ebb616a0e8cb5..43663afa6dfdf8 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -109,7 +109,7 @@ Status S3FileWriter::_abort_impl() { _wait_until_finish( fmt::format("wait s3 file {} before abort", _obj_storage_path_opts.path.native())); _pending_buf.reset(); - if (_obj_storage_path_opts.upload_id.has_value()) { + if (_multipart_upload_started) { const auto& client = _obj_client->get(); if (client == nullptr) { return Status::InternalError("invalid obj storage client"); @@ -131,6 +131,8 @@ Status S3FileWriter::_create_multi_upload_request() { } auto resp = client->create_multipart_upload(_obj_storage_path_opts); if (resp.resp.status.code == ErrorCode::OK) { + // Some providers identify staged uploads by block IDs instead of a server-issued upload ID. + _multipart_upload_started = true; _obj_storage_path_opts.upload_id = resp.upload_id; } return {resp.resp.status.code, std::move(resp.resp.status.msg)}; diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h index bfebae917b1386..eb40772eea095e 100644 --- a/be/src/io/fs/s3_file_writer.h +++ b/be/src/io/fs/s3_file_writer.h @@ -119,6 +119,7 @@ class S3FileWriter final : public FileWriter { std::shared_ptr _obj_client; std::optional _first_append_timestamp; bool _close_latency_recorded = false; + bool _multipart_upload_started = false; }; } // namespace io diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 7b3cae3d86d5f2..fde7ca3bc2d0ad 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -69,15 +69,16 @@ Status RuntimeState::add_iceberg_commit_datas(TIcebergCommitData iceberg_commit_ const size_t thrift_limit = static_cast(std::max(config::thrift_max_message_size, 0)); const size_t commit_data_limit = thrift_limit > report_envelope_headroom ? thrift_limit - report_envelope_headroom : 0; - std::lock_guard lock(_iceberg_commit_datas_mutex); - // Reject metadata while its file can still be removed; a later oversized report would strand every output. - if (_iceberg_commit_datas_serialized_bytes + serialized_size + sizeof(uint32_t) > + std::lock_guard budget_lock(_iceberg_commit_data_budget->mutex); + // Parallel task states share this budget because FE receives their vectors in one fragment report. + if (_iceberg_commit_data_budget->serialized_bytes + serialized_size + sizeof(uint32_t) > commit_data_limit) { return Status::InternalError( "Iceberg commit metadata exceeds the Thrift report limit; reduce output file " "count"); } - _iceberg_commit_datas_serialized_bytes += serialized_size + sizeof(uint32_t); + std::lock_guard data_lock(_iceberg_commit_datas_mutex); + _iceberg_commit_data_budget->serialized_bytes += serialized_size + sizeof(uint32_t); _iceberg_commit_datas.emplace_back(std::move(iceberg_commit_data)); return Status::OK(); } diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index d440e3b45b9636..9ae8484d81709a 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -76,6 +76,14 @@ class RuntimeFilterConsumer; class RuntimeFilterProducer; class TaskExecutionContext; +class IcebergCommitDataBudget { + friend class RuntimeState; + +private: + std::mutex mutex; + size_t serialized_bytes = 0; +}; + // A collection of items that are part of the global state of a // query and shared across all execution nodes of that query. class RuntimeState { @@ -532,6 +540,14 @@ class RuntimeState { Status add_iceberg_commit_datas(TIcebergCommitData iceberg_commit_data); + void set_iceberg_commit_data_budget(std::shared_ptr budget) { + _iceberg_commit_data_budget = std::move(budget); + } + + const std::shared_ptr& iceberg_commit_data_budget() const { + return _iceberg_commit_data_budget; + } + std::vector mc_commit_datas() const { std::lock_guard lock(_mc_commit_datas_mutex); return _mc_commit_datas; @@ -975,7 +991,8 @@ class RuntimeState { mutable std::mutex _iceberg_commit_datas_mutex; std::vector _iceberg_commit_datas; - size_t _iceberg_commit_datas_serialized_bytes = 0; + std::shared_ptr _iceberg_commit_data_budget = + std::make_shared(); mutable std::mutex _mc_commit_datas_mutex; std::vector _mc_commit_datas; diff --git a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp new file mode 100644 index 00000000000000..87a22e4071fbcc --- /dev/null +++ b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/operator/spill_iceberg_table_sink_operator.h" + +#include + +namespace doris { + +TEST(SpillIcebergTableSinkOperatorTest, BoundsManyPartitionReservationToOneInputBlock) { + std::vector per_partition_reservations(128, 8 * 1024 * 1024); + + EXPECT_EQ(8 * 1024 * 1024, bounded_iceberg_reserve_size(per_partition_reservations)); +} + +} // namespace doris diff --git a/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp b/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp index 264c9895990a4b..e38da59cfbc176 100644 --- a/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp +++ b/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp @@ -17,6 +17,9 @@ #include +#include +#include + #include "core/column/column_vector.h" #include "core/data_type/data_type_number.h" #include "exec/sink/writer/iceberg/viceberg_table_writer.h" @@ -28,6 +31,12 @@ namespace { class FakePartitionWriter final : public IPartitionWriterBase { public: + explicit FakePartitionWriter(std::atomic* destroyed = nullptr) : _destroyed(destroyed) {} + ~FakePartitionWriter() override { + if (_destroyed != nullptr) { + ++(*_destroyed); + } + } Status open(RuntimeState*, RuntimeProfile*, const RowDescriptor*) override { return Status::OK(); } @@ -39,6 +48,7 @@ class FakePartitionWriter final : public IPartitionWriterBase { private: std::string _name = "fake"; + std::atomic* _destroyed; }; TDataSink make_sink() { @@ -62,6 +72,15 @@ class VIcebergTableWriterTest : public testing::Test { std::make_shared()); } + static void add_writer(VIcebergTableWriter* writer, std::string partition, + std::shared_ptr partition_writer) { + writer->_partitions_to_writers.emplace(std::move(partition), std::move(partition_writer)); + } + + static void clear_writers(VIcebergTableWriter* writer) { + writer->_partitions_to_writers.clear(); + } + static void publish_active_writers(VIcebergTableWriter* writer) { writer->_publish_active_writers(); } @@ -97,4 +116,29 @@ TEST_F(VIcebergTableWriterTest, ActiveWriterSnapshotContainsEveryOpenPartition) EXPECT_EQ(writer.active_writers()->size(), 2); } +TEST_F(VIcebergTableWriterTest, LoadedSnapshotRetainsWritersDuringConcurrentPublication) { + VIcebergTableWriter writer(make_sink(), {}, nullptr, nullptr); + std::atomic destroyed = 0; + add_writer(&writer, "p=1", std::make_shared(&destroyed)); + publish_active_writers(&writer); + std::promise snapshot_loaded; + std::promise replacement_published; + + auto reader = std::async(std::launch::async, [&]() { + auto snapshot = writer.active_writers(); + snapshot_loaded.set_value(); + replacement_published.get_future().wait(); + EXPECT_EQ(1, snapshot->size()); + EXPECT_EQ("fake", snapshot->front()->file_name()); + }); + + snapshot_loaded.get_future().wait(); + clear_writers(&writer); + publish_active_writers(&writer); + EXPECT_EQ(0, destroyed.load()); + replacement_published.set_value(); + reader.get(); + EXPECT_EQ(1, destroyed.load()); +} + } // namespace doris diff --git a/be/test/io/fs/azure_obj_storage_client_test.cpp b/be/test/io/fs/azure_obj_storage_client_test.cpp index 7591b4bf2ea997..d9f10e41e94caf 100644 --- a/be/test/io/fs/azure_obj_storage_client_test.cpp +++ b/be/test/io/fs/azure_obj_storage_client_test.cpp @@ -156,6 +156,26 @@ TEST_F(AzureObjStorageClientTest, delete_objects_recursively) { EXPECT_EQ(response.status.code, ErrorCode::OK); EXPECT_EQ(files.size(), 0); } + +TEST_F(AzureObjStorageClientTest, abort_multipart_upload_discards_staged_blocks) { + io::ObjectStoragePathOptions opts; + auto create_response = + AzureObjStorageClientTest::obj_storage_client->create_multipart_upload(opts); + ASSERT_EQ(create_response.resp.status.code, ErrorCode::OK); + ASSERT_TRUE(create_response.upload_id.has_value()); + opts.key = "AzureObjStorageClientTest/abort_multipart_upload_" + *create_response.upload_id; + opts.upload_id = create_response.upload_id; + + auto upload_response = + AzureObjStorageClientTest::obj_storage_client->upload_part(opts, "staged", 1); + ASSERT_EQ(upload_response.resp.status.code, ErrorCode::OK); + auto abort_response = + AzureObjStorageClientTest::obj_storage_client->abort_multipart_upload(opts); + ASSERT_EQ(abort_response.status.code, ErrorCode::OK); + + auto head_response = AzureObjStorageClientTest::obj_storage_client->head_object(opts); + EXPECT_EQ(head_response.resp.status.code, ErrorCode::NOT_FOUND); +} #else class AzureObjStorageClientTest : public testing::Test { diff --git a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp index 657b139c0a4fcd..6120347beb0f7c 100644 --- a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp +++ b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp @@ -62,6 +62,11 @@ class FakeObjStorageClient : public ObjStorageClient { ++calls; return ObjectStorageResponse::OK(); } + ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions& opts) override { + ++calls; + ++abort_multipart_upload_calls; + return ObjectStorageResponse::OK(); + } ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override { ++calls; return {}; @@ -106,6 +111,7 @@ class FakeObjStorageClient : public ObjStorageClient { int create_multipart_upload_calls = 0; int create_multipart_upload_provider_calls = 0; int create_multipart_upload_provider_calls_per_logical_call = 1; + int abort_multipart_upload_calls = 0; int delete_objects_recursively_calls = 0; int delete_objects_recursively_provider_calls = 0; int delete_objects_recursively_provider_calls_per_logical_call = 1; @@ -368,6 +374,23 @@ TEST(RateLimitedObjStorageClientTest, multipart_control_apis_map_to_put_qps_with EXPECT_EQ(-1, put_bytes->add(1)); } +TEST(RateLimitedObjStorageClientTest, abortBypassesAnExhaustedPutLimit) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + manager.qps_limiter(S3RateLimitType::PUT) + ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); + manager.bytes_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); + + auto fake = std::make_shared(); + RateLimitedObjStorageClient client(fake); + ObjectStoragePathOptions opts {.bucket = "b", .key = "k", .upload_id = "upload"}; + + EXPECT_EQ(0, client.create_multipart_upload(opts).resp.status.code); + EXPECT_EQ(0, client.abort_multipart_upload(opts).status.code); + EXPECT_EQ(1, fake->abort_multipart_upload_calls); +} + TEST(RateLimitedObjStorageClientTest, delete_apis_map_to_put_qps_without_bytes) { RateLimiterConfigGuard guard; config::enable_s3_rate_limiter = true; diff --git a/be/test/io/fs/s3_file_writer_test.cpp b/be/test/io/fs/s3_file_writer_test.cpp index a3033c9f1c07e4..b00f6c491c72d4 100644 --- a/be/test/io/fs/s3_file_writer_test.cpp +++ b/be/test/io/fs/s3_file_writer_test.cpp @@ -1169,6 +1169,14 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { return default_response; } + ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions& opts) override { + std::lock_guard lock(_mutex); + abort_multipart_count++; + last_opts = opts; + parts.clear(); + return default_response; + } + ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override { std::lock_guard lock(_mutex); return {.resp = ObjectStorageResponse::OK(), @@ -1243,6 +1251,7 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { int put_object_count = 0; int upload_part_count = 0; int complete_multipart_count = 0; + int abort_multipart_count = 0; // Structures to store input parameters for each call struct UploadPartParams { @@ -1281,6 +1290,7 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { put_object_count = 0; upload_part_count = 0; complete_multipart_count = 0; + abort_multipart_count = 0; create_multipart_params.clear(); put_object_params.clear(); @@ -1322,6 +1332,19 @@ create_s3_client(const std::string& path) { return {mock_client, s3_file_writer}; } +TEST_F(S3FileWriterTest, abortsProviderMultipartWithoutAnUploadId) { + auto [client, writer] = create_s3_client("provider_without_upload_id"); + client->default_upload_response.upload_id.reset(); + std::string data(config::s3_write_buffer_size, 'a'); + + ASSERT_TRUE(writer->append(Slice(data)).ok()); + ASSERT_TRUE(writer->abort().ok()); + + EXPECT_EQ(1, client->create_multipart_count); + EXPECT_EQ(1, client->abort_multipart_count); + EXPECT_EQ(FileWriter::State::CLOSED, writer->state()); +} + /** * Generate test data for S3FileWriter boundary tests. * Returns a vector of sizes that we'll use to generate data on demand. diff --git a/be/test/runtime/runtime_state_block_budget_test.cpp b/be/test/runtime/runtime_state_block_budget_test.cpp index b2b1e141508101..24afe920d197c5 100644 --- a/be/test/runtime/runtime_state_block_budget_test.cpp +++ b/be/test/runtime/runtime_state_block_budget_test.cpp @@ -40,6 +40,25 @@ TEST(RuntimeStateIcebergCommitDataTest, RejectsMetadataBeforeItCanExceedTheThrif EXPECT_TRUE(collected.empty()); } +TEST(RuntimeStateIcebergCommitDataTest, SharesTheReportBudgetAcrossParallelTasks) { + RuntimeState first; + RuntimeState second; + auto budget = std::make_shared(); + first.set_iceberg_commit_data_budget(budget); + second.set_iceberg_commit_data_budget(budget); + const int32_t saved_limit = config::thrift_max_message_size; + config::thrift_max_message_size = 1024 * 1024 + 512; + TIcebergCommitData commit_data; + commit_data.__set_file_path(std::string(300, 'x')); + + Status first_status = first.add_iceberg_commit_datas(commit_data); + Status second_status = second.add_iceberg_commit_datas(commit_data); + + config::thrift_max_message_size = saved_limit; + EXPECT_TRUE(first_status.ok()) << first_status; + EXPECT_FALSE(second_status.ok()); +} + // --------------------------------------------------------------------------- // RuntimeState::batch_size() // --------------------------------------------------------------------------- diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java index 0b353703ba723e..cbdf75169e4eb0 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java @@ -33,19 +33,25 @@ import org.apache.iceberg.ReachableFileUtil; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.FileInfo; import org.apache.iceberg.io.SupportsPrefixOperations; +import org.apache.iceberg.util.PropertyUtil; import java.io.IOException; import java.net.URI; +import java.time.Duration; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; /** Safely lists or deletes old files that are unreachable from every retained snapshot. */ public class IcebergRemoveOrphanFilesAction extends BaseIcebergAction { + private static final long MIN_RETENTION_MS = Duration.ofHours(24).toMillis(); public static final String OLDER_THAN = "older_than"; public static final String LOCATION = "location"; public static final String DRY_RUN = "dry_run"; @@ -84,6 +90,11 @@ protected List executeAction(Table table, ConnectorSession session) { if (!(table.io() instanceof SupportsPrefixOperations)) { throw new DorisConnectorException("remove_orphan_files requires FileIO prefix listing support"); } + if (!PropertyUtil.propertyAsBoolean(table.properties(), TableProperties.GC_ENABLED, + TableProperties.GC_ENABLED_DEFAULT)) { + // A GC-disabled table may share files with another table, so no destructive scan is safe. + throw new DorisConnectorException("Cannot remove orphan files: Iceberg GC is disabled"); + } String tableLocation = normalizeLocation(table.location()); String scanLocation = namedArguments.getString(LOCATION); scanLocation = scanLocation == null ? tableLocation : normalizeLocation(scanLocation); @@ -93,16 +104,21 @@ protected List executeAction(Table table, ConnectorSession session) { } try { - Set reachable = collectReachableFiles(table); + ReachableIndex reachable = new ReachableIndex(collectReachableFiles(table)); long orphanCount = 0; long deletedCount = 0; long olderThan = namedArguments.getLong(OLDER_THAN); + // The SQL procedure needs a retention fence because concurrent uploads are not reachable until commit. + if (olderThan > System.currentTimeMillis() - MIN_RETENTION_MS) { + throw new DorisConnectorException( + "older_than must retain at least 24 hours of files"); + } boolean dryRun = namedArguments.getBoolean(DRY_RUN); // Object stores use raw prefix matching, so the separator prevents "table_backup" siblings // from being treated as children of "table". String listingPrefix = scanLocation.endsWith("/") ? scanLocation : scanLocation + "/"; for (FileInfo file : ((SupportsPrefixOperations) table.io()).listPrefix(listingPrefix)) { - if (file.createdAtMillis() < olderThan && !reachable.contains(file.location())) { + if (file.createdAtMillis() < olderThan && !isReachable(file.location(), reachable)) { orphanCount++; if (!dryRun) { table.io().deleteFile(file.location()); @@ -118,6 +134,8 @@ protected List executeAction(Table table, ConnectorSession session) { private Set collectReachableFiles(Table table) throws IOException { Set reachable = new HashSet<>(ReachableFileUtil.metadataFileLocations(table, true)); + // Hadoop tables consult this live pointer even though it is not part of the metadata log. + reachable.add(ReachableFileUtil.versionHintLocation(table)); Set scannedDeleteManifests = new HashSet<>(); reachable.addAll(ReachableFileUtil.manifestListLocations(table)); reachable.addAll(ReachableFileUtil.statisticsFilesLocations(table)); @@ -146,6 +164,92 @@ private Set collectReachableFiles(Table table) throws IOException { return reachable; } + private static boolean isReachable(String candidate, ReachableIndex reachable) { + FileIdentity candidateIdentity = FileIdentity.of(candidate); + if (reachable.identities.contains(candidateIdentity)) { + return true; + } + if (reachable.paths.contains(candidateIdentity.path)) { + // A path collision across unknown providers/authorities cannot be classified safely. + throw new DorisConnectorException( + "Cannot determine whether listed and reachable file locations are equivalent"); + } + return false; + } + + static boolean sameFileIdentity(String first, String second) { + return FileIdentity.of(first).equals(FileIdentity.of(second)); + } + + static void verifyNoPrefixMismatch(String candidate, Set reachable) { + FileIdentity candidateIdentity = FileIdentity.of(candidate); + for (String retained : reachable) { + FileIdentity retainedIdentity = FileIdentity.of(retained); + // Matching paths with different providers/authorities are ambiguous; deletion must fail closed. + if (candidateIdentity.path.equals(retainedIdentity.path) + && !candidateIdentity.equals(retainedIdentity)) { + throw new DorisConnectorException( + "Cannot determine whether listed and reachable file locations are equivalent"); + } + } + } + + private static final class FileIdentity { + private final String scheme; + private final String authority; + private final String path; + + private FileIdentity(String scheme, String authority, String path) { + this.scheme = scheme; + this.authority = authority; + this.path = path; + } + + private static FileIdentity of(String location) { + URI uri = URI.create(location).normalize(); + String scheme = uri.getScheme(); + scheme = scheme == null ? "" : scheme.toLowerCase(Locale.ROOT); + if (scheme.equals("s3a") || scheme.equals("s3n")) { + scheme = "s3"; + } + String authority = uri.getAuthority(); + authority = authority == null ? "" : authority.toLowerCase(Locale.ROOT); + String path = uri.getPath(); + return new FileIdentity(scheme, authority, path == null ? "" : path); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof FileIdentity)) { + return false; + } + FileIdentity that = (FileIdentity) other; + return scheme.equals(that.scheme) && authority.equals(that.authority) + && path.equals(that.path); + } + + @Override + public int hashCode() { + return Objects.hash(scheme, authority, path); + } + } + + private static final class ReachableIndex { + private final Set identities = new HashSet<>(); + private final Set paths = new HashSet<>(); + + private ReachableIndex(Set locations) { + for (String location : locations) { + FileIdentity identity = FileIdentity.of(location); + identities.add(identity); + paths.add(identity.path); + } + } + } + private String normalizeLocation(String location) { String normalized = URI.create(location).normalize().toString(); return normalized.length() > 1 && normalized.endsWith("/") diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergProcedureOpsTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergProcedureOpsTest.java index 2ab67da81fd841..3d996d4926e93b 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergProcedureOpsTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergProcedureOpsTest.java @@ -86,7 +86,8 @@ public void getSupportedProceduresExportsFactoryNamesInLegacyOrder() { "expire_snapshots", "rewrite_data_files", "publish_changes", - "rewrite_manifests"), + "rewrite_manifests", + "remove_orphan_files"), newOps().getSupportedProcedures()); } @@ -120,7 +121,8 @@ public void executeRejectsUnknownProcedureWithLegacyMessage() { Assertions.assertEquals( "Unsupported Iceberg procedure: no_such_proc. Supported procedures: rollback_to_snapshot, " + "rollback_to_timestamp, set_current_snapshot, cherrypick_snapshot, fast_forward, " - + "expire_snapshots, rewrite_data_files, publish_changes, rewrite_manifests", + + "expire_snapshots, rewrite_data_files, publish_changes, rewrite_manifests, " + + "remove_orphan_files", e.getMessage()); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java new file mode 100644 index 00000000000000..fdccb1c026352a --- /dev/null +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.iceberg.action; + +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.procedure.ConnectorProcedureResult; + +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.ReachableFileUtil; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.hadoop.HadoopTables; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class IcebergRemoveOrphanFilesActionTest { + private static final long MIN_RETENTION_MS = Duration.ofHours(24).toMillis(); + + @Test + public void gcDisabledPreventsDeletion(@TempDir Path temp) throws Exception { + Table table = createTable(temp.resolve("table"), + Collections.singletonMap(TableProperties.GC_ENABLED, "false")); + Path orphan = createOldFile(temp.resolve("table/data/orphan.parquet")); + IcebergRemoveOrphanFilesAction action = action(System.currentTimeMillis() - MIN_RETENTION_MS, false); + action.validate(); + + Assertions.assertThrows(DorisConnectorException.class, + () -> action.execute(table, ActionTestTables.session("UTC"))); + Assertions.assertTrue(Files.exists(orphan)); + } + + @Test + public void recentCutoffCannotRaceAnUncommittedWriter(@TempDir Path temp) throws Exception { + Table table = createTable(temp.resolve("table"), Collections.emptyMap()); + Path uncommitted = createOldFile(temp.resolve("table/data/uncommitted.parquet")); + IcebergRemoveOrphanFilesAction action = action(System.currentTimeMillis(), false); + action.validate(); + + Assertions.assertThrows(DorisConnectorException.class, + () -> action.execute(table, ActionTestTables.session("UTC"))); + Assertions.assertTrue(Files.exists(uncommitted)); + } + + @Test + public void keepsVersionHintWhileDeletingAnOldOrphan(@TempDir Path temp) throws Exception { + Table table = createTable(temp.resolve("table"), Collections.emptyMap()); + Path orphan = createOldFile(temp.resolve("table/data/orphan.parquet")); + Path versionHint = Path.of(java.net.URI.create(ReachableFileUtil.versionHintLocation(table))); + Files.setLastModifiedTime(versionHint, FileTime.fromMillis(1)); + IcebergRemoveOrphanFilesAction action = action(System.currentTimeMillis() - MIN_RETENTION_MS, false); + action.validate(); + + ConnectorProcedureResult result = action.execute(table, ActionTestTables.session("UTC")); + + Assertions.assertEquals("1", result.getRows().get(0).get(0)); + Assertions.assertEquals("1", result.getRows().get(0).get(1)); + Assertions.assertFalse(Files.exists(orphan)); + Assertions.assertTrue(Files.exists(versionHint)); + } + + @Test + public void treatsS3SchemeAliasesAsTheSameFile() { + Assertions.assertTrue(IcebergRemoveOrphanFilesAction.sameFileIdentity( + "s3://bucket/path/data.parquet", "s3a://bucket/path/data.parquet")); + Assertions.assertTrue(IcebergRemoveOrphanFilesAction.sameFileIdentity( + "s3n://bucket/path/data.parquet", "s3://BUCKET/path/data.parquet")); + } + + @Test + public void rejectsUnresolvedPrefixMismatches() { + Assertions.assertThrows(DorisConnectorException.class, + () -> IcebergRemoveOrphanFilesAction.verifyNoPrefixMismatch( + "s3://first/path/data.parquet", + Collections.singleton("s3://second/path/data.parquet"))); + } + + private static IcebergRemoveOrphanFilesAction action(long olderThan, boolean dryRun) { + Map properties = new HashMap<>(); + properties.put(IcebergRemoveOrphanFilesAction.OLDER_THAN, String.valueOf(olderThan)); + properties.put(IcebergRemoveOrphanFilesAction.DRY_RUN, String.valueOf(dryRun)); + return new IcebergRemoveOrphanFilesAction(properties, Collections.emptyList(), null); + } + + private static Table createTable(Path location, Map properties) { + HadoopTables tables = new HadoopTables(new Configuration()); + return tables.create(ActionTestTables.SCHEMA, PartitionSpec.unpartitioned(), properties, + location.toUri().toString()); + } + + private static Path createOldFile(Path path) throws Exception { + Files.createDirectories(path.getParent()); + Files.write(path, new byte[] {1}); + Files.setLastModifiedTime(path, FileTime.fromMillis(1)); + return path; + } +} From 08b5ac7793ac7deb549228d3e0f8f7e3cfc0fed6 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 2 Aug 2026 11:02:42 +0800 Subject: [PATCH 03/29] [fix](iceberg) Preserve Azure data and deduplicate orphan scans ### What problem does this PR solve? Issue Number: None Related PR: #66348 Problem Summary: Azure abort could delete a pre-existing Put Blob object because an empty committed block list does not mean the key has no committed content. Iceberg orphan cleanup also reopened inherited data manifests once per retained snapshot, making long histories quadratic. Preserve Put Blob content while discarding only staged block-list replacements, and scan each unique retained manifest once. ### Release note Preserve existing Azure objects during failed Iceberg replacements and bound orphan cleanup manifest reads. ### Check List (For Author) - Test: Unit Test - Azure abort helper BE unit test - Iceberg orphan action FE unit tests - Behavior changed: Yes, Azure abort no longer deletes pre-existing Put Blob content and orphan cleanup deduplicates manifest reads. - Does this need documentation: No --- be/src/io/fs/azure_obj_storage_client.cpp | 11 +- be/src/io/fs/azure_obj_storage_client.h | 4 + .../io/fs/azure_obj_storage_client_test.cpp | 38 ++++++ .../IcebergRemoveOrphanFilesAction.java | 35 +++--- .../IcebergRemoveOrphanFilesActionTest.java | 112 ++++++++++++++++++ 5 files changed, 178 insertions(+), 22 deletions(-) diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp index 5bc0c8102e47b2..5a03c6831d05df 100644 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ b/be/src/io/fs/azure_obj_storage_client.cpp @@ -261,11 +261,14 @@ ObjectStorageResponse AzureObjStorageClient::abort_multipart_upload( GetBlockListOptions get_options; get_options.ListType = Models::BlockListType::All; auto block_list = client.GetBlockList(get_options); + const bool has_committed_blob = azure_block_list_has_committed_blob( + block_list.Value.CommittedBlocks.size(), block_list.Value.ETag.HasValue()); + if (!has_committed_blob) { + // Uncommitted blocks are invisible and expire without deleting a racing commit. + return; + } if (block_list.Value.CommittedBlocks.empty()) { - DeleteBlobOptions delete_options; - // The ETag fence, when present, protects a concurrently committed blob. - delete_options.AccessConditions.IfMatch = block_list.Value.ETag; - client.Delete(delete_options); + // Azure cannot selectively discard staged blocks without replacing Put Blob content. return; } std::vector committed_ids; diff --git a/be/src/io/fs/azure_obj_storage_client.h b/be/src/io/fs/azure_obj_storage_client.h index 58c8a24481de69..fe3cf01417aa3b 100644 --- a/be/src/io/fs/azure_obj_storage_client.h +++ b/be/src/io/fs/azure_obj_storage_client.h @@ -33,6 +33,10 @@ class ObjClientHolder; bool is_azure_tls_ca_error_message(std::string_view message); std::string build_azure_tls_debug_suffix(std::string_view error_message, std::string_view tls_debug_context); +inline bool azure_block_list_has_committed_blob(size_t committed_block_count, bool has_etag) { + // Put Blob creates committed content without block IDs; its ETag is the only safe discriminator. + return committed_block_count > 0 || has_etag; +} class AzureObjStorageClient final : public ObjStorageClient { public: diff --git a/be/test/io/fs/azure_obj_storage_client_test.cpp b/be/test/io/fs/azure_obj_storage_client_test.cpp index d9f10e41e94caf..560fc3664475c6 100644 --- a/be/test/io/fs/azure_obj_storage_client_test.cpp +++ b/be/test/io/fs/azure_obj_storage_client_test.cpp @@ -19,6 +19,8 @@ #include +#include + #include "io/fs/file_system.h" #include "io/fs/obj_storage_client.h" #include "util/s3_util.h" @@ -32,6 +34,12 @@ namespace doris { +TEST(AzureObjStorageClientAbortHelperTest, preserves_committed_put_blob_without_block_list) { + EXPECT_FALSE(io::azure_block_list_has_committed_blob(0, false)); + EXPECT_TRUE(io::azure_block_list_has_committed_blob(0, true)); + EXPECT_TRUE(io::azure_block_list_has_committed_blob(1, false)); +} + #ifdef USE_AZURE using namespace Azure::Storage::Blobs; @@ -176,6 +184,36 @@ TEST_F(AzureObjStorageClientTest, abort_multipart_upload_discards_staged_blocks) auto head_response = AzureObjStorageClientTest::obj_storage_client->head_object(opts); EXPECT_EQ(head_response.resp.status.code, ErrorCode::NOT_FOUND); } + +TEST_F(AzureObjStorageClientTest, abort_multipart_upload_preserves_existing_put_blob) { + io::ObjectStoragePathOptions opts; + auto create_response = + AzureObjStorageClientTest::obj_storage_client->create_multipart_upload(opts); + ASSERT_EQ(create_response.resp.status.code, ErrorCode::OK); + ASSERT_TRUE(create_response.upload_id.has_value()); + opts.key = "AzureObjStorageClientTest/abort_preserves_put_blob_" + *create_response.upload_id; + opts.upload_id = create_response.upload_id; + + auto put_response = AzureObjStorageClientTest::obj_storage_client->put_object(opts, "original"); + ASSERT_EQ(put_response.status.code, ErrorCode::OK); + auto upload_response = + AzureObjStorageClientTest::obj_storage_client->upload_part(opts, "replacement", 1); + ASSERT_EQ(upload_response.resp.status.code, ErrorCode::OK); + + auto abort_response = + AzureObjStorageClientTest::obj_storage_client->abort_multipart_upload(opts); + ASSERT_EQ(abort_response.status.code, ErrorCode::OK); + std::array contents {}; + size_t size_return = 0; + auto get_response = AzureObjStorageClientTest::obj_storage_client->get_object( + opts, contents.data(), 0, contents.size(), &size_return); + ASSERT_EQ(get_response.status.code, ErrorCode::OK); + EXPECT_EQ(size_return, contents.size()); + EXPECT_EQ(std::string_view(contents.data(), contents.size()), "original"); + + EXPECT_EQ(AzureObjStorageClientTest::obj_storage_client->delete_object(opts).status.code, + ErrorCode::OK); +} #else class AzureObjStorageClientTest : public testing::Test { diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java index cbdf75169e4eb0..af77d11b2ab677 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java @@ -25,8 +25,9 @@ import org.apache.doris.foundation.util.ArgumentParsers; import com.google.common.collect.Lists; +import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; -import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFile; import org.apache.iceberg.ManifestFiles; import org.apache.iceberg.ManifestReader; @@ -34,7 +35,6 @@ import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; -import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.FileInfo; import org.apache.iceberg.io.SupportsPrefixOperations; import org.apache.iceberg.util.PropertyUtil; @@ -136,28 +136,27 @@ private Set collectReachableFiles(Table table) throws IOException { Set reachable = new HashSet<>(ReachableFileUtil.metadataFileLocations(table, true)); // Hadoop tables consult this live pointer even though it is not part of the metadata log. reachable.add(ReachableFileUtil.versionHintLocation(table)); + Set scannedDataManifests = new HashSet<>(); Set scannedDeleteManifests = new HashSet<>(); reachable.addAll(ReachableFileUtil.manifestListLocations(table)); reachable.addAll(ReachableFileUtil.statisticsFilesLocations(table)); for (Snapshot snapshot : table.snapshots()) { for (ManifestFile manifest : snapshot.allManifests(table.io())) { reachable.add(manifest.path()); - } - for (ManifestFile manifest : snapshot.deleteManifests(table.io())) { - if (!scannedDeleteManifests.add(manifest.path())) { - continue; - } - // A retained delete file may not apply to any current data task, so read delete manifests directly. - try (ManifestReader deletes = - ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { - deletes.forEach(delete -> reachable.add(delete.location())); - } - } - try (CloseableIterable tasks = table.newScan() - .useSnapshot(snapshot.snapshotId()).planFiles()) { - for (FileScanTask task : tasks) { - reachable.add(task.file().location()); - task.deletes().forEach(delete -> reachable.add(delete.location())); + if (manifest.content() == ManifestContent.DATA) { + // Snapshots inherit manifests, so read each path once to keep work linear. + if (scannedDataManifests.add(manifest.path())) { + try (ManifestReader dataFiles = + ManifestFiles.read(manifest, table.io(), table.specs())) { + dataFiles.forEach(dataFile -> reachable.add(dataFile.location())); + } + } + } else if (scannedDeleteManifests.add(manifest.path())) { + // A retained delete file may not apply to any current data task, so read it directly. + try (ManifestReader deletes = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + deletes.forEach(delete -> reachable.add(delete.location())); + } } } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java index fdccb1c026352a..0e9ed5bc849371 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java @@ -21,11 +21,21 @@ import org.apache.doris.connector.api.procedure.ConnectorProcedureResult; import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.ReachableFileUtil; +import org.apache.iceberg.StaticTableOperations; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.FileInfo; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.io.SupportsPrefixOperations; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -36,7 +46,9 @@ import java.time.Duration; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; public class IcebergRemoveOrphanFilesActionTest { private static final long MIN_RETENTION_MS = Duration.ofHours(24).toMillis(); @@ -99,6 +111,37 @@ public void rejectsUnresolvedPrefixMismatches() { Collections.singleton("s3://second/path/data.parquet"))); } + @Test + public void readsEachSharedDataManifestOnlyOnce(@TempDir Path temp) throws Exception { + Map properties = new HashMap<>(); + properties.put(TableProperties.MANIFEST_MERGE_ENABLED, "false"); + Table table = createTable(temp.resolve("table"), properties); + appendDataFile(table, createOldFile(temp.resolve("table/data/first.parquet"))); + appendDataFile(table, createOldFile(temp.resolve("table/data/second.parquet"))); + + Set dataManifestPaths = new HashSet<>(); + int[] manifestReferenceCount = {0}; + table.snapshots().forEach(snapshot -> snapshot.dataManifests(table.io()) + .forEach(manifest -> { + manifestReferenceCount[0]++; + dataManifestPaths.add(manifest.path()); + })); + Assertions.assertTrue(manifestReferenceCount[0] > dataManifestPaths.size()); + RecordingFileIO recordingFileIO = new RecordingFileIO(table.io(), dataManifestPaths); + Table recordingTable = new BaseTable( + new StaticTableOperations(((HasTableOperations) table).operations().current(), recordingFileIO), + table.name()); + IcebergRemoveOrphanFilesAction action = action( + System.currentTimeMillis() - MIN_RETENTION_MS, true); + action.validate(); + + action.execute(recordingTable, ActionTestTables.session("UTC")); + + Assertions.assertEquals(dataManifestPaths.size(), recordingFileIO.manifestOpenCount()); + dataManifestPaths.forEach(path -> Assertions.assertEquals(1, + recordingFileIO.openCounts.getOrDefault(path, 0), path)); + } + private static IcebergRemoveOrphanFilesAction action(long olderThan, boolean dryRun) { Map properties = new HashMap<>(); properties.put(IcebergRemoveOrphanFilesAction.OLDER_THAN, String.valueOf(olderThan)); @@ -118,4 +161,73 @@ private static Path createOldFile(Path path) throws Exception { Files.setLastModifiedTime(path, FileTime.fromMillis(1)); return path; } + + private static void appendDataFile(Table table, Path path) { + DataFile dataFile = DataFiles.builder(table.spec()) + .withPath(path.toUri().toString()) + .withFileSizeInBytes(1) + .withRecordCount(1) + .build(); + table.newFastAppend().appendFile(dataFile).commit(); + } + + private static final class RecordingFileIO implements SupportsPrefixOperations { + private final FileIO delegate; + private final SupportsPrefixOperations prefixDelegate; + private final Set manifestPaths; + private final Map openCounts = new HashMap<>(); + + private RecordingFileIO(FileIO delegate, Set manifestPaths) { + this.delegate = delegate; + this.prefixDelegate = (SupportsPrefixOperations) delegate; + this.manifestPaths = manifestPaths; + } + + private void record(String path) { + if (manifestPaths.contains(path)) { + openCounts.merge(path, 1, Integer::sum); + } + } + + private int manifestOpenCount() { + return openCounts.values().stream().mapToInt(Integer::intValue).sum(); + } + + @Override + public InputFile newInputFile(String path) { + record(path); + return delegate.newInputFile(path); + } + + @Override + public InputFile newInputFile(String path, long length) { + record(path); + return delegate.newInputFile(path, length); + } + + @Override + public OutputFile newOutputFile(String path) { + return delegate.newOutputFile(path); + } + + @Override + public void deleteFile(String path) { + delegate.deleteFile(path); + } + + @Override + public Map properties() { + return delegate.properties(); + } + + @Override + public Iterable listPrefix(String prefix) { + return prefixDelegate.listPrefix(prefix); + } + + @Override + public void deletePrefix(String prefix) { + prefixDelegate.deletePrefix(prefix); + } + } } From 065e5df46b4f01f1c5fb936bfecfe74df294aea9 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 2 Aug 2026 12:30:11 +0800 Subject: [PATCH 04/29] [fix](iceberg) Address follow-up write safety review --- .../operator/iceberg_sorter_reserve_memory.h | 45 +++++++ .../spill_iceberg_table_sink_operator.cpp | 18 +-- .../spill_iceberg_table_sink_operator.h | 3 +- .../writer/iceberg/viceberg_sort_writer.cpp | 7 + .../writer/iceberg/viceberg_sort_writer.h | 2 + be/src/exec/sort/sorter.cpp | 17 ++- be/src/exec/sort/sorter.h | 9 ++ be/src/io/fs/azure_obj_storage_client.cpp | 5 +- ...spill_iceberg_table_sink_operator_test.cpp | 15 ++- .../io/fs/azure_obj_storage_client_test.cpp | 2 + .../iceberg/IcebergConnectorTransaction.java | 41 +++++- .../iceberg/IcebergScanPlanProvider.java | 5 +- .../connector/iceberg/IcebergTableHandle.java | 11 +- .../IcebergRemoveOrphanFilesAction.java | 83 +++++++++--- .../IcebergConnectorTransactionTest.java | 125 ++++++++++++++++++ .../iceberg/IcebergTableHandleTest.java | 11 ++ .../IcebergRemoveOrphanFilesActionTest.java | 75 +++++++++++ .../filesystem/azure/AzureObjStorage.java | 4 +- .../azure/AzureObjStorageExtensionTest.java | 43 ++++++ 19 files changed, 474 insertions(+), 47 deletions(-) create mode 100644 be/src/exec/operator/iceberg_sorter_reserve_memory.h diff --git a/be/src/exec/operator/iceberg_sorter_reserve_memory.h b/be/src/exec/operator/iceberg_sorter_reserve_memory.h new file mode 100644 index 00000000000000..1ad1454b4af5f3 --- /dev/null +++ b/be/src/exec/operator/iceberg_sorter_reserve_memory.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +namespace doris { + +struct IcebergSorterReserveMemory { + size_t retained_growth = 0; + size_t transient_workspace = 0; +}; + +inline size_t bounded_iceberg_reserve_size( + const std::vector& per_partition_reservations) { + size_t retained_growth = 0; + size_t transient_workspace = 0; + for (const auto& reservation : per_partition_reservations) { + retained_growth = std::min(std::numeric_limits::max() - retained_growth, + reservation.retained_growth) + + retained_growth; + transient_workspace = std::max(transient_workspace, reservation.transient_workspace); + } + return std::min(std::numeric_limits::max() - retained_growth, transient_workspace) + + retained_growth; +} + +} // namespace doris diff --git a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp index a974e575916b7f..5d0d1cb00a7916 100644 --- a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp +++ b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp @@ -17,8 +17,6 @@ #include "exec/operator/spill_iceberg_table_sink_operator.h" -#include - #include "common/status.h" #include "exec/operator/iceberg_table_sink_operator.h" #include "exec/sink/writer/iceberg/viceberg_sort_writer.h" @@ -26,13 +24,6 @@ namespace doris { -size_t bounded_iceberg_reserve_size(const std::vector& per_partition_reservations) { - return per_partition_reservations.empty() - ? 0 - : *std::max_element(per_partition_reservations.begin(), - per_partition_reservations.end()); -} - SpillIcebergTableSinkLocalState::SpillIcebergTableSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state) : Base(parent, state) {} @@ -64,15 +55,18 @@ size_t SpillIcebergTableSinkLocalState::get_reserve_mem_size(RuntimeState* state if (!_writer) { return 0; } - std::vector per_partition_reservations; + std::vector per_partition_reservations; auto active_writers = _writer->active_writers(); per_partition_reservations.reserve(active_writers->size()); for (const auto& writer : *active_writers) { if (auto* sort_writer = dynamic_cast(writer.get())) { - per_partition_reservations.push_back(sort_writer->get_reserve_mem_size(state, eos)); + auto reservation = sort_writer->get_reserve_mem_size_components(state, eos); + per_partition_reservations.push_back( + {.retained_growth = reservation.retained_growth, + .transient_workspace = reservation.transient_workspace}); } } - // One input block is partitioned among writers and consumed serially, so their full-batch estimates overlap. + // Column growth remains in every touched sorter, while sorting workspace is reused by serial dispatch. return bounded_iceberg_reserve_size(per_partition_reservations); } diff --git a/be/src/exec/operator/spill_iceberg_table_sink_operator.h b/be/src/exec/operator/spill_iceberg_table_sink_operator.h index 60eefd1bfb5242..5ffdd7505599ea 100644 --- a/be/src/exec/operator/spill_iceberg_table_sink_operator.h +++ b/be/src/exec/operator/spill_iceberg_table_sink_operator.h @@ -20,6 +20,7 @@ #include #include +#include "exec/operator/iceberg_sorter_reserve_memory.h" #include "exec/operator/operator.h" #include "exec/sink/writer/iceberg/viceberg_table_writer.h" @@ -28,8 +29,6 @@ namespace doris { class SpillIcebergTableSinkLocalState; class SpillIcebergTableSinkOperatorX; -size_t bounded_iceberg_reserve_size(const std::vector& per_partition_reservations); - class SpillIcebergTableSinkLocalState final : public AsyncWriterSink { public: diff --git a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp index 168199f1d00849..444aec8933ae6a 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp @@ -84,6 +84,13 @@ size_t VIcebergSortWriter::get_reserve_mem_size(RuntimeState* state, bool eos) c return _sorter == nullptr ? 0 : _sorter->get_reserve_mem_size(state, eos); } +SorterReserveMemory VIcebergSortWriter::get_reserve_mem_size_components(RuntimeState* state, + bool eos) const { + std::lock_guard lock(_sorter_mutex); + return _sorter == nullptr ? SorterReserveMemory {} + : _sorter->get_reserve_mem_size_components(state, eos); +} + Status VIcebergSortWriter::trigger_spill() { std::lock_guard lock(_sorter_mutex); if (_closed || _sorter == nullptr) { diff --git a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.h b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.h index e1e512f0a0cf79..37659eeca4bc89 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.h +++ b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.h @@ -105,6 +105,8 @@ class VIcebergSortWriter : public IPartitionWriterBase { size_t get_reserve_mem_size(RuntimeState* state, bool eos) const; + SorterReserveMemory get_reserve_mem_size_components(RuntimeState* state, bool eos) const; + // Called by the memory management system to trigger spilling data to disk Status trigger_spill(); diff --git a/be/src/exec/sort/sorter.cpp b/be/src/exec/sort/sorter.cpp index 2d9304adfa2f8e..c64d07b219a6cf 100644 --- a/be/src/exec/sort/sorter.cpp +++ b/be/src/exec/sort/sorter.cpp @@ -202,7 +202,12 @@ bool FullSorter::has_enough_capacity(Block* input_block, Block* unsorted_block) } size_t FullSorter::get_reserve_mem_size(RuntimeState* state, bool eos) const { - size_t size_to_reserve = 0; + return get_reserve_mem_size_components(state, eos).total(); +} + +SorterReserveMemory FullSorter::get_reserve_mem_size_components(RuntimeState* state, + bool eos) const { + SorterReserveMemory reserve; const auto rows = _state->unsorted_block()->rows(); if (rows != 0) { const auto bytes = _state->unsorted_block()->bytes(); @@ -213,24 +218,24 @@ size_t FullSorter::get_reserve_mem_size(RuntimeState* state, bool eos) const { auto new_rows = rows + state->batch_size(); // If the new size is greater than 85% of allocalted bytes, it maybe need to realloc. if ((new_block_bytes * 100 / allocated_bytes) >= 85) { - size_to_reserve += (size_t)(allocated_bytes * 1.15); + reserve.retained_growth += (size_t)(allocated_bytes * 1.15); } auto sort = new_rows > _buffered_block_size || new_block_bytes > _buffered_block_bytes; if (sort) { // new column is created when doing sort, reserve average size of one column // for estimation - size_to_reserve += new_block_bytes / _state->unsorted_block()->columns(); + reserve.transient_workspace += new_block_bytes / _state->unsorted_block()->columns(); // helping data structures used during sorting - size_to_reserve += new_rows * sizeof(IColumn::Permutation::value_type); + reserve.transient_workspace += new_rows * sizeof(IColumn::Permutation::value_type); auto sort_columns_count = _ordering_expr_ctxs.size(); if (1 != sort_columns_count) { - size_to_reserve += new_rows * sizeof(EqualRangeIterator); + reserve.transient_workspace += new_rows * sizeof(EqualRangeIterator); } } } - return size_to_reserve; + return reserve; } Status FullSorter::append_block(Block* block) { diff --git a/be/src/exec/sort/sorter.h b/be/src/exec/sort/sorter.h index 1651247eecc1ab..5c748f86a7f858 100644 --- a/be/src/exec/sort/sorter.h +++ b/be/src/exec/sort/sorter.h @@ -39,6 +39,13 @@ #include "runtime/runtime_state.h" namespace doris { + +struct SorterReserveMemory { + size_t retained_growth = 0; + size_t transient_workspace = 0; + + size_t total() const { return retained_growth + transient_workspace; } +}; class ObjectPool; class RowDescriptor; } // namespace doris @@ -194,6 +201,8 @@ class FullSorter final : public Sorter { size_t get_reserve_mem_size(RuntimeState* state, bool eos) const override; + SorterReserveMemory get_reserve_mem_size_components(RuntimeState* state, bool eos) const; + Status merge_sort_read_for_spill(RuntimeState* state, doris::Block* block, int batch_size, bool* eos) override; void reset() override; diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp index 5a03c6831d05df..daa215e3c58d9e 100644 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ b/be/src/io/fs/azure_obj_storage_client.cpp @@ -223,17 +223,20 @@ ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStora std::string_view stream, int part_num) { auto client = _client->GetBlockBlobClient(opts.key); + std::string block_id = azure_block_id(opts, part_num); auto resp = do_azure_client_call( [&]() { Azure::Core::IO::MemoryBodyStream memory_body( reinterpret_cast(stream.data()), stream.size()); // The blockId must be base64 encoded SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); - client.StageBlock(azure_block_id(opts, part_num), memory_body); + client.StageBlock(block_id, memory_body); }, opts, _tls_debug_context); return ObjectStorageUploadResponse { .resp = resp, + // Hive defers completion to FE, so the exact staged ID must cross that boundary. + .etag = block_id, }; } diff --git a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp index 87a22e4071fbcc..5d1fa473205f0b 100644 --- a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp +++ b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp @@ -15,16 +15,25 @@ // specific language governing permissions and limitations // under the License. -#include "exec/operator/spill_iceberg_table_sink_operator.h" - #include +#include "exec/operator/iceberg_sorter_reserve_memory.h" + namespace doris { TEST(SpillIcebergTableSinkOperatorTest, BoundsManyPartitionReservationToOneInputBlock) { - std::vector per_partition_reservations(128, 8 * 1024 * 1024); + std::vector per_partition_reservations( + 128, {.retained_growth = 0, .transient_workspace = 8 * 1024 * 1024}); EXPECT_EQ(8 * 1024 * 1024, bounded_iceberg_reserve_size(per_partition_reservations)); } +TEST(SpillIcebergTableSinkOperatorTest, AccumulatesRetainedGrowthAcrossTouchedPartitions) { + std::vector per_partition_reservations { + {.retained_growth = 3 * 1024 * 1024, .transient_workspace = 7 * 1024 * 1024}, + {.retained_growth = 4 * 1024 * 1024, .transient_workspace = 5 * 1024 * 1024}}; + + EXPECT_EQ(14 * 1024 * 1024, bounded_iceberg_reserve_size(per_partition_reservations)); +} + } // namespace doris diff --git a/be/test/io/fs/azure_obj_storage_client_test.cpp b/be/test/io/fs/azure_obj_storage_client_test.cpp index 560fc3664475c6..88e81ea7fe6ed0 100644 --- a/be/test/io/fs/azure_obj_storage_client_test.cpp +++ b/be/test/io/fs/azure_obj_storage_client_test.cpp @@ -177,6 +177,8 @@ TEST_F(AzureObjStorageClientTest, abort_multipart_upload_discards_staged_blocks) auto upload_response = AzureObjStorageClientTest::obj_storage_client->upload_part(opts, "staged", 1); ASSERT_EQ(upload_response.resp.status.code, ErrorCode::OK); + ASSERT_TRUE(upload_response.etag.has_value()); + EXPECT_FALSE(upload_response.etag->empty()); auto abort_response = AzureObjStorageClientTest::obj_storage_client->abort_multipart_upload(opts); ASSERT_EQ(abort_response.status.code, ErrorCode::OK); diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java index f21dfb99359082..9306f29bbcfd18 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java @@ -439,14 +439,51 @@ private void applyBeginGuards(IcebergWriteContext ctx, String tableName) { throw new IllegalArgumentException(branchName + " is a tag, not a branch. Tags cannot be targets for producing snapshots"); } - this.baseSnapshotId = op == WriteOperation.OVERWRITE ? branchRef.snapshotId() : null; + this.baseSnapshotId = op == WriteOperation.OVERWRITE + ? resolveOverwriteBaseSnapshot(ctx, branchRef.snapshotId(), tableName) : null; } else { this.branchName = null; - this.baseSnapshotId = op == WriteOperation.OVERWRITE ? getSnapshotIdIfPresent(table) : null; + this.baseSnapshotId = op == WriteOperation.OVERWRITE + ? resolveOverwriteBaseSnapshot(ctx, getSnapshotIdIfPresent(table), tableName) : null; } } } + private Long resolveOverwriteBaseSnapshot(IcebergWriteContext ctx, Long targetHead, String tableName) { + if (!ctx.isReadSnapshotResolved()) { + return targetHead; + } + long readSnapshotId = ctx.getReadSnapshotId(); + if (readSnapshotId < 0) { + // An explicit empty read must conflict with any snapshot created before beginWrite. + if (targetHead != null) { + throw new DorisConnectorException("Iceberg table " + tableName + + " changed after the statement read an empty snapshot"); + } + return null; + } + if (targetHead == null || !isAncestorOfTarget(readSnapshotId, targetHead)) { + throw new DorisConnectorException("Read snapshot " + readSnapshotId + + " is not an ancestor of the target branch for Iceberg table " + tableName); + } + return readSnapshotId; + } + + private boolean isAncestorOfTarget(long ancestorId, long targetHeadId) { + Long snapshotId = targetHeadId; + while (snapshotId != null) { + if (snapshotId == ancestorId) { + return true; + } + Snapshot snapshot = table.snapshot(snapshotId); + if (snapshot == null) { + return false; + } + snapshotId = snapshot.parentId(); + } + return false; + } + @Override public long getTransactionId() { return transactionId; diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java index 09f8fa44615093..bfabf1c986adba 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java @@ -936,7 +936,7 @@ private List doPlanPositionDeletesSystemTableScan(IcebergTab Table metadataTable, List columns, Optional filter, ConnectorSession session) { BatchScan scan = metadataTable.newBatchScan(); - if (handle.hasSnapshotPin()) { + if (handle.hasSnapshotSelection()) { if (handle.getRef() != null) { scan = scan.useRef(handle.getRef()); } else { @@ -1157,6 +1157,9 @@ private TableScan buildScan(Table table, IcebergTableHandle handle, Optional= 0 || ref != null; } + /** Whether the pin selects an Iceberg snapshot/ref rather than the explicit empty-table state. */ + public boolean hasSnapshotSelection() { + return snapshotId >= 0 || ref != null; + } + /** Whether snapshot resolution observed a table before its first snapshot was committed. */ public boolean isResolvedEmptySnapshot() { return snapshotResolved && snapshotId < 0 && ref == null; diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java index af77d11b2ab677..37226b21fde047 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java @@ -42,7 +42,9 @@ import java.io.IOException; import java.net.URI; import java.time.Duration; +import java.util.ArrayList; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -95,13 +97,7 @@ protected List executeAction(Table table, ConnectorSession session) { // A GC-disabled table may share files with another table, so no destructive scan is safe. throw new DorisConnectorException("Cannot remove orphan files: Iceberg GC is disabled"); } - String tableLocation = normalizeLocation(table.location()); - String scanLocation = namedArguments.getString(LOCATION); - scanLocation = scanLocation == null ? tableLocation : normalizeLocation(scanLocation); - // Normalize dot segments before the containment check so local FileIO paths cannot escape the table root. - if (!scanLocation.equals(tableLocation) && !scanLocation.startsWith(tableLocation + "/")) { - throw new DorisConnectorException("location must be within the Iceberg table location"); - } + List scanLocations = resolveScanLocations(table); try { ReachableIndex reachable = new ReachableIndex(collectReachableFiles(table)); @@ -114,15 +110,18 @@ protected List executeAction(Table table, ConnectorSession session) { "older_than must retain at least 24 hours of files"); } boolean dryRun = namedArguments.getBoolean(DRY_RUN); - // Object stores use raw prefix matching, so the separator prevents "table_backup" siblings - // from being treated as children of "table". - String listingPrefix = scanLocation.endsWith("/") ? scanLocation : scanLocation + "/"; - for (FileInfo file : ((SupportsPrefixOperations) table.io()).listPrefix(listingPrefix)) { - if (file.createdAtMillis() < olderThan && !isReachable(file.location(), reachable)) { - orphanCount++; - if (!dryRun) { - table.io().deleteFile(file.location()); - deletedCount++; + Set visitedFiles = new HashSet<>(); + for (String scanLocation : scanLocations) { + // Object stores use raw prefix matching, so the separator excludes sibling prefixes. + String listingPrefix = scanLocation.endsWith("/") ? scanLocation : scanLocation + "/"; + for (FileInfo file : ((SupportsPrefixOperations) table.io()).listPrefix(listingPrefix)) { + if (visitedFiles.add(file.location()) && file.createdAtMillis() < olderThan + && !isReachable(file.location(), reachable)) { + orphanCount++; + if (!dryRun) { + table.io().deleteFile(file.location()); + deletedCount++; + } } } } @@ -132,6 +131,58 @@ protected List executeAction(Table table, ConnectorSession session) { } } + private List resolveScanLocations(Table table) { + String tableRoot = normalizeLocation(table.location()); + String dataRoot = normalizeLocation(resolveDataLocation(table, tableRoot)); + Set ownedRoots = new LinkedHashSet<>(); + ownedRoots.add(tableRoot); + ownedRoots.add(dataRoot); + + String requested = namedArguments.getString(LOCATION); + if (requested != null) { + String normalized = normalizeLocation(requested); + boolean owned = ownedRoots.stream().anyMatch(root -> isWithin(normalized, root)); + if (!owned) { + throw new DorisConnectorException( + "location must be within an Iceberg table-owned metadata or data location"); + } + return Lists.newArrayList(normalized); + } + + List roots = new ArrayList<>(); + for (String candidate : ownedRoots) { + // Avoid listing a nested default data directory twice when the table root already covers it. + if (ownedRoots.stream().noneMatch(other -> !other.equals(candidate) && isWithin(candidate, other))) { + roots.add(candidate); + } + } + return roots; + } + + private String resolveDataLocation(Table table, String tableRoot) { + Map properties = table.properties(); + String dataLocation = nonEmpty(properties.get(TableProperties.WRITE_DATA_LOCATION)); + if (dataLocation == null && Boolean.parseBoolean(properties.get(TableProperties.OBJECT_STORE_ENABLED))) { + dataLocation = nonEmpty(properties.get(TableProperties.OBJECT_STORE_PATH)); + } + if (dataLocation == null) { + dataLocation = nonEmpty(properties.get(TableProperties.WRITE_FOLDER_STORAGE_LOCATION)); + } + return dataLocation == null ? tableRoot + "/data" : dataLocation; + } + + private String nonEmpty(String location) { + return location == null || location.isEmpty() ? null : location; + } + + private boolean isWithin(String location, String root) { + FileIdentity child = FileIdentity.of(location); + FileIdentity parent = FileIdentity.of(root); + String pathPrefix = parent.path.endsWith("/") ? parent.path : parent.path + "/"; + return child.scheme.equals(parent.scheme) && child.authority.equals(parent.authority) + && (child.path.equals(parent.path) || child.path.startsWith(pathPrefix)); + } + private Set collectReachableFiles(Table table) throws IOException { Set reachable = new HashSet<>(ReachableFileUtil.metadataFileLocations(table, true)); // Hadoop tables consult this live pointer even though it is not part of the metadata log. diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java index 1fa84febaac271..0df759d7a215a9 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java @@ -138,6 +138,11 @@ private static IcebergWriteContext overwriteToBranch(String branch) { WriteOperation.OVERWRITE, true, Collections.emptyMap(), Optional.of(branch)); } + private static IcebergWriteContext overwriteCtxPinned(long readSnapshotId) { + return new IcebergWriteContext(WriteOperation.OVERWRITE, true, Collections.emptyMap(), Optional.empty(), + readSnapshotId); + } + private static IcebergWriteContext overwriteStaticCtx(Table table, Map staticValues) { IcebergWriteSchemaContext schemaContext = IcebergWriteSchemaContext.create(table, table.name(), Optional.empty(), false, false); @@ -614,6 +619,84 @@ public void overwriteDynamicRejectsConcurrentDataInReplacedPartition() { "dynamic overwrite must not silently replace data committed after its base snapshot"); } + @Test + public void overwriteDynamicRejectsDataCommittedBetweenScanAndBegin() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t1"); + PartitionSpec spec = PartitionSpec.builderFor(PART_SCHEMA).identity("region").build(); + Table table = catalog.createTable(id, PART_SCHEMA, spec, props("write.format.default", "parquet")); + table.newAppend().appendFile(partitionedDataFile(spec, + "s3://b/db1/t1/region=us/seed.parquet", 1L, "region=us")).commit(); + long readSnapshotId = table.currentSnapshot().snapshotId(); + table.newAppend().appendFile(partitionedDataFile(spec, + "s3://b/db1/t1/region=us/between-scan-and-begin.parquet", 1L, "region=us")).commit(); + + IcebergConnectorTransaction txn = txnFor( + opsReturning(catalog.loadTable(id)), new RecordingConnectorContext()); + txn.beginWrite(SESSION, "db1", "t1", overwriteCtxPinned(readSnapshotId)); + txn.addCommitData(commitBytes( + dataFileItem("s3://b/db1/t1/region=us/replacement.parquet", 2L, 1024L, + Collections.singletonList("us")))); + + Assertions.assertThrows(DorisConnectorException.class, txn::commit); + } + + @Test + public void overwriteRejectsFirstSnapshotCommittedAfterEmptyRead() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t1"); + Table table = catalog.createTable(id, SCHEMA, PartitionSpec.unpartitioned(), + props("write.format.default", "parquet")); + table.newAppend().appendFile(dataFile(table.spec(), + "s3://b/db1/t1/between-scan-and-begin.parquet", 1L)).commit(); + IcebergConnectorTransaction txn = txnFor( + opsReturning(catalog.loadTable(id)), new RecordingConnectorContext()); + + Assertions.assertThrows(DorisConnectorException.class, + () -> txn.beginWrite(SESSION, "db1", "t1", overwriteCtxPinned(-1L))); + } + + @Test + public void overwriteRejectsFirstSnapshotCommittedAfterBeginFromEmptyRead() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t1"); + Table table = catalog.createTable(id, SCHEMA, PartitionSpec.unpartitioned(), + props("write.format.default", "parquet")); + IcebergConnectorTransaction txn = txnFor( + opsReturning(catalog.loadTable(id)), new RecordingConnectorContext()); + txn.beginWrite(SESSION, "db1", "t1", overwriteCtxPinned(-1L)); + + table.newAppend().appendFile(dataFile(table.spec(), + "s3://b/db1/t1/after-begin.parquet", 1L)).commit(); + + Assertions.assertThrows(DorisConnectorException.class, txn::commit); + } + + @Test + public void overwriteBranchUsesTheSnapshotReadFromThatBranch() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t1"); + PartitionSpec spec = PartitionSpec.builderFor(PART_SCHEMA).identity("region").build(); + Table table = catalog.createTable(id, PART_SCHEMA, spec, props("write.format.default", "parquet")); + table.newAppend().appendFile(partitionedDataFile(spec, + "s3://b/db1/t1/region=us/seed.parquet", 1L, "region=us")).commit(); + long readSnapshotId = table.currentSnapshot().snapshotId(); + table.manageSnapshots().createBranch("b1", readSnapshotId).commit(); + table.newAppend().toBranch("b1").appendFile(partitionedDataFile(spec, + "s3://b/db1/t1/region=us/between-scan-and-begin.parquet", 1L, "region=us")).commit(); + + IcebergConnectorTransaction txn = txnFor( + opsReturning(catalog.loadTable(id)), new RecordingConnectorContext()); + txn.beginWrite(SESSION, "db1", "t1", + new IcebergWriteContext(WriteOperation.OVERWRITE, true, Collections.emptyMap(), + Optional.of("b1"), readSnapshotId)); + txn.addCommitData(commitBytes( + dataFileItem("s3://b/db1/t1/region=us/replacement.parquet", 2L, 1024L, + Collections.singletonList("us")))); + + Assertions.assertThrows(DorisConnectorException.class, txn::commit); + } + @Test public void overwriteEmptyUnpartitionedClearsTable() { InMemoryCatalog catalog = freshCatalog(); @@ -664,6 +747,24 @@ public void overwriteEmptyUnpartitionedBranchClearsOnlyBranchFiles() { } } + @Test + public void overwriteEmptyUnpartitionedRejectsDataCommittedBetweenScanAndBegin() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t1"); + Table table = catalog.createTable(id, SCHEMA, PartitionSpec.unpartitioned(), + props("write.format.default", "parquet")); + table.newAppend().appendFile(dataFile(table.spec(), "s3://b/db1/t1/seed.parquet", 1L)).commit(); + long readSnapshotId = table.currentSnapshot().snapshotId(); + table.newAppend().appendFile(dataFile(table.spec(), + "s3://b/db1/t1/between-scan-and-begin.parquet", 1L)).commit(); + + IcebergConnectorTransaction txn = txnFor( + opsReturning(catalog.loadTable(id)), new RecordingConnectorContext()); + txn.beginWrite(SESSION, "db1", "t1", overwriteCtxPinned(readSnapshotId)); + + Assertions.assertThrows(DorisConnectorException.class, txn::commit); + } + @Test public void overwriteStaticPartitionUsesRowFilter() { InMemoryCatalog catalog = freshCatalog(); @@ -727,6 +828,30 @@ public void overwriteStaticRejectsConcurrentDataInTargetPartition() { "static overwrite must reject concurrent data matching its target partition filter"); } + @Test + public void overwriteStaticRejectsDataCommittedBetweenScanAndBegin() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t1"); + PartitionSpec spec = PartitionSpec.builderFor(PART_SCHEMA).identity("region").build(); + Table table = catalog.createTable(id, PART_SCHEMA, spec, props("write.format.default", "parquet")); + table.newAppend().appendFile(partitionedDataFile(spec, + "s3://b/db1/t1/region=us/seed.parquet", 1L, "region=us")).commit(); + long readSnapshotId = table.currentSnapshot().snapshotId(); + table.newAppend().appendFile(partitionedDataFile(spec, + "s3://b/db1/t1/region=us/between-scan-and-begin.parquet", 1L, "region=us")).commit(); + + IcebergConnectorTransaction txn = txnFor( + opsReturning(catalog.loadTable(id)), new RecordingConnectorContext()); + txn.beginWrite(SESSION, "db1", "t1", + new IcebergWriteContext(WriteOperation.OVERWRITE, true, + Collections.singletonMap("region", "us"), Optional.empty(), readSnapshotId)); + txn.addCommitData(commitBytes( + dataFileItem("s3://b/db1/t1/region=us/replacement.parquet", 2L, 1024L, + Collections.singletonList("us")))); + + Assertions.assertThrows(DorisConnectorException.class, txn::commit); + } + @Test public void deleteWritesRowDeltaDeleteFiles() { InMemoryCatalog catalog = freshCatalog(); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableHandleTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableHandleTest.java index f5e928afaaa50c..4492d097b96dd8 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableHandleTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableHandleTest.java @@ -57,6 +57,17 @@ public void withSnapshotPinsByIdAndCarriesSchemaId() { Assertions.assertEquals("t1", pinned.getTableName()); } + @Test + public void explicitEmptySnapshotIsDistinctFromNoPin() { + IcebergTableHandle bare = new IcebergTableHandle("db1", "t1"); + IcebergTableHandle empty = bare.withSnapshot(-1L, null, -1L); + + Assertions.assertFalse(empty.hasSnapshotPin()); + Assertions.assertFalse(empty.hasSnapshotSelection()); + Assertions.assertTrue(empty.isSnapshotResolved()); + Assertions.assertNotEquals(bare, empty); + } + @Test public void withSnapshotPinsByRef() { IcebergTableHandle pinned = new IcebergTableHandle("db1", "t1").withSnapshot(7L, "b1", 2L); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java index 0e9ed5bc849371..05d3dc1ae4e048 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java @@ -142,10 +142,85 @@ public void readsEachSharedDataManifestOnlyOnce(@TempDir Path temp) throws Excep recordingFileIO.openCounts.getOrDefault(path, 0), path)); } + @Test + public void scansConfiguredDataRootOutsideTableLocationByDefault(@TempDir Path temp) throws Exception { + Path dataRoot = temp.resolve("owned-data"); + Table table = createTable(temp.resolve("metadata"), + Collections.singletonMap(TableProperties.WRITE_DATA_LOCATION, + dataRoot.toUri().toString())); + Path orphan = createOldFile(dataRoot.resolve("orphan.parquet")); + IcebergRemoveOrphanFilesAction action = action( + System.currentTimeMillis() - MIN_RETENTION_MS, false); + action.validate(); + + ConnectorProcedureResult result = action.execute(table, ActionTestTables.session("UTC")); + + Assertions.assertEquals("1", result.getRows().get(0).get(0)); + Assertions.assertEquals("1", result.getRows().get(0).get(1)); + Assertions.assertFalse(Files.exists(orphan)); + } + + @Test + public void allowsExplicitConfiguredDataRootButRejectsArbitraryRoot(@TempDir Path temp) throws Exception { + Path dataRoot = temp.resolve("owned-data"); + Table table = createTable(temp.resolve("metadata"), + Collections.singletonMap(TableProperties.WRITE_DATA_LOCATION, + dataRoot.toUri().toString())); + Path orphan = createOldFile(dataRoot.resolve("orphan.parquet")); + + IcebergRemoveOrphanFilesAction configured = action( + System.currentTimeMillis() - MIN_RETENTION_MS, false, dataRoot.toUri().toString()); + configured.validate(); + configured.execute(table, ActionTestTables.session("UTC")); + Assertions.assertFalse(Files.exists(orphan)); + + IcebergRemoveOrphanFilesAction arbitrary = action( + System.currentTimeMillis() - MIN_RETENTION_MS, false, + temp.resolve("unowned").toUri().toString()); + arbitrary.validate(); + Assertions.assertThrows(DorisConnectorException.class, + () -> arbitrary.execute(table, ActionTestTables.session("UTC"))); + } + + @Test + public void scansObjectStoreAndFolderStorageFallbackRoots(@TempDir Path temp) throws Exception { + Path objectRoot = temp.resolve("object-data"); + Map objectProperties = new HashMap<>(); + objectProperties.put(TableProperties.OBJECT_STORE_ENABLED, "true"); + objectProperties.put(TableProperties.OBJECT_STORE_PATH, objectRoot.toUri().toString()); + Table objectTable = createTable(temp.resolve("object-metadata"), objectProperties); + Path objectOrphan = createOldFile(objectRoot.resolve("orphan.parquet")); + + Path folderRoot = temp.resolve("folder-data"); + Table folderTable = createTable(temp.resolve("folder-metadata"), + Collections.singletonMap(TableProperties.WRITE_FOLDER_STORAGE_LOCATION, + folderRoot.toUri().toString())); + Path folderOrphan = createOldFile(folderRoot.resolve("orphan.parquet")); + + IcebergRemoveOrphanFilesAction objectAction = action( + System.currentTimeMillis() - MIN_RETENTION_MS, false); + objectAction.validate(); + objectAction.execute(objectTable, ActionTestTables.session("UTC")); + IcebergRemoveOrphanFilesAction folderAction = action( + System.currentTimeMillis() - MIN_RETENTION_MS, false); + folderAction.validate(); + folderAction.execute(folderTable, ActionTestTables.session("UTC")); + + Assertions.assertFalse(Files.exists(objectOrphan)); + Assertions.assertFalse(Files.exists(folderOrphan)); + } + private static IcebergRemoveOrphanFilesAction action(long olderThan, boolean dryRun) { + return action(olderThan, dryRun, null); + } + + private static IcebergRemoveOrphanFilesAction action(long olderThan, boolean dryRun, String location) { Map properties = new HashMap<>(); properties.put(IcebergRemoveOrphanFilesAction.OLDER_THAN, String.valueOf(olderThan)); properties.put(IcebergRemoveOrphanFilesAction.DRY_RUN, String.valueOf(dryRun)); + if (location != null) { + properties.put(IcebergRemoveOrphanFilesAction.LOCATION, location); + } return new IcebergRemoveOrphanFilesAction(properties, Collections.emptyList(), null); } diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java index e97a8284136b05..197b6c64a7851e 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java @@ -250,7 +250,9 @@ public void completeMultipartUpload(String remotePath, String uploadId, List sorted = new ArrayList<>(parts); sorted.sort((a, b) -> Integer.compare(a.partNumber(), b.partNumber())); for (UploadPartResult part : sorted) { - blockIds.add(toBlockId(part.partNumber())); + // New BEs carry their exact UUID-prefixed ID; the fallback completes uploads from older BEs. + blockIds.add(part.etag() == null || part.etag().isEmpty() + ? toBlockId(part.partNumber()) : part.etag()); } blockBlobClient.commitBlockList(blockIds); } catch (BlobStorageException e) { diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java index eca48e0c4bf867..b6a0bc6f973abb 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java @@ -17,6 +17,7 @@ package org.apache.doris.filesystem.azure; +import org.apache.doris.filesystem.UploadPartResult; import org.apache.doris.filesystem.spi.RemoteObjects; import com.azure.storage.blob.BlobClient; @@ -378,6 +379,48 @@ void deleteObjectsByKeys_attachesPerKeyExceptionsAsSuppressed() throws Exception // F20 — abortMultipartUpload safe-noop / commit-empty behaviour // ------------------------------------------------------------------ + @Test + void completeMultipartUpload_usesExactBlockIdsReportedByBe() throws Exception { + com.azure.storage.blob.specialized.BlockBlobClient blockClient = + Mockito.mock(com.azure.storage.blob.specialized.BlockBlobClient.class); + BlobClient blobClient = Mockito.mock(BlobClient.class); + Mockito.when(blobClient.getBlockBlobClient()).thenReturn(blockClient); + BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); + Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); + BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); + Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); + TestableAzureObjStorage storage = new TestableAzureObjStorage(buildBasicProps(), serviceClient); + + String firstBlockId = "YmUtZ2VuZXJhdGVkLXVwbG9hZC1pZDowMDAwMDAwMDAx"; + String secondBlockId = "YmUtZ2VuZXJhdGVkLXVwbG9hZC1pZDowMDAwMDAwMDAy"; + storage.completeMultipartUpload( + "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", + "be-generated-upload-id", + Arrays.asList(new UploadPartResult(2, secondBlockId), + new UploadPartResult(1, firstBlockId))); + + Mockito.verify(blockClient).commitBlockList(Arrays.asList(firstBlockId, secondBlockId)); + } + + @Test + void completeMultipartUpload_fallsBackForOlderBeWithoutBlockIds() throws Exception { + com.azure.storage.blob.specialized.BlockBlobClient blockClient = + Mockito.mock(com.azure.storage.blob.specialized.BlockBlobClient.class); + BlobClient blobClient = Mockito.mock(BlobClient.class); + Mockito.when(blobClient.getBlockBlobClient()).thenReturn(blockClient); + BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); + Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); + BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); + Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); + TestableAzureObjStorage storage = new TestableAzureObjStorage(buildBasicProps(), serviceClient); + + storage.completeMultipartUpload( + "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", + "legacy-upload-id", Collections.singletonList(new UploadPartResult(1, ""))); + + Mockito.verify(blockClient).commitBlockList(Collections.singletonList("AQAAAA==")); + } + @Test void abortMultipartUpload_safeNoopWhenCommittedBlobExists() throws Exception { com.azure.storage.blob.models.BlobProperties props = From 87b32359b0e0f38db4fec96a6bd2bcd20977405f Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 2 Aug 2026 14:25:57 +0800 Subject: [PATCH 05/29] [fix](iceberg) Address multipart and snapshot review feedback --- .../operator/iceberg_sorter_reserve_memory.h | 9 +++ .../spill_iceberg_table_sink_operator.cpp | 3 +- be/src/io/fs/azure_obj_storage_client.cpp | 74 ++++++++++++------- be/src/io/fs/azure_obj_storage_client.h | 5 +- be/src/io/fs/obj_storage_client.h | 1 + be/src/io/fs/s3_file_writer.cpp | 1 + ...spill_iceberg_table_sink_operator_test.cpp | 6 ++ .../io/fs/azure_obj_storage_client_test.cpp | 36 ++++++++- .../iceberg/IcebergConnectorTransaction.java | 14 ++-- .../IcebergRemoveOrphanFilesAction.java | 59 +++++++++++---- .../IcebergConnectorTransactionTest.java | 37 +++++++++- .../iceberg/IcebergScanPlanProviderTest.java | 17 +++++ .../IcebergRemoveOrphanFilesActionTest.java | 18 +++++ .../filesystem/azure/AzureObjStorage.java | 67 ++++++++--------- .../azure/AzureObjStorageExtensionTest.java | 66 +++++++++-------- 15 files changed, 285 insertions(+), 128 deletions(-) diff --git a/be/src/exec/operator/iceberg_sorter_reserve_memory.h b/be/src/exec/operator/iceberg_sorter_reserve_memory.h index 1ad1454b4af5f3..835343cfec806b 100644 --- a/be/src/exec/operator/iceberg_sorter_reserve_memory.h +++ b/be/src/exec/operator/iceberg_sorter_reserve_memory.h @@ -42,4 +42,13 @@ inline size_t bounded_iceberg_reserve_size( retained_growth; } +inline size_t iceberg_reserve_size( + const std::vector& per_partition_reservations, + size_t incoming_block_bytes) { + size_t sorter_reserve = bounded_iceberg_reserve_size(per_partition_reservations); + // The incoming block creates cold partition writers before they can appear in the published snapshot. + return std::min(std::numeric_limits::max() - sorter_reserve, incoming_block_bytes) + + sorter_reserve; +} + } // namespace doris diff --git a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp index 5d0d1cb00a7916..1fde8f882b280e 100644 --- a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp +++ b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp @@ -67,7 +67,8 @@ size_t SpillIcebergTableSinkLocalState::get_reserve_mem_size(RuntimeState* state } } // Column growth remains in every touched sorter, while sorting workspace is reused by serial dispatch. - return bounded_iceberg_reserve_size(per_partition_reservations); + return iceberg_reserve_size(per_partition_reservations, + eos ? 0 : state->minimum_operator_memory_required_bytes()); } size_t SpillIcebergTableSinkLocalState::get_revocable_mem_size(RuntimeState* state) const { diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp index daa215e3c58d9e..a870193deeefe7 100644 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ b/be/src/io/fs/azure_obj_storage_client.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -74,6 +75,16 @@ std::string azure_block_id(const doris::io::ObjectStoragePathOptions& opts, int return Aws::Utils::HashingUtils::Base64Encode(bytes); } +std::string legacy_azure_block_id(int part_num) { + std::array bytes { + static_cast(part_num & 0xff), + static_cast((part_num >> 8) & 0xff), + static_cast((part_num >> 16) & 0xff), + static_cast((part_num >> 24) & 0xff)}; + Aws::Utils::ByteBuffer buffer(bytes.data(), bytes.size()); + return Aws::Utils::HashingUtils::Base64Encode(buffer); +} + // Rate limiting is applied by RateLimitedObjStorageClient, the decorator that // S3ClientFactory wraps around this client when the bucket is subject to limiting. @@ -83,6 +94,10 @@ constexpr char BlobNotFound[] = "BlobNotFound"; namespace doris::io { +std::string azure_multipart_temp_key(std::string_view key, std::string_view upload_id) { + return fmt::format("{}.__doris_multipart/{}", key, upload_id); +} + // As Azure's doc said, the batch size is 256 // You can find out the num in https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id // > Each batch request supports a maximum of 256 subrequests. @@ -222,7 +237,8 @@ ObjectStorageResponse AzureObjStorageClient::put_object(const ObjectStoragePathO ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStoragePathOptions& opts, std::string_view stream, int part_num) { - auto client = _client->GetBlockBlobClient(opts.key); + DCHECK(opts.upload_id.has_value()); + auto client = _client->GetBlockBlobClient(azure_multipart_temp_key(opts.key, *opts.upload_id)); std::string block_id = azure_block_id(opts, part_num); auto resp = do_azure_client_call( [&]() { @@ -231,6 +247,13 @@ ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStora // The blockId must be base64 encoded SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); client.StageBlock(block_id, memory_body); + if (opts.deferred_completion) { + // During rolling upgrades an old FE still commits deterministic IDs on the final blob. + Azure::Core::IO::MemoryBodyStream legacy_body( + reinterpret_cast(stream.data()), stream.size()); + _client->GetBlockBlobClient(opts.key).StageBlock( + legacy_azure_block_id(part_num), legacy_body); + } }, opts, _tls_debug_context); return ObjectStorageUploadResponse { @@ -243,46 +266,43 @@ ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStora ObjectStorageResponse AzureObjStorageClient::complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) { - auto client = _client->GetBlockBlobClient(opts.key); + DCHECK(opts.upload_id.has_value()); + auto temp_client = + _client->GetBlockBlobClient(azure_multipart_temp_key(opts.key, *opts.upload_id)); + auto target_client = _client->GetBlockBlobClient(opts.key); std::vector string_block_ids; std::ranges::transform( completed_parts, std::back_inserter(string_block_ids), [&opts](const ObjectCompleteMultiPart& i) { return azure_block_id(opts, i.part_num); }); - return do_azure_client_call( + auto response = do_azure_client_call( [&]() { SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); - client.CommitBlockList(string_block_ids); + // Per-upload temporary blobs keep Azure's blob-wide staged-block namespace isolated. + temp_client.CommitBlockList(string_block_ids); + auto copy = target_client.StartCopyFromUri(temp_client.GetUrl()); + copy.PollUntilDone(std::chrono::milliseconds(100)); }, opts, _tls_debug_context); + if (response.status.code != ErrorCode::OK) { + return response; + } + auto cleanup = do_azure_client_call([&]() { temp_client.Delete(); }, opts, _tls_debug_context); + if (cleanup.status.code != ErrorCode::OK && + cleanup.http_code != static_cast(Azure::Core::Http::HttpStatusCode::NotFound)) { + LOG(WARNING) << "Azure multipart temporary blob cleanup failed after publication"; + } + // Publication already succeeded; a cleanup failure must not turn a retry into a conflicting overwrite. + return response; } ObjectStorageResponse AzureObjStorageClient::abort_multipart_upload( const ObjectStoragePathOptions& opts) { - auto client = _client->GetBlockBlobClient(opts.key); + DCHECK(opts.upload_id.has_value()); + auto client = _client->GetBlockBlobClient(azure_multipart_temp_key(opts.key, *opts.upload_id)); auto response = do_azure_client_call( [&]() { - GetBlockListOptions get_options; - get_options.ListType = Models::BlockListType::All; - auto block_list = client.GetBlockList(get_options); - const bool has_committed_blob = azure_block_list_has_committed_blob( - block_list.Value.CommittedBlocks.size(), block_list.Value.ETag.HasValue()); - if (!has_committed_blob) { - // Uncommitted blocks are invisible and expire without deleting a racing commit. - return; - } - if (block_list.Value.CommittedBlocks.empty()) { - // Azure cannot selectively discard staged blocks without replacing Put Blob content. - return; - } - std::vector committed_ids; - committed_ids.reserve(block_list.Value.CommittedBlocks.size()); - std::ranges::transform(block_list.Value.CommittedBlocks, - std::back_inserter(committed_ids), - [](const Models::BlobBlock& block) { return block.Name; }); - CommitBlockListOptions commit_options; - commit_options.AccessConditions.IfMatch = block_list.Value.ETag; - // Recommitting only the old IDs discards this writer's unique staged blocks. - client.CommitBlockList(committed_ids, commit_options); + // Never recommit the final blob: a legacy staged block may shadow a committed block ID. + client.Delete(); }, opts, _tls_debug_context); // Azure creates no server-side object until the first block is staged, so absence is clean. diff --git a/be/src/io/fs/azure_obj_storage_client.h b/be/src/io/fs/azure_obj_storage_client.h index fe3cf01417aa3b..62968557490ab0 100644 --- a/be/src/io/fs/azure_obj_storage_client.h +++ b/be/src/io/fs/azure_obj_storage_client.h @@ -33,10 +33,7 @@ class ObjClientHolder; bool is_azure_tls_ca_error_message(std::string_view message); std::string build_azure_tls_debug_suffix(std::string_view error_message, std::string_view tls_debug_context); -inline bool azure_block_list_has_committed_blob(size_t committed_block_count, bool has_etag) { - // Put Blob creates committed content without block IDs; its ETag is the only safe discriminator. - return committed_block_count > 0 || has_etag; -} +std::string azure_multipart_temp_key(std::string_view key, std::string_view upload_id); class AzureObjStorageClient final : public ObjStorageClient { public: diff --git a/be/src/io/fs/obj_storage_client.h b/be/src/io/fs/obj_storage_client.h index db326a931719f9..6ed2803c53e13b 100644 --- a/be/src/io/fs/obj_storage_client.h +++ b/be/src/io/fs/obj_storage_client.h @@ -45,6 +45,7 @@ struct ObjectStoragePathOptions { std::string key = std::string(); // blob name in azure std::string prefix = std::string(); // for batch delete and recursive delete std::optional upload_id = std::nullopt; // provider-specific upload token + bool deferred_completion = false; // another process commits the uploaded parts }; struct ObjectCompleteMultiPart { diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index 43663afa6dfdf8..4d4f5d751cdf67 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -66,6 +66,7 @@ S3FileWriter::S3FileWriter(std::shared_ptr client, std::string .key = std::move(key)}), _used_by_s3_committer(opts ? opts->used_by_s3_committer : false), _obj_client(std::move(client)) { + _obj_storage_path_opts.deferred_completion = _used_by_s3_committer; s3_file_writer_total << 1; s3_file_being_written << 1; Aws::Http::SetCompliantRfc3986Encoding(true); diff --git a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp index 5d1fa473205f0b..62df6d4d210117 100644 --- a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp +++ b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp @@ -36,4 +36,10 @@ TEST(SpillIcebergTableSinkOperatorTest, AccumulatesRetainedGrowthAcrossTouchedPa EXPECT_EQ(14 * 1024 * 1024, bounded_iceberg_reserve_size(per_partition_reservations)); } +TEST(SpillIcebergTableSinkOperatorTest, ReservesIncomingBlockBeforeAnyPartitionWriterExists) { + std::vector no_published_sorters; + + EXPECT_EQ(6 * 1024 * 1024, iceberg_reserve_size(no_published_sorters, 6 * 1024 * 1024)); +} + } // namespace doris diff --git a/be/test/io/fs/azure_obj_storage_client_test.cpp b/be/test/io/fs/azure_obj_storage_client_test.cpp index 88e81ea7fe6ed0..83671a9d997d59 100644 --- a/be/test/io/fs/azure_obj_storage_client_test.cpp +++ b/be/test/io/fs/azure_obj_storage_client_test.cpp @@ -34,10 +34,11 @@ namespace doris { -TEST(AzureObjStorageClientAbortHelperTest, preserves_committed_put_blob_without_block_list) { - EXPECT_FALSE(io::azure_block_list_has_committed_blob(0, false)); - EXPECT_TRUE(io::azure_block_list_has_committed_blob(0, true)); - EXPECT_TRUE(io::azure_block_list_has_committed_blob(1, false)); +TEST(AzureObjStorageClientMultipartHelperTest, isolates_uploads_with_temporary_blob_keys) { + EXPECT_EQ("table/file.parquet.__doris_multipart/upload-a", + io::azure_multipart_temp_key("table/file.parquet", "upload-a")); + EXPECT_NE(io::azure_multipart_temp_key("table/file.parquet", "upload-a"), + io::azure_multipart_temp_key("table/file.parquet", "upload-b")); } #ifdef USE_AZURE @@ -216,6 +217,33 @@ TEST_F(AzureObjStorageClientTest, abort_multipart_upload_preserves_existing_put_ EXPECT_EQ(AzureObjStorageClientTest::obj_storage_client->delete_object(opts).status.code, ErrorCode::OK); } + +TEST_F(AzureObjStorageClientTest, concurrent_multipart_uploads_do_not_share_staged_blocks) { + io::ObjectStoragePathOptions first {.key = "AzureObjStorageClientTest/concurrent_multipart"}; + io::ObjectStoragePathOptions second = first; + auto first_create = obj_storage_client->create_multipart_upload(first); + auto second_create = obj_storage_client->create_multipart_upload(second); + ASSERT_TRUE(first_create.upload_id.has_value()); + ASSERT_TRUE(second_create.upload_id.has_value()); + first.upload_id = first_create.upload_id; + second.upload_id = second_create.upload_id; + + ASSERT_EQ(obj_storage_client->upload_part(first, "first", 1).resp.status.code, ErrorCode::OK); + ASSERT_EQ(obj_storage_client->upload_part(second, "second", 1).resp.status.code, ErrorCode::OK); + ASSERT_EQ(obj_storage_client->complete_multipart_upload(first, {{.part_num = 1}}).status.code, + ErrorCode::OK); + ASSERT_EQ(obj_storage_client->complete_multipart_upload(second, {{.part_num = 1}}).status.code, + ErrorCode::OK); + + std::array contents {}; + size_t size_return = 0; + ASSERT_EQ(obj_storage_client + ->get_object(second, contents.data(), 0, contents.size(), &size_return) + .status.code, + ErrorCode::OK); + EXPECT_EQ(std::string_view(contents.data(), size_return), "second"); + EXPECT_EQ(obj_storage_client->delete_object(second).status.code, ErrorCode::OK); +} #else class AzureObjStorageClientTest : public testing::Test { diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java index 9306f29bbcfd18..b4c2ce5111ba64 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java @@ -413,14 +413,14 @@ private void applyBeginGuards(IcebergWriteContext ctx, String tableName) { // scan used, S_read), threaded onto the write handle and carried on the ctx. The commit-time // removeDeletes (option D) re-derives from baseSnapshotId, and BE unions the scan-time (S_read) // old deletes into the new DV — anchoring both at S_read keeps supply and remove on one snapshot - // (no resurrection under a concurrent commit in the read->begin-write window). + // (no resurrection under a concurrent commit in the read->begin-write window). An explicitly pinned + // -1 is the empty-table generation and must remain an OCC fence; only an unpinned caller may fall + // back to the begin-time current snapshot. long pinnedReadSnapshot = ctx.getReadSnapshotId(); - if (ctx.isReadSnapshotResolved()) { - // An explicitly empty read stays null so validation covers a concurrent first append. - this.baseSnapshotId = pinnedReadSnapshot >= 0 ? Long.valueOf(pinnedReadSnapshot) : null; - } else { - this.baseSnapshotId = getSnapshotIdIfPresent(table); - } + // Keep both ternary arms boxed (Long): getSnapshotIdIfPresent returns null for an empty table + // (no snapshot), and a primitive arm would force-unbox that null into an NPE. + this.baseSnapshotId = ctx.isReadSnapshotResolved() + ? Long.valueOf(pinnedReadSnapshot) : getSnapshotIdIfPresent(table); if (table instanceof HasTableOperations) { int formatVersion = ((HasTableOperations) table).operations().current().formatVersion(); if (formatVersion < 2) { diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java index 37226b21fde047..9c4dd71f61c5a4 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java @@ -44,7 +44,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.HashSet; -import java.util.LinkedHashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -132,11 +132,7 @@ protected List executeAction(Table table, ConnectorSession session) { } private List resolveScanLocations(Table table) { - String tableRoot = normalizeLocation(table.location()); - String dataRoot = normalizeLocation(resolveDataLocation(table, tableRoot)); - Set ownedRoots = new LinkedHashSet<>(); - ownedRoots.add(tableRoot); - ownedRoots.add(dataRoot); + List ownedRoots = resolveOwnedRoots(table.location(), table.properties()); String requested = namedArguments.getString(LOCATION); if (requested != null) { @@ -149,18 +145,45 @@ private List resolveScanLocations(Table table) { return Lists.newArrayList(normalized); } - List roots = new ArrayList<>(); - for (String candidate : ownedRoots) { - // Avoid listing a nested default data directory twice when the table root already covers it. - if (ownedRoots.stream().noneMatch(other -> !other.equals(candidate) && isWithin(candidate, other))) { - roots.add(candidate); + return minimalOwnedRoots(ownedRoots); + } + + static List resolveOwnedRoots(String tableLocation, Map properties) { + String tableRoot = normalizeLocation(tableLocation); + String dataRoot = normalizeLocation(resolveDataLocation(properties, tableRoot)); + List configuredRoots = new ArrayList<>(); + configuredRoots.add(tableRoot); + configuredRoots.add(dataRoot); + // Iceberg may place metadata outside both table and data roots, so it remains an independent owned root. + String metadataRoot = nonEmpty(properties.get(TableProperties.WRITE_METADATA_LOCATION)); + if (metadataRoot != null) { + configuredRoots.add(normalizeLocation(metadataRoot)); + } + return canonicalOwnedRoots(configuredRoots); + } + + static List minimalOwnedRoots(List roots) { + List canonicalRoots = canonicalOwnedRoots(roots); + List minimal = new ArrayList<>(); + for (String candidate : canonicalRoots) { + // Canonically equal aliases are deduplicated first, so only strict containment removes a root. + if (canonicalRoots.stream().noneMatch(other -> !sameFileIdentity(other, candidate) + && isWithinLocation(candidate, other))) { + minimal.add(candidate); } } - return roots; + return minimal; } - private String resolveDataLocation(Table table, String tableRoot) { - Map properties = table.properties(); + private static List canonicalOwnedRoots(List roots) { + Map byIdentity = new LinkedHashMap<>(); + for (String root : roots) { + byIdentity.putIfAbsent(FileIdentity.of(root), root); + } + return new ArrayList<>(byIdentity.values()); + } + + private static String resolveDataLocation(Map properties, String tableRoot) { String dataLocation = nonEmpty(properties.get(TableProperties.WRITE_DATA_LOCATION)); if (dataLocation == null && Boolean.parseBoolean(properties.get(TableProperties.OBJECT_STORE_ENABLED))) { dataLocation = nonEmpty(properties.get(TableProperties.OBJECT_STORE_PATH)); @@ -171,11 +194,15 @@ private String resolveDataLocation(Table table, String tableRoot) { return dataLocation == null ? tableRoot + "/data" : dataLocation; } - private String nonEmpty(String location) { + private static String nonEmpty(String location) { return location == null || location.isEmpty() ? null : location; } private boolean isWithin(String location, String root) { + return isWithinLocation(location, root); + } + + private static boolean isWithinLocation(String location, String root) { FileIdentity child = FileIdentity.of(location); FileIdentity parent = FileIdentity.of(root); String pathPrefix = parent.path.endsWith("/") ? parent.path : parent.path + "/"; @@ -300,7 +327,7 @@ private ReachableIndex(Set locations) { } } - private String normalizeLocation(String location) { + private static String normalizeLocation(String location) { String normalized = URI.create(location).normalize().toString(); return normalized.length() > 1 && normalized.endsWith("/") ? normalized.substring(0, normalized.length() - 1) : normalized; diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java index 0df759d7a215a9..0219b12416abbd 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java @@ -164,12 +164,12 @@ private static IcebergWriteContext rewriteCtx() { private static IcebergWriteContext deleteCtxPinned(long readSnapshotId) { return new IcebergWriteContext( - WriteOperation.DELETE, false, Collections.emptyMap(), Optional.empty(), readSnapshotId); + WriteOperation.DELETE, false, Collections.emptyMap(), Optional.empty(), readSnapshotId, true); } private static IcebergWriteContext mergeCtxPinned(long readSnapshotId) { return new IcebergWriteContext( - WriteOperation.MERGE, false, Collections.emptyMap(), Optional.empty(), readSnapshotId); + WriteOperation.MERGE, false, Collections.emptyMap(), Optional.empty(), readSnapshotId, true); } /** @@ -522,6 +522,22 @@ public void beginMergeHonorsPinnedReadSnapshotOverCurrent() { "MERGE must anchor baseSnapshotId at the pinned read snapshot, not the current snapshot"); } + @Test + public void beginDeletePreservesExplicitEmptyReadSnapshot() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t1"); + Table table = catalog.createTable(id, SCHEMA, PartitionSpec.unpartitioned(), props("format-version", "2")); + table.newAppend().appendFile( + dataFile(table.spec(), "s3://b/db1/t1/concurrent.parquet", 1L)).commit(); + + IcebergConnectorTransaction txn = txnFor( + opsReturning(catalog.loadTable(id)), new RecordingConnectorContext()); + txn.beginWrite(SESSION, "db1", "t1", deleteCtxPinned(-1L)); + + Assertions.assertEquals(Long.valueOf(-1L), txn.getBaseSnapshotId(), + "an explicit empty read is an OCC fence, not an absent pin"); + } + @Test public void beginInsertDoesNotCaptureBaseSnapshotId() { InMemoryCatalog catalog = freshCatalog(); @@ -1046,6 +1062,23 @@ public void mergeFromResolvedEmptySnapshotRejectsConcurrentFirstAppend() throws committedFiles.get(0).path().toString()); } + @Test + public void deleteFromExplicitEmptySnapshotDetectsFirstConcurrentCommit() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t1"); + Table table = catalog.createTable(id, SCHEMA, PartitionSpec.unpartitioned(), props("format-version", "2")); + IcebergConnectorTransaction txn = txnFor(opsReturning(table), new RecordingConnectorContext()); + txn.beginWrite(SESSION, "db1", "t1", deleteCtxPinned(-1L)); + + catalog.loadTable(id).newAppend().appendFile( + dataFile(table.spec(), "s3://b/db1/t1/concurrent.parquet", 7L)).commit(); + txn.addCommitData(commitBytes(positionDeleteItem( + "s3://b/db1/t1/del.parquet", 1L, "s3://b/db1/t1/concurrent.parquet"))); + + Assertions.assertThrows(DorisConnectorException.class, txn::commit, + "validateFromSnapshot(-1) must reject the first snapshot committed after an empty read"); + } + @Test public void deletePassesValidationSuiteWhenNoConcurrentChange() { InMemoryCatalog catalog = freshCatalog(); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java index 26fa913062afea..d2d963e6829d9e 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java @@ -319,6 +319,23 @@ public void planScanEnumeratesOneRangePerDataFile() { Assertions.assertEquals(2048L, ranges.get(1).getLength()); } + @Test + public void explicitEmptySnapshotDoesNotDriftToFirstConcurrentCommit() { + Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); + IcebergTableHandle emptyPin = new IcebergTableHandle("db1", "t1") + .withSnapshot(-1L, null, -1L); + table.newAppend().appendFile( + dataFile(table.spec(), "s3://b/db/t1/concurrent.parquet", 1024, null, null)).commit(); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider( + Collections.emptyMap(), opsReturning(table)); + + List ranges = provider.planScan(null, + ConnectorScanRequest.builder(emptyPin, Collections.emptyList()).build()); + + Assertions.assertTrue(ranges.isEmpty(), + "an explicit empty MVCC pin must not be replaced by the table's first snapshot"); + } + @Test public void planScanRewriteFileScopeKeepsOnlyRawScopedFiles() { Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java index 05d3dc1ae4e048..f4aa7b126a52b4 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java @@ -103,6 +103,24 @@ public void treatsS3SchemeAliasesAsTheSameFile() { "s3n://bucket/path/data.parquet", "s3://BUCKET/path/data.parquet")); } + @Test + public void canonicalAliasRootsCollapseToOneOwnedScanRoot() { + Assertions.assertEquals(Collections.singletonList("s3a://bucket/table"), + IcebergRemoveOrphanFilesAction.minimalOwnedRoots( + java.util.List.of("s3a://bucket/table", "s3://BUCKET/table"))); + } + + @Test + public void configuredDataAndMetadataLocationsAreSeparateOwnedRoots() { + Map properties = new HashMap<>(); + properties.put(TableProperties.WRITE_DATA_LOCATION, "s3://bucket/data-root"); + properties.put(TableProperties.WRITE_METADATA_LOCATION, "s3://bucket/metadata-root"); + + Assertions.assertEquals(java.util.List.of( + "s3://bucket/table-root", "s3://bucket/data-root", "s3://bucket/metadata-root"), + IcebergRemoveOrphanFilesAction.resolveOwnedRoots("s3://bucket/table-root", properties)); + } + @Test public void rejectsUnresolvedPrefixMismatches() { Assertions.assertThrows(DorisConnectorException.class, diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java index 197b6c64a7851e..1772f5c72b8dca 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java @@ -56,6 +56,7 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.UUID; /** * Azure Blob Storage implementation of {@link ObjStorage}. @@ -219,8 +220,7 @@ public void copyObject(String srcPath, String dstPath) throws IOException { @Override public String initiateMultipartUpload(String remotePath) throws IOException { // Azure block blobs don't have an explicit "initiate" API. - // Return the path itself as the upload ID; block IDs are derived from part numbers. - return remotePath; + return UUID.randomUUID().toString(); } @Override @@ -229,7 +229,7 @@ public UploadPartResult uploadPart(String remotePath, String uploadId, int partN try { AzureUri uri = AzureUri.parse(remotePath); BlockBlobClient blockBlobClient = getClient().getBlobContainerClient(uri.container()) - .getBlobClient(uri.key()).getBlockBlobClient(); + .getBlobClient(multipartTempKey(uri.key(), uploadId)).getBlockBlobClient(); String blockId = toBlockId(partNum); blockBlobClient.stageBlock(blockId, body.content(), body.contentLength()); return new UploadPartResult(partNum, blockId); @@ -244,17 +244,31 @@ public void completeMultipartUpload(String remotePath, String uploadId, List parts) throws IOException { try { AzureUri uri = AzureUri.parse(remotePath); - BlockBlobClient blockBlobClient = getClient().getBlobContainerClient(uri.container()) - .getBlobClient(uri.key()).getBlockBlobClient(); + BlobContainerClient containerClient = getClient().getBlobContainerClient(uri.container()); List blockIds = new ArrayList<>(); List sorted = new ArrayList<>(parts); sorted.sort((a, b) -> Integer.compare(a.partNumber(), b.partNumber())); + boolean exactBlockIds = !sorted.isEmpty() && sorted.stream() + .allMatch(part -> part.etag() != null && !part.etag().isEmpty()); for (UploadPartResult part : sorted) { - // New BEs carry their exact UUID-prefixed ID; the fallback completes uploads from older BEs. - blockIds.add(part.etag() == null || part.etag().isEmpty() - ? toBlockId(part.partNumber()) : part.etag()); + // A mixed-version upload must use one namespace consistently; new BEs also stage legacy IDs. + blockIds.add(exactBlockIds ? part.etag() : toBlockId(part.partNumber())); + } + String commitKey = exactBlockIds ? multipartTempKey(uri.key(), uploadId) : uri.key(); + BlobClient commitBlob = containerClient.getBlobClient(commitKey); + commitBlob.getBlockBlobClient().commitBlockList(blockIds); + if (exactBlockIds) { + BlobClient targetBlob = containerClient.getBlobClient(uri.key()); + // The temporary blob is the provider-visible writer fence; only a completed copy publishes it. + targetBlob.beginCopy(commitBlob.getBlobUrl(), null).waitForCompletion(); + try { + commitBlob.delete(); + } catch (BlobStorageException cleanupFailure) { + // Publication already succeeded; cleanup must not make a retry overwrite a newer writer. + LOG.warn("Azure multipart temporary blob cleanup failed after publication: {}", + cleanupFailure.getMessage()); + } } - blockBlobClient.commitBlockList(blockIds); } catch (BlobStorageException e) { throw new IOException("completeMultipartUpload failed for " + remotePath + ": " + e.getMessage(), e); @@ -263,35 +277,12 @@ public void completeMultipartUpload(String remotePath, String uploadId, @Override public void abortMultipartUpload(String remotePath, String uploadId) throws IOException { - // Azure has no native "abort multipart upload" API; the closest equivalent is to - // commit an empty block list (which atomically discards any uncommitted blocks - // for that blob) and then delete the resulting empty blob so no trace remains. - // - // SAFETY: commitBlockList(empty) overwrites whatever is at the target blob, so we - // MUST refuse to run when a committed blob already exists at this path — otherwise - // an abort call could destroy real user data. In that case the staged blocks are - // left to expire on their own (Azure GCs them after the service-side timeout). try { AzureUri uri = AzureUri.parse(remotePath); - BlockBlobClient blockBlobClient = getClient().getBlobContainerClient(uri.container()) - .getBlobClient(uri.key()).getBlockBlobClient(); - boolean committedBlobExists; - try { - blockBlobClient.getProperties(); - committedBlobExists = true; - } catch (BlobStorageException e) { - if (e.getStatusCode() != HTTP_NOT_FOUND) { - throw e; - } - committedBlobExists = false; - } - if (committedBlobExists) { - LOG.warn("abortMultipartUpload skipped for {}: a committed blob already exists; " - + "uncommitted blocks will expire automatically.", remotePath); - return; - } - blockBlobClient.commitBlockList(Collections.emptyList()); - blockBlobClient.delete(); + // Abort is scoped to this writer's temporary blob. Legacy final-blob blocks are left to expire, + // because recommitting their IDs can select a staged replacement for committed user data. + getClient().getBlobContainerClient(uri.container()) + .getBlobClient(multipartTempKey(uri.key(), uploadId)).deleteIfExists(); } catch (BlobStorageException e) { // Best-effort: log and swallow rather than mask the original failure that // triggered the abort path. Uncommitted blocks will be GC'd by the service. @@ -526,4 +517,8 @@ private static String toBlockId(int partNum) { byte[] bytes = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN).putInt(partNum).array(); return Base64.getEncoder().encodeToString(bytes); } + + static String multipartTempKey(String key, String uploadId) { + return key + ".__doris_multipart/" + uploadId; + } } diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java index b6a0bc6f973abb..4e687917e034c4 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java @@ -379,14 +379,30 @@ void deleteObjectsByKeys_attachesPerKeyExceptionsAsSuppressed() throws Exception // F20 — abortMultipartUpload safe-noop / commit-empty behaviour // ------------------------------------------------------------------ + @Test + void multipartTempKey_isolatesSameTargetWriters() { + Assertions.assertEquals("stage/blob.__doris_multipart/upload-a", + AzureObjStorage.multipartTempKey("stage/blob", "upload-a")); + Assertions.assertNotEquals( + AzureObjStorage.multipartTempKey("stage/blob", "upload-a"), + AzureObjStorage.multipartTempKey("stage/blob", "upload-b")); + } + @Test void completeMultipartUpload_usesExactBlockIdsReportedByBe() throws Exception { com.azure.storage.blob.specialized.BlockBlobClient blockClient = Mockito.mock(com.azure.storage.blob.specialized.BlockBlobClient.class); - BlobClient blobClient = Mockito.mock(BlobClient.class); - Mockito.when(blobClient.getBlockBlobClient()).thenReturn(blockClient); + BlobClient tempBlob = Mockito.mock(BlobClient.class); + Mockito.when(tempBlob.getBlockBlobClient()).thenReturn(blockClient); + Mockito.when(tempBlob.getBlobUrl()).thenReturn("https://account/container/temp"); + BlobClient targetBlob = Mockito.mock(BlobClient.class); + com.azure.core.util.polling.SyncPoller poller = + Mockito.mock(com.azure.core.util.polling.SyncPoller.class); + Mockito.when(targetBlob.beginCopy("https://account/container/temp", null)).thenReturn(poller); BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); - Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); + Mockito.when(containerClient.getBlobClient( + "stage/blob.__doris_multipart/be-generated-upload-id")).thenReturn(tempBlob); + Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(targetBlob); BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); TestableAzureObjStorage storage = new TestableAzureObjStorage(buildBasicProps(), serviceClient); @@ -400,6 +416,8 @@ void completeMultipartUpload_usesExactBlockIdsReportedByBe() throws Exception { new UploadPartResult(1, firstBlockId))); Mockito.verify(blockClient).commitBlockList(Arrays.asList(firstBlockId, secondBlockId)); + Mockito.verify(poller).waitForCompletion(); + Mockito.verify(tempBlob).delete(); } @Test @@ -422,47 +440,33 @@ void completeMultipartUpload_fallsBackForOlderBeWithoutBlockIds() throws Excepti } @Test - void abortMultipartUpload_safeNoopWhenCommittedBlobExists() throws Exception { - com.azure.storage.blob.models.BlobProperties props = - Mockito.mock(com.azure.storage.blob.models.BlobProperties.class); - Mockito.when(props.getBlobSize()).thenReturn(1024L); - + void completeMultipartUpload_usesLegacyNamespaceWhenAnyBlockIdIsMissing() throws Exception { com.azure.storage.blob.specialized.BlockBlobClient blockClient = Mockito.mock(com.azure.storage.blob.specialized.BlockBlobClient.class); - Mockito.when(blockClient.getProperties()).thenReturn(props); - BlobClient blobClient = Mockito.mock(BlobClient.class); Mockito.when(blobClient.getBlockBlobClient()).thenReturn(blockClient); - BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); - BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); - TestableAzureObjStorage storage = new TestableAzureObjStorage(buildBasicProps(), serviceClient); - storage.abortMultipartUpload( - "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", "uploadId"); + storage.completeMultipartUpload( + "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", + "mixed-upload-id", + Arrays.asList(new UploadPartResult(1, "exact-id"), new UploadPartResult(2, ""))); - // The committed blob must NOT be touched (no commitBlockList, no delete). - Mockito.verify(blockClient, Mockito.never()).commitBlockList(Mockito.anyList()); - Mockito.verify(blockClient, Mockito.never()).delete(); + Mockito.verify(blockClient).commitBlockList(Arrays.asList("AQAAAA==", "AgAAAA==")); + Mockito.verify(containerClient, Mockito.never()).getBlobClient( + "stage/blob.__doris_multipart/mixed-upload-id"); } @Test - void abortMultipartUpload_commitsEmptyAndDeletesWhenNoCommittedBlob() throws Exception { - com.azure.storage.blob.specialized.BlockBlobClient blockClient = - Mockito.mock(com.azure.storage.blob.specialized.BlockBlobClient.class); - BlobStorageException notFoundEx = Mockito.mock(BlobStorageException.class); - Mockito.when(notFoundEx.getStatusCode()).thenReturn(404); - Mockito.when(blockClient.getProperties()).thenThrow(notFoundEx); - - BlobClient blobClient = Mockito.mock(BlobClient.class); - Mockito.when(blobClient.getBlockBlobClient()).thenReturn(blockClient); - + void abortMultipartUpload_deletesOnlyWriterTemporaryBlob() throws Exception { + BlobClient tempBlob = Mockito.mock(BlobClient.class); BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); - Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); + Mockito.when(containerClient.getBlobClient( + "stage/blob.__doris_multipart/uploadId")).thenReturn(tempBlob); BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); @@ -472,8 +476,8 @@ void abortMultipartUpload_commitsEmptyAndDeletesWhenNoCommittedBlob() throws Exc storage.abortMultipartUpload( "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", "uploadId"); - Mockito.verify(blockClient).commitBlockList(Collections.emptyList()); - Mockito.verify(blockClient).delete(); + Mockito.verify(tempBlob).deleteIfExists(); + Mockito.verify(containerClient, Mockito.never()).getBlobClient("stage/blob"); } // ------------------------------------------------------------------ From 25f5e23e9b9985e7b23f2c58c3b186dca56369e2 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 2 Aug 2026 16:56:21 +0800 Subject: [PATCH 06/29] [fix](iceberg) Address remaining write safety feedback Issue Number: None Related PR: #66348 Problem Summary: The latest review found that async memory admission ended before queued writes, Azure temporary-copy publication and its compatibility mirror were unsafe for same-key writers, BE did not know FE smaller Thrift limit, branch writes could lose the exact MVCC pin, and orphan cleanup assumed ownership for layouts it could not prove. Transfer reservations across the async queue, publish Azure block lists atomically on the target, negotiate the receiver limit, bind branch-aware pins after the target branch is known, and use ownership-aware bounded orphan scans. Harden Iceberg async write admission, Azure multipart publication, and orphan-file cleanup safety. - Test: Unit Test - BE targeted unit tests: 7 passed. - FE targeted unit tests: 37 passed; all 59 reactor modules built successfully. - Azure-enabled production and test translation units compiled successfully. - Behavior changed: Yes. Unsafe orphan scan roots now require an explicit guarded override, and conflicting Azure same-key multipart writers fail closed instead of mixing data. - Does this need documentation: No --- .../spill_iceberg_table_sink_operator.cpp | 3 +- .../exec/sink/writer/async_result_writer.cpp | 37 ++- be/src/exec/sink/writer/async_result_writer.h | 11 +- be/src/io/fs/azure_obj_storage_client.cpp | 74 ++---- be/src/io/fs/azure_obj_storage_client.h | 2 +- be/src/io/fs/obj_storage_client.h | 1 - be/src/io/fs/s3_file_writer.cpp | 1 - .../runtime/memory/thread_mem_tracker_mgr.cpp | 40 +++ .../runtime/memory/thread_mem_tracker_mgr.h | 40 +++ be/src/runtime/runtime_state.cpp | 9 +- .../io/fs/azure_obj_storage_client_test.cpp | 22 +- .../memory/thread_mem_tracker_mgr_test.cpp | 33 +++ .../runtime_state_block_budget_test.cpp | 14 + .../IcebergRemoveOrphanFilesAction.java | 251 +++++++++++------- .../IcebergRemoveOrphanFilesActionTest.java | 157 ++++++++--- .../translator/PhysicalPlanTranslator.java | 18 +- .../doris/planner/PluginDrivenTableSink.java | 50 +++- .../org/apache/doris/qe/SessionVariable.java | 2 + ...lPlanTranslatorIcebergRowLevelDmlTest.java | 25 +- .../PluginDrivenTableSinkBindingTest.java | 42 +++ .../apache/doris/qe/SessionVariablesTest.java | 9 + .../filesystem/azure/AzureObjStorage.java | 44 +-- .../azure/AzureObjStorageExtensionTest.java | 30 +-- gensrc/thrift/PaloInternalService.thrift | 2 + 24 files changed, 623 insertions(+), 294 deletions(-) diff --git a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp index 1fde8f882b280e..75cb06e62ef74a 100644 --- a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp +++ b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp @@ -67,8 +67,9 @@ size_t SpillIcebergTableSinkLocalState::get_reserve_mem_size(RuntimeState* state } } // Column growth remains in every touched sorter, while sorting workspace is reused by serial dispatch. + // The final queued item may contain rows and also owns the reservation used by async finish(). return iceberg_reserve_size(per_partition_reservations, - eos ? 0 : state->minimum_operator_memory_required_bytes()); + state->minimum_operator_memory_required_bytes()); } size_t SpillIcebergTableSinkLocalState::get_revocable_mem_size(RuntimeState* state) const { diff --git a/be/src/exec/sink/writer/async_result_writer.cpp b/be/src/exec/sink/writer/async_result_writer.cpp index 206f6adc8445d7..4bce381dfd7c31 100644 --- a/be/src/exec/sink/writer/async_result_writer.cpp +++ b/be/src/exec/sink/writer/async_result_writer.cpp @@ -44,6 +44,8 @@ Status AsyncResultWriter::sink(Block* block, bool eos) { add_block = _get_free_block(block, rows); } + // The pipeline reservation protects allocations performed after this block is dequeued. + auto reservation = thread_context()->thread_mem_tracker_mgr->take_reserved_memory(); std::lock_guard l(_m); // if io task failed, just return error status to // end the query @@ -55,9 +57,12 @@ Status AsyncResultWriter::sink(Block* block, bool eos) { if (_is_finished()) { _dependency->set_ready(); } - if (rows) { - _memory_used_counter->update(add_block->allocated_bytes()); - _data_queue.emplace_back(std::move(add_block)); + if (rows || eos) { + if (rows) { + _memory_used_counter->update(add_block->allocated_bytes()); + } + _data_queue.emplace_back(QueuedBlock { + .block = std::move(add_block), .reservation = std::move(reservation), .eos = eos}); if (!_data_queue_is_available() && !_is_finished()) { _dependency->block(); } @@ -71,17 +76,19 @@ Status AsyncResultWriter::sink(Block* block, bool eos) { return Status::OK(); } -std::unique_ptr AsyncResultWriter::_get_block_from_queue() { +AsyncResultWriter::QueuedBlock AsyncResultWriter::_get_block_from_queue() { std::lock_guard l(_m); DCHECK(!_data_queue.empty()); - auto block = std::move(_data_queue.front()); + auto queued = std::move(_data_queue.front()); _data_queue.pop_front(); DCHECK(_dependency); if (_data_queue_is_available()) { _dependency->set_ready(); } - _memory_used_counter->update(-block->allocated_bytes()); - return block; + if (queued.block) { + _memory_used_counter->update(-queued.block->allocated_bytes()); + } + return queued; } Status AsyncResultWriter::start_writer(RuntimeState* state, RuntimeProfile* operator_profile) { @@ -165,9 +172,12 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* opera } //2) get the block from data queue and write to downstream - auto block = _get_block_from_queue(); - auto status = write(state, *block); + auto queued = _get_block_from_queue(); + thread_context()->thread_mem_tracker_mgr->adopt_reserved_memory( + std::move(queued.reservation)); + Status status = queued.block ? write(state, *queued.block) : Status::OK(); if (!status.ok()) [[unlikely]] { + thread_context()->thread_mem_tracker_mgr->shrink_reserved(); std::unique_lock l(_m); _writer_status.update(status); if (_is_finished()) { @@ -176,7 +186,14 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* opera break; } - _return_free_block(std::move(block)); + if (queued.block) { + _return_free_block(std::move(queued.block)); + } + if (queued.eos) { + // Keep the final reservation through finish(), where buffered sorters are committed. + break; + } + thread_context()->thread_mem_tracker_mgr->shrink_reserved(); } bool need_finish = false; diff --git a/be/src/exec/sink/writer/async_result_writer.h b/be/src/exec/sink/writer/async_result_writer.h index 99d4f8eaa59eff..516201ebd818e9 100644 --- a/be/src/exec/sink/writer/async_result_writer.h +++ b/be/src/exec/sink/writer/async_result_writer.h @@ -23,6 +23,7 @@ #include "exec/sink/writer/result_writer.h" #include "exprs/vexpr_fwd.h" +#include "runtime/memory/thread_mem_tracker_mgr.h" #include "runtime/runtime_profile.h" namespace doris { @@ -77,18 +78,24 @@ class AsyncResultWriter : public ResultWriter { std::unique_ptr _get_free_block(Block*, size_t rows); private: + struct QueuedBlock { + std::unique_ptr block; + ReservedMemoryToken reservation; + bool eos = false; + }; + void process_block(RuntimeState* state, RuntimeProfile* operator_profile); [[nodiscard]] bool _data_queue_is_available() const { return _data_queue.size() < QUEUE_SIZE; } [[nodiscard]] bool _is_finished() const { return !_writer_status.ok() || _eos; } void _set_ready_to_finish(); void _return_free_block(std::unique_ptr); - std::unique_ptr _get_block_from_queue(); + QueuedBlock _get_block_from_queue(); static constexpr auto QUEUE_SIZE = 3; std::mutex _m; std::condition_variable _cv; - std::deque> _data_queue; + std::deque _data_queue; // Default value is ok AtomicStatus _writer_status; bool _eos = false; diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp index a870193deeefe7..7396c355a0ca70 100644 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ b/be/src/io/fs/azure_obj_storage_client.cpp @@ -21,7 +21,6 @@ #include #include -#include #include #include #include @@ -35,7 +34,6 @@ #include #include #include -#include #include #include #include @@ -66,25 +64,14 @@ std::string to_lower_ascii(std::string_view input) { return lowered; } -std::string azure_block_id(const doris::io::ObjectStoragePathOptions& opts, int part_num) { - DCHECK(opts.upload_id.has_value()); +std::string encode_azure_block_id(std::string_view upload_id, int part_num) { // Azure requires every block ID for one blob to have the same decoded length. - std::string raw_id = fmt::format("{}:{:010}", *opts.upload_id, part_num); + std::string raw_id = fmt::format("{}:{:010}", upload_id, part_num); Aws::Utils::ByteBuffer bytes(reinterpret_cast(raw_id.data()), raw_id.size()); return Aws::Utils::HashingUtils::Base64Encode(bytes); } -std::string legacy_azure_block_id(int part_num) { - std::array bytes { - static_cast(part_num & 0xff), - static_cast((part_num >> 8) & 0xff), - static_cast((part_num >> 16) & 0xff), - static_cast((part_num >> 24) & 0xff)}; - Aws::Utils::ByteBuffer buffer(bytes.data(), bytes.size()); - return Aws::Utils::HashingUtils::Base64Encode(buffer); -} - // Rate limiting is applied by RateLimitedObjStorageClient, the decorator that // S3ClientFactory wraps around this client when the bucket is subject to limiting. @@ -94,8 +81,8 @@ constexpr char BlobNotFound[] = "BlobNotFound"; namespace doris::io { -std::string azure_multipart_temp_key(std::string_view key, std::string_view upload_id) { - return fmt::format("{}.__doris_multipart/{}", key, upload_id); +std::string azure_multipart_block_id(std::string_view upload_id, int part_num) { + return encode_azure_block_id(upload_id, part_num); } // As Azure's doc said, the batch size is 256 @@ -238,22 +225,16 @@ ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStora std::string_view stream, int part_num) { DCHECK(opts.upload_id.has_value()); - auto client = _client->GetBlockBlobClient(azure_multipart_temp_key(opts.key, *opts.upload_id)); - std::string block_id = azure_block_id(opts, part_num); + auto client = _client->GetBlockBlobClient(opts.key); + std::string block_id = azure_multipart_block_id(*opts.upload_id, part_num); auto resp = do_azure_client_call( [&]() { Azure::Core::IO::MemoryBodyStream memory_body( reinterpret_cast(stream.data()), stream.size()); // The blockId must be base64 encoded SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); + // Upload-scoped IDs make a conflicting writer fail closed instead of selecting its blocks. client.StageBlock(block_id, memory_body); - if (opts.deferred_completion) { - // During rolling upgrades an old FE still commits deterministic IDs on the final blob. - Azure::Core::IO::MemoryBodyStream legacy_body( - reinterpret_cast(stream.data()), stream.size()); - _client->GetBlockBlobClient(opts.key).StageBlock( - legacy_azure_block_id(part_num), legacy_body); - } }, opts, _tls_debug_context); return ObjectStorageUploadResponse { @@ -267,48 +248,27 @@ ObjectStorageResponse AzureObjStorageClient::complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) { DCHECK(opts.upload_id.has_value()); - auto temp_client = - _client->GetBlockBlobClient(azure_multipart_temp_key(opts.key, *opts.upload_id)); auto target_client = _client->GetBlockBlobClient(opts.key); std::vector string_block_ids; - std::ranges::transform( - completed_parts, std::back_inserter(string_block_ids), - [&opts](const ObjectCompleteMultiPart& i) { return azure_block_id(opts, i.part_num); }); - auto response = do_azure_client_call( + std::ranges::transform(completed_parts, std::back_inserter(string_block_ids), + [&opts](const ObjectCompleteMultiPart& i) { + return azure_multipart_block_id(*opts.upload_id, i.part_num); + }); + return do_azure_client_call( [&]() { SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); - // Per-upload temporary blobs keep Azure's blob-wide staged-block namespace isolated. - temp_client.CommitBlockList(string_block_ids); - auto copy = target_client.StartCopyFromUri(temp_client.GetUrl()); - copy.PollUntilDone(std::chrono::milliseconds(100)); + // Put Block List atomically replaces the committed blob; no scan-visible staging blob exists. + target_client.CommitBlockList(string_block_ids); }, opts, _tls_debug_context); - if (response.status.code != ErrorCode::OK) { - return response; - } - auto cleanup = do_azure_client_call([&]() { temp_client.Delete(); }, opts, _tls_debug_context); - if (cleanup.status.code != ErrorCode::OK && - cleanup.http_code != static_cast(Azure::Core::Http::HttpStatusCode::NotFound)) { - LOG(WARNING) << "Azure multipart temporary blob cleanup failed after publication"; - } - // Publication already succeeded; a cleanup failure must not turn a retry into a conflicting overwrite. - return response; } ObjectStorageResponse AzureObjStorageClient::abort_multipart_upload( const ObjectStoragePathOptions& opts) { DCHECK(opts.upload_id.has_value()); - auto client = _client->GetBlockBlobClient(azure_multipart_temp_key(opts.key, *opts.upload_id)); - auto response = do_azure_client_call( - [&]() { - // Never recommit the final blob: a legacy staged block may shadow a committed block ID. - client.Delete(); - }, - opts, _tls_debug_context); - // Azure creates no server-side object until the first block is staged, so absence is clean. - return response.http_code == static_cast(Azure::Core::Http::HttpStatusCode::NotFound) - ? ObjectStorageResponse::OK() - : response; + // Azure cannot delete one upload's uncommitted blocks without changing the committed blob. + // Leaving them to service GC preserves the last successfully published value. + return ObjectStorageResponse::OK(); } ObjectStorageHeadResponse AzureObjStorageClient::head_object(const ObjectStoragePathOptions& opts) { diff --git a/be/src/io/fs/azure_obj_storage_client.h b/be/src/io/fs/azure_obj_storage_client.h index 62968557490ab0..de4ea5459a7cf7 100644 --- a/be/src/io/fs/azure_obj_storage_client.h +++ b/be/src/io/fs/azure_obj_storage_client.h @@ -33,7 +33,7 @@ class ObjClientHolder; bool is_azure_tls_ca_error_message(std::string_view message); std::string build_azure_tls_debug_suffix(std::string_view error_message, std::string_view tls_debug_context); -std::string azure_multipart_temp_key(std::string_view key, std::string_view upload_id); +std::string azure_multipart_block_id(std::string_view upload_id, int part_num); class AzureObjStorageClient final : public ObjStorageClient { public: diff --git a/be/src/io/fs/obj_storage_client.h b/be/src/io/fs/obj_storage_client.h index 6ed2803c53e13b..db326a931719f9 100644 --- a/be/src/io/fs/obj_storage_client.h +++ b/be/src/io/fs/obj_storage_client.h @@ -45,7 +45,6 @@ struct ObjectStoragePathOptions { std::string key = std::string(); // blob name in azure std::string prefix = std::string(); // for batch delete and recursive delete std::optional upload_id = std::nullopt; // provider-specific upload token - bool deferred_completion = false; // another process commits the uploaded parts }; struct ObjectCompleteMultiPart { diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index 4d4f5d751cdf67..43663afa6dfdf8 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -66,7 +66,6 @@ S3FileWriter::S3FileWriter(std::shared_ptr client, std::string .key = std::move(key)}), _used_by_s3_committer(opts ? opts->used_by_s3_committer : false), _obj_client(std::move(client)) { - _obj_storage_path_opts.deferred_completion = _used_by_s3_committer; s3_file_writer_total << 1; s3_file_being_written << 1; Aws::Http::SetCompliantRfc3986Encoding(true); diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.cpp b/be/src/runtime/memory/thread_mem_tracker_mgr.cpp index 5de03560c289d2..f86e155f9acd44 100644 --- a/be/src/runtime/memory/thread_mem_tracker_mgr.cpp +++ b/be/src/runtime/memory/thread_mem_tracker_mgr.cpp @@ -27,6 +27,46 @@ namespace doris { +void ReservedMemoryToken::release() { + if (_bytes == 0 && _untracked_bytes == 0) { + return; + } + // A queued item may be discarded after an async failure; its reservation still needs full rollback. + GlobalMemoryArbitrator::shrink_process_reserved(_bytes + _untracked_bytes); + _limiter_tracker->shrink_reserved(_bytes + _untracked_bytes); + _limiter_tracker->release(_bytes); + if (auto wg = _wg_wptr.lock()) { + wg->sub_wg_refresh_interval_memory_growth(_bytes); + } + _bytes = 0; + _untracked_bytes = 0; +} + +ReservedMemoryToken ThreadMemTrackerMgr::take_reserved_memory() { + CHECK(init()); + if (_reserved_mem == 0) { + return {}; + } + ReservedMemoryToken token(_limiter_tracker_sptr, _wg_wptr, _reserved_mem, _untracked_mem); + // Accounting remains reserved globally; only its thread-local ownership moves into the token. + _reserved_mem = 0; + _untracked_mem = 0; + return token; +} + +void ThreadMemTrackerMgr::adopt_reserved_memory(ReservedMemoryToken&& token) { + CHECK(init()); + if (token._bytes == 0 && token._untracked_bytes == 0) { + return; + } + flush_untracked_mem(); + CHECK(token._limiter_tracker == _limiter_tracker_sptr); + _reserved_mem += token._bytes; + _untracked_mem += token._untracked_bytes; + token._bytes = 0; + token._untracked_bytes = 0; +} + void ThreadMemTrackerMgr::attach_limiter_tracker( const std::shared_ptr& mem_tracker) { DCHECK(mem_tracker); diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.h b/be/src/runtime/memory/thread_mem_tracker_mgr.h index d9a915c439bc62..b4494997ca7cd1 100644 --- a/be/src/runtime/memory/thread_mem_tracker_mgr.h +++ b/be/src/runtime/memory/thread_mem_tracker_mgr.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include "common/be_mock_util.h" @@ -39,6 +40,42 @@ namespace doris { +class ReservedMemoryToken { +public: + ReservedMemoryToken() = default; + ReservedMemoryToken(const ReservedMemoryToken&) = delete; + ReservedMemoryToken& operator=(const ReservedMemoryToken&) = delete; + ReservedMemoryToken(ReservedMemoryToken&& other) noexcept { *this = std::move(other); } + ReservedMemoryToken& operator=(ReservedMemoryToken&& other) noexcept { + if (this != &other) { + release(); + _limiter_tracker = std::move(other._limiter_tracker); + _wg_wptr = std::move(other._wg_wptr); + _bytes = std::exchange(other._bytes, 0); + _untracked_bytes = std::exchange(other._untracked_bytes, 0); + } + return *this; + } + ~ReservedMemoryToken() { release(); } + [[nodiscard]] int64_t bytes() const { return _bytes; } + +private: + friend class ThreadMemTrackerMgr; + ReservedMemoryToken(std::shared_ptr limiter_tracker, + std::weak_ptr wg_wptr, int64_t bytes, + int64_t untracked_bytes) + : _limiter_tracker(std::move(limiter_tracker)), + _wg_wptr(std::move(wg_wptr)), + _bytes(bytes), + _untracked_bytes(untracked_bytes) {} + void release(); + + std::shared_ptr _limiter_tracker; + std::weak_ptr _wg_wptr; + int64_t _bytes = 0; + int64_t _untracked_bytes = 0; +}; + constexpr size_t SYNC_PROC_RESERVED_INTERVAL_BYTES = (1ULL << 20); // 1M static std::string MEMORY_ORPHAN_CHECK_MSG = "The ThreadContext of the current thread not attach a valid MemoryTracker. after the " @@ -99,6 +136,9 @@ class ThreadMemTrackerMgr { void shrink_reserved(); + ReservedMemoryToken take_reserved_memory(); + void adopt_reserved_memory(ReservedMemoryToken&& token); + MemTrackerLimiter* limiter_mem_tracker() { CHECK(init()); return _limiter_tracker; diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index fde7ca3bc2d0ad..c805b5310a1f17 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -66,7 +66,14 @@ Status RuntimeState::add_iceberg_commit_datas(TIcebergCommitData iceberg_commit_ RETURN_IF_ERROR(serializer.serialize(&iceberg_commit_data, &serialized_size, &buffer)); constexpr size_t report_envelope_headroom = 1024 * 1024; - const size_t thrift_limit = static_cast(std::max(config::thrift_max_message_size, 0)); + int32_t effective_thrift_limit = std::max(config::thrift_max_message_size, 0); + if (_query_options.__isset.coordinator_thrift_max_message_size && + _query_options.coordinator_thrift_max_message_size > 0) { + // An older FE omits this field; otherwise the receiver's smaller limit is authoritative. + effective_thrift_limit = std::min(effective_thrift_limit, + _query_options.coordinator_thrift_max_message_size); + } + const size_t thrift_limit = static_cast(effective_thrift_limit); const size_t commit_data_limit = thrift_limit > report_envelope_headroom ? thrift_limit - report_envelope_headroom : 0; std::lock_guard budget_lock(_iceberg_commit_data_budget->mutex); diff --git a/be/test/io/fs/azure_obj_storage_client_test.cpp b/be/test/io/fs/azure_obj_storage_client_test.cpp index 83671a9d997d59..21de949500cf80 100644 --- a/be/test/io/fs/azure_obj_storage_client_test.cpp +++ b/be/test/io/fs/azure_obj_storage_client_test.cpp @@ -34,15 +34,15 @@ namespace doris { -TEST(AzureObjStorageClientMultipartHelperTest, isolates_uploads_with_temporary_blob_keys) { - EXPECT_EQ("table/file.parquet.__doris_multipart/upload-a", - io::azure_multipart_temp_key("table/file.parquet", "upload-a")); - EXPECT_NE(io::azure_multipart_temp_key("table/file.parquet", "upload-a"), - io::azure_multipart_temp_key("table/file.parquet", "upload-b")); -} - #ifdef USE_AZURE +TEST(AzureObjStorageClientMultipartHelperTest, isolates_uploads_with_fixed_length_block_ids) { + EXPECT_NE(io::azure_multipart_block_id("upload-a", 1), + io::azure_multipart_block_id("upload-b", 1)); + EXPECT_EQ(io::azure_multipart_block_id("upload-a", 1).size(), + io::azure_multipart_block_id("upload-a", 999).size()); +} + using namespace Azure::Storage::Blobs; TEST(AzureObjStorageClientTlsHelperTest, detects_tls_ca_error) { @@ -166,7 +166,7 @@ TEST_F(AzureObjStorageClientTest, delete_objects_recursively) { EXPECT_EQ(files.size(), 0); } -TEST_F(AzureObjStorageClientTest, abort_multipart_upload_discards_staged_blocks) { +TEST_F(AzureObjStorageClientTest, abort_multipart_upload_leaves_no_visible_blob) { io::ObjectStoragePathOptions opts; auto create_response = AzureObjStorageClientTest::obj_storage_client->create_multipart_upload(opts); @@ -232,7 +232,9 @@ TEST_F(AzureObjStorageClientTest, concurrent_multipart_uploads_do_not_share_stag ASSERT_EQ(obj_storage_client->upload_part(second, "second", 1).resp.status.code, ErrorCode::OK); ASSERT_EQ(obj_storage_client->complete_multipart_upload(first, {{.part_num = 1}}).status.code, ErrorCode::OK); - ASSERT_EQ(obj_storage_client->complete_multipart_upload(second, {{.part_num = 1}}).status.code, + // Committing one writer discards all other uncommitted blocks on the blob. The loser must fail + // instead of publishing a mixture of blocks from two uploads. + EXPECT_NE(obj_storage_client->complete_multipart_upload(second, {{.part_num = 1}}).status.code, ErrorCode::OK); std::array contents {}; @@ -241,7 +243,7 @@ TEST_F(AzureObjStorageClientTest, concurrent_multipart_uploads_do_not_share_stag ->get_object(second, contents.data(), 0, contents.size(), &size_return) .status.code, ErrorCode::OK); - EXPECT_EQ(std::string_view(contents.data(), size_return), "second"); + EXPECT_EQ(std::string_view(contents.data(), size_return), "first"); EXPECT_EQ(obj_storage_client->delete_object(second).status.code, ErrorCode::OK); } #else diff --git a/be/test/runtime/memory/thread_mem_tracker_mgr_test.cpp b/be/test/runtime/memory/thread_mem_tracker_mgr_test.cpp index 716e5f2e9a4958..188f16e96a8ee6 100644 --- a/be/test/runtime/memory/thread_mem_tracker_mgr_test.cpp +++ b/be/test/runtime/memory/thread_mem_tracker_mgr_test.cpp @@ -331,6 +331,39 @@ TEST_F(ThreadMemTrackerMgrTest, ReserveMemory) { EXPECT_EQ(doris::GlobalMemoryArbitrator::process_reserved_memory(), 0); } +TEST_F(ThreadMemTrackerMgrTest, TransfersReservationBetweenAsyncTasks) { + auto tracker = MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER, + "UT-TransferReservation"); + auto resource_context = ResourceContext::create_shared(); + resource_context->memory_context()->set_mem_tracker(tracker); + ThreadContext producer; + ThreadContext consumer; + producer.attach_task(resource_context); + consumer.attach_task(resource_context); + constexpr int64_t reservation = 4 * 1024 * 1024; + + ASSERT_TRUE(producer.thread_mem_tracker_mgr->try_reserve(reservation).ok()); + auto token = producer.thread_mem_tracker_mgr->take_reserved_memory(); + EXPECT_EQ(producer.thread_mem_tracker_mgr->reserved_mem(), 0); + EXPECT_EQ(token.bytes(), reservation); + + consumer.thread_mem_tracker_mgr->adopt_reserved_memory(std::move(token)); + EXPECT_EQ(consumer.thread_mem_tracker_mgr->reserved_mem(), reservation); + consumer.thread_mem_tracker_mgr->consume(reservation); + EXPECT_EQ(consumer.thread_mem_tracker_mgr->reserved_mem(), 0); + + ASSERT_TRUE(producer.thread_mem_tracker_mgr->try_reserve(reservation).ok()); + { + auto abandoned = producer.thread_mem_tracker_mgr->take_reserved_memory(); + EXPECT_EQ(abandoned.bytes(), reservation); + } + EXPECT_EQ(GlobalMemoryArbitrator::process_reserved_memory(), 0); + + producer.detach_task(); + consumer.detach_task(); + EXPECT_EQ(GlobalMemoryArbitrator::process_reserved_memory(), 0); +} + TEST_F(ThreadMemTrackerMgrTest, NestedReserveMemory) { std::unique_ptr thread_context = std::make_unique(); std::shared_ptr t = MemTrackerLimiter::create_shared( diff --git a/be/test/runtime/runtime_state_block_budget_test.cpp b/be/test/runtime/runtime_state_block_budget_test.cpp index 24afe920d197c5..dc02d35bd32398 100644 --- a/be/test/runtime/runtime_state_block_budget_test.cpp +++ b/be/test/runtime/runtime_state_block_budget_test.cpp @@ -59,6 +59,20 @@ TEST(RuntimeStateIcebergCommitDataTest, SharesTheReportBudgetAcrossParallelTasks EXPECT_FALSE(second_status.ok()); } +TEST(RuntimeStateIcebergCommitDataTest, UsesTheSmallerCoordinatorThriftLimit) { + RuntimeState state; + const int32_t saved_limit = config::thrift_max_message_size; + config::thrift_max_message_size = 4 * 1024 * 1024; + state._query_options.__set_coordinator_thrift_max_message_size(1024 * 1024 + 128); + TIcebergCommitData commit_data; + commit_data.__set_file_path(std::string(256, 'x')); + + Status status = state.add_iceberg_commit_datas(commit_data); + + config::thrift_max_message_size = saved_limit; + EXPECT_FALSE(status.ok()); +} + // --------------------------------------------------------------------------- // RuntimeState::batch_size() // --------------------------------------------------------------------------- diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java index 9c4dd71f61c5a4..cef3b5d2a7ebed 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java @@ -50,13 +50,16 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.regex.Pattern; /** Safely lists or deletes old files that are unreachable from every retained snapshot. */ public class IcebergRemoveOrphanFilesAction extends BaseIcebergAction { private static final long MIN_RETENTION_MS = Duration.ofHours(24).toMillis(); + private static final int MAX_REACHABLE_FILES = 5_000_000; public static final String OLDER_THAN = "older_than"; public static final String LOCATION = "location"; public static final String DRY_RUN = "dry_run"; + public static final String ALLOW_UNSAFE_LOCATION = "allow_unsafe_location"; public IcebergRemoveOrphanFilesAction(Map properties, List partitionNames, ConnectorPredicate whereCondition) { @@ -67,10 +70,13 @@ public IcebergRemoveOrphanFilesAction(Map properties, List executeAction(Table table, ConnectorSession session) { // A GC-disabled table may share files with another table, so no destructive scan is safe. throw new DorisConnectorException("Cannot remove orphan files: Iceberg GC is disabled"); } - List scanLocations = resolveScanLocations(table); + long olderThan = namedArguments.getLong(OLDER_THAN); + // Reject an unsafe cutoff before opening any metadata or manifest file. + if (olderThan > System.currentTimeMillis() - MIN_RETENTION_MS) { + throw new DorisConnectorException("older_than must retain at least 24 hours of files"); + } + List scanScopes = resolveScanScopes(table); try { - ReachableIndex reachable = new ReachableIndex(collectReachableFiles(table)); + ReachableIndex reachable = collectReachableFiles(table); long orphanCount = 0; long deletedCount = 0; - long olderThan = namedArguments.getLong(OLDER_THAN); - // The SQL procedure needs a retention fence because concurrent uploads are not reachable until commit. - if (olderThan > System.currentTimeMillis() - MIN_RETENTION_MS) { - throw new DorisConnectorException( - "older_than must retain at least 24 hours of files"); - } boolean dryRun = namedArguments.getBoolean(DRY_RUN); - Set visitedFiles = new HashSet<>(); - for (String scanLocation : scanLocations) { + for (ScanScope scope : scanScopes) { // Object stores use raw prefix matching, so the separator excludes sibling prefixes. - String listingPrefix = scanLocation.endsWith("/") ? scanLocation : scanLocation + "/"; + String listingPrefix = scope.root.endsWith("/") ? scope.root : scope.root + "/"; for (FileInfo file : ((SupportsPrefixOperations) table.io()).listPrefix(listingPrefix)) { - if (visitedFiles.add(file.location()) && file.createdAtMillis() < olderThan + if (scope.owns(file.location()) && file.createdAtMillis() < olderThan && !isReachable(file.location(), reachable)) { orphanCount++; if (!dryRun) { @@ -131,67 +135,69 @@ protected List executeAction(Table table, ConnectorSession session) { } } - private List resolveScanLocations(Table table) { - List ownedRoots = resolveOwnedRoots(table.location(), table.properties()); - + private List resolveScanScopes(Table table) { + String tableRoot = normalizeLocation(table.location()); String requested = namedArguments.getString(LOCATION); if (requested != null) { String normalized = normalizeLocation(requested); - boolean owned = ownedRoots.stream().anyMatch(root -> isWithin(normalized, root)); - if (!owned) { - throw new DorisConnectorException( - "location must be within an Iceberg table-owned metadata or data location"); + if (isWithin(normalized, tableRoot)) { + return Lists.newArrayList(ScanScope.exclusive(normalized)); } - return Lists.newArrayList(normalized); - } - - return minimalOwnedRoots(ownedRoots); - } - - static List resolveOwnedRoots(String tableLocation, Map properties) { - String tableRoot = normalizeLocation(tableLocation); - String dataRoot = normalizeLocation(resolveDataLocation(properties, tableRoot)); - List configuredRoots = new ArrayList<>(); - configuredRoots.add(tableRoot); - configuredRoots.add(dataRoot); - // Iceberg may place metadata outside both table and data roots, so it remains an independent owned root. - String metadataRoot = nonEmpty(properties.get(TableProperties.WRITE_METADATA_LOCATION)); - if (metadataRoot != null) { - configuredRoots.add(normalizeLocation(metadataRoot)); - } - return canonicalOwnedRoots(configuredRoots); - } - - static List minimalOwnedRoots(List roots) { - List canonicalRoots = canonicalOwnedRoots(roots); - List minimal = new ArrayList<>(); - for (String candidate : canonicalRoots) { - // Canonically equal aliases are deduplicated first, so only strict containment removes a root. - if (canonicalRoots.stream().noneMatch(other -> !sameFileIdentity(other, candidate) - && isWithinLocation(candidate, other))) { - minimal.add(candidate); + if (!namedArguments.getBoolean(ALLOW_UNSAFE_LOCATION)) { + throw new DorisConnectorException( + "Cannot prove that location is owned by this table; set allow_unsafe_location=true " + + "only after verifying the prefix is exclusive to the table"); } + // This explicit escape hatch also covers historical roots after a table-location migration. + return Lists.newArrayList(ScanScope.exclusive(normalized)); } - return minimal; - } - - private static List canonicalOwnedRoots(List roots) { - Map byIdentity = new LinkedHashMap<>(); - for (String root : roots) { - byIdentity.putIfAbsent(FileIdentity.of(root), root); + if (nonEmpty(table.properties().get(TableProperties.WRITE_LOCATION_PROVIDER_IMPL)) != null) { + throw new DorisConnectorException( + "remove_orphan_files cannot infer ownership for a custom write.location-provider.impl; " + + "provide location with allow_unsafe_location=true after verifying exclusivity"); } - return new ArrayList<>(byIdentity.values()); - } - - private static String resolveDataLocation(Map properties, String tableRoot) { - String dataLocation = nonEmpty(properties.get(TableProperties.WRITE_DATA_LOCATION)); - if (dataLocation == null && Boolean.parseBoolean(properties.get(TableProperties.OBJECT_STORE_ENABLED))) { - dataLocation = nonEmpty(properties.get(TableProperties.OBJECT_STORE_PATH)); + String metadataRoot = nonEmpty(table.properties().get(TableProperties.WRITE_METADATA_LOCATION)); + if (metadataRoot != null && !isWithin(normalizeLocation(metadataRoot), tableRoot)) { + throw new DorisConnectorException( + "Cannot prove that the configured external metadata location is table-exclusive; " + + "provide location with allow_unsafe_location=true after verifying exclusivity"); } - if (dataLocation == null) { - dataLocation = nonEmpty(properties.get(TableProperties.WRITE_FOLDER_STORAGE_LOCATION)); + List scopes = new ArrayList<>(); + scopes.add(ScanScope.exclusive(tableRoot)); + if (Boolean.parseBoolean(table.properties().get(TableProperties.OBJECT_STORE_ENABLED))) { + // Match Iceberg's ObjectStoreLocationProvider precedence exactly. + String objectRoot = nonEmpty(table.properties().get(TableProperties.WRITE_DATA_LOCATION)); + if (objectRoot == null) { + objectRoot = nonEmpty(table.properties().get(TableProperties.OBJECT_STORE_PATH)); + } + if (objectRoot == null) { + objectRoot = nonEmpty(table.properties().get(TableProperties.WRITE_FOLDER_STORAGE_LOCATION)); + } + if (objectRoot != null) { + String normalizedObjectRoot = normalizeLocation(objectRoot); + if (!isWithin(normalizedObjectRoot, tableRoot)) { + if (normalizedObjectRoot.startsWith(tableRoot)) { + // Iceberg omits table context for this raw-prefix case, so ownership is not recoverable. + throw new DorisConnectorException( + "Cannot prove object-store ownership because its path has the table location " + + "as a non-directory prefix; provide a verified explicit location"); + } + scopes.add(ScanScope.objectStore(normalizedObjectRoot, tableRoot)); + } + } + } else { + String externalDataRoot = nonEmpty(table.properties().get(TableProperties.WRITE_DATA_LOCATION)); + if (externalDataRoot == null) { + externalDataRoot = nonEmpty( + table.properties().get(TableProperties.WRITE_FOLDER_STORAGE_LOCATION)); + } + if (externalDataRoot != null && !isWithin(normalizeLocation(externalDataRoot), tableRoot)) { + throw new DorisConnectorException( + "Cannot prove that the configured external data location is table-exclusive; " + + "provide location with allow_unsafe_location=true after verifying exclusivity"); + } } - return dataLocation == null ? tableRoot + "/data" : dataLocation; + return scopes; } private static String nonEmpty(String location) { @@ -210,8 +216,9 @@ private static boolean isWithinLocation(String location, String root) { && (child.path.equals(parent.path) || child.path.startsWith(pathPrefix)); } - private Set collectReachableFiles(Table table) throws IOException { - Set reachable = new HashSet<>(ReachableFileUtil.metadataFileLocations(table, true)); + private ReachableIndex collectReachableFiles(Table table) throws IOException { + ReachableIndex reachable = new ReachableIndex(MAX_REACHABLE_FILES); + reachable.addAll(ReachableFileUtil.metadataFileLocations(table, true)); // Hadoop tables consult this live pointer even though it is not part of the metadata log. reachable.add(ReachableFileUtil.versionHintLocation(table)); Set scannedDataManifests = new HashSet<>(); @@ -243,10 +250,11 @@ private Set collectReachableFiles(Table table) throws IOException { private static boolean isReachable(String candidate, ReachableIndex reachable) { FileIdentity candidateIdentity = FileIdentity.of(candidate); - if (reachable.identities.contains(candidateIdentity)) { + FileIdentity retainedIdentity = reachable.byPath.get(candidateIdentity.path); + if (candidateIdentity.equals(retainedIdentity)) { return true; } - if (reachable.paths.contains(candidateIdentity.path)) { + if (retainedIdentity != null) { // A path collision across unknown providers/authorities cannot be classified safely. throw new DorisConnectorException( "Cannot determine whether listed and reachable file locations are equivalent"); @@ -258,19 +266,6 @@ static boolean sameFileIdentity(String first, String second) { return FileIdentity.of(first).equals(FileIdentity.of(second)); } - static void verifyNoPrefixMismatch(String candidate, Set reachable) { - FileIdentity candidateIdentity = FileIdentity.of(candidate); - for (String retained : reachable) { - FileIdentity retainedIdentity = FileIdentity.of(retained); - // Matching paths with different providers/authorities are ambiguous; deletion must fail closed. - if (candidateIdentity.path.equals(retainedIdentity.path) - && !candidateIdentity.equals(retainedIdentity)) { - throw new DorisConnectorException( - "Cannot determine whether listed and reachable file locations are equivalent"); - } - } - } - private static final class FileIdentity { private final String scheme; private final String authority; @@ -314,16 +309,92 @@ public int hashCode() { } } + static void verifyReachableIndexLimit(Set locations, int maxEntries) { + ReachableIndex index = new ReachableIndex(maxEntries); + index.addAll(locations); + } + + static boolean isOwnedObjectStorePath(String candidate, String storageRoot, String tableLocation) { + return ScanScope.objectStore(normalizeLocation(storageRoot), normalizeLocation(tableLocation)) + .owns(candidate); + } + private static final class ReachableIndex { - private final Set identities = new HashSet<>(); - private final Set paths = new HashSet<>(); - - private ReachableIndex(Set locations) { - for (String location : locations) { - FileIdentity identity = FileIdentity.of(location); - identities.add(identity); - paths.add(identity.path); + private final Map byPath = new LinkedHashMap<>(); + private final int maxEntries; + + private ReachableIndex(int maxEntries) { + this.maxEntries = maxEntries; + } + + private void addAll(Iterable locations) { + locations.forEach(this::add); + } + + private void add(String location) { + FileIdentity identity = FileIdentity.of(location); + FileIdentity existing = byPath.putIfAbsent(identity.path, identity); + if (existing != null && !existing.equals(identity)) { + throw new DorisConnectorException( + "Cannot determine whether reachable file locations are equivalent"); + } + if (existing == null && byPath.size() > maxEntries) { + throw new DorisConnectorException( + "Reachable file index exceeds the safe in-memory limit of " + maxEntries); + } + } + } + + private static final class ScanScope { + private final String root; + private final Pattern ownedRelativePath; + + private ScanScope(String root, Pattern ownedRelativePath) { + this.root = root; + this.ownedRelativePath = ownedRelativePath; + } + + private static ScanScope exclusive(String root) { + return new ScanScope(root, null); + } + + private static ScanScope objectStore(String root, String tableLocation) { + URI tableUri = URI.create(tableLocation); + String[] segments = tableUri.getPath().split("/"); + List names = new ArrayList<>(); + for (String segment : segments) { + if (!segment.isEmpty()) { + names.add(segment); + } + } + if (names.isEmpty()) { + throw new DorisConnectorException( + "Cannot infer an object-store table context from the table location"); + } + String context = names.size() > 1 + ? names.get(names.size() - 2) + "/" + names.get(names.size() - 1) + : names.get(names.size() - 1); + return new ScanScope(root, Pattern.compile( + "[01]{4}/[01]{4}/[01]{4}/[01]{4}/[01]{4}/" + + Pattern.quote(context) + "/.+")); + } + + private boolean owns(String candidate) { + if (ownedRelativePath == null) { + return isWithinLocation(candidate, root); + } + FileIdentity child = FileIdentity.of(candidate); + FileIdentity parent = FileIdentity.of(root); + if (!child.scheme.equals(parent.scheme) || !child.authority.equals(parent.authority)) { + return false; + } + if (!isWithinLocation(candidate, root)) { + return false; } + String relative = child.path.substring(Math.min(child.path.length(), parent.path.length())); + relative = relative.startsWith("/") ? relative.substring(1) : relative; + // Iceberg's ObjectStoreLocationProvider prefixes five 4-bit hash directories before context. + return ownedRelativePath.matcher(relative).matches(); } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java index f4aa7b126a52b4..c48eac67816251 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java @@ -69,12 +69,20 @@ public void gcDisabledPreventsDeletion(@TempDir Path temp) throws Exception { @Test public void recentCutoffCannotRaceAnUncommittedWriter(@TempDir Path temp) throws Exception { Table table = createTable(temp.resolve("table"), Collections.emptyMap()); + Set manifestPaths = new HashSet<>(); + table.snapshots().forEach(snapshot -> snapshot.allManifests(table.io()) + .forEach(manifest -> manifestPaths.add(manifest.path()))); + RecordingFileIO recordingFileIO = new RecordingFileIO(table.io(), manifestPaths); + Table recordingTable = new BaseTable( + new StaticTableOperations(((HasTableOperations) table).operations().current(), recordingFileIO), + table.name()); Path uncommitted = createOldFile(temp.resolve("table/data/uncommitted.parquet")); IcebergRemoveOrphanFilesAction action = action(System.currentTimeMillis(), false); action.validate(); Assertions.assertThrows(DorisConnectorException.class, - () -> action.execute(table, ActionTestTables.session("UTC"))); + () -> action.execute(recordingTable, ActionTestTables.session("UTC"))); + Assertions.assertEquals(0, recordingFileIO.manifestOpenCount()); Assertions.assertTrue(Files.exists(uncommitted)); } @@ -103,32 +111,6 @@ public void treatsS3SchemeAliasesAsTheSameFile() { "s3n://bucket/path/data.parquet", "s3://BUCKET/path/data.parquet")); } - @Test - public void canonicalAliasRootsCollapseToOneOwnedScanRoot() { - Assertions.assertEquals(Collections.singletonList("s3a://bucket/table"), - IcebergRemoveOrphanFilesAction.minimalOwnedRoots( - java.util.List.of("s3a://bucket/table", "s3://BUCKET/table"))); - } - - @Test - public void configuredDataAndMetadataLocationsAreSeparateOwnedRoots() { - Map properties = new HashMap<>(); - properties.put(TableProperties.WRITE_DATA_LOCATION, "s3://bucket/data-root"); - properties.put(TableProperties.WRITE_METADATA_LOCATION, "s3://bucket/metadata-root"); - - Assertions.assertEquals(java.util.List.of( - "s3://bucket/table-root", "s3://bucket/data-root", "s3://bucket/metadata-root"), - IcebergRemoveOrphanFilesAction.resolveOwnedRoots("s3://bucket/table-root", properties)); - } - - @Test - public void rejectsUnresolvedPrefixMismatches() { - Assertions.assertThrows(DorisConnectorException.class, - () -> IcebergRemoveOrphanFilesAction.verifyNoPrefixMismatch( - "s3://first/path/data.parquet", - Collections.singleton("s3://second/path/data.parquet"))); - } - @Test public void readsEachSharedDataManifestOnlyOnce(@TempDir Path temp) throws Exception { Map properties = new HashMap<>(); @@ -161,7 +143,7 @@ public void readsEachSharedDataManifestOnlyOnce(@TempDir Path temp) throws Excep } @Test - public void scansConfiguredDataRootOutsideTableLocationByDefault(@TempDir Path temp) throws Exception { + public void rejectsUnprovenExternalDataRootByDefault(@TempDir Path temp) throws Exception { Path dataRoot = temp.resolve("owned-data"); Table table = createTable(temp.resolve("metadata"), Collections.singletonMap(TableProperties.WRITE_DATA_LOCATION, @@ -171,15 +153,14 @@ public void scansConfiguredDataRootOutsideTableLocationByDefault(@TempDir Path t System.currentTimeMillis() - MIN_RETENTION_MS, false); action.validate(); - ConnectorProcedureResult result = action.execute(table, ActionTestTables.session("UTC")); - - Assertions.assertEquals("1", result.getRows().get(0).get(0)); - Assertions.assertEquals("1", result.getRows().get(0).get(1)); - Assertions.assertFalse(Files.exists(orphan)); + Assertions.assertThrows(DorisConnectorException.class, + () -> action.execute(table, ActionTestTables.session("UTC"))); + Assertions.assertTrue(Files.exists(orphan)); } @Test - public void allowsExplicitConfiguredDataRootButRejectsArbitraryRoot(@TempDir Path temp) throws Exception { + public void guardedExplicitDataRootDeletesButUnguardedArbitraryRootIsRejected(@TempDir Path temp) + throws Exception { Path dataRoot = temp.resolve("owned-data"); Table table = createTable(temp.resolve("metadata"), Collections.singletonMap(TableProperties.WRITE_DATA_LOCATION, @@ -187,7 +168,7 @@ public void allowsExplicitConfiguredDataRootButRejectsArbitraryRoot(@TempDir Pat Path orphan = createOldFile(dataRoot.resolve("orphan.parquet")); IcebergRemoveOrphanFilesAction configured = action( - System.currentTimeMillis() - MIN_RETENTION_MS, false, dataRoot.toUri().toString()); + System.currentTimeMillis() - MIN_RETENTION_MS, false, dataRoot.toUri().toString(), true); configured.validate(); configured.execute(table, ActionTestTables.session("UTC")); Assertions.assertFalse(Files.exists(orphan)); @@ -201,13 +182,62 @@ public void allowsExplicitConfiguredDataRootButRejectsArbitraryRoot(@TempDir Pat } @Test - public void scansObjectStoreAndFolderStorageFallbackRoots(@TempDir Path temp) throws Exception { + public void guardedExplicitLocationCoversFormerTableRootAfterMigration(@TempDir Path temp) throws Exception { + Table table = createTable(temp.resolve("current-table"), Collections.emptyMap()); + Path sharedFormerRoot = temp.resolve("former-root-now-shared"); + Path formerDataRoot = sharedFormerRoot.resolve("old-table-data"); + Path formerMetadataRoot = sharedFormerRoot.resolve("old-table-metadata"); + Path dataOrphan = createOldFile(formerDataRoot.resolve("orphan.parquet")); + Path metadataOrphan = createOldFile(formerMetadataRoot.resolve("orphan.metadata.json")); + Path neighborFile = createOldFile(sharedFormerRoot.resolve("neighbor-table/live.parquet")); + + IcebergRemoveOrphanFilesAction unguarded = action( + System.currentTimeMillis() - MIN_RETENTION_MS, false, + sharedFormerRoot.toUri().toString()); + unguarded.validate(); + Assertions.assertThrows(DorisConnectorException.class, + () -> unguarded.execute(table, ActionTestTables.session("UTC"))); + Assertions.assertTrue(Files.exists(dataOrphan)); + Assertions.assertTrue(Files.exists(metadataOrphan)); + Assertions.assertTrue(Files.exists(neighborFile)); + + IcebergRemoveOrphanFilesAction dataAction = action( + System.currentTimeMillis() - MIN_RETENTION_MS, false, + formerDataRoot.toUri().toString(), true); + dataAction.validate(); + dataAction.execute(table, ActionTestTables.session("UTC")); + IcebergRemoveOrphanFilesAction metadataAction = action( + System.currentTimeMillis() - MIN_RETENTION_MS, false, + formerMetadataRoot.toUri().toString(), true); + metadataAction.validate(); + metadataAction.execute(table, ActionTestTables.session("UTC")); + + Assertions.assertFalse(Files.exists(dataOrphan)); + Assertions.assertFalse(Files.exists(metadataOrphan)); + Assertions.assertTrue(Files.exists(neighborFile)); + } + + @Test + public void objectStoreOwnershipExcludesNeighborTableAndFolderRootFailsClosed(@TempDir Path temp) + throws Exception { Path objectRoot = temp.resolve("object-data"); Map objectProperties = new HashMap<>(); objectProperties.put(TableProperties.OBJECT_STORE_ENABLED, "true"); - objectProperties.put(TableProperties.OBJECT_STORE_PATH, objectRoot.toUri().toString()); + objectProperties.put(TableProperties.WRITE_DATA_LOCATION, objectRoot.toUri().toString()); + objectProperties.put(TableProperties.OBJECT_STORE_PATH, + temp.resolve("lower-precedence-object-path").toUri().toString()); + Files.createDirectories(objectRoot); Table objectTable = createTable(temp.resolve("object-metadata"), objectProperties); - Path objectOrphan = createOldFile(objectRoot.resolve("orphan.parquet")); + Path ownOrphan = createOldFile(objectRoot.resolve( + "0000/0001/0010/0011/0100/" + temp.getFileName() + "/object-metadata/own.parquet")); + Path neighborFile = createOldFile(objectRoot.resolve( + "0000/0001/0010/0011/0100/" + temp.getFileName() + "/neighbor/live.parquet")); + Assertions.assertTrue(IcebergRemoveOrphanFilesAction.isOwnedObjectStorePath( + "s3://bucket/shared/0000/0001/0010/0011/0100/db/table/file.parquet", + "s3://bucket/shared", "s3://bucket/warehouse/db/table")); + Assertions.assertFalse(IcebergRemoveOrphanFilesAction.isOwnedObjectStorePath( + "s3://bucket/shared/0000/0001/0010/0011/0100/db/neighbor/file.parquet", + "s3://bucket/shared", "s3://bucket/warehouse/db/table")); Path folderRoot = temp.resolve("folder-data"); Table folderTable = createTable(temp.resolve("folder-metadata"), @@ -219,13 +249,51 @@ public void scansObjectStoreAndFolderStorageFallbackRoots(@TempDir Path temp) th System.currentTimeMillis() - MIN_RETENTION_MS, false); objectAction.validate(); objectAction.execute(objectTable, ActionTestTables.session("UTC")); + Assertions.assertFalse(Files.exists(ownOrphan)); + Assertions.assertTrue(Files.exists(neighborFile)); + + Map prefixCollisionProperties = new HashMap<>(); + prefixCollisionProperties.put(TableProperties.OBJECT_STORE_ENABLED, "true"); + prefixCollisionProperties.put(TableProperties.WRITE_DATA_LOCATION, + temp.resolve("prefix-table-shared").toUri().toString()); + Table prefixCollisionTable = createTable(temp.resolve("prefix-table"), prefixCollisionProperties); + Assertions.assertThrows(DorisConnectorException.class, + () -> objectAction.execute(prefixCollisionTable, ActionTestTables.session("UTC"))); + IcebergRemoveOrphanFilesAction folderAction = action( System.currentTimeMillis() - MIN_RETENTION_MS, false); folderAction.validate(); - folderAction.execute(folderTable, ActionTestTables.session("UTC")); + Assertions.assertThrows(DorisConnectorException.class, + () -> folderAction.execute(folderTable, ActionTestTables.session("UTC"))); + Assertions.assertTrue(Files.exists(folderOrphan)); + } + + @Test + public void reachableIndexHasExplicitSafetyCap() { + Set files = Set.of("s3://bucket/table/a", "s3://bucket/table/b", "s3://bucket/table/c"); + Assertions.assertThrows(DorisConnectorException.class, + () -> IcebergRemoveOrphanFilesAction.verifyReachableIndexLimit(files, 2)); + } - Assertions.assertFalse(Files.exists(objectOrphan)); - Assertions.assertFalse(Files.exists(folderOrphan)); + @Test + public void customLocationProviderRequiresGuardedExplicitLocation(@TempDir Path temp) throws Exception { + Table table = createTable(temp.resolve("table"), Collections.singletonMap( + TableProperties.WRITE_LOCATION_PROVIDER_IMPL, "example.CustomProvider")); + Path providerRoot = temp.resolve("provider-data"); + Path orphan = createOldFile(providerRoot.resolve("orphan.parquet")); + IcebergRemoveOrphanFilesAction action = action( + System.currentTimeMillis() - MIN_RETENTION_MS, true); + action.validate(); + Assertions.assertThrows(DorisConnectorException.class, + () -> action.execute(table, ActionTestTables.session("UTC"))); + Assertions.assertTrue(Files.exists(orphan)); + + IcebergRemoveOrphanFilesAction guarded = action( + System.currentTimeMillis() - MIN_RETENTION_MS, false, + providerRoot.toUri().toString(), true); + guarded.validate(); + guarded.execute(table, ActionTestTables.session("UTC")); + Assertions.assertFalse(Files.exists(orphan)); } private static IcebergRemoveOrphanFilesAction action(long olderThan, boolean dryRun) { @@ -233,9 +301,16 @@ private static IcebergRemoveOrphanFilesAction action(long olderThan, boolean dry } private static IcebergRemoveOrphanFilesAction action(long olderThan, boolean dryRun, String location) { + return action(olderThan, dryRun, location, false); + } + + private static IcebergRemoveOrphanFilesAction action( + long olderThan, boolean dryRun, String location, boolean allowUnsafeLocation) { Map properties = new HashMap<>(); properties.put(IcebergRemoveOrphanFilesAction.OLDER_THAN, String.valueOf(olderThan)); properties.put(IcebergRemoveOrphanFilesAction.DRY_RUN, String.valueOf(dryRun)); + properties.put(IcebergRemoveOrphanFilesAction.ALLOW_UNSAFE_LOCATION, + String.valueOf(allowUnsafeLocation)); if (location != null) { properties.put(IcebergRemoveOrphanFilesAction.LOCATION, location); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 09c8bc2462835e..c81869b6b0e57f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -54,7 +54,6 @@ import org.apache.doris.datasource.doris.RemoteDorisExternalTable; import org.apache.doris.datasource.doris.RemoteOlapTable; import org.apache.doris.datasource.doris.source.RemoteDorisScanNode; -import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; import org.apache.doris.datasource.plugin.PluginDrivenMetadata; @@ -645,13 +644,11 @@ private PluginDrivenTableSink buildPluginRowLevelDmlSink( "Connector '" + catalog.getName() + "' (type: " + catalog.getType() + ") does not support row-level DML operations"); } - providerTableHandle = PluginDrivenScanNode.applyMvccSnapshotPin( - metadata, connSession, providerTableHandle, MvccUtil.getSnapshotFromContext(targetTable)); // writeSortInfo == null: a row-level DML has no engine-resolved write sort (MERGE's sort lives in the // connector's TIcebergMergeSink.sort_fields, DELETE is unsorted). return new PluginDrivenTableSink(targetTable, writePlanProvider, connSession, providerTableHandle, connectorColumns, connectorColumns, null, writeOperation, - requireMergeCardinalityCheck, boundWriteMetadataIdentity); + requireMergeCardinalityCheck, boundWriteMetadataIdentity, metadata); } @Override @@ -710,20 +707,9 @@ public PlanFragment visitPhysicalConnectorTableSink( + ") does not support INSERT operations"); } - // Thread the statement's MVCC snapshot pin onto the WRITE handle, reusing the exact scan-side pin - // logic so a DML's write anchors at the SAME snapshot its scan read (the pin is keyed by - // catalog/db/table in StatementContext, so the write target resolves the scan's pin). WHY: an MVCC - // connector's RowDelta DELETE/MERGE re-derives the deletes to remove from the write's base snapshot, - // while BE unions the scan-time deletes into the new DV — pinning both at the read snapshot keeps - // them on one snapshot ([SHOULD-2] / Fix B). A no-op for non-MVCC tables (jdbc/maxcompute) and any - // connector whose handle is not snapshot-pinned, so it is byte-identical for every current write path. - providerTableHandle = PluginDrivenScanNode.applyMvccSnapshotPin( - metadata, connSession, providerTableHandle, MvccUtil.getSnapshotFromContext(targetTable)); - // Preserve the generation captured from the exact remote table load that supplied the bound schema. // A live lookup here would silently move the fence after a concurrent drop/recreate. String boundWriteMetadataIdentity = connectorTableSink.getBoundWriteMetadataIdentity(); - // The connector declares its write-sort columns (e.g. an iceberg WRITE ORDERED BY) as positions // into the sink's full-schema output; the engine resolves them to bound slots and builds the // TSortInfo here (the connector's planWrite has no bound exprs). Empty for connectors with no @@ -743,7 +729,7 @@ public PlanFragment visitPhysicalConnectorTableSink( PluginDrivenTableSink providerSink = new PluginDrivenTableSink(targetTable, writePlanProvider, connSession, providerTableHandle, connectorColumns, boundTargetColumns, writeSortInfo, writeOperation, false, - boundWriteMetadataIdentity); + boundWriteMetadataIdentity, metadata); rootFragment.setSink(providerSink); return rootFragment; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PluginDrivenTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PluginDrivenTableSink.java index 79e53cd3f02968..a337477c963b0c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PluginDrivenTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PluginDrivenTableSink.java @@ -17,15 +17,19 @@ package org.apache.doris.planner; +import org.apache.doris.analysis.TableScanParams; import org.apache.doris.common.AnalysisException; import org.apache.doris.connector.spi.ConnectorColumn; +import org.apache.doris.connector.spi.ConnectorMetadata; import org.apache.doris.connector.spi.ConnectorSession; import org.apache.doris.connector.spi.handle.ConnectorTableHandle; import org.apache.doris.connector.spi.handle.ConnectorWriteHandle; import org.apache.doris.connector.spi.handle.WriteOperation; import org.apache.doris.connector.spi.write.ConnectorSinkPlan; import org.apache.doris.connector.spi.write.ConnectorWritePlanProvider; +import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; +import org.apache.doris.datasource.scan.PluginDrivenScanNode; import org.apache.doris.nereids.trees.plans.commands.insert.InsertCommandContext; import org.apache.doris.nereids.trees.plans.commands.insert.PluginDrivenInsertCommandContext; import org.apache.doris.thrift.TDataSink; @@ -59,6 +63,7 @@ public class PluginDrivenTableSink extends BaseExternalTableDataSink { private final ConnectorWritePlanProvider writePlanProvider; private final ConnectorSession connectorSession; private final ConnectorTableHandle tableHandle; + private final ConnectorMetadata connectorMetadata; private final List connectorColumns; private final List boundTargetColumns; // The engine-built BE sort instruction for a connector that declares write-sort columns (iceberg @@ -114,7 +119,7 @@ public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, ConnectorTableHandle tableHandle, List connectorColumns, TSortInfo writeSortInfo, WriteOperation writeOperation) { this(targetTable, writePlanProvider, connectorSession, tableHandle, connectorColumns, - writeSortInfo, writeOperation, false); + writeSortInfo, writeOperation, false, null); } /** @@ -127,7 +132,20 @@ public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, ConnectorTableHandle tableHandle, List connectorColumns, TSortInfo writeSortInfo, WriteOperation writeOperation, boolean requireMergeCardinalityCheck) { this(targetTable, writePlanProvider, connectorSession, tableHandle, connectorColumns, - connectorColumns, writeSortInfo, writeOperation, requireMergeCardinalityCheck); + writeSortInfo, writeOperation, requireMergeCardinalityCheck, null); + } + + /** + * Plan-provider mode with connector metadata used to resolve the exact branch-aware MVCC pin at bind time. + */ + public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, + ConnectorWritePlanProvider writePlanProvider, ConnectorSession connectorSession, + ConnectorTableHandle tableHandle, List connectorColumns, + TSortInfo writeSortInfo, WriteOperation writeOperation, boolean requireMergeCardinalityCheck, + ConnectorMetadata connectorMetadata) { + this(targetTable, writePlanProvider, connectorSession, tableHandle, connectorColumns, + connectorColumns, writeSortInfo, writeOperation, requireMergeCardinalityCheck, + null, connectorMetadata); } /** @@ -151,11 +169,27 @@ public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, List boundTargetColumns, TSortInfo writeSortInfo, WriteOperation writeOperation, boolean requireMergeCardinalityCheck, String boundWriteMetadataIdentity) { + this(targetTable, writePlanProvider, connectorSession, tableHandle, connectorColumns, + boundTargetColumns, writeSortInfo, writeOperation, requireMergeCardinalityCheck, + boundWriteMetadataIdentity, null); + } + + /** + * Complete plan-provider mode. The schema generation and branch-aware snapshot are separate invariants: + * the former must stay at bind time, while the latter is resolved for the exact write branch. + */ + public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, + ConnectorWritePlanProvider writePlanProvider, ConnectorSession connectorSession, + ConnectorTableHandle tableHandle, List connectorColumns, + List boundTargetColumns, TSortInfo writeSortInfo, + WriteOperation writeOperation, boolean requireMergeCardinalityCheck, + String boundWriteMetadataIdentity, ConnectorMetadata connectorMetadata) { super(); this.targetTable = targetTable; this.writePlanProvider = writePlanProvider; this.connectorSession = connectorSession; this.tableHandle = tableHandle; + this.connectorMetadata = connectorMetadata; this.connectorColumns = connectorColumns; // Keep this immutable bind-time snapshot distinct from the write subset. Re-reading the live // table here would move the conflict-detection baseline and could miss ordinal schema drift. @@ -221,8 +255,18 @@ public void bindDataSink(Optional insertCtx) writeContext = ctx.getStaticPartitionSpec(); branchName = ctx.getBranchName(); } + ConnectorTableHandle boundTableHandle = tableHandle; + if (connectorMetadata != null && targetTable != null) { + Optional scanParams = branchName.map(branch -> + new TableScanParams(TableScanParams.BRANCH, Collections.emptyMap(), + Collections.singletonList(branch))); + // The write target must use the pin for its exact branch, not another reference of the same table. + boundTableHandle = PluginDrivenScanNode.applyMvccSnapshotPin( + connectorMetadata, connectorSession, tableHandle, + MvccUtil.getSnapshotFromContext(targetTable, Optional.empty(), scanParams)); + } ConnectorWriteHandle handle = new PluginDrivenWriteHandle( - tableHandle, connectorColumns, boundTargetColumns, overwrite, writeContext, writeSortInfo, + boundTableHandle, connectorColumns, boundTargetColumns, overwrite, writeContext, writeSortInfo, boundWriteMetadataIdentity, branchName, writeOperation, requireMergeCardinalityCheck); ConnectorSinkPlan sinkPlan = writePlanProvider.planWrite(connectorSession, handle); this.tDataSink = sinkPlan.getDataSink(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index cc144759f24455..6a8da9c71f1377 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -5377,6 +5377,8 @@ public boolean isRequireSequenceInInsert() { */ public TQueryOptions toThrift() { TQueryOptions tResult = new TQueryOptions(); + // Fragment reports are decoded by FE, whose limit can be lower than a rolling-upgrade BE's. + tResult.setCoordinatorThriftMaxMessageSize(Config.thrift_max_message_size); tResult.setMemLimit(maxExecMemByte); tResult.setMaxScanMemRatio(maxScanMemRatio); tResult.setEnableAdaptiveScan(enableAdaptiveScan); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorIcebergRowLevelDmlTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorIcebergRowLevelDmlTest.java index ba275fd20246dc..0b5a3b1ecd4950 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorIcebergRowLevelDmlTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorIcebergRowLevelDmlTest.java @@ -33,8 +33,10 @@ import org.apache.doris.connector.spi.ConnectorStatementScope; import org.apache.doris.connector.spi.ConnectorType; import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.connector.spi.handle.ConnectorWriteHandle; import org.apache.doris.connector.spi.handle.WriteOperation; import org.apache.doris.connector.spi.mvcc.ConnectorMvccSnapshot; +import org.apache.doris.connector.spi.write.ConnectorSinkPlan; import org.apache.doris.connector.spi.write.ConnectorWritePlanProvider; import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.datasource.mvcc.PluginDrivenMvccSnapshot; @@ -50,6 +52,7 @@ import org.apache.doris.planner.DataSink; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.PluginDrivenTableSink; +import org.apache.doris.thrift.TDataSink; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; @@ -248,11 +251,10 @@ public void mergePluginArmRunsMaterializedNameLoopSoBeResolvesOperationColumn() } @Test - public void rowLevelDmlThreadsMvccReadSnapshotPinOntoTheWriteHandle() { + public void rowLevelDmlThreadsMvccReadSnapshotPinOntoTheWriteHandle() throws Exception { // Fix B: the write handle must carry the statement's pinned MVCC read snapshot, so a DELETE/MERGE - // re-derives its deletes from the SAME snapshot its scan read. The pin decision itself is unit-tested in - // PluginDrivenScanNodeMvccPinTest; this pins that the row-level-DML helper actually wires it onto the - // write handle (a mutation dropping the applyMvccSnapshotPin call would leave the raw, unpinned handle). + // re-derives its deletes from the SAME snapshot its scan read. Binding is intentionally late because + // that is the first point where the exact target branch is available. Plugin plugin = pluginTable(); ConnectorMvccSnapshot connectorSnapshot = Mockito.mock(ConnectorMvccSnapshot.class); PluginDrivenMvccSnapshot pinned = new PluginDrivenMvccSnapshot( @@ -271,13 +273,18 @@ public void rowLevelDmlThreadsMvccReadSnapshotPinOntoTheWriteHandle() { PlanTranslatorContext context = new PlanTranslatorContext(); PhysicalPlanTranslator translator = new PhysicalPlanTranslator(context, null); + translator.visitPhysicalExternalRowLevelDeleteSink(sink, context); + PluginDrivenTableSink pluginSink = capturePluginSink(childFragment); try (MockedStatic mvcc = Mockito.mockStatic(MvccUtil.class)) { - mvcc.when(() -> MvccUtil.getSnapshotFromContext(plugin.table)).thenReturn(Optional.of(pinned)); - translator.visitPhysicalExternalRowLevelDeleteSink(sink, context); + mvcc.when(() -> MvccUtil.getSnapshotFromContext( + plugin.table, Optional.empty(), Optional.empty())).thenReturn(Optional.of(pinned)); + pluginSink.bindDataSink(Optional.empty()); } - PluginDrivenTableSink pluginSink = capturePluginSink(childFragment); - Assertions.assertSame(pinnedHandle, Deencapsulation.getField(pluginSink, "tableHandle"), + ConnectorWritePlanProvider provider = Deencapsulation.getField(pluginSink, "writePlanProvider"); + ArgumentCaptor handle = ArgumentCaptor.forClass(ConnectorWriteHandle.class); + Mockito.verify(provider).planWrite(Mockito.any(), handle.capture()); + Assertions.assertSame(pinnedHandle, handle.getValue().getTableHandle(), "the row-level DML write handle must carry the snapshot-pinned table handle (Fix B), not the raw" + " latest-read handle"); } @@ -362,6 +369,8 @@ private static Plugin pluginTable() { // provider and admits on ITS supportedOperations containing DELETE/MERGE. Mockito.when(provider.supportedOperations()) .thenReturn(EnumSet.of(WriteOperation.DELETE, WriteOperation.MERGE)); + Mockito.when(provider.planWrite(Mockito.any(), Mockito.any())) + .thenReturn(new ConnectorSinkPlan(new TDataSink())); Mockito.when(connector.getMetadata(Mockito.any())).thenReturn(metadata); Mockito.when(metadata.getTableHandle(Mockito.any(), Mockito.any(), Mockito.any())) .thenReturn(Optional.of(handle)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkBindingTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkBindingTest.java index 5d8f4f28052a1b..4f80f34b6fe393 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkBindingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkBindingTest.java @@ -17,18 +17,26 @@ package org.apache.doris.planner; +import org.apache.doris.analysis.TableScanParams; import org.apache.doris.common.AnalysisException; import org.apache.doris.connector.ConnectorSessionBuilder; +import org.apache.doris.connector.spi.ConnectorMetadata; import org.apache.doris.connector.spi.ConnectorSession; import org.apache.doris.connector.spi.handle.ConnectorTableHandle; import org.apache.doris.connector.spi.handle.ConnectorWriteHandle; +import org.apache.doris.connector.spi.mvcc.ConnectorMvccSnapshot; import org.apache.doris.connector.spi.write.ConnectorSinkPlan; import org.apache.doris.connector.spi.write.ConnectorWritePlanProvider; +import org.apache.doris.datasource.mvcc.MvccUtil; +import org.apache.doris.datasource.mvcc.PluginDrivenMvccSnapshot; +import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; import org.apache.doris.nereids.trees.plans.commands.insert.PluginDrivenInsertCommandContext; import org.apache.doris.thrift.TDataSink; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import java.util.Collections; import java.util.HashMap; @@ -88,6 +96,40 @@ public void absentContextDefaultsToNonOverwriteEmptySpec() throws AnalysisExcept "a plain INSERT must pass an empty static partition spec"); } + @Test + public void branchTargetUsesItsExactVersionAwareSnapshotPin() throws AnalysisException { + RecordingWritePlanProvider provider = new RecordingWritePlanProvider(); + ConnectorSession session = ConnectorSessionBuilder.create().withCatalogName("iceberg").build(); + ConnectorTableHandle baseHandle = Mockito.mock(ConnectorTableHandle.class); + ConnectorTableHandle pinnedHandle = Mockito.mock(ConnectorTableHandle.class); + ConnectorMetadata metadata = Mockito.mock(ConnectorMetadata.class); + PluginDrivenExternalTable table = Mockito.mock(PluginDrivenExternalTable.class); + ConnectorMvccSnapshot connectorSnapshot = Mockito.mock(ConnectorMvccSnapshot.class); + PluginDrivenMvccSnapshot snapshot = new PluginDrivenMvccSnapshot( + connectorSnapshot, Collections.emptyMap(), Collections.emptyMap()); + Mockito.when(metadata.applySnapshot(session, baseHandle, connectorSnapshot)).thenReturn(pinnedHandle); + PluginDrivenTableSink sink = new PluginDrivenTableSink(table, provider, session, baseHandle, + Collections.emptyList(), null, null, false, metadata); + PluginDrivenInsertCommandContext ctx = new PluginDrivenInsertCommandContext(); + ctx.setBranchName(Optional.of("audit")); + + try (MockedStatic mvcc = Mockito.mockStatic(MvccUtil.class)) { + mvcc.when(() -> MvccUtil.getSnapshotFromContext( + Mockito.eq(table), Mockito.eq(Optional.empty()), Mockito.any())) + .thenAnswer(invocation -> { + Optional selector = invocation.getArgument(2); + Assertions.assertEquals(TableScanParams.BRANCH, + selector.orElseThrow().getParamType()); + Assertions.assertEquals(Collections.singletonList("audit"), + selector.orElseThrow().getListParams()); + return Optional.of(snapshot); + }); + sink.bindDataSink(Optional.of(ctx)); + } + + Assertions.assertSame(pinnedHandle, provider.capturedHandle.getTableHandle()); + } + private static PluginDrivenTableSink newPlanProviderSink(ConnectorWritePlanProvider provider) { ConnectorSession session = ConnectorSessionBuilder.create().withCatalogName("mc_cat").build(); ConnectorTableHandle tableHandle = new ConnectorTableHandle() { }; diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java index f1f0bd14033c47..5075f916251441 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java @@ -398,4 +398,13 @@ public void testFileCacheQueryLimitBytesToThrift() throws Exception { Assertions.assertTrue(queryOptions.isSetFileCacheQueryLimitBytes()); Assertions.assertEquals(262144L, queryOptions.getFileCacheQueryLimitBytes()); } + + @Test + public void testCoordinatorThriftLimitPropagatesToBackends() { + TQueryOptions queryOptions = new SessionVariable().toThrift(); + + Assertions.assertTrue(queryOptions.isSetCoordinatorThriftMaxMessageSize()); + Assertions.assertEquals(Config.thrift_max_message_size, + queryOptions.getCoordinatorThriftMaxMessageSize()); + } } diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java index 1772f5c72b8dca..bf327e439c19b3 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java @@ -48,12 +48,14 @@ import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Base64; import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; import java.util.UUID; @@ -229,8 +231,8 @@ public UploadPartResult uploadPart(String remotePath, String uploadId, int partN try { AzureUri uri = AzureUri.parse(remotePath); BlockBlobClient blockBlobClient = getClient().getBlobContainerClient(uri.container()) - .getBlobClient(multipartTempKey(uri.key(), uploadId)).getBlockBlobClient(); - String blockId = toBlockId(partNum); + .getBlobClient(uri.key()).getBlockBlobClient(); + String blockId = multipartBlockId(uploadId, partNum); blockBlobClient.stageBlock(blockId, body.content(), body.contentLength()); return new UploadPartResult(partNum, blockId); } catch (BlobStorageException e) { @@ -251,24 +253,11 @@ public void completeMultipartUpload(String remotePath, String uploadId, boolean exactBlockIds = !sorted.isEmpty() && sorted.stream() .allMatch(part -> part.etag() != null && !part.etag().isEmpty()); for (UploadPartResult part : sorted) { - // A mixed-version upload must use one namespace consistently; new BEs also stage legacy IDs. + // Missing IDs identify an older BE upload, whose blocks use the legacy namespace. blockIds.add(exactBlockIds ? part.etag() : toBlockId(part.partNumber())); } - String commitKey = exactBlockIds ? multipartTempKey(uri.key(), uploadId) : uri.key(); - BlobClient commitBlob = containerClient.getBlobClient(commitKey); - commitBlob.getBlockBlobClient().commitBlockList(blockIds); - if (exactBlockIds) { - BlobClient targetBlob = containerClient.getBlobClient(uri.key()); - // The temporary blob is the provider-visible writer fence; only a completed copy publishes it. - targetBlob.beginCopy(commitBlob.getBlobUrl(), null).waitForCompletion(); - try { - commitBlob.delete(); - } catch (BlobStorageException cleanupFailure) { - // Publication already succeeded; cleanup must not make a retry overwrite a newer writer. - LOG.warn("Azure multipart temporary blob cleanup failed after publication: {}", - cleanupFailure.getMessage()); - } - } + // Put Block List is the atomic publication point and does not expose a staging blob to scans. + containerClient.getBlobClient(uri.key()).getBlockBlobClient().commitBlockList(blockIds); } catch (BlobStorageException e) { throw new IOException("completeMultipartUpload failed for " + remotePath + ": " + e.getMessage(), e); @@ -277,18 +266,8 @@ public void completeMultipartUpload(String remotePath, String uploadId, @Override public void abortMultipartUpload(String remotePath, String uploadId) throws IOException { - try { - AzureUri uri = AzureUri.parse(remotePath); - // Abort is scoped to this writer's temporary blob. Legacy final-blob blocks are left to expire, - // because recommitting their IDs can select a staged replacement for committed user data. - getClient().getBlobContainerClient(uri.container()) - .getBlobClient(multipartTempKey(uri.key(), uploadId)).deleteIfExists(); - } catch (BlobStorageException e) { - // Best-effort: log and swallow rather than mask the original failure that - // triggered the abort path. Uncommitted blocks will be GC'd by the service. - LOG.warn("abortMultipartUpload best-effort cleanup failed for {}: {}", - remotePath, e.getMessage()); - } + // Azure has no per-upload abort for blocks staged on a shared blob. A no-op is the only + // choice that cannot delete or recommit the last successfully published value. } /** @@ -518,7 +497,8 @@ private static String toBlockId(int partNum) { return Base64.getEncoder().encodeToString(bytes); } - static String multipartTempKey(String key, String uploadId) { - return key + ".__doris_multipart/" + uploadId; + static String multipartBlockId(String uploadId, int partNum) { + String rawId = String.format(Locale.ROOT, "%s:%010d", uploadId, partNum); + return Base64.getEncoder().encodeToString(rawId.getBytes(StandardCharsets.UTF_8)); } } diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java index 4e687917e034c4..59b43f010198ae 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java @@ -380,28 +380,22 @@ void deleteObjectsByKeys_attachesPerKeyExceptionsAsSuppressed() throws Exception // ------------------------------------------------------------------ @Test - void multipartTempKey_isolatesSameTargetWriters() { - Assertions.assertEquals("stage/blob.__doris_multipart/upload-a", - AzureObjStorage.multipartTempKey("stage/blob", "upload-a")); + void multipartBlockId_isolatesSameTargetWritersWithFixedLengthIds() { Assertions.assertNotEquals( - AzureObjStorage.multipartTempKey("stage/blob", "upload-a"), - AzureObjStorage.multipartTempKey("stage/blob", "upload-b")); + AzureObjStorage.multipartBlockId("upload-a", 1), + AzureObjStorage.multipartBlockId("upload-b", 1)); + Assertions.assertEquals( + AzureObjStorage.multipartBlockId("upload-a", 1).length(), + AzureObjStorage.multipartBlockId("upload-a", 999).length()); } @Test void completeMultipartUpload_usesExactBlockIdsReportedByBe() throws Exception { com.azure.storage.blob.specialized.BlockBlobClient blockClient = Mockito.mock(com.azure.storage.blob.specialized.BlockBlobClient.class); - BlobClient tempBlob = Mockito.mock(BlobClient.class); - Mockito.when(tempBlob.getBlockBlobClient()).thenReturn(blockClient); - Mockito.when(tempBlob.getBlobUrl()).thenReturn("https://account/container/temp"); BlobClient targetBlob = Mockito.mock(BlobClient.class); - com.azure.core.util.polling.SyncPoller poller = - Mockito.mock(com.azure.core.util.polling.SyncPoller.class); - Mockito.when(targetBlob.beginCopy("https://account/container/temp", null)).thenReturn(poller); + Mockito.when(targetBlob.getBlockBlobClient()).thenReturn(blockClient); BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); - Mockito.when(containerClient.getBlobClient( - "stage/blob.__doris_multipart/be-generated-upload-id")).thenReturn(tempBlob); Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(targetBlob); BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); @@ -416,8 +410,7 @@ void completeMultipartUpload_usesExactBlockIdsReportedByBe() throws Exception { new UploadPartResult(1, firstBlockId))); Mockito.verify(blockClient).commitBlockList(Arrays.asList(firstBlockId, secondBlockId)); - Mockito.verify(poller).waitForCompletion(); - Mockito.verify(tempBlob).delete(); + Mockito.verify(targetBlob, Mockito.never()).beginCopy(Mockito.anyString(), Mockito.isNull()); } @Test @@ -462,11 +455,8 @@ void completeMultipartUpload_usesLegacyNamespaceWhenAnyBlockIdIsMissing() throws } @Test - void abortMultipartUpload_deletesOnlyWriterTemporaryBlob() throws Exception { - BlobClient tempBlob = Mockito.mock(BlobClient.class); + void abortMultipartUpload_doesNotMutatePublishedTarget() throws Exception { BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); - Mockito.when(containerClient.getBlobClient( - "stage/blob.__doris_multipart/uploadId")).thenReturn(tempBlob); BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); @@ -476,8 +466,8 @@ void abortMultipartUpload_deletesOnlyWriterTemporaryBlob() throws Exception { storage.abortMultipartUpload( "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", "uploadId"); - Mockito.verify(tempBlob).deleteIfExists(); Mockito.verify(containerClient, Mockito.never()).getBlobClient("stage/blob"); + Mockito.verifyNoInteractions(containerClient); } // ------------------------------------------------------------------ diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index ec615dcac47bd8..9b498c8c0a6c3f 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -512,6 +512,8 @@ struct TQueryOptions { 226: optional bool enable_prune_nested_column = false; 227: optional bool new_version_bitmap_op_count = false; 228: optional bool enable_local_exchange_before_streaming_agg = false; + // FE is the receiver of fragment reports, so BE must also honor its message limit. + 229: optional i32 coordinator_thrift_max_message_size; // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query. From febfd1a2dfbb10a564d2e6c011b0487693be1b39 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 3 Aug 2026 10:48:49 +0800 Subject: [PATCH 07/29] [fix](iceberg) Match object store hash layout --- .../action/IcebergRemoveOrphanFilesAction.java | 4 ++-- .../IcebergRemoveOrphanFilesActionTest.java | 15 +++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java index cef3b5d2a7ebed..b4d057c3c932c0 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java @@ -375,7 +375,7 @@ private static ScanScope objectStore(String root, String tableLocation) { ? names.get(names.size() - 2) + "/" + names.get(names.size() - 1) : names.get(names.size() - 1); return new ScanScope(root, Pattern.compile( - "[01]{4}/[01]{4}/[01]{4}/[01]{4}/[01]{4}/" + "[01]{4}/[01]{4}/[01]{4}/[01]{8}/" + Pattern.quote(context) + "/.+")); } @@ -393,7 +393,7 @@ private boolean owns(String candidate) { } String relative = child.path.substring(Math.min(child.path.length(), parent.path.length())); relative = relative.startsWith("/") ? relative.substring(1) : relative; - // Iceberg's ObjectStoreLocationProvider prefixes five 4-bit hash directories before context. + // Iceberg 1.10.1 splits its 20-bit hash into 4/4/4/8-bit directories before context. return ownedRelativePath.matcher(relative).matches(); } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java index c48eac67816251..08d9018f229631 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java @@ -228,16 +228,15 @@ public void objectStoreOwnershipExcludesNeighborTableAndFolderRootFailsClosed(@T temp.resolve("lower-precedence-object-path").toUri().toString()); Files.createDirectories(objectRoot); Table objectTable = createTable(temp.resolve("object-metadata"), objectProperties); - Path ownOrphan = createOldFile(objectRoot.resolve( - "0000/0001/0010/0011/0100/" + temp.getFileName() + "/object-metadata/own.parquet")); - Path neighborFile = createOldFile(objectRoot.resolve( - "0000/0001/0010/0011/0100/" + temp.getFileName() + "/neighbor/live.parquet")); + String ownLocation = objectTable.locationProvider().newDataLocation("own.parquet"); + Path ownOrphan = createOldFile(Path.of(java.net.URI.create(ownLocation))); + Table neighborTable = createTable(temp.resolve("neighbor"), objectProperties); + String neighborLocation = neighborTable.locationProvider().newDataLocation("live.parquet"); + Path neighborFile = createOldFile(Path.of(java.net.URI.create(neighborLocation))); Assertions.assertTrue(IcebergRemoveOrphanFilesAction.isOwnedObjectStorePath( - "s3://bucket/shared/0000/0001/0010/0011/0100/db/table/file.parquet", - "s3://bucket/shared", "s3://bucket/warehouse/db/table")); + ownLocation, objectRoot.toUri().toString(), objectTable.location())); Assertions.assertFalse(IcebergRemoveOrphanFilesAction.isOwnedObjectStorePath( - "s3://bucket/shared/0000/0001/0010/0011/0100/db/neighbor/file.parquet", - "s3://bucket/shared", "s3://bucket/warehouse/db/table")); + neighborLocation, objectRoot.toUri().toString(), objectTable.location())); Path folderRoot = temp.resolve("folder-data"); Table folderTable = createTable(temp.resolve("folder-metadata"), From 5f8b26ff97856f262ed6b124937bdc5a01c709e3 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 3 Aug 2026 12:13:57 +0800 Subject: [PATCH 08/29] [fix](azure) Keep multipart block IDs rolling-compatible --- be/src/io/fs/azure_obj_storage_client.cpp | 18 +++++++--- .../io/fs/azure_obj_storage_client_test.cpp | 5 +++ .../filesystem/azure/AzureObjStorage.java | 11 ++++-- .../azure/AzureObjStorageExtensionTest.java | 34 +++++++++++++++++++ 4 files changed, 61 insertions(+), 7 deletions(-) diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp index 7396c355a0ca70..e52148ae877738 100644 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ b/be/src/io/fs/azure_obj_storage_client.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -34,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -65,10 +67,18 @@ std::string to_lower_ascii(std::string_view input) { } std::string encode_azure_block_id(std::string_view upload_id, int part_num) { - // Azure requires every block ID for one blob to have the same decoded length. - std::string raw_id = fmt::format("{}:{:010}", upload_id, part_num); - Aws::Utils::ByteBuffer bytes(reinterpret_cast(raw_id.data()), - raw_id.size()); + uint32_t upload_namespace = 0x811C9DC5U; + for (unsigned char byte : upload_id) { + upload_namespace = (upload_namespace ^ byte) * 0x01000193U; + } + uint32_t namespaced_part = upload_namespace + static_cast(part_num); + // Four decoded bytes remain compatible with legacy residual blocks while retaining an + // upload-specific namespace for part IDs. + std::array raw_id {}; + for (size_t i = 0; i < raw_id.size(); ++i) { + raw_id[i] = static_cast(namespaced_part >> (i * 8)); + } + Aws::Utils::ByteBuffer bytes(raw_id.data(), raw_id.size()); return Aws::Utils::HashingUtils::Base64Encode(bytes); } diff --git a/be/test/io/fs/azure_obj_storage_client_test.cpp b/be/test/io/fs/azure_obj_storage_client_test.cpp index 21de949500cf80..6fb70cac4719f0 100644 --- a/be/test/io/fs/azure_obj_storage_client_test.cpp +++ b/be/test/io/fs/azure_obj_storage_client_test.cpp @@ -26,6 +26,8 @@ #include "util/s3_util.h" #ifdef USE_AZURE +#include + #include #include #include @@ -37,10 +39,13 @@ namespace doris { #ifdef USE_AZURE TEST(AzureObjStorageClientMultipartHelperTest, isolates_uploads_with_fixed_length_block_ids) { + EXPECT_EQ("p3w3DA==", io::azure_multipart_block_id("upload-a", 1)); EXPECT_NE(io::azure_multipart_block_id("upload-a", 1), io::azure_multipart_block_id("upload-b", 1)); EXPECT_EQ(io::azure_multipart_block_id("upload-a", 1).size(), io::azure_multipart_block_id("upload-a", 999).size()); + EXPECT_EQ(4, Aws::Utils::HashingUtils::Base64Decode(io::azure_multipart_block_id("upload-a", 1)) + .GetLength()); } using namespace Azure::Storage::Blobs; diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java index bf327e439c19b3..bdd1e11a863140 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java @@ -55,7 +55,6 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; import java.util.UUID; @@ -498,7 +497,13 @@ private static String toBlockId(int partNum) { } static String multipartBlockId(String uploadId, int partNum) { - String rawId = String.format(Locale.ROOT, "%s:%010d", uploadId, partNum); - return Base64.getEncoder().encodeToString(rawId.getBytes(StandardCharsets.UTF_8)); + int uploadNamespace = 0x811C9DC5; + for (byte value : uploadId.getBytes(StandardCharsets.UTF_8)) { + uploadNamespace = (uploadNamespace ^ (value & 0xFF)) * 0x01000193; + } + int namespacedPart = uploadNamespace + partNum; + // Match the legacy four-byte length so a retry can coexist with pre-upgrade residual blocks. + byte[] rawId = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN).putInt(namespacedPart).array(); + return Base64.getEncoder().encodeToString(rawId); } } diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java index 59b43f010198ae..70dee75286e40c 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java @@ -19,6 +19,7 @@ import org.apache.doris.filesystem.UploadPartResult; import org.apache.doris.filesystem.spi.RemoteObjects; +import org.apache.doris.filesystem.spi.RequestBody; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; @@ -30,9 +31,11 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.time.OffsetDateTime; import java.util.Arrays; +import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -381,12 +384,43 @@ void deleteObjectsByKeys_attachesPerKeyExceptionsAsSuppressed() throws Exception @Test void multipartBlockId_isolatesSameTargetWritersWithFixedLengthIds() { + Assertions.assertEquals("p3w3DA==", AzureObjStorage.multipartBlockId("upload-a", 1)); Assertions.assertNotEquals( AzureObjStorage.multipartBlockId("upload-a", 1), AzureObjStorage.multipartBlockId("upload-b", 1)); Assertions.assertEquals( AzureObjStorage.multipartBlockId("upload-a", 1).length(), AzureObjStorage.multipartBlockId("upload-a", 999).length()); + Assertions.assertEquals(4, + Base64.getDecoder().decode(AzureObjStorage.multipartBlockId("upload-a", 1)).length); + } + + @Test + void uploadPart_acceptsLegacyResidualBlockLength() throws Exception { + com.azure.storage.blob.specialized.BlockBlobClient blockClient = + Mockito.mock(com.azure.storage.blob.specialized.BlockBlobClient.class); + int legacyDecodedLength = Base64.getDecoder().decode("AQAAAA==").length; + Mockito.doAnswer(invocation -> { + String blockId = invocation.getArgument(0); + if (Base64.getDecoder().decode(blockId).length != legacyDecodedLength) { + throw new IllegalStateException("Azure would reject a different block ID length"); + } + return null; + }).when(blockClient).stageBlock( + Mockito.anyString(), Mockito.any(java.io.InputStream.class), Mockito.anyLong()); + BlobClient blobClient = Mockito.mock(BlobClient.class); + Mockito.when(blobClient.getBlockBlobClient()).thenReturn(blockClient); + BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); + Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); + BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); + Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); + TestableAzureObjStorage storage = new TestableAzureObjStorage(buildBasicProps(), serviceClient); + + UploadPartResult result = storage.uploadPart( + "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", + "new-upload-id", 1, RequestBody.of(new ByteArrayInputStream(new byte[]{1}), 1)); + + Assertions.assertEquals(legacyDecodedLength, Base64.getDecoder().decode(result.etag()).length); } @Test From 82a09e1711baa227932891ff89b90449fe577a76 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 3 Aug 2026 12:15:50 +0800 Subject: [PATCH 09/29] [test](azure) Seed legacy residual block ID --- .../filesystem/azure/AzureObjStorageExtensionTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java index 70dee75286e40c..af5822e02cc78e 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java @@ -34,6 +34,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.time.OffsetDateTime; +import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; import java.util.Collections; @@ -399,12 +400,14 @@ void multipartBlockId_isolatesSameTargetWritersWithFixedLengthIds() { void uploadPart_acceptsLegacyResidualBlockLength() throws Exception { com.azure.storage.blob.specialized.BlockBlobClient blockClient = Mockito.mock(com.azure.storage.blob.specialized.BlockBlobClient.class); - int legacyDecodedLength = Base64.getDecoder().decode("AQAAAA==").length; + List stagedBlockIds = new ArrayList<>(Collections.singletonList("AQAAAA==")); Mockito.doAnswer(invocation -> { String blockId = invocation.getArgument(0); - if (Base64.getDecoder().decode(blockId).length != legacyDecodedLength) { + int requiredDecodedLength = Base64.getDecoder().decode(stagedBlockIds.get(0)).length; + if (Base64.getDecoder().decode(blockId).length != requiredDecodedLength) { throw new IllegalStateException("Azure would reject a different block ID length"); } + stagedBlockIds.add(blockId); return null; }).when(blockClient).stageBlock( Mockito.anyString(), Mockito.any(java.io.InputStream.class), Mockito.anyLong()); @@ -420,7 +423,7 @@ void uploadPart_acceptsLegacyResidualBlockLength() throws Exception { "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", "new-upload-id", 1, RequestBody.of(new ByteArrayInputStream(new byte[]{1}), 1)); - Assertions.assertEquals(legacyDecodedLength, Base64.getDecoder().decode(result.etag()).length); + Assertions.assertEquals(Arrays.asList("AQAAAA==", result.etag()), stagedBlockIds); } @Test From 2771934b8051c2735d21d0bde02728a6f41595e3 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 3 Aug 2026 15:41:40 +0800 Subject: [PATCH 10/29] [fix](azure) Fence multipart writers with blob leases --- be/src/io/fs/azure_obj_storage_client.cpp | 89 +++++++++-- be/test/io/client/s3_file_system_test.cpp | 13 +- .../io/fs/azure_obj_storage_client_test.cpp | 35 +++-- .../rate_limited_obj_storage_client_test.cpp | 10 +- .../filesystem/azure/AzureObjStorage.java | 90 ++++++++++- .../azure/AzureObjStorageExtensionTest.java | 143 +++++++++++++++++- 6 files changed, 325 insertions(+), 55 deletions(-) diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp index e52148ae877738..8c0a8201671a2b 100644 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ b/be/src/io/fs/azure_obj_storage_client.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -39,7 +40,6 @@ #include #include #include -#include #include #include "common/exception.h" @@ -49,7 +49,6 @@ #include "io/fs/obj_storage_client.h" #include "util/bvar_helper.h" #include "util/s3_util.h" -#include "util/uuid_generator.h" using namespace Azure::Storage::Blobs; @@ -72,8 +71,8 @@ std::string encode_azure_block_id(std::string_view upload_id, int part_num) { upload_namespace = (upload_namespace ^ byte) * 0x01000193U; } uint32_t namespaced_part = upload_namespace + static_cast(part_num); - // Four decoded bytes remain compatible with legacy residual blocks while retaining an - // upload-specific namespace for part IDs. + // Four decoded bytes remain compatible with legacy residual blocks. Writer isolation is + // enforced by the target blob lease because no 32-bit namespace can identify every upload. std::array raw_id {}; for (size_t i = 0; i < raw_id.size(); ++i) { raw_id[i] = static_cast(namespaced_part >> (i * 8)); @@ -82,6 +81,17 @@ std::string encode_azure_block_id(std::string_view upload_id, int part_num) { return Aws::Utils::HashingUtils::Base64Encode(bytes); } +constexpr std::string_view MULTIPART_LEASE_PREFIX = "doris-azure-lease-v1:"; +constexpr std::chrono::seconds MULTIPART_LEASE_DURATION {60}; + +std::optional azure_multipart_lease_id(std::string_view upload_id) { + if (upload_id.starts_with(MULTIPART_LEASE_PREFIX) && + upload_id.size() > MULTIPART_LEASE_PREFIX.size()) { + return upload_id.substr(MULTIPART_LEASE_PREFIX.size()); + } + return std::nullopt; +} + // Rate limiting is applied by RateLimitedObjStorageClient, the decorator that // S3ClientFactory wraps around this client when the bucket is subject to limiting. @@ -212,11 +222,27 @@ struct AzureBatchDeleter { ObjectStorageUploadResponse AzureObjStorageClient::create_multipart_upload( const ObjectStoragePathOptions& opts) { - std::stringstream upload_id; - upload_id << UUIDGenerator::instance()->next_uuid(); + auto target_blob = _client->GetBlobClient(opts.key); + auto target_client = target_blob.AsBlockBlobClient(); + std::string lease_id = BlobLeaseClient::CreateUniqueLeaseId(); + std::string upload_id = fmt::format("{}{}", MULTIPART_LEASE_PREFIX, lease_id); + auto resp = do_azure_client_call( + [&]() { + uint8_t empty = 0; + Azure::Core::IO::MemoryBodyStream empty_body(&empty, 0); + // The reservation makes an absent blob leaseable but remains uncommitted and + // invisible to normal listings until Put Block List publishes the real data. + target_client.StageBlock(azure_multipart_block_id(upload_id, 0), empty_body); + auto lease = + BlobLeaseClient(target_blob, lease_id).Acquire(MULTIPART_LEASE_DURATION); + upload_id = fmt::format("{}{}", MULTIPART_LEASE_PREFIX, lease.Value.LeaseId); + }, + opts, _tls_debug_context); return ObjectStorageUploadResponse { - .resp = ObjectStorageResponse::OK(), - .upload_id = upload_id.str(), + .resp = resp, + .upload_id = resp.status.code == ErrorCode::OK + ? std::make_optional(std::move(upload_id)) + : std::nullopt, }; } @@ -235,7 +261,8 @@ ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStora std::string_view stream, int part_num) { DCHECK(opts.upload_id.has_value()); - auto client = _client->GetBlockBlobClient(opts.key); + auto target_blob = _client->GetBlobClient(opts.key); + auto client = target_blob.AsBlockBlobClient(); std::string block_id = azure_multipart_block_id(*opts.upload_id, part_num); auto resp = do_azure_client_call( [&]() { @@ -243,8 +270,15 @@ ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStora reinterpret_cast(stream.data()), stream.size()); // The blockId must be base64 encoded SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); - // Upload-scoped IDs make a conflicting writer fail closed instead of selecting its blocks. - client.StageBlock(block_id, memory_body); + auto lease_id = azure_multipart_lease_id(*opts.upload_id); + if (lease_id.has_value()) { + BlobLeaseClient(target_blob, std::string(*lease_id)).Renew(); + StageBlockOptions stage_opts; + stage_opts.AccessConditions.LeaseId = std::string(*lease_id); + client.StageBlock(block_id, memory_body, stage_opts); + } else { + client.StageBlock(block_id, memory_body); + } }, opts, _tls_debug_context); return ObjectStorageUploadResponse { @@ -258,24 +292,51 @@ ObjectStorageResponse AzureObjStorageClient::complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) { DCHECK(opts.upload_id.has_value()); - auto target_client = _client->GetBlockBlobClient(opts.key); + auto target_blob = _client->GetBlobClient(opts.key); + auto target_client = target_blob.AsBlockBlobClient(); std::vector string_block_ids; std::ranges::transform(completed_parts, std::back_inserter(string_block_ids), [&opts](const ObjectCompleteMultiPart& i) { return azure_multipart_block_id(*opts.upload_id, i.part_num); }); - return do_azure_client_call( + auto resp = do_azure_client_call( [&]() { SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); // Put Block List atomically replaces the committed blob; no scan-visible staging blob exists. - target_client.CommitBlockList(string_block_ids); + auto lease_id = azure_multipart_lease_id(*opts.upload_id); + if (lease_id.has_value()) { + BlobLeaseClient(target_blob, std::string(*lease_id)).Renew(); + CommitBlockListOptions commit_opts; + commit_opts.AccessConditions.LeaseId = std::string(*lease_id); + target_client.CommitBlockList(string_block_ids, commit_opts); + } else { + target_client.CommitBlockList(string_block_ids); + } }, opts, _tls_debug_context); + if (resp.status.code == ErrorCode::OK) { + if (auto lease_id = azure_multipart_lease_id(*opts.upload_id); lease_id.has_value()) { + auto release_resp = do_azure_client_call( + [&]() { BlobLeaseClient(target_blob, std::string(*lease_id)).Release(); }, opts, + _tls_debug_context); + if (release_resp.status.code != ErrorCode::OK) { + LOG(WARNING) << "Azure multipart commit succeeded but its finite lease could not " + "be released; it will expire automatically"; + } + } + } + return resp; } ObjectStorageResponse AzureObjStorageClient::abort_multipart_upload( const ObjectStoragePathOptions& opts) { DCHECK(opts.upload_id.has_value()); + if (auto lease_id = azure_multipart_lease_id(*opts.upload_id); lease_id.has_value()) { + auto target_blob = _client->GetBlobClient(opts.key); + return do_azure_client_call( + [&]() { BlobLeaseClient(target_blob, std::string(*lease_id)).Release(); }, opts, + _tls_debug_context); + } // Azure cannot delete one upload's uncommitted blocks without changing the committed blob. // Leaving them to service GC preserves the last successfully published value. return ObjectStorageResponse::OK(); diff --git a/be/test/io/client/s3_file_system_test.cpp b/be/test/io/client/s3_file_system_test.cpp index 375ff3ff57f8e2..2745e87a5dd6e9 100644 --- a/be/test/io/client/s3_file_system_test.cpp +++ b/be/test/io/client/s3_file_system_test.cpp @@ -928,9 +928,8 @@ TEST_F(S3FileSystemTest, RateLimiterGetDownloadTest) { // Test: S3 rate limiter for PUT operations - multipart upload TEST_F(S3FileSystemTest, RateLimiterPutMultipartTest) { - // Skip if using Azure provider - Azure's create_multipart_upload is a no-op and doesn't - // consume rate limiter quota, while S3's CreateMultipartUpload does. This causes different - // failure timing that makes the test assertions invalid for Azure. + // This test asserts the S3 provider's exact multipart request/failure sequence; Azure uses + // lease coordination and therefore has different failure timing. if (config_->get_provider() == "AZURE") { GTEST_SKIP() << "This test relies on S3-specific multipart upload quota consumption " "behavior, not applicable for Azure"; @@ -1437,9 +1436,8 @@ TEST_F(S3FileSystemTest, RateLimiterGetDeleteDirectoryListTest) { // Test: S3 rate limiter for PUT operations - multipart upload with UploadPart failure TEST_F(S3FileSystemTest, RateLimiterPutMultipartUploadPartFailureTest) { - // Skip if using Azure provider - Azure's create_multipart_upload is a no-op and doesn't - // consume rate limiter quota, while S3's CreateMultipartUpload does. This causes different - // failure timing that makes the test assertions invalid for Azure. + // This test asserts the S3 provider's exact multipart request/failure sequence; Azure uses + // lease coordination and therefore has different failure timing. if (config_->get_provider() == "AZURE") { GTEST_SKIP() << "This test relies on S3-specific multipart upload quota consumption " "behavior, not applicable for Azure"; @@ -1866,8 +1864,7 @@ TEST_F(S3FileSystemTest, RateLimiterPutDeleteDirectoryDeleteObjectsTest) { // Test: S3 CreateMultipartUpload failure - simulates error when initiating multipart upload TEST_F(S3FileSystemTest, CreateMultipartUploadFailureTest) { - // Skip if using Azure provider - SyncPoint mechanism is S3-specific - // Also, Azure's create_multipart_upload is a no-op that always succeeds + // Skip if using Azure provider because the SyncPoint mechanism is S3-specific. if (config_->get_provider() == "AZURE") { GTEST_SKIP() << "This test uses S3-specific SyncPoint mechanism and multipart semantics, " "not applicable for Azure"; diff --git a/be/test/io/fs/azure_obj_storage_client_test.cpp b/be/test/io/fs/azure_obj_storage_client_test.cpp index 6fb70cac4719f0..dc7a2314dddb01 100644 --- a/be/test/io/fs/azure_obj_storage_client_test.cpp +++ b/be/test/io/fs/azure_obj_storage_client_test.cpp @@ -38,10 +38,12 @@ namespace doris { #ifdef USE_AZURE -TEST(AzureObjStorageClientMultipartHelperTest, isolates_uploads_with_fixed_length_block_ids) { +TEST(AzureObjStorageClientMultipartHelperTest, fixed_length_namespace_requires_target_lease) { EXPECT_EQ("p3w3DA==", io::azure_multipart_block_id("upload-a", 1)); - EXPECT_NE(io::azure_multipart_block_id("upload-a", 1), - io::azure_multipart_block_id("upload-b", 1)); + EXPECT_EQ("Sc7grw==", io::azure_multipart_block_id( + "doris-azure-lease-v1:09492e3d-e231-4ed9-bf84-b6fc772cda54", 1)); + EXPECT_EQ("Sc7grw==", io::azure_multipart_block_id( + "doris-azure-lease-v1:06996d15-1c2e-4ddd-8853-43816ea84a07", 1)); EXPECT_EQ(io::azure_multipart_block_id("upload-a", 1).size(), io::azure_multipart_block_id("upload-a", 999).size()); EXPECT_EQ(4, Aws::Utils::HashingUtils::Base64Decode(io::azure_multipart_block_id("upload-a", 1)) @@ -172,12 +174,11 @@ TEST_F(AzureObjStorageClientTest, delete_objects_recursively) { } TEST_F(AzureObjStorageClientTest, abort_multipart_upload_leaves_no_visible_blob) { - io::ObjectStoragePathOptions opts; + io::ObjectStoragePathOptions opts {.key = "AzureObjStorageClientTest/abort_multipart_upload"}; auto create_response = AzureObjStorageClientTest::obj_storage_client->create_multipart_upload(opts); ASSERT_EQ(create_response.resp.status.code, ErrorCode::OK); ASSERT_TRUE(create_response.upload_id.has_value()); - opts.key = "AzureObjStorageClientTest/abort_multipart_upload_" + *create_response.upload_id; opts.upload_id = create_response.upload_id; auto upload_response = @@ -194,16 +195,15 @@ TEST_F(AzureObjStorageClientTest, abort_multipart_upload_leaves_no_visible_blob) } TEST_F(AzureObjStorageClientTest, abort_multipart_upload_preserves_existing_put_blob) { - io::ObjectStoragePathOptions opts; + io::ObjectStoragePathOptions opts {.key = "AzureObjStorageClientTest/abort_preserves_put_blob"}; + auto put_response = AzureObjStorageClientTest::obj_storage_client->put_object(opts, "original"); + ASSERT_EQ(put_response.status.code, ErrorCode::OK); auto create_response = AzureObjStorageClientTest::obj_storage_client->create_multipart_upload(opts); ASSERT_EQ(create_response.resp.status.code, ErrorCode::OK); ASSERT_TRUE(create_response.upload_id.has_value()); - opts.key = "AzureObjStorageClientTest/abort_preserves_put_blob_" + *create_response.upload_id; opts.upload_id = create_response.upload_id; - auto put_response = AzureObjStorageClientTest::obj_storage_client->put_object(opts, "original"); - ASSERT_EQ(put_response.status.code, ErrorCode::OK); auto upload_response = AzureObjStorageClientTest::obj_storage_client->upload_part(opts, "replacement", 1); ASSERT_EQ(upload_response.resp.status.code, ErrorCode::OK); @@ -228,18 +228,21 @@ TEST_F(AzureObjStorageClientTest, concurrent_multipart_uploads_do_not_share_stag io::ObjectStoragePathOptions second = first; auto first_create = obj_storage_client->create_multipart_upload(first); auto second_create = obj_storage_client->create_multipart_upload(second); + ASSERT_EQ(first_create.resp.status.code, ErrorCode::OK); + ASSERT_NE(second_create.resp.status.code, ErrorCode::OK); ASSERT_TRUE(first_create.upload_id.has_value()); - ASSERT_TRUE(second_create.upload_id.has_value()); first.upload_id = first_create.upload_id; - second.upload_id = second_create.upload_id; ASSERT_EQ(obj_storage_client->upload_part(first, "first", 1).resp.status.code, ErrorCode::OK); - ASSERT_EQ(obj_storage_client->upload_part(second, "second", 1).resp.status.code, ErrorCode::OK); ASSERT_EQ(obj_storage_client->complete_multipart_upload(first, {{.part_num = 1}}).status.code, ErrorCode::OK); - // Committing one writer discards all other uncommitted blocks on the blob. The loser must fail - // instead of publishing a mixture of blocks from two uploads. - EXPECT_NE(obj_storage_client->complete_multipart_upload(second, {{.part_num = 1}}).status.code, + + second_create = obj_storage_client->create_multipart_upload(second); + ASSERT_EQ(second_create.resp.status.code, ErrorCode::OK); + ASSERT_TRUE(second_create.upload_id.has_value()); + second.upload_id = second_create.upload_id; + ASSERT_EQ(obj_storage_client->upload_part(second, "second", 1).resp.status.code, ErrorCode::OK); + ASSERT_EQ(obj_storage_client->complete_multipart_upload(second, {{.part_num = 1}}).status.code, ErrorCode::OK); std::array contents {}; @@ -248,7 +251,7 @@ TEST_F(AzureObjStorageClientTest, concurrent_multipart_uploads_do_not_share_stag ->get_object(second, contents.data(), 0, contents.size(), &size_return) .status.code, ErrorCode::OK); - EXPECT_EQ(std::string_view(contents.data(), size_return), "first"); + EXPECT_EQ(std::string_view(contents.data(), size_return), "second"); EXPECT_EQ(obj_storage_client->delete_object(second).status.code, ErrorCode::OK); } #else diff --git a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp index 6120347beb0f7c..7b7686e187de56 100644 --- a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp +++ b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp @@ -478,7 +478,7 @@ TEST(RateLimitedObjStorageClientTest, recursive_delete_charges_one_put_qps) { EXPECT_EQ(4, fake->delete_objects_recursively_provider_calls); } -TEST(RateLimitedObjStorageClientTest, azure_noop_multipart_create_charges_one_put_qps) { +TEST(RateLimitedObjStorageClientTest, multipart_create_charges_one_logical_put_qps) { RateLimiterConfigGuard guard; config::enable_s3_rate_limiter = true; auto& manager = S3RateLimiterManager::instance(); @@ -486,20 +486,20 @@ TEST(RateLimitedObjStorageClientTest, azure_noop_multipart_create_charges_one_pu manager.bytes_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); auto fake = std::make_shared(); - // Azure implements create_multipart_upload as a provider-side no-op. - fake->create_multipart_upload_provider_calls_per_logical_call = 0; + // Provider coordination may need multiple requests, but admission remains per logical API call. + fake->create_multipart_upload_provider_calls_per_logical_call = 2; RateLimitedObjStorageClient client(fake); ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; EXPECT_EQ(0, client.create_multipart_upload(opts).resp.status.code); EXPECT_EQ(1, fake->create_multipart_upload_calls); - EXPECT_EQ(0, fake->create_multipart_upload_provider_calls); + EXPECT_EQ(2, fake->create_multipart_upload_provider_calls); auto resp = client.create_multipart_upload(opts); EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, resp.resp.status.code); EXPECT_EQ(0, resp.resp.http_code); EXPECT_EQ(1, fake->create_multipart_upload_calls); - EXPECT_EQ(0, fake->create_multipart_upload_provider_calls); + EXPECT_EQ(2, fake->create_multipart_upload_provider_calls); } TEST(RateLimitedObjStorageClientTest, presigned_url_bypasses_rate_limiters) { diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java index bdd1e11a863140..9955d1a247bbb0 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java @@ -25,6 +25,8 @@ import com.azure.core.http.rest.PagedIterable; import com.azure.core.http.rest.PagedResponse; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; import com.azure.identity.ClientSecretCredentialBuilder; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; @@ -33,10 +35,14 @@ import com.azure.storage.blob.BlobServiceClientBuilder; import com.azure.storage.blob.models.BlobItem; import com.azure.storage.blob.models.BlobProperties; +import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.models.ListBlobsOptions; +import com.azure.storage.blob.options.BlockBlobCommitBlockListOptions; import com.azure.storage.blob.sas.BlobSasPermission; import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; +import com.azure.storage.blob.specialized.BlobLeaseClient; +import com.azure.storage.blob.specialized.BlobLeaseClientBuilder; import com.azure.storage.blob.specialized.BlockBlobClient; import com.azure.storage.common.StorageSharedKeyCredential; import com.azure.storage.common.sas.SasProtocol; @@ -71,6 +77,8 @@ public class AzureObjStorage implements ObjStorage { private static final Logger LOG = LogManager.getLogger(AzureObjStorage.class); private static final int HTTP_NOT_FOUND = 404; + private static final int MULTIPART_LEASE_SECONDS = 60; + private static final String MULTIPART_LEASE_PREFIX = "doris-azure-lease-v1:"; /** Validity period for presigned (SAS) URLs, in seconds. */ private static final int SESSION_EXPIRE_SECONDS = 3600; @@ -220,8 +228,23 @@ public void copyObject(String srcPath, String dstPath) throws IOException { @Override public String initiateMultipartUpload(String remotePath) throws IOException { - // Azure block blobs don't have an explicit "initiate" API. - return UUID.randomUUID().toString(); + try { + AzureUri uri = AzureUri.parse(remotePath); + BlobClient blobClient = getClient().getBlobContainerClient(uri.container()) + .getBlobClient(uri.key()); + BlockBlobClient blockBlobClient = blobClient.getBlockBlobClient(); + String leaseId = UUID.randomUUID().toString(); + String uploadId = MULTIPART_LEASE_PREFIX + leaseId; + // A zero-byte uncommitted block materializes an absent target without exposing it to + // normal listings, so Azure can fence every later block operation with a blob lease. + blockBlobClient.stageBlock(multipartBlockId(uploadId, 0), BinaryData.fromBytes(new byte[0])); + String acquiredLeaseId = createLeaseClient(blobClient, leaseId) + .acquireLease(MULTIPART_LEASE_SECONDS); + return MULTIPART_LEASE_PREFIX + acquiredLeaseId; + } catch (BlobStorageException e) { + throw new IOException("initiateMultipartUpload failed for " + remotePath + + ": " + e.getMessage(), e); + } } @Override @@ -229,10 +252,18 @@ public UploadPartResult uploadPart(String remotePath, String uploadId, int partN RequestBody body) throws IOException { try { AzureUri uri = AzureUri.parse(remotePath); - BlockBlobClient blockBlobClient = getClient().getBlobContainerClient(uri.container()) - .getBlobClient(uri.key()).getBlockBlobClient(); + BlobClient blobClient = getClient().getBlobContainerClient(uri.container()) + .getBlobClient(uri.key()); + BlockBlobClient blockBlobClient = blobClient.getBlockBlobClient(); String blockId = multipartBlockId(uploadId, partNum); - blockBlobClient.stageBlock(blockId, body.content(), body.contentLength()); + String leaseId = multipartLeaseId(uploadId); + if (leaseId == null) { + blockBlobClient.stageBlock(blockId, body.content(), body.contentLength()); + } else { + createLeaseClient(blobClient, leaseId).renewLease(); + blockBlobClient.stageBlockWithResponse(blockId, body.content(), body.contentLength(), + null, leaseId, null, Context.NONE); + } return new UploadPartResult(partNum, blockId); } catch (BlobStorageException e) { throw new IOException("uploadPart failed for " + remotePath + " part " + partNum @@ -256,7 +287,26 @@ public void completeMultipartUpload(String remotePath, String uploadId, blockIds.add(exactBlockIds ? part.etag() : toBlockId(part.partNumber())); } // Put Block List is the atomic publication point and does not expose a staging blob to scans. - containerClient.getBlobClient(uri.key()).getBlockBlobClient().commitBlockList(blockIds); + BlobClient blobClient = containerClient.getBlobClient(uri.key()); + BlockBlobClient blockBlobClient = blobClient.getBlockBlobClient(); + String leaseId = multipartLeaseId(uploadId); + if (leaseId == null) { + blockBlobClient.commitBlockList(blockIds); + } else { + BlobLeaseClient leaseClient = createLeaseClient(blobClient, leaseId); + leaseClient.renewLease(); + BlobRequestConditions conditions = new BlobRequestConditions().setLeaseId(leaseId); + blockBlobClient.commitBlockListWithResponse( + new BlockBlobCommitBlockListOptions(blockIds).setRequestConditions(conditions), + null, Context.NONE); + try { + leaseClient.releaseLease(); + } catch (BlobStorageException e) { + // Publication is already durable; the finite lease will expire without + // turning a successful commit into a retry that could overwrite new data. + LOG.warn("Failed to release Azure multipart lease after commit for {}", remotePath, e); + } + } } catch (BlobStorageException e) { throw new IOException("completeMultipartUpload failed for " + remotePath + ": " + e.getMessage(), e); @@ -265,8 +315,32 @@ public void completeMultipartUpload(String remotePath, String uploadId, @Override public void abortMultipartUpload(String remotePath, String uploadId) throws IOException { - // Azure has no per-upload abort for blocks staged on a shared blob. A no-op is the only - // choice that cannot delete or recommit the last successfully published value. + String leaseId = multipartLeaseId(uploadId); + if (leaseId == null) { + // Azure cannot selectively remove legacy uncommitted blocks without rewriting the blob. + return; + } + try { + AzureUri uri = AzureUri.parse(remotePath); + BlobClient blobClient = getClient().getBlobContainerClient(uri.container()) + .getBlobClient(uri.key()); + createLeaseClient(blobClient, leaseId).releaseLease(); + } catch (BlobStorageException e) { + throw new IOException("abortMultipartUpload failed for " + remotePath + + ": " + e.getMessage(), e); + } + } + + protected BlobLeaseClient createLeaseClient(BlobClient blobClient, String leaseId) { + return new BlobLeaseClientBuilder().blobClient(blobClient).leaseId(leaseId).buildClient(); + } + + private static String multipartLeaseId(String uploadId) { + if (uploadId != null && uploadId.startsWith(MULTIPART_LEASE_PREFIX) + && uploadId.length() > MULTIPART_LEASE_PREFIX.length()) { + return uploadId.substring(MULTIPART_LEASE_PREFIX.length()); + } + return null; } /** diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java index af5822e02cc78e..bb9db84189d8ad 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java @@ -25,7 +25,11 @@ import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.models.BlobProperties; +import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.options.BlockBlobCommitBlockListOptions; +import com.azure.storage.blob.specialized.BlobLeaseClient; +import com.azure.storage.blob.specialized.BlockBlobClient; import com.azure.storage.common.StorageSharedKeyCredential; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -384,11 +388,12 @@ void deleteObjectsByKeys_attachesPerKeyExceptionsAsSuppressed() throws Exception // ------------------------------------------------------------------ @Test - void multipartBlockId_isolatesSameTargetWritersWithFixedLengthIds() { + void multipartBlockId_keepsLegacyLengthButCannotIdentifyWriter() { Assertions.assertEquals("p3w3DA==", AzureObjStorage.multipartBlockId("upload-a", 1)); - Assertions.assertNotEquals( - AzureObjStorage.multipartBlockId("upload-a", 1), - AzureObjStorage.multipartBlockId("upload-b", 1)); + Assertions.assertEquals("Sc7grw==", AzureObjStorage.multipartBlockId( + "doris-azure-lease-v1:09492e3d-e231-4ed9-bf84-b6fc772cda54", 1)); + Assertions.assertEquals("Sc7grw==", AzureObjStorage.multipartBlockId( + "doris-azure-lease-v1:06996d15-1c2e-4ddd-8853-43816ea84a07", 1)); Assertions.assertEquals( AzureObjStorage.multipartBlockId("upload-a", 1).length(), AzureObjStorage.multipartBlockId("upload-a", 999).length()); @@ -396,6 +401,124 @@ void multipartBlockId_isolatesSameTargetWritersWithFixedLengthIds() { Base64.getDecoder().decode(AzureObjStorage.multipartBlockId("upload-a", 1)).length); } + @Test + void initiateMultipartUpload_reservesTargetAndAcquiresLease() throws Exception { + BlockBlobClient blockClient = Mockito.mock(BlockBlobClient.class); + BlobClient blobClient = Mockito.mock(BlobClient.class); + Mockito.when(blobClient.getBlockBlobClient()).thenReturn(blockClient); + BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); + Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); + BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); + Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); + BlobLeaseClient leaseClient = Mockito.mock(BlobLeaseClient.class); + Mockito.when(leaseClient.acquireLease(60)).thenReturn("lease-id"); + TestableAzureObjStorage storage = + new TestableAzureObjStorage(buildBasicProps(), serviceClient, leaseClient); + + String uploadId = storage.initiateMultipartUpload( + "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob"); + + Assertions.assertEquals("doris-azure-lease-v1:lease-id", uploadId); + Mockito.verify(blockClient).stageBlock( + Mockito.anyString(), Mockito.any(com.azure.core.util.BinaryData.class)); + Mockito.verify(leaseClient).acquireLease(60); + } + + @Test + void uploadPart_renewsLeaseAndFencesStagedBlock() throws Exception { + BlockBlobClient blockClient = Mockito.mock(BlockBlobClient.class); + BlobClient blobClient = Mockito.mock(BlobClient.class); + Mockito.when(blobClient.getBlockBlobClient()).thenReturn(blockClient); + BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); + Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); + BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); + Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); + BlobLeaseClient leaseClient = Mockito.mock(BlobLeaseClient.class); + TestableAzureObjStorage storage = + new TestableAzureObjStorage(buildBasicProps(), serviceClient, leaseClient); + + storage.uploadPart("wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", + "doris-azure-lease-v1:lease-id", 1, + RequestBody.of(new ByteArrayInputStream(new byte[]{1}), 1)); + + Mockito.verify(leaseClient).renewLease(); + Mockito.verify(blockClient).stageBlockWithResponse(Mockito.anyString(), + Mockito.any(java.io.InputStream.class), Mockito.eq(1L), Mockito.isNull(), + Mockito.eq("lease-id"), Mockito.isNull(), Mockito.any()); + } + + @Test + void completeMultipartUpload_fencesCommitAndReleasesLease() throws Exception { + BlockBlobClient blockClient = Mockito.mock(BlockBlobClient.class); + BlobClient blobClient = Mockito.mock(BlobClient.class); + Mockito.when(blobClient.getBlockBlobClient()).thenReturn(blockClient); + BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); + Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); + BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); + Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); + BlobLeaseClient leaseClient = Mockito.mock(BlobLeaseClient.class); + TestableAzureObjStorage storage = + new TestableAzureObjStorage(buildBasicProps(), serviceClient, leaseClient); + + storage.completeMultipartUpload( + "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", + "doris-azure-lease-v1:lease-id", + Collections.singletonList(new UploadPartResult(1, "AQAAAA=="))); + + Mockito.verify(leaseClient).renewLease(); + org.mockito.ArgumentCaptor options = + org.mockito.ArgumentCaptor.forClass(BlockBlobCommitBlockListOptions.class); + Mockito.verify(blockClient).commitBlockListWithResponse( + options.capture(), Mockito.isNull(), Mockito.any()); + BlobRequestConditions conditions = options.getValue().getRequestConditions(); + Assertions.assertEquals("lease-id", conditions.getLeaseId()); + Mockito.verify(leaseClient).releaseLease(); + } + + @Test + void completeMultipartUpload_lostLeaseFailsBeforePublication() throws Exception { + BlockBlobClient blockClient = Mockito.mock(BlockBlobClient.class); + BlobClient blobClient = Mockito.mock(BlobClient.class); + Mockito.when(blobClient.getBlockBlobClient()).thenReturn(blockClient); + BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); + Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); + BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); + Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); + BlobLeaseClient leaseClient = Mockito.mock(BlobLeaseClient.class); + BlobStorageException lostLease = Mockito.mock(BlobStorageException.class); + Mockito.when(leaseClient.renewLease()).thenThrow(lostLease); + TestableAzureObjStorage storage = + new TestableAzureObjStorage(buildBasicProps(), serviceClient, leaseClient); + + Assertions.assertThrows(IOException.class, () -> storage.completeMultipartUpload( + "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", + "doris-azure-lease-v1:lease-id", + Collections.singletonList(new UploadPartResult(1, "AQAAAA==")))); + + Mockito.verify(blockClient, Mockito.never()).commitBlockList(Mockito.anyList()); + Mockito.verify(blockClient, Mockito.never()).commitBlockListWithResponse( + Mockito.any(BlockBlobCommitBlockListOptions.class), Mockito.isNull(), Mockito.any()); + } + + @Test + void abortMultipartUpload_releasesLeasedSessionWithoutRewritingTarget() throws Exception { + BlobClient blobClient = Mockito.mock(BlobClient.class); + BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); + Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); + BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); + Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); + BlobLeaseClient leaseClient = Mockito.mock(BlobLeaseClient.class); + TestableAzureObjStorage storage = + new TestableAzureObjStorage(buildBasicProps(), serviceClient, leaseClient); + + storage.abortMultipartUpload( + "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", + "doris-azure-lease-v1:lease-id"); + + Mockito.verify(leaseClient).releaseLease(); + Mockito.verify(blobClient, Mockito.never()).delete(); + } + @Test void uploadPart_acceptsLegacyResidualBlockLength() throws Exception { com.azure.storage.blob.specialized.BlockBlobClient blockClient = @@ -524,13 +647,20 @@ private Map buildBasicProps() { */ private static class TestableAzureObjStorage extends AzureObjStorage { private final BlobServiceClient mockClient; + private final BlobLeaseClient mockLeaseClient; String stubbedSasUrl = "https://stubbed-sas-url"; String lastGenerateSasContainer; String lastGenerateSasBlobKey; TestableAzureObjStorage(Map props, BlobServiceClient mockClient) { + this(props, mockClient, null); + } + + TestableAzureObjStorage(Map props, BlobServiceClient mockClient, + BlobLeaseClient mockLeaseClient) { super(props); this.mockClient = mockClient; + this.mockLeaseClient = mockLeaseClient; } @Override @@ -538,6 +668,11 @@ protected BlobServiceClient buildClient() { return mockClient; } + @Override + protected BlobLeaseClient createLeaseClient(BlobClient blobClient, String leaseId) { + return mockLeaseClient; + } + @Override protected String generateSasUrl(String endpoint, String container, String blobKey, StorageSharedKeyCredential credential, OffsetDateTime expiresOn) { From b6f4f016bba2d42ba4518d233daa0a2ea1e72d2a Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 3 Aug 2026 17:28:55 +0800 Subject: [PATCH 11/29] [fix](iceberg) Close async and rolling boundaries --- .../spill_iceberg_table_sink_operator.cpp | 2 ++ .../exec/sink/writer/async_result_writer.cpp | 17 +++++++++- be/src/exec/sink/writer/async_result_writer.h | 32 +++++++++++++++++-- ...spill_iceberg_table_sink_operator_test.cpp | 20 ++++++++++++ .../hive/HiveConnectorTransaction.java | 23 +++++++++++++ .../doris/connector/hive/HiveWriteUtils.java | 6 +++- .../hive/HiveConnectorTransactionTest.java | 29 ++++++++++++++--- .../connector/hive/HiveWriteUtilsTest.java | 15 +++++++++ .../filesystem/azure/AzureObjStorage.java | 10 +++--- .../azure/AzureObjStorageExtensionTest.java | 18 +++++------ 10 files changed, 149 insertions(+), 23 deletions(-) diff --git a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp index 75cb06e62ef74a..1e70abd51e7813 100644 --- a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp +++ b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp @@ -30,6 +30,8 @@ SpillIcebergTableSinkLocalState::SpillIcebergTableSinkLocalState(DataSinkOperato Status SpillIcebergTableSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) { RETURN_IF_ERROR(Base::init(state, info)); + // Admission samples async sorter state, so the next block must wait until the prior append publishes it. + _writer->wait_for_processing_before_next_sink(); SCOPED_TIMER(exec_time_counter()); SCOPED_TIMER(_init_timer); diff --git a/be/src/exec/sink/writer/async_result_writer.cpp b/be/src/exec/sink/writer/async_result_writer.cpp index 4bce381dfd7c31..b49ab103fb23cf 100644 --- a/be/src/exec/sink/writer/async_result_writer.cpp +++ b/be/src/exec/sink/writer/async_result_writer.cpp @@ -81,6 +81,7 @@ AsyncResultWriter::QueuedBlock AsyncResultWriter::_get_block_from_queue() { DCHECK(!_data_queue.empty()); auto queued = std::move(_data_queue.front()); _data_queue.pop_front(); + _queue_admission.begin_processing(); DCHECK(_dependency); if (_data_queue_is_available()) { _dependency->set_ready(); @@ -91,6 +92,17 @@ AsyncResultWriter::QueuedBlock AsyncResultWriter::_get_block_from_queue() { return queued; } +void AsyncResultWriter::_notify_block_processed() { + if (!_queue_admission.waits_for_processing()) { + return; + } + std::lock_guard l(_m); + _queue_admission.finish_processing(); + if (_data_queue_is_available()) { + _dependency->set_ready(); + } +} + Status AsyncResultWriter::start_writer(RuntimeState* state, RuntimeProfile* operator_profile) { // Attention!!! // AsyncResultWriter::open is called asynchronously, @@ -179,8 +191,9 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* opera if (!status.ok()) [[unlikely]] { thread_context()->thread_mem_tracker_mgr->shrink_reserved(); std::unique_lock l(_m); + _queue_admission.finish_processing(); _writer_status.update(status); - if (_is_finished()) { + if (_is_finished() || _data_queue_is_available()) { _dependency->set_ready(); } break; @@ -191,9 +204,11 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* opera } if (queued.eos) { // Keep the final reservation through finish(), where buffered sorters are committed. + _notify_block_processed(); break; } thread_context()->thread_mem_tracker_mgr->shrink_reserved(); + _notify_block_processed(); } bool need_finish = false; diff --git a/be/src/exec/sink/writer/async_result_writer.h b/be/src/exec/sink/writer/async_result_writer.h index 516201ebd818e9..830fca659da5ef 100644 --- a/be/src/exec/sink/writer/async_result_writer.h +++ b/be/src/exec/sink/writer/async_result_writer.h @@ -37,6 +37,27 @@ class Dependency; class PipelineTask; class Block; + +inline constexpr size_t ASYNC_WRITER_QUEUE_SIZE = 3; + +class AsyncWriterQueueAdmission { +public: + void wait_for_processing_before_next_sink() { _wait_for_processing = true; } + void begin_processing() { _block_being_processed = _wait_for_processing; } + void finish_processing() { _block_being_processed = false; } + + [[nodiscard]] bool is_available(size_t queued_blocks) const { + return _wait_for_processing ? queued_blocks == 0 && !_block_being_processed + : queued_blocks < ASYNC_WRITER_QUEUE_SIZE; + } + + [[nodiscard]] bool waits_for_processing() const { return _wait_for_processing; } + +private: + bool _block_being_processed = false; + bool _wait_for_processing = false; +}; + /* * In the pipeline execution engine, there are usually a large number of io operations on the sink side that * will block the limited execution threads of the pipeline execution engine, resulting in a sharp performance @@ -70,6 +91,10 @@ class AsyncResultWriter : public ResultWriter { void set_low_memory_mode(); + void wait_for_processing_before_next_sink() { + _queue_admission.wait_for_processing_before_next_sink(); + } + protected: Status _projection_block(Block& input_block, Block* output_block); const VExprContextSPtrs& _vec_output_expr_ctxs; @@ -85,20 +110,23 @@ class AsyncResultWriter : public ResultWriter { }; void process_block(RuntimeState* state, RuntimeProfile* operator_profile); - [[nodiscard]] bool _data_queue_is_available() const { return _data_queue.size() < QUEUE_SIZE; } + [[nodiscard]] bool _data_queue_is_available() const { + return _queue_admission.is_available(_data_queue.size()); + } [[nodiscard]] bool _is_finished() const { return !_writer_status.ok() || _eos; } void _set_ready_to_finish(); void _return_free_block(std::unique_ptr); QueuedBlock _get_block_from_queue(); + void _notify_block_processed(); - static constexpr auto QUEUE_SIZE = 3; std::mutex _m; std::condition_variable _cv; std::deque _data_queue; // Default value is ok AtomicStatus _writer_status; bool _eos = false; + AsyncWriterQueueAdmission _queue_admission; std::atomic_bool _low_memory_mode = false; std::shared_ptr _dependency; diff --git a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp index 62df6d4d210117..028153eb444b3a 100644 --- a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp +++ b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp @@ -18,6 +18,7 @@ #include #include "exec/operator/iceberg_sorter_reserve_memory.h" +#include "exec/sink/writer/async_result_writer.h" namespace doris { @@ -42,4 +43,23 @@ TEST(SpillIcebergTableSinkOperatorTest, ReservesIncomingBlockBeforeAnyPartitionW EXPECT_EQ(6 * 1024 * 1024, iceberg_reserve_size(no_published_sorters, 6 * 1024 * 1024)); } +TEST(SpillIcebergTableSinkOperatorTest, WaitsUntilDequeuedBlockUpdatesSorterState) { + AsyncWriterQueueAdmission stateful_admission; + stateful_admission.wait_for_processing_before_next_sink(); + + EXPECT_TRUE(stateful_admission.is_available(0)); + EXPECT_FALSE(stateful_admission.is_available(1)); + stateful_admission.begin_processing(); + // Dequeueing does not admit block 2 until block 1 changes the state sampled by admission. + EXPECT_FALSE(stateful_admission.is_available(0)); + stateful_admission.finish_processing(); + EXPECT_TRUE(stateful_admission.is_available(0)); + + // Writers without state-dependent admission retain the existing three-block queue behavior. + AsyncWriterQueueAdmission buffered_admission; + buffered_admission.begin_processing(); + EXPECT_TRUE(buffered_admission.is_available(2)); + EXPECT_FALSE(buffered_admission.is_available(3)); +} + } // namespace doris diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorTransaction.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorTransaction.java index 375604ecd4a9e1..315404b82df529 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorTransaction.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorTransaction.java @@ -288,6 +288,8 @@ private ScheduledFuture startCommitLockHeartbeat(long lockId) { } private void commitWhileTableLocked() { + // Object-store files remain unpublished until FE consumes one completion record per file. + validateObjectStoreCommitRecords(); validateWriteMetadataBeforePublication(); // The classification (finishInsertTable) ran from the executor in the legacy class; the unified SPI // exposes only commit(), so it runs here (before the committer) to populate the action maps. If it @@ -436,6 +438,27 @@ private boolean isTargetTable(String dbName, String tableName) { && nameMapping.getRemoteTblName().equalsIgnoreCase(tableName); } + private void validateObjectStoreCommitRecords() { + if (fileType != TFileType.FILE_S3) { + return; + } + for (THivePartitionUpdate update : hivePartitionUpdates) { + int fileCount = update.getFileNames() == null ? 0 : update.getFileNames().size(); + List uploads = update.getS3MpuPendingUploads(); + int uploadCount = uploads == null ? 0 : uploads.size(); + boolean completeRecords = uploads != null && uploads.stream().allMatch(upload -> + upload != null && upload.getUploadId() != null && !upload.getUploadId().isEmpty() + && upload.getBucket() != null && !upload.getBucket().isEmpty() + && upload.getKey() != null && !upload.getKey().isEmpty()); + if (fileCount != uploadCount || (fileCount > 0 && !completeRecords)) { + throw new DorisConnectorException(String.format( + "Object-store write reported %d file(s) but %d valid multipart completion record(s); " + + "all backends must support deferred multipart completion before metadata commit", + fileCount, completeRecords ? uploadCount : 0)); + } + } + } + @Override public void rollback() { if (hmsCommitter == null) { diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveWriteUtils.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveWriteUtils.java index 8e4c46dee4c92e..80c5199d7f491a 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveWriteUtils.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveWriteUtils.java @@ -55,7 +55,11 @@ static List mergePartitions(List hiv THivePartitionUpdate old = merged.get(pu.getName()); old.setFileSize(old.getFileSize() + pu.getFileSize()); old.setRowCount(old.getRowCount() + pu.getRowCount()); - if (old.getS3MpuPendingUploads() != null && pu.getS3MpuPendingUploads() != null) { + if (pu.getS3MpuPendingUploads() != null && !pu.getS3MpuPendingUploads().isEmpty()) { + // A missing legacy list is empty state, not ownership of later completion records. + if (old.getS3MpuPendingUploads() == null) { + old.setS3MpuPendingUploads(new ArrayList<>()); + } old.getS3MpuPendingUploads().addAll(pu.getS3MpuPendingUploads()); } old.getFileNames().addAll(pu.getFileNames()); diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorTransactionTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorTransactionTest.java index 5b5ded04976ddb..769dbac8cd471d 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorTransactionTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorTransactionTest.java @@ -459,6 +459,23 @@ public void testCommitCompletesMultipartUploads() throws TException { "an unpartitioned INSERT_EXISTING must also update the table statistics; calls=" + client.calls); } + @Test + public void testCommitRejectsBaseBeObjectStoreUpdateWithoutPendingUpload() throws TException { + RecordingHmsClient client = new RecordingHmsClient(); + client.table = table(false, Collections.emptyMap()); + HiveConnectorTransaction txn = newTxn(client); + txn.beginWrite(null, DB, TBL, ctx(false)); + // The base Azure BE reports the file but omits this field because initiation returned no upload ID. + txn.addCommitData(serialize(pu("", TUpdateMode.APPEND, "s3://bucket/db/t", + Collections.singletonList("base-be-file"), 100, 4))); + + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, txn::commit); + + Assertions.assertTrue(ex.getMessage().contains("multipart completion"), ex.getMessage()); + Assertions.assertFalse(client.calls.stream().anyMatch(c -> c.startsWith("updateTableStatistics")), + "metadata must remain unchanged when the object is still uncommitted"); + } + @Test public void testRollbackAbortsPendingMultipartUploads() throws TException { // rollback() is NOT a no-op for hive (D9): data files are staged before commit, so a rollback must @@ -503,15 +520,17 @@ public void testCommitAddsNewPartitionOnce() throws TException { // GAP-7: the 20-at-a-time batching moved INTO ThriftHmsClient.addPartitions, so the committer must // call addPartitions ONCE with the whole list (not re-batch it). GAP-4: the new partition's storage // descriptor (values/location/columns) is rebuilt from the table at commit time. A genuinely-new - // partition takes the ADD path; on FILE_S3 the write path == target path, so no rename/MPU runs and - // the object-store FileSystem is never resolved (hence newTxn, not newTxnWithFs). + // partition takes the ADD path; on FILE_S3 the write path == target path, so FE completes the deferred + // multipart upload before adding HMS metadata. RecordingHmsClient client = new RecordingHmsClient(); client.table = table(true, Collections.emptyMap()); client.partitionExistsResult = false; - HiveConnectorTransaction txn = newTxn(client); + RecordingObjStorage objStorage = new RecordingObjStorage(); + HiveConnectorTransaction txn = newTxnWithFs(client, new RecordingObjFileSystem(objStorage)); txn.beginWrite(null, DB, TBL, ctx(false)); - txn.addCommitData(serialize(pu("dt=2024-01-01", TUpdateMode.NEW, "s3://bucket/db/t/dt=2024-01-01", - Collections.singletonList("f1"), 100, 4))); + txn.addCommitData(serialize(puWithMpu("dt=2024-01-01", TUpdateMode.NEW, + "s3://bucket/db/t/dt=2024-01-01", "bucket", "db/t/dt=2024-01-01/f1", + "upload-1", Collections.singletonMap(1, "etag-1")))); txn.commit(); txn.close(); diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveWriteUtilsTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveWriteUtilsTest.java index e4633e6e032fbe..cad9fd6cc9a86a 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveWriteUtilsTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveWriteUtilsTest.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -100,6 +101,20 @@ public void mergePartitionsToleratesNullPendingUploads() { Assertions.assertEquals(3L, merged.get(0).getFileSize()); } + @Test + public void mergePartitionsPreservesNewPendingUploadAfterLegacyUpdate() { + THivePartitionUpdate legacy = update("p=1", 1L, 1L, "legacy-file"); + THivePartitionUpdate current = update("p=1", 2L, 2L, "current-file"); + current.setS3MpuPendingUploads(new ArrayList<>(Collections.singletonList( + new TS3MPUPendingUpload().setUploadId("upload-1")))); + + THivePartitionUpdate merged = HiveWriteUtils.mergePartitions(Arrays.asList(legacy, current)).get(0); + + Assertions.assertNotNull(merged.getS3MpuPendingUploads()); + Assertions.assertEquals(1, merged.getS3MpuPendingUploads().size(), + "a legacy first update must not erase a later completion token"); + } + @Test public void isSubDirectoryHappyPath() { Assertions.assertTrue(HiveWriteUtils.isSubDirectory("/warehouse/table", "/warehouse/table/p=1")); diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java index 9955d1a247bbb0..6899bd2d5e3236 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java @@ -280,11 +280,13 @@ public void completeMultipartUpload(String remotePath, String uploadId, List blockIds = new ArrayList<>(); List sorted = new ArrayList<>(parts); sorted.sort((a, b) -> Integer.compare(a.partNumber(), b.partNumber())); - boolean exactBlockIds = !sorted.isEmpty() && sorted.stream() - .allMatch(part -> part.etag() != null && !part.etag().isEmpty()); for (UploadPartResult part : sorted) { - // Missing IDs identify an older BE upload, whose blocks use the legacy namespace. - blockIds.add(exactBlockIds ? part.etag() : toBlockId(part.partNumber())); + // Guessing a legacy ID cannot recover an old BE payload and can select another writer's block. + if (part.etag() == null || part.etag().isEmpty()) { + throw new IOException("Azure multipart completion requires the exact staged block ID " + + "for part " + part.partNumber()); + } + blockIds.add(part.etag()); } // Put Block List is the atomic publication point and does not expose a staging blob to scans. BlobClient blobClient = containerClient.getBlobClient(uri.key()); diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java index bb9db84189d8ad..aee95b4aea11f3 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java @@ -574,7 +574,7 @@ void completeMultipartUpload_usesExactBlockIdsReportedByBe() throws Exception { } @Test - void completeMultipartUpload_fallsBackForOlderBeWithoutBlockIds() throws Exception { + void completeMultipartUpload_rejectsOlderBeWithoutBlockIds() throws Exception { com.azure.storage.blob.specialized.BlockBlobClient blockClient = Mockito.mock(com.azure.storage.blob.specialized.BlockBlobClient.class); BlobClient blobClient = Mockito.mock(BlobClient.class); @@ -585,15 +585,15 @@ void completeMultipartUpload_fallsBackForOlderBeWithoutBlockIds() throws Excepti Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); TestableAzureObjStorage storage = new TestableAzureObjStorage(buildBasicProps(), serviceClient); - storage.completeMultipartUpload( + Assertions.assertThrows(IOException.class, () -> storage.completeMultipartUpload( "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", - "legacy-upload-id", Collections.singletonList(new UploadPartResult(1, ""))); + "legacy-upload-id", Collections.singletonList(new UploadPartResult(1, "")))); - Mockito.verify(blockClient).commitBlockList(Collections.singletonList("AQAAAA==")); + Mockito.verify(blockClient, Mockito.never()).commitBlockList(Mockito.anyList()); } @Test - void completeMultipartUpload_usesLegacyNamespaceWhenAnyBlockIdIsMissing() throws Exception { + void completeMultipartUpload_rejectsMixedExactAndMissingBlockIds() throws Exception { com.azure.storage.blob.specialized.BlockBlobClient blockClient = Mockito.mock(com.azure.storage.blob.specialized.BlockBlobClient.class); BlobClient blobClient = Mockito.mock(BlobClient.class); @@ -604,14 +604,12 @@ void completeMultipartUpload_usesLegacyNamespaceWhenAnyBlockIdIsMissing() throws Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); TestableAzureObjStorage storage = new TestableAzureObjStorage(buildBasicProps(), serviceClient); - storage.completeMultipartUpload( + Assertions.assertThrows(IOException.class, () -> storage.completeMultipartUpload( "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", "mixed-upload-id", - Arrays.asList(new UploadPartResult(1, "exact-id"), new UploadPartResult(2, ""))); + Arrays.asList(new UploadPartResult(1, "exact-id"), new UploadPartResult(2, "")))); - Mockito.verify(blockClient).commitBlockList(Arrays.asList("AQAAAA==", "AgAAAA==")); - Mockito.verify(containerClient, Mockito.never()).getBlobClient( - "stage/blob.__doris_multipart/mixed-upload-id"); + Mockito.verify(blockClient, Mockito.never()).commitBlockList(Mockito.anyList()); } @Test From 630af939d1aa9b9773831d7e60fd06349e2249be Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 3 Aug 2026 17:34:31 +0800 Subject: [PATCH 12/29] [test](iceberg) Isolate async admission policy --- be/src/exec/sink/writer/async_result_writer.h | 21 +-------- .../writer/async_writer_queue_admission.h | 44 +++++++++++++++++++ ...spill_iceberg_table_sink_operator_test.cpp | 2 +- 3 files changed, 46 insertions(+), 21 deletions(-) create mode 100644 be/src/exec/sink/writer/async_writer_queue_admission.h diff --git a/be/src/exec/sink/writer/async_result_writer.h b/be/src/exec/sink/writer/async_result_writer.h index 830fca659da5ef..fe851a9171aae4 100644 --- a/be/src/exec/sink/writer/async_result_writer.h +++ b/be/src/exec/sink/writer/async_result_writer.h @@ -21,6 +21,7 @@ #include #include // IWYU pragma: keep +#include "exec/sink/writer/async_writer_queue_admission.h" #include "exec/sink/writer/result_writer.h" #include "exprs/vexpr_fwd.h" #include "runtime/memory/thread_mem_tracker_mgr.h" @@ -38,26 +39,6 @@ class PipelineTask; class Block; -inline constexpr size_t ASYNC_WRITER_QUEUE_SIZE = 3; - -class AsyncWriterQueueAdmission { -public: - void wait_for_processing_before_next_sink() { _wait_for_processing = true; } - void begin_processing() { _block_being_processed = _wait_for_processing; } - void finish_processing() { _block_being_processed = false; } - - [[nodiscard]] bool is_available(size_t queued_blocks) const { - return _wait_for_processing ? queued_blocks == 0 && !_block_being_processed - : queued_blocks < ASYNC_WRITER_QUEUE_SIZE; - } - - [[nodiscard]] bool waits_for_processing() const { return _wait_for_processing; } - -private: - bool _block_being_processed = false; - bool _wait_for_processing = false; -}; - /* * In the pipeline execution engine, there are usually a large number of io operations on the sink side that * will block the limited execution threads of the pipeline execution engine, resulting in a sharp performance diff --git a/be/src/exec/sink/writer/async_writer_queue_admission.h b/be/src/exec/sink/writer/async_writer_queue_admission.h new file mode 100644 index 00000000000000..50d78409a1afa8 --- /dev/null +++ b/be/src/exec/sink/writer/async_writer_queue_admission.h @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace doris { + +inline constexpr size_t ASYNC_WRITER_QUEUE_SIZE = 3; + +class AsyncWriterQueueAdmission { +public: + void wait_for_processing_before_next_sink() { _wait_for_processing = true; } + void begin_processing() { _block_being_processed = _wait_for_processing; } + void finish_processing() { _block_being_processed = false; } + + [[nodiscard]] bool is_available(size_t queued_blocks) const { + return _wait_for_processing ? queued_blocks == 0 && !_block_being_processed + : queued_blocks < ASYNC_WRITER_QUEUE_SIZE; + } + + [[nodiscard]] bool waits_for_processing() const { return _wait_for_processing; } + +private: + bool _block_being_processed = false; + bool _wait_for_processing = false; +}; + +} // namespace doris diff --git a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp index 028153eb444b3a..8cefc3a58a9655 100644 --- a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp +++ b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp @@ -18,7 +18,7 @@ #include #include "exec/operator/iceberg_sorter_reserve_memory.h" -#include "exec/sink/writer/async_result_writer.h" +#include "exec/sink/writer/async_writer_queue_admission.h" namespace doris { From 40f328b19d5e95efe7771c588986e369186e7904 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 3 Aug 2026 20:45:06 +0800 Subject: [PATCH 13/29] [fix](iceberg) Close remaining write safety gaps --- .../operator/iceberg_sorter_reserve_memory.h | 16 +++++ .../pipeline/pipeline_fragment_context.cpp | 51 +++++++++++++--- .../exec/pipeline/report_exec_status_size.h | 42 +++++++++++++ be/src/exec/sink/viceberg_delete_sink.cpp | 16 ++++- be/src/exec/sink/viceberg_delete_sink.h | 1 + .../exec/sink/writer/async_result_writer.cpp | 21 ++++++- .../writer/async_writer_queue_admission.h | 9 +++ .../writer/hive_multipart_compatibility.h | 29 +++++++++ .../writer/iceberg/viceberg_sort_writer.cpp | 19 +++++- .../writer/iceberg/viceberg_table_writer.cpp | 16 ++++- .../writer/iceberg/viceberg_table_writer.h | 1 + .../sink/writer/vhive_partition_writer.cpp | 17 +++++- .../exec/sink/writer/vhive_partition_writer.h | 1 + be/src/io/fs/azure_obj_storage_client.cpp | 23 ++++++- be/src/io/fs/azure_obj_storage_client.h | 1 + be/src/runtime/runtime_state.cpp | 41 ++++++++++--- be/src/runtime/runtime_state.h | 7 +++ ...spill_iceberg_table_sink_operator_test.cpp | 40 ++++++++++++ .../io/fs/azure_obj_storage_client_test.cpp | 6 ++ .../runtime_state_block_budget_test.cpp | 28 +++++++++ .../connector/hive/HiveWritePlanProvider.java | 2 + .../hive/HiveWritePlanProviderTest.java | 11 ++++ .../iceberg/IcebergConnectorTransaction.java | 4 +- .../IcebergRemoveOrphanFilesAction.java | 61 +++---------------- .../IcebergConnectorTransactionTest.java | 35 +++++++++++ .../IcebergRemoveOrphanFilesActionTest.java | 39 ++++++++++-- .../filesystem/azure/AzureObjStorage.java | 22 ++++++- .../azure/AzureObjStorageExtensionTest.java | 27 ++++++++ gensrc/thrift/DataSinks.thrift | 1 + 29 files changed, 498 insertions(+), 89 deletions(-) create mode 100644 be/src/exec/pipeline/report_exec_status_size.h create mode 100644 be/src/exec/sink/writer/hive_multipart_compatibility.h diff --git a/be/src/exec/operator/iceberg_sorter_reserve_memory.h b/be/src/exec/operator/iceberg_sorter_reserve_memory.h index 835343cfec806b..d1f0adea434282 100644 --- a/be/src/exec/operator/iceberg_sorter_reserve_memory.h +++ b/be/src/exec/operator/iceberg_sorter_reserve_memory.h @@ -51,4 +51,20 @@ inline size_t iceberg_reserve_size( sorter_reserve; } +inline size_t iceberg_spill_merge_workspace(size_t spill_file_count, size_t spill_buffer_bytes, + size_t merge_limit_bytes) { + if (spill_file_count == 0 || spill_buffer_bytes == 0) { + return 0; + } + const size_t max_fan_in = std::max(2, merge_limit_bytes / spill_buffer_bytes); + const size_t input_count = std::min(spill_file_count, max_fan_in); + const size_t max_size = std::numeric_limits::max(); + const size_t input_bytes = input_count > max_size / spill_buffer_bytes + ? max_size + : input_count * spill_buffer_bytes; + // VSortedRunMerger materializes one block per input cursor plus the block being emitted. + return input_bytes > max_size - spill_buffer_bytes ? max_size + : input_bytes + spill_buffer_bytes; +} + } // namespace doris diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index 8ccaca43dbf3c0..150e8d8dd9454c 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -26,6 +26,7 @@ #include #include + // IWYU pragma: no_include #include #include @@ -117,6 +118,7 @@ #include "exec/operator/union_source_operator.h" #include "exec/pipeline/dependency.h" #include "exec/pipeline/pipeline_task.h" +#include "exec/pipeline/report_exec_status_size.h" #include "exec/pipeline/task_scheduler.h" #include "exec/runtime_filter/runtime_filter_mgr.h" #include "exec/sort/topn_sorter.h" @@ -2361,6 +2363,10 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r DCHECK(req.status.ok() || req.done); // if !status.ok() => done if (req.coord_addr.hostname == "external") { // External query (flink/spark read tablets) not need to report to FE. + if (req.done) { + // Without a coordinator acknowledgement no external-write file may escape rollback. + req.runtime_state->finalize_iceberg_report_cleanup(false); + } return; } int callback_retries = 10; @@ -2381,6 +2387,9 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r static_cast(req.cancel_fn(Status::InternalError( "query_id: {}, couldn't get a client for {}, reason is {}", uid.to_string(), PrintThriftNetworkAddress(req.coord_addr), coord_status.to_string()))); + if (req.done) { + req.runtime_state->finalize_iceberg_report_cleanup(false); + } return; } @@ -2549,6 +2558,16 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r params.__set_backend_id(_exec_env->cluster_info()->backend_id); } + Status report_size_status = validate_report_exec_status_size( + params, req.runtime_state->coordinator_thrift_message_limit()); + if (!report_size_status.ok()) { + if (req.done) { + req.runtime_state->finalize_iceberg_report_cleanup(false); + } + req.cancel_fn(report_size_status); + return; + } + TReportExecStatusResult res; Status rpc_status; @@ -2569,6 +2588,9 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r rpc_status = coord->reopen(); if (!rpc_status.ok()) { + if (req.done) { + req.runtime_state->finalize_iceberg_report_cleanup(false); + } req.cancel_fn(rpc_status); return; } @@ -2582,9 +2604,18 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r } if (!rpc_status.ok()) { + if (req.done) { + req.runtime_state->finalize_iceberg_report_cleanup(false); + } LOG_INFO("Going to cancel query {} since report exec status got rpc failed: {}", print_id(req.query_id), rpc_status.to_string()); req.cancel_fn(rpc_status); + } else if (req.done && req.status.ok()) { + // Files remain rollback-owned until the coordinator has acknowledged the final metadata report. + req.runtime_state->finalize_iceberg_report_cleanup(true); + } else if (req.done) { + // An acknowledged error report confirms that FE will not publish this write's files. + req.runtime_state->finalize_iceberg_report_cleanup(false); } } @@ -2631,13 +2662,19 @@ Status PipelineFragmentContext::send_report(bool done) { .first_error_msg = first_error_msg, .cancel_fn = [this](const Status& reason) { cancel(reason); }}; auto ctx = std::dynamic_pointer_cast(shared_from_this()); - return _exec_env->fragment_mgr()->get_thread_pool()->submit_func([this, req, ctx]() { - SCOPED_ATTACH_TASK(ctx->get_query_ctx()->query_mem_tracker()); - _coordinator_callback(req); - if (!req.done) { - ctx->refresh_next_report_time(); - } - }); + Status submit_status = + _exec_env->fragment_mgr()->get_thread_pool()->submit_func([this, req, ctx]() { + SCOPED_ATTACH_TASK(ctx->get_query_ctx()->query_mem_tracker()); + _coordinator_callback(req); + if (!req.done) { + ctx->refresh_next_report_time(); + } + }); + if (!submit_status.ok() && req.done) { + // A rejected final callback can never transfer ownership to the coordinator. + req.runtime_state->finalize_iceberg_report_cleanup(false); + } + return submit_status; } size_t PipelineFragmentContext::get_revocable_size(bool* has_running_task) const { diff --git a/be/src/exec/pipeline/report_exec_status_size.h b/be/src/exec/pipeline/report_exec_status_size.h new file mode 100644 index 00000000000000..b5920963bfbcd4 --- /dev/null +++ b/be/src/exec/pipeline/report_exec_status_size.h @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "util/thrift_util.h" + +namespace doris { + +inline Status validate_report_exec_status_size(const TReportExecStatusParams& params, + size_t thrift_limit) { + ThriftSerializer serializer(false, 256); + uint32_t serialized_size = 0; + uint8_t* buffer = nullptr; + RETURN_IF_ERROR(serializer.serialize(¶ms, &serialized_size, &buffer)); + // Include the args field header and RPC method/version/sequence envelope around the params. + constexpr size_t rpc_envelope_bytes = 64; + if (thrift_limit < rpc_envelope_bytes || serialized_size > thrift_limit - rpc_envelope_bytes) { + return Status::InternalError( + "ReportExecStatus exceeds the coordinator Thrift message limit"); + } + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/sink/viceberg_delete_sink.cpp b/be/src/exec/sink/viceberg_delete_sink.cpp index a3dcb1df22f657..74dad178364080 100644 --- a/be/src/exec/sink/viceberg_delete_sink.cpp +++ b/be/src/exec/sink/viceberg_delete_sink.cpp @@ -293,7 +293,7 @@ Status VIcebergDeleteSink::close(Status close_status) { } if (!_defer_file_cleanup_until_outer_close) { - _created_files.clear(); + _transfer_created_files_to_report_cleanup(); } return Status::OK(); @@ -303,11 +303,23 @@ void VIcebergDeleteSink::finish_deferred_file_cleanup(Status outer_status) { if (!outer_status.ok()) { _cleanup_created_files(); } else { - _created_files.clear(); + _transfer_created_files_to_report_cleanup(); } _defer_file_cleanup_until_outer_close = false; } +void VIcebergDeleteSink::_transfer_created_files_to_report_cleanup() { + DCHECK(_state != nullptr); + for (auto& created_file : _created_files) { + _state->add_failed_iceberg_report_cleanup([cleanup_fs = std::move(created_file.first), + cleanup_path = std::move(created_file.second)] { + WARN_IF_ERROR(cleanup_fs->delete_file(cleanup_path), + "failed to delete an Iceberg delete file after report failure"); + }); + } + _created_files.clear(); +} + void VIcebergDeleteSink::_cleanup_created_files() { for (const auto& [fs, path] : _created_files) { Status delete_status = fs->delete_file(path); diff --git a/be/src/exec/sink/viceberg_delete_sink.h b/be/src/exec/sink/viceberg_delete_sink.h index 55698ae0404b14..625134ba3a3d51 100644 --- a/be/src/exec/sink/viceberg_delete_sink.h +++ b/be/src/exec/sink/viceberg_delete_sink.h @@ -134,6 +134,7 @@ class VIcebergDeleteSink final : public AsyncResultWriter { Status _init_position_delete_output_exprs(); std::string _get_file_extension() const; void _cleanup_created_files(); + void _transfer_created_files_to_report_cleanup(); TDataSink _t_sink; RuntimeState* _state = nullptr; diff --git a/be/src/exec/sink/writer/async_result_writer.cpp b/be/src/exec/sink/writer/async_result_writer.cpp index b49ab103fb23cf..bce7cc4f196740 100644 --- a/be/src/exec/sink/writer/async_result_writer.cpp +++ b/be/src/exec/sink/writer/async_result_writer.cpp @@ -150,6 +150,7 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* opera } DCHECK(_dependency); + bool reservation_held_for_finish = false; while (_writer_status.ok()) { ThreadCpuStopWatch cpu_time_stop_watch; cpu_time_stop_watch.start(); @@ -178,7 +179,6 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* opera //check if eos or writer error if ((_eos && _data_queue.empty()) || !_writer_status.ok()) { - _data_queue.clear(); break; } } @@ -204,6 +204,7 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* opera } if (queued.eos) { // Keep the final reservation through finish(), where buffered sorters are committed. + reservation_held_for_finish = true; _notify_block_processed(); break; } @@ -211,6 +212,17 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* opera _notify_block_processed(); } + { + std::lock_guard l(_m); + drain_async_writer_queue(_data_queue, [this](const QueuedBlock& queued) { + if (queued.block) { + _memory_used_counter->update(-queued.block->allocated_bytes()); + } + }); + _queue_admission.finish_processing(); + _dependency->set_ready(); + } + bool need_finish = false; { // If the last block is sent successfuly, then call finish to clear the buffer or commit @@ -231,8 +243,13 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* opera Status st = finish(state); _writer_status.update(st); } + if (reservation_held_for_finish) { + thread_context()->thread_mem_tracker_mgr->shrink_reserved(); + } Status st = Status::OK(); - { st = _writer_status.status(); } + { + st = _writer_status.status(); + } Status close_st = close(st); { diff --git a/be/src/exec/sink/writer/async_writer_queue_admission.h b/be/src/exec/sink/writer/async_writer_queue_admission.h index 50d78409a1afa8..b5a73cb72aa881 100644 --- a/be/src/exec/sink/writer/async_writer_queue_admission.h +++ b/be/src/exec/sink/writer/async_writer_queue_admission.h @@ -41,4 +41,13 @@ class AsyncWriterQueueAdmission { bool _wait_for_processing = false; }; +template +void drain_async_writer_queue(Queue& queue, BeforeRelease before_release) { + for (const auto& queued : queue) { + before_release(queued); + } + // Queued reservation tokens must be destroyed as soon as the writer reaches a terminal state. + queue.clear(); +} + } // namespace doris diff --git a/be/src/exec/sink/writer/hive_multipart_compatibility.h b/be/src/exec/sink/writer/hive_multipart_compatibility.h new file mode 100644 index 00000000000000..c7546f314bfa8b --- /dev/null +++ b/be/src/exec/sink/writer/hive_multipart_compatibility.h @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "io/fs/obj_storage_client.h" + +namespace doris { + +inline bool hive_multipart_protocol_supported(io::ObjStorageType provider, + bool supports_deferred_azure_multipart) { + return provider != io::ObjStorageType::AZURE || supports_deferred_azure_multipart; +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp index 444aec8933ae6a..396d81ad87e88a 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp @@ -17,6 +17,7 @@ #include "exec/sink/writer/iceberg/viceberg_sort_writer.h" +#include "exec/operator/iceberg_sorter_reserve_memory.h" #include "exec/spill/spill_file_manager.h" #include "exec/spill/spill_file_reader.h" #include "exec/spill/spill_file_writer.h" @@ -87,8 +88,22 @@ size_t VIcebergSortWriter::get_reserve_mem_size(RuntimeState* state, bool eos) c SorterReserveMemory VIcebergSortWriter::get_reserve_mem_size_components(RuntimeState* state, bool eos) const { std::lock_guard lock(_sorter_mutex); - return _sorter == nullptr ? SorterReserveMemory {} - : _sorter->get_reserve_mem_size_components(state, eos); + if (_sorter == nullptr) { + return {}; + } + auto reservation = _sorter->get_reserve_mem_size_components(state, eos); + if (eos && !_sorted_spill_files.empty()) { + size_t spill_file_count = _sorted_spill_files.size(); + if (_sorter->data_size() > 0) { + ++spill_file_count; + } + const size_t merge_workspace = + iceberg_spill_merge_workspace(spill_file_count, state->spill_buffer_size_bytes(), + state->spill_sort_merge_mem_limit_bytes()); + reservation.transient_workspace = + std::max(reservation.transient_workspace, merge_workspace); + } + return reservation; } Status VIcebergSortWriter::trigger_spill() { diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp index 52157c5a80ab27..3b37d023f87806 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp @@ -583,7 +583,7 @@ Status VIcebergTableWriter::close(Status status) { if (!status.ok() || !result_status.ok()) { _cleanup_closed_files(); } else if (!_defer_file_cleanup_until_outer_close) { - _closed_files.clear(); + _transfer_closed_files_to_report_cleanup(); } return result_status; } @@ -594,11 +594,23 @@ void VIcebergTableWriter::finish_deferred_file_cleanup(Status outer_status) { if (!outer_status.ok()) { _cleanup_closed_files(); } else { - _closed_files.clear(); + _transfer_closed_files_to_report_cleanup(); } _defer_file_cleanup_until_outer_close = false; } +void VIcebergTableWriter::_transfer_closed_files_to_report_cleanup() { + DCHECK(_state != nullptr); + for (auto& closed_file : _closed_files) { + _state->add_failed_iceberg_report_cleanup([cleanup_fs = std::move(closed_file.first), + cleanup_path = std::move(closed_file.second)] { + WARN_IF_ERROR(cleanup_fs->delete_file(cleanup_path), + "failed to delete an Iceberg file after report failure"); + }); + } + _closed_files.clear(); +} + void VIcebergTableWriter::_cleanup_closed_files() { for (const auto& [fs, path] : _closed_files) { Status delete_status = fs->delete_file(path); diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h index ce3c920986388e..7947216c933e44 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h @@ -150,6 +150,7 @@ class VIcebergTableWriter final : public AsyncResultWriter { Status _write_prepared_block(Block& output_block); Status _process_row_lineage_columns(Block& block); void _cleanup_closed_files(); + void _transfer_closed_files_to_report_cleanup(); // Currently it is a copy, maybe it is better to use move semantics to eliminate it. TDataSink _t_sink; diff --git a/be/src/exec/sink/writer/vhive_partition_writer.cpp b/be/src/exec/sink/writer/vhive_partition_writer.cpp index 8331efac54bd47..a7960850e3d198 100644 --- a/be/src/exec/sink/writer/vhive_partition_writer.cpp +++ b/be/src/exec/sink/writer/vhive_partition_writer.cpp @@ -21,10 +21,12 @@ #include "core/block/materialize_block.h" #include "core/column/column_map.h" +#include "exec/sink/writer/hive_multipart_compatibility.h" #include "format/transformer/vcsv_transformer.h" #include "format/transformer/vorc_transformer.h" #include "format/transformer/vparquet_transformer.h" #include "io/file_factory.h" +#include "io/fs/s3_file_system.h" #include "io/fs/s3_file_writer.h" #include "runtime/runtime_state.h" @@ -50,7 +52,10 @@ VHivePartitionWriter::VHivePartitionWriter(const TDataSink& t_sink, std::string _file_format_type(file_format_type), _hive_compress_type(hive_compress_type), _hive_serde_properties(hive_serde_properties), - _hadoop_conf(hadoop_conf) {} + _hadoop_conf(hadoop_conf), + _supports_deferred_azure_multipart( + t_sink.hive_table_sink.__isset.supports_deferred_azure_multipart && + t_sink.hive_table_sink.supports_deferred_azure_multipart) {} Status VHivePartitionWriter::open(RuntimeState* state, RuntimeProfile* operator_profile) { _state = state; @@ -64,6 +69,16 @@ Status VHivePartitionWriter::open(RuntimeState* state, RuntimeProfile* operator_ .path = fmt::format("{}/{}", _write_info.write_path, _get_target_file_name()), .fs_name {}}; _fs = DORIS_TRY(FileFactory::create_fs(fs_properties, file_description)); + if (auto* s3_fs = dynamic_cast(_fs.get()); + s3_fs != nullptr && + !hive_multipart_protocol_supported(s3_fs->client_holder()->s3_client_conf().provider, + _supports_deferred_azure_multipart)) { + // An old coordinator cannot publish namespaced Azure block IDs; lease expiry is not a + // compatibility fence, so reject before creating an upload that it could corrupt. + return Status::NotSupported( + "Azure Hive writes require a coordinator that supports deferred multipart " + "completion"); + } io::FileWriterOptions file_writer_options = {.used_by_s3_committer = true}; RETURN_IF_ERROR(_fs->create_file(file_description.path, &_file_writer, &file_writer_options)); diff --git a/be/src/exec/sink/writer/vhive_partition_writer.h b/be/src/exec/sink/writer/vhive_partition_writer.h index 0b124108623fa1..92e316a95c8e10 100644 --- a/be/src/exec/sink/writer/vhive_partition_writer.h +++ b/be/src/exec/sink/writer/vhive_partition_writer.h @@ -101,6 +101,7 @@ class VHivePartitionWriter { TFileCompressType::type _hive_compress_type; const THiveSerDeProperties* _hive_serde_properties; const std::map& _hadoop_conf; + bool _supports_deferred_azure_multipart = false; std::shared_ptr _fs = nullptr; diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp index 8c0a8201671a2b..d6d96c25e95ce6 100644 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ b/be/src/io/fs/azure_obj_storage_client.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -92,6 +93,19 @@ std::optional azure_multipart_lease_id(std::string_view upload return std::nullopt; } +void renew_or_reacquire_azure_multipart_lease(BlobClient& target_blob, std::string_view lease_id) { + BlobLeaseClient lease(target_blob, std::string(lease_id)); + try { + lease.Renew(); + } catch (const Azure::Storage::StorageException& e) { + if (!doris::io::azure_multipart_lease_can_be_reacquired(static_cast(e.StatusCode))) { + throw; + } + // The same proposed ID can be reacquired only while no competing writer owns the blob. + lease.Acquire(MULTIPART_LEASE_DURATION); + } +} + // Rate limiting is applied by RateLimitedObjStorageClient, the decorator that // S3ClientFactory wraps around this client when the bucket is subject to limiting. @@ -105,6 +119,11 @@ std::string azure_multipart_block_id(std::string_view upload_id, int part_num) { return encode_azure_block_id(upload_id, part_num); } +bool azure_multipart_lease_can_be_reacquired(int http_status) { + return http_status == static_cast(Azure::Core::Http::HttpStatusCode::Conflict) || + http_status == static_cast(Azure::Core::Http::HttpStatusCode::PreconditionFailed); +} + // As Azure's doc said, the batch size is 256 // You can find out the num in https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id // > Each batch request supports a maximum of 256 subrequests. @@ -272,7 +291,7 @@ ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStora SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); auto lease_id = azure_multipart_lease_id(*opts.upload_id); if (lease_id.has_value()) { - BlobLeaseClient(target_blob, std::string(*lease_id)).Renew(); + renew_or_reacquire_azure_multipart_lease(target_blob, *lease_id); StageBlockOptions stage_opts; stage_opts.AccessConditions.LeaseId = std::string(*lease_id); client.StageBlock(block_id, memory_body, stage_opts); @@ -305,7 +324,7 @@ ObjectStorageResponse AzureObjStorageClient::complete_multipart_upload( // Put Block List atomically replaces the committed blob; no scan-visible staging blob exists. auto lease_id = azure_multipart_lease_id(*opts.upload_id); if (lease_id.has_value()) { - BlobLeaseClient(target_blob, std::string(*lease_id)).Renew(); + renew_or_reacquire_azure_multipart_lease(target_blob, *lease_id); CommitBlockListOptions commit_opts; commit_opts.AccessConditions.LeaseId = std::string(*lease_id); target_client.CommitBlockList(string_block_ids, commit_opts); diff --git a/be/src/io/fs/azure_obj_storage_client.h b/be/src/io/fs/azure_obj_storage_client.h index de4ea5459a7cf7..8373af40739f99 100644 --- a/be/src/io/fs/azure_obj_storage_client.h +++ b/be/src/io/fs/azure_obj_storage_client.h @@ -34,6 +34,7 @@ bool is_azure_tls_ca_error_message(std::string_view message); std::string build_azure_tls_debug_suffix(std::string_view error_message, std::string_view tls_debug_context); std::string azure_multipart_block_id(std::string_view upload_id, int part_num); +bool azure_multipart_lease_can_be_reacquired(int http_status); class AzureObjStorageClient final : public ObjStorageClient { public: diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index c805b5310a1f17..db0f9c058b3d24 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -65,15 +65,9 @@ Status RuntimeState::add_iceberg_commit_datas(TIcebergCommitData iceberg_commit_ uint8_t* buffer = nullptr; RETURN_IF_ERROR(serializer.serialize(&iceberg_commit_data, &serialized_size, &buffer)); + // This is an early per-vector guard only; the assembled RPC is measured again before send. constexpr size_t report_envelope_headroom = 1024 * 1024; - int32_t effective_thrift_limit = std::max(config::thrift_max_message_size, 0); - if (_query_options.__isset.coordinator_thrift_max_message_size && - _query_options.coordinator_thrift_max_message_size > 0) { - // An older FE omits this field; otherwise the receiver's smaller limit is authoritative. - effective_thrift_limit = std::min(effective_thrift_limit, - _query_options.coordinator_thrift_max_message_size); - } - const size_t thrift_limit = static_cast(effective_thrift_limit); + const size_t thrift_limit = coordinator_thrift_message_limit(); const size_t commit_data_limit = thrift_limit > report_envelope_headroom ? thrift_limit - report_envelope_headroom : 0; std::lock_guard budget_lock(_iceberg_commit_data_budget->mutex); @@ -90,6 +84,37 @@ Status RuntimeState::add_iceberg_commit_datas(TIcebergCommitData iceberg_commit_ return Status::OK(); } +size_t RuntimeState::coordinator_thrift_message_limit() const { + int32_t effective_thrift_limit = std::max(config::thrift_max_message_size, 0); + if (_query_options.__isset.coordinator_thrift_max_message_size && + _query_options.coordinator_thrift_max_message_size > 0) { + // An older FE omits this field; otherwise the receiver's smaller limit is authoritative. + effective_thrift_limit = std::min(effective_thrift_limit, + _query_options.coordinator_thrift_max_message_size); + } + return static_cast(effective_thrift_limit); +} + +void RuntimeState::add_failed_iceberg_report_cleanup(std::function cleanup) { + std::lock_guard lock(_iceberg_commit_data_budget->mutex); + _iceberg_commit_data_budget->failed_report_cleanups.emplace_back(std::move(cleanup)); +} + +void RuntimeState::finalize_iceberg_report_cleanup(bool report_acknowledged) { + std::vector> cleanups; + { + std::lock_guard lock(_iceberg_commit_data_budget->mutex); + if (report_acknowledged) { + _iceberg_commit_data_budget->failed_report_cleanups.clear(); + return; + } + cleanups.swap(_iceberg_commit_data_budget->failed_report_cleanups); + } + for (auto& cleanup : cleanups) { + cleanup(); + } +} + RuntimeState::RuntimeState(const TPlanFragmentExecParams& fragment_exec_params, const TQueryOptions& query_options, const TQueryGlobals& query_globals, ExecEnv* exec_env, QueryContext* ctx, diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 9ae8484d81709a..15b255e83a3c2c 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -82,6 +82,7 @@ class IcebergCommitDataBudget { private: std::mutex mutex; size_t serialized_bytes = 0; + std::vector> failed_report_cleanups; }; // A collection of items that are part of the global state of a @@ -540,6 +541,12 @@ class RuntimeState { Status add_iceberg_commit_datas(TIcebergCommitData iceberg_commit_data); + size_t coordinator_thrift_message_limit() const; + + void add_failed_iceberg_report_cleanup(std::function cleanup); + + void finalize_iceberg_report_cleanup(bool report_acknowledged); + void set_iceberg_commit_data_budget(std::shared_ptr budget) { _iceberg_commit_data_budget = std::move(budget); } diff --git a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp index 8cefc3a58a9655..ea24310a77909e 100644 --- a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp +++ b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp @@ -17,8 +17,12 @@ #include +#include +#include + #include "exec/operator/iceberg_sorter_reserve_memory.h" #include "exec/sink/writer/async_writer_queue_admission.h" +#include "exec/sink/writer/hive_multipart_compatibility.h" namespace doris { @@ -43,6 +47,13 @@ TEST(SpillIcebergTableSinkOperatorTest, ReservesIncomingBlockBeforeAnyPartitionW EXPECT_EQ(6 * 1024 * 1024, iceberg_reserve_size(no_published_sorters, 6 * 1024 * 1024)); } +TEST(SpillIcebergTableSinkOperatorTest, ReservesAllMergeInputsAndOutputAtEos) { + constexpr size_t MB = 1024 * 1024; + + EXPECT_EQ(72 * MB, iceberg_spill_merge_workspace(12, 8 * MB, 64 * MB)); + EXPECT_EQ(32 * MB, iceberg_spill_merge_workspace(3, 8 * MB, 64 * MB)); +} + TEST(SpillIcebergTableSinkOperatorTest, WaitsUntilDequeuedBlockUpdatesSorterState) { AsyncWriterQueueAdmission stateful_admission; stateful_admission.wait_for_processing_before_next_sink(); @@ -62,4 +73,33 @@ TEST(SpillIcebergTableSinkOperatorTest, WaitsUntilDequeuedBlockUpdatesSorterStat EXPECT_FALSE(buffered_admission.is_available(3)); } +TEST(SpillIcebergTableSinkOperatorTest, TerminalWriterDrainsQueuedReservations) { + int live_reservations = 0; + struct Reservation { + explicit Reservation(int* live) : live(live) { ++*live; } + ~Reservation() { --*live; } + int* live; + }; + struct Queued { + size_t bytes; + std::unique_ptr reservation; + }; + std::deque queue; + queue.push_back({7, std::make_unique(&live_reservations)}); + queue.push_back({11, std::make_unique(&live_reservations)}); + size_t released_bytes = 0; + + drain_async_writer_queue(queue, [&](const Queued& queued) { released_bytes += queued.bytes; }); + + EXPECT_TRUE(queue.empty()); + EXPECT_EQ(0, live_reservations); + EXPECT_EQ(18, released_bytes); +} + +TEST(SpillIcebergTableSinkOperatorTest, AzureDeferredMultipartRequiresCoordinatorCapability) { + EXPECT_TRUE(hive_multipart_protocol_supported(io::ObjStorageType::AWS, false)); + EXPECT_FALSE(hive_multipart_protocol_supported(io::ObjStorageType::AZURE, false)); + EXPECT_TRUE(hive_multipart_protocol_supported(io::ObjStorageType::AZURE, true)); +} + } // namespace doris diff --git a/be/test/io/fs/azure_obj_storage_client_test.cpp b/be/test/io/fs/azure_obj_storage_client_test.cpp index dc7a2314dddb01..2b26573d8f601f 100644 --- a/be/test/io/fs/azure_obj_storage_client_test.cpp +++ b/be/test/io/fs/azure_obj_storage_client_test.cpp @@ -50,6 +50,12 @@ TEST(AzureObjStorageClientMultipartHelperTest, fixed_length_namespace_requires_t .GetLength()); } +TEST(AzureObjStorageClientMultipartHelperTest, expired_lease_can_be_reacquired_fail_closed) { + EXPECT_TRUE(io::azure_multipart_lease_can_be_reacquired(409)); + EXPECT_TRUE(io::azure_multipart_lease_can_be_reacquired(412)); + EXPECT_FALSE(io::azure_multipart_lease_can_be_reacquired(500)); +} + using namespace Azure::Storage::Blobs; TEST(AzureObjStorageClientTlsHelperTest, detects_tls_ca_error) { diff --git a/be/test/runtime/runtime_state_block_budget_test.cpp b/be/test/runtime/runtime_state_block_budget_test.cpp index dc02d35bd32398..30a0256bfc6b67 100644 --- a/be/test/runtime/runtime_state_block_budget_test.cpp +++ b/be/test/runtime/runtime_state_block_budget_test.cpp @@ -18,6 +18,7 @@ #include #include "common/config.h" +#include "exec/pipeline/report_exec_status_size.h" #include "runtime/runtime_state.h" #include "testutil/mock/mock_runtime_state.h" #include "util/block_budget.h" @@ -73,6 +74,33 @@ TEST(RuntimeStateIcebergCommitDataTest, UsesTheSmallerCoordinatorThriftLimit) { EXPECT_FALSE(status.ok()); } +TEST(RuntimeStateIcebergCommitDataTest, ValidatesTheCompleteReportEnvelope) { + TReportExecStatusParams params; + params.__set_error_log({std::string(2 * 1024 * 1024, 'x')}); + + EXPECT_FALSE(validate_report_exec_status_size(params, 1024 * 1024).ok()); + EXPECT_TRUE(validate_report_exec_status_size(params, 3 * 1024 * 1024).ok()); +} + +TEST(RuntimeStateIcebergCommitDataTest, RetainsFileCleanupUntilReportAcknowledgement) { + RuntimeState coordinator_state; + RuntimeState task_state; + auto report_state = std::make_shared(); + coordinator_state.set_iceberg_commit_data_budget(report_state); + task_state.set_iceberg_commit_data_budget(report_state); + int cleanup_count = 0; + task_state.add_failed_iceberg_report_cleanup([&] { ++cleanup_count; }); + + coordinator_state.finalize_iceberg_report_cleanup(false); + coordinator_state.finalize_iceberg_report_cleanup(false); + + EXPECT_EQ(1, cleanup_count); + + task_state.add_failed_iceberg_report_cleanup([&] { ++cleanup_count; }); + coordinator_state.finalize_iceberg_report_cleanup(true); + EXPECT_EQ(1, cleanup_count); +} + // --------------------------------------------------------------------------- // RuntimeState::batch_size() // --------------------------------------------------------------------------- diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveWritePlanProvider.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveWritePlanProvider.java index 586d38f2436186..4ae66aae99c63b 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveWritePlanProvider.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveWritePlanProvider.java @@ -295,6 +295,8 @@ private THiveTableSink buildSink(ConnectorSession session, HiveTableHandle table // Hadoop config (BE-canonical static creds; hive has no vended overlay). tSink.setHadoopConfig(buildHadoopConfig()); + // New coordinators publish Azure's exact staged block IDs after BE writers finish. + tSink.setSupportsDeferredAzureMultipart(true); tSink.setOverwrite(handle.isOverwrite()); return tSink; } diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveWritePlanProviderTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveWritePlanProviderTest.java index eb549709a0a937..f6f8da32875ba2 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveWritePlanProviderTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveWritePlanProviderTest.java @@ -329,6 +329,17 @@ public void planWriteSetsBucketInfo() { Assertions.assertEquals(8, sink.getBucketInfo().getBucketCount()); } + @Test + public void planWriteAdvertisesDeferredAzureMultipartProtocol() { + RecordingHmsClient client = new RecordingHmsClient(); + client.table = tableBuilder().build(); + + THiveTableSink sink = planSink(client, new RecordingConnectorContext(), handle()); + + Assertions.assertTrue(sink.isSetSupportsDeferredAzureMultipart()); + Assertions.assertTrue(sink.isSupportsDeferredAzureMultipart()); + } + // ───────────────────────────── file format ───────────────────────────── @Test diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java index b4c2ce5111ba64..e2209ea0cbd6cf 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java @@ -460,7 +460,9 @@ private Long resolveOverwriteBaseSnapshot(IcebergWriteContext ctx, Long targetHe throw new DorisConnectorException("Iceberg table " + tableName + " changed after the statement read an empty snapshot"); } - return null; + // Keep the empty generation pinned across Transactions.newTransaction(), whose refresh may + // otherwise adopt a first snapshot committed between the guard and transaction creation. + return readSnapshotId; } if (targetHead == null || !isAncestorOfTarget(readSnapshotId, targetHead)) { throw new DorisConnectorException("Read snapshot " + readSnapshotId diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java index b4d057c3c932c0..40a5f312fb8a43 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java @@ -50,7 +50,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.regex.Pattern; /** Safely lists or deletes old files that are unreachable from every retained snapshot. */ public class IcebergRemoveOrphanFilesAction extends BaseIcebergAction { @@ -176,13 +175,11 @@ private List resolveScanScopes(Table table) { if (objectRoot != null) { String normalizedObjectRoot = normalizeLocation(objectRoot); if (!isWithin(normalizedObjectRoot, tableRoot)) { - if (normalizedObjectRoot.startsWith(tableRoot)) { - // Iceberg omits table context for this raw-prefix case, so ownership is not recoverable. - throw new DorisConnectorException( - "Cannot prove object-store ownership because its path has the table location " - + "as a non-directory prefix; provide a verified explicit location"); - } - scopes.add(ScanScope.objectStore(normalizedObjectRoot, tableRoot)); + // Iceberg's hashed path retains only a suffix of the table location; that suffix is not + // a globally unique ownership key when multiple catalogs share an object-store root. + throw new DorisConnectorException( + "Cannot prove that the configured object-store root is table-exclusive; " + + "provide location with allow_unsafe_location=true after verifying exclusivity"); } } } else { @@ -314,11 +311,6 @@ static void verifyReachableIndexLimit(Set locations, int maxEntries) { index.addAll(locations); } - static boolean isOwnedObjectStorePath(String candidate, String storageRoot, String tableLocation) { - return ScanScope.objectStore(normalizeLocation(storageRoot), normalizeLocation(tableLocation)) - .owns(candidate); - } - private static final class ReachableIndex { private final Map byPath = new LinkedHashMap<>(); private final int maxEntries; @@ -347,54 +339,17 @@ private void add(String location) { private static final class ScanScope { private final String root; - private final Pattern ownedRelativePath; - private ScanScope(String root, Pattern ownedRelativePath) { + private ScanScope(String root) { this.root = root; - this.ownedRelativePath = ownedRelativePath; } private static ScanScope exclusive(String root) { - return new ScanScope(root, null); - } - - private static ScanScope objectStore(String root, String tableLocation) { - URI tableUri = URI.create(tableLocation); - String[] segments = tableUri.getPath().split("/"); - List names = new ArrayList<>(); - for (String segment : segments) { - if (!segment.isEmpty()) { - names.add(segment); - } - } - if (names.isEmpty()) { - throw new DorisConnectorException( - "Cannot infer an object-store table context from the table location"); - } - String context = names.size() > 1 - ? names.get(names.size() - 2) + "/" + names.get(names.size() - 1) - : names.get(names.size() - 1); - return new ScanScope(root, Pattern.compile( - "[01]{4}/[01]{4}/[01]{4}/[01]{8}/" - + Pattern.quote(context) + "/.+")); + return new ScanScope(root); } private boolean owns(String candidate) { - if (ownedRelativePath == null) { - return isWithinLocation(candidate, root); - } - FileIdentity child = FileIdentity.of(candidate); - FileIdentity parent = FileIdentity.of(root); - if (!child.scheme.equals(parent.scheme) || !child.authority.equals(parent.authority)) { - return false; - } - if (!isWithinLocation(candidate, root)) { - return false; - } - String relative = child.path.substring(Math.min(child.path.length(), parent.path.length())); - relative = relative.startsWith("/") ? relative.substring(1) : relative; - // Iceberg 1.10.1 splits its 20-bit hash into 4/4/4/8-bit directories before context. - return ownedRelativePath.matcher(relative).matches(); + return isWithinLocation(candidate, root); } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java index 0219b12416abbd..1f5af3cac19590 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java @@ -34,6 +34,7 @@ import org.apache.doris.thrift.TIcebergColumnStats; import org.apache.doris.thrift.TIcebergCommitData; +import org.apache.iceberg.BaseTable; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.DeleteFile; @@ -41,10 +42,12 @@ import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileMetadata; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; +import org.apache.iceberg.Transaction; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.expressions.Expression; @@ -681,6 +684,8 @@ public void overwriteRejectsFirstSnapshotCommittedAfterBeginFromEmptyRead() { IcebergConnectorTransaction txn = txnFor( opsReturning(catalog.loadTable(id)), new RecordingConnectorContext()); txn.beginWrite(SESSION, "db1", "t1", overwriteCtxPinned(-1L)); + Assertions.assertEquals(-1L, txn.getBaseSnapshotId(), + "the empty-read generation must remain the transaction OCC fence"); table.newAppend().appendFile(dataFile(table.spec(), "s3://b/db1/t1/after-begin.parquet", 1L)).commit(); @@ -688,6 +693,36 @@ public void overwriteRejectsFirstSnapshotCommittedAfterBeginFromEmptyRead() { Assertions.assertThrows(DorisConnectorException.class, txn::commit); } + @Test + public void overwriteRejectsFirstSnapshotCommittedDuringTransactionRefresh() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t1"); + Table loaded = catalog.createTable(id, SCHEMA, PartitionSpec.unpartitioned(), + props("write.format.default", "parquet")); + Table racing = new BaseTable(((HasTableOperations) loaded).operations(), loaded.name()) { + private boolean injected; + + @Override + public Transaction newTransaction() { + if (!injected) { + injected = true; + Table concurrent = catalog.loadTable(id); + concurrent.newAppend().appendFile(dataFile(concurrent.spec(), + "s3://b/db1/t1/during-refresh.parquet", 1L)).commit(); + } + return super.newTransaction(); + } + }; + IcebergConnectorTransaction txn = txnFor( + opsReturning(racing), new RecordingConnectorContext()); + + txn.beginWrite(SESSION, "db1", "t1", overwriteCtxPinned(-1L)); + txn.addCommitData(commitBytes(dataFileItem( + "s3://b/db1/t1/replacement.parquet", 1L, 1024L, Collections.emptyList()))); + + Assertions.assertThrows(DorisConnectorException.class, txn::commit); + } + @Test public void overwriteBranchUsesTheSnapshotReadFromThatBranch() { InMemoryCatalog catalog = freshCatalog(); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java index 08d9018f229631..b09ce892e2ea30 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java @@ -233,11 +233,6 @@ public void objectStoreOwnershipExcludesNeighborTableAndFolderRootFailsClosed(@T Table neighborTable = createTable(temp.resolve("neighbor"), objectProperties); String neighborLocation = neighborTable.locationProvider().newDataLocation("live.parquet"); Path neighborFile = createOldFile(Path.of(java.net.URI.create(neighborLocation))); - Assertions.assertTrue(IcebergRemoveOrphanFilesAction.isOwnedObjectStorePath( - ownLocation, objectRoot.toUri().toString(), objectTable.location())); - Assertions.assertFalse(IcebergRemoveOrphanFilesAction.isOwnedObjectStorePath( - neighborLocation, objectRoot.toUri().toString(), objectTable.location())); - Path folderRoot = temp.resolve("folder-data"); Table folderTable = createTable(temp.resolve("folder-metadata"), Collections.singletonMap(TableProperties.WRITE_FOLDER_STORAGE_LOCATION, @@ -247,7 +242,16 @@ public void objectStoreOwnershipExcludesNeighborTableAndFolderRootFailsClosed(@T IcebergRemoveOrphanFilesAction objectAction = action( System.currentTimeMillis() - MIN_RETENTION_MS, false); objectAction.validate(); - objectAction.execute(objectTable, ActionTestTables.session("UTC")); + Assertions.assertThrows(DorisConnectorException.class, + () -> objectAction.execute(objectTable, ActionTestTables.session("UTC"))); + Assertions.assertTrue(Files.exists(ownOrphan)); + Assertions.assertTrue(Files.exists(neighborFile)); + + IcebergRemoveOrphanFilesAction guardedObjectAction = action( + System.currentTimeMillis() - MIN_RETENTION_MS, false, + ownOrphan.getParent().toUri().toString(), true); + guardedObjectAction.validate(); + guardedObjectAction.execute(objectTable, ActionTestTables.session("UTC")); Assertions.assertFalse(Files.exists(ownOrphan)); Assertions.assertTrue(Files.exists(neighborFile)); @@ -267,6 +271,29 @@ public void objectStoreOwnershipExcludesNeighborTableAndFolderRootFailsClosed(@T Assertions.assertTrue(Files.exists(folderOrphan)); } + @Test + public void sharedObjectRootWithSameTableSuffixFailsClosed(@TempDir Path temp) throws Exception { + Path sharedRoot = temp.resolve("shared-object-root"); + Map properties = new HashMap<>(); + properties.put(TableProperties.OBJECT_STORE_ENABLED, "true"); + properties.put(TableProperties.WRITE_DATA_LOCATION, sharedRoot.toUri().toString()); + Table first = createTable(temp.resolve("catalog-a/db/t"), properties); + Table second = createTable(temp.resolve("catalog-b/db/t"), properties); + Path firstFile = createOldFile(Path.of(java.net.URI.create( + first.locationProvider().newDataLocation("first.parquet")))); + Path secondFile = createOldFile(Path.of(java.net.URI.create( + second.locationProvider().newDataLocation("second.parquet")))); + + IcebergRemoveOrphanFilesAction action = action( + System.currentTimeMillis() - MIN_RETENTION_MS, false); + action.validate(); + + Assertions.assertThrows(DorisConnectorException.class, + () -> action.execute(first, ActionTestTables.session("UTC"))); + Assertions.assertTrue(Files.exists(firstFile)); + Assertions.assertTrue(Files.exists(secondFile)); + } + @Test public void reachableIndexHasExplicitSafetyCap() { Set files = Set.of("s3://bucket/table/a", "s3://bucket/table/b", "s3://bucket/table/c"); diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java index 6899bd2d5e3236..4cc0bd7c885396 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java @@ -77,6 +77,8 @@ public class AzureObjStorage implements ObjStorage { private static final Logger LOG = LogManager.getLogger(AzureObjStorage.class); private static final int HTTP_NOT_FOUND = 404; + private static final int HTTP_CONFLICT = 409; + private static final int HTTP_PRECONDITION_FAILED = 412; private static final int MULTIPART_LEASE_SECONDS = 60; private static final String MULTIPART_LEASE_PREFIX = "doris-azure-lease-v1:"; /** Validity period for presigned (SAS) URLs, in seconds. */ @@ -260,7 +262,7 @@ public UploadPartResult uploadPart(String remotePath, String uploadId, int partN if (leaseId == null) { blockBlobClient.stageBlock(blockId, body.content(), body.contentLength()); } else { - createLeaseClient(blobClient, leaseId).renewLease(); + renewOrReacquireLease(blobClient, leaseId); blockBlobClient.stageBlockWithResponse(blockId, body.content(), body.contentLength(), null, leaseId, null, Context.NONE); } @@ -295,8 +297,7 @@ public void completeMultipartUpload(String remotePath, String uploadId, if (leaseId == null) { blockBlobClient.commitBlockList(blockIds); } else { - BlobLeaseClient leaseClient = createLeaseClient(blobClient, leaseId); - leaseClient.renewLease(); + BlobLeaseClient leaseClient = renewOrReacquireLease(blobClient, leaseId); BlobRequestConditions conditions = new BlobRequestConditions().setLeaseId(leaseId); blockBlobClient.commitBlockListWithResponse( new BlockBlobCommitBlockListOptions(blockIds).setRequestConditions(conditions), @@ -337,6 +338,21 @@ protected BlobLeaseClient createLeaseClient(BlobClient blobClient, String leaseI return new BlobLeaseClientBuilder().blobClient(blobClient).leaseId(leaseId).buildClient(); } + private BlobLeaseClient renewOrReacquireLease(BlobClient blobClient, String leaseId) { + BlobLeaseClient leaseClient = createLeaseClient(blobClient, leaseId); + try { + leaseClient.renewLease(); + } catch (BlobStorageException e) { + if (e.getStatusCode() != HTTP_CONFLICT && e.getStatusCode() != HTTP_PRECONDITION_FAILED) { + throw e; + } + // A finite lease may expire while FE waits for every writer. Reacquiring the same ID is safe: + // it succeeds only if no competing writer currently owns the target. + leaseClient.acquireLease(MULTIPART_LEASE_SECONDS); + } + return leaseClient; + } + private static String multipartLeaseId(String uploadId) { if (uploadId != null && uploadId.startsWith(MULTIPART_LEASE_PREFIX) && uploadId.length() > MULTIPART_LEASE_PREFIX.length()) { diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java index aee95b4aea11f3..fcc89ceedca788 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java @@ -500,6 +500,33 @@ void completeMultipartUpload_lostLeaseFailsBeforePublication() throws Exception Mockito.any(BlockBlobCommitBlockListOptions.class), Mockito.isNull(), Mockito.any()); } + @Test + void completeMultipartUpload_reacquiresExpiredLeaseBeforePublication() throws Exception { + BlockBlobClient blockClient = Mockito.mock(BlockBlobClient.class); + BlobClient blobClient = Mockito.mock(BlobClient.class); + Mockito.when(blobClient.getBlockBlobClient()).thenReturn(blockClient); + BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); + Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); + BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); + Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); + BlobLeaseClient leaseClient = Mockito.mock(BlobLeaseClient.class); + BlobStorageException expiredLease = Mockito.mock(BlobStorageException.class); + Mockito.when(expiredLease.getStatusCode()).thenReturn(409); + Mockito.when(leaseClient.renewLease()).thenThrow(expiredLease); + Mockito.when(leaseClient.acquireLease(60)).thenReturn("lease-id"); + TestableAzureObjStorage storage = + new TestableAzureObjStorage(buildBasicProps(), serviceClient, leaseClient); + + storage.completeMultipartUpload( + "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", + "doris-azure-lease-v1:lease-id", + Collections.singletonList(new UploadPartResult(1, "AQAAAA=="))); + + Mockito.verify(leaseClient).acquireLease(60); + Mockito.verify(blockClient).commitBlockListWithResponse( + Mockito.any(BlockBlobCommitBlockListOptions.class), Mockito.isNull(), Mockito.any()); + } + @Test void abortMultipartUpload_releasesLeasedSessionWithoutRewritingTarget() throws Exception { BlobClient blobClient = Mockito.mock(BlobClient.class); diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift index 9efe29a51eb421..d2cb4d534bb49b 100644 --- a/gensrc/thrift/DataSinks.thrift +++ b/gensrc/thrift/DataSinks.thrift @@ -384,6 +384,7 @@ struct THiveTableSink { 10: optional bool overwrite 11: optional THiveSerDeProperties serde_properties 12: optional list broker_addresses; + 13: optional bool supports_deferred_azure_multipart } enum TUpdateMode { From d0025535c57a8dd6b1b2e82542982679360e728c Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 3 Aug 2026 21:54:39 +0800 Subject: [PATCH 14/29] [chore](be) Fix clang formatting --- be/src/exec/sink/writer/async_result_writer.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/be/src/exec/sink/writer/async_result_writer.cpp b/be/src/exec/sink/writer/async_result_writer.cpp index bce7cc4f196740..84a2f2ef6c76f2 100644 --- a/be/src/exec/sink/writer/async_result_writer.cpp +++ b/be/src/exec/sink/writer/async_result_writer.cpp @@ -247,9 +247,7 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* opera thread_context()->thread_mem_tracker_mgr->shrink_reserved(); } Status st = Status::OK(); - { - st = _writer_status.status(); - } + { st = _writer_status.status(); } Status close_st = close(st); { From 3f1b4756afd339a0ded6cb860e47ecf89f1030dd Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 4 Aug 2026 11:47:29 +0800 Subject: [PATCH 15/29] [fix](iceberg) Close final write ownership gaps --- .../operator/iceberg_sorter_reserve_memory.h | 4 + be/src/exec/operator/operator.h | 4 + .../spill_iceberg_table_sink_operator.cpp | 30 +++- .../spill_iceberg_table_sink_operator.h | 4 +- .../pipeline/pipeline_fragment_context.cpp | 34 +++- be/src/exec/pipeline/pipeline_task.cpp | 3 +- .../writer/iceberg/viceberg_table_writer.cpp | 7 + be/src/io/fs/azure_obj_storage_client.cpp | 22 +-- be/src/io/fs/azure_obj_storage_client.h | 1 - be/src/runtime/runtime_state.cpp | 8 +- be/src/runtime/runtime_state.h | 4 +- ...spill_iceberg_table_sink_operator_test.cpp | 16 ++ .../iceberg/iceberg_table_writer_test.cpp | 12 ++ .../io/fs/azure_obj_storage_client_test.cpp | 6 - .../runtime_state_block_budget_test.cpp | 12 +- .../apache/doris/qe/AbstractJobProcessor.java | 10 +- .../java/org/apache/doris/qe/Coordinator.java | 165 +++++++++++------- .../org/apache/doris/qe/JobProcessor.java | 2 +- .../apache/doris/qe/NereidsCoordinator.java | 4 +- .../org/apache/doris/qe/QeProcessorImpl.java | 58 +++++- .../org/apache/doris/qe/SessionVariable.java | 1 + .../doris/qe/runtime/LoadProcessor.java | 46 ++--- .../runtime/SingleFragmentPipelineTask.java | 11 +- .../transaction/CommitDataSerializer.java | 23 ++- .../qe/QeProcessorImplReportAckTest.java | 136 +++++++++++++++ .../apache/doris/qe/SessionVariablesTest.java | 1 + .../SingleFragmentPipelineTaskTest.java | 15 ++ .../filesystem/azure/AzureObjStorage.java | 20 +-- .../azure/AzureObjStorageExtensionTest.java | 36 ++-- gensrc/thrift/FrontendService.thrift | 2 + gensrc/thrift/PaloInternalService.thrift | 2 + 31 files changed, 531 insertions(+), 168 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java diff --git a/be/src/exec/operator/iceberg_sorter_reserve_memory.h b/be/src/exec/operator/iceberg_sorter_reserve_memory.h index d1f0adea434282..3026922219e2aa 100644 --- a/be/src/exec/operator/iceberg_sorter_reserve_memory.h +++ b/be/src/exec/operator/iceberg_sorter_reserve_memory.h @@ -23,6 +23,8 @@ namespace doris { +class Block; + struct IcebergSorterReserveMemory { size_t retained_growth = 0; size_t transient_workspace = 0; @@ -51,6 +53,8 @@ inline size_t iceberg_reserve_size( sorter_reserve; } +size_t iceberg_cold_writer_reserve_size(const Block& block, size_t writer_workspace_bytes); + inline size_t iceberg_spill_merge_workspace(size_t spill_file_count, size_t spill_buffer_bytes, size_t merge_limit_bytes) { if (spill_file_count == 0 || spill_buffer_bytes == 0) { diff --git a/be/src/exec/operator/operator.h b/be/src/exec/operator/operator.h index becffbb171ee55..d60c8fe23209f4 100644 --- a/be/src/exec/operator/operator.h +++ b/be/src/exec/operator/operator.h @@ -631,6 +631,10 @@ class DataSinkOperatorXBase : public OperatorBase { [[nodiscard]] virtual size_t get_reserve_mem_size(RuntimeState* state, bool eos) { return state->minimum_operator_memory_required_bytes(); } + [[nodiscard]] virtual size_t get_reserve_mem_size(RuntimeState* state, bool eos, + const Block* block) { + return get_reserve_mem_size(state, eos); + } bool is_blockable(RuntimeState* state) const override { return state->get_sink_local_state()->is_blockable(); } diff --git a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp index 1e70abd51e7813..266dc654f4315f 100644 --- a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp +++ b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp @@ -18,12 +18,27 @@ #include "exec/operator/spill_iceberg_table_sink_operator.h" #include "common/status.h" +#include "core/block/block.h" #include "exec/operator/iceberg_table_sink_operator.h" #include "exec/sink/writer/iceberg/viceberg_sort_writer.h" #include "exec/sink/writer/iceberg/viceberg_table_writer.h" namespace doris { +size_t iceberg_cold_writer_reserve_size(const Block& block, size_t writer_workspace_bytes) { + const size_t block_bytes = block.allocated_bytes(); + const size_t row_index_bytes = + std::min(std::numeric_limits::max() / sizeof(size_t), block.rows()) * + sizeof(size_t); + const size_t selected_and_retained_bytes = + std::min(std::numeric_limits::max() / 2, block_bytes) * 2; + size_t reserve = std::min(std::numeric_limits::max() - writer_workspace_bytes, + selected_and_retained_bytes) + + writer_workspace_bytes; + // Cold dispatch may allocate a selected block and a retained sorter copy before publication. + return std::min(std::numeric_limits::max() - reserve, row_index_bytes) + reserve; +} + SpillIcebergTableSinkLocalState::SpillIcebergTableSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state) : Base(parent, state) {} @@ -53,7 +68,8 @@ bool SpillIcebergTableSinkLocalState::is_blockable() const { return true; } -size_t SpillIcebergTableSinkLocalState::get_reserve_mem_size(RuntimeState* state, bool eos) { +size_t SpillIcebergTableSinkLocalState::get_reserve_mem_size(RuntimeState* state, bool eos, + const Block* block) { if (!_writer) { return 0; } @@ -70,8 +86,11 @@ size_t SpillIcebergTableSinkLocalState::get_reserve_mem_size(RuntimeState* state } // Column growth remains in every touched sorter, while sorting workspace is reused by serial dispatch. // The final queued item may contain rows and also owns the reservation used by async finish(). - return iceberg_reserve_size(per_partition_reservations, - state->minimum_operator_memory_required_bytes()); + const size_t incoming_reserve = + block == nullptr ? state->minimum_operator_memory_required_bytes() + : iceberg_cold_writer_reserve_size( + *block, state->minimum_operator_memory_required_bytes()); + return iceberg_reserve_size(per_partition_reservations, incoming_reserve); } size_t SpillIcebergTableSinkLocalState::get_revocable_mem_size(RuntimeState* state) const { @@ -142,9 +161,10 @@ Status SpillIcebergTableSinkOperatorX::sink_impl(RuntimeState* state, Block* in_ return local_state.sink(state, in_block, eos); } -size_t SpillIcebergTableSinkOperatorX::get_reserve_mem_size(RuntimeState* state, bool eos) { +size_t SpillIcebergTableSinkOperatorX::get_reserve_mem_size(RuntimeState* state, bool eos, + const Block* block) { auto& local_state = get_local_state(state); - return local_state.get_reserve_mem_size(state, eos); + return local_state.get_reserve_mem_size(state, eos, block); } size_t SpillIcebergTableSinkOperatorX::revocable_mem_size(RuntimeState* state) const { diff --git a/be/src/exec/operator/spill_iceberg_table_sink_operator.h b/be/src/exec/operator/spill_iceberg_table_sink_operator.h index 5ffdd7505599ea..bd981531896c6c 100644 --- a/be/src/exec/operator/spill_iceberg_table_sink_operator.h +++ b/be/src/exec/operator/spill_iceberg_table_sink_operator.h @@ -43,7 +43,7 @@ class SpillIcebergTableSinkLocalState final Status open(RuntimeState* state) override; bool is_blockable() const override; - [[nodiscard]] size_t get_reserve_mem_size(RuntimeState* state, bool eos); + [[nodiscard]] size_t get_reserve_mem_size(RuntimeState* state, bool eos, const Block* block); Status revoke_memory(RuntimeState* state); size_t get_revocable_mem_size(RuntimeState* state) const; @@ -67,7 +67,7 @@ class SpillIcebergTableSinkOperatorX final Status sink_impl(RuntimeState* state, Block* in_block, bool eos) override; - size_t get_reserve_mem_size(RuntimeState* state, bool eos) override; + size_t get_reserve_mem_size(RuntimeState* state, bool eos, const Block* block) override; size_t revocable_mem_size(RuntimeState* state) const override; diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index 150e8d8dd9454c..6c471f8817be9d 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -2365,7 +2365,7 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r // External query (flink/spark read tablets) not need to report to FE. if (req.done) { // Without a coordinator acknowledgement no external-write file may escape rollback. - req.runtime_state->finalize_iceberg_report_cleanup(false); + req.runtime_state->finalize_iceberg_report_cleanup(IcebergReportOutcome::REJECTED); } return; } @@ -2388,7 +2388,7 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r "query_id: {}, couldn't get a client for {}, reason is {}", uid.to_string(), PrintThriftNetworkAddress(req.coord_addr), coord_status.to_string()))); if (req.done) { - req.runtime_state->finalize_iceberg_report_cleanup(false); + req.runtime_state->finalize_iceberg_report_cleanup(IcebergReportOutcome::REJECTED); } return; } @@ -2562,7 +2562,7 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r params, req.runtime_state->coordinator_thrift_message_limit()); if (!report_size_status.ok()) { if (req.done) { - req.runtime_state->finalize_iceberg_report_cleanup(false); + req.runtime_state->finalize_iceberg_report_cleanup(IcebergReportOutcome::REJECTED); } req.cancel_fn(report_size_status); return; @@ -2570,6 +2570,7 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r TReportExecStatusResult res; Status rpc_status; + bool report_outcome_ambiguous = false; VLOG_DEBUG << "reportExecStatus params is " << apache::thrift::ThriftDebugString(params).c_str(); @@ -2582,14 +2583,18 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r try { (*coord)->reportExecStatus(res, params); } catch (apache::thrift::transport::TTransportException& e) { + report_outcome_ambiguous = true; LOG(WARNING) << "Retrying ReportExecStatus. query id: " << print_id(req.query_id) << ", instance id: " << print_id(req.fragment_instance_id) << " to " << req.coord_addr << ", err: " << e.what(); rpc_status = coord->reopen(); if (!rpc_status.ok()) { + // The first request may have been consumed; keep files until metadata or orphan cleanup wins. + report_outcome_ambiguous = true; if (req.done) { - req.runtime_state->finalize_iceberg_report_cleanup(false); + req.runtime_state->finalize_iceberg_report_cleanup( + IcebergReportOutcome::AMBIGUOUS); } req.cancel_fn(rpc_status); return; @@ -2599,23 +2604,34 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r rpc_status = Status::create(res.status); } catch (apache::thrift::TException& e) { + report_outcome_ambiguous = true; rpc_status = Status::InternalError("ReportExecStatus() to {} failed: {}", PrintThriftNetworkAddress(req.coord_addr), e.what()); } + const bool requires_external_file_ack = params.__isset.iceberg_commit_datas; + if (rpc_status.ok() && requires_external_file_ack && + (!res.__isset.external_file_commit_data_accepted || + !res.external_file_commit_data_accepted)) { + rpc_status = Status::InternalError( + "Coordinator did not accept ownership of the external-file report"); + } + if (!rpc_status.ok()) { - if (req.done) { - req.runtime_state->finalize_iceberg_report_cleanup(false); + if (req.done && !report_outcome_ambiguous) { + req.runtime_state->finalize_iceberg_report_cleanup(IcebergReportOutcome::REJECTED); + } else if (req.done) { + req.runtime_state->finalize_iceberg_report_cleanup(IcebergReportOutcome::AMBIGUOUS); } LOG_INFO("Going to cancel query {} since report exec status got rpc failed: {}", print_id(req.query_id), rpc_status.to_string()); req.cancel_fn(rpc_status); } else if (req.done && req.status.ok()) { // Files remain rollback-owned until the coordinator has acknowledged the final metadata report. - req.runtime_state->finalize_iceberg_report_cleanup(true); + req.runtime_state->finalize_iceberg_report_cleanup(IcebergReportOutcome::ACKNOWLEDGED); } else if (req.done) { // An acknowledged error report confirms that FE will not publish this write's files. - req.runtime_state->finalize_iceberg_report_cleanup(false); + req.runtime_state->finalize_iceberg_report_cleanup(IcebergReportOutcome::REJECTED); } } @@ -2672,7 +2688,7 @@ Status PipelineFragmentContext::send_report(bool done) { }); if (!submit_status.ok() && req.done) { // A rejected final callback can never transfer ownership to the coordinator. - req.runtime_state->finalize_iceberg_report_cleanup(false); + req.runtime_state->finalize_iceberg_report_cleanup(IcebergReportOutcome::REJECTED); } return submit_status; } diff --git a/be/src/exec/pipeline/pipeline_task.cpp b/be/src/exec/pipeline/pipeline_task.cpp index 064e34fe2ca07c..fe1cbba6cbec92 100644 --- a/be/src/exec/pipeline/pipeline_task.cpp +++ b/be/src/exec/pipeline/pipeline_task.cpp @@ -664,7 +664,8 @@ Status PipelineTask::execute(bool* done) { ->task_controller() ->is_enable_reserve_memory() && workload_group && !(_wake_up_early || _dry_run)) { - const auto sink_reserve_size = _sink->get_reserve_mem_size(_state, _eos); + const auto sink_reserve_size = + _sink->get_reserve_mem_size(_state, _eos, _block.get()); if (sink_reserve_size > 0 && _should_trigger_revoking(sink_reserve_size)) { LOG(INFO) << fmt::format( diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp index 3b37d023f87806..9a81c43f80e2cb 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp @@ -56,6 +56,13 @@ Status VIcebergTableWriter::open(RuntimeState* state, RuntimeProfile* profile) { _state = state; _operator_profile = profile; + if (!state->query_options().__isset.supports_external_file_report_ack || + !state->query_options().supports_external_file_report_ack) { + // Do not create files unless the coordinator can make ownership transfer retry-safe. + return Status::NotSupported( + "Iceberg writes require a coordinator that acknowledges external-file reports"); + } + // Get target file size from query options // If value is 0 or not set, use config::iceberg_sink_max_file_size _target_file_size_bytes = config::iceberg_sink_max_file_size; diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp index d6d96c25e95ce6..52dea823f18302 100644 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ b/be/src/io/fs/azure_obj_storage_client.cpp @@ -93,17 +93,10 @@ std::optional azure_multipart_lease_id(std::string_view upload return std::nullopt; } -void renew_or_reacquire_azure_multipart_lease(BlobClient& target_blob, std::string_view lease_id) { +void renew_azure_multipart_lease(BlobClient& target_blob, std::string_view lease_id) { BlobLeaseClient lease(target_blob, std::string(lease_id)); - try { - lease.Renew(); - } catch (const Azure::Storage::StorageException& e) { - if (!doris::io::azure_multipart_lease_can_be_reacquired(static_cast(e.StatusCode))) { - throw; - } - // The same proposed ID can be reacquired only while no competing writer owns the blob. - lease.Acquire(MULTIPART_LEASE_DURATION); - } + // A renewal failure loses the upload-generation fence even if the same ID is acquirable later. + lease.Renew(); } // Rate limiting is applied by RateLimitedObjStorageClient, the decorator that @@ -119,11 +112,6 @@ std::string azure_multipart_block_id(std::string_view upload_id, int part_num) { return encode_azure_block_id(upload_id, part_num); } -bool azure_multipart_lease_can_be_reacquired(int http_status) { - return http_status == static_cast(Azure::Core::Http::HttpStatusCode::Conflict) || - http_status == static_cast(Azure::Core::Http::HttpStatusCode::PreconditionFailed); -} - // As Azure's doc said, the batch size is 256 // You can find out the num in https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id // > Each batch request supports a maximum of 256 subrequests. @@ -291,7 +279,7 @@ ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStora SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); auto lease_id = azure_multipart_lease_id(*opts.upload_id); if (lease_id.has_value()) { - renew_or_reacquire_azure_multipart_lease(target_blob, *lease_id); + renew_azure_multipart_lease(target_blob, *lease_id); StageBlockOptions stage_opts; stage_opts.AccessConditions.LeaseId = std::string(*lease_id); client.StageBlock(block_id, memory_body, stage_opts); @@ -324,7 +312,7 @@ ObjectStorageResponse AzureObjStorageClient::complete_multipart_upload( // Put Block List atomically replaces the committed blob; no scan-visible staging blob exists. auto lease_id = azure_multipart_lease_id(*opts.upload_id); if (lease_id.has_value()) { - renew_or_reacquire_azure_multipart_lease(target_blob, *lease_id); + renew_azure_multipart_lease(target_blob, *lease_id); CommitBlockListOptions commit_opts; commit_opts.AccessConditions.LeaseId = std::string(*lease_id); target_client.CommitBlockList(string_block_ids, commit_opts); diff --git a/be/src/io/fs/azure_obj_storage_client.h b/be/src/io/fs/azure_obj_storage_client.h index 8373af40739f99..de4ea5459a7cf7 100644 --- a/be/src/io/fs/azure_obj_storage_client.h +++ b/be/src/io/fs/azure_obj_storage_client.h @@ -34,7 +34,6 @@ bool is_azure_tls_ca_error_message(std::string_view message); std::string build_azure_tls_debug_suffix(std::string_view error_message, std::string_view tls_debug_context); std::string azure_multipart_block_id(std::string_view upload_id, int part_num); -bool azure_multipart_lease_can_be_reacquired(int http_status); class AzureObjStorageClient final : public ObjStorageClient { public: diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index db0f9c058b3d24..3c1e756e90ec59 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -100,14 +100,18 @@ void RuntimeState::add_failed_iceberg_report_cleanup(std::function clean _iceberg_commit_data_budget->failed_report_cleanups.emplace_back(std::move(cleanup)); } -void RuntimeState::finalize_iceberg_report_cleanup(bool report_acknowledged) { +void RuntimeState::finalize_iceberg_report_cleanup(IcebergReportOutcome outcome) { std::vector> cleanups; { std::lock_guard lock(_iceberg_commit_data_budget->mutex); - if (report_acknowledged) { + if (outcome == IcebergReportOutcome::ACKNOWLEDGED) { _iceberg_commit_data_budget->failed_report_cleanups.clear(); return; } + if (outcome == IcebergReportOutcome::AMBIGUOUS) { + // A consumed request with a lost ACK may already be publishing these files. + return; + } cleanups.swap(_iceberg_commit_data_budget->failed_report_cleanups); } for (auto& cleanup : cleanups) { diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 15b255e83a3c2c..d8163a3c32e04d 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -85,6 +85,8 @@ class IcebergCommitDataBudget { std::vector> failed_report_cleanups; }; +enum class IcebergReportOutcome { ACKNOWLEDGED, REJECTED, AMBIGUOUS }; + // A collection of items that are part of the global state of a // query and shared across all execution nodes of that query. class RuntimeState { @@ -545,7 +547,7 @@ class RuntimeState { void add_failed_iceberg_report_cleanup(std::function cleanup); - void finalize_iceberg_report_cleanup(bool report_acknowledged); + void finalize_iceberg_report_cleanup(IcebergReportOutcome outcome); void set_iceberg_commit_data_budget(std::shared_ptr budget) { _iceberg_commit_data_budget = std::move(budget); diff --git a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp index ea24310a77909e..92aa896daebcf2 100644 --- a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp +++ b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp @@ -19,7 +19,11 @@ #include #include +#include +#include "core/block/block.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type_string.h" #include "exec/operator/iceberg_sorter_reserve_memory.h" #include "exec/sink/writer/async_writer_queue_admission.h" #include "exec/sink/writer/hive_multipart_compatibility.h" @@ -47,6 +51,18 @@ TEST(SpillIcebergTableSinkOperatorTest, ReservesIncomingBlockBeforeAnyPartitionW EXPECT_EQ(6 * 1024 * 1024, iceberg_reserve_size(no_published_sorters, 6 * 1024 * 1024)); } +TEST(SpillIcebergTableSinkOperatorTest, ColdWriterReserveUsesFirstBlockLargerThanOperatorFloor) { + constexpr size_t operator_floor = 32 * 1024 * 1024; + auto strings = ColumnString::create(); + std::string payload(40 * 1024 * 1024, 'x'); + strings->insert_data(payload.data(), payload.size()); + Block block; + block.insert({std::move(strings), std::make_shared(), "payload"}); + + ASSERT_GT(block.allocated_bytes(), operator_floor); + EXPECT_GT(iceberg_cold_writer_reserve_size(block, operator_floor), 2 * block.allocated_bytes()); +} + TEST(SpillIcebergTableSinkOperatorTest, ReservesAllMergeInputsAndOutputAtEos) { constexpr size_t MB = 1024 * 1024; diff --git a/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp b/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp index e38da59cfbc176..5ca983fca439f8 100644 --- a/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp +++ b/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp @@ -24,6 +24,7 @@ #include "core/data_type/data_type_number.h" #include "exec/sink/writer/iceberg/viceberg_table_writer.h" #include "exec/sink/writer/iceberg/vpartition_writer_base.h" +#include "runtime/runtime_state.h" namespace doris { @@ -86,6 +87,17 @@ class VIcebergTableWriterTest : public testing::Test { } }; +TEST_F(VIcebergTableWriterTest, RejectsCoordinatorWithoutExternalFileReportAck) { + VIcebergTableWriter writer(make_sink(), {}, nullptr, nullptr); + RuntimeState state; + RuntimeProfile profile("test"); + + Status status = writer.open(&state, &profile); + + EXPECT_TRUE(status.is()); + EXPECT_NE(std::string::npos, status.to_string().find("acknowledges external-file reports")); +} + TEST_F(VIcebergTableWriterTest, SelectBlockUsesRowPermutation) { VIcebergTableWriter writer(make_sink(), {}, nullptr, nullptr); auto values = ColumnInt32::create(); diff --git a/be/test/io/fs/azure_obj_storage_client_test.cpp b/be/test/io/fs/azure_obj_storage_client_test.cpp index 2b26573d8f601f..dc7a2314dddb01 100644 --- a/be/test/io/fs/azure_obj_storage_client_test.cpp +++ b/be/test/io/fs/azure_obj_storage_client_test.cpp @@ -50,12 +50,6 @@ TEST(AzureObjStorageClientMultipartHelperTest, fixed_length_namespace_requires_t .GetLength()); } -TEST(AzureObjStorageClientMultipartHelperTest, expired_lease_can_be_reacquired_fail_closed) { - EXPECT_TRUE(io::azure_multipart_lease_can_be_reacquired(409)); - EXPECT_TRUE(io::azure_multipart_lease_can_be_reacquired(412)); - EXPECT_FALSE(io::azure_multipart_lease_can_be_reacquired(500)); -} - using namespace Azure::Storage::Blobs; TEST(AzureObjStorageClientTlsHelperTest, detects_tls_ca_error) { diff --git a/be/test/runtime/runtime_state_block_budget_test.cpp b/be/test/runtime/runtime_state_block_budget_test.cpp index 30a0256bfc6b67..802e9c54b5f1b7 100644 --- a/be/test/runtime/runtime_state_block_budget_test.cpp +++ b/be/test/runtime/runtime_state_block_budget_test.cpp @@ -91,14 +91,20 @@ TEST(RuntimeStateIcebergCommitDataTest, RetainsFileCleanupUntilReportAcknowledge int cleanup_count = 0; task_state.add_failed_iceberg_report_cleanup([&] { ++cleanup_count; }); - coordinator_state.finalize_iceberg_report_cleanup(false); - coordinator_state.finalize_iceberg_report_cleanup(false); + coordinator_state.finalize_iceberg_report_cleanup(IcebergReportOutcome::REJECTED); + coordinator_state.finalize_iceberg_report_cleanup(IcebergReportOutcome::REJECTED); EXPECT_EQ(1, cleanup_count); task_state.add_failed_iceberg_report_cleanup([&] { ++cleanup_count; }); - coordinator_state.finalize_iceberg_report_cleanup(true); + coordinator_state.finalize_iceberg_report_cleanup(IcebergReportOutcome::ACKNOWLEDGED); EXPECT_EQ(1, cleanup_count); + + task_state.add_failed_iceberg_report_cleanup([&] { ++cleanup_count; }); + coordinator_state.finalize_iceberg_report_cleanup(IcebergReportOutcome::AMBIGUOUS); + EXPECT_EQ(1, cleanup_count); + coordinator_state.finalize_iceberg_report_cleanup(IcebergReportOutcome::REJECTED); + EXPECT_EQ(2, cleanup_count); } // --------------------------------------------------------------------------- diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AbstractJobProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AbstractJobProcessor.java index 54e607dd27afa0..991f839a511bf4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/AbstractJobProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AbstractJobProcessor.java @@ -84,14 +84,18 @@ public void tryFinishSchedule() { } @Override - public final void updateFragmentExecStatus(TReportExecStatusParams params) { + public final boolean updateFragmentExecStatus(TReportExecStatusParams params) { if (params.status.status_code == TStatusCode.FINISHED) { params.status = new TStatus(TStatusCode.OK); } SingleFragmentPipelineTask fragmentTask = backendFragmentTasks.get().get( new BackendFragmentId(params.getBackendId(), params.getFragmentId())); if (fragmentTask == null) { - return; + if (params.isSetHivePartitionUpdates() || params.isSetIcebergCommitDatas() + || params.isSetMcCommitDatas()) { + throw new IllegalStateException("Missing fragment handler for external-file report"); + } + return false; } TUniqueId queryId = coordinatorContext.queryId; @@ -117,6 +121,8 @@ public final void updateFragmentExecStatus(TReportExecStatusParams params) { } } doProcessReportExecStatus(params, fragmentTask); + return !params.isSetHivePartitionUpdates() && !params.isSetIcebergCommitDatas() + && !params.isSetMcCommitDatas() || fragmentTask.isDone(); } private Map buildBackendFragmentTasks( diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 82979d901db526..35780a456b26f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -2558,7 +2558,7 @@ private void updateScanRangeNumByScanRange(TScanRangeParams param) { } // update job progress from BE - public void updateFragmentExecStatus(TReportExecStatusParams params) { + public boolean updateFragmentExecStatus(TReportExecStatusParams params) { if (params.isSetLoadedRows() && jobId != -1) { if (params.isSetFragmentInstanceReports()) { for (TFragmentInstanceReport report : params.getFragmentInstanceReports()) { @@ -2578,82 +2578,102 @@ public void updateFragmentExecStatus(TReportExecStatusParams params) { } PipelineExecContext ctx = pipelineExecContexts.get(Pair.of(params.getFragmentId(), params.getBackendId())); - if (ctx == null || !ctx.updatePipelineStatus(params)) { + boolean hasExternalCommitData = params.isSetHivePartitionUpdates() + || params.isSetIcebergCommitDatas() || params.isSetMcCommitDatas(); + if (ctx == null) { + if (hasExternalCommitData) { + throw new IllegalStateException("Missing fragment handler for external-file report"); + } + return false; + } + if (!ctx.updatePipelineStatus(params)) { + if (hasExternalCommitData && !ctx.done) { + throw new IllegalStateException("External-file report was not a completed fragment report"); + } LOG.debug("Fragment {} is not done, ignore report status: {}", params.getFragmentId(), params.toString()); - return; + return ctx.done; } - Status status = new Status(params.status); - // for now, abort the query if we see any error except if the error is cancelled - // and returned_all_results_ is true. - // (UpdateStatus() initiates cancellation, if it hasn't already been initiated) - if (!status.ok()) { - if (returnedAllResults && status.isCancelled()) { - LOG.warn("Query {} has returned all results, fragment_id={} instance_id={}, be={}" - + " is reporting failed status {}", - DebugUtil.printId(queryId), params.getFragmentId(), - DebugUtil.printId(params.getFragmentInstanceId()), - params.getBackendId(), - status.toString()); - } else { - LOG.warn("one instance report fail, query_id={} fragment_id={} instance_id={}, be={}," - + " error message: {}", - DebugUtil.printId(queryId), params.getFragmentId(), - DebugUtil.printId(params.getFragmentInstanceId()), - params.getBackendId(), status.toString()); - updateStatus(status); + boolean accepted = false; + try { + Status status = new Status(params.status); + // for now, abort the query if we see any error except if the error is cancelled + // and returned_all_results_ is true. + // (UpdateStatus() initiates cancellation, if it hasn't already been initiated) + if (!status.ok()) { + if (returnedAllResults && status.isCancelled()) { + LOG.warn("Query {} has returned all results, fragment_id={} instance_id={}, be={}" + + " is reporting failed status {}", + DebugUtil.printId(queryId), params.getFragmentId(), + DebugUtil.printId(params.getFragmentInstanceId()), + params.getBackendId(), + status.toString()); + } else { + LOG.warn("one instance report fail, query_id={} fragment_id={} instance_id={}, be={}," + + " error message: {}", + DebugUtil.printId(queryId), params.getFragmentId(), + DebugUtil.printId(params.getFragmentInstanceId()), + params.getBackendId(), status.toString()); + updateStatus(status); + } } - } - if (params.isSetDeltaUrls() && deltaUrls != null) { - updateDeltas(params.getDeltaUrls()); - } - if (params.isSetLoadCounters() && loadCounters != null) { - updateLoadCounters(params.getLoadCounters()); - } - if (params.isSetTrackingUrl()) { - LOG.info("query_id={} tracking_url: {}", DebugUtil.printId(queryId), params.getTrackingUrl()); - trackingUrl = params.getTrackingUrl(); - } - if (params.isSetFirstErrorMsg()) { - LOG.info("query_id={} first_error_msg: {}", DebugUtil.printId(queryId), params.getFirstErrorMsg()); - firstErrorMsg = params.getFirstErrorMsg(); - } - if (params.isSetTxnId()) { - txnId = params.getTxnId(); - } - if (params.isSetLabel()) { - label = params.getLabel(); - } - if (params.isSetExportFiles()) { - updateExportFiles(params.getExportFiles()); - } - if (params.isSetCommitInfos()) { - updateCommitInfos(params.getCommitInfos()); - } - if (params.isSetErrorTabletInfos()) { - updateErrorTabletInfos(params.getErrorTabletInfos()); - } - if (params.isSetHivePartitionUpdates() || params.isSetIcebergCommitDatas() || params.isSetMcCommitDatas()) { - Transaction txn = Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr().getTxnById(txnId); - if (params.isSetHivePartitionUpdates()) { - CommitDataSerializer.feed(txn, params.getHivePartitionUpdates()); + if (params.isSetDeltaUrls() && deltaUrls != null) { + updateDeltas(params.getDeltaUrls()); } - if (params.isSetIcebergCommitDatas()) { - CommitDataSerializer.feed(txn, params.getIcebergCommitDatas()); + if (params.isSetLoadCounters() && loadCounters != null) { + updateLoadCounters(params.getLoadCounters()); } - if (params.isSetMcCommitDatas()) { - CommitDataSerializer.feed(txn, params.getMcCommitDatas()); + if (params.isSetTrackingUrl()) { + LOG.info("query_id={} tracking_url: {}", DebugUtil.printId(queryId), params.getTrackingUrl()); + trackingUrl = params.getTrackingUrl(); } + if (params.isSetFirstErrorMsg()) { + LOG.info("query_id={} first_error_msg: {}", DebugUtil.printId(queryId), params.getFirstErrorMsg()); + firstErrorMsg = params.getFirstErrorMsg(); + } + if (params.isSetTxnId()) { + txnId = params.getTxnId(); + } + if (params.isSetLabel()) { + label = params.getLabel(); + } + if (params.isSetExportFiles()) { + updateExportFiles(params.getExportFiles()); + } + if (params.isSetCommitInfos()) { + updateCommitInfos(params.getCommitInfos()); + } + if (params.isSetErrorTabletInfos()) { + updateErrorTabletInfos(params.getErrorTabletInfos()); + } + if (params.isSetHivePartitionUpdates() || params.isSetIcebergCommitDatas() + || params.isSetMcCommitDatas()) { + Transaction txn = Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr().getTxnById(txnId); + if (params.isSetHivePartitionUpdates()) { + CommitDataSerializer.feed(txn, params.getHivePartitionUpdates()); + } + if (params.isSetIcebergCommitDatas()) { + CommitDataSerializer.feed(txn, params.getIcebergCommitDatas()); + } + if (params.isSetMcCommitDatas()) { + CommitDataSerializer.feed(txn, params.getMcCommitDatas()); + } + } + + accepted = true; + } finally { + ctx.finishPipelineStatus(accepted); } - if (ctx.done) { + if (accepted) { if (LOG.isDebugEnabled()) { LOG.debug("Query {} fragment {} is marked done", DebugUtil.printId(queryId), ctx.fragmentId); } fragmentsDoneLatch.markedCountDown(params.getFragmentId(), params.getBackendId()); } + return accepted; } /* @@ -3061,6 +3081,7 @@ public static class PipelineExecContext { PlanFragmentId fragmentId; boolean initiated; boolean done; + boolean processingDoneReport; TNetworkAddress brpcAddress; TNetworkAddress address; @@ -3117,10 +3138,30 @@ public synchronized boolean updatePipelineStatus(TReportExecStatusParams params) // duplicate packet return false; } - this.done = true; + while (processingDoneReport) { + try { + wait(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for a duplicate report", e); + } + if (this.done) { + return false; + } + } + // Serialize ownership processing so no duplicate can be acknowledged before acceptance finishes. + processingDoneReport = true; return true; } + public synchronized void finishPipelineStatus(boolean accepted) { + if (accepted) { + this.done = true; + } + processingDoneReport = false; + notifyAll(); + } + public boolean isBackendStateHealthy() { if (backend.getLastMissingHeartbeatTime() > lastMissingHeartbeatTime && !backend.isAlive()) { LOG.warn("backend {} is down while joining the coordinator. job id: {}", diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/JobProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/JobProcessor.java index 6365f26c4f9c2a..bc2b991032adef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/JobProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/JobProcessor.java @@ -26,7 +26,7 @@ public interface JobProcessor { void cancel(Status cancelReason); - void updateFragmentExecStatus(TReportExecStatusParams params); + boolean updateFragmentExecStatus(TReportExecStatusParams params); void tryFinishSchedule(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java index bc4c74335ca0b7..35bc335e30468f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java @@ -271,8 +271,8 @@ public boolean isDone() { } @Override - public void updateFragmentExecStatus(TReportExecStatusParams params) { - coordinatorContext.getJobProcessor().updateFragmentExecStatus(params); + public boolean updateFragmentExecStatus(TReportExecStatusParams params) { + return coordinatorContext.getJobProcessor().updateFragmentExecStatus(params); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java b/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java index 2c4202c3078757..eafbb23dfc704e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java @@ -37,6 +37,8 @@ import org.apache.doris.thrift.TUniqueId; import com.google.common.base.Strings; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -47,6 +49,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; public final class QeProcessorImpl implements QeProcessor { @@ -57,6 +60,10 @@ public final class QeProcessorImpl implements QeProcessor { private Map queryToInstancesNum; private Map userToInstancesCount; private ExecutorService writeProfileExecutor; + private final Cache acceptedExternalFileReports = CacheBuilder.newBuilder() + .maximumSize(1_000_000) + .expireAfterWrite(30, TimeUnit.MINUTES) + .build(); private final QueryFinishCallbackRegistry queryFinishCallbackRegistry = new QueryFinishCallbackRegistry(); public static final QeProcessor INSTANCE; @@ -284,22 +291,67 @@ public TReportExecStatusResult reportExecStatus(TReportExecStatusParams params, } } + boolean hasExternalCommitData = hasExternalCommitData(params); + String reportKey = hasExternalCommitData ? externalFileReportKey(params) : null; + if (hasExternalCommitData && reportKey == null) { + return rejectedExternalFileReport(result, "External-file report is missing its identity fields"); + } + if (hasExternalCommitData && acceptedExternalFileReports.getIfPresent(reportKey) != null) { + // Keep acceptance available after coordinator removal so a lost response is retry-safe. + result.setStatus(new TStatus(TStatusCode.OK)); + result.setExternalFileCommitDataAccepted(true); + return result; + } + final QueryInfo info = coordinatorMap.get(params.query_id); result.setStatus(new TStatus(TStatusCode.OK)); if (info == null) { // Currently, the execution of query is splited from the exec status process. // So, it is very likely that when exec status arrived on FE asynchronously, coordinator // has been removed from coordinatorMap. - return result; + return hasExternalCommitData + ? rejectedExternalFileReport(result, "Coordinator no longer owns this external-file report") + : result; } try { - info.getCoord().updateFragmentExecStatus(params); + boolean accepted = info.getCoord().updateFragmentExecStatus(params); + if (hasExternalCommitData && !accepted) { + return rejectedExternalFileReport(result, "FE has not accepted the external-file report"); + } } catch (Exception e) { LOG.warn("Exception during handle report, response: {}, query: {}, instance: {}", result.toString(), DebugUtil.printId(params.query_id), DebugUtil.printId(params.fragment_instance_id), e); - return result; + return hasExternalCommitData + ? rejectedExternalFileReport(result, "FE did not accept the external-file report") + : result; } result.setStatus(new TStatus(TStatusCode.OK)); + if (hasExternalCommitData) { + // Publish the retry token before replying; a transport loss cannot revoke FE ownership. + acceptedExternalFileReports.put(reportKey, Boolean.TRUE); + result.setExternalFileCommitDataAccepted(true); + } + return result; + } + + private static boolean hasExternalCommitData(TReportExecStatusParams params) { + return params.isSetHivePartitionUpdates() || params.isSetIcebergCommitDatas() || params.isSetMcCommitDatas(); + } + + private static String externalFileReportKey(TReportExecStatusParams params) { + if (!params.isSetQueryId() || !params.isSetFragmentId() || !params.isSetBackendId()) { + return null; + } + return params.getQueryId().getHi() + ":" + params.getQueryId().getLo() + ":" + + params.getFragmentId() + ":" + params.getBackendId(); + } + + private static TReportExecStatusResult rejectedExternalFileReport( + TReportExecStatusResult result, String message) { + TStatus status = new TStatus(TStatusCode.INTERNAL_ERROR); + status.addToErrorMsgs(message); + result.setStatus(status); + result.setExternalFileCommitDataAccepted(false); return result; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 6a8da9c71f1377..f0445ae9192e00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -5379,6 +5379,7 @@ public TQueryOptions toThrift() { TQueryOptions tResult = new TQueryOptions(); // Fragment reports are decoded by FE, whose limit can be lower than a rolling-upgrade BE's. tResult.setCoordinatorThriftMaxMessageSize(Config.thrift_max_message_size); + tResult.setSupportsExternalFileReportAck(true); tResult.setMemLimit(maxExecMemByte); tResult.setMaxScanMemRatio(maxScanMemRatio); tResult.setEnableAdaptiveScan(enableAdaptiveScan); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java index 069cd5e8a8924b..d4878ff99a3d6c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java @@ -186,12 +186,37 @@ protected void doProcessReportExecStatus(TReportExecStatusParams params, SingleF } } - if (!fragmentTask.processReportExecStatus(params)) { + if (!fragmentTask.processReportExecStatus(params, () -> acceptFinalReport(params))) { + if ((params.isSetHivePartitionUpdates() || params.isSetIcebergCommitDatas() + || params.isSetMcCommitDatas()) && !fragmentTask.isDone()) { + throw new IllegalStateException("External-file report was not a completed fragment report"); + } LOG.debug("Fragment {} is not done, ignore report status: {}", params.getFragmentId(), params.toString()); return; } + if (fragmentTask.isDone()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Query {} fragment {} is marked done", + DebugUtil.printId(coordinatorContext.queryId), params.getFragmentId()); + } + MarkedCountDownLatch latch = this.latch.get(); + latch.markedCountDown(params.getFragmentId(), params.getBackendId()); + + int topFragmentId = coordinatorContext.topDistributedPlan + .getFragmentJob().getFragment().getFragmentId().asInt(); + if (topFragmentId == params.getFragmentId()) { + MarkedCountDownLatch topFragmentLatch = this.topFragmentLatch.get(); + topFragmentLatch.markedCountDown(params.getFragmentId(), params.getBackendId()); + if (topFragmentLatch.getCount() == 0) { + tryFinishSchedule(); + } + } + } + } + + private void acceptFinalReport(TReportExecStatusParams params) { LoadContext loadContext = coordinatorContext.asLoadProcessor().loadContext; if (params.isSetDeltaUrls()) { loadContext.updateDeltaUrls(params.getDeltaUrls()); @@ -233,25 +258,6 @@ protected void doProcessReportExecStatus(TReportExecStatusParams params, SingleF CommitDataSerializer.feed(txn, params.getMcCommitDatas()); } } - - if (fragmentTask.isDone()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Query {} fragment {} is marked done", - DebugUtil.printId(coordinatorContext.queryId), params.getFragmentId()); - } - MarkedCountDownLatch latch = this.latch.get(); - latch.markedCountDown(params.getFragmentId(), params.getBackendId()); - - int topFragmentId = coordinatorContext.topDistributedPlan - .getFragmentJob().getFragment().getFragmentId().asInt(); - if (topFragmentId == params.getFragmentId()) { - MarkedCountDownLatch topFragmentLatch = this.topFragmentLatch.get(); - topFragmentLatch.markedCountDown(params.getFragmentId(), params.getBackendId()); - if (topFragmentLatch.getCount() == 0) { - tryFinishSchedule(); - } - } - } } // Check backend health for every unfinished load fragment task. diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTask.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTask.java index c6110d6a35be01..2b5b685bdd0f27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTask.java @@ -58,12 +58,19 @@ public SingleFragmentPipelineTask(Backend backend, int fragmentId, Set> fragments) { try { TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory()); - for (TBase fragment : fragments) { - txn.addCommitData(serializer.serialize(fragment)); + List serializedFragments = fragments.stream().map(fragment -> { + try { + return serializer.serialize(fragment); + } catch (TException e) { + throw new CommitDataSerializationException(e); + } + }).collect(Collectors.toList()); + // Serialize the complete vector before mutating the transaction so malformed input is retry-safe. + for (byte[] serializedFragment : serializedFragments) { + txn.addCommitData(serializedFragment); } } catch (TException e) { - throw new RuntimeException("failed to serialize connector commit data", e); + throw new RuntimeException("failed to initialize connector commit-data serialization", e); + } catch (CommitDataSerializationException e) { + throw new RuntimeException("failed to serialize connector commit data", e.getCause()); + } + } + + private static final class CommitDataSerializationException extends RuntimeException { + private CommitDataSerializationException(TException cause) { + super(cause); } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java new file mode 100644 index 00000000000000..fd8f34d90e73ab --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.common.profile.ExecutionProfile; +import org.apache.doris.planner.PlanFragmentId; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TIcebergCommitData; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TQueryOptions; +import org.apache.doris.thrift.TReportExecStatusParams; +import org.apache.doris.thrift.TReportExecStatusResult; +import org.apache.doris.thrift.TStatus; +import org.apache.doris.thrift.TStatusCode; +import org.apache.doris.thrift.TUniqueId; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Collections; + +class QeProcessorImplReportAckTest { + private TUniqueId registeredQueryId; + + @AfterEach + void cleanup() { + if (registeredQueryId != null) { + QeProcessorImpl.INSTANCE.unregisterQuery(registeredQueryId); + } + } + + @Test + void rejectsExternalReportWithoutCoordinator() { + TReportExecStatusResult result = report(params(new TUniqueId(12345, 1))); + + Assertions.assertEquals(TStatusCode.INTERNAL_ERROR, result.getStatus().getStatusCode()); + Assertions.assertFalse(result.isExternalFileCommitDataAccepted()); + } + + @Test + void rejectsExternalReportWhenHandlerThrows() throws Exception { + TUniqueId queryId = new TUniqueId(12345, 2); + Coordinator coordinator = register(queryId); + Mockito.doThrow(new RuntimeException("injected acceptance failure")) + .when(coordinator).updateFragmentExecStatus(Mockito.any()); + + TReportExecStatusResult result = report(params(queryId)); + + Assertions.assertEquals(TStatusCode.INTERNAL_ERROR, result.getStatus().getStatusCode()); + Assertions.assertFalse(result.isExternalFileCommitDataAccepted()); + } + + @Test + void rejectsExternalReportWhenHandlerDoesNotAcceptIt() throws Exception { + TUniqueId queryId = new TUniqueId(12345, 4); + register(queryId); + + TReportExecStatusResult result = report(params(queryId)); + + Assertions.assertEquals(TStatusCode.INTERNAL_ERROR, result.getStatus().getStatusCode()); + Assertions.assertFalse(result.isExternalFileCommitDataAccepted()); + } + + @Test + void retriesAcceptedExternalReportAfterCoordinatorRemoval() throws Exception { + TUniqueId queryId = new TUniqueId(12345, 3); + Coordinator coordinator = register(queryId); + Mockito.when(coordinator.updateFragmentExecStatus(Mockito.any())).thenReturn(true); + TReportExecStatusParams params = params(queryId); + + TReportExecStatusResult first = report(params); + QeProcessorImpl.INSTANCE.unregisterQuery(queryId); + registeredQueryId = null; + TReportExecStatusResult retry = report(params); + + Assertions.assertTrue(first.isExternalFileCommitDataAccepted()); + Assertions.assertTrue(retry.isExternalFileCommitDataAccepted()); + Assertions.assertEquals(TStatusCode.OK, retry.getStatus().getStatusCode()); + Mockito.verify(coordinator, Mockito.times(1)).updateFragmentExecStatus(params); + } + + @Test + void legacyCoordinatorRetriesFailedAcceptanceBeforeMarkingDone() { + Backend backend = Mockito.mock(Backend.class); + Mockito.when(backend.getHost()).thenReturn("127.0.0.1"); + ExecutionProfile profile = Mockito.mock(ExecutionProfile.class); + Coordinator.PipelineExecContext context = new Coordinator.PipelineExecContext( + new PlanFragmentId(7), null, backend, profile, -1); + TReportExecStatusParams report = new TReportExecStatusParams().setDone(true); + + Assertions.assertTrue(context.updatePipelineStatus(report)); + context.finishPipelineStatus(false); + Assertions.assertTrue(context.updatePipelineStatus(report)); + context.finishPipelineStatus(true); + Assertions.assertFalse(context.updatePipelineStatus(report)); + } + + private Coordinator register(TUniqueId queryId) throws Exception { + Coordinator coordinator = Mockito.mock(Coordinator.class); + Mockito.when(coordinator.getQueryOptions()).thenReturn(new TQueryOptions()); + QeProcessorImpl.INSTANCE.registerQuery(queryId, new QeProcessorImpl.QueryInfo(coordinator)); + registeredQueryId = queryId; + return coordinator; + } + + private static TReportExecStatusParams params(TUniqueId queryId) { + return new TReportExecStatusParams() + .setQueryId(queryId) + .setFragmentId(7) + .setBackendId(9) + .setDone(true) + .setStatus(new TStatus(TStatusCode.OK)) + .setIcebergCommitDatas(Collections.emptyList()); + } + + private static TReportExecStatusResult report(TReportExecStatusParams params) { + return QeProcessorImpl.INSTANCE.reportExecStatus(params, new TNetworkAddress("127.0.0.1", 9050)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java index 5075f916251441..9febc45b150e6d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java @@ -406,5 +406,6 @@ public void testCoordinatorThriftLimitPropagatesToBackends() { Assertions.assertTrue(queryOptions.isSetCoordinatorThriftMaxMessageSize()); Assertions.assertEquals(Config.thrift_max_message_size, queryOptions.getCoordinatorThriftMaxMessageSize()); + Assertions.assertTrue(queryOptions.isSupportsExternalFileReportAck()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTaskTest.java index 31becae01dc9db..281da9ab3dfe1d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTaskTest.java @@ -19,6 +19,7 @@ import org.apache.doris.common.Status; import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TReportExecStatusParams; import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TUniqueId; @@ -28,6 +29,20 @@ import java.util.Collections; class SingleFragmentPipelineTaskTest { + @Test + void failedAcceptanceLeavesFinalReportRetryable() { + SingleFragmentPipelineTask task = createTask(createBackend(100L)); + TReportExecStatusParams report = new TReportExecStatusParams().setDone(true); + + Assertions.assertThrows(RuntimeException.class, + () -> task.processReportExecStatus(report, () -> { + throw new RuntimeException("injected failure"); + })); + Assertions.assertFalse(task.isDone()); + Assertions.assertTrue(task.processReportExecStatus(report, () -> { })); + Assertions.assertTrue(task.isDone()); + } + @Test void backendWithUnchangedProcessEpochIsHealthy() { Backend backend = createBackend(100L); diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java index 4cc0bd7c885396..a269ceca30c5f0 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java @@ -77,8 +77,6 @@ public class AzureObjStorage implements ObjStorage { private static final Logger LOG = LogManager.getLogger(AzureObjStorage.class); private static final int HTTP_NOT_FOUND = 404; - private static final int HTTP_CONFLICT = 409; - private static final int HTTP_PRECONDITION_FAILED = 412; private static final int MULTIPART_LEASE_SECONDS = 60; private static final String MULTIPART_LEASE_PREFIX = "doris-azure-lease-v1:"; /** Validity period for presigned (SAS) URLs, in seconds. */ @@ -262,7 +260,7 @@ public UploadPartResult uploadPart(String remotePath, String uploadId, int partN if (leaseId == null) { blockBlobClient.stageBlock(blockId, body.content(), body.contentLength()); } else { - renewOrReacquireLease(blobClient, leaseId); + renewMultipartLease(blobClient, leaseId); blockBlobClient.stageBlockWithResponse(blockId, body.content(), body.contentLength(), null, leaseId, null, Context.NONE); } @@ -297,7 +295,7 @@ public void completeMultipartUpload(String remotePath, String uploadId, if (leaseId == null) { blockBlobClient.commitBlockList(blockIds); } else { - BlobLeaseClient leaseClient = renewOrReacquireLease(blobClient, leaseId); + BlobLeaseClient leaseClient = renewMultipartLease(blobClient, leaseId); BlobRequestConditions conditions = new BlobRequestConditions().setLeaseId(leaseId); blockBlobClient.commitBlockListWithResponse( new BlockBlobCommitBlockListOptions(blockIds).setRequestConditions(conditions), @@ -338,18 +336,10 @@ protected BlobLeaseClient createLeaseClient(BlobClient blobClient, String leaseI return new BlobLeaseClientBuilder().blobClient(blobClient).leaseId(leaseId).buildClient(); } - private BlobLeaseClient renewOrReacquireLease(BlobClient blobClient, String leaseId) { + private BlobLeaseClient renewMultipartLease(BlobClient blobClient, String leaseId) { BlobLeaseClient leaseClient = createLeaseClient(blobClient, leaseId); - try { - leaseClient.renewLease(); - } catch (BlobStorageException e) { - if (e.getStatusCode() != HTTP_CONFLICT && e.getStatusCode() != HTTP_PRECONDITION_FAILED) { - throw e; - } - // A finite lease may expire while FE waits for every writer. Reacquiring the same ID is safe: - // it succeeds only if no competing writer currently owns the target. - leaseClient.acquireLease(MULTIPART_LEASE_SECONDS); - } + // A renewal failure loses the upload-generation fence even if the same ID is acquirable later. + leaseClient.renewLease(); return leaseClient; } diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java index fcc89ceedca788..b7aa4bcf9f19f0 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java @@ -501,7 +501,7 @@ void completeMultipartUpload_lostLeaseFailsBeforePublication() throws Exception } @Test - void completeMultipartUpload_reacquiresExpiredLeaseBeforePublication() throws Exception { + void completeMultipartUpload_expiredLeaseFailsClosedAfterCollidingWriterStagesAndReleases() throws Exception { BlockBlobClient blockClient = Mockito.mock(BlockBlobClient.class); BlobClient blobClient = Mockito.mock(BlobClient.class); Mockito.when(blobClient.getBlockBlobClient()).thenReturn(blockClient); @@ -509,21 +509,35 @@ void completeMultipartUpload_reacquiresExpiredLeaseBeforePublication() throws Ex Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); - BlobLeaseClient leaseClient = Mockito.mock(BlobLeaseClient.class); + BlobLeaseClient staleLeaseClient = Mockito.mock(BlobLeaseClient.class); + BlobLeaseClient competingLeaseClient = Mockito.mock(BlobLeaseClient.class); BlobStorageException expiredLease = Mockito.mock(BlobStorageException.class); Mockito.when(expiredLease.getStatusCode()).thenReturn(409); - Mockito.when(leaseClient.renewLease()).thenThrow(expiredLease); - Mockito.when(leaseClient.acquireLease(60)).thenReturn("lease-id"); - TestableAzureObjStorage storage = - new TestableAzureObjStorage(buildBasicProps(), serviceClient, leaseClient); + Mockito.when(staleLeaseClient.renewLease()).thenThrow(expiredLease); + TestableAzureObjStorage staleStorage = + new TestableAzureObjStorage(buildBasicProps(), serviceClient, staleLeaseClient); + TestableAzureObjStorage competingStorage = + new TestableAzureObjStorage(buildBasicProps(), serviceClient, competingLeaseClient); + String staleUpload = "doris-azure-lease-v1:09492e3d-e231-4ed9-bf84-b6fc772cda54"; + String competingUpload = "doris-azure-lease-v1:06996d15-1c2e-4ddd-8853-43816ea84a07"; + String collidingBlockId = AzureObjStorage.multipartBlockId(staleUpload, 1); + + competingStorage.uploadPart( + "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", + competingUpload, 1, RequestBody.of(new ByteArrayInputStream(new byte[]{2}), 1)); + competingStorage.abortMultipartUpload( + "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", competingUpload); - storage.completeMultipartUpload( + Assertions.assertThrows(IOException.class, () -> staleStorage.completeMultipartUpload( "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", - "doris-azure-lease-v1:lease-id", - Collections.singletonList(new UploadPartResult(1, "AQAAAA=="))); + staleUpload, Collections.singletonList(new UploadPartResult(1, collidingBlockId)))); - Mockito.verify(leaseClient).acquireLease(60); - Mockito.verify(blockClient).commitBlockListWithResponse( + Mockito.verify(blockClient).stageBlockWithResponse(Mockito.eq(collidingBlockId), + Mockito.any(java.io.InputStream.class), Mockito.eq(1L), Mockito.isNull(), + Mockito.eq("06996d15-1c2e-4ddd-8853-43816ea84a07"), Mockito.isNull(), Mockito.any()); + Mockito.verify(competingLeaseClient).releaseLease(); + Mockito.verify(staleLeaseClient, Mockito.never()).acquireLease(Mockito.anyInt()); + Mockito.verify(blockClient, Mockito.never()).commitBlockListWithResponse( Mockito.any(BlockBlobCommitBlockListOptions.class), Mockito.isNull(), Mockito.any()); } diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 0d0fef3eacce79..5d98075023cdfc 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -183,6 +183,8 @@ struct TListPrivilegesResult{ struct TReportExecStatusResult { // required in V1 1: optional Status.TStatus status + // Set only after FE accepts the external-file commit vectors for this report. + 2: optional bool external_file_commit_data_accepted } // Service Protocol Details diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 9b498c8c0a6c3f..1f05f5f2312fc4 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -514,6 +514,8 @@ struct TQueryOptions { 228: optional bool enable_local_exchange_before_streaming_agg = false; // FE is the receiver of fragment reports, so BE must also honor its message limit. 229: optional i32 coordinator_thrift_max_message_size; + // FE can explicitly and idempotently acknowledge external-file commit reports. + 230: optional bool supports_external_file_report_ack = false; // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query. From d01bebcb06bb511d7bbf86cf0337d49d781deb6d Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 4 Aug 2026 15:15:54 +0800 Subject: [PATCH 16/29] [fix](iceberg) Gate standalone deletes on report ACK ### What problem does this PR solve? Issue Number: None Related PR: #66348 Problem Summary: A standalone Iceberg DELETE bypassed the table-writer capability gate, so a new BE could create delete files for an old coordinator that cannot acknowledge ownership transfer. Share the report-acknowledgement validation across table and delete writers and reject the direct delete path before it can create files. ### Release note Prevent standalone Iceberg deletes from creating files when the coordinator cannot acknowledge external-file ownership transfer. ### Check List (For Author) - Test: Unit Test - VIcebergDeleteSinkTest.* - VIcebergTableWriterTest.RejectsCoordinatorWithoutExternalFileReportAck - Behavior changed: Yes. Mixed-version standalone Iceberg DELETE now fails before file creation when the coordinator lacks report acknowledgement support. - Does this need documentation: No --- be/src/exec/sink/viceberg_delete_sink.cpp | 3 ++ .../iceberg/iceberg_writer_compatibility.h | 35 +++++++++++++++++++ .../writer/iceberg/viceberg_table_writer.cpp | 8 ++--- .../exec/sink/viceberg_delete_sink_test.cpp | 12 +++++++ 4 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 be/src/exec/sink/writer/iceberg/iceberg_writer_compatibility.h diff --git a/be/src/exec/sink/viceberg_delete_sink.cpp b/be/src/exec/sink/viceberg_delete_sink.cpp index 74dad178364080..1a572f57da7260 100644 --- a/be/src/exec/sink/viceberg_delete_sink.cpp +++ b/be/src/exec/sink/viceberg_delete_sink.cpp @@ -34,6 +34,7 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "exec/common/endian.h" +#include "exec/sink/writer/iceberg/iceberg_writer_compatibility.h" #include "exprs/vexpr.h" #include "format/table/deletion_vector.h" #include "format/table/iceberg_delete_file_reader_helper.h" @@ -203,6 +204,8 @@ Status VIcebergDeleteSink::init_properties(ObjectPool* pool) { Status VIcebergDeleteSink::open(RuntimeState* state, RuntimeProfile* profile) { _state = state; + RETURN_IF_ERROR(validate_iceberg_external_file_report_ack(state->query_options())); + // Initialize counters _written_rows_counter = ADD_COUNTER(profile, "RowsWritten", TUnit::UNIT); _send_data_timer = ADD_TIMER(profile, "SendDataTime"); diff --git a/be/src/exec/sink/writer/iceberg/iceberg_writer_compatibility.h b/be/src/exec/sink/writer/iceberg/iceberg_writer_compatibility.h new file mode 100644 index 00000000000000..ae06f4847a4c82 --- /dev/null +++ b/be/src/exec/sink/writer/iceberg/iceberg_writer_compatibility.h @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "common/status.h" +#include "gen_cpp/PaloInternalService_types.h" + +namespace doris { + +inline Status validate_iceberg_external_file_report_ack(const TQueryOptions& query_options) { + if (!query_options.__isset.supports_external_file_report_ack || + !query_options.supports_external_file_report_ack) { + // A pre-ACK coordinator cannot safely take ownership of files created by this sink. + return Status::NotSupported( + "Iceberg writes require a coordinator that acknowledges external-file reports"); + } + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp index 9a81c43f80e2cb..05175baeca73c4 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp @@ -31,6 +31,7 @@ #include "core/data_type/data_type_struct.h" #include "core/data_type_serde/data_type_serde.h" #include "exec/sink/writer/iceberg/iceberg_partition_path.h" +#include "exec/sink/writer/iceberg/iceberg_writer_compatibility.h" #include "exec/sink/writer/iceberg/partition_transformers.h" #include "exec/sink/writer/iceberg/viceberg_partition_writer.h" #include "exec/sink/writer/iceberg/viceberg_sort_writer.h" @@ -56,12 +57,7 @@ Status VIcebergTableWriter::open(RuntimeState* state, RuntimeProfile* profile) { _state = state; _operator_profile = profile; - if (!state->query_options().__isset.supports_external_file_report_ack || - !state->query_options().supports_external_file_report_ack) { - // Do not create files unless the coordinator can make ownership transfer retry-safe. - return Status::NotSupported( - "Iceberg writes require a coordinator that acknowledges external-file reports"); - } + RETURN_IF_ERROR(validate_iceberg_external_file_report_ack(state->query_options())); // Get target file size from query options // If value is 0 or not set, use config::iceberg_sink_max_file_size diff --git a/be/test/exec/sink/viceberg_delete_sink_test.cpp b/be/test/exec/sink/viceberg_delete_sink_test.cpp index 7faa77ed702c68..e2897125e497f3 100644 --- a/be/test/exec/sink/viceberg_delete_sink_test.cpp +++ b/be/test/exec/sink/viceberg_delete_sink_test.cpp @@ -94,6 +94,18 @@ TEST_F(VIcebergDeleteSinkTest, TestInitProperties) { ASSERT_TRUE(status.ok()); } +TEST_F(VIcebergDeleteSinkTest, RejectsCoordinatorWithoutExternalFileReportAck) { + VExprContextSPtrs output_exprs; + auto sink = std::make_shared(_t_data_sink, output_exprs, nullptr, nullptr); + RuntimeState state; + RuntimeProfile profile("test"); + + Status status = sink->open(&state, &profile); + + EXPECT_TRUE(status.is()); + EXPECT_NE(std::string::npos, status.to_string().find("acknowledges external-file reports")); +} + TEST_F(VIcebergDeleteSinkTest, TestGetRowIdColumnIndex) { VExprContextSPtrs output_exprs; auto sink = std::make_shared(_t_data_sink, output_exprs, nullptr, nullptr); From 6799e4eedada9f9570e16a25e3e087b0327ca3ec Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 4 Aug 2026 20:54:55 +0800 Subject: [PATCH 17/29] [fix](iceberg) Preserve external write ownership until final report Issue Number: None Related PR: #66348 Problem Summary: Periodic reports could prematurely send external commit vectors, while final-report rejection and Hive pre-commit failures could lose the last cleanup owner for deferred object-store uploads. Keep ownership-bearing vectors final-only, retain provider cleanup callbacks until the final report is accepted, and self-rollback Hive validation or classification failures. Also align the orphan-action imports with the connector SPI package. None - Test: Unit Test - HiveConnectorTransactionTest: 17 tests passed - Iceberg connector focused suites: 283 tests passed - Azure object-storage extension suite: 26 tests passed - FE core focused suites: 30 tests passed - Changed BE production and unit-test objects compiled successfully - clang-format v16 and connector import gate passed - Behavior changed: Yes, external-write ownership is transferred only by an accepted final report - Does this need documentation: No --- .../pipeline/pipeline_fragment_context.cpp | 71 ++++++------------- be/src/exec/sink/viceberg_delete_sink.cpp | 11 +-- .../writer/iceberg/viceberg_table_writer.cpp | 11 +-- .../sink/writer/vhive_partition_writer.cpp | 10 +++ be/src/io/fs/s3_file_writer.cpp | 20 ++++++ be/src/io/fs/s3_file_writer.h | 3 + be/src/runtime/runtime_state.cpp | 46 ++++++++---- be/src/runtime/runtime_state.h | 26 +++---- be/test/io/fs/s3_file_writer_test.cpp | 16 ++++- .../runtime_state_block_budget_test.cpp | 53 ++++++++++---- .../hive/HiveConnectorTransaction.java | 57 ++++++++++----- .../hive/HiveConnectorTransactionTest.java | 46 ++++++++++++ .../IcebergRemoveOrphanFilesAction.java | 10 +-- .../IcebergRemoveOrphanFilesActionTest.java | 4 +- 14 files changed, 263 insertions(+), 121 deletions(-) diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index 6c471f8817be9d..06b5a737a9df99 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -470,8 +470,8 @@ Status PipelineFragmentContext::_build_pipeline_tasks_for_instance( _params.query_options, _query_ctx->query_globals, _exec_env, _query_ctx.get()); { // Initialize runtime state for this task - task_runtime_state->set_iceberg_commit_data_budget( - _runtime_state->iceberg_commit_data_budget()); + task_runtime_state->set_external_file_report_state( + _runtime_state->external_file_report_state()); task_runtime_state->set_query_mem_tracker(_query_ctx->query_mem_tracker()); task_runtime_state->set_task_execution_context(shared_from_this()); @@ -2365,7 +2365,8 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r // External query (flink/spark read tablets) not need to report to FE. if (req.done) { // Without a coordinator acknowledgement no external-write file may escape rollback. - req.runtime_state->finalize_iceberg_report_cleanup(IcebergReportOutcome::REJECTED); + req.runtime_state->finalize_external_file_report_cleanup( + ExternalFileReportOutcome::REJECTED); } return; } @@ -2388,7 +2389,8 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r "query_id: {}, couldn't get a client for {}, reason is {}", uid.to_string(), PrintThriftNetworkAddress(req.coord_addr), coord_status.to_string()))); if (req.done) { - req.runtime_state->finalize_iceberg_report_cleanup(IcebergReportOutcome::REJECTED); + req.runtime_state->finalize_external_file_report_cleanup( + ExternalFileReportOutcome::REJECTED); } return; } @@ -2513,42 +2515,9 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r } } } - if (auto hpu = req.runtime_state->hive_partition_updates(); !hpu.empty()) { - params.__isset.hive_partition_updates = true; - params.hive_partition_updates.insert(params.hive_partition_updates.end(), hpu.begin(), - hpu.end()); - } else if (!req.runtime_states.empty()) { - for (auto* rs : req.runtime_states) { - if (auto rs_hpu = rs->hive_partition_updates(); !rs_hpu.empty()) { - params.__isset.hive_partition_updates = true; - params.hive_partition_updates.insert(params.hive_partition_updates.end(), - rs_hpu.begin(), rs_hpu.end()); - } - } - } - req.runtime_state->append_iceberg_commit_datas(¶ms.iceberg_commit_datas); - if (!params.iceberg_commit_datas.empty()) { - params.__isset.iceberg_commit_datas = true; - } else if (!req.runtime_states.empty()) { - for (auto* rs : req.runtime_states) { - rs->append_iceberg_commit_datas(¶ms.iceberg_commit_datas); - if (!params.iceberg_commit_datas.empty()) { - params.__isset.iceberg_commit_datas = true; - } - } - } - - if (auto mcd = req.runtime_state->mc_commit_datas(); !mcd.empty()) { - params.__isset.mc_commit_datas = true; - params.mc_commit_datas.insert(params.mc_commit_datas.end(), mcd.begin(), mcd.end()); - } else if (!req.runtime_states.empty()) { - for (auto* rs : req.runtime_states) { - if (auto rs_mcd = rs->mc_commit_datas(); !rs_mcd.empty()) { - params.__isset.mc_commit_datas = true; - params.mc_commit_datas.insert(params.mc_commit_datas.end(), rs_mcd.begin(), - rs_mcd.end()); - } - } + req.runtime_state->append_external_file_commit_data(¶ms, req.done); + for (auto* rs : req.runtime_states) { + rs->append_external_file_commit_data(¶ms, req.done); } req.runtime_state->get_unreported_errors(&(params.error_log)); @@ -2562,7 +2531,8 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r params, req.runtime_state->coordinator_thrift_message_limit()); if (!report_size_status.ok()) { if (req.done) { - req.runtime_state->finalize_iceberg_report_cleanup(IcebergReportOutcome::REJECTED); + req.runtime_state->finalize_external_file_report_cleanup( + ExternalFileReportOutcome::REJECTED); } req.cancel_fn(report_size_status); return; @@ -2593,8 +2563,8 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r // The first request may have been consumed; keep files until metadata or orphan cleanup wins. report_outcome_ambiguous = true; if (req.done) { - req.runtime_state->finalize_iceberg_report_cleanup( - IcebergReportOutcome::AMBIGUOUS); + req.runtime_state->finalize_external_file_report_cleanup( + ExternalFileReportOutcome::AMBIGUOUS); } req.cancel_fn(rpc_status); return; @@ -2619,19 +2589,23 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r if (!rpc_status.ok()) { if (req.done && !report_outcome_ambiguous) { - req.runtime_state->finalize_iceberg_report_cleanup(IcebergReportOutcome::REJECTED); + req.runtime_state->finalize_external_file_report_cleanup( + ExternalFileReportOutcome::REJECTED); } else if (req.done) { - req.runtime_state->finalize_iceberg_report_cleanup(IcebergReportOutcome::AMBIGUOUS); + req.runtime_state->finalize_external_file_report_cleanup( + ExternalFileReportOutcome::AMBIGUOUS); } LOG_INFO("Going to cancel query {} since report exec status got rpc failed: {}", print_id(req.query_id), rpc_status.to_string()); req.cancel_fn(rpc_status); } else if (req.done && req.status.ok()) { // Files remain rollback-owned until the coordinator has acknowledged the final metadata report. - req.runtime_state->finalize_iceberg_report_cleanup(IcebergReportOutcome::ACKNOWLEDGED); + req.runtime_state->finalize_external_file_report_cleanup( + ExternalFileReportOutcome::ACKNOWLEDGED); } else if (req.done) { // An acknowledged error report confirms that FE will not publish this write's files. - req.runtime_state->finalize_iceberg_report_cleanup(IcebergReportOutcome::REJECTED); + req.runtime_state->finalize_external_file_report_cleanup( + ExternalFileReportOutcome::REJECTED); } } @@ -2688,7 +2662,8 @@ Status PipelineFragmentContext::send_report(bool done) { }); if (!submit_status.ok() && req.done) { // A rejected final callback can never transfer ownership to the coordinator. - req.runtime_state->finalize_iceberg_report_cleanup(IcebergReportOutcome::REJECTED); + req.runtime_state->finalize_external_file_report_cleanup( + ExternalFileReportOutcome::REJECTED); } return submit_status; } diff --git a/be/src/exec/sink/viceberg_delete_sink.cpp b/be/src/exec/sink/viceberg_delete_sink.cpp index 1a572f57da7260..92d26b560f7160 100644 --- a/be/src/exec/sink/viceberg_delete_sink.cpp +++ b/be/src/exec/sink/viceberg_delete_sink.cpp @@ -314,11 +314,12 @@ void VIcebergDeleteSink::finish_deferred_file_cleanup(Status outer_status) { void VIcebergDeleteSink::_transfer_created_files_to_report_cleanup() { DCHECK(_state != nullptr); for (auto& created_file : _created_files) { - _state->add_failed_iceberg_report_cleanup([cleanup_fs = std::move(created_file.first), - cleanup_path = std::move(created_file.second)] { - WARN_IF_ERROR(cleanup_fs->delete_file(cleanup_path), - "failed to delete an Iceberg delete file after report failure"); - }); + _state->add_rejected_external_file_report_cleanup( + [cleanup_fs = std::move(created_file.first), + cleanup_path = std::move(created_file.second)] { + WARN_IF_ERROR(cleanup_fs->delete_file(cleanup_path), + "failed to delete an Iceberg delete file after report failure"); + }); } _created_files.clear(); } diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp index 05175baeca73c4..5b7c31d6c0d9cb 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp @@ -605,11 +605,12 @@ void VIcebergTableWriter::finish_deferred_file_cleanup(Status outer_status) { void VIcebergTableWriter::_transfer_closed_files_to_report_cleanup() { DCHECK(_state != nullptr); for (auto& closed_file : _closed_files) { - _state->add_failed_iceberg_report_cleanup([cleanup_fs = std::move(closed_file.first), - cleanup_path = std::move(closed_file.second)] { - WARN_IF_ERROR(cleanup_fs->delete_file(cleanup_path), - "failed to delete an Iceberg file after report failure"); - }); + _state->add_rejected_external_file_report_cleanup( + [cleanup_fs = std::move(closed_file.first), + cleanup_path = std::move(closed_file.second)] { + WARN_IF_ERROR(cleanup_fs->delete_file(cleanup_path), + "failed to delete an Iceberg file after report failure"); + }); } _closed_files.clear(); } diff --git a/be/src/exec/sink/writer/vhive_partition_writer.cpp b/be/src/exec/sink/writer/vhive_partition_writer.cpp index a7960850e3d198..658da3dbc8194f 100644 --- a/be/src/exec/sink/writer/vhive_partition_writer.cpp +++ b/be/src/exec/sink/writer/vhive_partition_writer.cpp @@ -162,6 +162,12 @@ Status VHivePartitionWriter::close(const Status& status) { } if (status_ok) { auto partition_update = _build_partition_update(); + if (partition_update.__isset.s3_mpu_pending_uploads) { + auto* s3_writer = dynamic_cast(_file_writer.get()); + DCHECK(s3_writer != nullptr); + // Until FE accepts the final report, BE remains the cleanup owner of staged uploads. + _state->add_rejected_external_file_report_cleanup(s3_writer->failed_report_cleanup()); + } _state->add_hive_partition_updates(partition_update); } return result_status; @@ -224,6 +230,10 @@ void VHivePartitionWriter::_add_s3_mpu_pending_upload_for_rollback() { if (!_build_s3_mpu_pending_upload(&s3_mpu_pending_upload)) { return; } + auto* s3_writer = dynamic_cast(_file_writer.get()); + DCHECK(s3_writer != nullptr); + // A failed write still relies on the final report to hand its staged upload to FE rollback. + _state->add_rejected_external_file_report_cleanup(s3_writer->failed_report_cleanup()); THivePartitionUpdate hive_partition_update; hive_partition_update.__set_name(_partition_name); diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index 43663afa6dfdf8..63eb6e32c44c25 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -105,6 +105,26 @@ Status S3FileWriter::abort() { return Status::OK(); } +std::function S3FileWriter::failed_report_cleanup() const { + auto client_holder = _obj_client; + auto path_opts = _obj_storage_path_opts; + return [client_holder = std::move(client_holder), path_opts = std::move(path_opts)] { + // The writer is already CLOSED, but ownership was not transferred; bypass abort()'s + // state guard while retaining the provider client and the exact upload identity. + const auto& client = client_holder->get(); + if (client == nullptr) { + LOG(WARNING) << "failed to abort a rejected external-file report: invalid object " + "storage client"; + return; + } + auto response = client->abort_multipart_upload(path_opts); + if (response.status.code != ErrorCode::OK) { + LOG(WARNING) << "failed to abort a rejected external-file report for " + << path_opts.path.native() << ": " << response.status.msg; + } + }; +} + Status S3FileWriter::_abort_impl() { _wait_until_finish( fmt::format("wait s3 file {} before abort", _obj_storage_path_opts.path.native())); diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h index eb40772eea095e..83a2100c0ff943 100644 --- a/be/src/io/fs/s3_file_writer.h +++ b/be/src/io/fs/s3_file_writer.h @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -74,6 +75,8 @@ class S3FileWriter final : public FileWriter { Status abort() override; Status try_finish_close() override; + std::function failed_report_cleanup() const; + private: Status _abort_impl(); Status _close_impl(); diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 3c1e756e90ec59..f2052910faf11b 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -70,16 +70,16 @@ Status RuntimeState::add_iceberg_commit_datas(TIcebergCommitData iceberg_commit_ const size_t thrift_limit = coordinator_thrift_message_limit(); const size_t commit_data_limit = thrift_limit > report_envelope_headroom ? thrift_limit - report_envelope_headroom : 0; - std::lock_guard budget_lock(_iceberg_commit_data_budget->mutex); + std::lock_guard budget_lock(_external_file_report_state->mutex); // Parallel task states share this budget because FE receives their vectors in one fragment report. - if (_iceberg_commit_data_budget->serialized_bytes + serialized_size + sizeof(uint32_t) > + if (_external_file_report_state->iceberg_serialized_bytes + serialized_size + sizeof(uint32_t) > commit_data_limit) { return Status::InternalError( "Iceberg commit metadata exceeds the Thrift report limit; reduce output file " "count"); } std::lock_guard data_lock(_iceberg_commit_datas_mutex); - _iceberg_commit_data_budget->serialized_bytes += serialized_size + sizeof(uint32_t); + _external_file_report_state->iceberg_serialized_bytes += serialized_size + sizeof(uint32_t); _iceberg_commit_datas.emplace_back(std::move(iceberg_commit_data)); return Status::OK(); } @@ -95,24 +95,46 @@ size_t RuntimeState::coordinator_thrift_message_limit() const { return static_cast(effective_thrift_limit); } -void RuntimeState::add_failed_iceberg_report_cleanup(std::function cleanup) { - std::lock_guard lock(_iceberg_commit_data_budget->mutex); - _iceberg_commit_data_budget->failed_report_cleanups.emplace_back(std::move(cleanup)); +void RuntimeState::append_external_file_commit_data(TReportExecStatusParams* params, + bool final_report) const { + if (!final_report) { + // Ownership-bearing commit vectors must only appear in the final report that transfers them. + return; + } + if (auto updates = hive_partition_updates(); !updates.empty()) { + params->__isset.hive_partition_updates = true; + params->hive_partition_updates.insert(params->hive_partition_updates.end(), updates.begin(), + updates.end()); + } + append_iceberg_commit_datas(¶ms->iceberg_commit_datas); + if (!params->iceberg_commit_datas.empty()) { + params->__isset.iceberg_commit_datas = true; + } + if (auto commit_datas = mc_commit_datas(); !commit_datas.empty()) { + params->__isset.mc_commit_datas = true; + params->mc_commit_datas.insert(params->mc_commit_datas.end(), commit_datas.begin(), + commit_datas.end()); + } +} + +void RuntimeState::add_rejected_external_file_report_cleanup(std::function cleanup) { + std::lock_guard lock(_external_file_report_state->mutex); + _external_file_report_state->rejected_report_cleanups.emplace_back(std::move(cleanup)); } -void RuntimeState::finalize_iceberg_report_cleanup(IcebergReportOutcome outcome) { +void RuntimeState::finalize_external_file_report_cleanup(ExternalFileReportOutcome outcome) { std::vector> cleanups; { - std::lock_guard lock(_iceberg_commit_data_budget->mutex); - if (outcome == IcebergReportOutcome::ACKNOWLEDGED) { - _iceberg_commit_data_budget->failed_report_cleanups.clear(); + std::lock_guard lock(_external_file_report_state->mutex); + if (outcome == ExternalFileReportOutcome::ACKNOWLEDGED) { + _external_file_report_state->rejected_report_cleanups.clear(); return; } - if (outcome == IcebergReportOutcome::AMBIGUOUS) { + if (outcome == ExternalFileReportOutcome::AMBIGUOUS) { // A consumed request with a lost ACK may already be publishing these files. return; } - cleanups.swap(_iceberg_commit_data_budget->failed_report_cleanups); + cleanups.swap(_external_file_report_state->rejected_report_cleanups); } for (auto& cleanup : cleanups) { cleanup(); diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index d8163a3c32e04d..2cd5503a23aa7d 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -76,16 +76,16 @@ class RuntimeFilterConsumer; class RuntimeFilterProducer; class TaskExecutionContext; -class IcebergCommitDataBudget { +class ExternalFileReportState { friend class RuntimeState; private: std::mutex mutex; - size_t serialized_bytes = 0; - std::vector> failed_report_cleanups; + size_t iceberg_serialized_bytes = 0; + std::vector> rejected_report_cleanups; }; -enum class IcebergReportOutcome { ACKNOWLEDGED, REJECTED, AMBIGUOUS }; +enum class ExternalFileReportOutcome { ACKNOWLEDGED, REJECTED, AMBIGUOUS }; // A collection of items that are part of the global state of a // query and shared across all execution nodes of that query. @@ -545,16 +545,18 @@ class RuntimeState { size_t coordinator_thrift_message_limit() const; - void add_failed_iceberg_report_cleanup(std::function cleanup); + void append_external_file_commit_data(TReportExecStatusParams* params, bool final_report) const; - void finalize_iceberg_report_cleanup(IcebergReportOutcome outcome); + void add_rejected_external_file_report_cleanup(std::function cleanup); - void set_iceberg_commit_data_budget(std::shared_ptr budget) { - _iceberg_commit_data_budget = std::move(budget); + void finalize_external_file_report_cleanup(ExternalFileReportOutcome outcome); + + void set_external_file_report_state(std::shared_ptr report_state) { + _external_file_report_state = std::move(report_state); } - const std::shared_ptr& iceberg_commit_data_budget() const { - return _iceberg_commit_data_budget; + const std::shared_ptr& external_file_report_state() const { + return _external_file_report_state; } std::vector mc_commit_datas() const { @@ -1000,8 +1002,8 @@ class RuntimeState { mutable std::mutex _iceberg_commit_datas_mutex; std::vector _iceberg_commit_datas; - std::shared_ptr _iceberg_commit_data_budget = - std::make_shared(); + std::shared_ptr _external_file_report_state = + std::make_shared(); mutable std::mutex _mc_commit_datas_mutex; std::vector _mc_commit_datas; diff --git a/be/test/io/fs/s3_file_writer_test.cpp b/be/test/io/fs/s3_file_writer_test.cpp index b00f6c491c72d4..c0c42239a1858a 100644 --- a/be/test/io/fs/s3_file_writer_test.cpp +++ b/be/test/io/fs/s3_file_writer_test.cpp @@ -1319,8 +1319,9 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { * @return A tuple containing the mock S3 client and the S3FileWriter. */ std::tuple, std::shared_ptr> -create_s3_client(const std::string& path) { +create_s3_client(const std::string& path, bool used_by_s3_committer = false) { doris::io::FileWriterOptions opts; + opts.used_by_s3_committer = used_by_s3_committer; io::FileWriterPtr file_writer; auto st = s3_fs->create_file(path, &file_writer, &opts); EXPECT_TRUE(st.ok()) << st; @@ -1345,6 +1346,19 @@ TEST_F(S3FileWriterTest, abortsProviderMultipartWithoutAnUploadId) { EXPECT_EQ(FileWriter::State::CLOSED, writer->state()); } +TEST_F(S3FileWriterTest, failedReportCleanupAbortsDeferredProviderUploadAfterClose) { + auto [client, writer] = create_s3_client("deferred_report_rejected", true); + std::string data(config::s3_write_buffer_size, 'a'); + ASSERT_TRUE(writer->append(Slice(data)).ok()); + ASSERT_TRUE(writer->close().ok()); + ASSERT_EQ(FileWriter::State::CLOSED, writer->state()); + auto cleanup = writer->failed_report_cleanup(); + + cleanup(); + + EXPECT_EQ(1, client->abort_multipart_count); +} + /** * Generate test data for S3FileWriter boundary tests. * Returns a vector of sizes that we'll use to generate data on demand. diff --git a/be/test/runtime/runtime_state_block_budget_test.cpp b/be/test/runtime/runtime_state_block_budget_test.cpp index 802e9c54b5f1b7..16a078bd64c8bf 100644 --- a/be/test/runtime/runtime_state_block_budget_test.cpp +++ b/be/test/runtime/runtime_state_block_budget_test.cpp @@ -44,9 +44,9 @@ TEST(RuntimeStateIcebergCommitDataTest, RejectsMetadataBeforeItCanExceedTheThrif TEST(RuntimeStateIcebergCommitDataTest, SharesTheReportBudgetAcrossParallelTasks) { RuntimeState first; RuntimeState second; - auto budget = std::make_shared(); - first.set_iceberg_commit_data_budget(budget); - second.set_iceberg_commit_data_budget(budget); + auto budget = std::make_shared(); + first.set_external_file_report_state(budget); + second.set_external_file_report_state(budget); const int32_t saved_limit = config::thrift_max_message_size; config::thrift_max_message_size = 1024 * 1024 + 512; TIcebergCommitData commit_data; @@ -82,28 +82,53 @@ TEST(RuntimeStateIcebergCommitDataTest, ValidatesTheCompleteReportEnvelope) { EXPECT_TRUE(validate_report_exec_status_size(params, 3 * 1024 * 1024).ok()); } +TEST(RuntimeStateIcebergCommitDataTest, PeriodicReportOmitsExternalCommitData) { + RuntimeState state; + THivePartitionUpdate hive_update; + state.add_hive_partition_updates(hive_update); + TIcebergCommitData iceberg_data; + iceberg_data.__set_file_path("data.parquet"); + ASSERT_TRUE(state.add_iceberg_commit_datas(iceberg_data).ok()); + TMCCommitData mc_data; + state.add_mc_commit_datas(mc_data); + TReportExecStatusParams periodic_params; + + state.append_external_file_commit_data(&periodic_params, false); + + EXPECT_FALSE(periodic_params.__isset.hive_partition_updates); + EXPECT_FALSE(periodic_params.__isset.iceberg_commit_datas); + EXPECT_FALSE(periodic_params.__isset.mc_commit_datas); + + TReportExecStatusParams final_params; + state.append_external_file_commit_data(&final_params, true); + EXPECT_TRUE(final_params.__isset.hive_partition_updates); + EXPECT_TRUE(final_params.__isset.iceberg_commit_datas); + EXPECT_TRUE(final_params.__isset.mc_commit_datas); +} + TEST(RuntimeStateIcebergCommitDataTest, RetainsFileCleanupUntilReportAcknowledgement) { RuntimeState coordinator_state; RuntimeState task_state; - auto report_state = std::make_shared(); - coordinator_state.set_iceberg_commit_data_budget(report_state); - task_state.set_iceberg_commit_data_budget(report_state); + auto report_state = std::make_shared(); + coordinator_state.set_external_file_report_state(report_state); + task_state.set_external_file_report_state(report_state); int cleanup_count = 0; - task_state.add_failed_iceberg_report_cleanup([&] { ++cleanup_count; }); + task_state.add_rejected_external_file_report_cleanup([&] { ++cleanup_count; }); - coordinator_state.finalize_iceberg_report_cleanup(IcebergReportOutcome::REJECTED); - coordinator_state.finalize_iceberg_report_cleanup(IcebergReportOutcome::REJECTED); + coordinator_state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::REJECTED); + coordinator_state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::REJECTED); EXPECT_EQ(1, cleanup_count); - task_state.add_failed_iceberg_report_cleanup([&] { ++cleanup_count; }); - coordinator_state.finalize_iceberg_report_cleanup(IcebergReportOutcome::ACKNOWLEDGED); + task_state.add_rejected_external_file_report_cleanup([&] { ++cleanup_count; }); + coordinator_state.finalize_external_file_report_cleanup( + ExternalFileReportOutcome::ACKNOWLEDGED); EXPECT_EQ(1, cleanup_count); - task_state.add_failed_iceberg_report_cleanup([&] { ++cleanup_count; }); - coordinator_state.finalize_iceberg_report_cleanup(IcebergReportOutcome::AMBIGUOUS); + task_state.add_rejected_external_file_report_cleanup([&] { ++cleanup_count; }); + coordinator_state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::AMBIGUOUS); EXPECT_EQ(1, cleanup_count); - coordinator_state.finalize_iceberg_report_cleanup(IcebergReportOutcome::REJECTED); + coordinator_state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::REJECTED); EXPECT_EQ(2, cleanup_count); } diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorTransaction.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorTransaction.java index 315404b82df529..e7a95fe6999c19 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorTransaction.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorTransaction.java @@ -288,15 +288,23 @@ private ScheduledFuture startCommitLockHeartbeat(long lockId) { } private void commitWhileTableLocked() { - // Object-store files remain unpublished until FE consumes one completion record per file. - validateObjectStoreCommitRecords(); - validateWriteMetadataBeforePublication(); - // The classification (finishInsertTable) ran from the executor in the legacy class; the unified SPI - // exposes only commit(), so it runs here (before the committer) to populate the action maps. If it - // throws, the committer was never created and the engine's subsequent rollback() cleans up. - finishInsertTable(nameMapping); - // Classification can perform metastore reads, so close that interval before any file or HMS mutation. - validateWriteMetadataBeforePublication(); + try { + // Object-store files remain unpublished until FE consumes one completion record per file. + validateObjectStoreCommitRecords(); + validateWriteMetadataBeforePublication(); + // Classification can perform metastore reads, so validate once more before any publication. + finishInsertTable(nameMapping); + validateWriteMetadataBeforePublication(); + } catch (Throwable t) { + // The transaction manager removes this connector before commit(), so this is the last owner + // capable of aborting deferred uploads when pre-commit validation or classification fails. + try { + rollback(); + } catch (Throwable cleanupFailure) { + t.addSuppressed(new Exception("Failed to clean up after pre-commit failure", cleanupFailure)); + } + throw t; + } hmsCommitter = new HmsCommitter(); try { for (Map.Entry> entry : tableActions.entrySet()) { @@ -446,10 +454,8 @@ private void validateObjectStoreCommitRecords() { int fileCount = update.getFileNames() == null ? 0 : update.getFileNames().size(); List uploads = update.getS3MpuPendingUploads(); int uploadCount = uploads == null ? 0 : uploads.size(); - boolean completeRecords = uploads != null && uploads.stream().allMatch(upload -> - upload != null && upload.getUploadId() != null && !upload.getUploadId().isEmpty() - && upload.getBucket() != null && !upload.getBucket().isEmpty() - && upload.getKey() != null && !upload.getKey().isEmpty()); + boolean completeRecords = uploads != null && uploads.stream() + .allMatch(HiveConnectorTransaction::isCompleteObjectStoreUpload); if (fileCount != uploadCount || (fileCount > 0 && !completeRecords)) { throw new DorisConnectorException(String.format( "Object-store write reported %d file(s) but %d valid multipart completion record(s); " @@ -631,15 +637,32 @@ void finishInsertTable(NameMapping nameMapping) { private void collectUncompletedMpuPendingUploads(List hivePartitionUpdates) { for (THivePartitionUpdate pu : hivePartitionUpdates) { - if (pu.getS3MpuPendingUploads() != null) { - for (TS3MPUPendingUpload s3MpuPendingUpload : pu.getS3MpuPendingUploads()) { - uncompletedMpuPendingUploads.add( - new UncompletedMpuPendingUpload(s3MpuPendingUpload, pu.getLocation().getWritePath())); + List uploads = pu.getS3MpuPendingUploads(); + if (uploads == null) { + continue; + } + String writePath = pu.getLocation() == null ? null : pu.getLocation().getWritePath(); + if (writePath == null || writePath.isEmpty()) { + // A malformed record must not prevent other valid uploads from being aborted. + LOG.warn("Skipping MPU cleanup record without a write path"); + continue; + } + for (TS3MPUPendingUpload upload : uploads) { + if (!isCompleteObjectStoreUpload(upload)) { + LOG.warn("Skipping incomplete MPU cleanup record for write path {}", writePath); + continue; } + uncompletedMpuPendingUploads.add(new UncompletedMpuPendingUpload(upload, writePath)); } } } + private static boolean isCompleteObjectStoreUpload(TS3MPUPendingUpload upload) { + return upload != null && upload.getUploadId() != null && !upload.getUploadId().isEmpty() + && upload.getBucket() != null && !upload.getBucket().isEmpty() + && upload.getKey() != null && !upload.getKey().isEmpty(); + } + private void convertToInsertExistingPartitionAction( NameMapping nameMapping, List> partitions) { diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorTransactionTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorTransactionTest.java index 769dbac8cd471d..f14ceec77d9623 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorTransactionTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorTransactionTest.java @@ -476,6 +476,52 @@ public void testCommitRejectsBaseBeObjectStoreUpdateWithoutPendingUpload() throw "metadata must remain unchanged when the object is still uncommitted"); } + @Test + public void testCommitValidationFailureAbortsEveryValidPendingUpload() throws TException { + RecordingHmsClient client = new RecordingHmsClient(); + client.table = table(false, Collections.emptyMap()); + RecordingObjStorage objStorage = new RecordingObjStorage(); + HiveConnectorTransaction txn = newTxnWithFs(client, new RecordingObjFileSystem(objStorage)); + txn.beginWrite(null, DB, TBL, ctx(false)); + txn.addCommitData(serialize(puWithMpu("", TUpdateMode.APPEND, "s3://bucket/db/t", + "bucket", "db/t/valid", "upload-valid", Collections.singletonMap(1, "etag-1")))); + THivePartitionUpdate malformed = puWithMpu("", TUpdateMode.APPEND, "s3://bucket/db/t", + "bucket", "db/t/malformed", "upload-malformed", Collections.singletonMap(1, "etag-1")); + malformed.unsetLocation(); + txn.addCommitData(serialize(malformed)); + txn.addCommitData(serialize(pu("", TUpdateMode.APPEND, "s3://bucket/db/t", + Collections.singletonList("missing-completion"), 100, 4))); + + Assertions.assertThrows(DorisConnectorException.class, txn::commit); + + Assertions.assertEquals(Collections.singletonList("abort:s3://bucket/db/t/valid:upload-valid"), + objStorage.calls, + "a pre-committer validation failure must abort every valid provider upload it rejects"); + Assertions.assertFalse(client.calls.stream().anyMatch(c -> c.startsWith("updateTableStatistics")), + "validation failure must not publish HMS metadata"); + } + + @Test + public void testCommitClassificationFailureAbortsPendingUpload() throws TException { + RecordingHmsClient client = new RecordingHmsClient(); + client.table = table(false, Collections.emptyMap()); + RecordingObjStorage objStorage = new RecordingObjStorage(); + HiveConnectorTransaction txn = newTxnWithFs(client, new RecordingObjFileSystem(objStorage)); + txn.beginWrite(null, DB, TBL, ctx(false)); + txn.addCommitData(serialize(puWithMpu("", TUpdateMode.NEW, "s3://bucket/db/t", + "bucket", "db/t/unclassified", "upload-unclassified", + Collections.singletonMap(1, "etag-1")))); + + Assertions.assertThrows(RuntimeException.class, txn::commit); + + Assertions.assertEquals( + Collections.singletonList("abort:s3://bucket/db/t/unclassified:upload-unclassified"), + objStorage.calls, + "a classification failure before HmsCommitter creation must abort its provider upload"); + Assertions.assertFalse(client.calls.stream().anyMatch(c -> c.startsWith("updateTableStatistics")), + "classification failure must not publish HMS metadata"); + } + @Test public void testRollbackAbortsPendingMultipartUploads() throws TException { // rollback() is NOT a no-op for hive (D9): data files are staged before commit, so a rollback must diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java index 40a5f312fb8a43..7fb4fbeb608564 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java @@ -17,11 +17,11 @@ package org.apache.doris.connector.iceberg.action; -import org.apache.doris.connector.api.ConnectorColumn; -import org.apache.doris.connector.api.ConnectorSession; -import org.apache.doris.connector.api.ConnectorType; -import org.apache.doris.connector.api.DorisConnectorException; -import org.apache.doris.connector.api.pushdown.ConnectorPredicate; +import org.apache.doris.connector.spi.ConnectorColumn; +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorType; +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.connector.spi.pushdown.ConnectorPredicate; import org.apache.doris.foundation.util.ArgumentParsers; import com.google.common.collect.Lists; diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java index b09ce892e2ea30..bce05901cead38 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java @@ -17,8 +17,8 @@ package org.apache.doris.connector.iceberg.action; -import org.apache.doris.connector.api.DorisConnectorException; -import org.apache.doris.connector.api.procedure.ConnectorProcedureResult; +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.connector.spi.procedure.ConnectorProcedureResult; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.BaseTable; From 403b77a48d28d7d8eb43d7f7125539258b60fa22 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 5 Aug 2026 10:58:00 +0800 Subject: [PATCH 18/29] [fix](iceberg) Avoid writer test suite collision --- .../exec/sink/writer/iceberg/viceberg_table_writer.h | 3 ++- .../sink/writer/iceberg/iceberg_table_writer_test.cpp | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h index 7947216c933e44..0d6bede9a1a783 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h @@ -79,7 +79,8 @@ class VIcebergTableWriter final : public AsyncResultWriter { private: FRIEND_TEST(VIcebergTableWriterTest, RejectMissingPartitionSource); FRIEND_TEST(VIcebergTableWriterTest, ResolvesNestedPartitionSource); - friend class VIcebergTableWriterTest; + // The lifecycle fixture inspects snapshots to verify that cross-thread writer ownership stays stable. + friend class VIcebergTableWriterLifecycleTest; // The spill thread needs a stable view of every partition sorter, while the async writer owns the map. doris::atomic_shared_ptr _active_writers; diff --git a/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp b/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp index 5ca983fca439f8..d486672c0c83db 100644 --- a/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp +++ b/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp @@ -61,7 +61,7 @@ TDataSink make_sink() { } // namespace -class VIcebergTableWriterTest : public testing::Test { +class VIcebergTableWriterLifecycleTest : public testing::Test { protected: static Status select_block(VIcebergTableWriter* writer, Block& input, const IColumn::Permutation& rows, Block* selected) { @@ -87,7 +87,7 @@ class VIcebergTableWriterTest : public testing::Test { } }; -TEST_F(VIcebergTableWriterTest, RejectsCoordinatorWithoutExternalFileReportAck) { +TEST_F(VIcebergTableWriterLifecycleTest, RejectsCoordinatorWithoutExternalFileReportAck) { VIcebergTableWriter writer(make_sink(), {}, nullptr, nullptr); RuntimeState state; RuntimeProfile profile("test"); @@ -98,7 +98,7 @@ TEST_F(VIcebergTableWriterTest, RejectsCoordinatorWithoutExternalFileReportAck) EXPECT_NE(std::string::npos, status.to_string().find("acknowledges external-file reports")); } -TEST_F(VIcebergTableWriterTest, SelectBlockUsesRowPermutation) { +TEST_F(VIcebergTableWriterLifecycleTest, SelectBlockUsesRowPermutation) { VIcebergTableWriter writer(make_sink(), {}, nullptr, nullptr); auto values = ColumnInt32::create(); values->insert_value(10); @@ -117,7 +117,7 @@ TEST_F(VIcebergTableWriterTest, SelectBlockUsesRowPermutation) { EXPECT_EQ(result.get_element(1), 10); } -TEST_F(VIcebergTableWriterTest, ActiveWriterSnapshotContainsEveryOpenPartition) { +TEST_F(VIcebergTableWriterLifecycleTest, ActiveWriterSnapshotContainsEveryOpenPartition) { VIcebergTableWriter writer(make_sink(), {}, nullptr, nullptr); add_writer(&writer, "p=1"); add_writer(&writer, "p=2"); @@ -128,7 +128,7 @@ TEST_F(VIcebergTableWriterTest, ActiveWriterSnapshotContainsEveryOpenPartition) EXPECT_EQ(writer.active_writers()->size(), 2); } -TEST_F(VIcebergTableWriterTest, LoadedSnapshotRetainsWritersDuringConcurrentPublication) { +TEST_F(VIcebergTableWriterLifecycleTest, LoadedSnapshotRetainsWritersDuringConcurrentPublication) { VIcebergTableWriter writer(make_sink(), {}, nullptr, nullptr); std::atomic destroyed = 0; add_writer(&writer, "p=1", std::make_shared(&destroyed)); From 025bd21deac0deec5ae50db2560adc466839f072 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 5 Aug 2026 13:01:15 +0800 Subject: [PATCH 19/29] [test](be) Cover deferred upload report lifecycle ### What problem does this PR solve? Issue Number: None Related PR: #66348 Problem Summary: The external-file report tests covered periodic metadata suppression and deferred provider cleanup separately, but did not exercise their ownership handoff as one lifecycle. Add a Hive writer regression that verifies a successful close retains the exact pending upload, periodic reports omit it, final reports include it, and a definite coordinator rejection aborts the retained provider upload. ### Release note None ### Check List (For Author) - Test: Unit Test - VHivePartitionWriterReportLifecycleTest.* - RuntimeStateIcebergCommitDataTest.* - S3FileWriterTest.failedReportCleanupAbortsDeferredProviderUploadAfterClose - 8 focused ASAN BE tests passed - clang-format v16 check passed - Behavior changed: No - Does this need documentation: No --- .../pipeline/pipeline_fragment_context.cpp | 14 +- .../exec/pipeline/pipeline_fragment_context.h | 2 + ...partition_writer_report_lifecycle_test.cpp | 208 ++++++++++++++++++ 3 files changed, 220 insertions(+), 4 deletions(-) create mode 100644 be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index 06b5a737a9df99..2fa064c8a68e09 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -2352,6 +2352,15 @@ std::string PipelineFragmentContext::_to_http_path(const std::string& file_name) return url.str(); } +void PipelineFragmentContext::_append_external_file_commit_data( + const ReportStatusRequest& req, TReportExecStatusParams* params) const { + // External-file cleanup remains BE-owned until the final report transfers commit metadata. + req.runtime_state->append_external_file_commit_data(params, req.done); + for (auto* rs : req.runtime_states) { + rs->append_external_file_commit_data(params, req.done); + } +} + void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& req) { DBUG_EXECUTE_IF("FragmentMgr::coordinator_callback.report_delay", { int random_seconds = req.status.is() ? 8 : 2; @@ -2515,10 +2524,7 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r } } } - req.runtime_state->append_external_file_commit_data(¶ms, req.done); - for (auto* rs : req.runtime_states) { - rs->append_external_file_commit_data(¶ms, req.done); - } + _append_external_file_commit_data(req, ¶ms); req.runtime_state->get_unreported_errors(&(params.error_log)); params.__isset.error_log = (!params.error_log.empty()); diff --git a/be/src/exec/pipeline/pipeline_fragment_context.h b/be/src/exec/pipeline/pipeline_fragment_context.h index 1a63426738bef8..7243b0214d9fa0 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.h +++ b/be/src/exec/pipeline/pipeline_fragment_context.h @@ -153,6 +153,8 @@ class PipelineFragmentContext : public TaskExecutionContext { private: void _coordinator_callback(const ReportStatusRequest& req); + void _append_external_file_commit_data(const ReportStatusRequest& req, + TReportExecStatusParams* params) const; std::string _to_http_path(const std::string& file_name) const; void _release_resource(); diff --git a/be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp b/be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp new file mode 100644 index 00000000000000..21234083984ca4 --- /dev/null +++ b/be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp @@ -0,0 +1,208 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#include "exec/pipeline/pipeline_fragment_context.h" +#include "exec/sink/writer/vhive_partition_writer.h" +#include "format/transformer/vfile_format_transformer.h" +#include "io/fs/s3_file_system.h" +#include "io/fs/s3_file_writer.h" +#include "runtime/exec_env.h" +#include "testutil/mock/mock_runtime_state.h" + +namespace doris { +namespace { + +class RecordingObjStorageClient final : public io::ObjStorageClient { +public: + io::ObjectStorageUploadResponse create_multipart_upload( + const io::ObjectStoragePathOptions&) override { + return {.resp = io::ObjectStorageResponse::OK(), .upload_id = "upload-id"}; + } + + io::ObjectStorageResponse put_object(const io::ObjectStoragePathOptions&, + std::string_view) override { + return io::ObjectStorageResponse::OK(); + } + + io::ObjectStorageUploadResponse upload_part(const io::ObjectStoragePathOptions&, + std::string_view, int) override { + return {.resp = io::ObjectStorageResponse::OK(), .etag = "etag"}; + } + + io::ObjectStorageResponse complete_multipart_upload( + const io::ObjectStoragePathOptions&, + const std::vector&) override { + return io::ObjectStorageResponse::OK(); + } + + io::ObjectStorageResponse abort_multipart_upload( + const io::ObjectStoragePathOptions& opts) override { + ++abort_count; + aborted_upload_id = opts.upload_id.value_or(""); + return io::ObjectStorageResponse::OK(); + } + + io::ObjectStorageHeadResponse head_object(const io::ObjectStoragePathOptions&) override { + return {.resp = io::ObjectStorageResponse::OK(), .file_size = 0}; + } + + io::ObjectStorageResponse get_object(const io::ObjectStoragePathOptions&, void*, size_t, size_t, + size_t*) override { + return io::ObjectStorageResponse::OK(); + } + + io::ObjectStorageResponse list_objects(const io::ObjectStoragePathOptions&, + std::vector*) override { + return io::ObjectStorageResponse::OK(); + } + + io::ObjectStorageResponse delete_objects(const io::ObjectStoragePathOptions&, + std::vector) override { + return io::ObjectStorageResponse::OK(); + } + + io::ObjectStorageResponse delete_object(const io::ObjectStoragePathOptions&) override { + return io::ObjectStorageResponse::OK(); + } + + io::ObjectStorageResponse delete_objects_recursively( + const io::ObjectStoragePathOptions&) override { + return io::ObjectStorageResponse::OK(); + } + + std::string generate_presigned_url(const io::ObjectStoragePathOptions&, int64_t, + const S3ClientConf&) override { + return {}; + } + + int abort_count = 0; + std::string aborted_upload_id; +}; + +class FixedLengthTransformer final : public VFileFormatTransformer { +public: + explicit FixedLengthTransformer(const VExprContextSPtrs& output_exprs) + : VFileFormatTransformer(nullptr, output_exprs, false) {} + + Status open() override { return Status::OK(); } + Status write(const Block&) override { return Status::OK(); } + Status close() override { return Status::OK(); } + int64_t written_len() override { return 64; } +}; + +std::unique_ptr create_closed_hive_writer( + RuntimeState* state, const VExprContextSPtrs& output_exprs, + const std::shared_ptr& client) { + THiveTableSink hive_sink; + TDataSink sink; + sink.__set_type(TDataSinkType::HIVE_TABLE_SINK); + sink.__set_hive_table_sink(hive_sink); + VHivePartitionWriter::WriteInfo write_info {.write_path = "s3://bucket/staging", + .original_write_path = "s3://bucket/table", + .target_path = "s3://bucket/table", + .file_type = TFileType::FILE_S3, + .broker_addresses = {}}; + static const std::map hadoop_conf; + auto writer = std::make_unique( + sink, "", TUpdateMode::APPEND, output_exprs, std::vector {}, + std::move(write_info), "part", 0, TFileFormatType::FORMAT_PARQUET, + TFileCompressType::PLAIN, nullptr, hadoop_conf); + + auto holder = std::make_shared(S3ClientConf {}); + holder->_client = client; + io::FileWriterOptions options {.used_by_s3_committer = true}; + auto file_writer = + std::make_unique(holder, "bucket", "table/part.parquet", &options); + file_writer->_obj_storage_path_opts.upload_id = "upload-id"; + file_writer->_state = io::FileWriter::State::CLOSED; + writer->_file_writer = std::move(file_writer); + writer->_file_format_transformer = std::make_unique(output_exprs); + writer->_state = state; + EXPECT_TRUE(writer->close(Status::OK()).ok()); + return writer; +} + +std::shared_ptr create_fragment_context(TUniqueId query_id) { + auto query_ctx = MockQueryContext::create(query_id); + return std::make_shared(query_id, TPipelineFragmentParams(), query_ctx, + ExecEnv::GetInstance(), + [](RuntimeState*, Status*) {}); +} + +ReportStatusRequest report_request(RuntimeState* state, bool done) { + TNetworkAddress address; + address.hostname = "external"; + return {.status = Status::OK(), + .runtime_states = {}, + .done = done, + .coord_addr = address, + .query_id = TUniqueId(), + .fragment_id = 0, + .fragment_instance_id = TUniqueId(), + .backend_num = 0, + .runtime_state = state, + .load_error_url = "", + .first_error_msg = "", + .cancel_fn = [](const Status&) {}}; +} + +} // namespace + +TEST(VHivePartitionWriterReportLifecycleTest, + PeriodicReportDefersMetadataAndRejectedFinalReportAbortsProviderUpload) { + MockRuntimeState state; + VExprContextSPtrs output_exprs; + auto client = std::make_shared(); + auto writer = create_closed_hive_writer(&state, output_exprs, client); + auto context = create_fragment_context(TUniqueId()); + + TReportExecStatusParams periodic_params; + auto periodic_request = report_request(&state, false); + context->_append_external_file_commit_data(periodic_request, &periodic_params); + + EXPECT_FALSE(periodic_params.__isset.hive_partition_updates); + EXPECT_EQ(0, client->abort_count); + + TReportExecStatusParams final_params; + auto final_request = report_request(&state, true); + context->_append_external_file_commit_data(final_request, &final_params); + ASSERT_TRUE(final_params.__isset.hive_partition_updates); + ASSERT_EQ(1, final_params.hive_partition_updates.size()); + ASSERT_TRUE(final_params.hive_partition_updates[0].__isset.s3_mpu_pending_uploads); + ASSERT_EQ(1, final_params.hive_partition_updates[0].s3_mpu_pending_uploads.size()); + const auto& pending_upload = final_params.hive_partition_updates[0].s3_mpu_pending_uploads[0]; + EXPECT_EQ("bucket", pending_upload.bucket); + EXPECT_EQ("table/part.parquet", pending_upload.key); + EXPECT_EQ("upload-id", pending_upload.upload_id); + + // A definite coordinator rejection must consume the same cleanup owner that was retained + // while the periodic report deliberately withheld the pending-upload record. + context->_coordinator_callback(final_request); + + EXPECT_EQ(1, client->abort_count); + EXPECT_EQ("upload-id", client->aborted_upload_id); +} + +} // namespace doris From 44cb06dab9fc9aa31fd26cdf5baea3c71209a4f1 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 6 Aug 2026 12:00:04 +0800 Subject: [PATCH 20/29] [fix](be) Declare external report Thrift parameter ### What problem does this PR solve? Issue Number: None Related PR: #66348 Problem Summary: runtime_state.h exposed a pointer to TReportExecStatusParams without declaring the generated Thrift class. Translation units that included RuntimeState without FrontendService types failed the full BE build. Forward-declare the pointer-only type so the header is self-contained without importing the full frontend service header. ### Release note None ### Check List (For Author) - Test: Manual test - Reproduced the failing ANN translation-unit compile before the fix. - All four affected ANN translation units passed ASAN syntax-only compilation after the fix. - RuntimeState and PipelineFragmentContext translation units passed ASAN syntax-only compilation. - clang-format v16 check passed for the affected header. - Full local BE build and focused BE UT were attempted but blocked before source compilation by an incomplete external Arrow thirdparty installation. - Behavior changed: No - Does this need documentation: No --- be/src/runtime/runtime_state.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 2cd5503a23aa7d..29c17da078154e 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -76,6 +76,9 @@ class RuntimeFilterConsumer; class RuntimeFilterProducer; class TaskExecutionContext; +// Keep RuntimeState self-contained without importing the full frontend Thrift service header. +class TReportExecStatusParams; + class ExternalFileReportState { friend class RuntimeState; From 7dbfd9658e0f11bc31e198dab5f5250823157a55 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 6 Aug 2026 13:21:26 +0800 Subject: [PATCH 21/29] [fix](iceberg) Bound partition sorter memory admission ### What problem does this PR solve? Issue Number: None Related PR: #66348 Problem Summary: Iceberg sorted writes could either sum a full batch estimate for every active partition or under-account cold partition dispatch and near-capacity sorter growth. The existing regressions also tested several ownership helpers in isolation instead of exercising the asynchronous writer, EOS merge, real partition selection, and Azure final-report paths. Derive safe row and byte upper bounds from the actual incoming block, preserve cumulative growth for every sorter that can be touched, reserve cold dispatch copies conservatively, and use saturating arithmetic for all reservation totals. Add production-path tests for reservation transfer, terminal cleanup, merge fan-in, many-partition selection, and Azure provider cleanup. ### Release note Harden Iceberg sorted-write memory admission for many-partition input and asynchronous writer lifecycle boundaries. ### Check List (For Author) - Test: Unit Test - 29 focused ASAN BE tests passed across spill admission, async writer, Iceberg partition/table writers, Hive report lifecycle, sorter, and reservation transfer suites. - All affected production and test objects compiled successfully in the ASAN build. - All 12 affected C/C++ files passed clang-format 16. - Focused clang-tidy findings in the changed sorter code were fixed; full translation-unit analysis remains blocked by pre-existing unconditional static assertions in be/src/util/jni-util.h. - Behavior changed: Yes. Memory admission now uses the actual input rows and bytes while retaining safe cumulative growth and cold-writer bounds. - Does this need documentation: No --- .../operator/iceberg_sorter_reserve_memory.h | 80 ++++++-- .../spill_iceberg_table_sink_operator.cpp | 23 ++- .../writer/iceberg/viceberg_sort_writer.cpp | 23 ++- .../writer/iceberg/viceberg_sort_writer.h | 7 + be/src/exec/sort/sorter.cpp | 54 +++++- be/src/exec/sort/sorter.h | 12 +- ...spill_iceberg_table_sink_operator_test.cpp | 27 ++- .../sink/writer/async_result_writer_test.cpp | 180 ++++++++++++++++++ .../iceberg/iceberg_partition_writer_test.cpp | 27 +++ .../iceberg/iceberg_table_writer_test.cpp | 24 +++ ...partition_writer_report_lifecycle_test.cpp | 37 +++- be/test/exec/sort/full_sort_test.cpp | 10 +- 12 files changed, 467 insertions(+), 37 deletions(-) create mode 100644 be/test/exec/sink/writer/async_result_writer_test.cpp diff --git a/be/src/exec/operator/iceberg_sorter_reserve_memory.h b/be/src/exec/operator/iceberg_sorter_reserve_memory.h index 3026922219e2aa..8b09a0af4dfbfc 100644 --- a/be/src/exec/operator/iceberg_sorter_reserve_memory.h +++ b/be/src/exec/operator/iceberg_sorter_reserve_memory.h @@ -27,30 +27,88 @@ class Block; struct IcebergSorterReserveMemory { size_t retained_growth = 0; + size_t retained_growth_trigger_bytes = 0; size_t transient_workspace = 0; }; +inline size_t iceberg_saturating_add(size_t lhs, size_t rhs) { + return std::min(std::numeric_limits::max() - lhs, rhs) + lhs; +} + inline size_t bounded_iceberg_reserve_size( - const std::vector& per_partition_reservations) { - size_t retained_growth = 0; + const std::vector& per_partition_reservations, + size_t incoming_rows = std::numeric_limits::max(), + size_t incoming_bytes = std::numeric_limits::max()) { size_t transient_workspace = 0; for (const auto& reservation : per_partition_reservations) { - retained_growth = std::min(std::numeric_limits::max() - retained_growth, - reservation.retained_growth) + - retained_growth; transient_workspace = std::max(transient_workspace, reservation.transient_workspace); } - return std::min(std::numeric_limits::max() - retained_growth, transient_workspace) + - retained_growth; + + std::vector growth_candidates; + growth_candidates.reserve(per_partition_reservations.size()); + for (const auto& reservation : per_partition_reservations) { + if (reservation.retained_growth > 0) { + growth_candidates.push_back(&reservation); + } + } + + std::sort(growth_candidates.begin(), growth_candidates.end(), + [](const auto* lhs, const auto* rhs) { + return lhs->retained_growth > rhs->retained_growth; + }); + size_t row_bound = 0; + for (size_t i = 0; i < std::min(incoming_rows, growth_candidates.size()); ++i) { + row_bound = iceberg_saturating_add(row_bound, growth_candidates[i]->retained_growth); + } + + size_t byte_bound = 0; + std::vector positive_trigger_candidates; + positive_trigger_candidates.reserve(growth_candidates.size()); + for (const auto* reservation : growth_candidates) { + if (reservation->retained_growth_trigger_bytes == 0) { + byte_bound = iceberg_saturating_add(byte_bound, reservation->retained_growth); + } else { + positive_trigger_candidates.push_back(reservation); + } + } + std::sort(positive_trigger_candidates.begin(), positive_trigger_candidates.end(), + [](const auto* lhs, const auto* rhs) { + return static_cast(lhs->retained_growth) * + rhs->retained_growth_trigger_bytes > + static_cast(rhs->retained_growth) * + lhs->retained_growth_trigger_bytes; + }); + size_t remaining_bytes = incoming_bytes; + for (const auto* reservation : positive_trigger_candidates) { + if (reservation->retained_growth_trigger_bytes <= remaining_bytes) { + byte_bound = iceberg_saturating_add(byte_bound, reservation->retained_growth); + remaining_bytes -= reservation->retained_growth_trigger_bytes; + continue; + } + const auto numerator = + static_cast(reservation->retained_growth) * remaining_bytes + + reservation->retained_growth_trigger_bytes - 1; + const auto fractional_growth = + std::min(numerator / reservation->retained_growth_trigger_bytes, + std::numeric_limits::max()); + byte_bound = iceberg_saturating_add(byte_bound, static_cast(fractional_growth)); + break; + } + + // A block's rows and bytes are divided across partition sorters. The two fractional-relaxation + // bounds avoid charging the complete input block to every active partition while remaining safe. + const size_t retained_growth = std::min(row_bound, byte_bound); + return iceberg_saturating_add(retained_growth, transient_workspace); } inline size_t iceberg_reserve_size( const std::vector& per_partition_reservations, - size_t incoming_block_bytes) { - size_t sorter_reserve = bounded_iceberg_reserve_size(per_partition_reservations); + size_t incoming_block_reserve, size_t incoming_rows = std::numeric_limits::max(), + size_t incoming_bytes = std::numeric_limits::max()) { + size_t sorter_reserve = + bounded_iceberg_reserve_size(per_partition_reservations, incoming_rows, incoming_bytes); // The incoming block creates cold partition writers before they can appear in the published snapshot. - return std::min(std::numeric_limits::max() - sorter_reserve, incoming_block_bytes) + - sorter_reserve; + return iceberg_saturating_add(sorter_reserve, incoming_block_reserve); } size_t iceberg_cold_writer_reserve_size(const Block& block, size_t writer_workspace_bytes); diff --git a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp index 266dc654f4315f..c7e4615568d127 100644 --- a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp +++ b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp @@ -30,13 +30,13 @@ size_t iceberg_cold_writer_reserve_size(const Block& block, size_t writer_worksp const size_t row_index_bytes = std::min(std::numeric_limits::max() / sizeof(size_t), block.rows()) * sizeof(size_t); - const size_t selected_and_retained_bytes = - std::min(std::numeric_limits::max() / 2, block_bytes) * 2; - size_t reserve = std::min(std::numeric_limits::max() - writer_workspace_bytes, - selected_and_retained_bytes) + - writer_workspace_bytes; - // Cold dispatch may allocate a selected block and a retained sorter copy before publication. - return std::min(std::numeric_limits::max() - reserve, row_index_bytes) + reserve; + const size_t dispatch_copies = block_bytes > std::numeric_limits::max() / 4 + ? std::numeric_limits::max() + : block_bytes * 4; + size_t reserve = iceberg_saturating_add(writer_workspace_bytes, dispatch_copies); + // A transform, selected blocks, and retained sorter copies can coexist; one extra block-sized + // allowance covers allocator fragmentation when a block is split into many tiny partitions. + return iceberg_saturating_add(reserve, row_index_bytes); } SpillIcebergTableSinkLocalState::SpillIcebergTableSinkLocalState(DataSinkOperatorXBase* parent, @@ -74,13 +74,17 @@ size_t SpillIcebergTableSinkLocalState::get_reserve_mem_size(RuntimeState* state return 0; } std::vector per_partition_reservations; + const size_t incoming_rows = block == nullptr ? 0 : block->rows(); + const size_t incoming_bytes = block == nullptr ? 0 : block->allocated_bytes(); auto active_writers = _writer->active_writers(); per_partition_reservations.reserve(active_writers->size()); for (const auto& writer : *active_writers) { if (auto* sort_writer = dynamic_cast(writer.get())) { - auto reservation = sort_writer->get_reserve_mem_size_components(state, eos); + auto reservation = sort_writer->get_reserve_mem_size_components( + state, eos, incoming_rows, incoming_bytes); per_partition_reservations.push_back( {.retained_growth = reservation.retained_growth, + .retained_growth_trigger_bytes = reservation.retained_growth_trigger_bytes, .transient_workspace = reservation.transient_workspace}); } } @@ -90,7 +94,8 @@ size_t SpillIcebergTableSinkLocalState::get_reserve_mem_size(RuntimeState* state block == nullptr ? state->minimum_operator_memory_required_bytes() : iceberg_cold_writer_reserve_size( *block, state->minimum_operator_memory_required_bytes()); - return iceberg_reserve_size(per_partition_reservations, incoming_reserve); + return iceberg_reserve_size(per_partition_reservations, incoming_reserve, incoming_rows, + incoming_bytes); } size_t SpillIcebergTableSinkLocalState::get_revocable_mem_size(RuntimeState* state) const { diff --git a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp index 396d81ad87e88a..bfcce657afeb1d 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp @@ -92,6 +92,24 @@ SorterReserveMemory VIcebergSortWriter::get_reserve_mem_size_components(RuntimeS return {}; } auto reservation = _sorter->get_reserve_mem_size_components(state, eos); + _include_spill_merge_reservation(state, eos, &reservation); + return reservation; +} + +SorterReserveMemory VIcebergSortWriter::get_reserve_mem_size_components( + RuntimeState* state, bool eos, size_t incoming_rows, size_t incoming_bytes) const { + std::lock_guard lock(_sorter_mutex); + if (_sorter == nullptr) { + return {}; + } + auto reservation = + _sorter->get_reserve_mem_size_components(state, eos, incoming_rows, incoming_bytes); + _include_spill_merge_reservation(state, eos, &reservation); + return reservation; +} + +void VIcebergSortWriter::_include_spill_merge_reservation(RuntimeState* state, bool eos, + SorterReserveMemory* reservation) const { if (eos && !_sorted_spill_files.empty()) { size_t spill_file_count = _sorted_spill_files.size(); if (_sorter->data_size() > 0) { @@ -100,10 +118,9 @@ SorterReserveMemory VIcebergSortWriter::get_reserve_mem_size_components(RuntimeS const size_t merge_workspace = iceberg_spill_merge_workspace(spill_file_count, state->spill_buffer_size_bytes(), state->spill_sort_merge_mem_limit_bytes()); - reservation.transient_workspace = - std::max(reservation.transient_workspace, merge_workspace); + reservation->transient_workspace = + std::max(reservation->transient_workspace, merge_workspace); } - return reservation; } Status VIcebergSortWriter::trigger_spill() { diff --git a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.h b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.h index 37659eeca4bc89..b41c31828431f1 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.h +++ b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.h @@ -107,10 +107,17 @@ class VIcebergSortWriter : public IPartitionWriterBase { SorterReserveMemory get_reserve_mem_size_components(RuntimeState* state, bool eos) const; + SorterReserveMemory get_reserve_mem_size_components(RuntimeState* state, bool eos, + size_t incoming_rows, + size_t incoming_bytes) const; + // Called by the memory management system to trigger spilling data to disk Status trigger_spill(); private: + void _include_spill_merge_reservation(RuntimeState* state, bool eos, + SorterReserveMemory* reservation) const; + // Calculate average row size from the first non-empty block to determine // the optimal batch row count for spill operations void _update_spill_block_batch_row_count(const Block& block); diff --git a/be/src/exec/sort/sorter.cpp b/be/src/exec/sort/sorter.cpp index c64d07b219a6cf..01ff069d9451cb 100644 --- a/be/src/exec/sort/sorter.cpp +++ b/be/src/exec/sort/sorter.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -39,6 +40,20 @@ #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" +namespace { + +size_t saturating_add_size(size_t lhs, size_t rhs) { + return std::min(std::numeric_limits::max() - lhs, rhs) + lhs; +} + +size_t saturating_multiply_size(size_t lhs, size_t rhs) { + return lhs == 0 || rhs <= std::numeric_limits::max() / lhs + ? lhs * rhs + : std::numeric_limits::max(); +} + +} // namespace + namespace doris { class RowDescriptor; } // namespace doris @@ -207,31 +222,52 @@ size_t FullSorter::get_reserve_mem_size(RuntimeState* state, bool eos) const { SorterReserveMemory FullSorter::get_reserve_mem_size_components(RuntimeState* state, bool eos) const { + const auto rows = _state->unsorted_block()->rows(); + const auto bytes = _state->unsorted_block()->bytes(); + const auto bytes_per_row = rows == 0 ? 0 : bytes / rows; + return get_reserve_mem_size_components( + state, eos, state->batch_size(), + saturating_multiply_size(bytes_per_row, state->batch_size())); +} + +SorterReserveMemory FullSorter::get_reserve_mem_size_components(RuntimeState* state, bool eos, + size_t incoming_rows, + size_t incoming_bytes) const { SorterReserveMemory reserve; const auto rows = _state->unsorted_block()->rows(); if (rows != 0) { const auto bytes = _state->unsorted_block()->bytes(); const auto allocated_bytes = _state->unsorted_block()->allocated_bytes(); - const auto bytes_per_row = bytes / rows; - const auto estimated_size_of_next_block = bytes_per_row * state->batch_size(); - auto new_block_bytes = estimated_size_of_next_block + bytes; - auto new_rows = rows + state->batch_size(); + auto new_block_bytes = saturating_add_size(bytes, incoming_bytes); + auto new_rows = saturating_add_size(rows, incoming_rows); // If the new size is greater than 85% of allocalted bytes, it maybe need to realloc. - if ((new_block_bytes * 100 / allocated_bytes) >= 85) { - reserve.retained_growth += (size_t)(allocated_bytes * 1.15); + const auto growth_threshold = static_cast( + (static_cast(allocated_bytes) * 85 + 99) / 100); + const size_t growth_trigger_bytes = growth_threshold > bytes ? growth_threshold - bytes : 0; + if (incoming_rows > 0 && growth_trigger_bytes <= incoming_bytes) { + reserve.retained_growth = static_cast(std::min( + (static_cast(allocated_bytes) * 115 + 99) / 100, + std::numeric_limits::max())); + reserve.retained_growth_trigger_bytes = growth_trigger_bytes; } auto sort = new_rows > _buffered_block_size || new_block_bytes > _buffered_block_bytes; if (sort) { // new column is created when doing sort, reserve average size of one column // for estimation - reserve.transient_workspace += new_block_bytes / _state->unsorted_block()->columns(); + reserve.transient_workspace = + saturating_add_size(reserve.transient_workspace, + new_block_bytes / _state->unsorted_block()->columns()); // helping data structures used during sorting - reserve.transient_workspace += new_rows * sizeof(IColumn::Permutation::value_type); + reserve.transient_workspace = saturating_add_size( + reserve.transient_workspace, + saturating_multiply_size(new_rows, sizeof(IColumn::Permutation::value_type))); auto sort_columns_count = _ordering_expr_ctxs.size(); if (1 != sort_columns_count) { - reserve.transient_workspace += new_rows * sizeof(EqualRangeIterator); + reserve.transient_workspace = saturating_add_size( + reserve.transient_workspace, + saturating_multiply_size(new_rows, sizeof(EqualRangeIterator))); } } } diff --git a/be/src/exec/sort/sorter.h b/be/src/exec/sort/sorter.h index 5c748f86a7f858..2a47438f0b3e6e 100644 --- a/be/src/exec/sort/sorter.h +++ b/be/src/exec/sort/sorter.h @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -42,9 +43,14 @@ namespace doris { struct SorterReserveMemory { size_t retained_growth = 0; + size_t retained_growth_trigger_bytes = 0; size_t transient_workspace = 0; - size_t total() const { return retained_growth + transient_workspace; } + size_t total() const { + return retained_growth > std::numeric_limits::max() - transient_workspace + ? std::numeric_limits::max() + : retained_growth + transient_workspace; + } }; class ObjectPool; class RowDescriptor; @@ -203,6 +209,10 @@ class FullSorter final : public Sorter { SorterReserveMemory get_reserve_mem_size_components(RuntimeState* state, bool eos) const; + SorterReserveMemory get_reserve_mem_size_components(RuntimeState* state, bool eos, + size_t incoming_rows, + size_t incoming_bytes) const; + Status merge_sort_read_for_spill(RuntimeState* state, doris::Block* block, int batch_size, bool* eos) override; void reset() override; diff --git a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp index 92aa896daebcf2..3e004051220504 100644 --- a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp +++ b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp @@ -45,6 +45,30 @@ TEST(SpillIcebergTableSinkOperatorTest, AccumulatesRetainedGrowthAcrossTouchedPa EXPECT_EQ(14 * 1024 * 1024, bounded_iceberg_reserve_size(per_partition_reservations)); } +TEST(SpillIcebergTableSinkOperatorTest, BoundsRetainedGrowthByOneInputBlock) { + constexpr size_t MB = 1024 * 1024; + std::vector per_partition_reservations( + 128, {.retained_growth = 8 * MB, + .retained_growth_trigger_bytes = 8 * MB, + .transient_workspace = 4 * MB}); + + // Only one 8 MiB growth threshold can be crossed by this block. Treating the block as a full + // batch for every active partition would incorrectly reserve more than 1 GiB here. + EXPECT_EQ(12 * MB, bounded_iceberg_reserve_size(per_partition_reservations, 128, 8 * MB)); +} + +TEST(SpillIcebergTableSinkOperatorTest, RetainsNearCapacityGrowthAcrossAllPossiblePartitions) { + constexpr size_t MB = 1024 * 1024; + std::vector per_partition_reservations( + 4, {.retained_growth = 3 * MB, + .retained_growth_trigger_bytes = 0, + .transient_workspace = 2 * MB}); + + // A one-row append can grow every already-near-capacity sorter, so persistent growth remains + // cumulative even though the serially used workspace is shared. + EXPECT_EQ(14 * MB, bounded_iceberg_reserve_size(per_partition_reservations, 4, 1)); +} + TEST(SpillIcebergTableSinkOperatorTest, ReservesIncomingBlockBeforeAnyPartitionWriterExists) { std::vector no_published_sorters; @@ -60,7 +84,8 @@ TEST(SpillIcebergTableSinkOperatorTest, ColdWriterReserveUsesFirstBlockLargerTha block.insert({std::move(strings), std::make_shared(), "payload"}); ASSERT_GT(block.allocated_bytes(), operator_floor); - EXPECT_GT(iceberg_cold_writer_reserve_size(block, operator_floor), 2 * block.allocated_bytes()); + EXPECT_GE(iceberg_cold_writer_reserve_size(block, operator_floor), + 4 * block.allocated_bytes() + operator_floor); } TEST(SpillIcebergTableSinkOperatorTest, ReservesAllMergeInputsAndOutputAtEos) { diff --git a/be/test/exec/sink/writer/async_result_writer_test.cpp b/be/test/exec/sink/writer/async_result_writer_test.cpp new file mode 100644 index 00000000000000..840be91f5e0df6 --- /dev/null +++ b/be/test/exec/sink/writer/async_result_writer_test.cpp @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/async_result_writer.h" + +#include + +#include "core/block/block.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "exec/pipeline/dependency.h" +#include "runtime/exec_env.h" +#include "runtime/fragment_mgr.h" +#include "runtime/memory/global_memory_arbitrator.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/thread_context.h" +#include "runtime/workload_management/resource_context.h" + +namespace doris { + +namespace { + +const VExprContextSPtrs EMPTY_OUTPUT_EXPRS; + +Block make_block() { + auto values = ColumnInt32::create(); + values->insert_value(1); + Block block; + block.insert({std::move(values), std::make_shared(), "value"}); + return block; +} + +class RecordingAsyncWriter final : public AsyncResultWriter { +public: + RecordingAsyncWriter(std::shared_ptr dependency, + std::shared_ptr finish_dependency, Status open_status) + : AsyncResultWriter(EMPTY_OUTPUT_EXPRS, std::move(dependency), + std::move(finish_dependency)), + _open_status(std::move(open_status)) {} + + Status open(RuntimeState*, RuntimeProfile*) override { return _open_status; } + + Status write(RuntimeState*, Block&) override { + reservation_seen_by_write = thread_context()->thread_mem_tracker_mgr->reserved_mem(); + return Status::OK(); + } + + Status finish(RuntimeState*) override { + reservation_seen_by_finish = thread_context()->thread_mem_tracker_mgr->reserved_mem(); + return Status::OK(); + } + + Status close(Status) override { return Status::OK(); } + + int64_t reservation_seen_by_write = 0; + int64_t reservation_seen_by_finish = 0; + +private: + Status _open_status; +}; + +struct AsyncWriterHarness { + AsyncWriterHarness() + : dependency(std::make_shared(0, 0, "writer", true)), + finish_dependency(std::make_shared(0, 0, "finish", false)), + common_profile("CommonCounters"), + memory_usage(common_profile.AddHighWaterMarkCounter("MemoryUsage", TUnit::BYTES)) {} + + void prepare(AsyncResultWriter* writer) { + writer->_operator_profile = &operator_profile; + writer->_memory_used_counter = memory_usage; + } + + void process(AsyncResultWriter* writer) { writer->process_block(nullptr, &operator_profile); } + + std::shared_ptr dependency; + std::shared_ptr finish_dependency; + RuntimeProfile operator_profile {"operator"}; + RuntimeProfile common_profile; + RuntimeProfile::Counter* memory_usage; +}; + +} // namespace + +class AsyncResultWriterTest : public testing::Test { +protected: + void SetUp() override { + _exec_env = ExecEnv::GetInstance(); + if (_exec_env->fragment_mgr() == nullptr) { + _fragment_mgr = std::make_unique(_exec_env); + _exec_env->_fragment_mgr = _fragment_mgr.get(); + } + _tracker = MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER, + "UT-AsyncResultWriterReservation"); + _resource_context = ResourceContext::create_shared(); + _resource_context->memory_context()->set_mem_tracker(_tracker); + thread_context()->attach_task(_resource_context); + } + + void TearDown() override { + thread_context()->thread_mem_tracker_mgr->shrink_reserved(); + thread_context()->detach_task(); + EXPECT_EQ(0, GlobalMemoryArbitrator::process_reserved_memory()); + if (_fragment_mgr != nullptr) { + _fragment_mgr->stop(); + _exec_env->_fragment_mgr = nullptr; + _fragment_mgr.reset(); + } + } + + std::shared_ptr _tracker; + std::shared_ptr _resource_context; + ExecEnv* _exec_env = nullptr; + std::unique_ptr _fragment_mgr; +}; + +TEST_F(AsyncResultWriterTest, TransfersQueuedReservationIntoActualWrite) { + AsyncWriterHarness harness; + RecordingAsyncWriter writer(harness.dependency, harness.finish_dependency, Status::OK()); + harness.prepare(&writer); + constexpr int64_t reservation = 4 * 1024 * 1024; + Block block = make_block(); + ASSERT_TRUE(thread_context()->thread_mem_tracker_mgr->try_reserve(reservation).ok()); + + ASSERT_TRUE(writer.sink(&block, true).ok()); + EXPECT_EQ(0, thread_context()->thread_mem_tracker_mgr->reserved_mem()); + harness.process(&writer); + + EXPECT_GT(writer.reservation_seen_by_write, 0); + EXPECT_LE(writer.reservation_seen_by_write, reservation); + EXPECT_EQ(0, thread_context()->thread_mem_tracker_mgr->reserved_mem()); +} + +TEST_F(AsyncResultWriterTest, RetainsEosReservationThroughActualFinish) { + AsyncWriterHarness harness; + RecordingAsyncWriter writer(harness.dependency, harness.finish_dependency, Status::OK()); + harness.prepare(&writer); + constexpr int64_t reservation = 4 * 1024 * 1024; + ASSERT_TRUE(thread_context()->thread_mem_tracker_mgr->try_reserve(reservation).ok()); + Block block; + + ASSERT_TRUE(writer.sink(&block, true).ok()); + harness.process(&writer); + + EXPECT_EQ(reservation, writer.reservation_seen_by_finish); + EXPECT_EQ(0, thread_context()->thread_mem_tracker_mgr->reserved_mem()); +} + +TEST_F(AsyncResultWriterTest, OpenFailureDrainsQueuedReservation) { + AsyncWriterHarness harness; + RecordingAsyncWriter writer(harness.dependency, harness.finish_dependency, + Status::IOError("injected open failure")); + harness.prepare(&writer); + constexpr int64_t reservation = 4 * 1024 * 1024; + Block block = make_block(); + ASSERT_TRUE(thread_context()->thread_mem_tracker_mgr->try_reserve(reservation).ok()); + + ASSERT_TRUE(writer.sink(&block, true).ok()); + harness.process(&writer); + + EXPECT_FALSE(writer.get_writer_status().ok()); + EXPECT_EQ(0, thread_context()->thread_mem_tracker_mgr->reserved_mem()); + EXPECT_EQ(0, GlobalMemoryArbitrator::process_reserved_memory()); +} + +} // namespace doris diff --git a/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp b/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp index 18af0af7cb23bd..d5a7e3f9ab3351 100644 --- a/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp +++ b/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp @@ -21,7 +21,9 @@ #include "exec/sink/writer/iceberg/viceberg_partition_writer.h" #include "exec/sink/writer/iceberg/viceberg_sort_writer.h" +#include "testutil/mock/mock_descriptors.h" #include "testutil/mock/mock_runtime_state.h" +#include "testutil/mock/mock_slot_ref.h" namespace doris { @@ -131,4 +133,29 @@ TEST_F(VIcebergPartitionWriterTest, SortWriterPropagatesUnderlyingCloseFailure) EXPECT_NE(status.to_string().find("injected close failure"), std::string::npos); } +TEST_F(VIcebergPartitionWriterTest, EosReservationIncludesActualSpillFanIn) { + VExprContextSPtrs output_exprs; + iceberg::Schema schema(std::vector {}); + std::string schema_json; + std::map hadoop_conf; + auto partition_writer = std::shared_ptr( + make_writer(make_table_sink(false), output_exprs, schema, &schema_json, hadoop_conf)); + VIcebergSortWriter sort_writer(partition_writer, TSortInfo(), 1024); + MockRuntimeState state; + ObjectPool pool; + auto row_desc = std::make_unique( + std::vector {std::make_shared()}, &pool); + auto ordering_expr_ctxs = + MockSlotRef::create_mock_contexts(0, std::make_shared()); + std::vector is_asc_order {true}; + std::vector nulls_first {false}; + sort_writer._sorter = FullSorter::create_unique(ordering_expr_ctxs, -1, 0, &pool, is_asc_order, + nulls_first, *row_desc, &state, nullptr); + sort_writer._sorted_spill_files.resize(12); + + const auto reservation = sort_writer.get_reserve_mem_size_components(&state, true, 0, 0); + + EXPECT_EQ(72 * 1024 * 1024, reservation.transient_workspace); +} + } // namespace doris diff --git a/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp b/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp index d486672c0c83db..ac6c100b37e143 100644 --- a/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp +++ b/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp @@ -20,8 +20,11 @@ #include #include +#include "core/column/column_string.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "exec/operator/iceberg_sorter_reserve_memory.h" #include "exec/sink/writer/iceberg/viceberg_table_writer.h" #include "exec/sink/writer/iceberg/vpartition_writer_base.h" #include "runtime/runtime_state.h" @@ -117,6 +120,27 @@ TEST_F(VIcebergTableWriterLifecycleTest, SelectBlockUsesRowPermutation) { EXPECT_EQ(result.get_element(1), 10); } +TEST_F(VIcebergTableWriterLifecycleTest, ColdReserveCoversManyRealPartitionSelections) { + VIcebergTableWriter writer(make_sink(), {}, nullptr, nullptr); + auto values = ColumnString::create(); + for (size_t i = 0; i < 128; ++i) { + const std::string value(256 + i, static_cast('a' + i % 26)); + values->insert_data(value.data(), value.size()); + } + Block input; + input.insert({std::move(values), std::make_shared(), "value"}); + size_t selected_bytes = 0; + for (size_t row = 0; row < input.rows(); ++row) { + Block selected; + ASSERT_TRUE(select_block(&writer, input, {row}, &selected).ok()); + selected_bytes += selected.allocated_bytes(); + } + + const size_t reserve = iceberg_cold_writer_reserve_size(input, 0); + EXPECT_GE(reserve, + input.allocated_bytes() + 2 * selected_bytes + input.rows() * sizeof(size_t)); +} + TEST_F(VIcebergTableWriterLifecycleTest, ActiveWriterSnapshotContainsEveryOpenPartition) { VIcebergTableWriter writer(make_sink(), {}, nullptr, nullptr); add_writer(&writer, "p=1"); diff --git a/be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp b/be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp index 21234083984ca4..ffc54b5f1b4936 100644 --- a/be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp +++ b/be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp @@ -114,7 +114,8 @@ class FixedLengthTransformer final : public VFileFormatTransformer { std::unique_ptr create_closed_hive_writer( RuntimeState* state, const VExprContextSPtrs& output_exprs, - const std::shared_ptr& client) { + const std::shared_ptr& client, + io::ObjStorageType provider = io::ObjStorageType::AWS, std::string staged_block_id = {}) { THiveTableSink hive_sink; TDataSink sink; sink.__set_type(TDataSinkType::HIVE_TABLE_SINK); @@ -130,12 +131,18 @@ std::unique_ptr create_closed_hive_writer( std::move(write_info), "part", 0, TFileFormatType::FORMAT_PARQUET, TFileCompressType::PLAIN, nullptr, hadoop_conf); - auto holder = std::make_shared(S3ClientConf {}); + S3ClientConf client_conf; + client_conf.provider = provider; + auto holder = std::make_shared(client_conf); holder->_client = client; io::FileWriterOptions options {.used_by_s3_committer = true}; auto file_writer = std::make_unique(holder, "bucket", "table/part.parquet", &options); file_writer->_obj_storage_path_opts.upload_id = "upload-id"; + if (!staged_block_id.empty()) { + file_writer->_completed_parts.push_back( + {.part_num = 1, .etag = std::move(staged_block_id)}); + } file_writer->_state = io::FileWriter::State::CLOSED; writer->_file_writer = std::move(file_writer); writer->_file_format_transformer = std::make_unique(output_exprs); @@ -205,4 +212,30 @@ TEST(VHivePartitionWriterReportLifecycleTest, EXPECT_EQ("upload-id", client->aborted_upload_id); } +TEST(VHivePartitionWriterReportLifecycleTest, + AzureFinalReportCarriesExactBlockIdentityAndRejectionAbortsUpload) { + MockRuntimeState state; + VExprContextSPtrs output_exprs; + auto client = std::make_shared(); + auto writer = create_closed_hive_writer(&state, output_exprs, client, io::ObjStorageType::AZURE, + "exact-block-id"); + auto context = create_fragment_context(TUniqueId()); + auto final_request = report_request(&state, true); + TReportExecStatusParams final_params; + + context->_append_external_file_commit_data(final_request, &final_params); + + ASSERT_TRUE(final_params.__isset.hive_partition_updates); + ASSERT_EQ(1, final_params.hive_partition_updates.size()); + const auto& pending_uploads = final_params.hive_partition_updates[0].s3_mpu_pending_uploads; + ASSERT_EQ(1, pending_uploads.size()); + EXPECT_EQ("upload-id", pending_uploads[0].upload_id); + EXPECT_EQ("exact-block-id", pending_uploads[0].etags.at(1)); + + // Azure staged blocks stay owned by BE until the final coordinator ACK transfers them. + context->_coordinator_callback(final_request); + EXPECT_EQ(1, client->abort_count); + EXPECT_EQ("upload-id", client->aborted_upload_id); +} + } // namespace doris diff --git a/be/test/exec/sort/full_sort_test.cpp b/be/test/exec/sort/full_sort_test.cpp index e182048c807dad..76bd9f3cd8151c 100644 --- a/be/test/exec/sort/full_sort_test.cpp +++ b/be/test/exec/sort/full_sort_test.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -63,6 +64,13 @@ struct FullSorterTest : public testing::Test { std::vector nulls_first {false}; }; +TEST(SorterReserveMemoryTest, TotalSaturatesOnOverflow) { + SorterReserveMemory reservation {.retained_growth = std::numeric_limits::max() - 1, + .transient_workspace = 2}; + + EXPECT_EQ(std::numeric_limits::max(), reservation.total()); +} + TEST_F(FullSorterTest, test_full_sorter1) { sorter = FullSorter::create_unique(ordering_expr_ctxs, -1, 0, &pool, is_asc_order, nulls_first, *row_desc, &_state, nullptr); @@ -113,4 +121,4 @@ TEST_F(FullSorterTest, test_full_sorter3) { EXPECT_EQ(sorter->_state->get_sorted_block()[1]->rows(), 4); } -} // namespace doris \ No newline at end of file +} // namespace doris From 78a36ec4652dace86d71fa241bdb40109c4c0f8f Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 6 Aug 2026 15:37:40 +0800 Subject: [PATCH 22/29] [fix](regression) Isolate MTMV job lookup ### What problem does this PR solve? Issue Number: None Related PR: #66348 Problem Summary: Parallel regression suites can share an MTMV database. The getJobName helper queried mv_infos(), which materializes external metadata for every MV in that database, so an unrelated transient catalog failure could fail job lookup in another suite. Query MV job metadata directly and scope the lookup by both database and MV name. ### Release note None ### Check List (For Author) - Test: Unit Test - SuiteJobLookupTest verifies the isolated jobs metadata query. - Regression framework tests: 4 passed. - Framework Java and Groovy compilation passed. - Behavior changed: No. Regression job lookup no longer evaluates unrelated MV status. - Does this need documentation: No --- .../doris/regression/suite/Suite.groovy | 12 +++++-- .../suite/SuiteJobLookupTest.groovy | 33 +++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteJobLookupTest.groovy diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index cedb00cc0e3e56..a363f523310561 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -2323,10 +2323,16 @@ class Suite implements GroovyInterceptable { } } + static String buildJobNameQuery(String dbName, String mtmvName) { + return ("select Name from jobs('type'='mv') where MvDatabaseName = '${dbName}' " + + "and MvName = '${mtmvName}'") + } + String getJobName(String dbName, String mtmvName) { - String showMTMV = "select JobName from mv_infos('database'='${dbName}') where Name = '${mtmvName}'"; - logger.info(showMTMV) - List> result = sql(showMTMV) + // Job lookup must not materialize unrelated MVs whose external metadata may be unavailable. + String showJob = buildJobNameQuery(dbName, mtmvName) + logger.info(showJob) + List> result = sql(showJob) logger.info("result: " + result.toString()) if (result.isEmpty()) { Assert.fail(); diff --git a/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteJobLookupTest.groovy b/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteJobLookupTest.groovy new file mode 100644 index 00000000000000..e66b1dad09ef86 --- /dev/null +++ b/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteJobLookupTest.groovy @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.suite + +import org.junit.jupiter.api.Test + +import static org.junit.jupiter.api.Assertions.assertEquals + +class SuiteJobLookupTest { + @Test + void jobLookupUsesJobMetadataWithoutMaterializingMvStatus() { + String query = Suite.buildJobNameQuery("db1", "mv1") + + assertEquals( + "select Name from jobs('type'='mv') where MvDatabaseName = 'db1' and MvName = 'mv1'", + query) + } +} From c5f07d4fb83c165c0d88e1fd51dd0cc652e8745e Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 6 Aug 2026 19:03:43 +0800 Subject: [PATCH 23/29] [fix](test) Model Iceberg report ACK in merge sink UT ### What problem does this PR solve? Issue Number: None Related PR: #66348 Problem Summary: The Iceberg merge sink tests used a default mock runtime state after Iceberg writers began requiring coordinator acknowledgement of external-file reports. The inner table writer therefore rejected open before twelve tests reached the behavior they intended to verify. Use an ACK-capable mock runtime state for successful merge writer scenarios while preserving the fail-closed production check. ### Release note None ### Check List (For Author) - Test: Unit Test - Reproduced 12 failures in VIcebergMergeSinkTest before the fixture fix. - 15 focused ASAN tests passed after the fix, including the no-ACK rejection test. - The affected test source compiled with the ASAN BE test flags. - The affected C++ file passed clang-format 16. - Behavior changed: No. This updates the test coordinator capability only. - Does this need documentation: No --- .../exec/sink/viceberg_merge_sink_test.cpp | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/be/test/exec/sink/viceberg_merge_sink_test.cpp b/be/test/exec/sink/viceberg_merge_sink_test.cpp index eb7c0159d5fd38..7eac5823d49725 100644 --- a/be/test/exec/sink/viceberg_merge_sink_test.cpp +++ b/be/test/exec/sink/viceberg_merge_sink_test.cpp @@ -47,6 +47,16 @@ namespace doris { +class IcebergWriteMockRuntimeState : public MockRuntimeState { +public: + IcebergWriteMockRuntimeState() { + auto query_options = this->query_options(); + // Successful writer tests must model a coordinator that can accept file ownership. + query_options.__set_supports_external_file_report_ack(true); + set_query_options(query_options); + } +}; + class VIcebergMergeSinkTest : public testing::Test { protected: static std::string test_schema_json() { @@ -172,7 +182,7 @@ class VIcebergMergeSinkTest : public testing::Test { TEST_F(VIcebergMergeSinkTest, TestUpdateProducesDeleteAndInsert) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -202,7 +212,7 @@ TEST_F(VIcebergMergeSinkTest, TestUpdateProducesDeleteAndInsert) { TEST_F(VIcebergMergeSinkTest, TestMissingOperationColumn) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -225,7 +235,7 @@ TEST_F(VIcebergMergeSinkTest, TestMissingOperationColumn) { TEST_F(VIcebergMergeSinkTest, TestMissingRowIdColumn) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -248,7 +258,7 @@ TEST_F(VIcebergMergeSinkTest, TestMissingRowIdColumn) { TEST_F(VIcebergMergeSinkTest, TestUnknownOperation) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -275,7 +285,7 @@ TEST_F(VIcebergMergeSinkTest, TestUnknownOperation) { TEST_F(VIcebergMergeSinkTest, TestUpdateInsertAndDeleteOperations) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -305,7 +315,7 @@ TEST_F(VIcebergMergeSinkTest, TestUpdateInsertAndDeleteOperations) { TEST_F(VIcebergMergeSinkTest, TestSchemaMismatch) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -332,7 +342,7 @@ TEST_F(VIcebergMergeSinkTest, TestSchemaMismatch) { TEST_F(VIcebergMergeSinkTest, TestRejectsDuplicateMatchedTargetAcrossBlocks) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -361,7 +371,7 @@ TEST_F(VIcebergMergeSinkTest, TestRejectsDuplicateMatchedTargetAcrossBlocks) { TEST_F(VIcebergMergeSinkTest, TestUpdateSkipsCardinalityState) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -389,7 +399,7 @@ TEST_F(VIcebergMergeSinkTest, TestUpdateSkipsCardinalityState) { TEST_F(VIcebergMergeSinkTest, TestOldFePlanSkipsCardinalityState) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -413,7 +423,7 @@ TEST_F(VIcebergMergeSinkTest, TestOldFePlanSkipsCardinalityState) { TEST_F(VIcebergMergeSinkTest, TestRollingUpgradeSkipsCardinalityState) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; state.set_be_exec_version(SUPPORT_ICEBERG_MERGE_CARDINALITY_VERSION - 1); DataTypes types {std::make_shared(), @@ -437,7 +447,7 @@ TEST_F(VIcebergMergeSinkTest, TestRollingUpgradeSkipsCardinalityState) { TEST_F(VIcebergMergeSinkTest, TestErrorCloseRemovesRolledDataFiles) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -469,7 +479,7 @@ TEST_F(VIcebergMergeSinkTest, TestErrorCloseRemovesRolledDataFiles) { TEST_F(VIcebergMergeSinkTest, TestDeleteCloseFailureRemovesBothInnerSinkFiles) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -519,7 +529,7 @@ TEST_F(VIcebergMergeSinkTest, TestDeleteCloseFailureRemovesBothInnerSinkFiles) { TEST_F(VIcebergMergeSinkTest, TestMatchedRowIdsUseCompactRetainedState) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -548,7 +558,7 @@ TEST_F(VIcebergMergeSinkTest, TestMatchedRowIdsUseCompactRetainedState) { TEST_F(VIcebergMergeSinkTest, TestMatchedRowIdStateAcrossManyFilesAndWrites) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), From 35a48c04ec875b7bc1aeabfa8206262c44594ec9 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 6 Aug 2026 22:52:57 +0800 Subject: [PATCH 24/29] [fix](test) Preserve rebase safety invariants --- .../hive/HiveConnectorTransactionTest.java | 17 +++++++++++------ .../IcebergConnectorTransactionTest.java | 3 ++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorTransactionTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorTransactionTest.java index f14ceec77d9623..e3fe52ba2eebe0 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorTransactionTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorTransactionTest.java @@ -268,10 +268,13 @@ public void testCommitRejectsSameShapeTableReplacementBeforePublishing() throws client.table = table(false, Collections.emptyMap(), 10, Collections.singletonList(col("c1", "int"))); client.currentNotificationEventId = 10; - HiveConnectorTransaction txn = newTxn(client); + RecordingObjStorage objStorage = new RecordingObjStorage(); + HiveConnectorTransaction txn = newTxnWithFs(client, new RecordingObjFileSystem(objStorage)); txn.beginWrite(null, DB, TBL, ctx(false)); - txn.addCommitData(serialize(pu("", TUpdateMode.APPEND, "s3://bucket/db/t", - Collections.singletonList("f1"), 100, 4))); + // A complete ownership record is required so this test reaches the generation fence instead of the + // earlier fail-closed multipart validation. + txn.addCommitData(serialize(puWithMpu("", TUpdateMode.APPEND, "s3://bucket/db/t", + "bucket", "db/t/f1", "upload-replaced", Collections.singletonMap(1, "etag-1")))); // Recreate the table with every digest-visible field unchanged. Only the HMS event stream can // distinguish this new object from the table whose schema and location were bound above. @@ -292,10 +295,12 @@ public void testCommitHoldsExclusiveTableLockThroughPublication() throws TExcept RecordingHmsClient client = new RecordingHmsClient(); client.table = table(false, Collections.emptyMap()); client.currentNotificationEventId = 10; - HiveConnectorTransaction txn = newTxn(client); + RecordingObjStorage objStorage = new RecordingObjStorage(); + HiveConnectorTransaction txn = newTxnWithFs(client, new RecordingObjFileSystem(objStorage)); txn.beginWrite(null, DB, TBL, ctx(false)); - txn.addCommitData(serialize(pu("", TUpdateMode.APPEND, "s3://bucket/db/t", - Collections.singletonList("f1"), 100, 4))); + // Keep multipart validation green so the assertion measures the lock around actual publication. + txn.addCommitData(serialize(puWithMpu("", TUpdateMode.APPEND, "s3://bucket/db/t", + "bucket", "db/t/f1", "upload-lock", Collections.singletonMap(1, "etag-1")))); txn.commit(); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java index 1f5af3cac19590..13ae750395437d 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java @@ -142,8 +142,9 @@ private static IcebergWriteContext overwriteToBranch(String branch) { } private static IcebergWriteContext overwriteCtxPinned(long readSnapshotId) { + // A resolved empty read also uses -1, so the explicit flag preserves it as an OCC fence. return new IcebergWriteContext(WriteOperation.OVERWRITE, true, Collections.emptyMap(), Optional.empty(), - readSnapshotId); + readSnapshotId, true); } private static IcebergWriteContext overwriteStaticCtx(Table table, Map staticValues) { From 190ebe5ac6f6469426a4f8957594badf10b037ea Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 7 Aug 2026 12:28:03 +0800 Subject: [PATCH 25/29] [fix](iceberg) Close external write review gaps ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: External-file report retries could delete files after an earlier request may already have transferred ownership, orphan cleanup treated unknown creation times as old files, and coordinator report state lacked complete cross-thread visibility. Preserve ambiguous ownership monotonically, fail closed on non-positive creation times, and safely publish report state while keeping each report transaction identity local. ### Release note External writes retain uploaded files after ambiguous report delivery, orphan cleanup skips files without reliable creation times, and concurrent report state is safely published. ### Check List (For Author) - Test: Unit Test (7 RuntimeState Iceberg commit-data tests, 13 Iceberg orphan-file tests, and 7 FE report-ack tests) - Behavior changed: Yes. Unknown file timestamps and ambiguous report ownership now fail closed. - Does this need documentation: No --- be/src/runtime/runtime_state.cpp | 6 ++- be/src/runtime/runtime_state.h | 1 + .../runtime_state_block_budget_test.cpp | 13 +++++- .../IcebergRemoveOrphanFilesAction.java | 4 +- .../IcebergRemoveOrphanFilesActionTest.java | 43 ++++++++++++++++++- .../java/org/apache/doris/qe/Coordinator.java | 12 ++++-- .../qe/QeProcessorImplReportAckTest.java | 39 +++++++++++++++++ 7 files changed, 110 insertions(+), 8 deletions(-) diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index f2052910faf11b..380bc8f8f72081 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -131,7 +131,11 @@ void RuntimeState::finalize_external_file_report_cleanup(ExternalFileReportOutco return; } if (outcome == ExternalFileReportOutcome::AMBIGUOUS) { - // A consumed request with a lost ACK may already be publishing these files. + // Once an ACK can have been lost, a later rejection cannot prove FE never accepted the files. + _external_file_report_state->ownership_may_have_transferred = true; + return; + } + if (_external_file_report_state->ownership_may_have_transferred) { return; } cleanups.swap(_external_file_report_state->rejected_report_cleanups); diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 29c17da078154e..a3cfc5e4cad782 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -85,6 +85,7 @@ class ExternalFileReportState { private: std::mutex mutex; size_t iceberg_serialized_bytes = 0; + bool ownership_may_have_transferred = false; std::vector> rejected_report_cleanups; }; diff --git a/be/test/runtime/runtime_state_block_budget_test.cpp b/be/test/runtime/runtime_state_block_budget_test.cpp index 16a078bd64c8bf..5a384378ec382b 100644 --- a/be/test/runtime/runtime_state_block_budget_test.cpp +++ b/be/test/runtime/runtime_state_block_budget_test.cpp @@ -129,7 +129,18 @@ TEST(RuntimeStateIcebergCommitDataTest, RetainsFileCleanupUntilReportAcknowledge coordinator_state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::AMBIGUOUS); EXPECT_EQ(1, cleanup_count); coordinator_state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::REJECTED); - EXPECT_EQ(2, cleanup_count); + EXPECT_EQ(1, cleanup_count); +} + +TEST(RuntimeStateIcebergCommitDataTest, AmbiguousOwnershipCannotBecomeRejected) { + RuntimeState state; + int cleanup_count = 0; + state.add_rejected_external_file_report_cleanup([&] { ++cleanup_count; }); + + state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::AMBIGUOUS); + state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::REJECTED); + + EXPECT_EQ(0, cleanup_count); } // --------------------------------------------------------------------------- diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java index 7fb4fbeb608564..24e87fedbea7a6 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesAction.java @@ -118,7 +118,9 @@ protected List executeAction(Table table, ConnectorSession session) { // Object stores use raw prefix matching, so the separator excludes sibling prefixes. String listingPrefix = scope.root.endsWith("/") ? scope.root : scope.root + "/"; for (FileInfo file : ((SupportsPrefixOperations) table.io()).listPrefix(listingPrefix)) { - if (scope.owns(file.location()) && file.createdAtMillis() < olderThan + // Unknown creation time cannot prove the file predates every in-flight writer. + if (scope.owns(file.location()) && file.createdAtMillis() > 0 + && file.createdAtMillis() < olderThan && !isReachable(file.location(), reachable)) { orphanCount++; if (!dryRun) { diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java index bce05901cead38..b1f5de5c588ad3 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/action/IcebergRemoveOrphanFilesActionTest.java @@ -47,8 +47,11 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; public class IcebergRemoveOrphanFilesActionTest { private static final long MIN_RETENTION_MS = Duration.ofHours(24).toMillis(); @@ -103,6 +106,44 @@ public void keepsVersionHintWhileDeletingAnOldOrphan(@TempDir Path temp) throws Assertions.assertTrue(Files.exists(versionHint)); } + @Test + public void keepsOrphansWhenFileCreationTimeIsUnknown(@TempDir Path temp) throws Exception { + Table table = createTable(temp.resolve("table"), Collections.emptyMap()); + Path zeroTimestamp = createOldFile(temp.resolve("table/data/zero.parquet")); + Path negativeTimestamp = createOldFile(temp.resolve("table/data/negative.parquet")); + RecordingFileIO recordingFileIO = new RecordingFileIO(table.io(), Collections.emptySet()) { + @Override + public Iterable listPrefix(String prefix) { + List files = StreamSupport.stream(super.listPrefix(prefix).spliterator(), false) + .map(file -> { + long createdAtMillis = file.createdAtMillis(); + if (IcebergRemoveOrphanFilesAction.sameFileIdentity( + file.location(), negativeTimestamp.toUri().toString())) { + createdAtMillis = -1; + } else if (IcebergRemoveOrphanFilesAction.sameFileIdentity( + file.location(), zeroTimestamp.toUri().toString())) { + createdAtMillis = 0; + } + return new FileInfo(file.location(), file.size(), createdAtMillis); + }) + .collect(Collectors.toList()); + return files; + } + }; + Table recordingTable = new BaseTable( + new StaticTableOperations(((HasTableOperations) table).operations().current(), recordingFileIO), + table.name()); + IcebergRemoveOrphanFilesAction action = action( + System.currentTimeMillis() - MIN_RETENTION_MS, false); + action.validate(); + + ConnectorProcedureResult result = action.execute(recordingTable, ActionTestTables.session("UTC")); + + Assertions.assertEquals("0", result.getRows().get(0).get(0)); + Assertions.assertTrue(Files.exists(zeroTimestamp)); + Assertions.assertTrue(Files.exists(negativeTimestamp)); + } + @Test public void treatsS3SchemeAliasesAsTheSameFile() { Assertions.assertTrue(IcebergRemoveOrphanFilesAction.sameFileIdentity( @@ -365,7 +406,7 @@ private static void appendDataFile(Table table, Path path) { table.newFastAppend().appendFile(dataFile).commit(); } - private static final class RecordingFileIO implements SupportsPrefixOperations { + private static class RecordingFileIO implements SupportsPrefixOperations { private final FileIO delegate; private final SupportsPrefixOperations prefixDelegate; private final Set manifestPaths; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 35780a456b26f3..06006086d4dfaa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -253,7 +253,8 @@ public class Coordinator implements CoordInterface { private String trackingUrl; private String firstErrorMsg; // related txnId and label of group commit - private long txnId; + // Final reports race with status readers, so the transaction identity must be safely published. + private volatile long txnId; private String label; // for export @@ -2632,8 +2633,10 @@ public boolean updateFragmentExecStatus(TReportExecStatusParams params) { LOG.info("query_id={} first_error_msg: {}", DebugUtil.printId(queryId), params.getFirstErrorMsg()); firstErrorMsg = params.getFirstErrorMsg(); } + // Keep this report's identity local so another report cannot redirect its commit data. + long reportTxnId = params.isSetTxnId() ? params.getTxnId() : txnId; if (params.isSetTxnId()) { - txnId = params.getTxnId(); + txnId = reportTxnId; } if (params.isSetLabel()) { label = params.getLabel(); @@ -2649,7 +2652,7 @@ public boolean updateFragmentExecStatus(TReportExecStatusParams params) { } if (params.isSetHivePartitionUpdates() || params.isSetIcebergCommitDatas() || params.isSetMcCommitDatas()) { - Transaction txn = Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr().getTxnById(txnId); + Transaction txn = Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr().getTxnById(reportTxnId); if (params.isSetHivePartitionUpdates()) { CommitDataSerializer.feed(txn, params.getHivePartitionUpdates()); } @@ -3080,7 +3083,8 @@ public static class PipelineExecContext { TPipelineFragmentParams rpcParams; PlanFragmentId fragmentId; boolean initiated; - boolean done; + // Non-final reports read this outside the monitor after updatePipelineStatus returns. + volatile boolean done; boolean processingDoneReport; TNetworkAddress brpcAddress; diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java index fd8f34d90e73ab..e10ee3073dad73 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java @@ -34,6 +34,10 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import com.google.common.cache.Cache; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.util.Collections; class QeProcessorImplReportAckTest { @@ -96,6 +100,34 @@ void retriesAcceptedExternalReportAfterCoordinatorRemoval() throws Exception { Mockito.verify(coordinator, Mockito.times(1)).updateFragmentExecStatus(params); } + @Test + void evictedAcceptanceTokenRejectsRetryAfterCoordinatorRemoval() throws Exception { + TUniqueId queryId = new TUniqueId(12345, 5); + Coordinator coordinator = register(queryId); + Mockito.when(coordinator.updateFragmentExecStatus(Mockito.any())).thenReturn(true); + TReportExecStatusParams params = params(queryId); + + TReportExecStatusResult first = report(params); + QeProcessorImpl.INSTANCE.unregisterQuery(queryId); + registeredQueryId = null; + acceptedExternalFileReports().invalidateAll(); + TReportExecStatusResult retry = report(params); + + Assertions.assertTrue(first.isExternalFileCommitDataAccepted()); + Assertions.assertFalse(retry.isExternalFileCommitDataAccepted()); + Assertions.assertEquals(TStatusCode.INTERNAL_ERROR, retry.getStatus().getStatusCode()); + Mockito.verify(coordinator, Mockito.times(1)).updateFragmentExecStatus(params); + } + + @Test + void coordinatorReportStateUsesCrossThreadVisibility() throws Exception { + Field done = Coordinator.PipelineExecContext.class.getDeclaredField("done"); + Field txnId = Coordinator.class.getDeclaredField("txnId"); + + Assertions.assertTrue(Modifier.isVolatile(done.getModifiers())); + Assertions.assertTrue(Modifier.isVolatile(txnId.getModifiers())); + } + @Test void legacyCoordinatorRetriesFailedAcceptanceBeforeMarkingDone() { Backend backend = Mockito.mock(Backend.class); @@ -133,4 +165,11 @@ private static TReportExecStatusParams params(TUniqueId queryId) { private static TReportExecStatusResult report(TReportExecStatusParams params) { return QeProcessorImpl.INSTANCE.reportExecStatus(params, new TNetworkAddress("127.0.0.1", 9050)); } + + @SuppressWarnings("unchecked") + private static Cache acceptedExternalFileReports() throws Exception { + Field field = QeProcessorImpl.class.getDeclaredField("acceptedExternalFileReports"); + field.setAccessible(true); + return (Cache) field.get(QeProcessorImpl.INSTANCE); + } } From b02a3ff09adbc35b0f2fb3cc9d169ed9506e2fde Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 7 Aug 2026 12:42:59 +0800 Subject: [PATCH 26/29] [fix](fe) Fix Checkstyle import ordering --- .../java/org/apache/doris/qe/QeProcessorImplReportAckTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java index e10ee3073dad73..2e15875c73ffd0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java @@ -29,13 +29,12 @@ import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TUniqueId; +import com.google.common.cache.Cache; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import com.google.common.cache.Cache; - import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.util.Collections; From 8314c9041270ca967f8c6ab4eebd4687ae307f33 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 7 Aug 2026 17:42:08 +0800 Subject: [PATCH 27/29] [fix](iceberg) Keep scan test compatible with catalog properties --- .../doris/connector/iceberg/IcebergScanPlanProviderTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java index d2d963e6829d9e..5766ebe8c71cc4 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java @@ -326,8 +326,8 @@ public void explicitEmptySnapshotDoesNotDriftToFirstConcurrentCommit() { .withSnapshot(-1L, null, -1L); table.newAppend().appendFile( dataFile(table.spec(), "s3://b/db/t1/concurrent.parquet", 1024, null, null)).commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider( - Collections.emptyMap(), opsReturning(table)); + // Keep construction behind the shared helper so catalog-property API migrations do not break this test. + IcebergScanPlanProvider provider = providerOver(table); List ranges = provider.planScan(null, ConnectorScanRequest.builder(emptyPin, Collections.emptyList()).build()); From 71abd40668dd002228bec77de06493982756e6aa Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 11 Aug 2026 00:17:11 +0800 Subject: [PATCH 28/29] [fix](azure) Isolate multipart blocks with upload UUIDs Issue Number: None Related PR: #66348 Problem Summary: Azure block uploads are scoped to a blob rather than a provider multipart session, while the BE abort path duplicated transaction cleanup ownership that belongs to FE. Generate a local UUID for each Azure writer, encode the full UUID in every block ID, and remove leases and the generic BE abort interface. Competing same-key writers now stage disjoint blocks; once one commits, a later stale block list fails closed. None - Test: Unit Test - AzureObjStorageExtensionTest: 21 tests passed - Affected BE production and test objects compiled successfully, including Azure with USE_AZURE enabled - Repository clang-format 16 check and git diff --check passed - FE reactor Checkstyle passed with 0 violations - clang-tidy was attempted but blocked by repository/toolchain baseline errors in types.h and missing system stddef.h - Behavior changed: Yes, Azure multipart writers use local UUID block namespaces without leases, and BE no longer owns provider abort cleanup - Does this need documentation: No --- .../iceberg/viceberg_partition_writer.cpp | 3 +- .../sink/writer/vhive_partition_writer.cpp | 11 - be/src/io/fs/azure_obj_storage_client.cpp | 116 ++------- be/src/io/fs/azure_obj_storage_client.h | 1 - be/src/io/fs/file_writer.h | 3 - be/src/io/fs/obj_storage_client.h | 9 +- .../io/fs/rate_limited_obj_storage_client.cpp | 6 - .../io/fs/rate_limited_obj_storage_client.h | 1 - be/src/io/fs/s3_file_writer.cpp | 66 +---- be/src/io/fs/s3_file_writer.h | 6 - be/src/io/fs/s3_obj_storage_client.cpp | 19 -- be/src/io/fs/s3_obj_storage_client.h | 1 - ...partition_writer_report_lifecycle_test.cpp | 28 +-- .../io/fs/azure_obj_storage_client_test.cpp | 118 ++++----- .../rate_limited_obj_storage_client_test.cpp | 23 -- be/test/io/fs/s3_file_writer_test.cpp | 54 +---- .../filesystem/azure/AzureObjStorage.java | 120 ++-------- .../azure/AzureObjStorageExtensionTest.java | 225 +++--------------- 18 files changed, 119 insertions(+), 691 deletions(-) diff --git a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp index 7faaebe9525e88..0d4653400e6530 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp @@ -125,8 +125,7 @@ Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profil } } if (!open_status.ok()) { - // A transformer failure happens after object creation, so abort multipart state before deleting the path. - WARN_IF_ERROR(_file_writer->abort(), "failed to abort Iceberg file after open error"); + // A transformer failure happens after object creation, so remove any published file. WARN_IF_ERROR(_fs->delete_file(_path), "failed to delete Iceberg file after open error"); } return open_status; diff --git a/be/src/exec/sink/writer/vhive_partition_writer.cpp b/be/src/exec/sink/writer/vhive_partition_writer.cpp index 658da3dbc8194f..40d7b38fc30236 100644 --- a/be/src/exec/sink/writer/vhive_partition_writer.cpp +++ b/be/src/exec/sink/writer/vhive_partition_writer.cpp @@ -162,12 +162,6 @@ Status VHivePartitionWriter::close(const Status& status) { } if (status_ok) { auto partition_update = _build_partition_update(); - if (partition_update.__isset.s3_mpu_pending_uploads) { - auto* s3_writer = dynamic_cast(_file_writer.get()); - DCHECK(s3_writer != nullptr); - // Until FE accepts the final report, BE remains the cleanup owner of staged uploads. - _state->add_rejected_external_file_report_cleanup(s3_writer->failed_report_cleanup()); - } _state->add_hive_partition_updates(partition_update); } return result_status; @@ -230,11 +224,6 @@ void VHivePartitionWriter::_add_s3_mpu_pending_upload_for_rollback() { if (!_build_s3_mpu_pending_upload(&s3_mpu_pending_upload)) { return; } - auto* s3_writer = dynamic_cast(_file_writer.get()); - DCHECK(s3_writer != nullptr); - // A failed write still relies on the final report to hand its staged upload to FE rollback. - _state->add_rejected_external_file_report_cleanup(s3_writer->failed_report_cleanup()); - THivePartitionUpdate hive_partition_update; hive_partition_update.__set_name(_partition_name); hive_partition_update.__set_update_mode(_update_mode); diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp index 52dea823f18302..67cb690a04fdba 100644 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ b/be/src/io/fs/azure_obj_storage_client.cpp @@ -21,7 +21,6 @@ #include #include -#include #include #include #include @@ -29,7 +28,6 @@ #include #include #include -#include #include #include #include @@ -42,6 +40,7 @@ #include #include #include +#include #include "common/exception.h" #include "common/logging.h" @@ -50,6 +49,7 @@ #include "io/fs/obj_storage_client.h" #include "util/bvar_helper.h" #include "util/s3_util.h" +#include "util/uid_util.h" using namespace Azure::Storage::Blobs; @@ -67,38 +67,17 @@ std::string to_lower_ascii(std::string_view input) { } std::string encode_azure_block_id(std::string_view upload_id, int part_num) { - uint32_t upload_namespace = 0x811C9DC5U; - for (unsigned char byte : upload_id) { - upload_namespace = (upload_namespace ^ byte) * 0x01000193U; - } - uint32_t namespaced_part = upload_namespace + static_cast(part_num); - // Four decoded bytes remain compatible with legacy residual blocks. Writer isolation is - // enforced by the target blob lease because no 32-bit namespace can identify every upload. - std::array raw_id {}; - for (size_t i = 0; i < raw_id.size(); ++i) { - raw_id[i] = static_cast(namespaced_part >> (i * 8)); + // Keep the full upload UUID in every block ID so independent writers cannot stage the + // same block IDs even though Azure has no per-upload multipart namespace. + std::vector raw_id(upload_id.begin(), upload_id.end()); + auto part = static_cast(part_num); + for (size_t i = 0; i < sizeof(part); ++i) { + raw_id.push_back(static_cast(part >> (i * 8))); } Aws::Utils::ByteBuffer bytes(raw_id.data(), raw_id.size()); return Aws::Utils::HashingUtils::Base64Encode(bytes); } -constexpr std::string_view MULTIPART_LEASE_PREFIX = "doris-azure-lease-v1:"; -constexpr std::chrono::seconds MULTIPART_LEASE_DURATION {60}; - -std::optional azure_multipart_lease_id(std::string_view upload_id) { - if (upload_id.starts_with(MULTIPART_LEASE_PREFIX) && - upload_id.size() > MULTIPART_LEASE_PREFIX.size()) { - return upload_id.substr(MULTIPART_LEASE_PREFIX.size()); - } - return std::nullopt; -} - -void renew_azure_multipart_lease(BlobClient& target_blob, std::string_view lease_id) { - BlobLeaseClient lease(target_blob, std::string(lease_id)); - // A renewal failure loses the upload-generation fence even if the same ID is acquirable later. - lease.Renew(); -} - // Rate limiting is applied by RateLimitedObjStorageClient, the decorator that // S3ClientFactory wraps around this client when the bucket is subject to limiting. @@ -228,28 +207,11 @@ struct AzureBatchDeleter { }; ObjectStorageUploadResponse AzureObjStorageClient::create_multipart_upload( - const ObjectStoragePathOptions& opts) { - auto target_blob = _client->GetBlobClient(opts.key); - auto target_client = target_blob.AsBlockBlobClient(); - std::string lease_id = BlobLeaseClient::CreateUniqueLeaseId(); - std::string upload_id = fmt::format("{}{}", MULTIPART_LEASE_PREFIX, lease_id); - auto resp = do_azure_client_call( - [&]() { - uint8_t empty = 0; - Azure::Core::IO::MemoryBodyStream empty_body(&empty, 0); - // The reservation makes an absent blob leaseable but remains uncommitted and - // invisible to normal listings until Put Block List publishes the real data. - target_client.StageBlock(azure_multipart_block_id(upload_id, 0), empty_body); - auto lease = - BlobLeaseClient(target_blob, lease_id).Acquire(MULTIPART_LEASE_DURATION); - upload_id = fmt::format("{}{}", MULTIPART_LEASE_PREFIX, lease.Value.LeaseId); - }, - opts, _tls_debug_context); + const ObjectStoragePathOptions&) { + // Azure has no multipart session; this local UUID only namespaces the writer's block IDs. return ObjectStorageUploadResponse { - .resp = resp, - .upload_id = resp.status.code == ErrorCode::OK - ? std::make_optional(std::move(upload_id)) - : std::nullopt, + .resp = ObjectStorageResponse::OK(), + .upload_id = generate_uuid_string(), }; } @@ -268,8 +230,7 @@ ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStora std::string_view stream, int part_num) { DCHECK(opts.upload_id.has_value()); - auto target_blob = _client->GetBlobClient(opts.key); - auto client = target_blob.AsBlockBlobClient(); + auto client = _client->GetBlockBlobClient(opts.key); std::string block_id = azure_multipart_block_id(*opts.upload_id, part_num); auto resp = do_azure_client_call( [&]() { @@ -277,15 +238,7 @@ ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStora reinterpret_cast(stream.data()), stream.size()); // The blockId must be base64 encoded SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); - auto lease_id = azure_multipart_lease_id(*opts.upload_id); - if (lease_id.has_value()) { - renew_azure_multipart_lease(target_blob, *lease_id); - StageBlockOptions stage_opts; - stage_opts.AccessConditions.LeaseId = std::string(*lease_id); - client.StageBlock(block_id, memory_body, stage_opts); - } else { - client.StageBlock(block_id, memory_body); - } + client.StageBlock(block_id, memory_body); }, opts, _tls_debug_context); return ObjectStorageUploadResponse { @@ -299,54 +252,19 @@ ObjectStorageResponse AzureObjStorageClient::complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) { DCHECK(opts.upload_id.has_value()); - auto target_blob = _client->GetBlobClient(opts.key); - auto target_client = target_blob.AsBlockBlobClient(); + auto target_client = _client->GetBlockBlobClient(opts.key); std::vector string_block_ids; std::ranges::transform(completed_parts, std::back_inserter(string_block_ids), [&opts](const ObjectCompleteMultiPart& i) { return azure_multipart_block_id(*opts.upload_id, i.part_num); }); - auto resp = do_azure_client_call( + return do_azure_client_call( [&]() { SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); // Put Block List atomically replaces the committed blob; no scan-visible staging blob exists. - auto lease_id = azure_multipart_lease_id(*opts.upload_id); - if (lease_id.has_value()) { - renew_azure_multipart_lease(target_blob, *lease_id); - CommitBlockListOptions commit_opts; - commit_opts.AccessConditions.LeaseId = std::string(*lease_id); - target_client.CommitBlockList(string_block_ids, commit_opts); - } else { - target_client.CommitBlockList(string_block_ids); - } + target_client.CommitBlockList(string_block_ids); }, opts, _tls_debug_context); - if (resp.status.code == ErrorCode::OK) { - if (auto lease_id = azure_multipart_lease_id(*opts.upload_id); lease_id.has_value()) { - auto release_resp = do_azure_client_call( - [&]() { BlobLeaseClient(target_blob, std::string(*lease_id)).Release(); }, opts, - _tls_debug_context); - if (release_resp.status.code != ErrorCode::OK) { - LOG(WARNING) << "Azure multipart commit succeeded but its finite lease could not " - "be released; it will expire automatically"; - } - } - } - return resp; -} - -ObjectStorageResponse AzureObjStorageClient::abort_multipart_upload( - const ObjectStoragePathOptions& opts) { - DCHECK(opts.upload_id.has_value()); - if (auto lease_id = azure_multipart_lease_id(*opts.upload_id); lease_id.has_value()) { - auto target_blob = _client->GetBlobClient(opts.key); - return do_azure_client_call( - [&]() { BlobLeaseClient(target_blob, std::string(*lease_id)).Release(); }, opts, - _tls_debug_context); - } - // Azure cannot delete one upload's uncommitted blocks without changing the committed blob. - // Leaving them to service GC preserves the last successfully published value. - return ObjectStorageResponse::OK(); } ObjectStorageHeadResponse AzureObjStorageClient::head_object(const ObjectStoragePathOptions& opts) { diff --git a/be/src/io/fs/azure_obj_storage_client.h b/be/src/io/fs/azure_obj_storage_client.h index de4ea5459a7cf7..6cf6493e082af8 100644 --- a/be/src/io/fs/azure_obj_storage_client.h +++ b/be/src/io/fs/azure_obj_storage_client.h @@ -50,7 +50,6 @@ class AzureObjStorageClient final : public ObjStorageClient { ObjectStorageResponse complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) override; - ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions& opts) override; ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override; ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, size_t offset, size_t bytes_read, diff --git a/be/src/io/fs/file_writer.h b/be/src/io/fs/file_writer.h index 08754ec3689a0f..9402fdef18303c 100644 --- a/be/src/io/fs/file_writer.h +++ b/be/src/io/fs/file_writer.h @@ -73,9 +73,6 @@ class FileWriter { // If there is no data appended, an empty file will be persisted. virtual Status close(bool non_block = false) = 0; - // Abandon an unpublished file. Remote writers should cancel multipart state instead of completing it. - virtual Status abort() { return close(); } - // Non-blocking probe for a previous close(true). // OK means close finished successfully. NeedSendAgain means close is still running. // Other errors mean close finished with error or the writer does not support this API. diff --git a/be/src/io/fs/obj_storage_client.h b/be/src/io/fs/obj_storage_client.h index db326a931719f9..2688ee70e02a4b 100644 --- a/be/src/io/fs/obj_storage_client.h +++ b/be/src/io/fs/obj_storage_client.h @@ -44,7 +44,7 @@ struct ObjectStoragePathOptions { std::string bucket = std::string(); // blob container in azure std::string key = std::string(); // blob name in azure std::string prefix = std::string(); // for batch delete and recursive delete - std::optional upload_id = std::nullopt; // provider-specific upload token + std::optional upload_id = std::nullopt; // token identifying this writer's parts }; struct ObjectCompleteMultiPart { @@ -86,7 +86,8 @@ struct ObjectStorageHeadResponse : ObjectStorageResponse { class ObjStorageClient { public: virtual ~ObjStorageClient() = default; - // Create a multi-part upload request. The returned provider token identifies this upload's parts. + // Create a multi-part upload request. The returned token may be provider-issued or local and + // identifies this writer's parts. // The input parameters should include the bucket and key for the object storage. virtual ObjectStorageUploadResponse create_multipart_upload( const ObjectStoragePathOptions& opts) = 0; @@ -106,10 +107,6 @@ class ObjStorageClient { virtual ObjectStorageResponse complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) = 0; - virtual ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions&) { - return {.status = {.code = ErrorCode::NOT_IMPLEMENTED_ERROR, - .msg = "abort multipart upload is not supported"}}; - } // According to the passed bucket and key, it will access whether the corresponding file exists in the object storage. // If it exists, it will return the corresponding file size virtual ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) = 0; diff --git a/be/src/io/fs/rate_limited_obj_storage_client.cpp b/be/src/io/fs/rate_limited_obj_storage_client.cpp index 218c39cb4b19ec..1b8730847162df 100644 --- a/be/src/io/fs/rate_limited_obj_storage_client.cpp +++ b/be/src/io/fs/rate_limited_obj_storage_client.cpp @@ -73,12 +73,6 @@ ObjectStorageResponse RateLimitedObjStorageClient::complete_multipart_upload( return _inner->complete_multipart_upload(opts, completed_parts); } -ObjectStorageResponse RateLimitedObjStorageClient::abort_multipart_upload( - const ObjectStoragePathOptions& opts) { - // Cleanup must reach the provider even when a hard PUT limit caused the upload failure. - return _inner->abort_multipart_upload(opts); -} - ObjectStorageHeadResponse RateLimitedObjStorageClient::head_object( const ObjectStoragePathOptions& opts) { S3RateLimitGuard guard(S3RateLimitType::GET, 0); diff --git a/be/src/io/fs/rate_limited_obj_storage_client.h b/be/src/io/fs/rate_limited_obj_storage_client.h index dc6fb1503c375d..00725d7edcb299 100644 --- a/be/src/io/fs/rate_limited_obj_storage_client.h +++ b/be/src/io/fs/rate_limited_obj_storage_client.h @@ -50,7 +50,6 @@ class RateLimitedObjStorageClient final : public ObjStorageClient { ObjectStorageResponse complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) override; - ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions& opts) override; ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override; ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, size_t offset, size_t bytes_read, diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index 63eb6e32c44c25..a85aa5ce405ee9 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -78,8 +78,6 @@ S3FileWriter::~S3FileWriter() { // For thread safety std::ignore = _async_close_pack->future.get(); _async_close_pack = nullptr; - } else if (state() == State::OPENED) { - WARN_IF_ERROR(abort(), "failed to abort unfinished S3 writer"); } else { // Consider one situation where the file writer is destructed after it submit at least one async task // without calling close(), then there exists one occasion where the async task is executed right after @@ -87,62 +85,14 @@ S3FileWriter::~S3FileWriter() { _wait_until_finish(fmt::format("wait s3 file {} upload to be finished", _obj_storage_path_opts.path.native())); } + // Deferred uploads are reported to FE for cleanup. Uploads that never reach FE are left to + // the provider lifecycle policy, so destroying a writer must not mutate provider state here. if (state() == State::OPENED && !_failed) { s3_bytes_written_total << _bytes_appended; } s3_file_being_written << -1; } -Status S3FileWriter::abort() { - if (state() == State::CLOSED) { - return Status::OK(); - } - if (state() == State::ASYNC_CLOSING) { - return Status::InternalError("cannot abort an asynchronously closing S3 writer"); - } - RETURN_IF_ERROR(_abort_impl()); - _state = State::CLOSED; - return Status::OK(); -} - -std::function S3FileWriter::failed_report_cleanup() const { - auto client_holder = _obj_client; - auto path_opts = _obj_storage_path_opts; - return [client_holder = std::move(client_holder), path_opts = std::move(path_opts)] { - // The writer is already CLOSED, but ownership was not transferred; bypass abort()'s - // state guard while retaining the provider client and the exact upload identity. - const auto& client = client_holder->get(); - if (client == nullptr) { - LOG(WARNING) << "failed to abort a rejected external-file report: invalid object " - "storage client"; - return; - } - auto response = client->abort_multipart_upload(path_opts); - if (response.status.code != ErrorCode::OK) { - LOG(WARNING) << "failed to abort a rejected external-file report for " - << path_opts.path.native() << ": " << response.status.msg; - } - }; -} - -Status S3FileWriter::_abort_impl() { - _wait_until_finish( - fmt::format("wait s3 file {} before abort", _obj_storage_path_opts.path.native())); - _pending_buf.reset(); - if (_multipart_upload_started) { - const auto& client = _obj_client->get(); - if (client == nullptr) { - return Status::InternalError("invalid obj storage client"); - } - auto response = client->abort_multipart_upload(_obj_storage_path_opts); - if (response.status.code != ErrorCode::OK) { - return {response.status.code, std::move(response.status.msg)}; - } - } - // Once abort returns, no destructor or retry may complete the abandoned upload. - return Status::OK(); -} - Status S3FileWriter::_create_multi_upload_request() { LOG(INFO) << "create_multi_upload_request " << _obj_storage_path_opts.path.native(); const auto& client = _obj_client->get(); @@ -151,8 +101,6 @@ Status S3FileWriter::_create_multi_upload_request() { } auto resp = client->create_multipart_upload(_obj_storage_path_opts); if (resp.resp.status.code == ErrorCode::OK) { - // Some providers identify staged uploads by block IDs instead of a server-issued upload ID. - _multipart_upload_started = true; _obj_storage_path_opts.upload_id = resp.upload_id; } return {resp.resp.status.code, std::move(resp.resp.status.msg)}; @@ -215,10 +163,6 @@ Status S3FileWriter::close(bool non_block) { s3_file_writer_async_close_queuing << -1; s3_file_writer_async_close_processing << 1; _st = _close_impl(); - if (!_st.ok()) { - // A failed completion must not leave server-side multipart state behind. - WARN_IF_ERROR(_abort_impl(), "failed to abort incomplete S3 upload"); - } _async_close_pack->promise.set_value(_st); s3_file_writer_async_close_processing << -1; }); @@ -229,18 +173,12 @@ Status S3FileWriter::close(bool non_block) { << _obj_storage_path_opts.path.native() << ", fallback to sync close, status=" << submit_status; _st = _close_impl(); - if (!_st.ok()) { - WARN_IF_ERROR(_abort_impl(), "failed to abort incomplete S3 upload"); - } _async_close_pack->promise.set_value(_st); return _st; } return Status::OK(); } _st = _close_impl(); - if (!_st.ok()) { - WARN_IF_ERROR(_abort_impl(), "failed to abort incomplete S3 upload"); - } _state = State::CLOSED; if (!non_block && _st.ok()) { _record_close_latency(); diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h index 83a2100c0ff943..83ec75c9184920 100644 --- a/be/src/io/fs/s3_file_writer.h +++ b/be/src/io/fs/s3_file_writer.h @@ -21,7 +21,6 @@ #include #include -#include #include #include #include @@ -72,13 +71,9 @@ class S3FileWriter final : public FileWriter { } Status close(bool non_block = false) override; - Status abort() override; Status try_finish_close() override; - std::function failed_report_cleanup() const; - private: - Status _abort_impl(); Status _close_impl(); [[nodiscard]] std::string _dump_completed_part() const; void _wait_until_finish(std::string_view task_name); @@ -122,7 +117,6 @@ class S3FileWriter final : public FileWriter { std::shared_ptr _obj_client; std::optional _first_append_timestamp; bool _close_latency_recorded = false; - bool _multipart_upload_started = false; }; } // namespace io diff --git a/be/src/io/fs/s3_obj_storage_client.cpp b/be/src/io/fs/s3_obj_storage_client.cpp index 54ba6b687e9790..0c0b0370f8097f 100644 --- a/be/src/io/fs/s3_obj_storage_client.cpp +++ b/be/src/io/fs/s3_obj_storage_client.cpp @@ -275,25 +275,6 @@ ObjectStorageResponse S3ObjStorageClient::complete_multipart_upload( return ObjectStorageResponse::OK(); } -ObjectStorageResponse S3ObjStorageClient::abort_multipart_upload( - const ObjectStoragePathOptions& opts) { - AbortMultipartUploadRequest request; - request.WithBucket(opts.bucket).WithKey(opts.key).WithUploadId(*opts.upload_id); - auto outcome = SYNC_POINT_HOOK_RETURN_VALUE(_client->AbortMultipartUpload(request), - "s3_file_writer::abort_multi_part", - std::cref(request).get()); - if (!outcome.IsSuccess()) { - record_s3_request_failed(outcome.GetError()); - auto status = s3fs_error(outcome.GetError(), - fmt::format("failed to AbortMultipartUpload: {}, upload_id={}", - opts.path.native(), *opts.upload_id)); - return {convert_to_obj_response(std::move(status)), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}; - } - return ObjectStorageResponse::OK(); -} - ObjectStorageHeadResponse S3ObjStorageClient::head_object(const ObjectStoragePathOptions& opts) { Aws::S3::Model::HeadObjectRequest request; request.WithBucket(opts.bucket).WithKey(opts.key); diff --git a/be/src/io/fs/s3_obj_storage_client.h b/be/src/io/fs/s3_obj_storage_client.h index 10bcf6b2e9495b..45294226594d81 100644 --- a/be/src/io/fs/s3_obj_storage_client.h +++ b/be/src/io/fs/s3_obj_storage_client.h @@ -43,7 +43,6 @@ class S3ObjStorageClient final : public ObjStorageClient { ObjectStorageResponse complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) override; - ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions& opts) override; ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override; ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, size_t offset, size_t bytes_read, diff --git a/be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp b/be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp index ffc54b5f1b4936..73862ac8b0580d 100644 --- a/be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp +++ b/be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp @@ -57,13 +57,6 @@ class RecordingObjStorageClient final : public io::ObjStorageClient { return io::ObjectStorageResponse::OK(); } - io::ObjectStorageResponse abort_multipart_upload( - const io::ObjectStoragePathOptions& opts) override { - ++abort_count; - aborted_upload_id = opts.upload_id.value_or(""); - return io::ObjectStorageResponse::OK(); - } - io::ObjectStorageHeadResponse head_object(const io::ObjectStoragePathOptions&) override { return {.resp = io::ObjectStorageResponse::OK(), .file_size = 0}; } @@ -96,9 +89,6 @@ class RecordingObjStorageClient final : public io::ObjStorageClient { const S3ClientConf&) override { return {}; } - - int abort_count = 0; - std::string aborted_upload_id; }; class FixedLengthTransformer final : public VFileFormatTransformer { @@ -178,7 +168,7 @@ ReportStatusRequest report_request(RuntimeState* state, bool done) { } // namespace TEST(VHivePartitionWriterReportLifecycleTest, - PeriodicReportDefersMetadataAndRejectedFinalReportAbortsProviderUpload) { + PeriodicReportDefersMetadataAndFinalReportTransfersUploadIdentity) { MockRuntimeState state; VExprContextSPtrs output_exprs; auto client = std::make_shared(); @@ -190,7 +180,6 @@ TEST(VHivePartitionWriterReportLifecycleTest, context->_append_external_file_commit_data(periodic_request, &periodic_params); EXPECT_FALSE(periodic_params.__isset.hive_partition_updates); - EXPECT_EQ(0, client->abort_count); TReportExecStatusParams final_params; auto final_request = report_request(&state, true); @@ -203,17 +192,9 @@ TEST(VHivePartitionWriterReportLifecycleTest, EXPECT_EQ("bucket", pending_upload.bucket); EXPECT_EQ("table/part.parquet", pending_upload.key); EXPECT_EQ("upload-id", pending_upload.upload_id); - - // A definite coordinator rejection must consume the same cleanup owner that was retained - // while the periodic report deliberately withheld the pending-upload record. - context->_coordinator_callback(final_request); - - EXPECT_EQ(1, client->abort_count); - EXPECT_EQ("upload-id", client->aborted_upload_id); } -TEST(VHivePartitionWriterReportLifecycleTest, - AzureFinalReportCarriesExactBlockIdentityAndRejectionAbortsUpload) { +TEST(VHivePartitionWriterReportLifecycleTest, AzureFinalReportCarriesExactBlockIdentity) { MockRuntimeState state; VExprContextSPtrs output_exprs; auto client = std::make_shared(); @@ -231,11 +212,6 @@ TEST(VHivePartitionWriterReportLifecycleTest, ASSERT_EQ(1, pending_uploads.size()); EXPECT_EQ("upload-id", pending_uploads[0].upload_id); EXPECT_EQ("exact-block-id", pending_uploads[0].etags.at(1)); - - // Azure staged blocks stay owned by BE until the final coordinator ACK transfers them. - context->_coordinator_callback(final_request); - EXPECT_EQ(1, client->abort_count); - EXPECT_EQ("upload-id", client->aborted_upload_id); } } // namespace doris diff --git a/be/test/io/fs/azure_obj_storage_client_test.cpp b/be/test/io/fs/azure_obj_storage_client_test.cpp index dc7a2314dddb01..a45db87bfe0268 100644 --- a/be/test/io/fs/azure_obj_storage_client_test.cpp +++ b/be/test/io/fs/azure_obj_storage_client_test.cpp @@ -20,6 +20,8 @@ #include #include +#include +#include #include "io/fs/file_system.h" #include "io/fs/obj_storage_client.h" @@ -38,16 +40,39 @@ namespace doris { #ifdef USE_AZURE -TEST(AzureObjStorageClientMultipartHelperTest, fixed_length_namespace_requires_target_lease) { - EXPECT_EQ("p3w3DA==", io::azure_multipart_block_id("upload-a", 1)); - EXPECT_EQ("Sc7grw==", io::azure_multipart_block_id( - "doris-azure-lease-v1:09492e3d-e231-4ed9-bf84-b6fc772cda54", 1)); - EXPECT_EQ("Sc7grw==", io::azure_multipart_block_id( - "doris-azure-lease-v1:06996d15-1c2e-4ddd-8853-43816ea84a07", 1)); - EXPECT_EQ(io::azure_multipart_block_id("upload-a", 1).size(), - io::azure_multipart_block_id("upload-a", 999).size()); - EXPECT_EQ(4, Aws::Utils::HashingUtils::Base64Decode(io::azure_multipart_block_id("upload-a", 1)) - .GetLength()); +TEST(AzureObjStorageClientMultipartHelperTest, full_upload_uuid_isolates_writer_blocks) { + constexpr std::string_view first_upload = "09492e3d-e231-4ed9-bf84-b6fc772cda54"; + constexpr std::string_view second_upload = "06996d15-1c2e-4ddd-8853-43816ea84a07"; + auto first_block = io::azure_multipart_block_id(first_upload, 1); + auto second_block = io::azure_multipart_block_id(second_upload, 1); + + EXPECT_NE(first_block, second_block); + EXPECT_EQ(first_block.size(), io::azure_multipart_block_id(first_upload, 999).size()); + auto decoded = Aws::Utils::HashingUtils::Base64Decode(first_block); + ASSERT_EQ(first_upload.size() + sizeof(uint32_t), decoded.GetLength()); + EXPECT_EQ(first_upload, + std::string_view(reinterpret_cast(decoded.GetUnderlyingData()), + first_upload.size())); + EXPECT_EQ(1, decoded.GetUnderlyingData()[first_upload.size()]); + EXPECT_EQ(0, decoded.GetUnderlyingData()[first_upload.size() + 1]); + EXPECT_EQ(0, decoded.GetUnderlyingData()[first_upload.size() + 2]); + EXPECT_EQ(0, decoded.GetUnderlyingData()[first_upload.size() + 3]); +} + +TEST(AzureObjStorageClientMultipartHelperTest, create_upload_is_provider_free) { + io::AzureObjStorageClient client( + std::shared_ptr {}); + + auto first = client.create_multipart_upload({}); + auto second = client.create_multipart_upload({}); + + ASSERT_EQ(ErrorCode::OK, first.resp.status.code); + ASSERT_EQ(ErrorCode::OK, second.resp.status.code); + ASSERT_TRUE(first.upload_id.has_value()); + ASSERT_TRUE(second.upload_id.has_value()); + EXPECT_EQ(36, first.upload_id->size()); + EXPECT_EQ(36, second.upload_id->size()); + EXPECT_NE(first.upload_id, second.upload_id); } using namespace Azure::Storage::Blobs; @@ -173,85 +198,36 @@ TEST_F(AzureObjStorageClientTest, delete_objects_recursively) { EXPECT_EQ(files.size(), 0); } -TEST_F(AzureObjStorageClientTest, abort_multipart_upload_leaves_no_visible_blob) { - io::ObjectStoragePathOptions opts {.key = "AzureObjStorageClientTest/abort_multipart_upload"}; - auto create_response = - AzureObjStorageClientTest::obj_storage_client->create_multipart_upload(opts); - ASSERT_EQ(create_response.resp.status.code, ErrorCode::OK); - ASSERT_TRUE(create_response.upload_id.has_value()); - opts.upload_id = create_response.upload_id; - - auto upload_response = - AzureObjStorageClientTest::obj_storage_client->upload_part(opts, "staged", 1); - ASSERT_EQ(upload_response.resp.status.code, ErrorCode::OK); - ASSERT_TRUE(upload_response.etag.has_value()); - EXPECT_FALSE(upload_response.etag->empty()); - auto abort_response = - AzureObjStorageClientTest::obj_storage_client->abort_multipart_upload(opts); - ASSERT_EQ(abort_response.status.code, ErrorCode::OK); - - auto head_response = AzureObjStorageClientTest::obj_storage_client->head_object(opts); - EXPECT_EQ(head_response.resp.status.code, ErrorCode::NOT_FOUND); -} - -TEST_F(AzureObjStorageClientTest, abort_multipart_upload_preserves_existing_put_blob) { - io::ObjectStoragePathOptions opts {.key = "AzureObjStorageClientTest/abort_preserves_put_blob"}; - auto put_response = AzureObjStorageClientTest::obj_storage_client->put_object(opts, "original"); - ASSERT_EQ(put_response.status.code, ErrorCode::OK); - auto create_response = - AzureObjStorageClientTest::obj_storage_client->create_multipart_upload(opts); - ASSERT_EQ(create_response.resp.status.code, ErrorCode::OK); - ASSERT_TRUE(create_response.upload_id.has_value()); - opts.upload_id = create_response.upload_id; - - auto upload_response = - AzureObjStorageClientTest::obj_storage_client->upload_part(opts, "replacement", 1); - ASSERT_EQ(upload_response.resp.status.code, ErrorCode::OK); - - auto abort_response = - AzureObjStorageClientTest::obj_storage_client->abort_multipart_upload(opts); - ASSERT_EQ(abort_response.status.code, ErrorCode::OK); - std::array contents {}; - size_t size_return = 0; - auto get_response = AzureObjStorageClientTest::obj_storage_client->get_object( - opts, contents.data(), 0, contents.size(), &size_return); - ASSERT_EQ(get_response.status.code, ErrorCode::OK); - EXPECT_EQ(size_return, contents.size()); - EXPECT_EQ(std::string_view(contents.data(), contents.size()), "original"); - - EXPECT_EQ(AzureObjStorageClientTest::obj_storage_client->delete_object(opts).status.code, - ErrorCode::OK); -} - TEST_F(AzureObjStorageClientTest, concurrent_multipart_uploads_do_not_share_staged_blocks) { io::ObjectStoragePathOptions first {.key = "AzureObjStorageClientTest/concurrent_multipart"}; io::ObjectStoragePathOptions second = first; auto first_create = obj_storage_client->create_multipart_upload(first); auto second_create = obj_storage_client->create_multipart_upload(second); ASSERT_EQ(first_create.resp.status.code, ErrorCode::OK); - ASSERT_NE(second_create.resp.status.code, ErrorCode::OK); + ASSERT_EQ(second_create.resp.status.code, ErrorCode::OK); ASSERT_TRUE(first_create.upload_id.has_value()); + ASSERT_TRUE(second_create.upload_id.has_value()); + ASSERT_NE(first_create.upload_id, second_create.upload_id); first.upload_id = first_create.upload_id; + second.upload_id = second_create.upload_id; - ASSERT_EQ(obj_storage_client->upload_part(first, "first", 1).resp.status.code, ErrorCode::OK); + auto first_part = obj_storage_client->upload_part(first, "first", 1); + auto second_part = obj_storage_client->upload_part(second, "second", 1); + ASSERT_EQ(first_part.resp.status.code, ErrorCode::OK); + ASSERT_EQ(second_part.resp.status.code, ErrorCode::OK); + ASSERT_NE(first_part.etag, second_part.etag); ASSERT_EQ(obj_storage_client->complete_multipart_upload(first, {{.part_num = 1}}).status.code, ErrorCode::OK); - - second_create = obj_storage_client->create_multipart_upload(second); - ASSERT_EQ(second_create.resp.status.code, ErrorCode::OK); - ASSERT_TRUE(second_create.upload_id.has_value()); - second.upload_id = second_create.upload_id; - ASSERT_EQ(obj_storage_client->upload_part(second, "second", 1).resp.status.code, ErrorCode::OK); - ASSERT_EQ(obj_storage_client->complete_multipart_upload(second, {{.part_num = 1}}).status.code, + ASSERT_NE(obj_storage_client->complete_multipart_upload(second, {{.part_num = 1}}).status.code, ErrorCode::OK); - std::array contents {}; + std::array contents {}; size_t size_return = 0; ASSERT_EQ(obj_storage_client ->get_object(second, contents.data(), 0, contents.size(), &size_return) .status.code, ErrorCode::OK); - EXPECT_EQ(std::string_view(contents.data(), size_return), "second"); + EXPECT_EQ(std::string_view(contents.data(), size_return), "first"); EXPECT_EQ(obj_storage_client->delete_object(second).status.code, ErrorCode::OK); } #else diff --git a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp index 7b7686e187de56..edaeb80bd2bbfc 100644 --- a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp +++ b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp @@ -62,11 +62,6 @@ class FakeObjStorageClient : public ObjStorageClient { ++calls; return ObjectStorageResponse::OK(); } - ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions& opts) override { - ++calls; - ++abort_multipart_upload_calls; - return ObjectStorageResponse::OK(); - } ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override { ++calls; return {}; @@ -111,7 +106,6 @@ class FakeObjStorageClient : public ObjStorageClient { int create_multipart_upload_calls = 0; int create_multipart_upload_provider_calls = 0; int create_multipart_upload_provider_calls_per_logical_call = 1; - int abort_multipart_upload_calls = 0; int delete_objects_recursively_calls = 0; int delete_objects_recursively_provider_calls = 0; int delete_objects_recursively_provider_calls_per_logical_call = 1; @@ -374,23 +368,6 @@ TEST(RateLimitedObjStorageClientTest, multipart_control_apis_map_to_put_qps_with EXPECT_EQ(-1, put_bytes->add(1)); } -TEST(RateLimitedObjStorageClientTest, abortBypassesAnExhaustedPutLimit) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - manager.qps_limiter(S3RateLimitType::PUT) - ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); - manager.bytes_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k", .upload_id = "upload"}; - - EXPECT_EQ(0, client.create_multipart_upload(opts).resp.status.code); - EXPECT_EQ(0, client.abort_multipart_upload(opts).status.code); - EXPECT_EQ(1, fake->abort_multipart_upload_calls); -} - TEST(RateLimitedObjStorageClientTest, delete_apis_map_to_put_qps_without_bytes) { RateLimiterConfigGuard guard; config::enable_s3_rate_limiter = true; diff --git a/be/test/io/fs/s3_file_writer_test.cpp b/be/test/io/fs/s3_file_writer_test.cpp index c0c42239a1858a..3937d6e38561fe 100644 --- a/be/test/io/fs/s3_file_writer_test.cpp +++ b/be/test/io/fs/s3_file_writer_test.cpp @@ -316,21 +316,6 @@ class S3FileWriterTest : public testing::Test { } }; -TEST_F(S3FileWriterTest, abort_cleans_up_multipart_upload) { - mock_client = std::make_shared(); - doris::io::FileWriterOptions options; - - io::FileWriterPtr writer; - ASSERT_TRUE(s3_fs->create_file("abort_multipart", &writer, &options).ok()); - std::string data(config::s3_write_buffer_size, 'a'); - ASSERT_TRUE(writer->append(Slice(data)).ok()); - ASSERT_FALSE(static_cast(writer.get())->upload_id().empty()); - - ASSERT_TRUE(writer->abort().ok()); - EXPECT_EQ(writer->state(), io::FileWriter::State::CLOSED); - EXPECT_TRUE(mock_client->contents().empty()); -} - TEST_F(S3FileWriterTest, multi_part_io_error) { mock_client = std::make_shared(); doris::io::FileWriterOptions state; @@ -1169,14 +1154,6 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { return default_response; } - ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions& opts) override { - std::lock_guard lock(_mutex); - abort_multipart_count++; - last_opts = opts; - parts.clear(); - return default_response; - } - ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override { std::lock_guard lock(_mutex); return {.resp = ObjectStorageResponse::OK(), @@ -1251,7 +1228,6 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { int put_object_count = 0; int upload_part_count = 0; int complete_multipart_count = 0; - int abort_multipart_count = 0; // Structures to store input parameters for each call struct UploadPartParams { @@ -1290,7 +1266,6 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { put_object_count = 0; upload_part_count = 0; complete_multipart_count = 0; - abort_multipart_count = 0; create_multipart_params.clear(); put_object_params.clear(); @@ -1319,9 +1294,8 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { * @return A tuple containing the mock S3 client and the S3FileWriter. */ std::tuple, std::shared_ptr> -create_s3_client(const std::string& path, bool used_by_s3_committer = false) { +create_s3_client(const std::string& path) { doris::io::FileWriterOptions opts; - opts.used_by_s3_committer = used_by_s3_committer; io::FileWriterPtr file_writer; auto st = s3_fs->create_file(path, &file_writer, &opts); EXPECT_TRUE(st.ok()) << st; @@ -1333,32 +1307,6 @@ create_s3_client(const std::string& path, bool used_by_s3_committer = false) { return {mock_client, s3_file_writer}; } -TEST_F(S3FileWriterTest, abortsProviderMultipartWithoutAnUploadId) { - auto [client, writer] = create_s3_client("provider_without_upload_id"); - client->default_upload_response.upload_id.reset(); - std::string data(config::s3_write_buffer_size, 'a'); - - ASSERT_TRUE(writer->append(Slice(data)).ok()); - ASSERT_TRUE(writer->abort().ok()); - - EXPECT_EQ(1, client->create_multipart_count); - EXPECT_EQ(1, client->abort_multipart_count); - EXPECT_EQ(FileWriter::State::CLOSED, writer->state()); -} - -TEST_F(S3FileWriterTest, failedReportCleanupAbortsDeferredProviderUploadAfterClose) { - auto [client, writer] = create_s3_client("deferred_report_rejected", true); - std::string data(config::s3_write_buffer_size, 'a'); - ASSERT_TRUE(writer->append(Slice(data)).ok()); - ASSERT_TRUE(writer->close().ok()); - ASSERT_EQ(FileWriter::State::CLOSED, writer->state()); - auto cleanup = writer->failed_report_cleanup(); - - cleanup(); - - EXPECT_EQ(1, client->abort_multipart_count); -} - /** * Generate test data for S3FileWriter boundary tests. * Returns a vector of sizes that we'll use to generate data on demand. diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java index a269ceca30c5f0..837e739abd184b 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java @@ -25,8 +25,6 @@ import com.azure.core.http.rest.PagedIterable; import com.azure.core.http.rest.PagedResponse; -import com.azure.core.util.BinaryData; -import com.azure.core.util.Context; import com.azure.identity.ClientSecretCredentialBuilder; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; @@ -35,14 +33,10 @@ import com.azure.storage.blob.BlobServiceClientBuilder; import com.azure.storage.blob.models.BlobItem; import com.azure.storage.blob.models.BlobProperties; -import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.models.ListBlobsOptions; -import com.azure.storage.blob.options.BlockBlobCommitBlockListOptions; import com.azure.storage.blob.sas.BlobSasPermission; import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; -import com.azure.storage.blob.specialized.BlobLeaseClient; -import com.azure.storage.blob.specialized.BlobLeaseClientBuilder; import com.azure.storage.blob.specialized.BlockBlobClient; import com.azure.storage.common.StorageSharedKeyCredential; import com.azure.storage.common.sas.SasProtocol; @@ -77,8 +71,6 @@ public class AzureObjStorage implements ObjStorage { private static final Logger LOG = LogManager.getLogger(AzureObjStorage.class); private static final int HTTP_NOT_FOUND = 404; - private static final int MULTIPART_LEASE_SECONDS = 60; - private static final String MULTIPART_LEASE_PREFIX = "doris-azure-lease-v1:"; /** Validity period for presigned (SAS) URLs, in seconds. */ private static final int SESSION_EXPIRE_SECONDS = 3600; @@ -228,23 +220,8 @@ public void copyObject(String srcPath, String dstPath) throws IOException { @Override public String initiateMultipartUpload(String remotePath) throws IOException { - try { - AzureUri uri = AzureUri.parse(remotePath); - BlobClient blobClient = getClient().getBlobContainerClient(uri.container()) - .getBlobClient(uri.key()); - BlockBlobClient blockBlobClient = blobClient.getBlockBlobClient(); - String leaseId = UUID.randomUUID().toString(); - String uploadId = MULTIPART_LEASE_PREFIX + leaseId; - // A zero-byte uncommitted block materializes an absent target without exposing it to - // normal listings, so Azure can fence every later block operation with a blob lease. - blockBlobClient.stageBlock(multipartBlockId(uploadId, 0), BinaryData.fromBytes(new byte[0])); - String acquiredLeaseId = createLeaseClient(blobClient, leaseId) - .acquireLease(MULTIPART_LEASE_SECONDS); - return MULTIPART_LEASE_PREFIX + acquiredLeaseId; - } catch (BlobStorageException e) { - throw new IOException("initiateMultipartUpload failed for " + remotePath - + ": " + e.getMessage(), e); - } + // Azure has no multipart session; this local UUID only namespaces the writer's block IDs. + return UUID.randomUUID().toString(); } @Override @@ -252,18 +229,10 @@ public UploadPartResult uploadPart(String remotePath, String uploadId, int partN RequestBody body) throws IOException { try { AzureUri uri = AzureUri.parse(remotePath); - BlobClient blobClient = getClient().getBlobContainerClient(uri.container()) - .getBlobClient(uri.key()); - BlockBlobClient blockBlobClient = blobClient.getBlockBlobClient(); + BlockBlobClient blockBlobClient = getClient().getBlobContainerClient(uri.container()) + .getBlobClient(uri.key()).getBlockBlobClient(); String blockId = multipartBlockId(uploadId, partNum); - String leaseId = multipartLeaseId(uploadId); - if (leaseId == null) { - blockBlobClient.stageBlock(blockId, body.content(), body.contentLength()); - } else { - renewMultipartLease(blobClient, leaseId); - blockBlobClient.stageBlockWithResponse(blockId, body.content(), body.contentLength(), - null, leaseId, null, Context.NONE); - } + blockBlobClient.stageBlock(blockId, body.content(), body.contentLength()); return new UploadPartResult(partNum, blockId); } catch (BlobStorageException e) { throw new IOException("uploadPart failed for " + remotePath + " part " + partNum @@ -289,25 +258,7 @@ public void completeMultipartUpload(String remotePath, String uploadId, blockIds.add(part.etag()); } // Put Block List is the atomic publication point and does not expose a staging blob to scans. - BlobClient blobClient = containerClient.getBlobClient(uri.key()); - BlockBlobClient blockBlobClient = blobClient.getBlockBlobClient(); - String leaseId = multipartLeaseId(uploadId); - if (leaseId == null) { - blockBlobClient.commitBlockList(blockIds); - } else { - BlobLeaseClient leaseClient = renewMultipartLease(blobClient, leaseId); - BlobRequestConditions conditions = new BlobRequestConditions().setLeaseId(leaseId); - blockBlobClient.commitBlockListWithResponse( - new BlockBlobCommitBlockListOptions(blockIds).setRequestConditions(conditions), - null, Context.NONE); - try { - leaseClient.releaseLease(); - } catch (BlobStorageException e) { - // Publication is already durable; the finite lease will expire without - // turning a successful commit into a retry that could overwrite new data. - LOG.warn("Failed to release Azure multipart lease after commit for {}", remotePath, e); - } - } + containerClient.getBlobClient(uri.key()).getBlockBlobClient().commitBlockList(blockIds); } catch (BlobStorageException e) { throw new IOException("completeMultipartUpload failed for " + remotePath + ": " + e.getMessage(), e); @@ -316,39 +267,8 @@ public void completeMultipartUpload(String remotePath, String uploadId, @Override public void abortMultipartUpload(String remotePath, String uploadId) throws IOException { - String leaseId = multipartLeaseId(uploadId); - if (leaseId == null) { - // Azure cannot selectively remove legacy uncommitted blocks without rewriting the blob. - return; - } - try { - AzureUri uri = AzureUri.parse(remotePath); - BlobClient blobClient = getClient().getBlobContainerClient(uri.container()) - .getBlobClient(uri.key()); - createLeaseClient(blobClient, leaseId).releaseLease(); - } catch (BlobStorageException e) { - throw new IOException("abortMultipartUpload failed for " + remotePath - + ": " + e.getMessage(), e); - } - } - - protected BlobLeaseClient createLeaseClient(BlobClient blobClient, String leaseId) { - return new BlobLeaseClientBuilder().blobClient(blobClient).leaseId(leaseId).buildClient(); - } - - private BlobLeaseClient renewMultipartLease(BlobClient blobClient, String leaseId) { - BlobLeaseClient leaseClient = createLeaseClient(blobClient, leaseId); - // A renewal failure loses the upload-generation fence even if the same ID is acquirable later. - leaseClient.renewLease(); - return leaseClient; - } - - private static String multipartLeaseId(String uploadId) { - if (uploadId != null && uploadId.startsWith(MULTIPART_LEASE_PREFIX) - && uploadId.length() > MULTIPART_LEASE_PREFIX.length()) { - return uploadId.substring(MULTIPART_LEASE_PREFIX.length()); - } - return null; + // Azure cannot selectively remove one writer's uncommitted blocks. They are isolated by + // upload UUID and left for the service to garbage-collect without touching published data. } /** @@ -569,23 +489,15 @@ private static String requireProperty(String value, String key, String descripti return value; } - /** - * Converts a part number to a Base64-encoded block ID using little-endian byte order, - * consistent with the existing fe-core Azure multipart upload implementation. - */ - private static String toBlockId(int partNum) { - byte[] bytes = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN).putInt(partNum).array(); - return Base64.getEncoder().encodeToString(bytes); - } - static String multipartBlockId(String uploadId, int partNum) { - int uploadNamespace = 0x811C9DC5; - for (byte value : uploadId.getBytes(StandardCharsets.UTF_8)) { - uploadNamespace = (uploadNamespace ^ (value & 0xFF)) * 0x01000193; - } - int namespacedPart = uploadNamespace + partNum; - // Match the legacy four-byte length so a retry can coexist with pre-upgrade residual blocks. - byte[] rawId = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN).putInt(namespacedPart).array(); + byte[] uploadBytes = uploadId.getBytes(StandardCharsets.UTF_8); + // Keep the full upload UUID in every block ID so independent writers cannot stage the + // same block IDs even though Azure has no per-upload multipart namespace. + byte[] rawId = ByteBuffer.allocate(uploadBytes.length + Integer.BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .put(uploadBytes) + .putInt(partNum) + .array(); return Base64.getEncoder().encodeToString(rawId); } } diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java index b7aa4bcf9f19f0..2945b141cef296 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java @@ -25,10 +25,7 @@ import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.models.BlobProperties; -import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.BlobStorageException; -import com.azure.storage.blob.options.BlockBlobCommitBlockListOptions; -import com.azure.storage.blob.specialized.BlobLeaseClient; import com.azure.storage.blob.specialized.BlockBlobClient; import com.azure.storage.common.StorageSharedKeyCredential; import org.junit.jupiter.api.Assertions; @@ -37,14 +34,15 @@ import java.io.ByteArrayInputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.time.OffsetDateTime; -import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; /** * Unit tests for the cloud extension methods added to {@link AzureObjStorage}: @@ -384,124 +382,41 @@ void deleteObjectsByKeys_attachesPerKeyExceptionsAsSuppressed() throws Exception } // ------------------------------------------------------------------ - // F20 — abortMultipartUpload safe-noop / commit-empty behaviour + // F20 — multipart upload identity and safe no-op abort behaviour // ------------------------------------------------------------------ @Test - void multipartBlockId_keepsLegacyLengthButCannotIdentifyWriter() { - Assertions.assertEquals("p3w3DA==", AzureObjStorage.multipartBlockId("upload-a", 1)); - Assertions.assertEquals("Sc7grw==", AzureObjStorage.multipartBlockId( - "doris-azure-lease-v1:09492e3d-e231-4ed9-bf84-b6fc772cda54", 1)); - Assertions.assertEquals("Sc7grw==", AzureObjStorage.multipartBlockId( - "doris-azure-lease-v1:06996d15-1c2e-4ddd-8853-43816ea84a07", 1)); - Assertions.assertEquals( - AzureObjStorage.multipartBlockId("upload-a", 1).length(), - AzureObjStorage.multipartBlockId("upload-a", 999).length()); - Assertions.assertEquals(4, - Base64.getDecoder().decode(AzureObjStorage.multipartBlockId("upload-a", 1)).length); + void multipartBlockId_embedsFullUploadUuid() { + String firstUpload = "09492e3d-e231-4ed9-bf84-b6fc772cda54"; + String secondUpload = "06996d15-1c2e-4ddd-8853-43816ea84a07"; + String firstBlock = AzureObjStorage.multipartBlockId(firstUpload, 1); + String secondBlock = AzureObjStorage.multipartBlockId(secondUpload, 1); + + Assertions.assertNotEquals(firstBlock, secondBlock); + Assertions.assertEquals(firstBlock.length(), + AzureObjStorage.multipartBlockId(firstUpload, 999).length()); + byte[] decoded = Base64.getDecoder().decode(firstBlock); + Assertions.assertEquals(firstUpload.length() + Integer.BYTES, decoded.length); + Assertions.assertArrayEquals(firstUpload.getBytes(StandardCharsets.UTF_8), + Arrays.copyOf(decoded, firstUpload.length())); + Assertions.assertArrayEquals(new byte[]{1, 0, 0, 0}, + Arrays.copyOfRange(decoded, firstUpload.length(), decoded.length)); } @Test - void initiateMultipartUpload_reservesTargetAndAcquiresLease() throws Exception { - BlockBlobClient blockClient = Mockito.mock(BlockBlobClient.class); - BlobClient blobClient = Mockito.mock(BlobClient.class); - Mockito.when(blobClient.getBlockBlobClient()).thenReturn(blockClient); - BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); - Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); + void initiateMultipartUpload_returnsLocalUuidWithoutTouchingProvider() throws Exception { BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); - Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); - BlobLeaseClient leaseClient = Mockito.mock(BlobLeaseClient.class); - Mockito.when(leaseClient.acquireLease(60)).thenReturn("lease-id"); - TestableAzureObjStorage storage = - new TestableAzureObjStorage(buildBasicProps(), serviceClient, leaseClient); + TestableAzureObjStorage storage = new TestableAzureObjStorage(buildBasicProps(), serviceClient); String uploadId = storage.initiateMultipartUpload( "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob"); - Assertions.assertEquals("doris-azure-lease-v1:lease-id", uploadId); - Mockito.verify(blockClient).stageBlock( - Mockito.anyString(), Mockito.any(com.azure.core.util.BinaryData.class)); - Mockito.verify(leaseClient).acquireLease(60); - } - - @Test - void uploadPart_renewsLeaseAndFencesStagedBlock() throws Exception { - BlockBlobClient blockClient = Mockito.mock(BlockBlobClient.class); - BlobClient blobClient = Mockito.mock(BlobClient.class); - Mockito.when(blobClient.getBlockBlobClient()).thenReturn(blockClient); - BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); - Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); - BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); - Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); - BlobLeaseClient leaseClient = Mockito.mock(BlobLeaseClient.class); - TestableAzureObjStorage storage = - new TestableAzureObjStorage(buildBasicProps(), serviceClient, leaseClient); - - storage.uploadPart("wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", - "doris-azure-lease-v1:lease-id", 1, - RequestBody.of(new ByteArrayInputStream(new byte[]{1}), 1)); - - Mockito.verify(leaseClient).renewLease(); - Mockito.verify(blockClient).stageBlockWithResponse(Mockito.anyString(), - Mockito.any(java.io.InputStream.class), Mockito.eq(1L), Mockito.isNull(), - Mockito.eq("lease-id"), Mockito.isNull(), Mockito.any()); - } - - @Test - void completeMultipartUpload_fencesCommitAndReleasesLease() throws Exception { - BlockBlobClient blockClient = Mockito.mock(BlockBlobClient.class); - BlobClient blobClient = Mockito.mock(BlobClient.class); - Mockito.when(blobClient.getBlockBlobClient()).thenReturn(blockClient); - BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); - Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); - BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); - Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); - BlobLeaseClient leaseClient = Mockito.mock(BlobLeaseClient.class); - TestableAzureObjStorage storage = - new TestableAzureObjStorage(buildBasicProps(), serviceClient, leaseClient); - - storage.completeMultipartUpload( - "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", - "doris-azure-lease-v1:lease-id", - Collections.singletonList(new UploadPartResult(1, "AQAAAA=="))); - - Mockito.verify(leaseClient).renewLease(); - org.mockito.ArgumentCaptor options = - org.mockito.ArgumentCaptor.forClass(BlockBlobCommitBlockListOptions.class); - Mockito.verify(blockClient).commitBlockListWithResponse( - options.capture(), Mockito.isNull(), Mockito.any()); - BlobRequestConditions conditions = options.getValue().getRequestConditions(); - Assertions.assertEquals("lease-id", conditions.getLeaseId()); - Mockito.verify(leaseClient).releaseLease(); - } - - @Test - void completeMultipartUpload_lostLeaseFailsBeforePublication() throws Exception { - BlockBlobClient blockClient = Mockito.mock(BlockBlobClient.class); - BlobClient blobClient = Mockito.mock(BlobClient.class); - Mockito.when(blobClient.getBlockBlobClient()).thenReturn(blockClient); - BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); - Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); - BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); - Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); - BlobLeaseClient leaseClient = Mockito.mock(BlobLeaseClient.class); - BlobStorageException lostLease = Mockito.mock(BlobStorageException.class); - Mockito.when(leaseClient.renewLease()).thenThrow(lostLease); - TestableAzureObjStorage storage = - new TestableAzureObjStorage(buildBasicProps(), serviceClient, leaseClient); - - Assertions.assertThrows(IOException.class, () -> storage.completeMultipartUpload( - "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", - "doris-azure-lease-v1:lease-id", - Collections.singletonList(new UploadPartResult(1, "AQAAAA==")))); - - Mockito.verify(blockClient, Mockito.never()).commitBlockList(Mockito.anyList()); - Mockito.verify(blockClient, Mockito.never()).commitBlockListWithResponse( - Mockito.any(BlockBlobCommitBlockListOptions.class), Mockito.isNull(), Mockito.any()); + Assertions.assertEquals(uploadId, UUID.fromString(uploadId).toString()); + Mockito.verifyNoInteractions(serviceClient); } @Test - void completeMultipartUpload_expiredLeaseFailsClosedAfterCollidingWriterStagesAndReleases() throws Exception { + void uploadPart_stagesBlockWithFullUploadUuid() throws Exception { BlockBlobClient blockClient = Mockito.mock(BlockBlobClient.class); BlobClient blobClient = Mockito.mock(BlobClient.class); Mockito.when(blobClient.getBlockBlobClient()).thenReturn(blockClient); @@ -509,85 +424,17 @@ void completeMultipartUpload_expiredLeaseFailsClosedAfterCollidingWriterStagesAn Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); - BlobLeaseClient staleLeaseClient = Mockito.mock(BlobLeaseClient.class); - BlobLeaseClient competingLeaseClient = Mockito.mock(BlobLeaseClient.class); - BlobStorageException expiredLease = Mockito.mock(BlobStorageException.class); - Mockito.when(expiredLease.getStatusCode()).thenReturn(409); - Mockito.when(staleLeaseClient.renewLease()).thenThrow(expiredLease); - TestableAzureObjStorage staleStorage = - new TestableAzureObjStorage(buildBasicProps(), serviceClient, staleLeaseClient); - TestableAzureObjStorage competingStorage = - new TestableAzureObjStorage(buildBasicProps(), serviceClient, competingLeaseClient); - String staleUpload = "doris-azure-lease-v1:09492e3d-e231-4ed9-bf84-b6fc772cda54"; - String competingUpload = "doris-azure-lease-v1:06996d15-1c2e-4ddd-8853-43816ea84a07"; - String collidingBlockId = AzureObjStorage.multipartBlockId(staleUpload, 1); - - competingStorage.uploadPart( - "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", - competingUpload, 1, RequestBody.of(new ByteArrayInputStream(new byte[]{2}), 1)); - competingStorage.abortMultipartUpload( - "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", competingUpload); - - Assertions.assertThrows(IOException.class, () -> staleStorage.completeMultipartUpload( - "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", - staleUpload, Collections.singletonList(new UploadPartResult(1, collidingBlockId)))); - - Mockito.verify(blockClient).stageBlockWithResponse(Mockito.eq(collidingBlockId), - Mockito.any(java.io.InputStream.class), Mockito.eq(1L), Mockito.isNull(), - Mockito.eq("06996d15-1c2e-4ddd-8853-43816ea84a07"), Mockito.isNull(), Mockito.any()); - Mockito.verify(competingLeaseClient).releaseLease(); - Mockito.verify(staleLeaseClient, Mockito.never()).acquireLease(Mockito.anyInt()); - Mockito.verify(blockClient, Mockito.never()).commitBlockListWithResponse( - Mockito.any(BlockBlobCommitBlockListOptions.class), Mockito.isNull(), Mockito.any()); - } - - @Test - void abortMultipartUpload_releasesLeasedSessionWithoutRewritingTarget() throws Exception { - BlobClient blobClient = Mockito.mock(BlobClient.class); - BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); - Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); - BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); - Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); - BlobLeaseClient leaseClient = Mockito.mock(BlobLeaseClient.class); - TestableAzureObjStorage storage = - new TestableAzureObjStorage(buildBasicProps(), serviceClient, leaseClient); - - storage.abortMultipartUpload( - "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", - "doris-azure-lease-v1:lease-id"); - - Mockito.verify(leaseClient).releaseLease(); - Mockito.verify(blobClient, Mockito.never()).delete(); - } - - @Test - void uploadPart_acceptsLegacyResidualBlockLength() throws Exception { - com.azure.storage.blob.specialized.BlockBlobClient blockClient = - Mockito.mock(com.azure.storage.blob.specialized.BlockBlobClient.class); - List stagedBlockIds = new ArrayList<>(Collections.singletonList("AQAAAA==")); - Mockito.doAnswer(invocation -> { - String blockId = invocation.getArgument(0); - int requiredDecodedLength = Base64.getDecoder().decode(stagedBlockIds.get(0)).length; - if (Base64.getDecoder().decode(blockId).length != requiredDecodedLength) { - throw new IllegalStateException("Azure would reject a different block ID length"); - } - stagedBlockIds.add(blockId); - return null; - }).when(blockClient).stageBlock( - Mockito.anyString(), Mockito.any(java.io.InputStream.class), Mockito.anyLong()); - BlobClient blobClient = Mockito.mock(BlobClient.class); - Mockito.when(blobClient.getBlockBlobClient()).thenReturn(blockClient); - BlobContainerClient containerClient = Mockito.mock(BlobContainerClient.class); - Mockito.when(containerClient.getBlobClient("stage/blob")).thenReturn(blobClient); - BlobServiceClient serviceClient = Mockito.mock(BlobServiceClient.class); - Mockito.when(serviceClient.getBlobContainerClient("mycontainer")).thenReturn(containerClient); TestableAzureObjStorage storage = new TestableAzureObjStorage(buildBasicProps(), serviceClient); + String uploadId = "09492e3d-e231-4ed9-bf84-b6fc772cda54"; UploadPartResult result = storage.uploadPart( - "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", - "new-upload-id", 1, RequestBody.of(new ByteArrayInputStream(new byte[]{1}), 1)); + "wasb://mycontainer@myaccount.blob.core.windows.net/stage/blob", uploadId, 1, + RequestBody.of(new ByteArrayInputStream(new byte[]{1}), 1)); - Assertions.assertEquals(Arrays.asList("AQAAAA==", result.etag()), stagedBlockIds); + String expectedBlockId = AzureObjStorage.multipartBlockId(uploadId, 1); + Assertions.assertEquals(expectedBlockId, result.etag()); + Mockito.verify(blockClient).stageBlock(Mockito.eq(expectedBlockId), + Mockito.any(java.io.InputStream.class), Mockito.eq(1L)); } @Test @@ -686,20 +533,13 @@ private Map buildBasicProps() { */ private static class TestableAzureObjStorage extends AzureObjStorage { private final BlobServiceClient mockClient; - private final BlobLeaseClient mockLeaseClient; String stubbedSasUrl = "https://stubbed-sas-url"; String lastGenerateSasContainer; String lastGenerateSasBlobKey; TestableAzureObjStorage(Map props, BlobServiceClient mockClient) { - this(props, mockClient, null); - } - - TestableAzureObjStorage(Map props, BlobServiceClient mockClient, - BlobLeaseClient mockLeaseClient) { super(props); this.mockClient = mockClient; - this.mockLeaseClient = mockLeaseClient; } @Override @@ -707,11 +547,6 @@ protected BlobServiceClient buildClient() { return mockClient; } - @Override - protected BlobLeaseClient createLeaseClient(BlobClient blobClient, String leaseId) { - return mockLeaseClient; - } - @Override protected String generateSasUrl(String endpoint, String container, String blobKey, StorageSharedKeyCredential credential, OffsetDateTime expiresOn) { From 64eba05b2dd02d5906db8be815c11ab915c45bad Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 11 Aug 2026 22:09:05 +0800 Subject: [PATCH 29/29] [fix](iceberg) Preserve EOS memory reservation through close --- be/src/exec/sink/writer/async_result_writer.cpp | 15 +++++++++------ be/src/exec/sort/sorter.cpp | 11 ++++++----- .../exec/sink/writer/async_result_writer_test.cpp | 9 +++++++-- be/test/exec/sort/full_sort_test.cpp | 14 ++++++++++++++ 4 files changed, 36 insertions(+), 13 deletions(-) diff --git a/be/src/exec/sink/writer/async_result_writer.cpp b/be/src/exec/sink/writer/async_result_writer.cpp index 84a2f2ef6c76f2..46c6cdf159cbf9 100644 --- a/be/src/exec/sink/writer/async_result_writer.cpp +++ b/be/src/exec/sink/writer/async_result_writer.cpp @@ -150,7 +150,12 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* opera } DCHECK(_dependency); - bool reservation_held_for_finish = false; + bool reservation_held_for_finalize = false; + Defer release_final_reservation {[&]() { + if (reservation_held_for_finalize) { + thread_context()->thread_mem_tracker_mgr->shrink_reserved(); + } + }}; while (_writer_status.ok()) { ThreadCpuStopWatch cpu_time_stop_watch; cpu_time_stop_watch.start(); @@ -203,8 +208,9 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* opera _return_free_block(std::move(queued.block)); } if (queued.eos) { - // Keep the final reservation through finish(), where buffered sorters are committed. - reservation_held_for_finish = true; + // Some writers finalize buffered data in close(), so the EOS reservation must outlive + // both finish() and close() instead of being released between the two callbacks. + reservation_held_for_finalize = true; _notify_block_processed(); break; } @@ -243,9 +249,6 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* opera Status st = finish(state); _writer_status.update(st); } - if (reservation_held_for_finish) { - thread_context()->thread_mem_tracker_mgr->shrink_reserved(); - } Status st = Status::OK(); { st = _writer_status.status(); } diff --git a/be/src/exec/sort/sorter.cpp b/be/src/exec/sort/sorter.cpp index 01ff069d9451cb..5f878ecd7279ea 100644 --- a/be/src/exec/sort/sorter.cpp +++ b/be/src/exec/sort/sorter.cpp @@ -250,13 +250,14 @@ SorterReserveMemory FullSorter::get_reserve_mem_size_components(RuntimeState* st std::numeric_limits::max())); reserve.retained_growth_trigger_bytes = growth_trigger_bytes; } - auto sort = new_rows > _buffered_block_size || new_block_bytes > _buffered_block_bytes; + // Iceberg close forces every nonempty pending run to sort at EOS, even when the generic + // append thresholds are not reached, so admission must cover that final allocation too. + auto sort = (eos && new_rows > 0) || new_rows > _buffered_block_size || + new_block_bytes > _buffered_block_bytes; if (sort) { - // new column is created when doing sort, reserve average size of one column - // for estimation + // sort_block keeps the source columns live while materializing a fully permuted destination. reserve.transient_workspace = - saturating_add_size(reserve.transient_workspace, - new_block_bytes / _state->unsorted_block()->columns()); + saturating_add_size(reserve.transient_workspace, new_block_bytes); // helping data structures used during sorting reserve.transient_workspace = saturating_add_size( diff --git a/be/test/exec/sink/writer/async_result_writer_test.cpp b/be/test/exec/sink/writer/async_result_writer_test.cpp index 840be91f5e0df6..18339b818de3c8 100644 --- a/be/test/exec/sink/writer/async_result_writer_test.cpp +++ b/be/test/exec/sink/writer/async_result_writer_test.cpp @@ -64,10 +64,14 @@ class RecordingAsyncWriter final : public AsyncResultWriter { return Status::OK(); } - Status close(Status) override { return Status::OK(); } + Status close(Status) override { + reservation_seen_by_close = thread_context()->thread_mem_tracker_mgr->reserved_mem(); + return Status::OK(); + } int64_t reservation_seen_by_write = 0; int64_t reservation_seen_by_finish = 0; + int64_t reservation_seen_by_close = 0; private: Status _open_status; @@ -145,7 +149,7 @@ TEST_F(AsyncResultWriterTest, TransfersQueuedReservationIntoActualWrite) { EXPECT_EQ(0, thread_context()->thread_mem_tracker_mgr->reserved_mem()); } -TEST_F(AsyncResultWriterTest, RetainsEosReservationThroughActualFinish) { +TEST_F(AsyncResultWriterTest, RetainsEosReservationThroughActualClose) { AsyncWriterHarness harness; RecordingAsyncWriter writer(harness.dependency, harness.finish_dependency, Status::OK()); harness.prepare(&writer); @@ -157,6 +161,7 @@ TEST_F(AsyncResultWriterTest, RetainsEosReservationThroughActualFinish) { harness.process(&writer); EXPECT_EQ(reservation, writer.reservation_seen_by_finish); + EXPECT_EQ(reservation, writer.reservation_seen_by_close); EXPECT_EQ(0, thread_context()->thread_mem_tracker_mgr->reserved_mem()); } diff --git a/be/test/exec/sort/full_sort_test.cpp b/be/test/exec/sort/full_sort_test.cpp index 76bd9f3cd8151c..bd8f91b03cc863 100644 --- a/be/test/exec/sort/full_sort_test.cpp +++ b/be/test/exec/sort/full_sort_test.cpp @@ -102,6 +102,20 @@ TEST_F(FullSorterTest, test_full_sorter2) { std::cout << sorter->get_reserve_mem_size(&_state, false) << std::endl; } +TEST_F(FullSorterTest, EosReservationIncludesForcedSortBelowAppendThresholds) { + sorter = FullSorter::create_unique(ordering_expr_ctxs, -1, 0, &pool, is_asc_order, nulls_first, + *row_desc, &_state, nullptr); + Block block = ColumnHelper::create_block({10, 9, 8, 7, 6, 5, 4, 3, 2, 1}); + const size_t buffered_bytes = block.bytes(); + const size_t buffered_rows = block.rows(); + ASSERT_TRUE(sorter->append_block(&block).ok()); + + const auto reservation = sorter->get_reserve_mem_size_components(&_state, true, 0, 0); + + EXPECT_GE(reservation.transient_workspace, + buffered_bytes + buffered_rows * sizeof(IColumn::Permutation::value_type)); +} + TEST_F(FullSorterTest, test_full_sorter3) { sorter = FullSorter::create_unique(ordering_expr_ctxs, 3, 3, &pool, is_asc_order, nulls_first, *row_desc, &_state, nullptr);