From b24413f7b009f91a8b9d23dbfababcb8f45ded48 Mon Sep 17 00:00:00 2001 From: Matthijs Brobbel Date: Fri, 5 Jun 2026 11:49:54 +0000 Subject: [PATCH] Decouple core library from libcudf Remove libcudf as a dependency of the cuCascade core library; RMM (librmm) becomes the only direct RAPIDS dependency. The cuDF-backed GPU/host representations and all built-in tier converters are removed and become the responsibility of an external domain layer that links libcudf and registers converters via register_converter(). Library / build: - Drop find_package(cudf) and cudf::cudf from link libs; drop find_dependency(cudf) from the installed CMake config. - pixi: swap libcudf -> librmm, rename cudf-* features to rmm-*. - Delete gpu/cpu_data_representation, host_table(_packed), the ~1800-line built-in converters, register_builtin_converters(), the bandwidth profiler, and the cuDF-coupled benchmarks / tests / test utils. - Extract a generic memory::column_metadata (opaque int32_t type tag) into include/cucascade/memory/column_metadata.hpp for the disk tier. - Move record_writer_event()/get_writer_event() to the idata_representation base (no-op / nullptr defaults). - disk_data_representation is now the only in-library concrete representation. Docs / CI: - Update README, docs/*, and CLAUDE.md to the decoupled architecture; remove docs/bandwidth-profiler.md. - Disable the CI benchmark job (its converter benchmarks moved out with the cuDF code). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 5 + CLAUDE.md | 39 +- CMakeLists.txt | 9 +- README.md | 16 +- benchmark/CMakeLists.txt | 15 +- benchmark/README.md | 43 +- benchmark/benchmark_disk_converter.cpp | 1089 -------- .../benchmark_representation_converter.cpp | 615 ----- cmake/cuCascadeConfig.cmake.in | 1 - docs/ARCHITECTURE.md | 44 +- docs/README.md | 4 +- docs/bandwidth-profiler.md | 159 -- docs/data-management.md | 187 +- docs/development-guide.md | 54 +- docs/memory-management.md | 3 +- include/cucascade/data/bandwidth_profiler.hpp | 164 -- include/cucascade/data/common.hpp | 25 + .../data/cpu_data_representation.hpp | 160 -- include/cucascade/data/data_batch.hpp | 21 +- .../data/gpu_data_representation.hpp | 211 -- .../data/representation_converter.hpp | 18 +- include/cucascade/memory/column_metadata.hpp | 58 + include/cucascade/memory/common.hpp | 6 +- include/cucascade/memory/disk_table.hpp | 10 +- include/cucascade/memory/host_table.hpp | 133 - .../cucascade/memory/host_table_packed.hpp | 53 - pixi.toml | 28 +- src/data/CMakeLists.txt | 5 +- src/data/bandwidth_profiler.cpp | 357 --- src/data/cpu_data_representation.cpp | 324 --- src/data/gpu_data_representation.cpp | 111 - src/data/representation_converter.cpp | 1834 +------------ test/CMakeLists.txt | 6 - test/data/test_bandwidth_profiler.cpp | 443 ---- test/data/test_data_batch.cpp | 160 -- test/data/test_data_representation.cpp | 2286 ----------------- test/data/test_disk_host_converters.cpp | 648 ----- test/data/test_gpu_disk_converters.cpp | 702 ----- test/data/test_representation_converter.cpp | 136 - test/utils/cudf_test_utils.cpp | 418 --- test/utils/cudf_test_utils.hpp | 37 - test/utils/mock_test_utils.hpp | 66 - 42 files changed, 259 insertions(+), 10444 deletions(-) delete mode 100644 benchmark/benchmark_disk_converter.cpp delete mode 100644 benchmark/benchmark_representation_converter.cpp delete mode 100644 docs/bandwidth-profiler.md delete mode 100644 include/cucascade/data/bandwidth_profiler.hpp delete mode 100644 include/cucascade/data/cpu_data_representation.hpp delete mode 100644 include/cucascade/data/gpu_data_representation.hpp create mode 100644 include/cucascade/memory/column_metadata.hpp delete mode 100644 include/cucascade/memory/host_table.hpp delete mode 100644 include/cucascade/memory/host_table_packed.hpp delete mode 100644 src/data/bandwidth_profiler.cpp delete mode 100644 src/data/cpu_data_representation.cpp delete mode 100644 src/data/gpu_data_representation.cpp delete mode 100644 test/data/test_bandwidth_profiler.cpp delete mode 100644 test/data/test_data_representation.cpp delete mode 100644 test/data/test_disk_host_converters.cpp delete mode 100644 test/data/test_gpu_disk_converters.cpp delete mode 100644 test/utils/cudf_test_utils.cpp delete mode 100644 test/utils/cudf_test_utils.hpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91fbb49..a3a2b8a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,11 @@ jobs: benchmark: needs: build + # Disabled: the cuDF-dependent converter/throughput benchmarks were removed with the + # cuDF-backed representations (issue #142), so benchmark/CMakeLists.txt ships no sources + # and the cucascade_benchmarks target is skipped. Re-enable once cuDF-free benchmarks + # (e.g. raw-buffer disk I/O) are added back to benchmark/CMakeLists.txt. + if: false runs-on: linux-amd64-gpu-t4-latest-1 strategy: matrix: diff --git a/CLAUDE.md b/CLAUDE.md index e9a362e..ad2be9c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,10 +32,9 @@ Performance optimization of cuCascade's disk I/O backends (GDS and kvikIO) to ap - Pixi >= 0.59 - Config: `pixi.toml` - Lockfile: `pixi.lock` (committed) -- Channels: `rapidsai-nightly`, `conda-forge` (default); `rapidsai`, `conda-forge` (cudf-stable feature) +- Channels: `rapidsai-nightly`, `conda-forge` (default / `rmm-nightly` feature); `rapidsai`, `conda-forge` (`rmm-stable` feature) ## Frameworks -- RMM (RAPIDS Memory Manager) - GPU/host memory resource abstraction; provides `rmm::mr::device_memory_resource`, `rmm::cuda_stream_view`, `rmm::out_of_memory`, `rmm::bad_alloc`; pulled in via `find_package(rmm REQUIRED CONFIG)` from libcudf installation -- libcudf 26.06 (nightly) / 26.02 (stable) - Columnar data representation; provides `cudf::table`, `cudf::column`, `cudf::type_id`, `cudf::pack`/`unpack`; pulled in via `find_package(cudf REQUIRED CONFIG)` +- RMM (RAPIDS Memory Manager) - GPU/host memory resource abstraction; provides `rmm::mr::device_memory_resource`, `rmm::cuda_stream_view`, `rmm::out_of_memory`, `rmm::bad_alloc`; the library's only direct RAPIDS dependency, pulled in via `find_package(rmm REQUIRED CONFIG)` (conda package `librmm`) - Catch2 v2.13.10 - Unit test framework; fetched via `FetchContent` in `test/CMakeLists.txt`; test executable: `cucascade_tests` - Google Benchmark v1.8.3 - Microbenchmark framework; fetched via `FetchContent` in `benchmark/CMakeLists.txt`; benchmark executable: `cucascade_benchmarks` - Ninja - Build generator (configured in `CMakePresets.json`) @@ -46,8 +45,8 @@ Performance optimization of cuCascade's disk I/O backends (GDS and kvikIO) to ap - codespell v2.4.1 - Spell checking via pre-commit (ignore list: `.codespell_words`) - Doxygen - API documentation generation; config: `Doxyfile`; output parsed by `scripts/generate_api_docs.py` ## Key Dependencies -- `libcudf` 26.06 / 26.02 - Core data representation; `cudf::table` is the GPU-tier data container; all column type handling (LIST, STRUCT, STRING, DICTIONARY32, etc.) delegates to cudf -- `RMM` (via cudf) - `rmm::mr::device_memory_resource` is the base class for all custom allocators; `rmm::cuda_stream_view` is used throughout for CUDA stream propagation +- `RMM` (conda package `librmm`) - direct dependency (no longer pulled in transitively via cudf); `rmm::mr::device_memory_resource` is the base class for all custom allocators; `rmm::cuda_stream_view` is used throughout for CUDA stream propagation +- `libcudf` - NOT a dependency of the core library; only the optional external domain layer (which links cuCascade + libcudf) uses `cudf::table` and registers concrete GPU/HOST representations and converters at runtime - `CUDA::cudart` - Direct CUDA runtime API calls (`cudaMalloc`, `cudaMemcpyAsync`, `cudaStreamSynchronize`, `cudaFree`, `cudaMallocHost`, `cudaFreeHost`) - `CUDA::nvml` - GPU topology discovery via NVML in `src/memory/topology_discovery.cpp` - `kvikio` 26.06 / 26.02 - Async disk I/O with automatic GDS/POSIX fallback; used in `src/data/kvikio_io_backend.cpp` via `kvikio::FileHandle`; linked PRIVATE via `kvikio::kvikio` @@ -93,7 +92,7 @@ Performance optimization of cuCascade's disk I/O backends (GDS and kvikIO) to ap - CUDA headers: `snake_case.cuh` (e.g., `test/memory/test_gpu_kernels.cuh`) - Source: `snake_case.cpp` for C++, `snake_case.cu` for CUDA kernels - Test files: `test_.cpp` (e.g., `test/data/test_disk_io_backend.cpp`) -- Benchmark files: `benchmark_.cpp` (e.g., `benchmark/benchmark_disk_converter.cpp`) +- Benchmark files: `benchmark_.cpp` (under `benchmark/`) - `snake_case` for all: `memory_space`, `data_batch`, `disk_data_representation` - Interface classes prefixed with `i`: `idata_representation`, `idisk_io_backend` - Config structs suffixed with `_config`: `gpu_memory_space_config`, `disk_memory_space_config` @@ -105,7 +104,7 @@ Performance optimization of cuCascade's disk I/O backends (GDS and kvikIO) to ap - `snake_case`: `get_available_memory()`, `make_reservation_or_null()` - Getters prefixed with `get_`: `get_tier()`, `get_device_id()`, `get_batch_id()` - Boolean queries prefixed with `should_`, `has_`, or `is_`: `should_downgrade_memory()` -- Factory functions prefixed with `make_` or `create_`: `make_mock_memory_space()`, `create_simple_cudf_table()` +- Factory functions prefixed with `make_` or `create_`: `make_mock_memory_space()`, `create_conversion_test_configs()` - Try-pattern methods prefixed with `try_to_`: `try_to_create_task()`, `try_to_lock_for_processing()` - Blocking wait methods prefixed with `wait_to_`: `wait_to_create_task()` - Member variables prefixed with underscore: `_id`, `_capacity`, `_mutex`, `_disk_table` @@ -139,11 +138,9 @@ Performance optimization of cuCascade's disk I/O backends (GDS and kvikIO) to ap - `-Wnull-dereference -Wdouble-promotion -Wformat=2 -Wimplicit-fallthrough` ## License Header ## Include Organization -#include "utils/cudf_test_utils.hpp" // quoted local -#include "utils/mock_test_utils.hpp" +#include "utils/mock_test_utils.hpp" // quoted local #include // cucascade #include -#include // cuDF #include // RMM #include // system with dot #include // STL @@ -157,7 +154,7 @@ Performance optimization of cuCascade's disk I/O backends (GDS and kvikIO) to ap - Nested namespaces use traditional form: `namespace cucascade { namespace test {` (not C++17 `::`) - Use anonymous `namespace { }` for file-local helpers in `.cpp` and test files - `using namespace cucascade;` at file scope in test files is acceptable -- Specific test utilities imported explicitly: `using cucascade::test::create_simple_cudf_table;` +- Specific test utilities imported explicitly: `using cucascade::test::make_mock_memory_space;` - C++17 nested namespace shorthand used in `test_memory_resources.hpp`: `namespace cucascade::test {` ## Error Handling - `CUCASCADE_CUDA_TRY(call)` — wraps CUDA runtime calls; throws `cucascade::cuda_error` on failure @@ -255,17 +252,17 @@ Performance optimization of cuCascade's disk I/O backends (GDS and kvikIO) to ap - Depends on: Config layer, memory resource layer - Used by: `memory_reservation_manager`, `idata_representation` - Purpose: Tier-specific data storage format; all derive from `idata_representation` -- Location: `include/cucascade/data/common.hpp`, `include/cucascade/data/gpu_data_representation.hpp`, `include/cucascade/data/cpu_data_representation.hpp`, `include/cucascade/data/disk_data_representation.hpp` -- Contains: `idata_representation` (abstract: `get_size_in_bytes()`, `get_uncompressed_data_size_in_bytes()`, `clone()`, templated `cast()`), four concrete types -- `disk_data_representation` owns a `disk_table_allocation` (file path + `column_metadata` vector); destructor deletes the file (RAII) -- Depends on: `memory_space`, cuDF (`cudf::table`) +- Location: `include/cucascade/data/common.hpp`, `include/cucascade/data/disk_data_representation.hpp` +- Contains: `idata_representation` (abstract: `get_size_in_bytes()`, `get_uncompressed_data_size_in_bytes()`, `clone()`, templated `cast()`, virtual `record_writer_event()` / `get_writer_event()` with no-op / nullptr defaults); `disk_data_representation` is the only in-library concrete type. GPU/HOST representations are provided by the domain layer (user code that links cuCascade + libcudf). +- `disk_data_representation` owns a `disk_table_allocation` (file path + `std::vector`); destructor deletes the file (RAII) +- Depends on: `memory_space` - Used by: `data_batch`, converter registry - Purpose: Type-pair dispatch table for converting between representation types - Location: `include/cucascade/data/representation_converter.hpp`, `src/data/representation_converter.cpp` - Contains: `representation_converter_registry`, `converter_key` (`{source_type_index, target_type_index}`), `representation_converter_fn` - Registration: `register_converter(fn)` with static_assert constraints - Lookup: `convert(source, memory_space, stream)` uses `typeid(source)` at runtime -- `register_builtin_converters()` registers GPU↔HOST and GPU↔DISK and HOST↔DISK converters; overload accepts `shared_ptr` to select I/O backend +- The registry ships empty — cuCascade ships no built-in converters. The domain layer registers tier-to-tier converters (GPU↔HOST, GPU↔DISK, HOST↔DISK) at runtime via `register_converter()` - Depends on: `idata_representation`, `idisk_io_backend` - Used by: `data_batch::convert_to()`, `data_batch::clone_to()` - Purpose: Abstract disk I/O; concrete backends selectable at runtime @@ -273,7 +270,7 @@ Performance optimization of cuCascade's disk I/O backends (GDS and kvikIO) to ap - Contains: `idisk_io_backend` (abstract with `write_device`, `read_device`, `write_host`, `read_host`, `write_device_batch`, `read_device_batch`), `io_backend_type` enum (`KVIKIO`, `GDS`, `PIPELINE`), `make_io_backend(type)` factory - `GDS` uses raw cuFile batch API; `KVIKIO` uses kvikIO with automatic GDS/POSIX fallback; `PIPELINE` uses double-buffered pinned host transfer for D2H overlap with disk writes - Depends on: kvikIO, cuFile (GDS) -- Used by: built-in disk converters registered via `register_builtin_converters()` +- Used by: domain-layer disk converters registered via `register_converter()` - Purpose: Lifecycle management; substription-count reference counting, read-only and mutable locking - Location: `include/cucascade/data/data_batch.hpp`, `src/data/data_batch.cpp` - Contains: `data_batch` (owns `unique_ptr`), `batch_state` enum, `data_batch_processing_handle` (RAII, holds `weak_ptr`), `idata_batch_probe` interface, `lock_for_processing_result` @@ -302,7 +299,7 @@ Performance optimization of cuCascade's disk I/O backends (GDS and kvikIO) to ap - Purpose: Uniform interface for tier-specific storage formats - Location: `include/cucascade/data/common.hpp` - Pattern: Abstract base with `get_size_in_bytes()`, `get_uncompressed_data_size_in_bytes()`, `clone()`, and templated `cast()` (requires `std::derived_from`) -- Concrete types: `gpu_table_representation` (`include/cucascade/data/gpu_data_representation.hpp`), `host_data_representation`, `host_data_packed_representation` (`include/cucascade/data/cpu_data_representation.hpp`), `disk_data_representation` (`include/cucascade/data/disk_data_representation.hpp`) +- In-library concrete type: `disk_data_representation` (`include/cucascade/data/disk_data_representation.hpp`). GPU/HOST concrete representations are provided by the domain layer (user code linking cuCascade + libcudf) and registered via `register_converter()`. - Purpose: Owns a single tier+device memory budget and its allocator - Location: `include/cucascade/memory/memory_space.hpp` - Pattern: Non-copyable/non-movable; variant-based allocator dispatch; exposes `make_reservation_or_null()`, `should_downgrade_memory()`, `get_disk_mount_path()` @@ -313,9 +310,9 @@ Performance optimization of cuCascade's disk I/O backends (GDS and kvikIO) to ap - Purpose: Unit of data movement; read-only and mutable locking - Location: `include/cucascade/data/data_batch.hpp` - Pattern: Owns `unique_ptr`; `data_batch_processing_handle` holds `weak_ptr` so handle doesn't keep batch alive; `idata_batch_probe` for external state observation callbacks -- Purpose: On-disk file descriptor and binary format for a serialized cuDF table -- Location: `include/cucascade/memory/disk_table.hpp`, `include/cucascade/data/disk_file_format.hpp` -- Pattern: File starts with 32-byte `disk_file_header` (magic `0x43554353`, version, num_columns, metadata_size, data_offset); column metadata serialized depth-first; column data aligned to 4096-byte boundaries for GDS DMA +- Purpose: On-disk file descriptor and binary format for a serialized columnar table (described by domain-agnostic `memory::column_metadata`, whose `type_id` is an opaque `int32_t` tag cuCascade never interprets) +- Location: `include/cucascade/memory/disk_table.hpp`, `include/cucascade/data/disk_file_format.hpp`, `include/cucascade/memory/column_metadata.hpp` +- Pattern: The file holds only raw column buffers (per column, depth-first: null mask then data), each aligned to `DISK_FILE_ALIGNMENT` (4096 bytes) for direct DMA. No header or metadata is serialized to disk — the per-column `memory::column_metadata` is kept in-memory in `disk_table_allocation`, so a disk file is only meaningful together with its allocation. I/O is dispatched via the disk `memory_space`'s `get_io_backend()` - Purpose: Abstraction over GDS, kvikIO, and pipeline I/O strategies - Location: `include/cucascade/data/disk_io_backend.hpp` - Pattern: Interface with `write_device`/`read_device`/`write_host`/`read_host` and batch variants; `make_io_backend(io_backend_type)` factory diff --git a/CMakeLists.txt b/CMakeLists.txt index db2a888..7bac701 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,12 +47,9 @@ list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules") find_package(CUDAToolkit REQUIRED) find_package(Threads REQUIRED) -# Find RMM from libcudf installation +# Find RMM (RAPIDS Memory Manager) -- core to cuCascade's memory abstractions find_package(rmm REQUIRED CONFIG) -# Find cudf for data representation support -find_package(cudf REQUIRED CONFIG) - # Find numa (provided by numactl-devel or libnuma-dev depending on the package # manager) find_library(NUMA_LIB numa REQUIRED) @@ -130,8 +127,8 @@ set(CUCASCADE_PUBLIC_INCLUDE_DIRS $ $) -set(CUCASCADE_PUBLIC_LINK_LIBS rmm::rmm cudf::cudf CUDA::cudart_static - Threads::Threads ${NUMA_LIB}) +set(CUCASCADE_PUBLIC_LINK_LIBS rmm::rmm CUDA::cudart_static Threads::Threads + ${NUMA_LIB}) # Set include directories for the object library target_include_directories(cucascade_objects diff --git a/README.md b/README.md index 52a6498..db185e8 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ A high-performance GPU memory management library for data-intensive applications - **Memory Reservation System**: Avoid oversubscribing your GPU by making reservations and using allocators that respect reservations - **Hardware Topology Discovery**: Automatic detection of NUMA regions and GPU-CPU affinity for optimal memory placement - **Stream-Aware Tracking**: Per-stream memory usage tracking and reservation enforcement -- **cuDF Integration**: Native support for GPU DataFrames with batch processing capabilities and spilling to Host or Disk +- **Pluggable Data Representations**: A tiered memory and data-movement substrate; concrete dataframe representations and tier-to-tier converters (e.g. a cuDF-based domain layer) are supplied by external user code and registered at runtime, with batch processing and spilling to Host or Disk - **Pluggable Policies**: Control what happens when you OOM, try to allocate more than a reservation, how you pick what data to spill, by creating policies that plug into the system. # Getting Started @@ -46,7 +46,7 @@ pixi run benchmarks - **Compiler**: C++20 compatible compiler - **Build Tools**: CMake 4.1+, Ninja - **GPU/Drivers**: CUDA 13+, compatible NVIDIA driver -- **Dependencies**: libcudf 25.10+ +- **Dependencies**: RMM (librmm) # Usage @@ -152,8 +152,10 @@ cuCascade/ │ │ ├── data_batch.hpp # Batch processing for data │ │ ├── data_repository.hpp # Data storage abstraction │ │ ├── data_repository_manager.hpp -│ │ ├── cpu_data_representation.hpp -│ │ └── gpu_data_representation.hpp +│ │ ├── representation_converter.hpp # Converter registry (ships empty) +│ │ ├── disk_data_representation.hpp # On-disk representation +│ │ ├── disk_file_format.hpp # On-disk binary format +│ │ └── disk_io_backend.hpp # GDS / kvikIO / pipeline backends │ └── memory/ # Memory management headers │ ├── common.hpp # Tier enum, memory_space_id, utilities │ ├── memory_reservation_manager.hpp # Central reservation coordinator @@ -174,9 +176,8 @@ cuCascade/ ├── test/ │ ├── data/ # Data module tests │ ├── memory/ # Memory module tests -│ └── utils/ # Test utilities (cuDF helpers) +│ └── utils/ # Test utilities (mock helpers) ├── benchmark/ # Performance benchmarks -│ ├── benchmark_representation_converter.cpp # Converter benchmarks │ └── README.md # Benchmark documentation ├── cmake/ # CMake configuration modules ├── CMakeLists.txt # Main CMake configuration @@ -186,7 +187,8 @@ cuCascade/ # References -- [RAPIDS cuDF](https://github.com/rapidsai/cudf) - GPU DataFrame library +- [RAPIDS RMM](https://github.com/rapidsai/rmm) - RAPIDS Memory Manager (core dependency) +- [RAPIDS cuDF](https://github.com/rapidsai/cudf) - GPU DataFrame library (used by the optional domain layer) - [Pixi](https://pixi.sh/) - Package management tool # License diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 4921ee2..fb75c22 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -15,6 +15,16 @@ # the License. # ============================================================================= +# Collect all benchmark sources. The previous converter/profiler benchmarks were +# cudf-dependent and moved out with the cudf representations (issue #142). Add +# new cudf-free benchmarks (e.g. raw-buffer disk I/O) here. +set(BENCHMARK_SOURCES) + +if(NOT BENCHMARK_SOURCES) + message(STATUS "cuCascade: no benchmark sources; skipping benchmark target") + return() +endif() + # Fetch Google Benchmark include(FetchContent) FetchContent_Declare( @@ -36,11 +46,6 @@ set(BENCHMARK_ENABLE_INSTALL FetchContent_MakeAvailable(benchmark) -# Collect all benchmark sources -set(BENCHMARK_SOURCES - ${CMAKE_CURRENT_SOURCE_DIR}/benchmark_representation_converter.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/benchmark_disk_converter.cpp) - # Create benchmark executable add_executable(cucascade_benchmarks ${BENCHMARK_SOURCES}) diff --git a/benchmark/README.md b/benchmark/README.md index 0a7d04d..6fa387d 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -4,10 +4,11 @@ This directory contains performance benchmarks for the cuCascade library using G ## Building the Benchmarks -The benchmarks are built by default when you configure the project. To disable them: +The benchmarks are built by default when you configure the project (when the suite +contains sources — see [Available Benchmarks](#available-benchmarks)). To disable them: ```bash -cmake -DBUILD_BENCHMARKS=OFF .. +cmake -DCUCASCADE_BUILD_BENCHMARKS=OFF .. ``` To build the project with benchmarks enabled: @@ -30,42 +31,20 @@ After building, you can run all benchmarks: ### Running Specific Benchmarks -To run only specific benchmarks, use filters: +To run a subset of benchmarks, use a filter pattern: ```bash -# Run only conversion benchmarks -./benchmark/cucascade_benchmarks --benchmark_filter=Convert - -# Run only throughput benchmarks -./benchmark/cucascade_benchmarks --benchmark_filter=Throughput +./benchmark/cucascade_benchmarks --benchmark_filter= ``` ## Available Benchmarks -### Representation Converter Benchmarks - -Located in `benchmark_representation_converter.cpp`: - -1. **BM_ConvertGpuToHost**: Benchmarks GPU to HOST memory conversion with varying data sizes - - Tests with different data sizes - - Tests with different column counts - - Reports throughput in bytes/second - -2. **BM_ConvertHostToGpu**: Benchmarks HOST to GPU memory conversion - - Similar parameterization as GPU to HOST - - Measures upload performance - -5. **BM_GpuToHostThroughput**: Focuses on memory bandwidth for GPU→HOST transfers - - Tests with data sizes - - Reports throughput in GiB/s - -6. **BM_HostToGpuThroughput**: Focuses on memory bandwidth for HOST→GPU transfers - - Similar parameterization as GPU to HOST throughput - - Reports throughput in GiB/s - -All benchmarks measure different thread counts. -The multi-threading is explicitly implemented instead of relying on googlebenchmark's built-in threading functionality, -because that resulted in improper results. +> **None currently.** The previous representation-converter and throughput benchmarks +> (`BM_ConvertGpuToHost`, `BM_ConvertHostToGpu`, `BM_GpuToHostThroughput`, +> `BM_HostToGpuThroughput`) were cuDF-dependent and were removed together with the +> cuDF-backed data representations (issue #142). `benchmark/CMakeLists.txt` currently +> sets an empty `BENCHMARK_SOURCES` and returns early, so the `cucascade_benchmarks` +> target is skipped until new cuDF-free benchmarks (e.g. raw-buffer disk I/O) are added. ## Adding New Benchmarks diff --git a/benchmark/benchmark_disk_converter.cpp b/benchmark/benchmark_disk_converter.cpp deleted file mode 100644 index e4cc4c2..0000000 --- a/benchmark/benchmark_disk_converter.cpp +++ /dev/null @@ -1,1089 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "cucascade/data/cpu_data_representation.hpp" -#include "cucascade/data/disk_data_representation.hpp" -#include "cucascade/data/gpu_data_representation.hpp" -#include "cucascade/data/representation_converter.hpp" -#include "cucascade/memory/config.hpp" -#include "cucascade/memory/memory_reservation_manager.hpp" - -#include - -#include -#include -#include - -#include - -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace { - -using namespace cucascade; -using namespace cucascade::memory; - -constexpr uint64_t KiB = 1024ULL; -constexpr uint64_t MiB = 1024ULL * KiB; -constexpr uint64_t GiB = 1024ULL * MiB; - -/// Fraction of GPU memory that must remain free after allocating the benchmark table. -/// Leave headroom for cudf internals, RMM pool overhead, and converter temporaries. -constexpr double GPU_MEMORY_SAFETY_FACTOR = 0.75; - -/** - * @brief Skip the benchmark if the requested data size exceeds available GPU memory. - * - * Queries free GPU memory via cudaMemGetInfo and skips (with a message) if the - * benchmark's data allocation would exceed the safety threshold. - * - * @return true if the benchmark should be skipped. - */ -bool skip_if_oom(benchmark::State& state, int64_t total_bytes) -{ - std::size_t free_bytes = 0; - std::size_t total_gpu = 0; - cudaMemGetInfo(&free_bytes, &total_gpu); - - auto available = static_cast(static_cast(free_bytes) * GPU_MEMORY_SAFETY_FACTOR); - if (total_bytes > available) { - state.SkipWithMessage( - "OOM: need " + std::to_string(total_bytes / static_cast(MiB)) + " MiB but only " + - std::to_string(available / static_cast(MiB)) + " MiB available (GPU has " + - std::to_string(free_bytes / static_cast(MiB)) + " MiB free, " + - std::to_string(static_cast(GPU_MEMORY_SAFETY_FACTOR * 100)) + "% safety)"); - return true; - } - return false; -} - -// Hardware baselines from gdsio on /dev/nvme1n1 (/mnt/disk_2, ext4) -// Measured: gdsio -D /mnt/disk_2/gdsio_test -d 0 -w 4 -s 4G -x 0 -I 1 -// Write: 6.73 GiB/s, Read: 13.35 GiB/s -constexpr double GDSIO_WRITE_GIBS = 6.73; -constexpr double GDSIO_READ_GIBS = 13.35; -constexpr double GIBS_TO_BYTES = 1024.0 * 1024.0 * 1024.0; - -/** - * @brief Flush NVMe write cache and drop OS page cache for a file. - * - * Call after writes and before reads in benchmark loops to ensure each iteration - * measures real disk I/O without OS cache interference. - */ -void drop_os_cache(const std::string& path) -{ - int fd = ::open(path.c_str(), O_RDONLY); - if (fd >= 0) { - ::fdatasync(fd); - ::posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); - ::close(fd); - } -} - -/** - * @brief Create a converter registry with all built-in converters. - */ -std::unique_ptr make_benchmark_registry() -{ - auto registry = std::make_unique(); - register_builtin_converters(*registry); - return registry; -} - -// For non-NUMA systems, this should be -1, causing the allocator to use cudaHostAlloc instead of -// cudaHostRegister -constexpr int hostDevId = -1; - -// Global shared memory manager - managed via setup/teardown functions -static std::shared_ptr g_shared_memory_manager; - -/** - * @brief Create memory manager configs for benchmarking (GPU, HOST, and DISK). - */ -std::vector create_benchmark_configs() -{ - std::vector configs; - - // Query actual GPU memory and use 90% of it (leave room for driver/OS) - std::size_t free_bytes = 0; - std::size_t total_gpu = 0; - cudaMemGetInfo(&free_bytes, &total_gpu); - auto gpu_capacity = static_cast(static_cast(free_bytes) * 0.9); - - gpu_memory_space_config gpu_config; - gpu_config.device_id = 0; - gpu_config.memory_capacity = gpu_capacity; - gpu_config.mr_factory_fn = make_default_allocator_for_tier(Tier::GPU); - configs.emplace_back(gpu_config); - - host_memory_space_config host_config; - host_config.numa_id = hostDevId; - host_config.memory_capacity = gpu_capacity; // match GPU capacity - host_config.mr_factory_fn = make_default_allocator_for_tier(Tier::HOST); - host_config.initial_number_pools = 16; - configs.emplace_back(host_config); - - disk_memory_space_config disk_config; - disk_config.disk_id = 0; - disk_config.memory_capacity = 32 * GiB; - disk_config.mount_paths = "/tmp"; - configs.emplace_back(disk_config); - - return configs; -} - -/** - * @brief Get shared memory reservation manager. - */ -std::shared_ptr get_shared_memory_manager() -{ - return g_shared_memory_manager; -} - -/** - * @brief Setup function called before benchmarks. - */ -void DoSetup([[maybe_unused]] const benchmark::State& state) -{ - if (!g_shared_memory_manager) { - g_shared_memory_manager = - std::make_shared(create_benchmark_configs()); - } -} - -/** - * @brief Teardown function called after benchmarks. - * - * Intentionally does NOT reset the memory manager — disk benchmarks create - * disk_data_representation objects whose files live under the disk memory space - * mount path. Resetting the manager between benchmarks would invalidate - * disk files still referenced by later benchmark iterations. - */ -void DoTeardown([[maybe_unused]] const benchmark::State& state) -{ - // no-op: manager persists across benchmark functions -} - -/** - * @brief Create a cuDF table from bytes and columns specification (int64/float64 only). - * - * @param total_bytes Total size in bytes - * @param num_columns Number of columns (alternates between INT64 and FLOAT64) - * @return cudf::table The generated table - */ -cudf::table create_benchmark_table_from_bytes(int64_t total_bytes, int num_columns) -{ - constexpr size_t bytes_per_element = 8; - - int64_t total_elements = total_bytes / static_cast(bytes_per_element); - int64_t num_rows = total_elements / static_cast(num_columns); - - if (num_rows < 1) num_rows = 1; - - std::vector> columns; - - for (int i = 0; i < num_columns; ++i) { - cudf::data_type dtype; - uint8_t fill_value; - - if (i % 2 == 0) { - dtype = cudf::data_type{cudf::type_id::INT64}; - fill_value = 0x22; - } else { - dtype = cudf::data_type{cudf::type_id::FLOAT64}; - fill_value = 0x44; - } - - auto col = - cudf::make_numeric_column(dtype, static_cast(num_rows), cudf::mask_state::UNALLOCATED); - - if (num_rows > 0) { - auto view = col->mutable_view(); - auto type_size = cudf::size_of(dtype); - auto bytes = static_cast(num_rows) * (type_size); - CUCASCADE_CUDA_TRY(cudaMemset(const_cast(view.head()), fill_value, bytes)); - } - - columns.push_back(std::move(col)); - } - - return cudf::table(std::move(columns)); -} - -/** - * @brief Create a cuDF table with STRING columns for benchmarking. - * - * Each column has repeated 8-char strings ("benchstr") to approximate the requested total bytes. - * - * @param total_bytes Total approximate data size in bytes - * @param num_columns Number of string columns - * @return cudf::table The generated table - */ -cudf::table create_string_benchmark_table(int64_t total_bytes, int num_columns) -{ - constexpr int chars_per_string = 8; - int64_t bytes_per_column = total_bytes / static_cast(num_columns); - int64_t num_strings = - bytes_per_column / static_cast(chars_per_string + static_cast(sizeof(int32_t))); - if (num_strings < 1) num_strings = 1; - int64_t total_chars = num_strings * chars_per_string; - - rmm::cuda_stream stream; - - std::vector> columns; - - for (int c = 0; c < num_columns; ++c) { - // Build offsets: 0, 8, 16, 24, ... - std::vector host_offsets(static_cast(num_strings) + 1); - for (int64_t i = 0; i <= num_strings; ++i) { - host_offsets[static_cast(i)] = static_cast(i * chars_per_string); - } - rmm::device_buffer offsets_buf( - host_offsets.data(), host_offsets.size() * sizeof(int32_t), stream.view()); - auto offsets_col = - std::make_unique(cudf::data_type{cudf::type_id::INT32}, - static_cast(host_offsets.size()), - std::move(offsets_buf), - rmm::device_buffer{}, - 0); - - // Build chars: repeated "benchstr" - std::vector host_chars(static_cast(total_chars)); - const char pattern[] = "benchstr"; - for (int64_t i = 0; i < total_chars; ++i) { - host_chars[static_cast(i)] = - static_cast(pattern[static_cast(i % chars_per_string)]); - } - rmm::device_buffer chars_buf(host_chars.data(), host_chars.size(), stream.view()); - - stream.synchronize(); - - auto str_col = cudf::make_strings_column(static_cast(num_strings), - std::move(offsets_col), - std::move(chars_buf), - 0, - rmm::device_buffer{}); - columns.push_back(std::move(str_col)); - } - - return cudf::table(std::move(columns)); -} - -/** - * @brief Create a cuDF table with LIST columns for benchmarking. - * - * Each list element is a fixed-length list of 4 INT32s. - * - * @param total_bytes Total approximate data size in bytes - * @param num_columns Number of list columns - * @return cudf::table The generated table - */ -cudf::table create_list_benchmark_table(int64_t total_bytes, int num_columns) -{ - constexpr int elements_per_list = 4; - constexpr int64_t bytes_per_list_element_row = elements_per_list * sizeof(int32_t); - int64_t bytes_per_column = total_bytes / static_cast(num_columns); - int64_t num_lists = bytes_per_column / bytes_per_list_element_row; - if (num_lists < 1) num_lists = 1; - int64_t num_values = num_lists * elements_per_list; - - rmm::cuda_stream stream; - - std::vector> columns; - - for (int c = 0; c < num_columns; ++c) { - // Build offsets: 0, 4, 8, 12, ... - std::vector host_offsets(static_cast(num_lists) + 1); - for (int64_t i = 0; i <= num_lists; ++i) { - host_offsets[static_cast(i)] = static_cast(i * elements_per_list); - } - rmm::device_buffer offsets_buf( - host_offsets.data(), host_offsets.size() * sizeof(int32_t), stream.view()); - auto offsets_col = - std::make_unique(cudf::data_type{cudf::type_id::INT32}, - static_cast(host_offsets.size()), - std::move(offsets_buf), - rmm::device_buffer{}, - 0); - - // Build values column - auto values_col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - static_cast(num_values), - cudf::mask_state::UNALLOCATED, - stream.view()); - if (num_values > 0) { - auto view = values_col->mutable_view(); - CUCASCADE_CUDA_TRY(cudaMemset( - const_cast(view.head()), 0x33, static_cast(num_values) * sizeof(int32_t))); - } - - stream.synchronize(); - - auto list_col = cudf::make_lists_column(static_cast(num_lists), - std::move(offsets_col), - std::move(values_col), - 0, - {}); - columns.push_back(std::move(list_col)); - } - - return cudf::table(std::move(columns)); -} - -/** - * @brief Create a cuDF table with STRUCT columns for benchmarking. - * - * Each struct row = 16 bytes (8 + 8). - * - * @param total_bytes Total approximate data size in bytes - * @param num_columns Number of struct columns - * @return cudf::table The generated table - */ -cudf::table create_struct_benchmark_table(int64_t total_bytes, int num_columns) -{ - constexpr int64_t bytes_per_struct_row = 16; - int64_t bytes_per_column = total_bytes / static_cast(num_columns); - int64_t num_rows = bytes_per_column / bytes_per_struct_row; - if (num_rows < 1) num_rows = 1; - - rmm::cuda_stream stream; - - std::vector> columns; - - for (int c = 0; c < num_columns; ++c) { - auto field0 = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT64}, - static_cast(num_rows), - cudf::mask_state::UNALLOCATED, - stream.view()); - auto field1 = cudf::make_numeric_column(cudf::data_type{cudf::type_id::FLOAT64}, - static_cast(num_rows), - cudf::mask_state::UNALLOCATED, - stream.view()); - stream.synchronize(); - - std::vector> children; - children.push_back(std::move(field0)); - children.push_back(std::move(field1)); - auto struct_col = - cudf::make_structs_column(static_cast(num_rows), std::move(children), 0, {}); - - columns.push_back(std::move(struct_col)); - } - - return cudf::table(std::move(columns)); -} - -// ============================================================================= -// BENCH-01 / BENCH-04: GPU <-> Disk Size-Sweep Benchmarks (Numeric Columns) -// ============================================================================= - -/** - * @brief Benchmark GPU to Disk conversion with varying data sizes. - * @param state.range(0) Total bytes - * @param state.range(1) Number of columns - */ -void BM_ConvertGpuToDisk(benchmark::State& state) -{ - int64_t total_bytes = state.range(0); - if (skip_if_oom(state, total_bytes)) return; - int num_columns = static_cast(state.range(1)); - - auto mgr = get_shared_memory_manager(); - - auto registry = make_benchmark_registry(); - - const memory_space* gpu_space = mgr->get_memory_space(Tier::GPU, 0); - const memory_space* disk_space = mgr->get_memory_space(Tier::DISK, 0); - - rmm::cuda_stream stream; - - // Create GPU representation - auto table = create_benchmark_table_from_bytes(total_bytes, num_columns); - auto gpu_rep = - std::make_unique(std::make_unique(std::move(table)), - *const_cast(gpu_space), - stream.view()); - - // Warmup - auto warmup_table = create_benchmark_table_from_bytes(1 * KiB, 2); - auto warmup_repr = std::make_unique( - std::make_unique(std::move(warmup_table)), - *const_cast(gpu_space), - stream.view()); - auto warmup_result = - registry->convert(*warmup_repr, disk_space, stream.view()); - stream.synchronize(); - - size_t bytes_transferred = gpu_rep->get_size_in_bytes(); - - for ([[maybe_unused]] auto _ : state) { - // No cache eviction needed — pipeline uses O_DIRECT - auto disk_result = - registry->convert(*gpu_rep, disk_space, stream.view()); - stream.synchronize(); - drop_os_cache(disk_result->get_disk_table().file_path); - } - - state.SetBytesProcessed(static_cast(state.iterations()) * - static_cast(bytes_transferred)); - state.counters["columns"] = static_cast(num_columns); - state.counters["bytes"] = static_cast(bytes_transferred); - state.counters["gdsio_write_GiBs"] = GDSIO_WRITE_GIBS; - state.counters["gdsio_read_GiBs"] = GDSIO_READ_GIBS; -} - -/** - * @brief Benchmark Disk to GPU conversion with varying data sizes. - * @param state.range(0) Total bytes - * @param state.range(1) Number of columns - */ -void BM_ConvertDiskToGpu(benchmark::State& state) -{ - int64_t total_bytes = state.range(0); - if (skip_if_oom(state, total_bytes)) return; - int num_columns = static_cast(state.range(1)); - - auto mgr = get_shared_memory_manager(); - - auto registry = make_benchmark_registry(); - - const memory_space* gpu_space = mgr->get_memory_space(Tier::GPU, 0); - const memory_space* disk_space = mgr->get_memory_space(Tier::DISK, 0); - - rmm::cuda_stream stream; - - // Create GPU representation then convert to disk once - auto table = create_benchmark_table_from_bytes(total_bytes, num_columns); - auto gpu_rep = - std::make_unique(std::make_unique(std::move(table)), - *const_cast(gpu_space), - stream.view()); - - auto disk_rep = registry->convert(*gpu_rep, disk_space, stream.view()); - stream.synchronize(); - - size_t bytes_transferred = disk_rep->get_size_in_bytes(); - - for ([[maybe_unused]] auto _ : state) { - // No cache eviction needed — pipeline uses O_DIRECT - auto gpu_result = - registry->convert(*disk_rep, gpu_space, stream.view()); - stream.synchronize(); - } - - state.SetBytesProcessed(static_cast(state.iterations()) * - static_cast(bytes_transferred)); - state.counters["columns"] = static_cast(num_columns); - state.counters["bytes"] = static_cast(bytes_transferred); - state.counters["gdsio_write_GiBs"] = GDSIO_WRITE_GIBS; - state.counters["gdsio_read_GiBs"] = GDSIO_READ_GIBS; -} - -/** - * @brief Benchmark Host to Disk conversion with varying data sizes. - * @param state.range(0) Total bytes - * @param state.range(1) Number of columns - */ -void BM_ConvertHostToDisk(benchmark::State& state) -{ - int64_t total_bytes = state.range(0); - if (skip_if_oom(state, total_bytes)) return; - int num_columns = static_cast(state.range(1)); - - auto mgr = get_shared_memory_manager(); - - auto registry = std::make_unique(); - register_builtin_converters(*registry); - - const memory_space* gpu_space = mgr->get_memory_space(Tier::GPU, 0); - const memory_space* host_space = mgr->get_memory_space(Tier::HOST, hostDevId); - const memory_space* disk_space = mgr->get_memory_space(Tier::DISK, 0); - - rmm::cuda_stream stream; - - // Create GPU table, convert to host_data first - auto table = create_benchmark_table_from_bytes(total_bytes, num_columns); - auto gpu_rep = - std::make_unique(std::make_unique(std::move(table)), - *const_cast(gpu_space), - stream.view()); - - auto host_rep = registry->convert(*gpu_rep, host_space, stream.view()); - stream.synchronize(); - - size_t bytes_transferred = host_rep->get_size_in_bytes(); - - for ([[maybe_unused]] auto _ : state) { - // No cache eviction needed — pipeline uses O_DIRECT - auto disk_result = - registry->convert(*host_rep, disk_space, stream.view()); - stream.synchronize(); - drop_os_cache(disk_result->get_disk_table().file_path); - } - - state.SetBytesProcessed(static_cast(state.iterations()) * - static_cast(bytes_transferred)); - state.counters["columns"] = static_cast(num_columns); - state.counters["bytes"] = static_cast(bytes_transferred); - state.counters["gdsio_write_GiBs"] = GDSIO_WRITE_GIBS; - state.counters["gdsio_read_GiBs"] = GDSIO_READ_GIBS; -} - -/** - * @brief Benchmark Disk to Host conversion with varying data sizes. - * @param state.range(0) Total bytes - * @param state.range(1) Number of columns - */ -void BM_ConvertDiskToHost(benchmark::State& state) -{ - int64_t total_bytes = state.range(0); - if (skip_if_oom(state, total_bytes)) return; - int num_columns = static_cast(state.range(1)); - - auto mgr = get_shared_memory_manager(); - - auto registry = std::make_unique(); - register_builtin_converters(*registry); - - const memory_space* gpu_space = mgr->get_memory_space(Tier::GPU, 0); - const memory_space* host_space = mgr->get_memory_space(Tier::HOST, hostDevId); - const memory_space* disk_space = mgr->get_memory_space(Tier::DISK, 0); - - rmm::cuda_stream stream; - - // Create GPU table, convert to host, then to disk - auto table = create_benchmark_table_from_bytes(total_bytes, num_columns); - auto gpu_rep = - std::make_unique(std::make_unique(std::move(table)), - *const_cast(gpu_space), - stream.view()); - - auto host_rep = registry->convert(*gpu_rep, host_space, stream.view()); - stream.synchronize(); - - auto disk_rep = registry->convert(*host_rep, disk_space, stream.view()); - stream.synchronize(); - - size_t bytes_transferred = disk_rep->get_size_in_bytes(); - - for ([[maybe_unused]] auto _ : state) { - // No cache eviction needed — pipeline uses O_DIRECT - auto host_result = - registry->convert(*disk_rep, host_space, stream.view()); - stream.synchronize(); - } - - state.SetBytesProcessed(static_cast(state.iterations()) * - static_cast(bytes_transferred)); - state.counters["columns"] = static_cast(num_columns); - state.counters["bytes"] = static_cast(bytes_transferred); - state.counters["gdsio_write_GiBs"] = GDSIO_WRITE_GIBS; - state.counters["gdsio_read_GiBs"] = GDSIO_READ_GIBS; -} - -// ============================================================================= -// BENCH-02: Column Type Sweep Benchmarks -// ============================================================================= - -/** - * @brief Benchmark GPU to Disk conversion with STRING columns. - * @param state.range(0) Total bytes - * @param state.range(1) Number of columns - */ -void BM_ConvertGpuToDiskStringColumns(benchmark::State& state) -{ - int64_t total_bytes = state.range(0); - if (skip_if_oom(state, total_bytes)) return; - int num_columns = static_cast(state.range(1)); - - auto mgr = get_shared_memory_manager(); - - auto registry = make_benchmark_registry(); - - const memory_space* gpu_space = mgr->get_memory_space(Tier::GPU, 0); - const memory_space* disk_space = mgr->get_memory_space(Tier::DISK, 0); - - rmm::cuda_stream stream; - - auto table = create_string_benchmark_table(total_bytes, num_columns); - auto gpu_rep = - std::make_unique(std::make_unique(std::move(table)), - *const_cast(gpu_space), - stream.view()); - - size_t bytes_transferred = gpu_rep->get_size_in_bytes(); - - for ([[maybe_unused]] auto _ : state) { - // No cache eviction needed — pipeline uses O_DIRECT - auto disk_result = - registry->convert(*gpu_rep, disk_space, stream.view()); - stream.synchronize(); - drop_os_cache(disk_result->get_disk_table().file_path); - } - - state.SetBytesProcessed(static_cast(state.iterations()) * - static_cast(bytes_transferred)); - state.counters["columns"] = static_cast(num_columns); - state.counters["bytes"] = static_cast(bytes_transferred); - state.counters["gdsio_write_GiBs"] = GDSIO_WRITE_GIBS; - state.counters["gdsio_read_GiBs"] = GDSIO_READ_GIBS; -} - -/** - * @brief Benchmark GPU to Disk conversion with LIST columns. - * @param state.range(0) Total bytes - * @param state.range(1) Number of columns - */ -void BM_ConvertGpuToDiskListColumns(benchmark::State& state) -{ - int64_t total_bytes = state.range(0); - if (skip_if_oom(state, total_bytes)) return; - int num_columns = static_cast(state.range(1)); - - auto mgr = get_shared_memory_manager(); - - auto registry = make_benchmark_registry(); - - const memory_space* gpu_space = mgr->get_memory_space(Tier::GPU, 0); - const memory_space* disk_space = mgr->get_memory_space(Tier::DISK, 0); - - rmm::cuda_stream stream; - - auto table = create_list_benchmark_table(total_bytes, num_columns); - auto gpu_rep = - std::make_unique(std::make_unique(std::move(table)), - *const_cast(gpu_space), - stream.view()); - - size_t bytes_transferred = gpu_rep->get_size_in_bytes(); - - for ([[maybe_unused]] auto _ : state) { - // No cache eviction needed — pipeline uses O_DIRECT - auto disk_result = - registry->convert(*gpu_rep, disk_space, stream.view()); - stream.synchronize(); - drop_os_cache(disk_result->get_disk_table().file_path); - } - - state.SetBytesProcessed(static_cast(state.iterations()) * - static_cast(bytes_transferred)); - state.counters["columns"] = static_cast(num_columns); - state.counters["bytes"] = static_cast(bytes_transferred); - state.counters["gdsio_write_GiBs"] = GDSIO_WRITE_GIBS; - state.counters["gdsio_read_GiBs"] = GDSIO_READ_GIBS; -} - -/** - * @brief Benchmark GPU to Disk conversion with STRUCT columns. - * @param state.range(0) Total bytes - * @param state.range(1) Number of columns - */ -void BM_ConvertGpuToDiskStructColumns(benchmark::State& state) -{ - int64_t total_bytes = state.range(0); - if (skip_if_oom(state, total_bytes)) return; - int num_columns = static_cast(state.range(1)); - - auto mgr = get_shared_memory_manager(); - - auto registry = make_benchmark_registry(); - - const memory_space* gpu_space = mgr->get_memory_space(Tier::GPU, 0); - const memory_space* disk_space = mgr->get_memory_space(Tier::DISK, 0); - - rmm::cuda_stream stream; - - auto table = create_struct_benchmark_table(total_bytes, num_columns); - auto gpu_rep = - std::make_unique(std::make_unique(std::move(table)), - *const_cast(gpu_space), - stream.view()); - - size_t bytes_transferred = gpu_rep->get_size_in_bytes(); - - for ([[maybe_unused]] auto _ : state) { - // No cache eviction needed — pipeline uses O_DIRECT - auto disk_result = - registry->convert(*gpu_rep, disk_space, stream.view()); - stream.synchronize(); - drop_os_cache(disk_result->get_disk_table().file_path); - } - - state.SetBytesProcessed(static_cast(state.iterations()) * - static_cast(bytes_transferred)); - state.counters["columns"] = static_cast(num_columns); - state.counters["bytes"] = static_cast(bytes_transferred); - state.counters["gdsio_write_GiBs"] = GDSIO_WRITE_GIBS; - state.counters["gdsio_read_GiBs"] = GDSIO_READ_GIBS; -} - -// ============================================================================= -// Benchmark Registrations -// ============================================================================= - -// BENCH-01 / BENCH-04: Size sweep with numeric columns (GPU <-> Disk) -BENCHMARK(BM_ConvertGpuToDisk) - ->Setup(DoSetup) - ->Teardown(DoTeardown) - ->Args({1 * MiB, 4}) - ->Args({64 * MiB, 4}) - ->Args({512 * MiB, 4}) - ->Args({4 * GiB, 4}) - ->Unit(benchmark::kMillisecond) - ->UseRealTime(); - -BENCHMARK(BM_ConvertDiskToGpu) - ->Setup(DoSetup) - ->Teardown(DoTeardown) - ->Args({1 * MiB, 4}) - ->Args({64 * MiB, 4}) - ->Args({512 * MiB, 4}) - ->Args({4 * GiB, 4}) - ->Unit(benchmark::kMillisecond) - ->UseRealTime(); - -// BENCH-01 / BENCH-04: Size sweep with numeric columns (Host <-> Disk) -BENCHMARK(BM_ConvertHostToDisk) - ->Setup(DoSetup) - ->Teardown(DoTeardown) - ->Args({1 * MiB, 4}) - ->Args({64 * MiB, 4}) - ->Args({512 * MiB, 4}) - ->Args({4 * GiB, 4}) - ->Unit(benchmark::kMillisecond) - ->UseRealTime(); - -BENCHMARK(BM_ConvertDiskToHost) - ->Setup(DoSetup) - ->Teardown(DoTeardown) - ->Args({1 * MiB, 4}) - ->Args({64 * MiB, 4}) - ->Args({512 * MiB, 4}) - ->Args({4 * GiB, 4}) - ->Unit(benchmark::kMillisecond) - ->UseRealTime(); - -// BENCH-02: Column type sweep (64 MiB fixed size) -BENCHMARK(BM_ConvertGpuToDiskStringColumns) - ->Setup(DoSetup) - ->Teardown(DoTeardown) - ->Args({64 * MiB, 2}) - ->Args({64 * MiB, 4}) - ->Args({64 * MiB, 8}) - ->Unit(benchmark::kMillisecond) - ->UseRealTime(); - -BENCHMARK(BM_ConvertGpuToDiskListColumns) - ->Setup(DoSetup) - ->Teardown(DoTeardown) - ->Args({64 * MiB, 2}) - ->Args({64 * MiB, 4}) - ->Args({64 * MiB, 8}) - ->Unit(benchmark::kMillisecond) - ->UseRealTime(); - -BENCHMARK(BM_ConvertGpuToDiskStructColumns) - ->Setup(DoSetup) - ->Teardown(DoTeardown) - ->Args({64 * MiB, 2}) - ->Args({64 * MiB, 4}) - ->Args({64 * MiB, 8}) - ->Unit(benchmark::kMillisecond) - ->UseRealTime(); - -// BENCH-03: Pipeline backend (double-buffered pinned host) -void BM_ConvertGpuToDiskPipeline(benchmark::State& state) -{ - int64_t total_bytes = state.range(0); - if (skip_if_oom(state, total_bytes)) return; - int num_columns = static_cast(state.range(1)); - - auto mgr = get_shared_memory_manager(); - - auto registry = std::make_unique(); - register_builtin_converters(*registry); - - const memory_space* gpu_space = mgr->get_memory_space(Tier::GPU, 0); - const memory_space* disk_space = mgr->get_memory_space(Tier::DISK, 0); - - rmm::cuda_stream stream; - - auto table = create_benchmark_table_from_bytes(total_bytes, num_columns); - auto gpu_rep = - std::make_unique(std::make_unique(std::move(table)), - *const_cast(gpu_space), - stream.view()); - - size_t bytes_transferred = gpu_rep->get_size_in_bytes(); - - for ([[maybe_unused]] auto _ : state) { - // No cache eviction needed — pipeline uses O_DIRECT - auto disk_result = - registry->convert(*gpu_rep, disk_space, stream.view()); - stream.synchronize(); - drop_os_cache(disk_result->get_disk_table().file_path); - } - - state.SetBytesProcessed(static_cast(state.iterations()) * - static_cast(bytes_transferred)); - state.counters["columns"] = static_cast(num_columns); - state.counters["bytes"] = static_cast(bytes_transferred); - state.counters["gdsio_write_GiBs"] = GDSIO_WRITE_GIBS; - state.counters["gdsio_read_GiBs"] = GDSIO_READ_GIBS; - state.counters["backend"] = 2; // 2 = Pipeline -} - -BENCHMARK(BM_ConvertGpuToDiskPipeline) - ->Setup(DoSetup) - ->Teardown(DoTeardown) - ->Args({1 * MiB, 4}) - ->Args({64 * MiB, 4}) - ->Args({512 * MiB, 4}) - ->Args({4 * GiB, 4}) - ->Unit(benchmark::kMillisecond) - ->UseRealTime(); - -// ============================================================================= -// BENCH-04: Read benchmark (Disk -> GPU) using Pipeline backend -// ============================================================================= - -/** - * @brief Helper: write a GPU table to disk using the pipeline backend, - * returning the disk representation for read benchmarks. - */ -std::unique_ptr write_table_to_disk(cudf::table&& table, - const memory_space* gpu_space, - const memory_space* disk_space, - rmm::cuda_stream_view stream) -{ - auto registry = make_benchmark_registry(); - - auto gpu_rep = std::make_unique( - std::make_unique(std::move(table)), *const_cast(gpu_space), stream); - auto disk_rep = registry->convert(*gpu_rep, disk_space, stream); - stream.synchronize(); - return disk_rep; -} - -void BM_ConvertDiskToGpuPipeline(benchmark::State& state) -{ - int64_t total_bytes = state.range(0); - if (skip_if_oom(state, total_bytes)) return; - int num_columns = static_cast(state.range(1)); - - auto mgr = get_shared_memory_manager(); - - const memory_space* gpu_space = mgr->get_memory_space(Tier::GPU, 0); - const memory_space* disk_space = mgr->get_memory_space(Tier::DISK, 0); - - rmm::cuda_stream stream; - - auto disk_rep = write_table_to_disk(create_benchmark_table_from_bytes(total_bytes, num_columns), - gpu_space, - disk_space, - stream.view()); - - auto registry = make_benchmark_registry(); - - size_t bytes_transferred = disk_rep->get_size_in_bytes(); - const auto& disk_file = disk_rep->get_disk_table().file_path; - - for ([[maybe_unused]] auto _ : state) { - drop_os_cache(disk_file); - auto gpu_result = - registry->convert(*disk_rep, gpu_space, stream.view()); - stream.synchronize(); - } - - state.SetBytesProcessed(static_cast(state.iterations()) * - static_cast(bytes_transferred)); - state.counters["columns"] = static_cast(num_columns); - state.counters["bytes"] = static_cast(bytes_transferred); - state.counters["gdsio_write_GiBs"] = GDSIO_WRITE_GIBS; - state.counters["gdsio_read_GiBs"] = GDSIO_READ_GIBS; -} - -BENCHMARK(BM_ConvertDiskToGpuPipeline) - ->Setup(DoSetup) - ->Teardown(DoTeardown) - ->Args({1 * MiB, 4}) - ->Args({64 * MiB, 4}) - ->Args({512 * MiB, 4}) - ->Args({4 * GiB, 4}) - ->Unit(benchmark::kMillisecond) - ->UseRealTime(); - -// ============================================================================= -// BENCH-05: Column type sweep — write + read benchmarks (64 MiB) -// ============================================================================= - -/** - * @brief Helper: write benchmark for a given table creation function. - */ -template -void pipeline_write_benchmark(benchmark::State& state, TableFactory table_factory) -{ - int64_t total_bytes = state.range(0); - if (skip_if_oom(state, total_bytes)) return; - int num_columns = static_cast(state.range(1)); - - auto mgr = get_shared_memory_manager(); - - auto registry = make_benchmark_registry(); - - const memory_space* gpu_space = mgr->get_memory_space(Tier::GPU, 0); - const memory_space* disk_space = mgr->get_memory_space(Tier::DISK, 0); - - rmm::cuda_stream stream; - - auto table = table_factory(total_bytes, num_columns); - auto gpu_rep = - std::make_unique(std::make_unique(std::move(table)), - *const_cast(gpu_space), - stream.view()); - - size_t bytes_transferred = gpu_rep->get_size_in_bytes(); - - for ([[maybe_unused]] auto _ : state) { - auto disk_result = - registry->convert(*gpu_rep, disk_space, stream.view()); - stream.synchronize(); - drop_os_cache(disk_result->get_disk_table().file_path); - } - - state.SetBytesProcessed(static_cast(state.iterations()) * - static_cast(bytes_transferred)); - state.counters["columns"] = static_cast(num_columns); - state.counters["bytes"] = static_cast(bytes_transferred); - state.counters["gdsio_write_GiBs"] = GDSIO_WRITE_GIBS; - state.counters["gdsio_read_GiBs"] = GDSIO_READ_GIBS; -} - -/** - * @brief Helper: read benchmark for a given table creation function. - */ -template -void pipeline_read_benchmark(benchmark::State& state, TableFactory table_factory) -{ - int64_t total_bytes = state.range(0); - if (skip_if_oom(state, total_bytes)) return; - int num_columns = static_cast(state.range(1)); - - auto mgr = get_shared_memory_manager(); - - const memory_space* gpu_space = mgr->get_memory_space(Tier::GPU, 0); - const memory_space* disk_space = mgr->get_memory_space(Tier::DISK, 0); - - rmm::cuda_stream stream; - - auto disk_rep = write_table_to_disk( - table_factory(total_bytes, num_columns), gpu_space, disk_space, stream.view()); - - auto registry = make_benchmark_registry(); - - size_t bytes_transferred = disk_rep->get_size_in_bytes(); - const auto& disk_file = disk_rep->get_disk_table().file_path; - - for ([[maybe_unused]] auto _ : state) { - drop_os_cache(disk_file); - auto gpu_result = - registry->convert(*disk_rep, gpu_space, stream.view()); - stream.synchronize(); - } - - state.SetBytesProcessed(static_cast(state.iterations()) * - static_cast(bytes_transferred)); - state.counters["columns"] = static_cast(num_columns); - state.counters["bytes"] = static_cast(bytes_transferred); - state.counters["gdsio_write_GiBs"] = GDSIO_WRITE_GIBS; - state.counters["gdsio_read_GiBs"] = GDSIO_READ_GIBS; -} - -// --- Numeric write/read --- -void BM_WriteNumericPipeline(benchmark::State& s) -{ - pipeline_write_benchmark(s, create_benchmark_table_from_bytes); -} -void BM_ReadNumericPipeline(benchmark::State& s) -{ - pipeline_read_benchmark(s, create_benchmark_table_from_bytes); -} - -// --- String write/read --- -void BM_WriteStringPipeline(benchmark::State& s) -{ - pipeline_write_benchmark(s, create_string_benchmark_table); -} -void BM_ReadStringPipeline(benchmark::State& s) -{ - pipeline_read_benchmark(s, create_string_benchmark_table); -} - -// --- List write/read --- -void BM_WriteListPipeline(benchmark::State& s) -{ - pipeline_write_benchmark(s, create_list_benchmark_table); -} -void BM_ReadListPipeline(benchmark::State& s) -{ - pipeline_read_benchmark(s, create_list_benchmark_table); -} - -// --- Struct write/read --- -void BM_WriteStructPipeline(benchmark::State& s) -{ - pipeline_write_benchmark(s, create_struct_benchmark_table); -} -void BM_ReadStructPipeline(benchmark::State& s) -{ - pipeline_read_benchmark(s, create_struct_benchmark_table); -} - -// Size sweep args for type comparison -#define DISK_BACKEND_ARGS \ - ->Setup(DoSetup) \ - ->Teardown(DoTeardown) \ - ->Args({64 * MiB, 4}) \ - ->Args({512 * MiB, 4}) \ - ->Args({4 * GiB, 4}) \ - ->Unit(benchmark::kMillisecond) \ - ->UseRealTime() - -BENCHMARK(BM_WriteNumericPipeline) DISK_BACKEND_ARGS; -BENCHMARK(BM_ReadNumericPipeline) DISK_BACKEND_ARGS; - -BENCHMARK(BM_WriteStringPipeline) DISK_BACKEND_ARGS; -BENCHMARK(BM_ReadStringPipeline) DISK_BACKEND_ARGS; - -BENCHMARK(BM_WriteListPipeline) DISK_BACKEND_ARGS; -BENCHMARK(BM_ReadListPipeline) DISK_BACKEND_ARGS; - -BENCHMARK(BM_WriteStructPipeline) DISK_BACKEND_ARGS; -BENCHMARK(BM_ReadStructPipeline) DISK_BACKEND_ARGS; - -} // namespace diff --git a/benchmark/benchmark_representation_converter.cpp b/benchmark/benchmark_representation_converter.cpp deleted file mode 100644 index 57d22c3..0000000 --- a/benchmark/benchmark_representation_converter.cpp +++ /dev/null @@ -1,615 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "cucascade/data/cpu_data_representation.hpp" -#include "cucascade/data/gpu_data_representation.hpp" -#include "cucascade/data/representation_converter.hpp" -#include "cucascade/memory/memory_reservation_manager.hpp" - -#include - -#include -#include -#include - -#include - -#include - -#include - -#include -#include -#include -#include - -namespace { - -using namespace cucascade; -using namespace cucascade::memory; - -constexpr uint64_t KiB = 1024ULL; -constexpr uint64_t MiB = 1024ULL * KiB; -constexpr uint64_t GiB = 1024ULL * MiB; - -// For non-NUMA systems, this should be -1, causing the allocator to use cudaHostAlloc instead of -// cudaHostRegister -constexpr int hostDevId = -1; - -// Global shared memory manager - managed via setup/teardown functions -static std::shared_ptr g_shared_memory_manager; - -/** - * @brief Create memory manager configs for benchmarking (one GPU and one HOST). - */ -std::vector create_benchmark_configs() -{ - std::vector configs; - // Large memory limits for benchmarking - gpu_memory_space_config gpu_config; - gpu_config.device_id = 0; - gpu_config.memory_capacity = 8 * GiB; - gpu_config.mr_factory_fn = make_default_allocator_for_tier(Tier::GPU); - configs.emplace_back(gpu_config); - - host_memory_space_config host_config; - host_config.numa_id = hostDevId; - host_config.memory_capacity = 16 * GiB; - host_config.mr_factory_fn = make_default_allocator_for_tier(Tier::HOST); - // Pre-allocate enough pinned blocks for the largest benchmark case (512 MiB × 4 threads = 2 GiB) - // so that expand_pool() is never called during timed iterations. - // Each pool = pool_size (128) × block_size (1 MiB) = 128 MiB; 16 pools = 2 GiB. - host_config.initial_number_pools = 16; - configs.emplace_back(host_config); - return configs; -} - -/** - * @brief Get shared memory reservation manager. - */ -std::shared_ptr get_shared_memory_manager() -{ - return g_shared_memory_manager; -} - -/** - * @brief Setup function called before benchmarks. - */ -void DoSetup([[maybe_unused]] const benchmark::State& state) -{ - if (!g_shared_memory_manager) { - g_shared_memory_manager = - std::make_shared(create_benchmark_configs()); - } -} - -/** - * @brief Teardown function called after benchmarks. - */ -void DoTeardown([[maybe_unused]] const benchmark::State& state) -{ - // Reset the shared memory manager after benchmark - g_shared_memory_manager.reset(); -} - -/** - * @brief Create a cuDF table from bytes and columns specification (int64/float64 only). - * - * @param total_bytes Total size in bytes - * @param num_columns Number of columns (alternates between INT64 and FLOAT64) - * @return cudf::table The generated table - */ -cudf::table create_benchmark_table_from_bytes(int64_t total_bytes, int num_columns) -{ - // Both INT64 and FLOAT64 are 8 bytes - constexpr size_t bytes_per_element = 8; - - // Calculate number of rows - int64_t total_elements = total_bytes / static_cast(bytes_per_element); - int64_t num_rows = total_elements / static_cast(num_columns); - - // Ensure at least 1 row - if (num_rows < 1) num_rows = 1; - - std::vector> columns; - - for (int i = 0; i < num_columns; ++i) { - cudf::data_type dtype; - uint8_t fill_value; - - // Alternate between INT64 and FLOAT64 - if (i % 2 == 0) { - dtype = cudf::data_type{cudf::type_id::INT64}; - fill_value = 0x22; - } else { - dtype = cudf::data_type{cudf::type_id::FLOAT64}; - fill_value = 0x44; - } - - auto col = - cudf::make_numeric_column(dtype, static_cast(num_rows), cudf::mask_state::UNALLOCATED); - - if (num_rows > 0) { - auto view = col->mutable_view(); - auto type_size = cudf::size_of(dtype); - auto bytes = static_cast(num_rows) * (type_size); - CUCASCADE_CUDA_TRY(cudaMemset(const_cast(view.head()), fill_value, bytes)); - } - - columns.push_back(std::move(col)); - } - - return cudf::table(std::move(columns)); -} - -// ============================================================================= -// GPU <-> HOST Conversion Benchmarks -// ============================================================================= - -/** - * @brief Benchmark GPU to HOST conversion with varying data sizes. - * @param state.range(0) Total bytes - * @param state.range(1) Number of columns - * @param state.range(2) Manual thread count - */ -void BM_ConvertGpuToHost(benchmark::State& state) -{ - int64_t total_bytes = state.range(0); - int num_columns = static_cast(state.range(1)); - uint64_t thread_count = static_cast(state.range(2)); - - // Use shared memory manager across all threads - auto mgr = get_shared_memory_manager(); - - auto registry = std::make_unique(); - register_builtin_converters(*registry); - - const memory_space* gpu_space = mgr->get_memory_space(Tier::GPU, 0); - const memory_space* host_space = mgr->get_memory_space(Tier::HOST, hostDevId); - - // Create separate table and representation for each thread BEFORE warmup - std::vector> thread_gpu_reprs; - thread_gpu_reprs.reserve(thread_count); - std::vector streams(thread_count); - - for (uint64_t t = 0; t < thread_count; ++t) { - auto table = create_benchmark_table_from_bytes(total_bytes, num_columns); - thread_gpu_reprs.push_back( - std::make_unique(std::make_unique(std::move(table)), - *const_cast(gpu_space), - rmm::cuda_stream_view{})); - } - - // Warm-up - rmm::cuda_stream warmup_stream; - auto warmup_table = create_benchmark_table_from_bytes(1 * KiB, 2); - auto warmup_repr = std::make_unique( - std::make_unique(std::move(warmup_table)), - *const_cast(gpu_space), - warmup_stream.view()); - auto warmup_result = - registry->convert(*warmup_repr, host_space, warmup_stream); - warmup_stream.synchronize(); - - size_t bytes_transferred = thread_gpu_reprs[0]->get_size_in_bytes() * thread_count; - - // Benchmark loop - for (auto _ : state) { - std::vector threads; - threads.reserve(thread_count); - - // Create threads - for (uint64_t t = 0; t < thread_count; ++t) { - threads.emplace_back([&, t]() { - auto host_result = registry->convert( - *thread_gpu_reprs[t], host_space, streams[t]); - streams[t].synchronize(); - }); - } - - // Join all threads - for (auto& thread : threads) { - thread.join(); - } - } - - // Update counters in main thread after loop - state.SetBytesProcessed(static_cast(state.iterations()) * - static_cast(bytes_transferred)); - state.counters["columns"] = static_cast(num_columns); - state.counters["bytes"] = static_cast(bytes_transferred); - state.counters["num_threads"] = static_cast(thread_count); -} - -/** - * @brief Benchmark HOST to GPU conversion with varying data sizes. - * @param state.range(0) Total bytes - * @param state.range(1) Number of columns - * @param state.range(2) Manual thread count - */ -void BM_ConvertHostToGpu(benchmark::State& state) -{ - int64_t total_bytes = state.range(0); - int num_columns = static_cast(state.range(1)); - uint64_t thread_count = static_cast(state.range(2)); - - // Use shared memory manager across all threads - auto mgr = get_shared_memory_manager(); - - auto registry = std::make_unique(); - register_builtin_converters(*registry); - - const memory_space* gpu_space = mgr->get_memory_space(Tier::GPU, 0); - const memory_space* host_space = mgr->get_memory_space(Tier::HOST, hostDevId); - - // Create separate table and representation for each thread BEFORE warmup - std::vector> thread_host_reprs; - thread_host_reprs.reserve(thread_count); - std::vector streams(thread_count); - - rmm::cuda_stream setup_stream; - for (uint64_t t = 0; t < thread_count; ++t) { - auto table = create_benchmark_table_from_bytes(total_bytes, num_columns); - auto gpu_repr_temp = - std::make_unique(std::make_unique(std::move(table)), - *const_cast(gpu_space), - setup_stream.view()); - auto host_repr = - registry->convert(*gpu_repr_temp, host_space, setup_stream); - setup_stream.synchronize(); - thread_host_reprs.push_back(std::move(host_repr)); - } - - // Warm-up - rmm::cuda_stream warmup_stream; - auto warmup_result = - registry->convert(*thread_host_reprs[0], gpu_space, warmup_stream); - warmup_stream.synchronize(); - - size_t bytes_transferred = thread_host_reprs[0]->get_size_in_bytes() * thread_count; - - // Benchmark loop - for (auto _ : state) { - std::vector threads; - threads.reserve(thread_count); - - // Create threads - for (uint64_t t = 0; t < thread_count; ++t) { - threads.emplace_back([&, t]() { - auto gpu_result = - registry->convert(*thread_host_reprs[t], gpu_space, streams[t]); - streams[t].synchronize(); - }); - } - - // Join all threads - for (auto& thread : threads) { - thread.join(); - } - } - - // Update counters in main thread after loop - state.SetBytesProcessed(static_cast(state.iterations()) * - static_cast(bytes_transferred)); - state.counters["columns"] = static_cast(num_columns); - state.counters["bytes"] = static_cast(bytes_transferred); - state.counters["num_threads"] = static_cast(thread_count); -} - -// ============================================================================= -// GPU <-> HOST_FAST Conversion Benchmarks (fast path: no cudf::pack) -// ============================================================================= - -/** - * @brief Benchmark GPU to HOST_FAST conversion (fast path, no cudf::pack). - * @param state.range(0) Total bytes - * @param state.range(1) Number of columns - * @param state.range(2) Manual thread count - */ -void BM_ConvertGpuToHostFast(benchmark::State& state) -{ - int64_t total_bytes = state.range(0); - int num_columns = static_cast(state.range(1)); - uint64_t thread_count = static_cast(state.range(2)); - - auto mgr = get_shared_memory_manager(); - - auto registry = std::make_unique(); - register_builtin_converters(*registry); - - const memory_space* gpu_space = mgr->get_memory_space(Tier::GPU, 0); - const memory_space* host_space = mgr->get_memory_space(Tier::HOST, hostDevId); - - std::vector> thread_gpu_reprs; - thread_gpu_reprs.reserve(thread_count); - std::vector streams(thread_count); - - for (uint64_t t = 0; t < thread_count; ++t) { - auto table = create_benchmark_table_from_bytes(total_bytes, num_columns); - thread_gpu_reprs.push_back( - std::make_unique(std::make_unique(std::move(table)), - *const_cast(gpu_space), - rmm::cuda_stream_view{})); - } - - // Warm-up: small transfer to prime CUDA driver - rmm::cuda_stream warmup_stream; - auto warmup_table = create_benchmark_table_from_bytes(1 * KiB, 2); - auto warmup_repr = std::make_unique( - std::make_unique(std::move(warmup_table)), - *const_cast(gpu_space), - warmup_stream.view()); - auto warmup_result = - registry->convert(*warmup_repr, host_space, warmup_stream); - warmup_stream.synchronize(); - - size_t bytes_transferred = thread_gpu_reprs[0]->get_size_in_bytes() * thread_count; - - for (auto _ : state) { - std::vector threads; - threads.reserve(thread_count); - - for (uint64_t t = 0; t < thread_count; ++t) { - threads.emplace_back([&, t]() { - auto host_result = - registry->convert(*thread_gpu_reprs[t], host_space, streams[t]); - streams[t].synchronize(); - }); - } - - for (auto& thread : threads) { - thread.join(); - } - } - - state.SetBytesProcessed(static_cast(state.iterations()) * - static_cast(bytes_transferred)); - state.counters["columns"] = static_cast(num_columns); - state.counters["bytes"] = static_cast(bytes_transferred); - state.counters["num_threads"] = static_cast(thread_count); -} - -/** - * @brief Benchmark HOST_FAST to GPU conversion (fast path, no cudf::unpack). - * @param state.range(0) Total bytes - * @param state.range(1) Number of columns - * @param state.range(2) Manual thread count - */ -void BM_ConvertHostFastToGpu(benchmark::State& state) -{ - int64_t total_bytes = state.range(0); - int num_columns = static_cast(state.range(1)); - uint64_t thread_count = static_cast(state.range(2)); - - auto mgr = get_shared_memory_manager(); - - auto registry = std::make_unique(); - register_builtin_converters(*registry); - - const memory_space* gpu_space = mgr->get_memory_space(Tier::GPU, 0); - const memory_space* host_space = mgr->get_memory_space(Tier::HOST, hostDevId); - - std::vector> thread_host_reprs; - thread_host_reprs.reserve(thread_count); - std::vector streams(thread_count); - - rmm::cuda_stream setup_stream; - for (uint64_t t = 0; t < thread_count; ++t) { - auto table = create_benchmark_table_from_bytes(total_bytes, num_columns); - auto gpu_repr_temp = - std::make_unique(std::make_unique(std::move(table)), - *const_cast(gpu_space), - setup_stream.view()); - auto host_repr = - registry->convert(*gpu_repr_temp, host_space, setup_stream); - setup_stream.synchronize(); - thread_host_reprs.push_back(std::move(host_repr)); - } - - // Warm-up - rmm::cuda_stream warmup_stream; - auto warmup_result = - registry->convert(*thread_host_reprs[0], gpu_space, warmup_stream); - warmup_stream.synchronize(); - - size_t bytes_transferred = thread_host_reprs[0]->get_size_in_bytes() * thread_count; - - for (auto _ : state) { - std::vector threads; - threads.reserve(thread_count); - - for (uint64_t t = 0; t < thread_count; ++t) { - threads.emplace_back([&, t]() { - auto gpu_result = - registry->convert(*thread_host_reprs[t], gpu_space, streams[t]); - streams[t].synchronize(); - }); - } - - for (auto& thread : threads) { - thread.join(); - } - } - - state.SetBytesProcessed(static_cast(state.iterations()) * - static_cast(bytes_transferred)); - state.counters["columns"] = static_cast(num_columns); - state.counters["bytes"] = static_cast(bytes_transferred); - state.counters["num_threads"] = static_cast(thread_count); -} - -// ============================================================================= -// Memory Throughput Benchmarks -// ============================================================================= - -/** - * @brief Benchmark GPU to HOST memory bandwidth. - * @param state.range(0) Total bytes - * @param state.range(1) Manual thread count - */ -void BM_GpuToHostThroughput(benchmark::State& state) -{ - uint64_t total_bytes = static_cast(state.range(0)); - uint64_t thread_count = static_cast(state.range(1)); - - // Create separate buffers and streams for each thread BEFORE benchmark loop - std::vector d_buffers(thread_count); - std::vector h_buffers(thread_count); - std::vector streams(thread_count); - - for (uint64_t t = 0; t < thread_count; ++t) { - CUCASCADE_CUDA_TRY(cudaMalloc(&d_buffers[t], total_bytes)); - CUCASCADE_CUDA_TRY(cudaMallocHost(&h_buffers[t], total_bytes)); - CUCASCADE_CUDA_TRY(cudaMemset(d_buffers[t], 0x42, total_bytes)); - } - - uint64_t total_bytes_transferred = total_bytes * thread_count; - - // Benchmark loop - for (auto _ : state) { - std::vector threads; - threads.reserve(thread_count); - - // Create threads - for (uint64_t t = 0; t < thread_count; ++t) { - threads.emplace_back([&, t]() { - CUCASCADE_CUDA_TRY(cudaMemcpyAsync( - h_buffers[t], d_buffers[t], total_bytes, cudaMemcpyDeviceToHost, streams[t].value())); - streams[t].synchronize(); - }); - } - - // Join all threads - for (auto& thread : threads) { - thread.join(); - } - } - - // Cleanup buffers - for (uint64_t t = 0; t < thread_count; ++t) { - CUCASCADE_CUDA_TRY(cudaFree(d_buffers[t])); - CUCASCADE_CUDA_TRY(cudaFreeHost(h_buffers[t])); - } - - // Update counters in main thread after loop - state.SetBytesProcessed(static_cast(state.iterations()) * - static_cast(total_bytes_transferred)); - state.counters["MB"] = static_cast(total_bytes_transferred) / (1024.0 * 1024.0); - state.counters["num_threads"] = static_cast(thread_count); -} - -/** - * @brief Benchmark HOST to GPU memory bandwidth. - * @param state.range(0) Total bytes - * @param state.range(1) Manual thread count - */ -void BM_HostToGpuThroughput(benchmark::State& state) -{ - uint64_t total_bytes = static_cast(state.range(0)); - uint64_t thread_count = static_cast(state.range(1)); - - // Create separate buffers and streams for each thread BEFORE benchmark loop - std::vector d_buffers(thread_count); - std::vector h_buffers(thread_count); - std::vector streams(thread_count); - - for (uint64_t t = 0; t < thread_count; ++t) { - CUCASCADE_CUDA_TRY(cudaMalloc(&d_buffers[t], total_bytes)); - CUCASCADE_CUDA_TRY(cudaMallocHost(&h_buffers[t], total_bytes)); - std::memset(h_buffers[t], 0x42, total_bytes); - } - - uint64_t total_bytes_transferred = total_bytes * thread_count; - - // Benchmark loop - for (auto _ : state) { - std::vector threads; - threads.reserve(thread_count); - - // Create threads - for (uint64_t t = 0; t < thread_count; ++t) { - threads.emplace_back([&, t]() { - CUCASCADE_CUDA_TRY(cudaMemcpyAsync( - d_buffers[t], h_buffers[t], total_bytes, cudaMemcpyHostToDevice, streams[t].value())); - streams[t].synchronize(); - }); - } - - // Join all threads - for (auto& thread : threads) { - thread.join(); - } - } - - // Cleanup buffers - for (uint64_t t = 0; t < thread_count; ++t) { - CUCASCADE_CUDA_TRY(cudaFree(d_buffers[t])); - CUCASCADE_CUDA_TRY(cudaFreeHost(h_buffers[t])); - } - - // Update counters in main thread after loop - state.SetBytesProcessed(static_cast(state.iterations()) * - static_cast(total_bytes_transferred)); - state.counters["MB"] = static_cast(total_bytes_transferred) / (1024.0 * 1024.0); - state.counters["num_threads"] = static_cast(thread_count); -} - -BENCHMARK(BM_ConvertGpuToHost) - ->Setup(DoSetup) - ->Teardown(DoTeardown) - ->RangeMultiplier(4) - ->Ranges({{64 * KiB, 512 * MiB}, {2, 8}, {1, 4}}) - ->Unit(benchmark::kMillisecond) - ->UseRealTime(); - -BENCHMARK(BM_ConvertHostToGpu) - ->Setup(DoSetup) - ->Teardown(DoTeardown) - ->RangeMultiplier(4) - ->Ranges({{64 * KiB, 512 * MiB}, {2, 8}, {1, 4}}) - ->Unit(benchmark::kMillisecond) - ->UseRealTime(); - -BENCHMARK(BM_ConvertGpuToHostFast) - ->Setup(DoSetup) - ->Teardown(DoTeardown) - ->RangeMultiplier(4) - ->Ranges({{64 * KiB, 512 * MiB}, {2, 8}, {1, 4}}) - ->Unit(benchmark::kMillisecond) - ->UseRealTime(); - -BENCHMARK(BM_ConvertHostFastToGpu) - ->Setup(DoSetup) - ->Teardown(DoTeardown) - ->RangeMultiplier(4) - ->Ranges({{64 * KiB, 512 * MiB}, {2, 8}, {1, 4}}) - ->Unit(benchmark::kMillisecond) - ->UseRealTime(); - -BENCHMARK(BM_GpuToHostThroughput) - ->RangeMultiplier(4) - ->Ranges({{64 * KiB, 512 * MiB}, {1, 4}}) - ->Unit(benchmark::kMillisecond) - ->UseRealTime(); - -BENCHMARK(BM_HostToGpuThroughput) - ->RangeMultiplier(4) - ->Ranges({{64 * KiB, 512 * MiB}, {1, 4}}) - ->Unit(benchmark::kMillisecond) - ->UseRealTime(); - -} // namespace diff --git a/cmake/cuCascadeConfig.cmake.in b/cmake/cuCascadeConfig.cmake.in index 347d83d..eac3532 100644 --- a/cmake/cuCascadeConfig.cmake.in +++ b/cmake/cuCascadeConfig.cmake.in @@ -6,7 +6,6 @@ include(CMakeFindDependencyMacro) find_dependency(CUDAToolkit REQUIRED VERSION 12.9) find_dependency(Threads REQUIRED) find_dependency(rmm REQUIRED CONFIG) -find_dependency(cudf REQUIRED CONFIG) # Include the targets file include("${CMAKE_CURRENT_LIST_DIR}/cuCascadeTargets.cmake") diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 094de57..36baebf 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -47,8 +47,12 @@ graph TB DR[data_repository] DB[data_batch] RC[representation_converter_registry] - GPU_REP[gpu_table_representation] - HOST_REP[host_data_representation / host_data_packed_representation] + IREP[idata_representation interface] + DISK_REP[disk_data_representation] + end + + subgraph "Domain Layer (external)" + DOMAIN_REP[GPU / HOST representations + converters
registered via register_converter] end subgraph "Memory Module" @@ -73,8 +77,8 @@ graph TB APP --> MRM DRM --> DR DR --> DB - DB --> GPU_REP - DB --> HOST_REP + DB --> IREP + DISK_REP --> IREP DB --> RC MRM --> MS_GPU @@ -91,8 +95,8 @@ graph TB FSHMR --> HOST_MEM DAL --> DISK_MEM - RC --> MS_GPU - RC --> MS_HOST + DISK_REP --> MS_DISK + DOMAIN_REP --> RC ``` ## Memory Tier System @@ -203,7 +207,7 @@ sequenceDiagram **File**: `include/cucascade/data/data_batch.hpp` -A `data_batch` is the fundamental unit of data in cuCascade. It wraps a tier-specific data representation (GPU table or host table) and manages concurrent access through a RAII read-only and mutable accessor classes. +A `data_batch` is the fundamental unit of data in cuCascade. It wraps a tier-specific data representation (an `idata_representation`, such as the in-library `disk_data_representation` or a GPU/HOST representation provided by the domain layer) and manages concurrent access through a RAII read-only and mutable accessor classes. ```mermaid stateDiagram-v2 @@ -250,19 +254,7 @@ The `data_repository_manager` coordinates multiple repositories across operators The `representation_converter_registry` provides type-indexed conversion between data representations. Converters are registered as functions keyed by `(source_type, target_type)`. -**Built-in converters**: - -| Source | Target | Method | -|--------|--------|--------| -| GPU table | Host (direct) | `cudaMemcpyBatchAsync` (D2H) — copies column buffers directly; ~99% PCIe bandwidth | -| Host (direct) | GPU table | `cudaMemcpyBatchAsync` (H2D) — reconstructs `cudf::column` tree from metadata | -| GPU table | Host (packed) | `cudf::pack()` -> `cudaMemcpyAsync` (D2H) -> multi-block host allocation | -| Host (packed) | GPU table | `cudaMemcpyAsync` (H2D) -> `cudf::unpack()` on device | -| GPU table | GPU table | `cudf::pack()` -> `cudaMemcpyPeerAsync` -> `cudf::unpack()` (cross-device) | -| Host (packed) | Host (packed) | Block-by-block `std::memcpy` (cross-NUMA) | - -"Host (direct)" = `host_data_representation` — preferred, no intermediate GPU allocation. -"Host (packed)" = `host_data_packed_representation` — uses cudf's pack/unpack serialization. +cuCascade ships no built-in converters; the registry starts empty. The domain layer (user code that links cuCascade together with libcudf) registers the tier-to-tier converters it needs (e.g. GPU <-> HOST <-> DISK) at runtime via `register_converter(fn)`. ### Topology Discovery @@ -289,7 +281,7 @@ A typical lifecycle of data through cuCascade: ``` 1. INGESTION - Create data representation (e.g., gpu_table_representation wrapping a cuDF table) + Create a data representation (provided by the domain layer, or disk_data_representation for the DISK tier) -> Wrap in data_batch with unique ID from data_repository_manager -> Add to data_repository @@ -306,7 +298,7 @@ A typical lifecycle of data through cuCascade: 4. MEMORY PRESSURE (downgrade) memory_space.should_downgrade_memory() [threshold exceeded] batch.try_to_lock_for_in_transit() [idle -> in_transit] - converter_registry.convert(...) + converter_registry.convert(...) [domain-registered converter] batch.set_data(new_representation) [data now on HOST] batch.try_to_release_in_transit() [in_transit -> idle] @@ -383,8 +375,7 @@ Key synchronization primitives: | `include/cucascade/memory/oom_handling_policy.hpp` | OOM handling strategies | | `include/cucascade/memory/error.hpp` | Custom error types and exceptions | | `include/cucascade/memory/numa_region_pinned_host_allocator.hpp` | NUMA-aware pinned host allocation | -| `include/cucascade/memory/host_table.hpp` | `host_table_allocation` + `column_metadata` for direct-copy host representations | -| `include/cucascade/memory/host_table_packed.hpp` | `host_table_packed_allocation` for packed (cudf::pack) host representations | +| `include/cucascade/memory/column_metadata.hpp` | `column_metadata` -- generic, domain-agnostic column buffer-layout descriptor used by the disk tier | | `include/cucascade/memory/null_device_memory_resource.hpp` | No-op resource for disk tier | ### Data Module @@ -395,8 +386,7 @@ Key synchronization primitives: | `include/cucascade/data/data_batch.hpp` | Batch lifecycle, read-only and mutable locking accessor classes| | `include/cucascade/data/data_repository.hpp` | Partitioned batch storage with blocking pops | | `include/cucascade/data/data_repository_manager.hpp` | Multi-pipeline repository coordination | -| `include/cucascade/data/representation_converter.hpp` | Type-indexed converter registry | -| `include/cucascade/data/gpu_data_representation.hpp` | GPU-resident cuDF table wrapper | -| `include/cucascade/data/cpu_data_representation.hpp` | `host_data_representation` (direct buffer copy) and `host_data_packed_representation` (cudf::pack) | +| `include/cucascade/data/representation_converter.hpp` | Type-indexed converter registry (ships empty; domain layer registers converters) | +| `include/cucascade/data/disk_data_representation.hpp` | Sole in-library concrete representation (DISK tier) | | `include/cucascade/utils/atomics.hpp` | `atomic_peak_tracker`, `atomic_bounded_counter` | | `include/cucascade/utils/overloaded.hpp` | Variant visitor helper | diff --git a/docs/README.md b/docs/README.md index d5f4290..1bb0f23 100644 --- a/docs/README.md +++ b/docs/README.md @@ -41,6 +41,6 @@ cuCascade is organized into two core modules: ## Related Resources - [Main README](../README.md) -- project overview, quick start, and usage examples -- [RAPIDS cuDF](https://github.com/rapidsai/cudf) -- GPU DataFrame library (core dependency) -- [RMM](https://github.com/rapidsai/rmm) -- RAPIDS Memory Manager (memory allocation backend) +- [RMM](https://github.com/rapidsai/rmm) -- RAPIDS Memory Manager (the only direct RAPIDS dependency) +- [RAPIDS cuDF](https://github.com/rapidsai/cudf) -- GPU DataFrame library (used by the optional domain layer, not the core library) - [Pixi](https://pixi.sh/) -- Package management tool used for builds diff --git a/docs/bandwidth-profiler.md b/docs/bandwidth-profiler.md deleted file mode 100644 index f18af57..0000000 --- a/docs/bandwidth-profiler.md +++ /dev/null @@ -1,159 +0,0 @@ -# Memory Space Bandwidth Profiler — Design - -**Status:** Design locked, pending implementation -**Date:** 2026-04-21 - -## Purpose - -Given a list of `memory_space*`, measure pairwise transfer throughput between them so engines can use the resulting bandwidth matrix for routing decisions (where to place / downgrade / upgrade data). - -Measurements flow through the **existing converter registry and `disk_io_backend`** — results reflect the actual transfer paths engines will pay at runtime, including user-registered converters. - -## Scope - -- Measure GPU↔GPU, GPU↔HOST, HOST↔HOST as a full all-pairs matrix. -- Measure DISK↔{GPU, HOST} for every (disk, non-disk) pair. **No disk↔disk entries** — not a routing path engines care about. -- Measure **both directions** of every pair separately (`src→dst` and `dst→src`). PCIe, NVLink asymmetries, and especially disk read/write asymmetry must be visible. -- Pure end-to-end function. No caching, no state. Runs at init time. -- Does **not** go through `memory_reservation_manager` — allocates directly from each space's allocator. Caller is responsible for running this before production load begins. - -## Decisions - -### D-01. Transfer mechanism -Use `representation_converter_registry::convert()` for every transfer. The registry is initialized (via `register_builtin_converters()` or user registrations) before the profiler runs. If a pair has no registered converter, that cell is left empty / marked unavailable. - -**Why:** Engines route based on what converters will actually do at runtime. Raw `cudaMemcpyAsync` / `cuFileRead` numbers would diverge from production overhead (packing, metadata handling, backend selection). - -### D-02. Disk backend selection -The profiler accepts an `std::shared_ptr` (same parameter as `register_builtin_converters`). Profiler results are scoped to that backend — a kvikIO profile is not interchangeable with a GDS profile. Callers wanting both profile twice. - -### D-03. Chunked-allocator detection — `chunked_resource_info` mixin -New interface lives at `include/cucascade/memory/chunked_resource_info.hpp`: - -```cpp -namespace cucascade::memory { - -struct chunked_resource_info { - virtual ~chunked_resource_info() = default; - [[nodiscard]] virtual std::size_t max_chunk_bytes() const = 0; -}; - -} // namespace cucascade::memory -``` - -- Inherited **alongside** `rmm::mr::device_memory_resource` by any resource that hands out fixed-size chunks rather than arbitrary contiguous ranges. Allocators that don't inherit it are treated as contiguous. -- `fixed_size_host_memory_resource` gets the mixin; `max_chunk_bytes()` forwards to the existing `get_block_size()`. -- The profiler probes with `dynamic_cast(mr)`; on hit, it iterates chunks; on miss, it performs a single contiguous allocation. -- The mixin is opt-in, non-intrusive, and does not alter the `device_memory_resource` contract. Any future allocator (GPU-side arenas, disk chunking) can opt in the same way. - -### D-04. Test-size semantics for chunked allocators -When probing a chunked allocator at test size `N`: -- Compute `k = ceil(N / max_chunk_bytes)` chunk allocations. -- Issue `k` converter calls — one per chunk — and measure the aggregate transfer rate. -- This matches production behavior: a `data_batch` backed by a chunked allocator will already be chunked, and real converters already handle the per-chunk dispatch. - -For contiguous allocators, a single allocation of size `N` is issued and one converter call measures the transfer. - -### D-05. Result shape -Both per-size detail and an aggregated summary (so engines have a single number for routing, and the full curve is available for diagnostics): - -```cpp -namespace cucascade::data { - -struct bandwidth_sample { - double gbps; // GB/s for this (pair, size) - std::chrono::duration mean_transfer_time; - std::size_t bytes_transferred; // effective bytes after chunk rounding -}; - -struct bandwidth_pair_result { - memory_space_id src; - memory_space_id dst; - std::map per_size; // keyed by test size in bytes - bandwidth_sample summary; // representative value (see D-06) - bool converter_available; // false => pair has no registered converter -}; - -struct bandwidth_profile { - std::vector pairs; // asymmetric: src→dst is distinct from dst→src - // Convenience accessors: - [[nodiscard]] double gbps(memory_space_id src, memory_space_id dst) const; // uses summary - [[nodiscard]] std::optional - sample(memory_space_id src, memory_space_id dst, std::size_t size_bytes) const; -}; - -} // namespace cucascade::data -``` - -### D-06. Summary-value aggregation -The `summary` field reports the **median gbps across all measured sizes** for that pair. Rationale: median is robust to outliers at either end of the size curve (small-size latency-dominated, largest-size capacity-constrained) while still reflecting realistic throughput. - -### D-07. Default measurement config -```cpp -struct bandwidth_profile_config { - std::vector test_sizes_bytes{ - 1ull << 20, // 1 MiB - 16ull << 20, // 16 MiB - 64ull << 20, // 64 MiB - 256ull << 20 // 256 MiB - }; - std::size_t warmup_iterations = 3; - std::size_t timed_iterations = 10; - std::chrono::milliseconds min_measurement_time{50}; // loop iterations until hit - bool measure_disk_pairs = true; // disable to skip slow disk paths -}; -``` - -- Multiple sizes chosen to surface the small-vs-large transfer curve without dominating init cost. -- `min_measurement_time` guards against cases where `timed_iterations` finishes faster than timer resolution. -- Disk measurements are opt-out because they're orders of magnitude slower than GPU/HOST pairs. - -### D-08. Bidirectional measurement -For every unique unordered pair `{A, B}` where at least one is non-disk (or both are non-disk), measure `A→B` and `B→A` as two independent entries in the result. For disk pairs, both directions are measured because write ≠ read throughput on NVMe. Self-pairs (`A→A`) are skipped. - -### D-09. Entry point -Pure function: -```cpp -namespace cucascade::data { - -[[nodiscard]] bandwidth_profile measure_bandwidth( - std::span spaces, - representation_converter_registry const& registry, - std::shared_ptr disk_backend = nullptr, - bandwidth_profile_config const& config = {}); - -} // namespace cucascade::data -``` - -- No caching, no internal state. Call it at init, hand the result to engines. -- `disk_backend` may be null if no DISK spaces are present; required when any DISK space is passed in. -- Errors in individual pairs (OOM, transfer failure) mark that pair as `converter_available = false` with zeroed samples; they do not abort the whole profile. - -### D-10. Header locations -- `include/cucascade/memory/chunked_resource_info.hpp` — the mixin trait (memory layer). -- `include/cucascade/data/bandwidth_profiler.hpp` — `bandwidth_profile`, `bandwidth_profile_config`, `measure_bandwidth()` (data layer, depends on converter registry + memory layer). - -Direction of dependency: data → memory (consistent with existing architecture). - -### D-11. Stream usage -Each transfer uses a stream acquired from the destination space's `exclusive_stream_pool` (policy `BLOCK`). The profiler never uses the default CUDA stream. Streams are released as the pool handle goes out of scope between iterations. - -### D-12. Thread safety -`measure_bandwidth` itself is not required to be reentrant — it's a one-shot init call. Callers must not mutate the converter registry or the passed-in memory spaces during execution. Internal transfers use per-pair streams, so individual converter calls run concurrently where the registry supports it. - -## Existing Code Touched - -- `include/cucascade/memory/fixed_size_host_memory_resource.hpp` — add `: public chunked_resource_info` and an override that forwards to `get_block_size()`. -- `include/cucascade/memory/chunked_resource_info.hpp` — **new**, the mixin interface. -- `include/cucascade/data/bandwidth_profiler.hpp` — **new**, public API. -- `src/data/bandwidth_profiler.cpp` — **new**, implementation. -- `test/data/test_bandwidth_profiler.cpp` — **new**, Catch2 tests using mock memory spaces + mock converters. -- Consumers: whatever engine code will call `measure_bandwidth()` at init and thread results into routing. Out of scope for this design — the profiler just returns the data. - -## Deferred / Out of Scope - -- **Caching or persistence of profiles across runs.** Bandwidth can drift (thermal, NUMA pinning, concurrent workload); callers that want persistence build it on top. -- **Dynamic re-profiling under load.** This is an init-time tool. -- **Concurrency bandwidth** (what happens when multiple engines transfer at once). Future work. -- **Raw-hardware baseline mode** (bypassing converters). Can be added as an alternative `measure_bandwidth_raw()` later if needed for debugging converter overhead. -- **Chunked disk allocators.** If `disk_access_limiter` grows chunked allocation semantics, it can opt into `chunked_resource_info` without touching the profiler. diff --git a/docs/data-management.md b/docs/data-management.md index b4128d0..057bcfb 100644 --- a/docs/data-management.md +++ b/docs/data-management.md @@ -7,9 +7,7 @@ A deep dive into cuCascade's data lifecycle, batch read-only and mutable locking - [Overview](#overview) - [Data Representations](#data-representations) - [Interface: idata_representation](#interface-idata_representation) - - [GPU Table Representation](#gpu-table-representation) - - [Host Table Representation (Direct Copy)](#host-table-representation-direct-copy) - - [Host Table Representation (Packed)](#host-table-representation-packed) + - [Concrete Representations](#concrete-representations) - [Data Batch Lifecycle](#data-batch-lifecycle) - [States](#states) - [State Transitions](#state-transitions) @@ -18,8 +16,6 @@ A deep dive into cuCascade's data lifecycle, batch read-only and mutable locking - [Cloning](#cloning) - [Representation Conversion](#representation-conversion) - [Converter Registry](#converter-registry) - - [Built-in Converters](#built-in-converters) - - [Conversion Flow](#conversion-flow) - [Data Repositories](#data-repositories) - [Add and Pop Semantics](#add-and-pop-semantics) - [Partitioning](#partitioning) @@ -37,7 +33,7 @@ A deep dive into cuCascade's data lifecycle, batch read-only and mutable locking The data module manages the lifecycle of data as it flows through processing pipelines and moves between memory tiers. It provides: -- **Tier-agnostic data representations** -- abstract interface with GPU and host implementations +- **Tier-agnostic data representations** -- abstract interface; cuCascade ships the disk representation in-library, while GPU/host representations are provided by the domain layer - **Locking read-only and mutable accessor classes for batch lifecycle** -- prevents concurrent access conflicts during processing and tier movement - **Type-indexed conversion** -- extensible registry for converting data between representations - **Partitioned repositories** -- thread-safe storage with blocking retrieval @@ -70,87 +66,26 @@ public: Data representations are thin wrappers -- they hold the data but delegate storage details (tier, device, allocator) to their associated `memory_space`. -### GPU Table Representation +The interface also declares `record_writer_event(stream)` and `get_writer_event()` as base virtuals with no-op / `nullptr` defaults, used for cross-stream / cross-device synchronization. Representations whose memory is produced asynchronously on a CUDA stream (e.g. GPU representations in the domain layer) override them. -**File**: `include/cucascade/data/gpu_data_representation.hpp` +### Concrete Representations -Wraps a `cudf::table` residing in GPU device memory: +cuCascade is independent of libcudf and ships exactly **one** concrete representation in-library: +`disk_data_representation` (`include/cucascade/data/disk_data_representation.hpp`), which persists a +serialized table to disk and reads it back via the disk I/O backends (kvikIO / GDS / pipeline). -```cpp -class gpu_table_representation : public idata_representation { - std::unique_ptr _table; - -public: - std::unique_ptr release_table(rmm::cuda_stream_view stream); // Transfer ownership - - std::size_t get_size_in_bytes() const override; - std::unique_ptr clone(rmm::cuda_stream_view stream) override; -}; -``` - -- `clone()` performs a deep copy using `cudf::table(table.view(), stream)` -- Owns the `cudf::table` object but not the underlying GPU memory (managed by the allocator) - -### Host Table Representation (Direct Copy) - -**File**: `include/cucascade/data/cpu_data_representation.hpp` - -The preferred host representation. Directly copies each column's GPU device buffers into pinned -host memory without an intermediate GPU-side contiguous allocation. Column layout is described by -custom per-column `column_metadata` structs, enabling reconstruction without cudf's pack format. - -```cpp -class host_data_representation : public idata_representation { - std::unique_ptr _host_table; - -public: - const std::unique_ptr& get_host_table() const; - - std::size_t get_size_in_bytes() const override; - std::unique_ptr clone(rmm::cuda_stream_view stream) override; -}; -``` - -A `host_table_allocation` (`include/cucascade/memory/host_table.hpp`) contains: -- `allocation` -- `multiple_blocks_allocation` (vector of fixed-size pinned memory blocks) -- `columns` -- per-column `column_metadata` trees describing buffer offsets, sizes, and nesting -- `data_size` -- total bytes stored across all blocks - -Each `column_metadata` node captures `type_id`, `num_rows`, `null_count`, `scale` (for decimals), -plus buffer offsets/sizes for the null mask and data buffer, and recursive `children` for nested -types (LIST, STRUCT, STRING, DICTIONARY32). - -This representation avoids the extra GPU allocation and GPU-to-GPU copy that `cudf::pack` performs. -`cudaMemcpyBatchAsync` is used to transfer all column buffers in a single driver call per direction, -achieving ~99% of raw PCIe bandwidth even with multiple concurrent threads. - -`clone()` copies data block-by-block with `std::memcpy` and duplicates the column metadata tree. - -### Host Table Representation (Packed) +GPU and host representations -- which wrap concrete column types such as `cudf::table` -- are **not** +part of the core library. They are provided by an external **domain layer** that links cuCascade and +libcudf, derive from `idata_representation`, and are wired into the conversion pipeline at runtime via +`representation_converter_registry::register_converter()`. -**File**: `include/cucascade/data/cpu_data_representation.hpp` - -A legacy representation that uses `cudf::pack()` to serialize the table to a contiguous GPU buffer -before copying it to host memory. Useful when cudf's serialization format is required. - -```cpp -class host_data_packed_representation : public idata_representation { - std::unique_ptr _host_table; - -public: - const std::unique_ptr& get_host_table() const; - - std::size_t get_size_in_bytes() const override; - std::unique_ptr clone(rmm::cuda_stream_view stream) override; -}; -``` - -A `host_table_packed_allocation` (`include/cucascade/memory/host_table_packed.hpp`) contains: -- `allocation` -- `multiple_blocks_allocation` (vector of fixed-size pinned memory blocks) -- `metadata` -- serialized cuDF table metadata from `cudf::pack()` (for reconstruction via `cudf::unpack()`) -- `data_size` -- actual data size in bytes - -`clone()` copies data block-by-block with `std::memcpy` and duplicates the metadata vector. +Column buffer layout for tier transfers and disk storage is described by the generic, domain-agnostic +`memory::column_metadata` (`include/cucascade/memory/column_metadata.hpp`). Each node captures an +opaque `type_id` tag (the numeric value of a consumer's column-type enum, e.g. `cudf::type_id`, which +cuCascade never interprets), `num_rows`, `null_count`, `scale` (for decimals), buffer offsets/sizes for +the null mask and data buffer, and recursive `children` for nested types. The disk tier's +`disk_table_allocation` (`include/cucascade/memory/disk_table.hpp`) holds a +`std::vector`. --- @@ -207,9 +142,9 @@ Access to batch data is only possible through one of two RAII accessor classes: // Acquire shared lock (blocks if exclusive lock held) read_only_data_batch ro = batch->to_read_only(); -// Access data -auto* data = ro.get_data()->cast(); -process(data->get_table_view()); +// Access data (cast to the concrete representation registered by your domain layer) +auto& data = ro.get_data()->cast(); +process(data); // Release: either let ro go out of scope, or explicitly return to idle auto idle_batch = cucascade::data_batch::to_idle(std::move(ro)); @@ -270,8 +205,8 @@ This is used by the pipeline and downgrade executor to track which batches are s read_only_data_batch ro = batch->to_read_only(); auto cloned = ro.clone(new_batch_id, stream); -// With representation conversion -auto cloned = ro.clone_to(registry, new_batch_id, host_space, stream); +// With representation conversion (target type registered by your domain layer) +auto cloned = ro.clone_to(registry, new_batch_id, host_space, stream); ``` Cloning produces a new `shared_ptr` in `idle` state with the given ID. The clone @@ -289,77 +224,27 @@ contains a deep copy of the data representation, residing in the same memory spa The `representation_converter_registry` stores conversion functions indexed by `(source_type, target_type)`: ```cpp -// Register a custom converter -registry.register_converter( +// Register a custom converter (SourceRep / TargetRep are concrete representations +// derived from idata_representation, typically provided by the domain layer) +registry.register_converter( [](idata_representation& source, const memory_space* target, rmm::cuda_stream_view stream) -> std::unique_ptr { - // Convert GPU table to host (direct copy) + // Build the target representation from the source return ...; } ); // Convert data -auto host_data = registry.convert( - *gpu_data, target_host_space, stream); +auto converted = registry.convert(*source_data, target_space, stream); ``` The registry is thread-safe (all operations guarded by `std::mutex`). -### Built-in Converters - -Registered via `register_builtin_converters()`: - -| Source | Target | Method | -|--------|--------|--------| -| `gpu_table_representation` | `host_data_representation` | `cudaMemcpyBatchAsync` (D2H) — copies each column's buffers directly to pinned host blocks; ~99% PCIe bandwidth | -| `host_data_representation` | `gpu_table_representation` | Allocates `rmm::device_buffer` per column buffer, `cudaMemcpyBatchAsync` (H2D), reconstructs `cudf::column` tree | -| `gpu_table_representation` | `host_data_packed_representation` | `cudf::pack()` on GPU -> `cudaMemcpyAsync` (D2H) to multi-block host allocation | -| `host_data_packed_representation` | `gpu_table_representation` | `cudaMemcpyAsync` (H2D) from host blocks -> `cudf::unpack()` on device | -| `gpu_table_representation` | `gpu_table_representation` | `cudf::pack()` -> `cudaMemcpyPeerAsync` -> `cudf::unpack()` (cross-device copy) | -| `host_data_packed_representation` | `host_data_packed_representation` | Block-by-block `std::memcpy` + metadata copy (cross-NUMA copy) | - -### Conversion Flow - -GPU to HOST (direct copy — preferred): - -```mermaid -sequenceDiagram - participant Caller - participant Registry as converter_registry - participant GPU as gpu_table_representation - participant Host as host_data_representation - participant HostMR as fixed_size_host_memory_resource - - Caller->>Registry: convert(gpu_data, host_space, stream) - Registry->>HostMR: allocate_multiple_blocks(total_column_bytes) - Note over HostMR: Returns vector of 1MB pinned host blocks - Registry->>Registry: Traverse column tree, compute buffer offsets into host blocks - Registry->>Registry: cudaMemcpyBatchAsync(all_bufs_in_one_call, D2H, stream) - Note over Registry: Single driver call for all column buffers - Registry->>Host: Create host_data_representation(blocks, column_metadata_tree) - Registry-->>Caller: unique_ptr -``` - -GPU to HOST (packed — uses cudf serialization): - -```mermaid -sequenceDiagram - participant Caller - participant Registry as converter_registry - participant GPU as gpu_table_representation - participant Host as host_data_packed_representation - participant HostMR as fixed_size_host_memory_resource - - Caller->>Registry: convert(gpu_data, host_space, stream) - Registry->>GPU: cudf::pack(table) - Note over GPU: Serializes table to contiguous buffer + metadata - Registry->>HostMR: allocate_multiple_blocks(data_size) - Note over HostMR: Returns vector of 1MB pinned host blocks - Registry->>Registry: cudaMemcpyAsync(host_blocks, gpu_buffer, D2H, stream) - Note over Registry: Copies data block by block - Registry->>Host: Create host_data_packed_representation(blocks, metadata) - Registry-->>Caller: unique_ptr -``` +cuCascade ships **no** built-in converters -- the registry starts empty. Consumers (the domain +layer that links libcudf) register the converters they need at runtime via `register_converter()`. +At convert time the registry looks up the function keyed by `{typeid(source), typeid(target)}` and +invokes it; the registry itself is representation-agnostic and never names or interprets concrete +types. --- @@ -512,10 +397,8 @@ Application | `include/cucascade/data/data_repository.hpp` | `idata_repository`, `shared_data_repository`, `unique_data_repository` | | `include/cucascade/data/data_repository_manager.hpp` | `data_repository_manager`, `operator_port_key` | | `include/cucascade/data/representation_converter.hpp` | `representation_converter_registry`, `converter_key` | -| `include/cucascade/data/gpu_data_representation.hpp` | `gpu_table_representation` wrapping `cudf::table` | -| `include/cucascade/data/cpu_data_representation.hpp` | `host_data_representation` (direct buffer copy) and `host_data_packed_representation` (cudf::pack) | -| `include/cucascade/memory/host_table.hpp` | `host_table_allocation` and `column_metadata` for direct-copy host representations | -| `include/cucascade/memory/host_table_packed.hpp` | `host_table_packed_allocation` for packed (cudf::pack) host representations | +| `include/cucascade/data/disk_data_representation.hpp` | `disk_data_representation` (the only in-library concrete representation) | +| `include/cucascade/memory/column_metadata.hpp` | `memory::column_metadata` generic columnar buffer-layout descriptor | | `include/cucascade/utils/atomics.hpp` | `atomic_peak_tracker`, `atomic_bounded_counter` | | `include/cucascade/utils/overloaded.hpp` | Variant visitor helper | | `docs/data_batch_state_transitions.md` | Complete state machine reference | diff --git a/docs/development-guide.md b/docs/development-guide.md index 79628d9..0028640 100644 --- a/docs/development-guide.md +++ b/docs/development-guide.md @@ -42,8 +42,7 @@ Building, testing, benchmarking, and contributing to cuCascade. | Ninja | Any | | CUDA Toolkit | 13+ | | NVIDIA Driver | Compatible with CUDA 13 | -| libcudf | 25.10+ | -| RMM | Via cuDF dependency | +| RMM | librmm (direct dependency) | | numactl | Development headers | The easiest way to get all dependencies is via [Pixi](https://pixi.sh/), which manages the full environment. @@ -95,9 +94,10 @@ cuCascade/ │ │ ├── data_repository.hpp # Partitioned batch storage │ │ ├── data_repository_manager.hpp │ │ ├── representation_converter.hpp -│ │ ├── cpu_data_representation.hpp -│ │ └── gpu_data_representation.hpp -│ ├── memory/ # Memory module headers (17 files) +│ │ ├── disk_data_representation.hpp # Disk-tier representation (in-library) +│ │ ├── disk_file_format.hpp +│ │ └── disk_io_backend.hpp # GDS / kvikIO / pipeline backends +│ ├── memory/ # Memory module headers (20 files) │ │ ├── common.hpp # Tier enum, memory_space_id │ │ ├── config.hpp # Tier-specific config structs │ │ ├── memory_reservation_manager.hpp @@ -113,20 +113,20 @@ cuCascade/ │ │ ├── oom_handling_policy.hpp │ │ ├── error.hpp │ │ ├── numa_region_pinned_host_allocator.hpp -│ │ ├── host_table.hpp -│ │ ├── host_table_packed.hpp +│ │ ├── column_metadata.hpp # Domain-agnostic column buffer-layout descriptor +│ │ ├── disk_table.hpp # On-disk table allocation + binary format │ │ └── null_device_memory_resource.hpp │ └── utils/ # Utility headers │ ├── atomics.hpp # atomic_peak_tracker, atomic_bounded_counter │ └── overloaded.hpp # Variant visitor helper ├── src/ -│ ├── data/ # Data module implementation (6 .cpp files) -│ └── memory/ # Memory module implementation (15 .cpp files) +│ ├── data/ # Data module implementation (7 .cpp files) +│ └── memory/ # Memory module implementation (16 .cpp files) ├── test/ │ ├── unittest.cpp # Test runner with GPU pool setup -│ ├── data/ # Data module tests (5 files) +│ ├── data/ # Data module tests (6 files) │ ├── memory/ # Memory module tests (3 files + GPU kernels) -│ └── utils/ # Test utilities (mocks, cuDF helpers) +│ └── utils/ # Test utilities (mocks, memory resources) ├── benchmark/ # Google Benchmark suite ├── docs/ # Documentation (you are here) ├── cmake/ # CMake package config template @@ -152,7 +152,7 @@ cucascade_objects (OBJECT library) └── src/memory/*.cpp │ ├── cucascade_static (STATIC library) - │ Links: RMM, cuDF, CUDA runtime, CUDA NVML, pthreads, numa + │ Links: RMM, CUDA runtime, CUDA NVML, pthreads, numa │ └── cucascade_shared (SHARED library) Same links, SOVERSION 0 @@ -212,7 +212,7 @@ cuCascade uses [Catch2](https://github.com/catchorg/Catch2) v2.13.10 (fetched fr Tests use BDD-style assertions: ```cpp TEST_CASE("data batch transitions to task_created", "[data_batch]") { - auto batch = data_batch(1, make_gpu_representation()); + auto batch = data_batch(1, std::make_unique(memory::Tier::GPU)); REQUIRE(batch.get_state() == batch_state::idle); @@ -231,8 +231,9 @@ All tests compile into a single executable `cucascade_tests`: | `test/data/test_data_batch.cpp` | 47+ cases | State transitions, processing handles, cloning | | `test/data/test_data_repository.cpp` | 140+ cases | Add/pop, partitioning, blocking, threading | | `test/data/test_data_repository_manager.cpp` | 100+ cases | Multi-operator, batch IDs, concurrent access | -| `test/data/test_data_representation.cpp` | Representation interface | Size, tier, clone operations | | `test/data/test_representation_converter.cpp` | Converter registry | Registration, lookup, conversion | +| `test/data/test_disk_io_backend.cpp` | Disk I/O backends | GDS / kvikIO / pipeline read/write | +| `test/data/test_io_worker.cpp` | I/O worker | Background disk transfer scheduling | | `test/memory/test_memory_reservation_manager.cpp` | Reservation system | Strategies, limits, multi-space | | `test/memory/test_topology_discovery.cpp` | Hardware detection | NVML integration | | `test/memory/test_gpu_kernels.cu` | GPU kernel tests | Device-side operations | @@ -243,13 +244,6 @@ All tests compile into a single executable `cucascade_tests`: - `make_mock_memory_space()` -- lightweight memory spaces without real allocators - `mock_data_representation` -- implements `idata_representation` for testing - `create_conversion_test_configs()` -- memory manager configs (1 GPU + 1 HOST) -- `create_simple_cudf_table()` -- factory for test cuDF tables (INT32/INT64 columns) - -**`test/utils/cudf_test_utils.hpp`**: -- `cudf_tables_have_equal_contents_on_stream()` -- stream-aware table comparison -- `expect_cudf_tables_equal_on_stream()` -- assertion wrapper with detailed error reporting -- `logging_device_resource` -- RMM allocation tracing -- `shared_device_resource` -- shared allocator wrapper **`test/utils/test_memory_resources.hpp`**: - `make_shared_current_device_resource()` -- wraps the current RMM device resource for sharing @@ -281,14 +275,13 @@ cd build/release && ./test/cucascade_tests -c "specific test name" ### Available Benchmarks -Built with [Google Benchmark](https://github.com/google/benchmark) v1.8.3: +Built with [Google Benchmark](https://github.com/google/benchmark) v1.8.3. -| Benchmark | Description | Parameters | -|-----------|-------------|------------| -| `BM_ConvertGpuToHost` | GPU -> HOST data conversion | Size: 64KiB-512MiB, Cols: 2-8, Threads: 1-4 | -| `BM_ConvertHostToGpu` | HOST -> GPU data conversion | Same as above | -| `BM_GpuToHostThroughput` | Raw GPU -> HOST bandwidth | Size: 64KiB-512MiB, Threads: 1-4 | -| `BM_HostToGpuThroughput` | Raw HOST -> GPU bandwidth | Same as above | +The previous converter/throughput benchmarks were cuDF-dependent and were removed +along with the cuDF-backed representations (issue #142). The benchmark suite +currently ships no sources, so the `cucascade_benchmarks` target is skipped until +new cuDF-free benchmarks (e.g. raw-buffer disk I/O) are added to +`benchmark/CMakeLists.txt`. ### Running Benchmarks @@ -300,8 +293,7 @@ pixi run benchmarks cd build/release && ./benchmark/cucascade_benchmarks # Filter to specific benchmarks -./benchmark/cucascade_benchmarks --benchmark_filter=Convert -./benchmark/cucascade_benchmarks --benchmark_filter=Throughput +./benchmark/cucascade_benchmarks --benchmark_filter= # JSON output for comparison ./benchmark/cucascade_benchmarks \ @@ -421,7 +413,7 @@ Available targets: - `cuCascade::cucascade_shared` -- shared library explicitly - `cuCascade::cucascade_static` -- static library explicitly -Transitive dependencies (cuDF, RMM, CUDA Toolkit, Threads) are automatically found. +Transitive dependencies (RMM, CUDA Toolkit, Threads) are automatically found. --- diff --git a/docs/memory-management.md b/docs/memory-management.md index 6012606..58f46bd 100644 --- a/docs/memory-management.md +++ b/docs/memory-management.md @@ -404,7 +404,6 @@ Default pool size: **16 streams** per memory space. | `include/cucascade/memory/oom_handling_policy.hpp` | OOM handling strategies | | `include/cucascade/memory/error.hpp` | `MemoryError` enum, `cucascade_out_of_memory` exception | | `include/cucascade/memory/numa_region_pinned_host_allocator.hpp` | NUMA-aware pinned host allocation | -| `include/cucascade/memory/host_table.hpp` | `host_table_allocation` + `column_metadata` for direct-copy host representations | -| `include/cucascade/memory/host_table_packed.hpp` | `host_table_packed_allocation` for packed (cudf::pack) host representations | +| `include/cucascade/memory/column_metadata.hpp` | Generic per-column buffer-layout descriptor (`memory::column_metadata`) used by the disk tier and tier transfers | | `include/cucascade/memory/null_device_memory_resource.hpp` | No-op resource for disk tier | | `include/cucascade/utils/atomics.hpp` | `atomic_peak_tracker`, `atomic_bounded_counter` | diff --git a/include/cucascade/data/bandwidth_profiler.hpp b/include/cucascade/data/bandwidth_profiler.hpp deleted file mode 100644 index 198f262..0000000 --- a/include/cucascade/data/bandwidth_profiler.hpp +++ /dev/null @@ -1,164 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace cucascade { -namespace data { - -/** - * @brief A single timing measurement for a (pair, transfer-size) probe. - */ -struct bandwidth_sample { - double gbps = 0.0; ///< Achieved throughput in GB/s (10^9 bytes) - double mean_seconds = 0.0; ///< Mean wall-clock time per iteration - std::size_t bytes_transferred = 0; ///< Effective bytes moved per iteration - std::size_t iterations_timed = 0; ///< Number of timed iterations aggregated -}; - -/** - * @brief Bandwidth measurement for one ordered (src -> dst) pair of memory spaces. - * - * Results are asymmetric — `src -> dst` and `dst -> src` are separate entries. - * Disk pairs are only included when exactly one endpoint is DISK (no disk-to-disk). - */ -struct bandwidth_pair_result { - /// Source memory space. Overwritten before use; default values exist only because - /// `memory_space_id` has no default constructor. - memory::memory_space_id src{memory::Tier::GPU, 0}; - /// Destination memory space. See `src` note. - memory::memory_space_id dst{memory::Tier::GPU, 0}; - - /// Chunk size advertised by the source's allocator (0 if contiguous / not a chunked resource). - std::size_t src_max_chunk_bytes = 0; - /// Chunk size advertised by the destination's allocator (0 if contiguous / not a chunked - /// resource). - std::size_t dst_max_chunk_bytes = 0; - - /// Per-test-size detail. Keyed by requested transfer size in bytes. - std::map per_size; - /// Aggregated summary across per_size entries (median gbps). - bandwidth_sample summary; - - /// False when no converter was available for this pair, or a transient error suppressed it. - bool converter_available = true; - /// Human-readable detail when `converter_available == false`. - std::string unavailable_reason; -}; - -/** - * @brief Configuration for `measure_bandwidth`. - */ -struct bandwidth_profile_config { - /// Transfer sizes to probe per pair. Defaults to a sweep from 1 MiB up to 256 MiB. - std::vector test_sizes_bytes{ - 1ull << 20, // 1 MiB - 16ull << 20, // 16 MiB - 64ull << 20, // 64 MiB - 256ull << 20, // 256 MiB - }; - /// Untimed converter invocations run before measurement to warm caches, JIT, file cache, etc. - std::size_t warmup_iterations = 3; - /// Timed converter invocations aggregated into the per-size sample. - std::size_t timed_iterations = 10; - /// Skip disk pairs (useful when you only care about GPU/HOST cells). - bool measure_disk_pairs = true; - /// When true, call `posix_fadvise(POSIX_FADV_DONTNEED)` on disk source files between timed - /// iterations so every read hits the disk instead of the Linux page cache. Process-local; - /// no sudo required. Disable to measure warm-cache behavior. - bool drop_page_cache_between_iters = true; -}; - -/** - * @brief Asymmetric bandwidth profile across a set of memory spaces. - */ -struct bandwidth_profile { - std::vector pairs; - - /** - * @brief Summary gbps for the given ordered pair, or 0 if not found / unavailable. - */ - [[nodiscard]] double gbps(memory::memory_space_id src, - memory::memory_space_id dst) const noexcept; - - /** - * @brief Per-size sample for the given ordered pair and transfer size. - * - * @return The sample if the pair exists and size was measured; `std::nullopt` otherwise. - */ - [[nodiscard]] std::optional sample(memory::memory_space_id src, - memory::memory_space_id dst, - std::size_t size_bytes) const; - - /** - * @brief Find the result for a pair, if any. - */ - [[nodiscard]] const bandwidth_pair_result* find(memory::memory_space_id src, - memory::memory_space_id dst) const noexcept; -}; - -/** - * @brief Measure pairwise transfer throughput between a set of memory spaces. - * - * Results are asymmetric — both `src -> dst` and `dst -> src` are measured. - * - * Pair rules: - * - GPU and HOST spaces are measured against each other in both directions, including - * self-to-self across distinct device ids. - * - DISK spaces are measured against each GPU and HOST space only (no disk-to-disk). - * - Pairs that reduce to the same memory space id are skipped. - * - * Transfers flow through the converter registry, using the built-in canonical - * representation for each tier (GPU -> gpu_table_representation, - * HOST -> host_data_representation, DISK -> disk_data_representation). Pairs without a - * registered converter are marked `converter_available == false`. - * - * This function is pure and synchronous. It does not reserve through - * `memory_reservation_manager`; callers are expected to run it at init time when the - * target memory spaces are otherwise idle. - * - * Each DISK `memory_space` owns its own I/O backend, so the profile reflects whichever - * backend each disk space was constructed with — no separate backend parameter is needed. - * At least one GPU space must be present: it is used to materialize the canonical source - * cudf table that seeds every pairwise transfer. - * - * @param spaces The memory spaces to profile. - * @param registry Converter registry to dispatch through. Pass a registry populated by - * `register_builtin_converters` plus any user-supplied converters. - * @param config Measurement knobs; defaults probe a size sweep. - * - * @return `bandwidth_profile` containing one `bandwidth_pair_result` per ordered pair considered. - * - * @throws std::invalid_argument if no GPU space is present in `spaces`. - */ -[[nodiscard]] bandwidth_profile measure_bandwidth(std::span spaces, - const representation_converter_registry& registry, - const bandwidth_profile_config& config = {}); - -} // namespace data -} // namespace cucascade diff --git a/include/cucascade/data/common.hpp b/include/cucascade/data/common.hpp index 1942362..25cd4f1 100644 --- a/include/cucascade/data/common.hpp +++ b/include/cucascade/data/common.hpp @@ -21,6 +21,8 @@ #include +#include + #include #include #include @@ -110,6 +112,29 @@ class idata_representation { */ virtual std::unique_ptr clone(rmm::cuda_stream_view stream) = 0; + /** + * @brief Record a CUDA event marking completion of the most recent writes to this + * representation's memory, for cross-stream / cross-device synchronization. + * + * The default implementation is a no-op. Tier-specific representations whose memory is + * produced asynchronously on a CUDA stream (e.g. GPU representations) override this to + * record an event on @p writer_stream. + * + * @param writer_stream The stream on which the most recent writes were enqueued. + */ + virtual void record_writer_event([[maybe_unused]] rmm::cuda_stream_view writer_stream) {} + + /** + * @brief Get the writer event recorded by record_writer_event(), or nullptr if none. + * + * Readers that cross stream / device boundaries should call cudaStreamWaitEvent on the + * returned event (when non-null) before reading this representation's memory. The default + * implementation returns nullptr; representations that record writer events override this. + * + * @return cudaEvent_t The writer event, or nullptr if none has been recorded. + */ + [[nodiscard]] virtual cudaEvent_t get_writer_event() const { return nullptr; } + /** * @brief Casts this interface to a specific derived type. * diff --git a/include/cucascade/data/cpu_data_representation.hpp b/include/cucascade/data/cpu_data_representation.hpp deleted file mode 100644 index ac99015..0000000 --- a/include/cucascade/data/cpu_data_representation.hpp +++ /dev/null @@ -1,160 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include - -#include -#include -#include - -namespace cucascade { - -/** - * @brief Data representation for a table stored in host memory using direct buffer copies. - * - * Directly copies each column's GPU device buffers to pinned host memory without an intermediate - * GPU-side contiguous allocation. The column layout is described by custom per-column metadata - * stored in host_table_allocation, enabling reconstruction without cudf's pack format. - * - * This avoids the extra GPU allocation and GPU-to-GPU copy that cudf::pack performs when - * serializing a table, at the cost of managing per-column buffer metadata ourselves. - */ -class host_data_representation : public idata_representation { - public: - /** - * @brief Construct a host_data_representation. - * - * @param host_table Allocation owning the copied column buffers and their layout metadata - * @param memory_space The host memory space where the data resides - */ - host_data_representation(std::unique_ptr host_table, - memory::memory_space* memory_space); - - /** - * @brief Get the total number of bytes occupied by this representation. - * - * @return std::size_t The number of data bytes stored in the host allocation - */ - std::size_t get_size_in_bytes() const override; - - /** - * @copydoc idata_representation::get_logical_data_size_in_bytes - */ - std::size_t get_uncompressed_data_size_in_bytes() const override; - - /** - * @brief Create a deep copy of this representation in the same memory space. - * - * @param stream CUDA stream (unused for host-side copies) - * @return std::unique_ptr A new host_data_representation - */ - std::unique_ptr clone(rmm::cuda_stream_view stream) override; - - /** - * @brief Access the underlying host table allocation. - * - * @return const reference to the unique_ptr owning the allocation - */ - const std::unique_ptr& get_host_table() const; - - /// @brief Number of top-level columns held by the underlying allocation. - [[nodiscard]] std::size_t num_columns() const noexcept; - - /** - * @brief Total bytes (null mask + data, recursive over children) for column @p i. - * - * @throws std::out_of_range if @p i >= num_columns(). - */ - [[nodiscard]] std::size_t column_size(std::size_t i) const; - - /** - * @brief Create a host_data_representation that shares buffers but exposes only the - * requested columns. - * - * Forwards to host_table_allocation::slice(). The returned representation references the - * same underlying pinned buffers as the source; converting it to another tier yields a - * table containing only the selected columns. - */ - [[nodiscard]] std::unique_ptr slice( - std::span col_indices) const; - - private: - std::unique_ptr _host_table; -}; - -/** - * @brief Data representation for a table being stored in host memory using cudf::pack. - * - * This represents a table whose data is stored across multiple blocks (not necessarily contiguous) - * in host memory. The host_data_packed_representation doesn't own the actual data but is instead - * owned by the multiple_blocks_allocation. - */ -class host_data_packed_representation : public idata_representation { - public: - /** - * @brief Construct a new host_data_packed_representation object - * - * @param host_table The underlying allocation owning the actual data - * @param memory_space The memory space where the host table resides - */ - host_data_packed_representation( - std::unique_ptr host_table, - cucascade::memory::memory_space* memory_space); - - /** - * @brief Get the size of the data representation in bytes - * - * @return std::size_t The number of bytes used to store this representation - */ - std::size_t get_size_in_bytes() const override; - - /** - * @copydoc idata_representation::get_logical_data_size_in_bytes - */ - std::size_t get_uncompressed_data_size_in_bytes() const override; - - /** - * @brief Create a deep copy of this host table representation. - * - * The cloned representation will have its own copy of the underlying host table, - * residing in the same memory space as the original. - * - * @param stream CUDA stream (unused for host-side copies) - * @return std::unique_ptr A new host_data_packed_representation with - * copied data - */ - std::unique_ptr clone(rmm::cuda_stream_view stream) override; - - /** - * @brief Get the underlying host table allocation - * - * @return const reference to the unique_ptr owning the allocation - */ - const std::unique_ptr& get_host_table() const; - - private: - std::unique_ptr - _host_table; ///< The allocation where the actual data resides -}; - -} // namespace cucascade diff --git a/include/cucascade/data/data_batch.hpp b/include/cucascade/data/data_batch.hpp index 86480f3..a3a3d0b 100644 --- a/include/cucascade/data/data_batch.hpp +++ b/include/cucascade/data/data_batch.hpp @@ -18,10 +18,11 @@ #pragma once #include -#include #include #include +#include + #include #include #include @@ -295,27 +296,23 @@ class read_only_data_batch { memory::memory_space* get_memory_space() const { return _batch->get_memory_space(); } /** - * @brief Get the writer event from the underlying GPU representation, or nullptr. + * @brief Get the writer event from the underlying representation, or nullptr. * - * D-B3 proxy: delegates to gpu_table_representation::get_writer_event() via - * dynamic_cast. Returns nullptr when the underlying representation is not a - * gpu_table_representation (e.g., host or disk tier) or when no writer event has - * been recorded yet. + * D-B3 proxy: delegates polymorphically to idata_representation::get_writer_event(). + * Returns nullptr when there is no underlying representation, when the representation's + * tier records no writer event (the base-class default, e.g. host or disk tier), or when + * no writer event has been recorded yet. * * STREAM-LINEAGE: callers that cross stream / device boundaries should call * cudaStreamWaitEvent on the returned event (when non-null) before reading the * underlying memory of this batch. * - * @return cudaEvent_t The writer event, or nullptr if not a GPU representation or - * no event recorded. + * @return cudaEvent_t The writer event, or nullptr if none is available. */ [[nodiscard]] cudaEvent_t get_writer_event() const { auto* repr = get_data(); - if (!repr) { return nullptr; } - auto* gpu_repr = dynamic_cast(repr); - if (!gpu_repr) { return nullptr; } - return gpu_repr->get_writer_event(); + return repr ? repr->get_writer_event() : nullptr; } // -- Clone operations (D-18/D-19/D-20/CLONE-01/CLONE-02) -- diff --git a/include/cucascade/data/gpu_data_representation.hpp b/include/cucascade/data/gpu_data_representation.hpp deleted file mode 100644 index b7c4c47..0000000 --- a/include/cucascade/data/gpu_data_representation.hpp +++ /dev/null @@ -1,211 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include - -#include -#include - -#include - -#include - -#include -#include -#include -#include - -namespace cucascade { - -/** - * @brief Data representation for a table being stored in GPU memory. - * - * This class currently represents a table just as a cuDF table along with the allocation where the - * cudf's table data actually resides. The primary purpose for this is that the table can be - * directly passed to cuDF APIs for processing without any additional copying while the underlying - * memory is still owned/tracked by our memory allocator. - * - * TODO: Once the GPU memory resource is implemented, replace the allocation type from - * IAllocatedMemory to the concrete type returned by the GPU memory allocator. - */ -class gpu_table_representation : public idata_representation { - public: - /** - * @brief Construct a new gpu_table_representation object. - * - * STREAM-LINEAGE: every gpu_table_representation must be born with a recorded - * writer event so cross-stream / cross-device readers (notably - * representation_converter.cpp's convert_gpu_to_gpu()) can establish ordering - * via cudaStreamWaitEvent. The constructor calls record_writer_event(@p - * writer_stream) automatically — passing a default-constructed - * cuda_stream_view records no event (legacy, only acceptable for paths whose - * data was never produced on any stream). - * - * @param table Unique pointer to the cuDF table with the data (ownership is transferred) - * @param memory_space The memory space where the GPU table resides - * @param writer_stream The stream on which @p table's data was last written. - * MUST be the actual writer stream — passing the wrong - * stream re-introduces the race this contract closes. - */ - gpu_table_representation(std::unique_ptr table, - cucascade::memory::memory_space& memory_space, - rmm::cuda_stream_view writer_stream); - - /** - * @brief Construct a new gpu_table_representation object from a cudf::table_view. - * - * STREAM-LINEAGE: writer_stream is REQUIRED — must be the stream on which the - * underlying data of @p table_view was last written. See the simple-table ctor - * docstring for the writer_stream contract. - * - * @param table_view View of the cuDF table (data ownership lives in @p owner) - * @tparam Owner The type of the owner of the cuDF table (e.g., a specific operator or component) - * @param owner Owner of the underlying data (transferred via std::any storage) - * @param alloc_size Allocation size in bytes for the data - * @param memory_space The memory space where the GPU table resides - * @param writer_stream The stream on which @p table_view's data was last written. - */ - template - gpu_table_representation(cudf::table_view table_view, - Owner&& owner, - std::size_t alloc_size, - cucascade::memory::memory_space& memory_space, - rmm::cuda_stream_view writer_stream); - - /** - * @brief Destructor — destroys the writer-event if one was recorded. - * - * STREAM-LINEAGE: events recorded on a writer stream via record_writer_event() are - * owned by the representation and released on destruction. - */ - ~gpu_table_representation() override; - - // Non-copyable / non-movable: the representation owns a cudaEvent_t handle whose - // lifetime must be unique. Move semantics could be added but are not needed by - // any in-tree caller. - gpu_table_representation(const gpu_table_representation&) = delete; - gpu_table_representation(gpu_table_representation&&) = delete; - gpu_table_representation& operator=(const gpu_table_representation&) = delete; - gpu_table_representation& operator=(gpu_table_representation&&) = delete; - - /** - * @brief Get the size of the data representation in bytes - * - * @return std::size_t The number of bytes used to store this representation - */ - std::size_t get_size_in_bytes() const override; - - /** - * @copydoc idata_representation::get_logical_data_size_in_bytes - */ - std::size_t get_uncompressed_data_size_in_bytes() const override; - - /** - * @brief Create a deep copy of this GPU table representation. - * - * The cloned representation will have its own copy of the underlying cuDF table, - * residing in the same memory space as the original. - * - * @param stream CUDA stream for memory operations - * @return std::unique_ptr A new gpu_table_representation with copied data - */ - std::unique_ptr clone(rmm::cuda_stream_view stream) override; - - /** - * @brief Get the underlying cuDF table view - * - * @return cudf::table_view A view of the cuDF table - */ - cudf::table_view get_table_view() const; - - /** - * @brief Release ownership of the underlying cuDF table - * - * After calling this method, this representation no longer owns the table. - * - * @param stream CUDA stream (used to materialize the table from a view path before release) - * @return std::unique_ptr The cuDF table - */ - std::unique_ptr release_table(rmm::cuda_stream_view stream); - - /** - * @brief Record a CUDA event on @p writer_stream and store it as the writer event. - * - * STREAM-LINEAGE: any cross-stream / cross-device reader of this representation's - * memory must wait on this event before issuing reads. Concretely, - * representation_converter.cpp's convert_gpu_to_gpu() will call - * cudaStreamWaitEvent(reader_stream, get_writer_event(), 0) before peer-copying - * source buffers. - * - * Calling this multiple times overwrites the previously recorded event (the - * representation owns a single writer event handle that is reused). Passing a - * default-constructed cuda_stream_view records no event and clears any prior one. - * - * @param writer_stream The stream on which the most recent writes to this - * representation's memory were enqueued. - */ - void record_writer_event(rmm::cuda_stream_view writer_stream); - - /** - * @brief Get the writer event recorded by record_writer_event(), or nullptr if none. - * - * Readers that cross stream / device boundaries must call cudaStreamWaitEvent on - * this event (when non-null) before reading the underlying memory. When this - * returns nullptr, callers should fall back to a coarser sync (e.g. - * cudaDeviceSynchronize on the source device) — this is the legacy behavior - * preserved for representations constructed by code paths that have not yet - * been migrated to record writer events. - * - * @return cudaEvent_t The writer event, or nullptr if none has been recorded. - */ - [[nodiscard]] cudaEvent_t get_writer_event() const; - - private: - struct owning_table_view { - std::any owner; ///< The owner of the cuDF table - std::size_t alloc_size{0}; - cudf::table_view view; ///< A view of the owned table for easy access - }; - - std::variant, owning_table_view> - _table; ///< cudf::table is the underlying representation of the data - - /// Lazily-created CUDA event recording the completion of the most recent - /// writer-stream work that produced this representation. Null until the first - /// call to record_writer_event(). - cudaEvent_t _writer_event{nullptr}; -}; - -template -gpu_table_representation::gpu_table_representation(cudf::table_view table_view, - Owner&& owner, - std::size_t alloc_size, - cucascade::memory::memory_space& memory_space, - rmm::cuda_stream_view writer_stream) - : idata_representation(memory_space), - _table( - owning_table_view{std::make_any(std::forward(owner)), alloc_size, table_view}) -{ - // STREAM-LINEAGE: record writer event so cross-stream/cross-device readers - // can establish ordering via cudaStreamWaitEvent. - if (writer_stream.value() != nullptr) { record_writer_event(writer_stream); } -} - -} // namespace cucascade diff --git a/include/cucascade/data/representation_converter.hpp b/include/cucascade/data/representation_converter.hpp index d9274ce..3321476 100644 --- a/include/cucascade/data/representation_converter.hpp +++ b/include/cucascade/data/representation_converter.hpp @@ -92,8 +92,9 @@ struct converter_key_hash { * * The registry is thread-safe for concurrent registration and lookup. * - * @note Built-in converters between gpu_table_representation and - * host_data_packed_representation are registered via register_builtin_converters(). + * @note cuCascade ships no built-in converters. Concrete converters (e.g. between GPU, HOST, + * and DISK representations) are provided by the domain layer, which registers them via + * register_converter(). */ class representation_converter_registry { public: @@ -180,7 +181,7 @@ class representation_converter_registry { * @throws std::runtime_error if no converter is registered for the type pair * * @example - * auto result = registry.convert(source, space); + * auto result = registry.convert(source, space); */ template std::unique_ptr convert(idata_representation& source, @@ -245,15 +246,4 @@ class representation_converter_registry { std::unordered_map _converters; }; -/** - * @brief Initialize the built-in representation converters. - * - * Registers converters between all supported representation types (GPU, HOST, DISK). - * Disk converters resolve the I/O backend from the disk memory_space at conversion time, - * so each disk memory_space can use a different backend. - * - * @param registry The converter registry to register converters with. - */ -void register_builtin_converters(representation_converter_registry& registry); - } // namespace cucascade diff --git a/include/cucascade/memory/column_metadata.hpp b/include/cucascade/memory/column_metadata.hpp new file mode 100644 index 0000000..e7fa051 --- /dev/null +++ b/include/cucascade/memory/column_metadata.hpp @@ -0,0 +1,58 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +namespace cucascade { +namespace memory { + +/** + * @brief Generic columnar buffer-layout descriptor for a single (possibly nested) column. + * + * Captures everything needed to reconstruct a column from raw memory bytes: a domain-defined + * type tag, logical row/null counts, an optional null mask buffer, an optional flat data buffer, + * and child descriptors for nested types. Children are stored recursively. + * + * @note `type_id` is an opaque, domain-defined type tag that cuCascade never interprets. It is + * laid out to hold the numeric value of a consumer's column type enum (e.g. the integer value of + * a `cudf::type_id`), so consumers reconstruct the concrete type by `static_cast`-ing it back. + * Keeping it a plain `int32_t` (same width/values as the corresponding consumer enum) makes the + * on-disk and in-memory layouts binary-compatible across the cuCascade/consumer boundary. + */ +struct column_metadata { + int32_t type_id; ///< Domain-defined column type tag (numeric value of the consumer's enum) + int32_t num_rows; ///< Number of logical rows (elements) in this column + int32_t null_count; ///< Number of null elements (may be a domain-defined "unknown" sentinel) + int32_t scale; ///< Scale factor for fixed-point / decimal types; 0 for all others + + bool has_null_mask; ///< Whether a null mask buffer was copied + std::size_t null_mask_offset; ///< Byte offset of the null mask within the allocation + std::size_t null_mask_size; ///< Size of the null mask buffer in bytes + + bool has_data; ///< Whether a flat data buffer was copied (false for nested types) + std::size_t data_offset; ///< Byte offset of the data buffer within the allocation + std::size_t data_size; ///< Size of the data buffer in bytes + + std::vector children; ///< Metadata for child columns (nested types) +}; + +} // namespace memory +} // namespace cucascade diff --git a/include/cucascade/memory/common.hpp b/include/cucascade/memory/common.hpp index 8b49708..c47bfba 100644 --- a/include/cucascade/memory/common.hpp +++ b/include/cucascade/memory/common.hpp @@ -184,9 +184,9 @@ void enable_pool_peer_access_for_all_visible_devices(cudaMemPool_t pool, int own * pool access to ProtNone) tells the driver to fall back to its internal pinned * host-staging path for `cudaMemcpyPeer*` — slower than real peer DMA but correct. * - * Intended call site: cucascade::register_builtin_converters() — runs once after - * memory_spaces are constructed and the application has called - * `cudaDeviceEnablePeerAccess` for each pair. Idempotent. + * Intended call site: once after memory_spaces are constructed and the application has + * called `cudaDeviceEnablePeerAccess` for each pair (e.g. by the domain layer that + * registers tier converters). Idempotent. * * @param pools_by_device A vector indexed by device id (0..N-1) of the cucascade * pool to also reset access on. Pass an empty vector if cucascade pools diff --git a/include/cucascade/memory/disk_table.hpp b/include/cucascade/memory/disk_table.hpp index 4c78c48..0119744 100644 --- a/include/cucascade/memory/disk_table.hpp +++ b/include/cucascade/memory/disk_table.hpp @@ -17,7 +17,7 @@ #pragma once -#include +#include #include #include @@ -30,10 +30,10 @@ namespace memory { /** * @brief Disk-resident allocation containing per-column metadata and a file path. * - * Mirrors the role of host_table_allocation for the disk tier. Instead of pinned - * host memory blocks, data is stored in a single file on disk identified by - * file_path. The column_metadata descriptors are identical to those used by - * host_table_allocation, enabling straightforward conversion between tiers. + * Describes a table serialized to a single file on disk: the file path, the per-column + * buffer-layout metadata, and the total data size. The column_metadata descriptors are the + * same generic layout descriptors used for in-memory column buffers, enabling straightforward + * conversion between tiers. */ struct disk_table_allocation { std::string file_path; ///< Absolute path to the batch file on disk diff --git a/include/cucascade/memory/host_table.hpp b/include/cucascade/memory/host_table.hpp deleted file mode 100644 index 964e8e4..0000000 --- a/include/cucascade/memory/host_table.hpp +++ /dev/null @@ -1,133 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#include - -#include -#include -#include -#include -#include - -namespace cucascade { -namespace memory { - -class memory_space; - -/** - * @brief Metadata describing a single column's buffer layout within a host_table_allocation. - * - * Captures everything needed to reconstruct a cudf column from raw host memory bytes, without - * relying on the cudf::pack serialization format. Children are stored recursively for nested types - * (LIST, STRUCT, STRING, DICTIONARY32). - */ -struct column_metadata { - cudf::type_id type_id; ///< Column data type identifier - cudf::size_type num_rows; ///< Number of logical rows (elements) in this column - cudf::size_type null_count; ///< Number of null elements (may be UNKNOWN_NULL_COUNT) - int32_t scale; ///< Scale factor for DECIMAL32/64/128 types; 0 for all others - - bool has_null_mask; ///< Whether a null mask buffer was copied - std::size_t null_mask_offset; ///< Byte offset of the null mask within the allocation - std::size_t null_mask_size; ///< Size of the null mask buffer in bytes - - bool has_data; ///< Whether a flat data buffer was copied (false for nested types) - std::size_t data_offset; ///< Byte offset of the data buffer within the allocation - std::size_t data_size; ///< Size of the data buffer in bytes - - std::vector children; ///< Metadata for child columns (nested types) -}; - -/** - * @brief Host memory allocation containing directly-copied column buffers and custom metadata. - * - * Unlike host_table_packed_allocation (which relies on cudf::pack/unpack and therefore requires an - * intermediate GPU-side contiguous copy), this class copies each column's GPU buffers - * directly to pinned host memory. This avoids the extra GPU memory allocation and bandwidth - * that cudf::pack incurs. - * - * Buffers are reference-counted via shared_ptr so a host_table_allocation produced by slice() - * shares the underlying storage with its parent. Reconstruction uses the per-column - * column_metadata descriptors rather than cudf's packed-table serialization format. - */ -class host_table_allocation { - public: - using buffers_ptr = std::shared_ptr; - - /** - * @brief Construct a host_table_allocation taking ownership of @p buffers. - * - * The buffers are converted from a unique allocation to a shared one so that future slices - * of this allocation can share the underlying storage. - */ - static std::unique_ptr create(fixed_multiple_blocks_allocation buffers, - std::vector columns, - std::size_t data_size); - - /** - * @brief Create a slice that shares buffers but exposes only the requested columns. - * - * Column offsets within the returned allocation reference the same shared buffers; reading - * from the slice reads from the original storage. The reported `data_size` is the sum of the - * selected columns' buffer bytes (null masks + data, recursive over children). - * - * @param col_indices Column indices to retain in the slice. Must be in [0, num_columns()). - * @return A new host_table_allocation referencing the shared buffers. - * @throws std::out_of_range if any index is invalid. - */ - std::unique_ptr slice(std::span col_indices) const; - - /** - * @brief Deep-copy currently-present column buffers into @p target memory_space. - * - * Allocates fresh storage in the target host memory space sized to fit only the columns - * present in this allocation, copies each column's null-mask and data buffers into a compact - * 8-byte-aligned layout, and returns a new owning host_table_allocation. - * - * @param target Host-tier memory_space that owns a fixed_size_host_memory_resource. - */ - std::unique_ptr clone(memory_space& target) const; - - /// @brief Number of top-level columns described by this allocation. - [[nodiscard]] std::size_t num_columns() const noexcept { return columns.size(); } - - /** - * @brief Total bytes (null mask + data, recursive over children) for column @p i. - * - * @throws std::out_of_range if @p i >= num_columns(). - */ - [[nodiscard]] std::size_t column_size(std::size_t i) const; - - /// Shared pinned blocks containing all raw column buffer data. - buffers_ptr allocation; - /// One metadata entry per top-level column; children are stored recursively. - std::vector columns; - /// Total number of live data bytes referenced by `columns`. - std::size_t data_size{0}; - - private: - host_table_allocation(buffers_ptr buffers, - std::vector cols, - std::size_t data_sz); -}; - -} // namespace memory -} // namespace cucascade diff --git a/include/cucascade/memory/host_table_packed.hpp b/include/cucascade/memory/host_table_packed.hpp deleted file mode 100644 index 8b49e68..0000000 --- a/include/cucascade/memory/host_table_packed.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#include -#include -#include - -namespace cucascade { -namespace memory { - -/** - * @brief Structure containing both the host memory allocation and metadata for table recreation. - * - * This structure contains the actual memory allocation and the metadata required to recreate the - * table. The metadata is a vector of uint8_t that contains the metadata for the table. The - * data_size is the size of the data in the allocation. - */ -struct host_table_packed_allocation { - /// Fixed-size buffers containing the table data - memory::fixed_multiple_blocks_allocation allocation; - /// Metadata generated by cudf::pack, required for table reconstruction - std::unique_ptr> metadata; - /// Number of bytes stored in the allocation - std::size_t data_size; - - host_table_packed_allocation(memory::fixed_multiple_blocks_allocation alloc, - std::unique_ptr> meta, - std::size_t data_sz) - : allocation(std::move(alloc)), metadata(std::move(meta)), data_size(data_sz) - { - } -}; - -} // namespace memory -} // namespace cucascade diff --git a/pixi.toml b/pixi.toml index 45fb4b6..0c62b3d 100644 --- a/pixi.toml +++ b/pixi.toml @@ -12,7 +12,6 @@ cxx-compiler = "*" doxygen = "*" fmt = "*" jinja2 = "*" -libcurand-dev = "*" make = "*" ninja = "*" numactl = "*" @@ -33,25 +32,32 @@ system-requirements = { cuda = "12.9" } dependencies = { "cuda-version" = "12.9.*" } activation = { env = { CUDAARCHS = "75-real;80-real;86-real;90a-real;100f-real" } } -# cuDF (nightly channel, default) -[feature.cudf-nightly] +# RMM (RAPIDS Memory Manager) -- cuCascade's only RAPIDS dependency. Provides the +# rmm CMake config consumed by find_package(rmm CONFIG). Previously pulled in +# transitively via libcudf; now depended on directly so the library (and tests) +# build without libcudf. Pin matches the RAPIDS release train; verify the exact +# available version with `pixi search -c rapidsai-nightly librmm` if the solve fails. +[feature.rmm-nightly] channels = [{ channel = "rapidsai-nightly", priority = 1 }] -dependencies = { "libcudf" = "26.06.*" } +dependencies = { "librmm" = "26.06.*" } -[feature.cudf-stable] +[feature.rmm-stable] channels = [{ channel = "rapidsai", priority = 1 }] -dependencies = { "libcudf" = "26.04.*" } +dependencies = { "librmm" = "26.04.*" } [environments] -default = ["cuda-13", "cudf-nightly"] -cuda-13-nightly = ["cuda-13", "cudf-nightly"] -cuda-12-nightly = ["cuda-12", "cudf-nightly"] -cuda-13-stable = ["cuda-13", "cudf-stable"] -cuda-12-stable = ["cuda-12", "cudf-stable"] +default = ["cuda-13", "rmm-nightly"] +cuda-13-nightly = ["cuda-13", "rmm-nightly"] +cuda-12-nightly = ["cuda-12", "rmm-nightly"] +cuda-13-stable = ["cuda-13", "rmm-stable"] +cuda-12-stable = ["cuda-12", "rmm-stable"] [tasks] build = "cmake --preset release && cmake --build build/release" build-debug = "cmake --preset debug && cmake --build build/debug" +# Thin aliases matching the names referenced by issue #142's acceptance criteria. +cmake-release = "cmake --preset release" +build-release = "cmake --build build/release" clean = "rm -rf build" test = "cd build/release && ctest --output-on-failure" benchmarks = "cd build/release && ./benchmark/cucascade_benchmarks" diff --git a/src/data/CMakeLists.txt b/src/data/CMakeLists.txt index dc48a35..b38019d 100644 --- a/src/data/CMakeLists.txt +++ b/src/data/CMakeLists.txt @@ -17,13 +17,10 @@ target_sources( cucascade_objects - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bandwidth_profiler.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/cpu_data_representation.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/disk_data_representation.cpp + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/disk_data_representation.cpp ${CMAKE_CURRENT_SOURCE_DIR}/io_backend_registry.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pipeline_io_backend.cpp ${CMAKE_CURRENT_SOURCE_DIR}/data_batch.cpp ${CMAKE_CURRENT_SOURCE_DIR}/data_repository.cpp ${CMAKE_CURRENT_SOURCE_DIR}/data_repository_manager.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/gpu_data_representation.cpp ${CMAKE_CURRENT_SOURCE_DIR}/representation_converter.cpp) diff --git a/src/data/bandwidth_profiler.cpp b/src/data/bandwidth_profiler.cpp deleted file mode 100644 index 5f70998..0000000 --- a/src/data/bandwidth_profiler.cpp +++ /dev/null @@ -1,357 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace cucascade { -namespace data { - -namespace { - -/// Canonical `idata_representation` type for a tier — what the profiler uses as both the -/// source-in-that-tier and the target-in-that-tier representation. -std::type_index canonical_type_for(memory::Tier tier) -{ - switch (tier) { - case memory::Tier::GPU: return std::type_index(typeid(gpu_table_representation)); - case memory::Tier::HOST: return std::type_index(typeid(host_data_representation)); - case memory::Tier::DISK: return std::type_index(typeid(disk_data_representation)); - default: throw std::invalid_argument("bandwidth_profiler: unsupported memory tier"); - } -} - -/// Probe an allocator for the chunked-resource mixin. Returns 0 for contiguous allocators. -std::size_t probe_max_chunk_bytes(const memory::memory_space& space) -{ - auto const* chunked = space.get_chunked_resource_info(); - return chunked != nullptr ? chunked->max_chunk_bytes() : 0; -} - -/// Build a single-column INT32 cudf::table of approximately `size_bytes` bytes, allocated -/// through the provided GPU memory resource reference. -std::unique_ptr make_gpu_table_of_size(std::size_t size_bytes, - rmm::device_async_resource_ref gpu_mr, - rmm::cuda_stream_view stream) -{ - constexpr std::size_t bytes_per_row = sizeof(std::int32_t); - auto num_rows = static_cast( - std::max(1, (size_bytes + bytes_per_row - 1) / bytes_per_row)); - - std::vector> cols; - auto col = cudf::make_numeric_column( - cudf::data_type{cudf::type_id::INT32}, num_rows, cudf::mask_state::UNALLOCATED, stream, gpu_mr); - cols.push_back(std::move(col)); - return std::make_unique(std::move(cols)); -} - -/// Materialize a source `idata_representation` of the given tier and size, living in `src_space`. -/// Uses `bootstrap_gpu` as a scratch GPU space to build the initial cudf table. When `src_space` -/// is a GPU other than the bootstrap, the GPU->GPU converter moves data across devices itself -/// (it acquires a stream on the target GPU internally). -std::unique_ptr build_source_representation( - memory::memory_space* src_space, - memory::memory_space* bootstrap_gpu, - std::size_t size_bytes, - const representation_converter_registry& registry) -{ - // The initial cudf table MUST be allocated with the bootstrap GPU as the current CUDA device, - // otherwise cudf's scratch allocations land on the wrong GPU and we hit illegal-memory-access - // during subsequent cross-device work. - rmm::cuda_set_device_raii bootstrap_guard{rmm::cuda_device_id{bootstrap_gpu->get_device_id()}}; - - auto bootstrap_stream = bootstrap_gpu->acquire_stream(); - auto gpu_mr = bootstrap_gpu->get_default_allocator(); - auto table = make_gpu_table_of_size(size_bytes, gpu_mr, bootstrap_stream); - auto gpu_rep = - std::make_unique(std::move(table), *bootstrap_gpu, bootstrap_stream); - bootstrap_stream.synchronize(); - - // Step 2: land the data in the requested src space via the registry. The converter is - // responsible for switching device when moving data across GPUs. - if (src_space == bootstrap_gpu) { return gpu_rep; } - - auto src_type = canonical_type_for(src_space->get_tier()); - auto result = registry.convert(*gpu_rep, src_type, src_space, bootstrap_stream); - // The converter may have enqueued async GPU reads from `gpu_rep`'s table on - // `bootstrap_stream`. Sync before `gpu_rep` goes out of scope — otherwise its cuDF table's - // RMM deallocation races with the in-flight copy and corrupts the converted output. - bootstrap_stream.synchronize(); - return result; -} - -/// Average a per-size sample set — pick the sample whose gbps is closest to the median as summary. -bandwidth_sample compute_summary(const std::map& per_size) -{ - if (per_size.empty()) { return {}; } - std::vector samples; - samples.reserve(per_size.size()); - for (auto const& [sz, s] : per_size) { - samples.push_back(&s); - } - std::sort( - samples.begin(), samples.end(), [](auto const* a, auto const* b) { return a->gbps < b->gbps; }); - return *samples[samples.size() / 2]; -} - -/// Evict a file's contents from the OS page cache so the next read hits disk. -/// Uses `posix_fadvise(POSIX_FADV_DONTNEED)` which is process-local and needs no privileges. -/// Best-effort: silently ignores open/advise failures. -void evict_page_cache(const std::string& path) -{ - int fd = ::open(path.c_str(), O_RDONLY); - if (fd < 0) return; - // Kernel drops DONTNEED pages lazily — syncing first ensures dirty pages are written out - // so they're actually eligible to drop. - ::fdatasync(fd); - (void)::posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); - ::close(fd); -} - -/// Core per-pair probe — warmup + timed loop around `registry.convert(...)`. -bandwidth_sample measure_single_size(idata_representation& source, - std::type_index target_type, - memory::memory_space* dst_space, - const representation_converter_registry& registry, - rmm::cuda_stream_view stream, - std::size_t nominal_size_bytes, - std::size_t warmup_iters, - std::size_t timed_iters, - bool drop_page_cache_between_iters) -{ - // Grab the disk source's file path once so we can evict its page cache between iterations. - // For non-disk sources this stays empty and the evict call is skipped. - std::string disk_source_path; - if (auto const* disk_src = dynamic_cast(&source)) { - disk_source_path = disk_src->get_disk_table().file_path; - } - - auto evict_if_needed = [&]() { - if (drop_page_cache_between_iters && !disk_source_path.empty()) { - evict_page_cache(disk_source_path); - } - }; - - // Warmup — discard results. - for (std::size_t i = 0; i < warmup_iters; ++i) { - auto dst_rep = registry.convert(source, target_type, dst_space, stream); - stream.synchronize(); - dst_rep.reset(); - evict_if_needed(); - } - - // Timed loop. Time accumulated per-iteration around the transfer and synchronization only — - // page-cache eviction (fdatasync + posix_fadvise) happens AFTER each iteration's clock stops - // so it cannot inflate the reported bandwidth number. - using clock = std::chrono::steady_clock; - std::chrono::duration elapsed{0}; - for (std::size_t i = 0; i < timed_iters; ++i) { - auto iter_t0 = clock::now(); - auto dst_rep = registry.convert(source, target_type, dst_space, stream); - stream.synchronize(); - auto iter_t1 = clock::now(); - elapsed += (iter_t1 - iter_t0); - dst_rep.reset(); - evict_if_needed(); - } - bandwidth_sample s{}; - s.iterations_timed = timed_iters; - s.bytes_transferred = nominal_size_bytes; - s.mean_seconds = timed_iters > 0 ? elapsed.count() / static_cast(timed_iters) : 0.0; - s.gbps = - s.mean_seconds > 0.0 ? static_cast(nominal_size_bytes) / s.mean_seconds / 1.0e9 : 0.0; - return s; -} - -bool should_skip_pair(const memory::memory_space& src, - const memory::memory_space& dst, - bool measure_disk_pairs) -{ - if (src.get_id() == dst.get_id()) return true; - if (src.get_tier() == memory::Tier::DISK && dst.get_tier() == memory::Tier::DISK) return true; - if (!measure_disk_pairs && - (src.get_tier() == memory::Tier::DISK || dst.get_tier() == memory::Tier::DISK)) { - return true; - } - return false; -} - -} // namespace - -// --------------------------------------------------------------------------------------------- -// bandwidth_profile lookups -// --------------------------------------------------------------------------------------------- - -const bandwidth_pair_result* bandwidth_profile::find(memory::memory_space_id src, - memory::memory_space_id dst) const noexcept -{ - for (auto const& p : pairs) { - if (p.src == src && p.dst == dst) return &p; - } - return nullptr; -} - -double bandwidth_profile::gbps(memory::memory_space_id src, - memory::memory_space_id dst) const noexcept -{ - auto const* p = find(src, dst); - if (p == nullptr || !p->converter_available) return 0.0; - return p->summary.gbps; -} - -std::optional bandwidth_profile::sample(memory::memory_space_id src, - memory::memory_space_id dst, - std::size_t size_bytes) const -{ - auto const* p = find(src, dst); - if (p == nullptr) return std::nullopt; - auto it = p->per_size.find(size_bytes); - if (it == p->per_size.end()) return std::nullopt; - return it->second; -} - -// --------------------------------------------------------------------------------------------- -// measure_bandwidth -// --------------------------------------------------------------------------------------------- - -bandwidth_profile measure_bandwidth(std::span spaces, - const representation_converter_registry& registry, - const bandwidth_profile_config& config) -{ - bandwidth_profile profile; - - // Locate bootstrap GPU space — used to materialize the canonical cudf source table that feeds - // every subsequent conversion. The profiler requires at least one GPU space in `spaces`. - memory::memory_space* bootstrap_gpu = nullptr; - for (auto* s : spaces) { - if (s == nullptr) continue; - if (s->get_tier() == memory::Tier::GPU && bootstrap_gpu == nullptr) { - bootstrap_gpu = s; - break; - } - } - - if (bootstrap_gpu == nullptr) { - throw std::invalid_argument( - "bandwidth_profiler: at least one GPU memory_space must be present in `spaces`"); - } - - for (auto* src : spaces) { - if (src == nullptr) continue; - for (auto* dst : spaces) { - if (dst == nullptr) continue; - if (should_skip_pair(*src, *dst, config.measure_disk_pairs)) continue; - - bandwidth_pair_result result; - result.src = src->get_id(); - result.dst = dst->get_id(); - result.src_max_chunk_bytes = probe_max_chunk_bytes(*src); - result.dst_max_chunk_bytes = probe_max_chunk_bytes(*dst); - result.converter_available = true; - - auto const target_type = canonical_type_for(dst->get_tier()); - - // The registry lacks a type-only probe, so we detect unavailable converters lazily: - // if build_source_representation or the first convert() throws, we record the reason and - // skip remaining sizes for this pair. - // - // Streams are per-CUDA-context: passing a stream that belongs to a different device than - // where the source table's memory lives causes illegal-memory-access on cross-device copies - // (cudf::pack reads the source with the passed stream). So when the source tier is GPU we - // must use a stream from the source GPU; otherwise any GPU stream will do and we borrow - // the bootstrap's. - auto* stream_owner = src->get_tier() == memory::Tier::GPU - ? src - : (dst->get_tier() == memory::Tier::GPU ? dst : bootstrap_gpu); - auto stream = stream_owner->acquire_stream(); - - for (auto size_bytes : config.test_sizes_bytes) { - try { - auto source = build_source_representation(src, bootstrap_gpu, size_bytes, registry); - - // Pin the current CUDA context to the stream's device for the duration of the - // measurement loop. Some converter and disk-backend code paths allocate scratch on - // the current device rather than the stream's device — mismatch triggers - // cudaErrorInvalidValue / cudaErrorInvalidResourceHandle when src and dst straddle - // GPUs or the pipeline backend was initialized under a different context. This guard - // is placed AFTER build_source_representation so the bootstrap build can run with - // bootstrap_gpu as current. - std::optional device_guard; - if (stream_owner->get_tier() == memory::Tier::GPU) { - device_guard.emplace(rmm::cuda_device_id{stream_owner->get_device_id()}); - } - - // Ensure source construction is complete on the destination's stream (converters may - // enqueue work on it during the warmup iterations). - stream.synchronize(); - auto sample = measure_single_size(*source, - target_type, - dst, - registry, - stream, - size_bytes, - config.warmup_iterations, - config.timed_iterations, - config.drop_page_cache_between_iters); - result.per_size.emplace(size_bytes, sample); - } catch (const std::exception& e) { - result.converter_available = false; - result.unavailable_reason = e.what(); - result.per_size.clear(); - break; - } - } - - if (result.converter_available) { result.summary = compute_summary(result.per_size); } - profile.pairs.push_back(std::move(result)); - } - } - - return profile; -} - -} // namespace data -} // namespace cucascade diff --git a/src/data/cpu_data_representation.cpp b/src/data/cpu_data_representation.cpp deleted file mode 100644 index c180632..0000000 --- a/src/data/cpu_data_representation.cpp +++ /dev/null @@ -1,324 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include - -namespace cucascade { - -namespace { - -inline std::size_t align_up_8(std::size_t v) noexcept -{ - return (v + 7u) & ~static_cast(7u); -} - -std::size_t column_total_bytes(const memory::column_metadata& m) -{ - std::size_t bytes = 0; - if (m.has_null_mask) bytes += m.null_mask_size; - if (m.has_data) bytes += m.data_size; - for (const auto& c : m.children) { - bytes += column_total_bytes(c); - } - return bytes; -} - -memory::column_metadata recompute_compact_offsets(const memory::column_metadata& src, - std::size_t& cursor) -{ - memory::column_metadata dst{}; - dst.type_id = src.type_id; - dst.num_rows = src.num_rows; - dst.null_count = src.null_count; - dst.scale = src.scale; - - dst.has_null_mask = src.has_null_mask; - dst.null_mask_size = src.null_mask_size; - if (src.has_null_mask && src.null_mask_size > 0) { - cursor = align_up_8(cursor); - dst.null_mask_offset = cursor; - cursor += src.null_mask_size; - } else { - dst.null_mask_offset = 0; - } - - dst.has_data = src.has_data; - dst.data_size = src.data_size; - if (src.has_data && src.data_size > 0) { - cursor = align_up_8(cursor); - dst.data_offset = cursor; - cursor += src.data_size; - } else { - dst.data_offset = 0; - } - - dst.children.reserve(src.children.size()); - for (const auto& child : src.children) { - dst.children.push_back(recompute_compact_offsets(child, cursor)); - } - return dst; -} - -void copy_between_blocks( - const memory::fixed_size_host_memory_resource::multiple_blocks_allocation& src, - std::size_t src_offset, - memory::fixed_size_host_memory_resource::multiple_blocks_allocation& dst, - std::size_t dst_offset, - std::size_t size) -{ - if (size == 0) return; - const std::size_t sb = src.block_size(); - const std::size_t db = dst.block_size(); - std::size_t s_idx = src_offset / sb; - std::size_t s_off = src_offset % sb; - std::size_t d_idx = dst_offset / db; - std::size_t d_off = dst_offset % db; - std::size_t copied = 0; - while (copied < size) { - const std::size_t s_avail = sb - s_off; - const std::size_t d_avail = db - d_off; - const std::size_t chunk = std::min({size - copied, s_avail, d_avail}); - std::memcpy(dst.at(d_idx).data() + d_off, src.at(s_idx).data() + s_off, chunk); - copied += chunk; - s_off += chunk; - d_off += chunk; - if (s_off == sb) { - ++s_idx; - s_off = 0; - } - if (d_off == db) { - ++d_idx; - d_off = 0; - } - } -} - -void clone_column_buffers( - const memory::column_metadata& src_meta, - const memory::column_metadata& dst_meta, - const memory::fixed_size_host_memory_resource::multiple_blocks_allocation& src, - memory::fixed_size_host_memory_resource::multiple_blocks_allocation& dst) -{ - if (src_meta.has_null_mask && src_meta.null_mask_size > 0) { - copy_between_blocks( - src, src_meta.null_mask_offset, dst, dst_meta.null_mask_offset, src_meta.null_mask_size); - } - if (src_meta.has_data && src_meta.data_size > 0) { - copy_between_blocks(src, src_meta.data_offset, dst, dst_meta.data_offset, src_meta.data_size); - } - for (std::size_t i = 0; i < src_meta.children.size(); ++i) { - clone_column_buffers(src_meta.children[i], dst_meta.children[i], src, dst); - } -} - -} // namespace - -namespace memory { - -// ============================================================================= -// host_table_allocation -// ============================================================================= - -host_table_allocation::host_table_allocation(buffers_ptr buffers, - std::vector cols, - std::size_t data_sz) - : allocation(std::move(buffers)), columns(std::move(cols)), data_size(data_sz) -{ -} - -std::unique_ptr host_table_allocation::create( - fixed_multiple_blocks_allocation buffers, - std::vector columns, - std::size_t data_size) -{ - buffers_ptr shared_buffers(std::move(buffers)); - return std::unique_ptr( - new host_table_allocation(std::move(shared_buffers), std::move(columns), data_size)); -} - -std::size_t host_table_allocation::column_size(std::size_t i) const -{ - if (i >= columns.size()) { - throw std::out_of_range("host_table_allocation::column_size: index out of range"); - } - return column_total_bytes(columns[i]); -} - -std::unique_ptr host_table_allocation::slice( - std::span col_indices) const -{ - std::vector sliced_cols; - sliced_cols.reserve(col_indices.size()); - std::size_t sliced_size = 0; - for (std::size_t idx : col_indices) { - if (idx >= columns.size()) { - throw std::out_of_range("host_table_allocation::slice: column index out of range"); - } - sliced_cols.push_back(columns[idx]); - sliced_size += column_total_bytes(columns[idx]); - } - return std::unique_ptr( - new host_table_allocation(allocation, std::move(sliced_cols), sliced_size)); -} - -std::unique_ptr host_table_allocation::clone(memory_space& target) const -{ - auto* mr = target.get_memory_resource_as(); - if (mr == nullptr) { - throw std::runtime_error( - "host_table_allocation::clone: target memory_space has no fixed_size_host_memory_resource"); - } - - std::size_t cursor = 0; - std::vector new_cols; - new_cols.reserve(columns.size()); - for (const auto& c : columns) { - new_cols.push_back(recompute_compact_offsets(c, cursor)); - } - const std::size_t new_data_size = cursor; - - auto new_alloc = mr->allocate_multiple_blocks(new_data_size); - - if (allocation && new_alloc) { - for (std::size_t i = 0; i < columns.size(); ++i) { - clone_column_buffers(columns[i], new_cols[i], *allocation, *new_alloc); - } - } - - buffers_ptr shared_new(std::move(new_alloc)); - return std::unique_ptr( - new host_table_allocation(std::move(shared_new), std::move(new_cols), new_data_size)); -} - -} // namespace memory - -// ============================================================================= -// host_data_representation -// ============================================================================= - -host_data_representation::host_data_representation( - std::unique_ptr host_table, memory::memory_space* memory_space) - : idata_representation(*memory_space), _host_table(std::move(host_table)) -{ -} - -std::size_t host_data_representation::get_size_in_bytes() const { return _host_table->data_size; } - -std::size_t host_data_representation::get_uncompressed_data_size_in_bytes() const -{ - return get_size_in_bytes(); -} - -const std::unique_ptr& host_data_representation::get_host_table() - const -{ - return _host_table; -} - -std::size_t host_data_representation::num_columns() const noexcept -{ - return _host_table->num_columns(); -} - -std::size_t host_data_representation::column_size(std::size_t i) const -{ - return _host_table->column_size(i); -} - -std::unique_ptr host_data_representation::slice( - std::span col_indices) const -{ - auto sliced = _host_table->slice(col_indices); - return std::make_unique( - std::move(sliced), &const_cast(get_memory_space())); -} - -std::unique_ptr host_data_representation::clone( - [[maybe_unused]] rmm::cuda_stream_view stream) -{ - auto cloned = _host_table->clone(get_memory_space()); - return std::make_unique(std::move(cloned), &get_memory_space()); -} - -// ============================================================================= -// host_data_packed_representation -// ============================================================================= - -host_data_packed_representation::host_data_packed_representation( - std::unique_ptr host_table, - cucascade::memory::memory_space* memory_space) - : idata_representation(*memory_space), _host_table(std::move(host_table)) -{ -} - -std::size_t host_data_packed_representation::get_size_in_bytes() const -{ - return _host_table->data_size; -} - -std::size_t host_data_packed_representation::get_uncompressed_data_size_in_bytes() const -{ - return get_size_in_bytes(); -} - -const std::unique_ptr& -host_data_packed_representation::get_host_table() const -{ - return _host_table; -} - -std::unique_ptr host_data_packed_representation::clone( - [[maybe_unused]] rmm::cuda_stream_view stream) -{ - // Get the host memory resource from the memory space - auto* host_mr = get_memory_space().get_memory_resource_of(); - if (!host_mr) { - throw std::runtime_error( - "Cannot clone host_data_packed_representation: no host memory resource"); - } - - // Allocate new blocks for the copy - auto allocation_copy = host_mr->allocate_multiple_blocks(_host_table->data_size); - - // Copy data block by block - const auto& src_blocks = _host_table->allocation->get_blocks(); - auto dst_blocks = allocation_copy->get_blocks(); - std::size_t remaining = _host_table->data_size; - std::size_t block_size = _host_table->allocation->block_size(); - - for (std::size_t i = 0; i < src_blocks.size() && remaining > 0; ++i) { - std::size_t copy_size = std::min(remaining, block_size); - std::memcpy(dst_blocks[i], src_blocks[i], copy_size); - remaining -= copy_size; - } - - // Clone the metadata - auto metadata_copy = std::make_unique>(*_host_table->metadata); - - // Create the new host_table_packed_allocation - auto host_table_copy = std::make_unique( - std::move(allocation_copy), std::move(metadata_copy), _host_table->data_size); - - return std::make_unique(std::move(host_table_copy), - &get_memory_space()); -} - -} // namespace cucascade diff --git a/src/data/gpu_data_representation.cpp b/src/data/gpu_data_representation.cpp deleted file mode 100644 index 17eb025..0000000 --- a/src/data/gpu_data_representation.cpp +++ /dev/null @@ -1,111 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include -#include - -namespace cucascade { - -gpu_table_representation::gpu_table_representation(std::unique_ptr table, - cucascade::memory::memory_space& memory_space, - rmm::cuda_stream_view writer_stream) - : idata_representation(memory_space), _table(std::move(table)) -{ - // STREAM-LINEAGE: record the writer event in the constructor body so every - // representation is born with a recorded event. Skipping when the caller - // passes a default-constructed (per-thread default) stream view preserves - // legacy behavior for callers that genuinely have no writer stream — they - // will fall back to cudaDeviceSynchronize on the source device in - // convert_gpu_to_gpu(). All non-legacy callers MUST pass a real writer - // stream. - if (writer_stream.value() != nullptr) { record_writer_event(writer_stream); } -} - -gpu_table_representation::~gpu_table_representation() -{ - // STREAM-LINEAGE: release the writer event if one was recorded. Use - // CUCASCADE_ASSERT_CUDA_SUCCESS so destructor stays noexcept-safe (release - // builds discard the error; debug builds assert). Null-event branch is the - // common case for representations whose writer never recorded an event. - if (_writer_event != nullptr) { - CUCASCADE_ASSERT_CUDA_SUCCESS(cudaEventDestroy(_writer_event)); - _writer_event = nullptr; - } -} - -std::size_t gpu_table_representation::get_size_in_bytes() const -{ - if (std::holds_alternative>(_table)) { - return std::get>(_table)->alloc_size(); - } else if (std::holds_alternative(_table)) { - return std::get(_table).alloc_size; - } - return 0; -} - -std::size_t gpu_table_representation::get_uncompressed_data_size_in_bytes() const -{ - return get_size_in_bytes(); -} - -cudf::table_view gpu_table_representation::get_table_view() const -{ - if (std::holds_alternative>(_table)) { - return std::get>(_table)->view(); - } else { - return std::get(_table).view; - } -} - -std::unique_ptr gpu_table_representation::release_table( - [[maybe_unused]] rmm::cuda_stream_view stream) -{ - if (std::holds_alternative(_table)) { - _table = std::make_unique(std::get(_table).view, stream); - } - return std::move(std::get>(_table)); -} - -std::unique_ptr gpu_table_representation::clone(rmm::cuda_stream_view stream) -{ - // Create a deep copy of the cuDF table using the provided stream. - // STREAM-LINEAGE: the clone has been written by `stream`; record an event on - // it so any cross-stream/cross-device reader of the clone honors the - // producer-consumer ordering established by record_writer_event(). - cudf::table_view view = get_table_view(); - auto cloned = std::make_unique( - std::make_unique(view, stream), get_memory_space(), stream); - return cloned; -} - -void gpu_table_representation::record_writer_event(rmm::cuda_stream_view writer_stream) -{ - // STREAM-LINEAGE: lazily allocate the event handle on first call. The event - // uses cudaEventDisableTiming for cheaper record/wait (we never query the - // elapsed time on it — it's used solely for cross-stream ordering). - if (_writer_event == nullptr) { - CUCASCADE_CUDA_TRY(cudaEventCreateWithFlags(&_writer_event, cudaEventDisableTiming)); - } - CUCASCADE_CUDA_TRY(cudaEventRecord(_writer_event, writer_stream.value())); -} - -cudaEvent_t gpu_table_representation::get_writer_event() const { return _writer_event; } - -} // namespace cucascade diff --git a/src/data/representation_converter.cpp b/src/data/representation_converter.cpp index 161001e..18c17d7 100644 --- a/src/data/representation_converter.cpp +++ b/src/data/representation_converter.cpp @@ -15,54 +15,15 @@ * limitations under the License. */ -#include "cudf/contiguous_split.hpp" - -#include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include +#include #include #include +#include #include -#include namespace cucascade { @@ -137,1793 +98,4 @@ void representation_converter_registry::clear() _converters.clear(); } -// ============================================================================= -// Built-in converter implementations -// ============================================================================= - -namespace { - -// Forward declaration. convert_gpu_to_gpu is defined below convert_gpu_to_host_fast -// so it can reuse BatchCopyAccumulator and the column-tree reconstruction helpers, -// peer-copying each column buffer directly and avoiding cudf::pack (whose internal -// scratch allocator races with the cucascade resource ref under multi-GPU and -// produces uninitialized dst_buf_info reads inside compute_splits). -std::unique_ptr convert_gpu_to_gpu( - idata_representation& source, - const memory::memory_space* target_memory_space, - rmm::cuda_stream_view stream); - -/** - * @brief Convert gpu_table_representation to host_data_packed_representation - */ -std::unique_ptr convert_gpu_to_host( - idata_representation& source, - const memory::memory_space* target_memory_space, - rmm::cuda_stream_view stream) -{ - // Synchronize the stream to ensure any prior operations (like table creation) - // are complete before we read from the source table - stream.synchronize(); - - auto& gpu_source = source.cast(); - auto packed_data = cudf::pack(gpu_source.get_table_view(), stream); - - auto mr = target_memory_space->get_memory_resource_as(); - auto allocation = mr->allocate_multiple_blocks(packed_data.gpu_data->size()); - - size_t block_index = 0; - size_t block_offset = 0; - size_t source_offset = 0; - const size_t block_size = allocation->block_size(); - while (source_offset < packed_data.gpu_data->size()) { - size_t remaining_bytes = packed_data.gpu_data->size() - source_offset; - size_t bytes_to_copy = std::min(remaining_bytes, block_size - block_offset); - std::span block_ptr = allocation->at(block_index); - CUCASCADE_CUDA_TRY( - cudaMemcpyAsync(block_ptr.data() + block_offset, - static_cast(packed_data.gpu_data->data()) + source_offset, - bytes_to_copy, - cudaMemcpyDeviceToHost, - stream.value())); - source_offset += bytes_to_copy; - block_offset += bytes_to_copy; - if (block_offset == block_size) { - block_index++; - block_offset = 0; - } - } - stream.synchronize(); - auto host_table_packed_allocation = std::make_unique( - std::move(allocation), std::move(packed_data.metadata), packed_data.gpu_data->size()); - return std::make_unique( - std::move(host_table_packed_allocation), - const_cast(target_memory_space)); -} - -/** - * @brief Convert host_data_packed_representation to gpu_table_representation - */ -std::unique_ptr convert_host_to_gpu( - idata_representation& source, - const memory::memory_space* target_memory_space, - rmm::cuda_stream_view stream) -{ - auto& host_source = source.cast(); - auto& host_table = host_source.get_host_table(); - auto const data_size = host_table->data_size; - - auto mr = target_memory_space->get_default_allocator(); - rmm::cuda_set_device_raii device_guard{rmm::cuda_device_id{target_memory_space->get_device_id()}}; - - rmm::device_buffer dst_buffer(data_size, stream, mr); - size_t src_block_index = 0; - size_t src_block_offset = 0; - size_t dst_offset = 0; - size_t const src_block_size = host_table->allocation->block_size(); - while (dst_offset < data_size) { - size_t remaining_bytes = data_size - dst_offset; - size_t bytes_available_in_src_block = src_block_size - src_block_offset; - size_t bytes_to_copy = std::min(remaining_bytes, bytes_available_in_src_block); - auto src_block = host_table->allocation->at(src_block_index); - CUCASCADE_CUDA_TRY(cudaMemcpyAsync(static_cast(dst_buffer.data()) + dst_offset, - src_block.data() + src_block_offset, - bytes_to_copy, - cudaMemcpyHostToDevice, - stream.value())); - dst_offset += bytes_to_copy; - src_block_offset += bytes_to_copy; - if (src_block_offset == src_block_size) { - src_block_index++; - src_block_offset = 0; - } - } - - auto new_gpu_data = std::make_unique(std::move(dst_buffer)); - // cudf::unpack only reads metadata to produce a non-owning table_view — no GPU work. - // Use source metadata directly since it remains alive for the duration of this call. - // The H→D memcpy and table copy are on the same stream, so ordering is guaranteed. - auto new_table_view = - cudf::unpack(host_table->metadata->data(), static_cast(new_gpu_data->data())); - auto new_table = std::make_unique(new_table_view, stream, mr); - stream.synchronize(); - - // STREAM-LINEAGE: the resulting representation was written by `stream`; - // record an event on it so cross-stream/cross-device readers honor producer - // ordering. - return std::make_unique( - std::move(new_table), *const_cast(target_memory_space), stream); -} - -/** - * @brief Convert host_data_packed_representation to host_data_packed_representation (cross-host - * copy) - */ -std::unique_ptr convert_host_to_host( - idata_representation& source, - const memory::memory_space* target_memory_space, - rmm::cuda_stream_view /*stream*/) -{ - auto& host_source = source.cast(); - auto& host_table = host_source.get_host_table(); - auto const data_size = host_table->data_size; - - assert(source.get_device_id() != target_memory_space->get_device_id()); - auto mr = target_memory_space->get_memory_resource_as(); - if (mr == nullptr) { - throw std::runtime_error( - "Target HOST memory_space does not have a fixed_size_host_memory_resource"); - } - auto dst_allocation = mr->allocate_multiple_blocks(data_size); - size_t src_block_index = 0; - size_t src_block_offset = 0; - size_t dst_block_index = 0; - size_t dst_block_offset = 0; - size_t const src_block_size = host_table->allocation->block_size(); - size_t const dst_block_size = dst_allocation->block_size(); - size_t copied = 0; - while (copied < data_size) { - size_t remaining = data_size - copied; - size_t src_avail = src_block_size - src_block_offset; - size_t dst_avail = dst_block_size - dst_block_offset; - size_t bytes_to_copy = std::min(remaining, std::min(src_avail, dst_avail)); - auto* src_ptr = host_table->allocation->at(src_block_index).data() + src_block_offset; - auto* dst_ptr = dst_allocation->at(dst_block_index).data() + dst_block_offset; - std::memcpy(dst_ptr, src_ptr, bytes_to_copy); - copied += bytes_to_copy; - src_block_offset += bytes_to_copy; - dst_block_offset += bytes_to_copy; - if (src_block_offset == src_block_size) { - src_block_index++; - src_block_offset = 0; - } - if (dst_block_offset == dst_block_size) { - dst_block_index++; - dst_block_offset = 0; - } - } - auto metadata_copy = std::make_unique>(*host_table->metadata); - auto host_table_packed_allocation = std::make_unique( - std::move(dst_allocation), std::move(metadata_copy), data_size); - return std::make_unique( - std::move(host_table_packed_allocation), - const_cast(target_memory_space)); -} - -// ============================================================================= -// Fast GPU -> Host converter (direct buffer copy, no cudf::pack intermediate) -// ============================================================================= - -/** - * @brief Align a byte offset up to the given alignment (must be a power of two). - */ -static std::size_t align_up_fast(std::size_t offset, std::size_t alignment) noexcept -{ - return (offset + alignment - 1u) & ~(alignment - 1u); -} - -/** - * @brief Returns true for column types that store element data in a flat device buffer. - * - * STRING, LIST, STRUCT, and EMPTY have no flat data buffer of their own; their payload - * lives in their children. Everything else (fixed-width types, DICTIONARY32) has one. - */ -static bool column_has_data_buffer(const cudf::column_view& col) noexcept -{ - switch (col.type().id()) { - case cudf::type_id::STRING: - case cudf::type_id::LIST: - case cudf::type_id::STRUCT: - case cudf::type_id::DICTIONARY32: - case cudf::type_id::EMPTY: return false; - default: return true; - } -} - -/** - * @brief Returns the byte size of one element in a column's data buffer. - * - * For DICTIONARY32 the element type is always int32_t (the index type). - * For all other types with a data buffer, delegates to cudf::size_of. - */ -static std::size_t element_size_bytes(const cudf::column_view& col) -{ - return cudf::size_of(col.type()); -} - -/** - * @brief Accumulates (dst, src, size) copy operations for a single cudaMemcpyBatchAsync call. - * - * Collect all copies first, then call flush() once to submit them all to the GPU in one driver - * call. This eliminates per-buffer driver overhead versus issuing individual cudaMemcpyAsync calls. - */ -struct BatchCopyAccumulator { - std::vector dsts; - std::vector srcs; - std::vector sizes; - - void add(void* dst, const void* src, std::size_t size) - { - if (size == 0 || src == nullptr || dst == nullptr) { return; } - dsts.push_back(dst); - srcs.push_back(src); - sizes.push_back(size); - } - - std::size_t count() const { return dsts.size(); } - - void flush(rmm::cuda_stream_view stream, cudaMemcpySrcAccessOrder src_order) - { - if (count() == 0) { return; } -#if CUDART_VERSION >= 12080 - cudaMemcpyAttributes attr{}; - attr.srcAccessOrder = src_order; - attr.flags = cudaMemcpyFlagDefault; - // Single-attribute template overload (cuda_runtime.h): deduces direction from pointer types. - // NOTE: cudaMemcpyBatchAsync requires a real (non-default) CUDA stream. - // CUDA 12.x has a failIdx parameter that was removed in CUDA 13. -#if CUDART_VERSION < 13000 - CUCASCADE_CUDA_TRY(cudaMemcpyBatchAsync( - dsts.data(), srcs.data(), sizes.data(), count(), attr, nullptr, stream.value())); -#else - CUCASCADE_CUDA_TRY( - cudaMemcpyBatchAsync(dsts.data(), srcs.data(), sizes.data(), count(), attr, stream.value())); -#endif -#else - // cudaMemcpyBatchAsync requires CUDA 12.8+; fall back to individual copies. - (void)src_order; - for (std::size_t i = 0; i < count(); ++i) { - CUCASCADE_CUDA_TRY( - cudaMemcpyAsync(dsts[i], srcs[i], sizes[i], cudaMemcpyDefault, stream.value())); - } -#endif - // Clear so subsequent add()+flush() cycles do not resubmit already-issued ops. - dsts.clear(); - srcs.clear(); - sizes.clear(); - } -}; - -/** - * @brief Recursively plan the buffer layout for one column, filling in column_metadata. - * - * Advances @p current_offset by the bytes needed for the column's null mask and data buffer - * (8-byte aligned), then recurses into any children. - * - * @note We assume the column view has offset == 0 (i.e., the table is not a slice). - */ -static memory::column_metadata plan_column_copy(const cudf::column_view& col, - std::size_t& current_offset, - rmm::cuda_stream_view stream) -{ - assert(col.offset() == 0 && "column_view with non-zero offset is not supported"); - - memory::column_metadata meta{}; - meta.type_id = col.type().id(); - meta.num_rows = col.size(); - meta.null_count = col.null_count(); - meta.scale = 0; - - if (meta.type_id == cudf::type_id::DECIMAL32 || meta.type_id == cudf::type_id::DECIMAL64 || - meta.type_id == cudf::type_id::DECIMAL128) { - meta.scale = col.type().scale(); - } - - // Null mask - if (col.nullable()) { - meta.has_null_mask = true; - meta.null_mask_size = cudf::bitmask_allocation_size_bytes(col.size()); - current_offset = align_up_fast(current_offset, 8u); - meta.null_mask_offset = current_offset; - current_offset += meta.null_mask_size; - } else { - meta.has_null_mask = false; - meta.null_mask_offset = 0; - meta.null_mask_size = 0; - } - - // Flat data buffer - if (col.type().id() == cudf::type_id::STRING) { - // STRING stores chars in the column's data buffer; child(0) holds the offsets, - // which may be INT32 or INT64 (large strings). Use strings_column_view::chars_size() - // to correctly handle both offset types without a raw cudaMemcpy. - if (col.size() > 0 && col.num_children() > 0 && col.data() != nullptr) { - auto const chars_bytes = cudf::strings_column_view(col).chars_size(stream); - if (chars_bytes > 0) { - current_offset = align_up_fast(current_offset, 8u); - meta.has_data = true; - meta.data_offset = current_offset; - meta.data_size = static_cast(chars_bytes); - current_offset += meta.data_size; - } else { - meta.has_data = false; - meta.data_offset = 0; - meta.data_size = 0; - } - } else { - meta.has_data = false; - meta.data_offset = 0; - meta.data_size = 0; - } - } else if (column_has_data_buffer(col) && col.size() > 0) { - // Fixed-width types and DICTIONARY32 indices - meta.has_data = true; - meta.data_size = static_cast(col.size()) * element_size_bytes(col); - current_offset = align_up_fast(current_offset, 8u); - meta.data_offset = current_offset; - current_offset += meta.data_size; - } else { - meta.has_data = false; - meta.data_offset = 0; - meta.data_size = 0; - } - - // Recurse into children (STRING offsets+chars, LIST offsets+values, STRUCT fields, etc.) - meta.children.reserve(static_cast(col.num_children())); - for (cudf::size_type i = 0; i < col.num_children(); ++i) { - meta.children.push_back(plan_column_copy(col.child(i), current_offset, stream)); - } - - return meta; -} - -/** - * @brief Append block-boundary-split copy ops for @p size device bytes → host allocation. - * - * Does NOT issue any CUDA calls; the caller fires one cudaMemcpyBatchAsync after collecting all - * ops. - */ -static void collect_d2h_ops( - const void* src, - std::size_t size, - std::size_t alloc_offset, - memory::fixed_size_host_memory_resource::multiple_blocks_allocation& alloc, - BatchCopyAccumulator& batch) -{ - if (size == 0 || src == nullptr) { return; } - - const std::size_t block_size = alloc.block_size(); - std::size_t block_idx = alloc_offset / block_size; - std::size_t block_off = alloc_offset % block_size; - std::size_t src_off = 0; - - while (src_off < size) { - std::size_t remaining = size - src_off; - std::size_t space_in_block = block_size - block_off; - std::size_t bytes_to_copy = std::min(remaining, space_in_block); - - auto block = alloc.at(block_idx); - batch.add(block.data() + block_off, static_cast(src) + src_off, bytes_to_copy); - src_off += bytes_to_copy; - block_off += bytes_to_copy; - if (block_off == block_size) { - ++block_idx; - block_off = 0; - } - } -} - -/** - * @brief Recursively collect D→H copy ops for a column's null mask, data buffer, and children. - */ -static void collect_column_d2h_ops( - const cudf::column_view& col, - const memory::column_metadata& meta, - memory::fixed_size_host_memory_resource::multiple_blocks_allocation& alloc, - BatchCopyAccumulator& batch) -{ - if (meta.has_null_mask) { - collect_d2h_ops(col.null_mask(), meta.null_mask_size, meta.null_mask_offset, alloc, batch); - } - if (meta.has_data) { - collect_d2h_ops(col.data(), meta.data_size, meta.data_offset, alloc, batch); - } - for (cudf::size_type i = 0; i < col.num_children(); ++i) { - collect_column_d2h_ops(col.child(i), meta.children[static_cast(i)], alloc, batch); - } -} - -/** - * @brief Convert gpu_table_representation to host_data_representation. - * - * Copies each column's device buffers directly to pinned host memory, bypassing the - * intermediate GPU-side contiguous buffer that cudf::pack allocates. - */ -std::unique_ptr convert_gpu_to_host_fast( - idata_representation& source, - const memory::memory_space* target_memory_space, - rmm::cuda_stream_view stream) -{ - auto& gpu_source = source.cast(); - const cudf::table_view view = gpu_source.get_table_view(); - - // --- Pass 1: plan the allocation layout --- - std::size_t current_offset = 0; - std::vector columns; - columns.reserve(static_cast(view.num_columns())); - for (cudf::size_type i = 0; i < view.num_columns(); ++i) { - columns.push_back(plan_column_copy(view.column(i), current_offset, stream)); - } - const std::size_t total_size = current_offset; - - // --- Pass 2: allocate pinned host blocks --- - auto mr = target_memory_space->get_memory_resource_as(); - auto allocation = mr->allocate_multiple_blocks(total_size); - - // --- Pass 3: collect all D→H copy ops, then fire one batched call --- - BatchCopyAccumulator batch; - for (cudf::size_type i = 0; i < view.num_columns(); ++i) { - collect_column_d2h_ops( - view.column(i), columns[static_cast(i)], *allocation, batch); - } - batch.flush(stream, cudaMemcpySrcAccessOrderStream); - stream.synchronize(); - - auto host_alloc = - memory::host_table_allocation::create(std::move(allocation), std::move(columns), total_size); - - return std::make_unique( - std::move(host_alloc), const_cast(target_memory_space)); -} - -// ============================================================================= -// Fast GPU -> GPU converter (direct per-buffer peer copy, no cudf::pack) -// ============================================================================= - -/** - * @brief Best-effort lookup of the NUMA node a GPU's PCI device is attached to. - * Returns -1 if the device id is invalid, the PCI bus id is unknown, or - * /sys/bus/pci/devices//numa_node is unreadable / -1. Pure topology - * query — no allocation, no side effects. - */ -static int resolve_gpu_numa_node(int device_id) noexcept -{ - if (device_id < 0) { return -1; } - char pci_buf[32] = {0}; - if (cudaDeviceGetPCIBusId(pci_buf, sizeof(pci_buf), device_id) != cudaSuccess) { return -1; } - std::string pci(pci_buf); - for (auto& c : pci) { - c = static_cast(std::tolower(c)); - } - std::ifstream f("/sys/bus/pci/devices/" + pci + "/numa_node"); - if (!f.is_open()) { return -1; } - std::string content; - std::getline(f, content); - if (content.empty()) { return -1; } - try { - return std::stoi(content); - } catch (...) { - return -1; - } -} - -/** - * @brief Allocate a target-side device buffer and copy @p size bytes from @p src_ptr - * (on @p src_device) into it. - * - * Routing is decided per-pair by the empirical probe in cucascade::memory - * (see ensure_p2p_probed): - * - probe says peer DMA works → cudaMemcpyPeerAsync (direct device-to-device - * DMA over NVLink / supported PCIe) - * - probe says peer DMA broken → explicit host-staged copy through the - * pre-pinned HOST memory_space pool (registered by memory_space's HOST - * constructor in register_host_pool). The staging copy chunks across the - * pool's fixed block size (1 MB by default), exactly like - * convert_gpu_to_host. We do the staging ourselves rather than relying on - * the driver's auto-fallback, so the correctness path is identical and - * observable regardless of driver/CUDA version quirks. - * - * NEVER calls cudaHostAlloc per transfer — the pool is allocated once at - * memory_reservation_manager construction time. - */ -static rmm::device_buffer alloc_and_peer_copy_async(const void* src_ptr, - int src_device, - std::size_t size, - int dst_device, - rmm::cuda_stream_view target_stream, - rmm::device_async_resource_ref target_mr) -{ - rmm::device_buffer buf(size, target_stream, target_mr); - if (size == 0 || src_ptr == nullptr) { return buf; } - - if (memory::probe_peer_dma_works(src_device, dst_device)) { - // Real peer DMA works on this hardware — direct path. - CUCASCADE_CUDA_TRY(cudaMemcpyPeerAsync( - buf.data(), dst_device, src_ptr, src_device, size, target_stream.value())); - return buf; - } - - // Peer DMA broken — stage through the pre-pinned HOST pool. - // - // Block layout: the host pool allocates non-contiguous fixed-size blocks - // (1 MB by default). We chunk the D2H and H2D legs across blocks just like - // convert_gpu_to_host does. Stream-ordering inside the loop preserves the - // same-stream invariant the original implementation relied on. - memory::fixed_size_host_memory_resource* host_pool = - memory::find_host_pool(resolve_gpu_numa_node(dst_device)); - if (host_pool == nullptr) { - // No HOST memory_space has been registered. This should only happen in - // standalone test builds that exercise the converter without constructing - // a memory_reservation_manager. Fall back to a one-shot pinned allocation - // (slow path — preserves correctness, never reached in production Sirius). - memory::numa_region_pinned_host_memory_resource mr(resolve_gpu_numa_node(dst_device), - /*make_portable=*/true); - void* host_buf = mr.allocate_sync(size); - { - rmm::cuda_set_device_raii src_guard{rmm::cuda_device_id{src_device}}; - CUCASCADE_CUDA_TRY( - cudaMemcpyAsync(host_buf, src_ptr, size, cudaMemcpyDeviceToHost, target_stream.value())); - CUCASCADE_CUDA_TRY(cudaStreamSynchronize(target_stream.value())); - } - { - rmm::cuda_set_device_raii dst_guard{rmm::cuda_device_id{dst_device}}; - CUCASCADE_CUDA_TRY( - cudaMemcpyAsync(buf.data(), host_buf, size, cudaMemcpyHostToDevice, target_stream.value())); - CUCASCADE_CUDA_TRY(cudaStreamSynchronize(target_stream.value())); - } - mr.deallocate_sync(host_buf, size); - return buf; - } - - auto allocation = host_pool->allocate_multiple_blocks(size); - const std::size_t block_sz = allocation->block_size(); - - // Pass 1: device-to-host, chunked across blocks. Issued on target_stream - // under the src_device CUDA context (matches the same-stream invariant - // the original implementation enforced). - { - rmm::cuda_set_device_raii src_guard{rmm::cuda_device_id{src_device}}; - std::size_t block_index = 0; - std::size_t block_offset = 0; - std::size_t src_offset = 0; - while (src_offset < size) { - const std::size_t remaining = size - src_offset; - const std::size_t bytes_left = block_sz - block_offset; - const std::size_t to_copy = std::min(remaining, bytes_left); - std::span block = allocation->at(block_index); - CUCASCADE_CUDA_TRY(cudaMemcpyAsync(block.data() + block_offset, - static_cast(src_ptr) + src_offset, - to_copy, - cudaMemcpyDeviceToHost, - target_stream.value())); - src_offset += to_copy; - block_offset += to_copy; - if (block_offset == block_sz) { - ++block_index; - block_offset = 0; - } - } - CUCASCADE_CUDA_TRY(cudaStreamSynchronize(target_stream.value())); - } - - // Pass 2: host-to-device, chunked across the same blocks. Switch the CUDA - // context to dst_device — same guard the original code documented as - // required when peer DMA is broken and the outer guard does not propagate - // through the column tree walk. - { - rmm::cuda_set_device_raii dst_guard{rmm::cuda_device_id{dst_device}}; - std::size_t block_index = 0; - std::size_t block_offset = 0; - std::size_t dst_offset = 0; - while (dst_offset < size) { - const std::size_t remaining = size - dst_offset; - const std::size_t bytes_left = block_sz - block_offset; - const std::size_t to_copy = std::min(remaining, bytes_left); - std::span block = allocation->at(block_index); - CUCASCADE_CUDA_TRY(cudaMemcpyAsync(static_cast(buf.data()) + dst_offset, - block.data() + block_offset, - to_copy, - cudaMemcpyHostToDevice, - target_stream.value())); - dst_offset += to_copy; - block_offset += to_copy; - if (block_offset == block_sz) { - ++block_index; - block_offset = 0; - } - } - CUCASCADE_CUDA_TRY(cudaStreamSynchronize(target_stream.value())); - } - // allocation's destructor returns blocks to the host pool — no cudaFreeHost. - return buf; -} - -/** - * @brief Synchronous version of alloc_and_peer_copy_async. Used for null masks - * because cudf column factories may inspect them during column construction. - */ -static rmm::device_buffer alloc_and_peer_copy_sync(const void* src_ptr, - int src_device, - std::size_t size, - int dst_device, - rmm::cuda_stream_view target_stream, - rmm::device_async_resource_ref target_mr) -{ - auto buf = - alloc_and_peer_copy_async(src_ptr, src_device, size, dst_device, target_stream, target_mr); - if (size == 0 || src_ptr == nullptr) { return buf; } - target_stream.synchronize(); - return buf; -} - -/** - * @brief Recursively rebuild a cudf::column on the target device, peer-copying each - * leaf buffer (null mask, data, offsets, children) directly. Mirrors the structure - * of reconstruct_column() (host→gpu) but reads from a source cudf::column_view on a - * peer device using cudaMemcpyPeerAsync. - * - * Null masks are copied synchronously because cudf factories may read them during - * column construction. Data and offset buffers are issued asynchronously on - * @p target_stream; the caller must sync that stream before the resulting cudf::table - * is observed by another stream. - * - * @note Assumes the source column_view has offset == 0 (no slicing), matching the - * same constraint imposed by plan_column_copy() on the GPU↔Host fast path. - */ -static std::unique_ptr reconstruct_column_p2p(const cudf::column_view& src, - int src_device, - int dst_device, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - assert(src.offset() == 0 && "column_view with non-zero offset is not supported"); - - rmm::device_buffer null_mask{}; - if (src.nullable()) { - auto const null_mask_size = cudf::bitmask_allocation_size_bytes(src.size()); - null_mask = - alloc_and_peer_copy_sync(src.null_mask(), src_device, null_mask_size, dst_device, stream, mr); - } - cudf::size_type const null_count = src.nullable() ? src.null_count() : 0; - - if (src.type().id() == cudf::type_id::STRING) { - if (src.num_children() < 1) { - // Empty / degenerate STRING column with no offsets child (cudf produces - // these for empty intermediate results). Use cudf::make_empty_column - // which builds the correct internal layout (offsets child of size 1 - // holding [0], chars in parent's data buffer of size 0). - return cudf::make_empty_column(cudf::data_type{cudf::type_id::STRING}); - } - // Copy offsets as-is (preserve source's INT32 vs INT64 type). make_strings_column - // accepts either offset type. - auto offsets_col = reconstruct_column_p2p(src.child(0), src_device, dst_device, stream, mr); - - rmm::device_buffer chars_buf{}; - if (src.size() > 0 && src.data() != nullptr) { - // Read total chars size from offsets[size-1]. Reading from the *source* column - // here is unsafe: cudf::strings_column_view::chars_size() ultimately calls - // cudaMemcpyAsync(... cudaMemcpyDefault ...), which on cudaMallocAsync pool - // memory across devices silently returns success without copying bytes — the - // host buffer keeps its uninitialized stack value (often -1). Read instead - // from the target-side offsets buffer we just peer-copied; that's a same-device - // D→H read on the target stream and works reliably regardless of pool peer - // access semantics. - auto const offsets_view = offsets_col->view(); - auto const last_idx = offsets_view.size() - 1; - int64_t chars_bytes = 0; - stream.synchronize(); - if (offsets_view.type().id() == cudf::type_id::INT32) { - int32_t value = 0; - CUCASCADE_CUDA_TRY(cudaMemcpyAsync(&value, - offsets_view.head() + last_idx, - sizeof(int32_t), - cudaMemcpyDeviceToHost, - stream.value())); - stream.synchronize(); - chars_bytes = value; - } else { - int64_t value = 0; - CUCASCADE_CUDA_TRY(cudaMemcpyAsync(&value, - offsets_view.head() + last_idx, - sizeof(int64_t), - cudaMemcpyDeviceToHost, - stream.value())); - stream.synchronize(); - chars_bytes = value; - } - if (chars_bytes > 0) { - chars_buf = alloc_and_peer_copy_async(src.data(), - src_device, - static_cast(chars_bytes), - dst_device, - stream, - mr); - } - } - return cudf::make_strings_column( - src.size(), std::move(offsets_col), std::move(chars_buf), null_count, std::move(null_mask)); - } - - if (src.type().id() == cudf::type_id::LIST) { - if (src.num_children() < 2) { - throw std::invalid_argument( - "reconstruct_column_p2p: LIST column must have two children (offsets, values)"); - } - // Preserve source's offsets type — make_lists_column accepts INT32 or INT64. - auto offsets_col = reconstruct_column_p2p(src.child(0), src_device, dst_device, stream, mr); - auto values_col = reconstruct_column_p2p(src.child(1), src_device, dst_device, stream, mr); - return cudf::make_lists_column( - src.size(), std::move(offsets_col), std::move(values_col), null_count, std::move(null_mask)); - } - - if (src.type().id() == cudf::type_id::STRUCT) { - std::vector> fields; - fields.reserve(static_cast(src.num_children())); - for (cudf::size_type i = 0; i < src.num_children(); ++i) { - fields.push_back(reconstruct_column_p2p(src.child(i), src_device, dst_device, stream, mr)); - } - // Construct directly to skip make_structs_column's superimpose_nulls kernel — - // our peer-copied null masks are already consistent (mirrors reconstruct_column()). - return std::make_unique(cudf::data_type{cudf::type_id::STRUCT}, - src.size(), - rmm::device_buffer{}, - std::move(null_mask), - null_count, - std::move(fields)); - } - - if (src.type().id() == cudf::type_id::DICTIONARY32) { - if (src.num_children() < 2) { - throw std::invalid_argument( - "reconstruct_column_p2p: DICTIONARY32 column must have two children " - "(indices, keys)"); - } - // cudf DICTIONARY32 child order: [0]=indices, [1]=keys. - // make_dictionary_column parameter order: (keys, indices, ...). - auto indices_col = reconstruct_column_p2p(src.child(0), src_device, dst_device, stream, mr); - auto keys_col = reconstruct_column_p2p(src.child(1), src_device, dst_device, stream, mr); - return cudf::make_dictionary_column( - std::move(keys_col), std::move(indices_col), std::move(null_mask), null_count); - } - - // Fixed-width / fixed-point — single flat data buffer. - rmm::device_buffer data_buf{}; - if (src.size() > 0 && src.head() != nullptr) { - auto const data_size = static_cast(src.size()) * cudf::size_of(src.type()); - data_buf = alloc_and_peer_copy_async(src.head(), src_device, data_size, dst_device, stream, mr); - } - return std::make_unique( - src.type(), src.size(), std::move(data_buf), std::move(null_mask), null_count); -} - -/** - * @brief Convert gpu_table_representation to gpu_table_representation (cross-GPU copy). - * - * Replaces the previous cudf::pack-based path. cudf::pack's compute_splits internally - * launches kernels that read scratch buffers allocated through the current_device - * resource ref; under sirius's multi-GPU setup the cucascade resource adapter created - * a stream-ordered race that materialized as uninitialized dst_buf_info reads (caught - * by compute-sanitizer) and silently produced garbage packed bytes. This path mirrors - * convert_gpu_to_host_fast / convert_host_fast_to_gpu: walk the source column tree, - * allocate target-side buffers, and peer-copy each leaf buffer directly. - */ -std::unique_ptr convert_gpu_to_gpu( - idata_representation& source, - const memory::memory_space* target_memory_space, - rmm::cuda_stream_view stream) -{ - // Sync the caller's stream so the source table's buffers are stable on the source - // device before we issue peer copies. The caller's stream is the one that produced - // (or last touched) the source representation. - stream.synchronize(); - - auto& gpu_source = source.cast(); - - // Same-device case: clone via source's own clone() method. - if (source.get_device_id() == target_memory_space->get_device_id()) { - return source.clone(stream); - } - - auto const src_device_id = gpu_source.get_device_id(); - auto const dst_device_id = target_memory_space->get_device_id(); - - // STREAM-LINEAGE INVARIANT: cross-device peer copies of cudaMallocAsync - // allocations require explicit event-ordered synchronization with the - // writer stream. A source-device-wide cudaDeviceSynchronize() does NOT - // establish the cross-mempool visibility the driver needs — under - // compute-sanitizer this site emits hundreds of stream-ordered-race errors - // even with a brute-force device sync. Producer-consumer pairing: - // producer = the stream that wrote gpu_source (recorded via - // gpu_table_representation::record_writer_event) - // consumer = target_stream (acquired from target memory space below) - // We resolve this in two passes: - // 1) Wait on the writer event (if recorded) on the *target* stream so the - // reader sees the writer's allocation/copy ordering. This is the precise - // primitive the sanitizer recognizes as closing the race. - // 2) Keep the source-device cudaDeviceSynchronize() as defense-in-depth for - // callers that have not yet been migrated to record writer events - // (get_writer_event() == nullptr). When the writer event is set the - // cudaDeviceSynchronize is technically redundant but harmless. - cudaEvent_t const writer_event = gpu_source.get_writer_event(); - - rmm::cuda_set_device_raii target_guard{rmm::cuda_device_id{dst_device_id}}; - - // Target-bound stream from the target memory_space's stream pool. All peer copies - // and target-side allocations are issued on this stream so they observe in-order - // completion without explicit cross-stream events. - auto target_stream = target_memory_space->acquire_stream(); - auto mr = target_memory_space->get_default_allocator(); - - if (writer_event != nullptr) { - // STREAM-LINEAGE pass 1: tie the reader stream's timeline to the writer's - // recorded event. After this point the target_stream observes all - // writer-side cudaMallocAsync allocations and writes in proper order. - CUCASCADE_CUDA_TRY(cudaStreamWaitEvent(target_stream.value(), writer_event, 0)); - } else { - // STREAM-LINEAGE pass 2 (fallback): no writer event recorded — fall back to - // a coarser source-device sync. This path is documented as insufficient for - // cross-mempool cudaMallocAsync allocations but is preserved for - // representations produced by code paths that have not yet been migrated to - // record_writer_event(). - rmm::cuda_set_device_raii src_sync_guard{rmm::cuda_device_id{src_device_id}}; - CUCASCADE_CUDA_TRY(cudaDeviceSynchronize()); - } - - cudf::table_view const src_view = gpu_source.get_table_view(); - - std::vector> target_columns; - target_columns.reserve(static_cast(src_view.num_columns())); - for (cudf::size_type i = 0; i < src_view.num_columns(); ++i) { - target_columns.push_back( - reconstruct_column_p2p(src_view.column(i), src_device_id, dst_device_id, target_stream, mr)); - } - - auto new_table = std::make_unique(std::move(target_columns)); - // Sync so all peer copies and any cudf::cast launches complete before the new - // table is observed by another stream. - target_stream.synchronize(); - - // STREAM-LINEAGE: the resulting representation was written by target_stream; - // the constructor records an event on it so any subsequent cross-device - // reader of this new representation observes the producer-consumer ordering. - // Although we just synchronized target_stream above (so a same-stream reader - // needs no further wait), the event is still required for readers that - // arrive on a different stream. - return std::make_unique( - std::move(new_table), *const_cast(target_memory_space), target_stream); -} - -/** - * @brief Allocate a device buffer of @p size bytes and schedule its H→D copy ops into @p batch. - * - * The buffer is allocated immediately; the actual copies are deferred until the caller calls - * batch.flush(). The buffer must remain alive until flush() completes (it will, since it is - * owned by the column tree being assembled in the caller). - */ -static rmm::device_buffer alloc_and_schedule_h2d( - memory::fixed_size_host_memory_resource::multiple_blocks_allocation& alloc, - std::size_t alloc_offset, - std::size_t size, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr, - BatchCopyAccumulator& batch) -{ - rmm::device_buffer buf(size, stream, mr); - if (size == 0) { return buf; } - if (alloc.size() == 0) { - throw std::invalid_argument( - "alloc_and_schedule_h2d: allocation is empty but copy size is non-zero"); - } - - const std::size_t block_size = alloc.block_size(); - std::size_t block_idx = alloc_offset / block_size; - std::size_t block_off = alloc_offset % block_size; - std::size_t dst_off = 0; - - while (dst_off < size) { - std::size_t remaining = size - dst_off; - std::size_t space_in_block = block_size - block_off; - std::size_t bytes_to_copy = std::min(remaining, space_in_block); - - auto block = alloc.at(block_idx); - batch.add(static_cast(buf.data()) + dst_off, block.data() + block_off, bytes_to_copy); - dst_off += bytes_to_copy; - block_off += bytes_to_copy; - if (block_off == block_size) { - ++block_idx; - block_off = 0; - } - } - return buf; -} - -/** - * @brief Copy data from host block allocation to a device buffer synchronously. - * - * Unlike alloc_and_schedule_h2d, this performs the copies immediately and synchronizes - * the stream. Used for null masks which cudf column factories access on construction - * (before batch.flush() runs). - */ -static rmm::device_buffer alloc_and_copy_h2d_sync( - memory::fixed_size_host_memory_resource::multiple_blocks_allocation& alloc, - std::size_t alloc_offset, - std::size_t size, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - rmm::device_buffer buf(size, stream, mr); - if (size == 0) { return buf; } - - const std::size_t block_size = alloc.block_size(); - std::size_t block_idx = alloc_offset / block_size; - std::size_t block_off = alloc_offset % block_size; - std::size_t dst_off = 0; - - while (dst_off < size) { - std::size_t remaining = size - dst_off; - std::size_t space_in_block = block_size - block_off; - std::size_t bytes_to_copy = std::min(remaining, space_in_block); - - auto block = alloc.at(block_idx); - CUCASCADE_CUDA_TRY(cudaMemcpyAsync(static_cast(buf.data()) + dst_off, - block.data() + block_off, - bytes_to_copy, - cudaMemcpyHostToDevice, - stream.value())); - dst_off += bytes_to_copy; - block_off += bytes_to_copy; - if (block_off == block_size) { - ++block_idx; - block_off = 0; - } - } - stream.synchronize(); - return buf; -} - -/** - * @brief Recursively reconstruct a cudf::column from column_metadata and host data. - * - * Allocates device buffers and schedules their H→D copies into @p batch. The caller must call - * batch.flush() after all columns are reconstructed to actually issue the transfers. - * - * Null masks are copied synchronously because cudf column factories (make_structs_column, - * make_strings_column, etc.) may access them immediately on construction. - */ -static std::unique_ptr reconstruct_column( - const memory::column_metadata& meta, - memory::fixed_size_host_memory_resource::multiple_blocks_allocation& alloc, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr, - BatchCopyAccumulator& batch) -{ - // Null mask — copied synchronously because cudf factories access it on construction - rmm::device_buffer null_mask{}; - if (meta.has_null_mask) { - null_mask = - alloc_and_copy_h2d_sync(alloc, meta.null_mask_offset, meta.null_mask_size, stream, mr); - } - const cudf::size_type null_count = meta.has_null_mask ? meta.null_count : 0; - - if (meta.type_id == cudf::type_id::STRING) { - // cudf::make_empty_column(STRING) produces a 0-row strings column with no children, and - // plan_column_copy faithfully records that. Reconstruct the canonical empty strings column - // here instead of failing the offsets-child check below. - if (meta.children.empty() && meta.num_rows == 0) { - return cudf::make_empty_column(cudf::data_type{cudf::type_id::STRING}); - } - if (meta.children.size() < 1) { - throw std::invalid_argument( - "reconstruct_column: STRING column metadata must have at least one child (offsets)"); - } - auto offsets_col = reconstruct_column(meta.children[0], alloc, stream, mr, batch); - if (offsets_col->type().id() == cudf::type_id::INT32) { - // Flush pending H2D copies so the INT32 offsets buffer has valid data on device - // before the cast reads from it. The cast replaces offsets_col, freeing the old - // INT32 buffer, so batch must not hold dangling pointers to it. - batch.flush(stream, cudaMemcpySrcAccessOrderDuringApiCall); - offsets_col = - cudf::cast(offsets_col->view(), cudf::data_type{cudf::type_id::INT64}, stream, mr); - } - return cudf::make_strings_column( - meta.num_rows, - std::move(offsets_col), - meta.has_data && meta.data_size > 0 - ? alloc_and_schedule_h2d(alloc, meta.data_offset, meta.data_size, stream, mr, batch) - : rmm::device_buffer{}, - null_count, - std::move(null_mask)); - } - - if (meta.type_id == cudf::type_id::LIST) { - if (meta.children.size() < 2) { - throw std::invalid_argument( - "reconstruct_column: LIST column metadata must have two children (offsets, values)"); - } - auto offsets_col = reconstruct_column(meta.children[0], alloc, stream, mr, batch); - if (offsets_col->type().id() == cudf::type_id::INT32) { - // Flush pending H2D copies so the INT32 offsets buffer has valid data on device - // before the cast reads from it. See STRING branch comment above. - batch.flush(stream, cudaMemcpySrcAccessOrderDuringApiCall); - offsets_col = - cudf::cast(offsets_col->view(), cudf::data_type{cudf::type_id::INT64}, stream, mr); - } - return cudf::make_lists_column(meta.num_rows, - std::move(offsets_col), - reconstruct_column(meta.children[1], alloc, stream, mr, batch), - null_count, - std::move(null_mask)); - } - - if (meta.type_id == cudf::type_id::STRUCT) { - std::vector> fields; - fields.reserve(meta.children.size()); - for (const auto& child_meta : meta.children) { - fields.push_back(reconstruct_column(child_meta, alloc, stream, mr, batch)); - } - // Construct directly instead of make_structs_column to avoid superimpose_nulls - // kernel launch — our serialized data already has consistent null masks. - return std::make_unique(cudf::data_type{cudf::type_id::STRUCT}, - meta.num_rows, - rmm::device_buffer{}, - std::move(null_mask), - null_count, - std::move(fields)); - } - - if (meta.type_id == cudf::type_id::DICTIONARY32) { - if (meta.children.size() < 2) { - throw std::invalid_argument( - "reconstruct_column: DICTIONARY32 column metadata must have two children " - "(indices, keys)"); - } - // cudf DICTIONARY32 children order: [0]=indices, [1]=keys. - // make_dictionary_column parameter order: (keys, indices, ...). - auto indices_col = reconstruct_column(meta.children[0], alloc, stream, mr, batch); - auto keys_col = reconstruct_column(meta.children[1], alloc, stream, mr, batch); - return cudf::make_dictionary_column( - std::move(keys_col), std::move(indices_col), std::move(null_mask), null_count); - } - - const cudf::data_type dtype = cudf::is_fixed_point(cudf::data_type{meta.type_id}) - ? cudf::data_type{meta.type_id, meta.scale} - : cudf::data_type{meta.type_id}; - return std::make_unique( - dtype, - meta.num_rows, - meta.has_data && meta.data_size > 0 - ? alloc_and_schedule_h2d(alloc, meta.data_offset, meta.data_size, stream, mr, batch) - : rmm::device_buffer{}, - std::move(null_mask), - null_count); -} - -/** - * @brief Convert host_data_representation to gpu_table_representation. - * - * Reconstructs each column from the stored column_metadata tree, copying - * buffers from pinned host blocks directly to device allocations. - */ -std::unique_ptr convert_host_fast_to_gpu( - idata_representation& source, - const memory::memory_space* target_memory_space, - rmm::cuda_stream_view stream) -{ - auto& fast_source = source.cast(); - const auto& fast_table = fast_source.get_host_table(); - if (!fast_table) { throw std::runtime_error("convert_host_fast_to_gpu: host table is null"); } - if (!fast_table->allocation) { - throw std::runtime_error("convert_host_fast_to_gpu: host table allocation is null"); - } - - // Sync the caller's stream so any upstream work that produced this - // host_data_representation is flushed before we read the pinned host blocks. - // The caller's stream may be bound to a non-target device under multi-GPU; - // synchronize is safe across devices. - stream.synchronize(); - - rmm::cuda_set_device_raii device_guard{rmm::cuda_device_id{target_memory_space->get_device_id()}}; - - // Acquire a target-bound stream from the target memory_space's stream pool. - // Using the caller's stream for the H2D batch under a target-device RAII - // guard raises cudaErrorInvalidValue when stream and current device belong - // to different CUDA contexts (multi-GPU case). - auto target_stream = target_memory_space->acquire_stream(); - auto mr = target_memory_space->get_default_allocator(); - - // Collect all H→D copy ops across all columns, then fire one batched call. - BatchCopyAccumulator batch; - std::vector> gpu_columns; - gpu_columns.reserve(fast_table->columns.size()); - for (const auto& col_meta : fast_table->columns) { - gpu_columns.push_back( - reconstruct_column(col_meta, *fast_table->allocation, target_stream, mr, batch)); - } - // Source is CPU-written pinned host memory: fully prepared before this call. - batch.flush(target_stream, cudaMemcpySrcAccessOrderDuringApiCall); - - auto new_table = std::make_unique(std::move(gpu_columns)); - target_stream.synchronize(); - - // STREAM-LINEAGE: writes happened on target_stream; record event so - // cross-stream readers observe ordering. - return std::make_unique( - std::move(new_table), *const_cast(target_memory_space), target_stream); -} - -/** - * @brief Convert host_data_representation to host_data_representation (cross-host copy) - */ -std::unique_ptr convert_host_fast_to_host_fast( - idata_representation& source, - const memory::memory_space* target_memory_space, - rmm::cuda_stream_view /*stream*/) -{ - auto& host_source = source.cast(); - auto& host_table = host_source.get_host_table(); - auto const data_size = host_table->data_size; - - assert(source.get_device_id() != target_memory_space->get_device_id()); - auto mr = target_memory_space->get_memory_resource_as(); - if (mr == nullptr) { - throw std::runtime_error( - "Target HOST memory_space does not have a fixed_size_host_memory_resource"); - } - auto dst_allocation = mr->allocate_multiple_blocks(data_size); - size_t src_block_index = 0; - size_t src_block_offset = 0; - size_t dst_block_index = 0; - size_t dst_block_offset = 0; - size_t const src_block_size = host_table->allocation->block_size(); - size_t const dst_block_size = dst_allocation->block_size(); - size_t copied = 0; - while (copied < data_size) { - size_t remaining = data_size - copied; - size_t src_avail = src_block_size - src_block_offset; - size_t dst_avail = dst_block_size - dst_block_offset; - size_t bytes_to_copy = std::min(remaining, std::min(src_avail, dst_avail)); - auto* src_ptr = host_table->allocation->at(src_block_index).data() + src_block_offset; - auto* dst_ptr = dst_allocation->at(dst_block_index).data() + dst_block_offset; - std::memcpy(dst_ptr, src_ptr, bytes_to_copy); - copied += bytes_to_copy; - src_block_offset += bytes_to_copy; - dst_block_offset += bytes_to_copy; - if (src_block_offset == src_block_size) { - src_block_index++; - src_block_offset = 0; - } - if (dst_block_offset == dst_block_size) { - dst_block_index++; - dst_block_offset = 0; - } - } - std::vector columns_copy = host_table->columns; - auto new_host_table = memory::host_table_allocation::create( - std::move(dst_allocation), std::move(columns_copy), data_size); - return std::make_unique( - std::move(new_host_table), const_cast(target_memory_space)); -} - -// ============================================================================= -// Disk <-> Host converters -// ============================================================================= - -/** - * @brief RAII guard that deletes a file on destruction unless released. - * - * Ensures partial files are cleaned up if an exception occurs during writing. - */ -struct disk_file_guard { - std::string path; - bool released{false}; - ~disk_file_guard() - { - if (!released && !path.empty()) { - std::error_code ec; - std::filesystem::remove(path, ec); - } - } - void release() { released = true; } -}; - -/** - * @brief Write data from a host block allocation to a disk file, handling block-boundary spanning. - */ -static void write_host_buffer_to_disk( - const memory::fixed_size_host_memory_resource::multiple_blocks_allocation& alloc, - std::size_t alloc_offset, - std::size_t size, - const std::filesystem::path& file_path, - std::size_t file_offset, - idisk_io_backend& backend) -{ - const std::size_t block_size = alloc.block_size(); - std::size_t block_idx = alloc_offset / block_size; - std::size_t block_off = alloc_offset % block_size; - std::size_t written = 0; - while (written < size) { - std::size_t remaining = size - written; - std::size_t avail = block_size - block_off; - std::size_t chunk = std::min(remaining, avail); - auto block = alloc.at(block_idx); - backend.write(file_path, block.data() + block_off, chunk, file_offset + written); - written += chunk; - block_off += chunk; - if (block_off == block_size) { - ++block_idx; - block_off = 0; - } - } -} - -/** - * @brief Read data from a disk file into a host block allocation, handling block-boundary spanning. - */ -static void read_disk_buffer_to_host( - const std::filesystem::path& file_path, - std::size_t file_offset, - std::size_t size, - memory::fixed_size_host_memory_resource::multiple_blocks_allocation& alloc, - std::size_t alloc_offset, - idisk_io_backend& backend) -{ - const std::size_t block_size = alloc.block_size(); - std::size_t block_idx = alloc_offset / block_size; - std::size_t block_off = alloc_offset % block_size; - std::size_t read_total = 0; - while (read_total < size) { - std::size_t remaining = size - read_total; - std::size_t avail = block_size - block_off; - std::size_t chunk = std::min(remaining, avail); - auto block = alloc.at(block_idx); - backend.read(file_path, block.data() + block_off, chunk, file_offset + read_total); - read_total += chunk; - block_off += chunk; - if (block_off == block_size) { - ++block_idx; - block_off = 0; - } - } -} - -/** - * @brief Recompute column_metadata offsets for 4KB-aligned disk file layout. - * - * Copies all non-offset fields from src, then assigns null_mask_offset and data_offset - * to 4KB-aligned positions within the file. The cursor is advanced past each buffer. - */ -static memory::column_metadata recompute_file_offsets(const memory::column_metadata& src, - std::size_t& cursor) -{ - memory::column_metadata dst{}; - dst.type_id = src.type_id; - dst.num_rows = src.num_rows; - dst.null_count = src.null_count; - dst.scale = src.scale; - - dst.has_null_mask = src.has_null_mask; - dst.null_mask_size = src.null_mask_size; - if (src.has_null_mask && src.null_mask_size > 0) { - cursor = rmm::align_up(cursor, DISK_FILE_ALIGNMENT); - dst.null_mask_offset = cursor; - cursor += src.null_mask_size; - } else { - dst.null_mask_offset = 0; - } - - dst.has_data = src.has_data; - dst.data_size = src.data_size; - if (src.has_data && src.data_size > 0) { - cursor = rmm::align_up(cursor, DISK_FILE_ALIGNMENT); - dst.data_offset = cursor; - cursor += src.data_size; - } else { - dst.data_offset = 0; - } - - dst.children.reserve(src.children.size()); - for (const auto& child : src.children) { - dst.children.push_back(recompute_file_offsets(child, cursor)); - } - return dst; -} - -/** - * @brief Recursively write column buffers from host blocks to disk at computed file offsets. - */ -static void write_column_buffers( - const memory::column_metadata& src_meta, - const memory::column_metadata& disk_meta, - const memory::fixed_size_host_memory_resource::multiple_blocks_allocation& alloc, - const std::filesystem::path& file_path, - idisk_io_backend& backend) -{ - if (src_meta.has_null_mask && src_meta.null_mask_size > 0) { - write_host_buffer_to_disk(alloc, - src_meta.null_mask_offset, - src_meta.null_mask_size, - file_path, - disk_meta.null_mask_offset, - backend); - } - if (src_meta.has_data && src_meta.data_size > 0) { - write_host_buffer_to_disk( - alloc, src_meta.data_offset, src_meta.data_size, file_path, disk_meta.data_offset, backend); - } - for (std::size_t i = 0; i < src_meta.children.size(); ++i) { - write_column_buffers(src_meta.children[i], disk_meta.children[i], alloc, file_path, backend); - } -} - -/** - * @brief Recompute column_metadata offsets for 8-byte-aligned host block layout. - * - * Used when reconstructing host_table_allocation from disk data. - */ -static memory::column_metadata recompute_host_offsets(const memory::column_metadata& src, - std::size_t& cursor) -{ - memory::column_metadata dst{}; - dst.type_id = src.type_id; - dst.num_rows = src.num_rows; - dst.null_count = src.null_count; - dst.scale = src.scale; - - dst.has_null_mask = src.has_null_mask; - dst.null_mask_size = src.null_mask_size; - if (src.has_null_mask && src.null_mask_size > 0) { - cursor = align_up_fast(cursor, 8u); - dst.null_mask_offset = cursor; - cursor += src.null_mask_size; - } else { - dst.null_mask_offset = 0; - } - - dst.has_data = src.has_data; - dst.data_size = src.data_size; - if (src.has_data && src.data_size > 0) { - cursor = align_up_fast(cursor, 8u); - dst.data_offset = cursor; - cursor += src.data_size; - } else { - dst.data_offset = 0; - } - - dst.children.reserve(src.children.size()); - for (const auto& child : src.children) { - dst.children.push_back(recompute_host_offsets(child, cursor)); - } - return dst; -} - -/** - * @brief Recursively read column buffers from disk into host blocks. - */ -static void read_column_buffers( - const memory::column_metadata& disk_meta, - const memory::column_metadata& host_meta, - const std::filesystem::path& file_path, - memory::fixed_size_host_memory_resource::multiple_blocks_allocation& alloc, - idisk_io_backend& backend) -{ - if (disk_meta.has_null_mask && disk_meta.null_mask_size > 0) { - read_disk_buffer_to_host(file_path, - disk_meta.null_mask_offset, - disk_meta.null_mask_size, - alloc, - host_meta.null_mask_offset, - backend); - } - if (disk_meta.has_data && disk_meta.data_size > 0) { - read_disk_buffer_to_host( - file_path, disk_meta.data_offset, disk_meta.data_size, alloc, host_meta.data_offset, backend); - } - for (std::size_t i = 0; i < disk_meta.children.size(); ++i) { - read_column_buffers(disk_meta.children[i], host_meta.children[i], file_path, alloc, backend); - } -} - -/** - * @brief Convert host_data_representation to disk_data_representation. - * - * Writes a complete disk file: header + serialized column_metadata + 4KB-aligned column data. - * An RAII file guard ensures partial files are cleaned up on exception. - */ -static std::unique_ptr convert_host_data_to_disk( - idata_representation& source, - const memory::memory_space* target_memory_space, - [[maybe_unused]] rmm::cuda_stream_view stream) -{ - auto& backend = target_memory_space->get_io_backend(); - auto& host_source = source.cast(); - const auto& host_table = host_source.get_host_table(); - - // Generate unique file path under the disk memory space's mount directory - auto mount_path = target_memory_space->get_disk_mount_path(); - auto file_path = memory::generate_disk_file_path(mount_path); - - disk_file_guard guard{file_path}; - - // Recompute column metadata with 4KB-aligned file offsets starting at offset 0. - // No header or serialized metadata is written — metadata stays in memory. - std::size_t cursor = 0; - std::vector disk_columns; - disk_columns.reserve(host_table->columns.size()); - for (const auto& col : host_table->columns) { - disk_columns.push_back(recompute_file_offsets(col, cursor)); - } - std::size_t total_file_data_size = cursor; - - // Write column data buffers - for (std::size_t i = 0; i < host_table->columns.size(); ++i) { - write_column_buffers( - host_table->columns[i], disk_columns[i], *host_table->allocation, file_path, backend); - } - - guard.release(); - - auto disk_table = std::make_unique( - std::move(file_path), std::move(disk_columns), total_file_data_size); - return std::make_unique( - std::move(disk_table), const_cast(*target_memory_space)); -} - -/** - * @brief Convert disk_data_representation to host_data_representation. - * - * Uses in-memory column metadata from the source disk representation, - * allocates host blocks, and reads data buffers from the disk file. - */ -static std::unique_ptr convert_disk_to_host_data( - idata_representation& source, - const memory::memory_space* target_memory_space, - [[maybe_unused]] rmm::cuda_stream_view stream) -{ - auto& backend = source.get_memory_space().get_io_backend(); - auto& disk_source = source.cast(); - const auto& disk_table = disk_source.get_disk_table(); - const auto& file_path = disk_table.file_path; - - // Column metadata is already in memory — no file header or metadata to read - const auto& disk_columns = disk_table.columns; - - // Compute host allocation size with 8-byte alignment - std::size_t host_cursor = 0; - std::vector host_columns; - host_columns.reserve(disk_columns.size()); - for (const auto& col : disk_columns) { - host_columns.push_back(recompute_host_offsets(col, host_cursor)); - } - std::size_t total_host_size = host_cursor; - - // Allocate host blocks - auto mr = target_memory_space->get_memory_resource_as(); - if (mr == nullptr) { - throw std::runtime_error( - "Target HOST memory_space does not have a fixed_size_host_memory_resource"); - } - auto allocation = mr->allocate_multiple_blocks(total_host_size); - - // Read column data from file into host blocks - for (std::size_t i = 0; i < disk_columns.size(); ++i) { - read_column_buffers(disk_columns[i], host_columns[i], file_path, *allocation, backend); - } - - auto host_alloc = memory::host_table_allocation::create( - std::move(allocation), std::move(host_columns), total_host_size); - return std::make_unique( - std::move(host_alloc), const_cast(target_memory_space)); -} - -// ============================================================================= -// Disk <-> GPU converters -// ============================================================================= - -/** - * @brief Recursively collect GPU column buffer I/O entries for batch submission. - * - * Instead of writing each buffer individually, collects all (ptr, size, file_offset) - * tuples into a vector for a single batch write_batch call. - */ -static void collect_gpu_column_io_entries(const cudf::column_view& col, - const memory::column_metadata& disk_meta, - std::vector& entries) -{ - if (disk_meta.has_null_mask && disk_meta.null_mask_size > 0) { - entries.push_back({col.null_mask(), disk_meta.null_mask_size, disk_meta.null_mask_offset}); - } - if (disk_meta.has_data && disk_meta.data_size > 0) { - entries.push_back({col.data(), disk_meta.data_size, disk_meta.data_offset}); - } - for (cudf::size_type i = 0; i < col.num_children(); ++i) { - collect_gpu_column_io_entries( - col.child(i), disk_meta.children[static_cast(i)], entries); - } -} - -/** - * @brief Convert gpu_table_representation to disk_data_representation. - * - * Writes GPU table buffers directly to disk via write_batch. - * File contains only 4KB-aligned column data — metadata stays in memory. - */ -static std::unique_ptr convert_gpu_to_disk( - idata_representation& source, - const memory::memory_space* target_memory_space, - rmm::cuda_stream_view stream) -{ - auto& backend = target_memory_space->get_io_backend(); - auto& gpu_source = source.cast(); - cudf::table_view tv = gpu_source.get_table_view(); - - // Generate unique file path under the disk memory space's mount directory - auto mount_path = target_memory_space->get_disk_mount_path(); - auto file_path = memory::generate_disk_file_path(mount_path); - - disk_file_guard guard{file_path}; - - // Plan column layout with 8-byte aligned offsets (same as host path) - std::size_t plan_offset = 0; - std::vector planned_columns; - planned_columns.reserve(static_cast(tv.num_columns())); - for (cudf::size_type i = 0; i < tv.num_columns(); ++i) { - planned_columns.push_back(plan_column_copy(tv.column(i), plan_offset, stream)); - } - - // Recompute with 4KB-aligned disk offsets starting at offset 0. - // No header or serialized metadata is written — metadata stays in memory. - std::size_t cursor = 0; - std::vector disk_columns; - disk_columns.reserve(planned_columns.size()); - for (const auto& col : planned_columns) { - disk_columns.push_back(recompute_file_offsets(col, cursor)); - } - std::size_t total_file_data_size = cursor; - - // Collect column data I/O entries - std::vector io_entries; - for (cudf::size_type i = 0; i < tv.num_columns(); ++i) { - collect_gpu_column_io_entries( - tv.column(i), disk_columns[static_cast(i)], io_entries); - } - - // Single file open, single batch submit - backend.write_batch(file_path, io_entries, stream); - - guard.release(); - - auto disk_table = std::make_unique( - std::move(file_path), std::move(disk_columns), total_file_data_size); - return std::make_unique( - std::move(disk_table), const_cast(*target_memory_space)); -} - -/** - * @brief Allocate an rmm::device_buffer and read data from disk directly into it. - */ -static rmm::device_buffer alloc_and_read_from_disk(const std::filesystem::path& file_path, - std::size_t file_offset, - std::size_t size, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr, - idisk_io_backend& backend) -{ - rmm::device_buffer buf(size, stream, mr); - if (size > 0) { backend.read(file_path, buf.data(), size, file_offset, stream); } - return buf; -} - -/** - * @brief Recursively reconstruct a cudf::column from disk column_metadata, reading directly - * from disk into device buffers via read. - */ -static std::unique_ptr reconstruct_column_from_disk( - const memory::column_metadata& meta, - const std::filesystem::path& file_path, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr, - idisk_io_backend& backend) -{ - // Null mask (shared by all type categories) - rmm::device_buffer null_mask{}; - if (meta.has_null_mask) { - null_mask = alloc_and_read_from_disk( - file_path, meta.null_mask_offset, meta.null_mask_size, stream, mr, backend); - } - const cudf::size_type null_count = meta.has_null_mask ? meta.null_count : 0; - - if (meta.type_id == cudf::type_id::STRING) { - // See reconstruct_column for the empty-strings-column rationale. cudf::make_empty_column( - // STRING) produces a column with no children; recreate it instead of erroring. - if (meta.children.empty() && meta.num_rows == 0) { - return cudf::make_empty_column(cudf::data_type{cudf::type_id::STRING}); - } - if (meta.children.size() < 1) { - throw std::invalid_argument( - "reconstruct_column_from_disk: STRING column metadata must have at least one child " - "(offsets)"); - } - // Reconstruct offsets and cast to INT64 if needed (cudf 26.06+ requires INT64 offsets) - auto offsets_col = - reconstruct_column_from_disk(meta.children[0], file_path, stream, mr, backend); - if (offsets_col->type().id() == cudf::type_id::INT32) { - offsets_col = - cudf::cast(offsets_col->view(), cudf::data_type{cudf::type_id::INT64}, stream, mr); - } - return cudf::make_strings_column( - meta.num_rows, - std::move(offsets_col), - meta.has_data && meta.data_size > 0 - ? alloc_and_read_from_disk(file_path, meta.data_offset, meta.data_size, stream, mr, backend) - : rmm::device_buffer{}, - null_count, - std::move(null_mask)); - } - - if (meta.type_id == cudf::type_id::LIST) { - if (meta.children.size() < 2) { - throw std::invalid_argument( - "reconstruct_column_from_disk: LIST column metadata must have two children (offsets, " - "values)"); - } - auto offsets_col = - reconstruct_column_from_disk(meta.children[0], file_path, stream, mr, backend); - if (offsets_col->type().id() == cudf::type_id::INT32) { - offsets_col = - cudf::cast(offsets_col->view(), cudf::data_type{cudf::type_id::INT64}, stream, mr); - } - return cudf::make_lists_column( - meta.num_rows, - std::move(offsets_col), - reconstruct_column_from_disk(meta.children[1], file_path, stream, mr, backend), - null_count, - std::move(null_mask)); - } - - if (meta.type_id == cudf::type_id::STRUCT) { - std::vector> fields; - fields.reserve(meta.children.size()); - for (const auto& child_meta : meta.children) { - fields.push_back(reconstruct_column_from_disk(child_meta, file_path, stream, mr, backend)); - } - return std::make_unique(cudf::data_type{cudf::type_id::STRUCT}, - meta.num_rows, - rmm::device_buffer{}, - std::move(null_mask), - null_count, - std::move(fields)); - } - - if (meta.type_id == cudf::type_id::DICTIONARY32) { - if (meta.children.size() < 2) { - throw std::invalid_argument( - "reconstruct_column_from_disk: DICTIONARY32 column metadata must have two children " - "(indices, keys)"); - } - // cudf DICTIONARY32 children order: [0]=indices, [1]=keys. - // make_dictionary_column parameter order: (keys, indices, ...). - auto indices_col = - reconstruct_column_from_disk(meta.children[0], file_path, stream, mr, backend); - auto keys_col = reconstruct_column_from_disk(meta.children[1], file_path, stream, mr, backend); - return cudf::make_dictionary_column( - std::move(keys_col), std::move(indices_col), std::move(null_mask), null_count); - } - - const cudf::data_type dtype = cudf::is_fixed_point(cudf::data_type{meta.type_id}) - ? cudf::data_type{meta.type_id, meta.scale} - : cudf::data_type{meta.type_id}; - return std::make_unique( - dtype, - meta.num_rows, - meta.has_data && meta.data_size > 0 - ? alloc_and_read_from_disk(file_path, meta.data_offset, meta.data_size, stream, mr, backend) - : rmm::device_buffer{}, - std::move(null_mask), - null_count); -} - -/** - * @brief Convert disk_data_representation to gpu_table_representation. - * - * Reads column data directly from disk into GPU device buffers. - * Column metadata comes from the in-memory disk_table_allocation. - */ -static std::unique_ptr convert_disk_to_gpu( - idata_representation& source, - const memory::memory_space* target_memory_space, - rmm::cuda_stream_view stream) -{ - auto& backend = source.get_memory_space().get_io_backend(); - auto& disk_source = source.cast(); - const auto& disk_table = disk_source.get_disk_table(); - const auto& file_path = disk_table.file_path; - - // Column metadata is already in memory — no file header or metadata to read - const auto& disk_columns = disk_table.columns; - - // Set CUDA device to target GPU (RAII restores previous device on scope exit) - rmm::cuda_set_device_raii device_guard{rmm::cuda_device_id{target_memory_space->get_device_id()}}; - - auto mr = target_memory_space->get_default_allocator(); - - // Reconstruct columns by reading directly from disk into device buffers - std::vector> gpu_columns; - gpu_columns.reserve(disk_columns.size()); - for (const auto& col_meta : disk_columns) { - gpu_columns.push_back(reconstruct_column_from_disk(col_meta, file_path, stream, mr, backend)); - } - - stream.synchronize(); - - auto new_table = std::make_unique(std::move(gpu_columns)); - // STREAM-LINEAGE: writes happened on `stream`; record event so cross-stream - // readers observe ordering. - return std::make_unique( - std::move(new_table), *const_cast(target_memory_space), stream); -} - -} // namespace - -void register_builtin_converters(representation_converter_registry& registry) -{ - // GPU -> GPU (cross-device copy). The convert_gpu_to_gpu implementation uses - // cudaMemcpyPeerAsync for each column buffer. Whether that takes the direct - // peer-DMA fast path or falls back to the driver's host-stage path is - // decided by cucascade::memory::ensure_p2p_probed() at the first - // memory_space construction — the probe runs once per process, detects the - // "lying enable" failure mode on consumer Intel chipsets, and disables peer - // access for any GPU pair where direct DMA does not actually move bytes. - registry.register_converter( - convert_gpu_to_gpu); - - // GPU -> HOST - registry.register_converter( - convert_gpu_to_host); - - // HOST -> GPU - registry.register_converter( - convert_host_to_gpu); - - // HOST -> HOST (cross-device copy) - registry.register_converter( - convert_host_to_host); - - // GPU -> HOST FAST (direct buffer copy, no intermediate GPU allocation) - registry.register_converter( - convert_gpu_to_host_fast); - - // HOST FAST -> GPU - registry.register_converter( - convert_host_fast_to_gpu); - - // HOST FAST -> HOST FAST (cross-device copy) - registry.register_converter( - convert_host_fast_to_host_fast); - - // HOST DATA -> DISK (backend resolved from target disk memory_space) - registry.register_converter( - convert_host_data_to_disk); - - // DISK -> HOST DATA (backend resolved from source disk memory_space) - registry.register_converter( - convert_disk_to_host_data); - - // GPU -> DISK (backend resolved from target disk memory_space) - registry.register_converter( - convert_gpu_to_disk); - - // DISK -> GPU (backend resolved from source disk memory_space) - registry.register_converter( - convert_disk_to_gpu); -} - } // namespace cucascade diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8fa4322..5abc006 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -26,22 +26,16 @@ FetchContent_MakeAvailable(Catch2) # Create test executable add_executable( cucascade_tests - # Utils - utils/cudf_test_utils.cpp # Memory tests memory/test_memory_reservation_manager.cpp memory/test_small_pinned_host_memory_resource.cpp memory/test_gpu_kernels.cu # Data tests - data/test_bandwidth_profiler.cpp data/test_data_batch.cpp data/test_data_repository.cpp data/test_data_repository_manager.cpp - data/test_data_representation.cpp data/test_disk_io_backend.cpp data/test_io_worker.cpp - data/test_disk_host_converters.cpp - data/test_gpu_disk_converters.cpp data/test_representation_converter.cpp # Main test runner unittest.cpp) diff --git a/test/data/test_bandwidth_profiler.cpp b/test/data/test_bandwidth_profiler.cpp deleted file mode 100644 index dfefadf..0000000 --- a/test/data/test_bandwidth_profiler.cpp +++ /dev/null @@ -1,443 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "utils/mock_test_utils.hpp" - -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace cucascade; -using cucascade::data::bandwidth_profile; -using cucascade::data::bandwidth_profile_config; -using cucascade::data::measure_bandwidth; -using cucascade::test::make_mock_memory_space; - -// Shared memory spaces across the bandwidth profiler test cases — avoids the CUDA-context -// degradation we've seen from repeated memory_space creation/destruction in the converter tests. -namespace { - -auto& shared_gpu_space() -{ - static auto s = make_mock_memory_space(memory::Tier::GPU, 0); - return s; -} - -auto& shared_host_space() -{ - static auto s = make_mock_memory_space(memory::Tier::HOST, 0); - return s; -} - -auto& shared_disk_space() -{ - static auto s = make_mock_memory_space(memory::Tier::DISK, 0); - return s; -} - -bandwidth_profile_config tiny_config() -{ - bandwidth_profile_config cfg; - cfg.test_sizes_bytes = {64ull * 1024}; // single 64 KiB size keeps the tests fast - cfg.warmup_iterations = 1; - cfg.timed_iterations = 2; - cfg.measure_disk_pairs = true; - return cfg; -} - -} // namespace - -// ============================================================================= -// chunked_resource_info mixin detection -// ============================================================================= - -TEST_CASE("chunked_resource_info is exposed by fixed_size_host_memory_resource", - "[bandwidth_profiler][chunked_resource_info]") -{ - // A HOST memory_space uses fixed_size_host_memory_resource as its reservation allocator, which - // inherits chunked_resource_info. GPU and DISK spaces do not. - auto& gpu = shared_gpu_space(); - auto& host = shared_host_space(); - auto& disk = shared_disk_space(); - - CHECK(gpu->get_chunked_resource_info() == nullptr); - CHECK(disk->get_chunked_resource_info() == nullptr); - - auto const* host_info = host->get_chunked_resource_info(); - REQUIRE(host_info != nullptr); - CHECK(host_info->max_chunk_bytes() > 0); -} - -// ============================================================================= -// Input validation -// ============================================================================= - -TEST_CASE("measure_bandwidth rejects input without a GPU space", "[bandwidth_profiler]") -{ - representation_converter_registry registry; - register_builtin_converters(registry); - - auto& host = shared_host_space(); - auto& disk = shared_disk_space(); - std::array spaces_array = {host.get(), disk.get()}; - std::span spaces{spaces_array}; - - REQUIRE_THROWS_AS(measure_bandwidth(spaces, registry, tiny_config()), std::invalid_argument); -} - -// ============================================================================= -// Pair enumeration rules — GPU/HOST/DISK, no disk-to-disk, no self, bidirectional -// ============================================================================= - -TEST_CASE("measure_bandwidth enumerates pairs with bidirectional entries and no disk-to-disk", - "[bandwidth_profiler]") -{ - representation_converter_registry registry; - register_builtin_converters(registry); - - auto& gpu = shared_gpu_space(); - auto& host = shared_host_space(); - auto& disk = shared_disk_space(); - auto& disk2 = []() -> auto& { - static auto s = make_mock_memory_space(memory::Tier::DISK, 1); - return s; - }(); - - std::array spaces_array = {gpu.get(), host.get(), disk.get(), disk2.get()}; - std::span spaces{spaces_array}; - - auto profile = measure_bandwidth(spaces, registry, tiny_config()); - - // No self-pair and no disk-to-disk. - for (auto const& pair : profile.pairs) { - CHECK(pair.src != pair.dst); - CHECK_FALSE((pair.src.tier == memory::Tier::DISK && pair.dst.tier == memory::Tier::DISK)); - } - - // Bidirectional: whenever (A -> B) is present, (B -> A) must also be present (even if - // one direction is marked unavailable). - for (auto const& ab : profile.pairs) { - auto const* ba = profile.find(ab.dst, ab.src); - INFO("missing reverse pair for " << static_cast(ab.src.tier) << "->" - << static_cast(ab.dst.tier)); - CHECK(ba != nullptr); - } - - // GPU-HOST pair should be present in both directions. - CHECK(profile.find(gpu->get_id(), host->get_id()) != nullptr); - CHECK(profile.find(host->get_id(), gpu->get_id()) != nullptr); - // GPU-DISK pair present. - CHECK(profile.find(gpu->get_id(), disk->get_id()) != nullptr); - CHECK(profile.find(disk->get_id(), gpu->get_id()) != nullptr); - // disk-to-disk absent. - CHECK(profile.find(disk->get_id(), disk2->get_id()) == nullptr); - CHECK(profile.find(disk2->get_id(), disk->get_id()) == nullptr); -} - -// ============================================================================= -// Size override is honored -// ============================================================================= - -TEST_CASE("measure_bandwidth records only the configured sizes per pair", "[bandwidth_profiler]") -{ - representation_converter_registry registry; - register_builtin_converters(registry); - - auto& gpu = shared_gpu_space(); - auto& host = shared_host_space(); - std::array spaces_arr = {gpu.get(), host.get()}; - std::span spaces{spaces_arr}; - - bandwidth_profile_config cfg; - cfg.test_sizes_bytes = {8ull * 1024, 64ull * 1024}; - cfg.warmup_iterations = 1; - cfg.timed_iterations = 2; - cfg.measure_disk_pairs = false; - - auto profile = measure_bandwidth(spaces, registry, cfg); - - auto const* gh = profile.find(gpu->get_id(), host->get_id()); - REQUIRE(gh != nullptr); - if (gh->converter_available) { - CHECK(gh->per_size.size() == cfg.test_sizes_bytes.size()); - for (auto sz : cfg.test_sizes_bytes) { - CHECK(gh->per_size.count(sz) == 1); - } - } -} - -// ============================================================================= -// Chunked detection is reflected in result metadata -// ============================================================================= - -TEST_CASE("per-pair result records chunk size for chunked source/destination spaces", - "[bandwidth_profiler][chunked_resource_info]") -{ - representation_converter_registry registry; - register_builtin_converters(registry); - - auto& gpu = shared_gpu_space(); - auto& host = shared_host_space(); - std::array spaces_arr = {gpu.get(), host.get()}; - std::span spaces{spaces_arr}; - - auto profile = measure_bandwidth(spaces, registry, tiny_config()); - - // GPU allocator is contiguous; HOST allocator is fixed_size_host_memory_resource (chunked). - // So for every pair the chunked byte count should be > 0 on the HOST side and 0 on the GPU side. - for (auto const& pair : profile.pairs) { - if (pair.src.tier == memory::Tier::HOST) { CHECK(pair.src_max_chunk_bytes > 0); } - if (pair.src.tier == memory::Tier::GPU) { CHECK(pair.src_max_chunk_bytes == 0); } - if (pair.dst.tier == memory::Tier::HOST) { CHECK(pair.dst_max_chunk_bytes > 0); } - if (pair.dst.tier == memory::Tier::GPU) { CHECK(pair.dst_max_chunk_bytes == 0); } - } -} - -// ============================================================================= -// Unavailable converter is reported, not thrown -// ============================================================================= - -TEST_CASE("pairs without a registered converter are reported as unavailable", - "[bandwidth_profiler]") -{ - // Empty registry — no converters registered. - representation_converter_registry registry; - - auto& gpu = shared_gpu_space(); - auto& host = shared_host_space(); - std::array spaces_arr = {gpu.get(), host.get()}; - std::span spaces{spaces_arr}; - - auto profile = measure_bandwidth(spaces, registry, tiny_config()); - - // All enumerated pairs should be marked unavailable with a non-empty reason. - REQUIRE_FALSE(profile.pairs.empty()); - for (auto const& pair : profile.pairs) { - CHECK_FALSE(pair.converter_available); - CHECK_FALSE(pair.unavailable_reason.empty()); - CHECK(pair.per_size.empty()); - } - - // Summary lookup returns 0 for unavailable pairs. - CHECK(profile.gbps(gpu->get_id(), host->get_id()) == 0.0); - CHECK_FALSE(profile.sample(gpu->get_id(), host->get_id(), 64ull * 1024).has_value()); -} - -// ============================================================================= -// End-to-end smoke: real converters produce positive throughput -// ============================================================================= - -// ============================================================================= -// Hidden: prints the full bandwidth matrix for the provided spaces. -// -// Run with: ./build/release/test/cucascade_tests "[bandwidth_matrix]" -// Or: pixi run test -- "[bandwidth_matrix]" -// -// The tag leads with `.` so it's excluded from the default `[~@nonunit]` run. -// ============================================================================= - -namespace { - -std::string format_space_id(memory::memory_space_id id) -{ - std::ostringstream os; - switch (id.tier) { - case memory::Tier::GPU: os << "GPU"; break; - case memory::Tier::HOST: os << "HOST"; break; - case memory::Tier::DISK: os << "DISK"; break; - default: os << "???"; break; - } - os << ":" << id.device_id; - return os.str(); -} - -std::string format_bytes(std::size_t bytes) -{ - std::ostringstream os; - if (bytes >= (1ull << 30)) { - os << (bytes / (1ull << 30)) << " GiB"; - } else if (bytes >= (1ull << 20)) { - os << (bytes / (1ull << 20)) << " MiB"; - } else if (bytes >= (1ull << 10)) { - os << (bytes / (1ull << 10)) << " KiB"; - } else { - os << bytes << " B"; - } - return os.str(); -} - -} // namespace - -TEST_CASE("print bandwidth matrix", "[.bandwidth_matrix]") -{ - representation_converter_registry registry; - register_builtin_converters(registry); - - int device_count = 0; - cudaGetDeviceCount(&device_count); - - auto& gpu = shared_gpu_space(); - auto& host = shared_host_space(); - auto& disk = shared_disk_space(); - - // Discover additional GPUs at runtime so the matrix test works on 1-GPU and N-GPU hosts. - std::vector> extra_gpus; - for (int d = 1; d < device_count; ++d) { - extra_gpus.push_back(make_mock_memory_space(memory::Tier::GPU, static_cast(d))); - } - - std::vector spaces_vec; - spaces_vec.push_back(gpu.get()); - for (auto& s : extra_gpus) { - spaces_vec.push_back(s.get()); - } - spaces_vec.push_back(host.get()); - spaces_vec.push_back(disk.get()); - std::span spaces{spaces_vec}; - - cucascade::data::bandwidth_profile_config cfg; - cfg.test_sizes_bytes = {1ull << 20, 16ull << 20, 64ull << 20}; - cfg.warmup_iterations = 2; - cfg.timed_iterations = 5; - cfg.measure_disk_pairs = true; - cfg.drop_page_cache_between_iters = true; - - auto profile = measure_bandwidth(spaces, registry, cfg); - - std::ostringstream out; - out << "\n\n=== Bandwidth Profile ===\n"; - out << "Detected " << device_count << " CUDA device(s); page-cache eviction " - << (cfg.drop_page_cache_between_iters ? "ON" : "off") << "\n"; - out << "Spaces:\n"; - for (std::size_t i = 0; i < spaces_vec.size(); ++i) { - auto const* mr_info = spaces_vec[i]->get_chunked_resource_info(); - out << " [" << i << "] " << format_space_id(spaces_vec[i]->get_id()); - if (mr_info != nullptr) { - out << " (chunked, " << format_bytes(mr_info->max_chunk_bytes()) << " blocks)"; - } else { - out << " (contiguous)"; - } - out << "\n"; - } - - out << "\nSummary GB/s (row = src, column = dst)\n"; - out << std::setw(12) << " "; - for (auto* dst : spaces_vec) { - out << std::setw(12) << format_space_id(dst->get_id()); - } - out << "\n"; - for (auto* src : spaces_vec) { - out << std::setw(12) << format_space_id(src->get_id()); - for (auto* dst : spaces_vec) { - if (src == dst) { - out << std::setw(12) << "-"; - } else { - auto const* pair = profile.find(src->get_id(), dst->get_id()); - if (pair == nullptr) { - out << std::setw(12) << "n/a"; - } else if (!pair->converter_available) { - out << std::setw(12) << "no-conv"; - } else { - std::ostringstream cell; - cell << std::fixed << std::setprecision(2) << pair->summary.gbps; - out << std::setw(12) << cell.str(); - } - } - } - out << "\n"; - } - - out << "\nPer-size detail (median-picked summary marked *):\n"; - for (auto const& pair : profile.pairs) { - out << " " << format_space_id(pair.src) << " -> " << format_space_id(pair.dst); - if (!pair.converter_available) { - out << " [unavailable: " << pair.unavailable_reason << "]\n"; - continue; - } - out << " (summary " << std::fixed << std::setprecision(2) << pair.summary.gbps << " GB/s)\n"; - for (auto const& [size_bytes, sample] : pair.per_size) { - bool is_summary = (sample.gbps == pair.summary.gbps); - out << " " << std::setw(8) << format_bytes(size_bytes) << ": " << std::fixed - << std::setprecision(2) << std::setw(7) << sample.gbps << " GB/s (" - << std::setprecision(3) << (sample.mean_seconds * 1e3) << " ms/iter, " - << sample.iterations_timed << " iters)" << (is_summary ? " *" : "") << "\n"; - } - } - out << "\n"; - - std::cerr << out.str(); - - // Also write to a stable file path so we can view the matrix from outside ctest - // (which suppresses stdout on success). - const char* out_path_env = std::getenv("CUCASCADE_BANDWIDTH_MATRIX_OUT"); - std::filesystem::path out_path = - out_path_env != nullptr ? out_path_env : "/tmp/cucascade_bandwidth_matrix.txt"; - { - std::ofstream f(out_path); - f << out.str(); - } - - // Sanity check so the test can still fail if the profiler produced nothing. - REQUIRE_FALSE(profile.pairs.empty()); -} - -TEST_CASE("measure_bandwidth produces positive gbps for GPU<->HOST with builtin converters", - "[bandwidth_profiler][integration]") -{ - representation_converter_registry registry; - register_builtin_converters(registry); - - auto& gpu = shared_gpu_space(); - auto& host = shared_host_space(); - std::array spaces_arr = {gpu.get(), host.get()}; - std::span spaces{spaces_arr}; - - auto profile = measure_bandwidth(spaces, registry, tiny_config()); - - auto const* gh = profile.find(gpu->get_id(), host->get_id()); - auto const* hg = profile.find(host->get_id(), gpu->get_id()); - REQUIRE(gh != nullptr); - REQUIRE(hg != nullptr); - - if (gh->converter_available) { - CHECK(gh->summary.gbps > 0.0); - CHECK(gh->summary.mean_seconds > 0.0); - CHECK(gh->summary.bytes_transferred > 0); - } - if (hg->converter_available) { - CHECK(hg->summary.gbps > 0.0); - CHECK(hg->summary.mean_seconds > 0.0); - } -} diff --git a/test/data/test_data_batch.cpp b/test/data/test_data_batch.cpp index 83de964..cf8e45d 100644 --- a/test/data/test_data_batch.cpp +++ b/test/data/test_data_batch.cpp @@ -15,11 +15,9 @@ * limitations under the License. */ -#include "utils/cudf_test_utils.hpp" #include "utils/mock_test_utils.hpp" #include -#include #include #include @@ -41,8 +39,6 @@ #include using namespace cucascade; -using cucascade::test::create_simple_cudf_table; -using cucascade::test::expect_cudf_tables_equal_on_stream; using cucascade::test::make_mock_memory_space; using cucascade::test::mock_data_representation; @@ -548,162 +544,6 @@ TEST_CASE("data_batch clone preserves tier information", "[data_batch]") } } -// ============================================================================= -// Real GPU data clone tests (TEST-05) -// ============================================================================= - -TEST_CASE("data_batch clone with real GPU data verifies data integrity", "[data_batch][gpu]") -{ - auto gpu_space = make_mock_memory_space(memory::Tier::GPU, 0); - rmm::cuda_stream stream; - - auto table = create_simple_cudf_table(100, 2, gpu_space->get_default_allocator(), stream.view()); - auto original_rows = table.num_rows(); - auto original_columns = table.num_columns(); - - auto gpu_repr = std::make_unique( - std::make_unique(std::move(table)), *gpu_space, rmm::cuda_stream_view{}); - auto batch = std::make_shared(1, std::move(gpu_repr)); - - auto ro = batch->to_read_only(); - auto cloned = ro.clone(2, stream.view()); - REQUIRE(cloned != nullptr); - REQUIRE(cloned->get_batch_id() == 2); - - auto ro_clone = cloned->to_read_only(); - - auto* original_repr = dynamic_cast(ro.get_data()); - auto* cloned_repr = dynamic_cast(ro_clone.get_data()); - REQUIRE(original_repr != nullptr); - REQUIRE(cloned_repr != nullptr); - - // Verify table shape matches - REQUIRE(cloned_repr->get_table_view().num_rows() == original_rows); - REQUIRE(cloned_repr->get_table_view().num_columns() == original_columns); - - stream.synchronize(); - expect_cudf_tables_equal_on_stream( - original_repr->get_table_view(), cloned_repr->get_table_view(), stream.view()); -} - -TEST_CASE("data_batch clone creates independent memory copies", "[data_batch][gpu]") -{ - auto gpu_space = make_mock_memory_space(memory::Tier::GPU, 0); - rmm::cuda_stream stream; - - auto table = create_simple_cudf_table(50, 2, gpu_space->get_default_allocator(), stream.view()); - auto gpu_repr = std::make_unique( - std::make_unique(std::move(table)), *gpu_space, rmm::cuda_stream_view{}); - auto batch = std::make_shared(1, std::move(gpu_repr)); - - auto ro = batch->to_read_only(); - auto cloned = ro.clone(2, stream.view()); - - auto ro_clone = cloned->to_read_only(); - - auto* original_repr = dynamic_cast(ro.get_data()); - auto* cloned_repr = dynamic_cast(ro_clone.get_data()); - - // Verify each column points to different memory - for (cudf::size_type i = 0; i < original_repr->get_table_view().num_columns(); ++i) { - REQUIRE(original_repr->get_table_view().column(i).head() != - cloned_repr->get_table_view().column(i).head()); - } -} - -TEST_CASE("data_batch multiple clones are all independent", "[data_batch][gpu]") -{ - auto gpu_space = make_mock_memory_space(memory::Tier::GPU, 0); - rmm::cuda_stream stream; - - auto table = create_simple_cudf_table(30, 2, gpu_space->get_default_allocator(), stream.view()); - auto gpu_repr = std::make_unique( - std::make_unique(std::move(table)), *gpu_space, rmm::cuda_stream_view{}); - auto batch = std::make_shared(1, std::move(gpu_repr)); - - // Clone 3 times from the same read_only accessor (clone does not consume the accessor) - auto ro = batch->to_read_only(); - auto clone1 = ro.clone(10, stream.view()); - auto clone2 = ro.clone(20, stream.view()); - auto clone3 = ro.clone(30, stream.view()); - - REQUIRE(clone1->get_batch_id() == 10); - REQUIRE(clone2->get_batch_id() == 20); - REQUIRE(clone3->get_batch_id() == 30); - - auto ro_c1 = clone1->to_read_only(); - auto ro_c2 = clone2->to_read_only(); - auto ro_c3 = clone3->to_read_only(); - - auto* original_repr = dynamic_cast(ro.get_data()); - auto* clone1_repr = dynamic_cast(ro_c1.get_data()); - auto* clone2_repr = dynamic_cast(ro_c2.get_data()); - auto* clone3_repr = dynamic_cast(ro_c3.get_data()); - - stream.synchronize(); - expect_cudf_tables_equal_on_stream( - original_repr->get_table_view(), clone1_repr->get_table_view(), stream.view()); - expect_cudf_tables_equal_on_stream( - original_repr->get_table_view(), clone2_repr->get_table_view(), stream.view()); - expect_cudf_tables_equal_on_stream( - original_repr->get_table_view(), clone3_repr->get_table_view(), stream.view()); -} - -TEST_CASE("data_batch clone with empty table", "[data_batch][gpu]") -{ - auto gpu_space = make_mock_memory_space(memory::Tier::GPU, 0); - rmm::cuda_stream stream; - - auto table = create_simple_cudf_table(0, 2, gpu_space->get_default_allocator(), stream.view()); - auto gpu_repr = std::make_unique( - std::make_unique(std::move(table)), *gpu_space, rmm::cuda_stream_view{}); - auto batch = std::make_shared(1, std::move(gpu_repr)); - - auto ro = batch->to_read_only(); - auto cloned = ro.clone(2, stream.view()); - REQUIRE(cloned != nullptr); - - auto ro_clone = cloned->to_read_only(); - auto* cloned_repr = dynamic_cast(ro_clone.get_data()); - REQUIRE(cloned_repr != nullptr); - REQUIRE(cloned_repr->get_table_view().num_rows() == 0); - REQUIRE(cloned_repr->get_table_view().num_columns() == 2); -} - -TEST_CASE("data_batch clone with large table", "[data_batch][gpu]") -{ - auto gpu_space = make_mock_memory_space(memory::Tier::GPU, 0); - rmm::cuda_stream stream; - - auto table = - create_simple_cudf_table(10000, 2, gpu_space->get_default_allocator(), stream.view()); - auto gpu_repr = std::make_unique( - std::make_unique(std::move(table)), *gpu_space, rmm::cuda_stream_view{}); - auto batch = std::make_shared(1, std::move(gpu_repr)); - - auto ro = batch->to_read_only(); - auto cloned = ro.clone(2, stream.view()); - REQUIRE(cloned != nullptr); - - auto ro_clone = cloned->to_read_only(); - - auto* original_repr = dynamic_cast(ro.get_data()); - auto* cloned_repr = dynamic_cast(ro_clone.get_data()); - - // Verify structure - REQUIRE(cloned_repr->get_table_view().num_rows() == 10000); - REQUIRE(cloned_repr->get_table_view().num_columns() == 2); - - stream.synchronize(); - expect_cudf_tables_equal_on_stream( - original_repr->get_table_view(), cloned_repr->get_table_view(), stream.view()); - - for (cudf::size_type i = 0; i < original_repr->get_table_view().num_columns(); ++i) { - REQUIRE(original_repr->get_table_view().column(i).head() != - cloned_repr->get_table_view().column(i).head()); - } -} - // ============================================================================= // Observable state tests (batch_state) // ============================================================================= diff --git a/test/data/test_data_representation.cpp b/test/data/test_data_representation.cpp deleted file mode 100644 index 0f8d4a3..0000000 --- a/test/data/test_data_representation.cpp +++ /dev/null @@ -1,2286 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "utils/cudf_test_utils.hpp" -#include "utils/mock_test_utils.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -#include - -#include -#include -#include -#include -#include - -using namespace cucascade; -using cucascade::test::create_conversion_test_configs; -using cucascade::test::create_simple_cudf_table; -using cucascade::test::make_mock_memory_space; - -// Note: Tests that require mock host_table_packed_allocation are disabled because -// fixed_size_host_memory_resource::multiple_blocks_allocation is now private. -// The real allocation tests below use actual memory resources. -[[maybe_unused]] static constexpr bool MOCK_HOST_ALLOCATION_DISABLED = true; - -// ============================================================================= -// host_data_packed_representation Tests -// ============================================================================= - -// Disabled: requires internal access to multiple_blocks_allocation constructor -TEST_CASE("host_data_packed_representation Construction", "[cpu_data_representation][.disabled]") -{ - SUCCEED("Test disabled - requires internal API access"); -} - -// Disabled: requires internal access to multiple_blocks_allocation constructor -TEST_CASE("host_data_packed_representation get_size_in_bytes", - "[cpu_data_representation][.disabled]") -{ - SUCCEED("Test disabled - requires internal API access"); -} - -// Disabled: requires internal access to multiple_blocks_allocation constructor -TEST_CASE("host_data_packed_representation memory tier", "[cpu_data_representation][.disabled]") -{ - SUCCEED("Test disabled - requires internal API access"); -} - -// Disabled: requires internal access to multiple_blocks_allocation constructor -TEST_CASE("host_data_packed_representation device_id", "[cpu_data_representation][.disabled]") -{ - SUCCEED("Test disabled - requires internal API access"); -} - -TEST_CASE("host_data_packed_representation converts to GPU and preserves contents", - "[cpu_data_representation][gpu_data_representation]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const memory::memory_space* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - const memory::memory_space* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - - // Start from a known cudf table; pack it and build a host_data_packed_representation - // Use the same stream for table creation and packing to avoid stream-ordered races - rmm::cuda_stream pack_stream; - auto original = - create_simple_cudf_table(128, 2, gpu_space->get_default_allocator(), pack_stream.view()); - auto view = original.view(); - auto packed = cudf::pack(view, pack_stream.view()); - pack_stream.synchronize(); - auto host_mr = host_space->get_memory_resource_as(); - REQUIRE(host_mr != nullptr); - - // Copy device buffer to host allocation - auto allocation = host_mr->allocate_multiple_blocks(packed.gpu_data->size()); - size_t copied = 0; - size_t block_idx = 0; - size_t block_off = 0; - const size_t block_size = allocation->block_size(); - while (copied < packed.gpu_data->size()) { - size_t remain = packed.gpu_data->size() - copied; - size_t bytes = std::min(remain, block_size - block_off); - void* dst_ptr = reinterpret_cast((*allocation)[block_idx].data()) + block_off; - CUCASCADE_CUDA_TRY(cudaMemcpy(dst_ptr, - static_cast(packed.gpu_data->data()) + copied, - bytes, - cudaMemcpyDeviceToHost)); - copied += bytes; - block_off += bytes; - if (block_off == block_size) { - block_off = 0; - block_idx++; - } - } - - auto meta_copy = std::make_unique>(*packed.metadata); - auto host_alloc = std::make_unique( - std::move(allocation), std::move(meta_copy), packed.gpu_data->size()); - host_data_packed_representation host_repr(std::move(host_alloc), - const_cast(host_space)); - - // Convert to GPU and compare cudf tables - auto gpu_stream = gpu_space->acquire_stream(); - auto gpu_any = registry.convert(host_repr, gpu_space, pack_stream); - pack_stream.synchronize(); - auto& gpu_repr = *gpu_any; - // Compare using the same stream used for conversion to avoid cross-stream hazards - cucascade::test::expect_cudf_tables_equal_on_stream( - original, gpu_repr.get_table_view(), pack_stream.view()); -} - -// ============================================================================= -// gpu_table_representation Tests -// ============================================================================= - -TEST_CASE("gpu_table_representation Construction", "[gpu_data_representation]") -{ - auto gpu_space = make_mock_memory_space(memory::Tier::GPU, 0); - auto table = create_simple_cudf_table(100, gpu_space->get_default_allocator()); - - gpu_table_representation repr( - std::make_unique(std::move(table)), *gpu_space, rmm::cuda_stream_view{}); - - REQUIRE(repr.get_current_tier() == memory::Tier::GPU); - REQUIRE(repr.get_device_id() == 0); - REQUIRE(repr.get_size_in_bytes() > 0); -} - -TEST_CASE("gpu_table_representation get_size_in_bytes", "[gpu_data_representation]") -{ - auto gpu_space = make_mock_memory_space(memory::Tier::GPU, 0); - - SECTION("100 rows") - { - auto table = create_simple_cudf_table(100, gpu_space->get_default_allocator()); - gpu_table_representation repr( - std::make_unique(std::move(table)), *gpu_space, rmm::cuda_stream_view{}); - - // Size should be at least 100 rows * (4 bytes for INT32 + 8 bytes for INT64) - std::size_t expected_min_size = 100 * (4 + 8); - REQUIRE(repr.get_size_in_bytes() >= expected_min_size); - } - - SECTION("1000 rows") - { - auto table = create_simple_cudf_table(1000, gpu_space->get_default_allocator()); - gpu_table_representation repr( - std::make_unique(std::move(table)), *gpu_space, rmm::cuda_stream_view{}); - - // Size should be at least 1000 rows * (4 bytes for INT32 + 8 bytes for INT64) - std::size_t expected_min_size = 1000 * (4 + 8); - REQUIRE(repr.get_size_in_bytes() >= expected_min_size); - } - - SECTION("Empty table") - { - auto table = create_simple_cudf_table(0, gpu_space->get_default_allocator()); - gpu_table_representation repr( - std::make_unique(std::move(table)), *gpu_space, rmm::cuda_stream_view{}); - - REQUIRE(repr.get_size_in_bytes() == 0); - } -} - -TEST_CASE("gpu_table_representation get_table", "[gpu_data_representation]") -{ - auto gpu_space = make_mock_memory_space(memory::Tier::GPU, 0); - auto table = create_simple_cudf_table(100, gpu_space->get_default_allocator()); - - // Store the number of columns before moving the table - auto num_columns = table.num_columns(); - - gpu_table_representation repr( - std::make_unique(std::move(table)), *gpu_space, rmm::cuda_stream_view{}); - - const cudf::table_view& retrieved_table = repr.get_table_view(); - REQUIRE(retrieved_table.num_columns() == num_columns); - REQUIRE(retrieved_table.num_rows() == 100); -} - -TEST_CASE("gpu_table_representation memory tier", "[gpu_data_representation]") -{ - SECTION("GPU tier") - { - auto gpu_space = make_mock_memory_space(memory::Tier::GPU, 0); - auto table = create_simple_cudf_table(100, gpu_space->get_default_allocator()); - gpu_table_representation repr( - std::make_unique(std::move(table)), *gpu_space, rmm::cuda_stream_view{}); - - REQUIRE(repr.get_current_tier() == memory::Tier::GPU); - } -} - -TEST_CASE("gpu_table_representation device_id", "[gpu_data_representation]") -{ - SECTION("Device 0") - { - auto gpu_space = make_mock_memory_space(memory::Tier::GPU, 0); - auto table = create_simple_cudf_table(100, gpu_space->get_default_allocator()); - gpu_table_representation repr( - std::make_unique(std::move(table)), *gpu_space, rmm::cuda_stream_view{}); - - REQUIRE(repr.get_device_id() == 0); - } - - SECTION("Device 1") - { - int device_count = 0; - if (cudaGetDeviceCount(&device_count) != cudaSuccess || device_count < 2) { - SUCCEED("Single GPU or CUDA not available; skipping device 1 section"); - return; - } - - auto gpu_space = make_mock_memory_space(memory::Tier::GPU, 1); - auto table = create_simple_cudf_table(100, gpu_space->get_default_allocator()); - gpu_table_representation repr( - std::make_unique(std::move(table)), *gpu_space, rmm::cuda_stream_view{}); - - REQUIRE(repr.get_device_id() == 1); - } -} - -TEST_CASE("gpu->host->gpu roundtrip preserves cudf table contents", "[gpu_data_representation]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const memory::memory_space* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const memory::memory_space* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - - // Use one stream for table creation and both conversions to enforce order - // and avoid stream-ordered races - auto chain_stream = gpu_space->acquire_stream(); - auto table = create_simple_cudf_table(100, 2, gpu_space->get_default_allocator(), chain_stream); - gpu_table_representation repr(std::make_unique(std::move(table)), - *const_cast(gpu_space), - rmm::cuda_stream_view{}); - - auto cpu_any = registry.convert(repr, host_space, chain_stream); - auto gpu_any = registry.convert(*cpu_any, gpu_space, chain_stream); - - auto& back = *gpu_any; - chain_stream.synchronize(); - cucascade::test::expect_cudf_tables_equal_on_stream( - repr.get_table_view(), back.get_table_view(), chain_stream); - // Stream is automatically managed - no explicit release needed -} - -// ============================================================================= -// Multi-GPU Cross-Device Conversion Test -// ============================================================================= -static std::unique_ptr create_multi_gpu_manager(int dev_a, - int dev_b) -{ - using namespace cucascade::memory; - std::vector configs; - configs.emplace_back(gpu_memory_space_config(dev_a, 2048ull * 1024 * 1024)); - configs.emplace_back(gpu_memory_space_config(dev_b, 2048ull * 1024 * 1024)); - return std::make_unique(std::move(configs)); -} - -TEST_CASE("gpu cross-device conversion when multiple GPUs are available", - "[gpu_data_representation][.multi-device]") -{ - int device_count = 0; - if (cudaGetDeviceCount(&device_count) != cudaSuccess || device_count < 2) { - SUCCEED("Single GPU or CUDA not available; skipping cross-device test"); - return; - } - - // Pick first two GPUs - int dev_src = 0; - int dev_dst = 1; - - auto mgr = create_multi_gpu_manager(dev_src, dev_dst); - representation_converter_registry registry; - register_builtin_converters(registry); - - const memory::memory_space* src_space = mgr->get_memory_space(memory::Tier::GPU, dev_src); - const memory::memory_space* dst_space = mgr->get_memory_space(memory::Tier::GPU, dev_dst); - REQUIRE(src_space != nullptr); - REQUIRE(dst_space != nullptr); - - // Use a single stream for table creation and peer copy to avoid stream-ordered races - auto xfer_stream = src_space->acquire_stream(); - - // Build a simple cudf table on source GPU and wrap it - auto table = create_simple_cudf_table(256, 2, src_space->get_default_allocator(), xfer_stream); - gpu_table_representation src_repr(std::make_unique(std::move(table)), - *const_cast(src_space), - rmm::cuda_stream_view{}); - - auto dst_any = registry.convert(src_repr, dst_space, xfer_stream); - auto& dst_repr = *dst_any; - - // Compare content equality using the same stream used for transfer - cucascade::test::expect_cudf_tables_equal_on_stream( - src_repr.get_table_view(), dst_repr.get_table_view(), xfer_stream); -} - -// ============================================================================= -// gpu_table_representation built from cudf::table_view + shared_ptr -// ============================================================================= - -TEST_CASE("gpu->host_packed->gpu roundtrip preserves contents (table_view+shared_ptr ctor)", - "[gpu_data_representation][table_view_ctor]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const memory::memory_space* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const memory::memory_space* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - - auto chain_stream = gpu_space->acquire_stream(); - auto table = create_simple_cudf_table(100, 2, gpu_space->get_default_allocator(), chain_stream); - - auto shared_table = std::make_shared(std::move(table)); - auto view = shared_table->view(); - auto alloc_size = shared_table->alloc_size(); - gpu_table_representation repr(view, - std::move(shared_table), - alloc_size, - *const_cast(gpu_space), - rmm::cuda_stream_view{}); - - auto cpu_any = registry.convert(repr, host_space, chain_stream); - auto gpu_any = registry.convert(*cpu_any, gpu_space, chain_stream); - - chain_stream.synchronize(); - cucascade::test::expect_cudf_tables_equal_on_stream( - repr.get_table_view(), gpu_any->get_table_view(), chain_stream); -} - -TEST_CASE("gpu->host_fast->gpu roundtrip preserves contents (table_view+shared_ptr ctor)", - "[gpu_data_representation][table_view_ctor]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const memory::memory_space* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const memory::memory_space* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - - rmm::cuda_stream stream; - constexpr int N = 64; - auto col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - CUCASCADE_CUDA_TRY( - cudaMemsetAsync(col->mutable_view().head(), 0xAB, N * sizeof(int32_t), stream.value())); - - std::vector> cols; - cols.push_back(std::move(col)); - auto shared_table = std::make_shared(std::move(cols)); - auto view = shared_table->view(); - auto alloc_size = shared_table->alloc_size(); - gpu_table_representation repr(view, - std::move(shared_table), - alloc_size, - *const_cast(gpu_space), - rmm::cuda_stream_view{}); - - auto host = registry.convert(repr, host_space, stream.view()); - auto back = registry.convert(*host, gpu_space, stream.view()); - stream.synchronize(); - - REQUIRE(back->get_table_view().num_columns() == 1); - REQUIRE(back->get_table_view().num_rows() == N); - cucascade::test::expect_cudf_tables_equal_on_stream( - repr.get_table_view(), back->get_table_view(), stream.view()); -} - -// ============================================================================= -// idata_representation Interface Tests -// ============================================================================= - -TEST_CASE("idata_representation cast functionality", - "[cpu_data_representation][gpu_data_representation]") -{ - // Note: host_data_packed_representation section disabled - requires internal API access - - SECTION("Cast gpu_table_representation") - { - auto gpu_space = make_mock_memory_space(memory::Tier::GPU, 0); - auto table = create_simple_cudf_table(100, gpu_space->get_default_allocator()); - gpu_table_representation repr( - std::make_unique(std::move(table)), *gpu_space, rmm::cuda_stream_view{}); - - idata_representation* base_ptr = &repr; - - // Cast to derived type - gpu_table_representation& casted = base_ptr->cast(); - REQUIRE(&casted == &repr); - REQUIRE(casted.get_table_view().num_rows() == 100); - } -} - -TEST_CASE("idata_representation const cast functionality", - "[cpu_data_representation][gpu_data_representation]") -{ - // Note: host_data_packed_representation section disabled - requires internal API access - - SECTION("Const cast gpu_table_representation") - { - auto gpu_space = make_mock_memory_space(memory::Tier::GPU, 0); - auto table = create_simple_cudf_table(100, gpu_space->get_default_allocator()); - gpu_table_representation repr( - std::make_unique(std::move(table)), *gpu_space, rmm::cuda_stream_view{}); - - const idata_representation* base_ptr = &repr; - - // Const cast to derived type - const gpu_table_representation& casted = base_ptr->cast(); - REQUIRE(&casted == &repr); - REQUIRE(casted.get_table_view().num_rows() == 100); - } -} - -// ============================================================================= -// Cross-Tier Comparison Tests -// ============================================================================= - -// Disabled: requires internal access to multiple_blocks_allocation constructor -TEST_CASE("Compare CPU and GPU representations", - "[cpu_data_representation][gpu_data_representation][.disabled]") -{ - SUCCEED("Test disabled - requires internal API access"); -} - -TEST_CASE("Multiple representations on same memory space", - "[cpu_data_representation][gpu_data_representation]") -{ - // Note: host_data_packed_representation section disabled - requires internal API access - - SECTION("Multiple GPU representations") - { - auto gpu_space = make_mock_memory_space(memory::Tier::GPU, 0); - - auto table1 = create_simple_cudf_table(100, gpu_space->get_default_allocator()); - gpu_table_representation repr1( - std::make_unique(std::move(table1)), *gpu_space, rmm::cuda_stream_view{}); - - auto table2 = create_simple_cudf_table(200, gpu_space->get_default_allocator()); - gpu_table_representation repr2( - std::make_unique(std::move(table2)), *gpu_space, rmm::cuda_stream_view{}); - - REQUIRE(repr1.get_current_tier() == repr2.get_current_tier()); - REQUIRE(repr1.get_device_id() == repr2.get_device_id()); - // Different row counts should result in different sizes - REQUIRE(repr1.get_size_in_bytes() != repr2.get_size_in_bytes()); - } -} - -// ============================================================================= -// Edge Case Tests -// ============================================================================= - -TEST_CASE("gpu_table_representation with single column", "[gpu_data_representation]") -{ - auto gpu_space = make_mock_memory_space(memory::Tier::GPU, 0); - - std::vector> columns; - auto col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - 100, - cudf::mask_state::UNALLOCATED, - rmm::cuda_stream_default, - gpu_space->get_default_allocator()); - columns.push_back(std::move(col)); - - auto table = std::make_unique(std::move(columns)); - gpu_table_representation repr(std::move(table), *gpu_space, rmm::cuda_stream_view{}); - - REQUIRE(repr.get_table_view().num_columns() == 1); - REQUIRE(repr.get_table_view().num_rows() == 100); - REQUIRE(repr.get_size_in_bytes() >= 100 * 4); // At least 100 rows * 4 bytes -} - -TEST_CASE("gpu_table_representation with multiple column types", "[gpu_data_representation]") -{ - auto gpu_space = make_mock_memory_space(memory::Tier::GPU, 0); - - std::vector> columns; - - // INT8 column - auto col1 = cudf::make_numeric_column( - cudf::data_type{cudf::type_id::INT8}, 100, cudf::mask_state::UNALLOCATED); - - // INT16 column - auto col2 = cudf::make_numeric_column( - cudf::data_type{cudf::type_id::INT16}, 100, cudf::mask_state::UNALLOCATED); - - // INT32 column - auto col3 = cudf::make_numeric_column( - cudf::data_type{cudf::type_id::INT32}, 100, cudf::mask_state::UNALLOCATED); - - // INT64 column - auto col4 = cudf::make_numeric_column( - cudf::data_type{cudf::type_id::INT64}, 100, cudf::mask_state::UNALLOCATED); - - columns.push_back(std::move(col1)); - columns.push_back(std::move(col2)); - columns.push_back(std::move(col3)); - columns.push_back(std::move(col4)); - - auto table = std::make_unique(std::move(columns)); - gpu_table_representation repr(std::move(table), *gpu_space, rmm::cuda_stream_view{}); - - REQUIRE(repr.get_table_view().num_columns() == 4); - REQUIRE(repr.get_table_view().num_rows() == 100); - // Size should be at least 100 * (1 + 2 + 4 + 8) = 1500 bytes - REQUIRE(repr.get_size_in_bytes() >= 1500); -} - -// Disabled: requires internal access to multiple_blocks_allocation constructor -TEST_CASE("Representations polymorphism", - "[cpu_data_representation][gpu_data_representation][.disabled]") -{ - SUCCEED("Test disabled - requires internal API access"); -} - -// ============================================================================= -// Clone Tests -// ============================================================================= - -TEST_CASE("gpu_table_representation clone creates independent copy", "[gpu_data_representation]") -{ - auto gpu_space = make_mock_memory_space(memory::Tier::GPU, 0); - auto table = create_simple_cudf_table(100, gpu_space->get_default_allocator()); - - gpu_table_representation repr( - std::make_unique(std::move(table)), *gpu_space, rmm::cuda_stream_view{}); - - // Clone the representation - auto cloned_base = repr.clone(rmm::cuda_stream_default); - REQUIRE(cloned_base != nullptr); - - // Verify it's a gpu_table_representation - auto* cloned = dynamic_cast(cloned_base.get()); - REQUIRE(cloned != nullptr); - - // Verify the cloned representation has the same properties - REQUIRE(cloned->get_current_tier() == repr.get_current_tier()); - REQUIRE(cloned->get_device_id() == repr.get_device_id()); - REQUIRE(cloned->get_size_in_bytes() == repr.get_size_in_bytes()); - - // Verify the tables have the same shape - REQUIRE(cloned->get_table_view().num_columns() == repr.get_table_view().num_columns()); - REQUIRE(cloned->get_table_view().num_rows() == repr.get_table_view().num_rows()); - - // Verify the data is equal - cucascade::test::expect_cudf_tables_equal_on_stream( - repr.get_table_view(), cloned->get_table_view(), rmm::cuda_stream_default); - - // Verify the tables are independent (different memory addresses) - for (cudf::size_type i = 0; i < repr.get_table_view().num_columns(); ++i) { - REQUIRE(repr.get_table_view().column(i).head() != cloned->get_table_view().column(i).head()); - } -} - -TEST_CASE("gpu_table_representation clone empty table", "[gpu_data_representation]") -{ - auto gpu_space = make_mock_memory_space(memory::Tier::GPU, 0); - auto table = create_simple_cudf_table(0, gpu_space->get_default_allocator()); - - gpu_table_representation repr( - std::make_unique(std::move(table)), *gpu_space, rmm::cuda_stream_view{}); - - auto cloned_base = repr.clone(rmm::cuda_stream_default); - REQUIRE(cloned_base != nullptr); - - auto* cloned = dynamic_cast(cloned_base.get()); - REQUIRE(cloned != nullptr); - REQUIRE(cloned->get_table_view().num_rows() == 0); - REQUIRE(cloned->get_size_in_bytes() == 0); -} - -TEST_CASE("host_data_packed_representation clone creates independent copy", - "[cpu_data_representation]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const memory::memory_space* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - const memory::memory_space* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - - // Create a host_data_packed_representation via conversion from GPU - // Use the same stream for table creation and conversion to avoid stream-ordered races - rmm::cuda_stream stream; - auto original = - create_simple_cudf_table(128, 2, gpu_space->get_default_allocator(), stream.view()); - gpu_table_representation gpu_repr(std::make_unique(std::move(original)), - *const_cast(gpu_space), - rmm::cuda_stream_view{}); - - auto host_repr_ptr = - registry.convert(gpu_repr, host_space, stream); - stream.synchronize(); - - // Clone the host representation - auto cloned_base = host_repr_ptr->clone(stream.view()); - REQUIRE(cloned_base != nullptr); - - auto* cloned = dynamic_cast(cloned_base.get()); - REQUIRE(cloned != nullptr); - - // Verify properties match - REQUIRE(cloned->get_current_tier() == host_repr_ptr->get_current_tier()); - REQUIRE(cloned->get_device_id() == host_repr_ptr->get_device_id()); - REQUIRE(cloned->get_size_in_bytes() == host_repr_ptr->get_size_in_bytes()); - - // Verify the underlying allocations are different (independent) - REQUIRE(cloned->get_host_table().get() != host_repr_ptr->get_host_table().get()); - REQUIRE(cloned->get_host_table()->allocation.get() != - host_repr_ptr->get_host_table()->allocation.get()); - - // Convert both back to GPU and verify data equality - auto cloned_gpu = registry.convert(*cloned, gpu_space, stream); - auto orig_gpu = registry.convert(*host_repr_ptr, gpu_space, stream); - stream.synchronize(); - - cucascade::test::expect_cudf_tables_equal_on_stream( - orig_gpu->get_table_view(), cloned_gpu->get_table_view(), stream.view()); -} - -// */ - -// ============================================================================= -// host_data_representation Tests -// ============================================================================= - -/// Read `size` bytes from a multi-block allocation starting at byte offset `offset`. -static std::vector read_from_alloc( - const memory::fixed_size_host_memory_resource::multiple_blocks_allocation& alloc, - std::size_t offset, - std::size_t size) -{ - std::vector result(size); - const std::size_t blk_sz = alloc.block_size(); - std::size_t bi = offset / blk_sz, bo = offset % blk_sz, di = 0; - while (di < size) { - std::size_t n = std::min(size - di, blk_sz - bo); - std::memcpy(result.data() + di, alloc.at(bi).data() + bo, n); - di += n; - bo += n; - if (bo == blk_sz) { - ++bi; - bo = 0; - } - } - return result; -} - -/// Synchronously copy `size` bytes from a GPU pointer to a host std::vector. -static std::vector gpu_bytes(const void* ptr, std::size_t size) -{ - std::vector result(size); - if (size > 0 && ptr != nullptr) { - CUCASCADE_CUDA_TRY(cudaMemcpy(result.data(), ptr, size, cudaMemcpyDeviceToHost)); - } - return result; -} - -/// Wrap a single column into a gpu_table_representation. -static gpu_table_representation wrap_column( - std::unique_ptr col, - memory::memory_space& gpu_space, - rmm::cuda_stream_view writer_stream = rmm::cuda_stream_view{}) -{ - std::vector> cols; - cols.push_back(std::move(col)); - return gpu_table_representation( - std::make_unique(std::move(cols)), gpu_space, writer_stream); -} - -/// Convert a gpu_table_representation to host_data_representation via the registry. -static std::unique_ptr fast_convert( - gpu_table_representation& src, - const memory::memory_space* host_space, - representation_converter_registry& registry, - rmm::cuda_stream_view stream) -{ - return registry.convert(src, host_space, stream); -} - -// ============================================================================= -// Converter registration -// ============================================================================= - -TEST_CASE("register_builtin_converters registers GPU->host_data_representation", - "[fast][registration]") -{ - representation_converter_registry registry; - register_builtin_converters(registry); - REQUIRE(registry.has_converter()); -} - -// ============================================================================= -// Fixed-width primitive types — metadata correctness -// ============================================================================= - -template -static void check_fixed_width_metadata(memory::memory_reservation_manager& mgr, - representation_converter_registry& registry) -{ - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - constexpr int N = 100; - auto col = cudf::make_numeric_column(cudf::data_type{TypeID}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - stream.synchronize(); - - auto repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - - REQUIRE(host != nullptr); - REQUIRE(host->get_current_tier() == memory::Tier::HOST); - REQUIRE(host->get_size_in_bytes() > 0); - - const auto& cols = host->get_host_table()->columns; - REQUIRE(cols.size() == 1); - REQUIRE(cols[0].type_id == TypeID); - REQUIRE(cols[0].num_rows == N); - REQUIRE(cols[0].has_data == true); - REQUIRE(cols[0].data_size == static_cast(N) * sizeof(CppType)); - REQUIRE(cols[0].has_null_mask == false); - REQUIRE(cols[0].children.empty()); -} - -TEST_CASE("Fast converter metadata: fixed-width primitive types", "[fast][metadata]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - // clang-format off - SECTION("INT8") { check_fixed_width_metadata(mgr, registry); } - SECTION("INT16") { check_fixed_width_metadata(mgr, registry); } - SECTION("INT32") { check_fixed_width_metadata(mgr, registry); } - SECTION("INT64") { check_fixed_width_metadata(mgr, registry); } - SECTION("UINT8") { check_fixed_width_metadata(mgr, registry); } - SECTION("UINT16") { check_fixed_width_metadata(mgr, registry); } - SECTION("UINT32") { check_fixed_width_metadata(mgr, registry); } - SECTION("UINT64") { check_fixed_width_metadata(mgr, registry); } - SECTION("FLOAT32"){ check_fixed_width_metadata(mgr, registry); } - SECTION("FLOAT64"){ check_fixed_width_metadata(mgr, registry); } - SECTION("BOOL8") { check_fixed_width_metadata(mgr, registry); } - // clang-format on -} - -// ============================================================================= -// Byte-level data integrity -// ============================================================================= - -TEST_CASE("Fast converter copies INT32 data bytes correctly", "[fast][data_integrity]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - constexpr int N = 200; - auto col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - CUCASCADE_CUDA_TRY( - cudaMemsetAsync(col->mutable_view().head(), 0xAB, N * sizeof(int32_t), stream.value())); - const void* gpu_ptr = col->view().data(); - - auto repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - - const auto& meta = host->get_host_table()->columns[0]; - auto actual_bytes = - read_from_alloc(*host->get_host_table()->allocation, meta.data_offset, meta.data_size); - auto expected_bytes = gpu_bytes(gpu_ptr, meta.data_size); - REQUIRE(actual_bytes == expected_bytes); -} - -TEST_CASE("Fast converter copies FLOAT64 data bytes correctly", "[fast][data_integrity]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - constexpr int N = 150; - auto col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::FLOAT64}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - CUCASCADE_CUDA_TRY( - cudaMemsetAsync(col->mutable_view().head(), 0xCD, N * sizeof(double), stream.value())); - const void* gpu_ptr = col->view().data(); - - auto repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - - const auto& meta = host->get_host_table()->columns[0]; - auto actual_bytes = - read_from_alloc(*host->get_host_table()->allocation, meta.data_offset, meta.data_size); - auto expected_bytes = gpu_bytes(gpu_ptr, meta.data_size); - REQUIRE(actual_bytes == expected_bytes); -} - -// ============================================================================= -// Nullable columns -// ============================================================================= - -TEST_CASE("Fast converter: nullable INT32 — null mask metadata and bytes", "[fast][nullable]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - constexpr int N = 100; - auto col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - N, - cudf::mask_state::ALL_VALID, - stream.view(), - gpu_space->get_default_allocator()); - const void* mask_ptr = col->view().null_mask(); - - auto repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - - const auto& meta = host->get_host_table()->columns[0]; - REQUIRE(meta.has_null_mask == true); - REQUIRE(meta.null_mask_size == cudf::bitmask_allocation_size_bytes(N)); - - auto actual_mask = read_from_alloc( - *host->get_host_table()->allocation, meta.null_mask_offset, meta.null_mask_size); - auto expected_mask = gpu_bytes(mask_ptr, meta.null_mask_size); - REQUIRE(actual_mask == expected_mask); -} - -TEST_CASE("Fast converter: nullable INT64 — both null mask and data bytes are correct", - "[fast][nullable]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - constexpr int N = 64; - auto col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT64}, - N, - cudf::mask_state::ALL_VALID, - stream.view(), - gpu_space->get_default_allocator()); - CUCASCADE_CUDA_TRY( - cudaMemsetAsync(col->mutable_view().head(), 0x77, N * sizeof(int64_t), stream.value())); - const void* data_ptr = col->view().data(); - const void* mask_ptr = col->view().null_mask(); - - auto repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - - const auto& meta = host->get_host_table()->columns[0]; - - auto actual_data = - read_from_alloc(*host->get_host_table()->allocation, meta.data_offset, meta.data_size); - auto expected_data = gpu_bytes(data_ptr, meta.data_size); - REQUIRE(actual_data == expected_data); - - auto actual_mask = read_from_alloc( - *host->get_host_table()->allocation, meta.null_mask_offset, meta.null_mask_size); - auto expected_mask = gpu_bytes(mask_ptr, meta.null_mask_size); - REQUIRE(actual_mask == expected_mask); -} - -// ============================================================================= -// Timestamp and Duration columns -// ============================================================================= - -TEST_CASE("Fast converter: timestamp columns metadata", "[fast][timestamp]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - constexpr int N = 50; - - SECTION("TIMESTAMP_DAYS — stored as int32_t") - { - auto col = cudf::make_timestamp_column(cudf::data_type{cudf::type_id::TIMESTAMP_DAYS}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - stream.synchronize(); - auto repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - const auto& meta = host->get_host_table()->columns[0]; - REQUIRE(meta.type_id == cudf::type_id::TIMESTAMP_DAYS); - REQUIRE(meta.has_data == true); - REQUIRE(meta.data_size == static_cast(N) * sizeof(int32_t)); - REQUIRE(meta.children.empty()); - } - - SECTION("TIMESTAMP_SECONDS — stored as int64_t") - { - auto col = cudf::make_timestamp_column(cudf::data_type{cudf::type_id::TIMESTAMP_SECONDS}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - stream.synchronize(); - auto repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - const auto& meta = host->get_host_table()->columns[0]; - REQUIRE(meta.type_id == cudf::type_id::TIMESTAMP_SECONDS); - REQUIRE(meta.has_data == true); - REQUIRE(meta.data_size == static_cast(N) * sizeof(int64_t)); - } - - SECTION("TIMESTAMP_MICROSECONDS — stored as int64_t") - { - auto col = cudf::make_timestamp_column(cudf::data_type{cudf::type_id::TIMESTAMP_MICROSECONDS}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - stream.synchronize(); - auto repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - const auto& meta = host->get_host_table()->columns[0]; - REQUIRE(meta.type_id == cudf::type_id::TIMESTAMP_MICROSECONDS); - REQUIRE(meta.data_size == static_cast(N) * sizeof(int64_t)); - } -} - -TEST_CASE("Fast converter: duration columns metadata", "[fast][duration]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - constexpr int N = 50; - - SECTION("DURATION_DAYS — stored as int32_t") - { - auto col = cudf::make_duration_column(cudf::data_type{cudf::type_id::DURATION_DAYS}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - stream.synchronize(); - auto repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - const auto& meta = host->get_host_table()->columns[0]; - REQUIRE(meta.type_id == cudf::type_id::DURATION_DAYS); - REQUIRE(meta.data_size == static_cast(N) * sizeof(int32_t)); - } - - SECTION("DURATION_MILLISECONDS — stored as int64_t") - { - auto col = cudf::make_duration_column(cudf::data_type{cudf::type_id::DURATION_MILLISECONDS}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - stream.synchronize(); - auto repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - const auto& meta = host->get_host_table()->columns[0]; - REQUIRE(meta.type_id == cudf::type_id::DURATION_MILLISECONDS); - REQUIRE(meta.data_size == static_cast(N) * sizeof(int64_t)); - } - - SECTION("DURATION_NANOSECONDS — stored as int64_t") - { - auto col = cudf::make_duration_column(cudf::data_type{cudf::type_id::DURATION_NANOSECONDS}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - stream.synchronize(); - auto repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - const auto& meta = host->get_host_table()->columns[0]; - REQUIRE(meta.type_id == cudf::type_id::DURATION_NANOSECONDS); - REQUIRE(meta.data_size == static_cast(N) * sizeof(int64_t)); - } -} - -// ============================================================================= -// Decimal columns — scale stored in metadata -// ============================================================================= - -TEST_CASE("Fast converter: decimal columns store scale in metadata", "[fast][decimal]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - constexpr int N = 50; - - SECTION("DECIMAL32 scale=-3, element size=4") - { - auto col = cudf::make_fixed_point_column(cudf::data_type{cudf::type_id::DECIMAL32, -3}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - stream.synchronize(); - auto repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - const auto& meta = host->get_host_table()->columns[0]; - REQUIRE(meta.type_id == cudf::type_id::DECIMAL32); - REQUIRE(meta.scale == -3); - REQUIRE(meta.has_data == true); - REQUIRE(meta.data_size == static_cast(N) * sizeof(int32_t)); - REQUIRE(meta.children.empty()); - } - - SECTION("DECIMAL64 scale=-6, element size=8") - { - auto col = cudf::make_fixed_point_column(cudf::data_type{cudf::type_id::DECIMAL64, -6}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - stream.synchronize(); - auto repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - const auto& meta = host->get_host_table()->columns[0]; - REQUIRE(meta.type_id == cudf::type_id::DECIMAL64); - REQUIRE(meta.scale == -6); - REQUIRE(meta.data_size == static_cast(N) * sizeof(int64_t)); - } - - SECTION("DECIMAL128 scale=-9, element size=16") - { - auto col = cudf::make_fixed_point_column(cudf::data_type{cudf::type_id::DECIMAL128, -9}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - stream.synchronize(); - auto repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - const auto& meta = host->get_host_table()->columns[0]; - REQUIRE(meta.type_id == cudf::type_id::DECIMAL128); - REQUIRE(meta.scale == -9); - REQUIRE(meta.data_size == static_cast(N) * 16); // 128-bit = 16 bytes - } -} - -// ============================================================================= -// STRING column — children metadata: offsets + chars -// ============================================================================= - -TEST_CASE("Fast converter: STRING column metadata structure", "[fast][string]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - // Build a strings column with 4 strings and 10 total characters - // offsets: [0, 2, 5, 7, 10] — child[0]: INT32, 5 elements - // chars: 10 bytes of 'x' — child[1]: INT8, 10 elements - constexpr int num_strings = 4; - constexpr int total_chars = 10; - - std::vector host_offsets = {0, 2, 5, 7, 10}; - rmm::device_buffer offsets_buf(host_offsets.data(), - host_offsets.size() * sizeof(int32_t), - stream.view(), - gpu_space->get_default_allocator()); - auto offsets_col = - std::make_unique(cudf::data_type{cudf::type_id::INT32}, - static_cast(host_offsets.size()), - std::move(offsets_buf), - rmm::device_buffer{}, - 0); - - std::vector host_chars(total_chars, 'x'); - rmm::device_buffer chars_buf( - host_chars.data(), host_chars.size(), stream.view(), gpu_space->get_default_allocator()); - - stream.synchronize(); - - auto strings_col = cudf::make_strings_column( - num_strings, std::move(offsets_col), std::move(chars_buf), 0, rmm::device_buffer{}); - - auto repr = wrap_column( - std::move(strings_col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - - const auto& meta = host->get_host_table()->columns[0]; - REQUIRE(meta.type_id == cudf::type_id::STRING); - REQUIRE(meta.num_rows == num_strings); - // In this cudf version, STRING stores chars in the column's data buffer. - REQUIRE(meta.has_data == true); - REQUIRE(meta.data_size == static_cast(total_chars)); - REQUIRE(meta.has_null_mask == false); - // Single child: offsets (INT32) - REQUIRE(meta.children.size() == 1); - - const auto& offsets_meta = meta.children[0]; - REQUIRE(offsets_meta.type_id == cudf::type_id::INT32); - REQUIRE(offsets_meta.num_rows == num_strings + 1); - REQUIRE(offsets_meta.has_data == true); - REQUIRE(offsets_meta.data_size == static_cast(num_strings + 1) * sizeof(int32_t)); -} - -// ============================================================================= -// LIST column — children metadata: offsets + values -// ============================================================================= - -TEST_CASE("Fast converter: LIST column metadata structure", "[fast][list]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - // 3 lists with 7 total elements: [0..1], [2..4], [5..6] - constexpr int num_lists = 3; - constexpr int num_values = 7; - - std::vector host_offsets = {0, 2, 5, 7}; - rmm::device_buffer offsets_buf(host_offsets.data(), - host_offsets.size() * sizeof(int32_t), - stream.view(), - gpu_space->get_default_allocator()); - auto offsets_col = - std::make_unique(cudf::data_type{cudf::type_id::INT32}, - static_cast(host_offsets.size()), - std::move(offsets_buf), - rmm::device_buffer{}, - 0); - - auto values_col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_values, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - stream.synchronize(); - - auto list_col = - cudf::make_lists_column(num_lists, std::move(offsets_col), std::move(values_col), 0, {}); - - auto repr = wrap_column( - std::move(list_col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - - const auto& meta = host->get_host_table()->columns[0]; - REQUIRE(meta.type_id == cudf::type_id::LIST); - REQUIRE(meta.num_rows == num_lists); - REQUIRE(meta.has_data == false); - REQUIRE(meta.has_null_mask == false); - REQUIRE(meta.children.size() == 2); - - const auto& offsets_meta = meta.children[0]; - REQUIRE(offsets_meta.type_id == cudf::type_id::INT32); - REQUIRE(offsets_meta.num_rows == num_lists + 1); - REQUIRE(offsets_meta.has_data == true); - REQUIRE(offsets_meta.data_size == static_cast(num_lists + 1) * sizeof(int32_t)); - - const auto& values_meta = meta.children[1]; - REQUIRE(values_meta.type_id == cudf::type_id::INT32); - REQUIRE(values_meta.num_rows == num_values); - REQUIRE(values_meta.has_data == true); - REQUIRE(values_meta.data_size == static_cast(num_values) * sizeof(int32_t)); -} - -TEST_CASE("Fast converter: nullable LIST preserves parent null mask", "[fast][list]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - constexpr int num_lists = 3; - constexpr int num_values = 5; - - std::vector host_offsets = {0, 2, 3, 5}; - rmm::device_buffer offsets_buf(host_offsets.data(), - host_offsets.size() * sizeof(int32_t), - stream.view(), - gpu_space->get_default_allocator()); - auto offsets_col = - std::make_unique(cudf::data_type{cudf::type_id::INT32}, - static_cast(host_offsets.size()), - std::move(offsets_buf), - rmm::device_buffer{}, - 0); - - auto values_col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_values, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - - // Build a null mask marking the first list as null (bit 0 = 0) - auto null_mask = cudf::create_null_mask(num_lists, cudf::mask_state::ALL_VALID, stream.view()); - stream.synchronize(); - - auto list_col = cudf::make_lists_column( - num_lists, std::move(offsets_col), std::move(values_col), 1, std::move(null_mask)); - - auto repr = wrap_column( - std::move(list_col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - - const auto& meta = host->get_host_table()->columns[0]; - REQUIRE(meta.type_id == cudf::type_id::LIST); - REQUIRE(meta.has_null_mask == true); - REQUIRE(meta.null_mask_size == cudf::bitmask_allocation_size_bytes(num_lists)); -} - -// ============================================================================= -// STRUCT column — field children metadata -// ============================================================================= - -TEST_CASE("Fast converter: STRUCT column metadata structure", "[fast][struct]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - constexpr int N = 8; - auto field0 = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - auto field1 = cudf::make_numeric_column(cudf::data_type{cudf::type_id::FLOAT64}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - stream.synchronize(); - - std::vector> children; - children.push_back(std::move(field0)); - children.push_back(std::move(field1)); - auto struct_col = cudf::make_structs_column(N, std::move(children), 0, {}); - - auto repr = wrap_column( - std::move(struct_col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - - const auto& meta = host->get_host_table()->columns[0]; - REQUIRE(meta.type_id == cudf::type_id::STRUCT); - REQUIRE(meta.num_rows == N); - REQUIRE(meta.has_data == false); - REQUIRE(meta.has_null_mask == false); - REQUIRE(meta.children.size() == 2); - - REQUIRE(meta.children[0].type_id == cudf::type_id::INT32); - REQUIRE(meta.children[0].num_rows == N); - REQUIRE(meta.children[0].has_data == true); - REQUIRE(meta.children[0].data_size == static_cast(N) * sizeof(int32_t)); - - REQUIRE(meta.children[1].type_id == cudf::type_id::FLOAT64); - REQUIRE(meta.children[1].num_rows == N); - REQUIRE(meta.children[1].has_data == true); - REQUIRE(meta.children[1].data_size == static_cast(N) * sizeof(double)); -} - -// ============================================================================= -// Nested types: LIST> -// ============================================================================= - -TEST_CASE("Fast converter: LIST> nested metadata", "[fast][nested]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - // Outer list: 2 lists of inner lists - // Inner lists: 3 inner lists with [2, 3, 2] elements = 7 total values - // outer_offsets: [0, 2, 3] — 2 outer lists - // inner_offsets: [0, 2, 5, 7] — 3 inner lists - // values: 7 INT32 elements - constexpr int num_outer = 2; - constexpr int num_inner = 3; - constexpr int num_values = 7; - - auto values = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_values, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - - std::vector inner_offs_h = {0, 2, 5, 7}; - rmm::device_buffer inner_offs_buf(inner_offs_h.data(), - inner_offs_h.size() * sizeof(int32_t), - stream.view(), - gpu_space->get_default_allocator()); - auto inner_offsets = - std::make_unique(cudf::data_type{cudf::type_id::INT32}, - static_cast(inner_offs_h.size()), - std::move(inner_offs_buf), - rmm::device_buffer{}, - 0); - stream.synchronize(); - - auto inner_list = - cudf::make_lists_column(num_inner, std::move(inner_offsets), std::move(values), 0, {}); - - std::vector outer_offs_h = {0, 2, 3}; - rmm::device_buffer outer_offs_buf(outer_offs_h.data(), - outer_offs_h.size() * sizeof(int32_t), - stream.view(), - gpu_space->get_default_allocator()); - auto outer_offsets = - std::make_unique(cudf::data_type{cudf::type_id::INT32}, - static_cast(outer_offs_h.size()), - std::move(outer_offs_buf), - rmm::device_buffer{}, - 0); - stream.synchronize(); - - auto outer_list = - cudf::make_lists_column(num_outer, std::move(outer_offsets), std::move(inner_list), 0, {}); - - auto repr = wrap_column( - std::move(outer_list), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - - // Outer LIST - const auto& outer_meta = host->get_host_table()->columns[0]; - REQUIRE(outer_meta.type_id == cudf::type_id::LIST); - REQUIRE(outer_meta.num_rows == num_outer); - REQUIRE(outer_meta.has_data == false); - REQUIRE(outer_meta.children.size() == 2); - - // outer.children[0] = outer offsets (INT32) - REQUIRE(outer_meta.children[0].type_id == cudf::type_id::INT32); - REQUIRE(outer_meta.children[0].num_rows == num_outer + 1); - REQUIRE(outer_meta.children[0].has_data == true); - - // outer.children[1] = inner LIST - const auto& inner_meta = outer_meta.children[1]; - REQUIRE(inner_meta.type_id == cudf::type_id::LIST); - REQUIRE(inner_meta.num_rows == num_inner); - REQUIRE(inner_meta.has_data == false); - REQUIRE(inner_meta.children.size() == 2); - - // inner.children[0] = inner offsets (INT32) - REQUIRE(inner_meta.children[0].type_id == cudf::type_id::INT32); - REQUIRE(inner_meta.children[0].num_rows == num_inner + 1); - - // inner.children[1] = values (INT32) - REQUIRE(inner_meta.children[1].type_id == cudf::type_id::INT32); - REQUIRE(inner_meta.children[1].num_rows == num_values); - REQUIRE(inner_meta.children[1].has_data == true); - REQUIRE(inner_meta.children[1].data_size == - static_cast(num_values) * sizeof(int32_t)); -} - -// ============================================================================= -// Nested types: LIST> -// ============================================================================= - -TEST_CASE("Fast converter: LIST> nested metadata", "[fast][nested]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - // 3 lists with 5 total struct elements - // offsets: [0, 2, 2, 5] — list[0]={s0,s1}, list[1]={} (empty), list[2]={s2,s3,s4} - constexpr int num_lists = 3; - constexpr int num_structs = 5; - - // Build the STRUCT child column (num_structs elements) - auto f0 = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_structs, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - auto f1 = cudf::make_numeric_column(cudf::data_type{cudf::type_id::FLOAT64}, - num_structs, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - stream.synchronize(); - - std::vector> struct_fields; - struct_fields.push_back(std::move(f0)); - struct_fields.push_back(std::move(f1)); - auto struct_col = cudf::make_structs_column(num_structs, std::move(struct_fields), 0, {}); - - // Build the offsets column for the LIST - std::vector offsets_h = {0, 2, 2, 5}; - rmm::device_buffer offsets_buf(offsets_h.data(), - offsets_h.size() * sizeof(int32_t), - stream.view(), - gpu_space->get_default_allocator()); - auto offsets_col = std::make_unique(cudf::data_type{cudf::type_id::INT32}, - static_cast(offsets_h.size()), - std::move(offsets_buf), - rmm::device_buffer{}, - 0); - stream.synchronize(); - - auto list_col = - cudf::make_lists_column(num_lists, std::move(offsets_col), std::move(struct_col), 0, {}); - - auto repr = wrap_column( - std::move(list_col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - - // Outer: LIST - const auto& list_meta = host->get_host_table()->columns[0]; - REQUIRE(list_meta.type_id == cudf::type_id::LIST); - REQUIRE(list_meta.num_rows == num_lists); - REQUIRE(list_meta.has_data == false); - REQUIRE(list_meta.children.size() == 2); - - // list.children[0] = offsets (INT32) - const auto& offs_meta = list_meta.children[0]; - REQUIRE(offs_meta.type_id == cudf::type_id::INT32); - REQUIRE(offs_meta.num_rows == num_lists + 1); - REQUIRE(offs_meta.has_data == true); - REQUIRE(offs_meta.data_size == static_cast(num_lists + 1) * sizeof(int32_t)); - - // list.children[1] = STRUCT - const auto& struct_meta = list_meta.children[1]; - REQUIRE(struct_meta.type_id == cudf::type_id::STRUCT); - REQUIRE(struct_meta.num_rows == num_structs); - REQUIRE(struct_meta.has_data == false); - REQUIRE(struct_meta.children.size() == 2); - - REQUIRE(struct_meta.children[0].type_id == cudf::type_id::INT32); - REQUIRE(struct_meta.children[0].num_rows == num_structs); - REQUIRE(struct_meta.children[0].has_data == true); - REQUIRE(struct_meta.children[0].data_size == - static_cast(num_structs) * sizeof(int32_t)); - - REQUIRE(struct_meta.children[1].type_id == cudf::type_id::FLOAT64); - REQUIRE(struct_meta.children[1].num_rows == num_structs); - REQUIRE(struct_meta.children[1].has_data == true); - REQUIRE(struct_meta.children[1].data_size == - static_cast(num_structs) * sizeof(double)); -} - -// ============================================================================= -// Nested types: STRUCT, FLOAT64> -// ============================================================================= - -TEST_CASE("Fast converter: STRUCT,FLOAT64> nested metadata", "[fast][nested]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - // STRUCT with 4 rows, where: - // field[0] = LIST: 4 lists with 9 total INT32 values - // offsets: [0, 3, 3, 7, 9] - // field[1] = FLOAT64: 4 scalar values - constexpr int num_rows = 4; - constexpr int num_values = 9; - - // Build LIST field - std::vector offs_h = {0, 3, 3, 7, 9}; - rmm::device_buffer offs_buf(offs_h.data(), - offs_h.size() * sizeof(int32_t), - stream.view(), - gpu_space->get_default_allocator()); - auto offs_col = std::make_unique(cudf::data_type{cudf::type_id::INT32}, - static_cast(offs_h.size()), - std::move(offs_buf), - rmm::device_buffer{}, - 0); - - auto vals_col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_values, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - stream.synchronize(); - - auto list_field = - cudf::make_lists_column(num_rows, std::move(offs_col), std::move(vals_col), 0, {}); - - // Build FLOAT64 field - auto float_field = cudf::make_numeric_column(cudf::data_type{cudf::type_id::FLOAT64}, - num_rows, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - stream.synchronize(); - - // Build STRUCT, FLOAT64> - std::vector> fields; - fields.push_back(std::move(list_field)); - fields.push_back(std::move(float_field)); - auto struct_col = cudf::make_structs_column(num_rows, std::move(fields), 0, {}); - - auto repr = wrap_column( - std::move(struct_col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - - // Top-level: STRUCT - const auto& struct_meta = host->get_host_table()->columns[0]; - REQUIRE(struct_meta.type_id == cudf::type_id::STRUCT); - REQUIRE(struct_meta.num_rows == num_rows); - REQUIRE(struct_meta.has_data == false); - REQUIRE(struct_meta.children.size() == 2); - - // struct.children[0] = LIST - const auto& list_meta = struct_meta.children[0]; - REQUIRE(list_meta.type_id == cudf::type_id::LIST); - REQUIRE(list_meta.num_rows == num_rows); - REQUIRE(list_meta.has_data == false); - REQUIRE(list_meta.children.size() == 2); - - // list offsets - const auto& list_offs_meta = list_meta.children[0]; - REQUIRE(list_offs_meta.type_id == cudf::type_id::INT32); - REQUIRE(list_offs_meta.num_rows == num_rows + 1); - REQUIRE(list_offs_meta.has_data == true); - REQUIRE(list_offs_meta.data_size == static_cast(num_rows + 1) * sizeof(int32_t)); - - // list values - const auto& list_vals_meta = list_meta.children[1]; - REQUIRE(list_vals_meta.type_id == cudf::type_id::INT32); - REQUIRE(list_vals_meta.num_rows == num_values); - REQUIRE(list_vals_meta.has_data == true); - REQUIRE(list_vals_meta.data_size == static_cast(num_values) * sizeof(int32_t)); - - // struct.children[1] = FLOAT64 - const auto& float_meta = struct_meta.children[1]; - REQUIRE(float_meta.type_id == cudf::type_id::FLOAT64); - REQUIRE(float_meta.num_rows == num_rows); - REQUIRE(float_meta.has_data == true); - REQUIRE(float_meta.data_size == static_cast(num_rows) * sizeof(double)); -} - -// ============================================================================= -// Empty table (0 rows) -// ============================================================================= - -TEST_CASE("Fast converter: empty table (0 rows)", "[fast][empty]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - auto col1 = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - 0, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - auto col2 = cudf::make_numeric_column(cudf::data_type{cudf::type_id::FLOAT64}, - 0, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - stream.synchronize(); - - std::vector> cols; - cols.push_back(std::move(col1)); - cols.push_back(std::move(col2)); - gpu_table_representation repr(std::make_unique(std::move(cols)), - *const_cast(gpu_space), - rmm::cuda_stream_view{}); - - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - - REQUIRE(host != nullptr); - REQUIRE(host->get_size_in_bytes() == 0); - REQUIRE(host->get_host_table()->columns.size() == 2); - - for (const auto& meta : host->get_host_table()->columns) { - REQUIRE(meta.num_rows == 0); - REQUIRE(meta.has_data == false); // size 0 → no data to copy - REQUIRE(meta.has_null_mask == false); - } -} - -// ============================================================================= -// Multi-column table spanning all primitive types -// ============================================================================= - -TEST_CASE("Fast converter: multi-column table with all primitive types", "[fast][multi_column]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - constexpr int N = 16; - - using TypePair = std::pair; - const std::vector type_sizes = { - {cudf::type_id::INT8, sizeof(int8_t)}, - {cudf::type_id::INT16, sizeof(int16_t)}, - {cudf::type_id::INT32, sizeof(int32_t)}, - {cudf::type_id::INT64, sizeof(int64_t)}, - {cudf::type_id::UINT8, sizeof(uint8_t)}, - {cudf::type_id::UINT16, sizeof(uint16_t)}, - {cudf::type_id::UINT32, sizeof(uint32_t)}, - {cudf::type_id::UINT64, sizeof(uint64_t)}, - {cudf::type_id::FLOAT32, sizeof(float)}, - {cudf::type_id::FLOAT64, sizeof(double)}, - {cudf::type_id::BOOL8, sizeof(bool)}, - }; - - std::vector> cols; - for (const auto& [tid, sz] : type_sizes) { - cols.push_back(cudf::make_numeric_column(cudf::data_type{tid}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator())); - } - stream.synchronize(); - - gpu_table_representation repr(std::make_unique(std::move(cols)), - *const_cast(gpu_space), - rmm::cuda_stream_view{}); - - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - - REQUIRE(host != nullptr); - REQUIRE(host->get_host_table()->columns.size() == type_sizes.size()); - - for (std::size_t i = 0; i < type_sizes.size(); ++i) { - const auto& [tid, sz] = type_sizes[i]; - const auto& meta = host->get_host_table()->columns[i]; - REQUIRE(meta.type_id == tid); - REQUIRE(meta.num_rows == N); - REQUIRE(meta.has_data == true); - REQUIRE(meta.data_size == static_cast(N) * sz); - REQUIRE(meta.has_null_mask == false); - REQUIRE(meta.children.empty()); - } -} - -// ============================================================================= -// clone() creates an independent copy with identical byte content -// ============================================================================= - -TEST_CASE("host_data_representation clone: same bytes, independent allocation", "[fast][clone]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - constexpr int N = 100; - auto col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - CUCASCADE_CUDA_TRY( - cudaMemsetAsync(col->mutable_view().head(), 0x55, N * sizeof(int32_t), stream.value())); - - auto repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - - auto cloned_base = host->clone(stream.view()); - REQUIRE(cloned_base != nullptr); - - auto* cloned = dynamic_cast(cloned_base.get()); - REQUIRE(cloned != nullptr); - - // Same logical properties - REQUIRE(cloned->get_size_in_bytes() == host->get_size_in_bytes()); - REQUIRE(cloned->get_current_tier() == host->get_current_tier()); - - // Different underlying allocations (fully independent) - REQUIRE(cloned->get_host_table().get() != host->get_host_table().get()); - REQUIRE(cloned->get_host_table()->allocation.get() != host->get_host_table()->allocation.get()); - - // Metadata structure is mirrored - REQUIRE(cloned->get_host_table()->columns.size() == host->get_host_table()->columns.size()); - REQUIRE(cloned->get_host_table()->columns[0].type_id == - host->get_host_table()->columns[0].type_id); - REQUIRE(cloned->get_host_table()->columns[0].data_size == - host->get_host_table()->columns[0].data_size); - - // Byte content is identical - const auto& orig_meta = host->get_host_table()->columns[0]; - const auto& clone_meta = cloned->get_host_table()->columns[0]; - auto orig_bytes = read_from_alloc( - *host->get_host_table()->allocation, orig_meta.data_offset, orig_meta.data_size); - auto clone_bytes = read_from_alloc( - *cloned->get_host_table()->allocation, clone_meta.data_offset, clone_meta.data_size); - REQUIRE(orig_bytes == clone_bytes); - REQUIRE(orig_bytes[0] == 0x55); // Known fill pattern -} - -TEST_CASE("host_data_representation clone: empty table", "[fast][clone]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - auto col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - 0, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - stream.synchronize(); - - auto repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(repr, host_space, registry, stream.view()); - stream.synchronize(); - - auto cloned_base = host->clone(stream.view()); - auto* cloned = dynamic_cast(cloned_base.get()); - REQUIRE(cloned != nullptr); - REQUIRE(cloned->get_size_in_bytes() == 0); - REQUIRE(cloned->get_host_table()->columns.size() == 1); - REQUIRE(cloned->get_host_table()->columns[0].num_rows == 0); -} - -// ============================================================================= -// Round-trip: HOST FAST → GPU -// ============================================================================= - -/// Convert a host_data_representation back to gpu_table_representation via registry. -static std::unique_ptr fast_back_convert( - host_data_representation& src, - const memory::memory_space* gpu_space, - representation_converter_registry& registry, - rmm::cuda_stream_view stream) -{ - return registry.convert(src, gpu_space, stream); -} - -TEST_CASE("register_builtin_converters registers host_data_representation->GPU", - "[fast][registration]") -{ - representation_converter_registry registry; - register_builtin_converters(registry); - REQUIRE(registry.has_converter()); -} - -TEST_CASE("Round-trip fast: INT32 column data preserved", "[fast][roundtrip]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - constexpr int N = 64; - auto col = cudf::make_numeric_column(cudf::data_type(cudf::type_id::INT32), - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - CUCASCADE_CUDA_TRY( - cudaMemsetAsync(col->mutable_view().head(), 0xAB, N * sizeof(int32_t), stream.view())); - - auto orig_repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(orig_repr, host_space, registry, stream.view()); - stream.synchronize(); - - auto back = fast_back_convert(*host, gpu_space, registry, stream.view()); - stream.synchronize(); - - REQUIRE(back != nullptr); - REQUIRE(back->get_table_view().num_columns() == 1); - REQUIRE(back->get_table_view().num_rows() == N); - - cudf::table_view back_tv = back->get_table_view(); - auto bytes = gpu_bytes(back_tv.column(0).data(), N * sizeof(int32_t)); - REQUIRE(bytes[0] == 0xAB); - REQUIRE(bytes[N * sizeof(int32_t) - 1] == 0xAB); -} - -TEST_CASE("Round-trip fast: nullable INT64 null mask preserved", "[fast][roundtrip]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - constexpr int N = 32; - auto col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT64}, - N, - cudf::mask_state::ALL_VALID, - stream.view(), - gpu_space->get_default_allocator()); - CUCASCADE_CUDA_TRY( - cudaMemsetAsync(col->mutable_view().head(), 0x77, N * sizeof(int64_t), stream.view())); - stream.synchronize(); - - auto orig_repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(orig_repr, host_space, registry, stream.view()); - stream.synchronize(); - - auto back = fast_back_convert(*host, gpu_space, registry, stream.view()); - stream.synchronize(); - - REQUIRE(back->get_table_view().num_columns() == 1); - cudf::table_view back_tv = back->get_table_view(); - REQUIRE(back_tv.column(0).nullable()); - REQUIRE(back_tv.column(0).null_count() == 0); - - auto data_bytes = gpu_bytes(back_tv.column(0).data(), N * sizeof(int64_t)); - REQUIRE(data_bytes[0] == 0x77); -} - -TEST_CASE("Round-trip fast: FLOAT64 byte integrity", "[fast][roundtrip]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - constexpr int N = 50; - auto col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::FLOAT64}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - CUCASCADE_CUDA_TRY( - cudaMemsetAsync(col->mutable_view().head(), 0xCD, N * sizeof(double), stream.view())); - - auto orig_repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(orig_repr, host_space, registry, stream.view()); - stream.synchronize(); - - auto back = fast_back_convert(*host, gpu_space, registry, stream.view()); - stream.synchronize(); - - cudf::table_view back_tv = back->get_table_view(); - auto data_bytes = gpu_bytes(back_tv.column(0).data(), N * sizeof(double)); - REQUIRE(data_bytes[0] == 0xCD); - REQUIRE(data_bytes[N * sizeof(double) - 1] == 0xCD); -} - -TEST_CASE("Round-trip fast: STRING column content preserved", "[fast][roundtrip]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - constexpr int num_strings = 4; - constexpr int total_chars = 10; - - std::vector host_offsets = {0, 2, 5, 7, 10}; - rmm::device_buffer offsets_buf(host_offsets.data(), - host_offsets.size() * sizeof(int32_t), - stream.view(), - gpu_space->get_default_allocator()); - auto offsets_col = - std::make_unique(cudf::data_type{cudf::type_id::INT32}, - static_cast(host_offsets.size()), - std::move(offsets_buf), - rmm::device_buffer{}, - 0); - - std::vector host_chars(total_chars, 'x'); - rmm::device_buffer chars_buf( - host_chars.data(), host_chars.size(), stream.view(), gpu_space->get_default_allocator()); - stream.synchronize(); - - auto strings_col = cudf::make_strings_column( - num_strings, std::move(offsets_col), std::move(chars_buf), 0, rmm::device_buffer{}); - - auto orig_repr = wrap_column( - std::move(strings_col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(orig_repr, host_space, registry, stream.view()); - stream.synchronize(); - - auto back = fast_back_convert(*host, gpu_space, registry, stream.view()); - stream.synchronize(); - - REQUIRE(back->get_table_view().num_columns() == 1); - REQUIRE(back->get_table_view().num_rows() == num_strings); - cudf::table_view back_tv = back->get_table_view(); - REQUIRE(back_tv.column(0).type().id() == cudf::type_id::STRING); - // Chars should be preserved - auto char_bytes = - gpu_bytes(back_tv.column(0).data(), static_cast(total_chars)); - REQUIRE(char_bytes[0] == static_cast('x')); - REQUIRE(char_bytes[total_chars - 1] == static_cast('x')); -} - -TEST_CASE("Round-trip fast: LIST structure preserved", "[fast][roundtrip]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - constexpr int num_lists = 3; - constexpr int num_values = 7; - - std::vector host_offsets = {0, 2, 5, 7}; - rmm::device_buffer offsets_buf(host_offsets.data(), - host_offsets.size() * sizeof(int32_t), - stream.view(), - gpu_space->get_default_allocator()); - auto offsets_col = - std::make_unique(cudf::data_type{cudf::type_id::INT32}, - static_cast(host_offsets.size()), - std::move(offsets_buf), - rmm::device_buffer{}, - 0); - auto values_col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_values, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - CUCASCADE_CUDA_TRY(cudaMemsetAsync( - values_col->mutable_view().head(), 0x33, num_values * sizeof(int32_t), stream.view())); - stream.synchronize(); - - auto list_col = - cudf::make_lists_column(num_lists, std::move(offsets_col), std::move(values_col), 0, {}); - - auto orig_repr = wrap_column( - std::move(list_col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(orig_repr, host_space, registry, stream.view()); - stream.synchronize(); - - auto back = fast_back_convert(*host, gpu_space, registry, stream.view()); - stream.synchronize(); - - REQUIRE(back->get_table_view().num_rows() == num_lists); - cudf::table_view back_tv = back->get_table_view(); - REQUIRE(back_tv.column(0).type().id() == cudf::type_id::LIST); - REQUIRE(back_tv.column(0).num_children() == 2); - // Values child bytes preserved - auto val_bytes = - gpu_bytes(back_tv.column(0).child(1).data(), num_values * sizeof(int32_t)); - REQUIRE(val_bytes[0] == 0x33); -} - -TEST_CASE("Round-trip fast: STRUCT fields preserved", "[fast][roundtrip]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - constexpr int N = 8; - auto f0 = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - auto f1 = cudf::make_numeric_column(cudf::data_type{cudf::type_id::FLOAT64}, - N, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - CUCASCADE_CUDA_TRY( - cudaMemsetAsync(f0->mutable_view().head(), 0x11, N * sizeof(int32_t), stream.view())); - CUCASCADE_CUDA_TRY( - cudaMemsetAsync(f1->mutable_view().head(), 0x22, N * sizeof(double), stream.view())); - stream.synchronize(); - - std::vector> fields; - fields.push_back(std::move(f0)); - fields.push_back(std::move(f1)); - auto struct_col = cudf::make_structs_column(N, std::move(fields), 0, {}); - - auto orig_repr = wrap_column( - std::move(struct_col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(orig_repr, host_space, registry, stream.view()); - stream.synchronize(); - - auto back = fast_back_convert(*host, gpu_space, registry, stream.view()); - stream.synchronize(); - - REQUIRE(back->get_table_view().num_rows() == N); - cudf::table_view back_tv = back->get_table_view(); - REQUIRE(back_tv.column(0).type().id() == cudf::type_id::STRUCT); - REQUIRE(back_tv.column(0).num_children() == 2); - - auto f0_bytes = gpu_bytes(back_tv.column(0).child(0).data(), N * sizeof(int32_t)); - REQUIRE(f0_bytes[0] == 0x11); - auto f1_bytes = gpu_bytes(back_tv.column(0).child(1).data(), N * sizeof(double)); - REQUIRE(f1_bytes[0] == 0x22); -} - -TEST_CASE("Round-trip fast: empty table (0 rows)", "[fast][roundtrip]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - auto col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - 0, - cudf::mask_state::UNALLOCATED, - stream.view(), - gpu_space->get_default_allocator()); - stream.synchronize(); - - auto orig_repr = wrap_column( - std::move(col), *const_cast(gpu_space), rmm::cuda_stream_view{}); - auto host = fast_convert(orig_repr, host_space, registry, stream.view()); - stream.synchronize(); - - auto back = fast_back_convert(*host, gpu_space, registry, stream.view()); - stream.synchronize(); - - REQUIRE(back != nullptr); - REQUIRE(back->get_table_view().num_columns() == 1); - REQUIRE(back->get_table_view().num_rows() == 0); - REQUIRE(back->get_table_view().column(0).type().id() == cudf::type_id::INT32); -} - -// ============================================================================= -// host_data_representation::slice round-trip -// ============================================================================= - -namespace { - -/// Build a GPU table with four primitive columns, each filled with a distinct byte pattern so a -/// mis-routed slice (wrong column or wrong bytes) is detectable byte-for-byte. -std::unique_ptr build_four_column_gpu_table(rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr, - int num_rows) -{ - auto fill = [&](cudf::data_type dt, std::size_t bytes_per_row, uint8_t pattern) { - auto col = cudf::make_numeric_column(dt, num_rows, cudf::mask_state::UNALLOCATED, stream, mr); - if (num_rows > 0) { - auto bytes = static_cast(num_rows) * bytes_per_row; - CUCASCADE_CUDA_TRY( - cudaMemsetAsync(col->mutable_view().head(), pattern, bytes, stream.value())); - } - return col; - }; - - std::vector> cols; - cols.push_back(fill(cudf::data_type{cudf::type_id::INT32}, sizeof(int32_t), 0xA1)); - cols.push_back(fill(cudf::data_type{cudf::type_id::INT64}, sizeof(int64_t), 0xB2)); - cols.push_back(fill(cudf::data_type{cudf::type_id::FLOAT32}, sizeof(float), 0xC3)); - cols.push_back(fill(cudf::data_type{cudf::type_id::FLOAT64}, sizeof(double), 0xD4)); - return std::make_unique(std::move(cols)); -} - -} // namespace - -TEST_CASE("host_data_representation::slice round-trip preserves selected columns", - "[fast][slice][roundtrip]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const auto* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const auto* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - rmm::cuda_stream stream; - - constexpr int N = 128; - auto orig_table = - build_four_column_gpu_table(stream.view(), gpu_space->get_default_allocator(), N); - stream.synchronize(); - const cudf::table_view orig_view = orig_table->view(); - - // GPU -> HOST - gpu_table_representation gpu_repr( - std::move(orig_table), *const_cast(gpu_space), stream.view()); - auto host = fast_convert(gpu_repr, host_space, registry, stream.view()); - stream.synchronize(); - REQUIRE(host->num_columns() == 4); - - SECTION("Slice keeps non-contiguous subset and matches original columns") - { - const std::array kept{0, 2}; - auto sliced = host->slice(std::span(kept.data(), kept.size())); - REQUIRE(sliced != nullptr); - REQUIRE(sliced->num_columns() == 2); - REQUIRE(sliced->column_size(0) == host->column_size(0)); - REQUIRE(sliced->column_size(1) == host->column_size(2)); - // Slice shares buffers with the source allocation. - REQUIRE(sliced->get_host_table()->allocation.get() == host->get_host_table()->allocation.get()); - - auto back = fast_back_convert(*sliced, gpu_space, registry, stream.view()); - stream.synchronize(); - REQUIRE(back != nullptr); - REQUIRE(back->get_table_view().num_columns() == 2); - - const cudf::table_view expected_view{{orig_view.column(kept[0]), orig_view.column(kept[1])}}; - cucascade::test::expect_cudf_tables_equal_on_stream( - expected_view, back->get_table_view(), stream.view()); - } - - SECTION("Reordered slice maps to the requested column order") - { - const std::array kept{3, 1, 0}; - auto sliced = host->slice(std::span(kept.data(), kept.size())); - REQUIRE(sliced->num_columns() == 3); - - auto back = fast_back_convert(*sliced, gpu_space, registry, stream.view()); - stream.synchronize(); - REQUIRE(back->get_table_view().num_columns() == 3); - - const cudf::table_view expected_view{ - {orig_view.column(kept[0]), orig_view.column(kept[1]), orig_view.column(kept[2])}}; - cucascade::test::expect_cudf_tables_equal_on_stream( - expected_view, back->get_table_view(), stream.view()); - } - - SECTION("Out-of-range index throws") - { - const std::array kept{4}; // valid range is [0, 4) - REQUIRE_THROWS_AS(host->slice(std::span(kept.data(), kept.size())), - std::out_of_range); - } -} diff --git a/test/data/test_disk_host_converters.cpp b/test/data/test_disk_host_converters.cpp deleted file mode 100644 index 42369ff..0000000 --- a/test/data/test_disk_host_converters.cpp +++ /dev/null @@ -1,648 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "utils/cudf_test_utils.hpp" -#include "utils/mock_test_utils.hpp" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include - -#include -#include -#include - -using namespace cucascade; - -namespace { - -/// Shared memory spaces — reused across all tests to avoid cumulative CUDA context -/// degradation from repeated creation/destruction of CUDA stream pools and pinned memory. -auto& shared_gpu_space() -{ - static auto s = test::make_mock_memory_space(memory::Tier::GPU, 0); - return s; -} -auto& shared_host_space() -{ - static auto s = test::make_mock_memory_space(memory::Tier::HOST, 0); - return s; -} -auto& shared_disk_space() -{ - static auto s = test::make_mock_memory_space(memory::Tier::DISK, 0); - return s; -} -rmm::cuda_stream_view shared_stream() -{ - static rmm::cuda_stream s; - return s.view(); -} - -/// Round-trip test helper: GPU -> host_data -> disk -> host_data -> GPU, compare tables. -void round_trip_test(std::unique_ptr original_table) -{ - auto stream = shared_stream(); - auto& gpu_space = shared_gpu_space(); - auto& host_space = shared_host_space(); - auto& disk_space = shared_disk_space(); - - representation_converter_registry registry; - register_builtin_converters(registry); - - // Create GPU representation from the original table - auto gpu_rep = std::make_unique( - std::move(original_table), *gpu_space, rmm::cuda_stream_view{}); - - // GPU -> host_data - auto host_rep = - registry.convert(*gpu_rep, host_space.get(), shared_stream()); - - // host_data -> disk - auto disk_rep = - registry.convert(*host_rep, disk_space.get(), shared_stream()); - - // disk -> host_data - auto host_rep2 = - registry.convert(*disk_rep, host_space.get(), shared_stream()); - - // host_data -> GPU - auto gpu_rep2 = - registry.convert(*host_rep2, gpu_space.get(), shared_stream()); - - // Compare original (from gpu_rep) with round-tripped - test::expect_cudf_tables_equal_on_stream( - gpu_rep->get_table_view(), gpu_rep2->get_table_view(), shared_stream()); -} - -/// Create a simple column of the given type with uninitialized data. -/// Returns a single-column cudf::table. -std::unique_ptr make_typed_table(cudf::type_id type, cudf::size_type num_rows) -{ - auto const dtype = cudf::data_type{type}; - std::vector> cols; - if (cudf::is_numeric(dtype)) { - // cudf::is_numeric includes BOOL8 - auto col = - cudf::make_numeric_column(dtype, num_rows, cudf::mask_state::UNALLOCATED, shared_stream()); - cols.push_back(std::move(col)); - } else if (cudf::is_timestamp(dtype)) { - auto col = - cudf::make_timestamp_column(dtype, num_rows, cudf::mask_state::UNALLOCATED, shared_stream()); - cols.push_back(std::move(col)); - } else if (cudf::is_duration(dtype)) { - auto col = - cudf::make_duration_column(dtype, num_rows, cudf::mask_state::UNALLOCATED, shared_stream()); - cols.push_back(std::move(col)); - } else if (cudf::is_fixed_point(dtype)) { - auto const fp_dtype = cudf::data_type{type, -2}; // scale = -2 - auto col = cudf::make_fixed_point_column( - fp_dtype, num_rows, cudf::mask_state::UNALLOCATED, shared_stream()); - cols.push_back(std::move(col)); - } - return std::make_unique(std::move(cols)); -} - -/// Create a table with ALL_NULL mask. -std::unique_ptr make_typed_table_with_nulls(cudf::type_id type, - cudf::size_type num_rows) -{ - auto const dtype = cudf::data_type{type}; - std::vector> cols; - if (cudf::is_numeric(dtype)) { - auto col = - cudf::make_numeric_column(dtype, num_rows, cudf::mask_state::ALL_NULL, shared_stream()); - cols.push_back(std::move(col)); - } else if (cudf::is_timestamp(dtype)) { - auto col = - cudf::make_timestamp_column(dtype, num_rows, cudf::mask_state::ALL_NULL, shared_stream()); - cols.push_back(std::move(col)); - } else if (cudf::is_duration(dtype)) { - auto col = - cudf::make_duration_column(dtype, num_rows, cudf::mask_state::ALL_NULL, shared_stream()); - cols.push_back(std::move(col)); - } else if (cudf::is_fixed_point(dtype)) { - auto const fp_dtype = cudf::data_type{type, -2}; - auto col = cudf::make_fixed_point_column( - fp_dtype, num_rows, cudf::mask_state::ALL_NULL, shared_stream()); - cols.push_back(std::move(col)); - } - return std::make_unique(std::move(cols)); -} - -} // namespace - -// ============================================================================= -// Numeric type round-trip tests (TEST-01) -// ============================================================================= - -TEST_CASE("host_data disk round-trip numeric types", "[disk][converter][numeric]") -{ - auto type = GENERATE(cudf::type_id::INT8, - cudf::type_id::INT16, - cudf::type_id::INT32, - cudf::type_id::INT64, - cudf::type_id::UINT8, - cudf::type_id::UINT16, - cudf::type_id::UINT32, - cudf::type_id::UINT64, - cudf::type_id::FLOAT32, - cudf::type_id::FLOAT64); - - SECTION("100 rows") { round_trip_test(make_typed_table(type, 100)); } -} - -// ============================================================================= -// BOOL8 round-trip tests (TEST-02) -// ============================================================================= - -TEST_CASE("host_data disk round-trip bool type", "[disk][converter][bool]") -{ - round_trip_test(make_typed_table(cudf::type_id::BOOL8, 100)); -} - -// ============================================================================= -// Timestamp round-trip tests (TEST-02) -// ============================================================================= - -TEST_CASE("host_data disk round-trip timestamp types", "[disk][converter][timestamp]") -{ - auto type = GENERATE(cudf::type_id::TIMESTAMP_DAYS, - cudf::type_id::TIMESTAMP_SECONDS, - cudf::type_id::TIMESTAMP_MILLISECONDS, - cudf::type_id::TIMESTAMP_MICROSECONDS, - cudf::type_id::TIMESTAMP_NANOSECONDS); - - round_trip_test(make_typed_table(type, 100)); -} - -// ============================================================================= -// Duration round-trip tests (TEST-02) -// ============================================================================= - -TEST_CASE("host_data disk round-trip duration types", "[disk][converter][duration]") -{ - auto type = GENERATE(cudf::type_id::DURATION_DAYS, - cudf::type_id::DURATION_SECONDS, - cudf::type_id::DURATION_MILLISECONDS, - cudf::type_id::DURATION_MICROSECONDS, - cudf::type_id::DURATION_NANOSECONDS); - - round_trip_test(make_typed_table(type, 100)); -} - -// ============================================================================= -// Decimal round-trip tests (TEST-03) -// ============================================================================= - -TEST_CASE("host_data disk round-trip decimal types", "[disk][converter][decimal]") -{ - auto type = - GENERATE(cudf::type_id::DECIMAL32, cudf::type_id::DECIMAL64, cudf::type_id::DECIMAL128); - - round_trip_test(make_typed_table(type, 100)); -} - -// ============================================================================= -// Null mask round-trip tests (TEST-08) -// ============================================================================= - -TEST_CASE("host_data disk round-trip all-null column", "[disk][converter][null]") -{ - round_trip_test(make_typed_table_with_nulls(cudf::type_id::INT32, 100)); -} - -TEST_CASE("host_data disk round-trip no-null column", "[disk][converter][null]") -{ - round_trip_test(make_typed_table(cudf::type_id::INT32, 100)); -} - -TEST_CASE("host_data disk round-trip mixed-null column", "[disk][converter][null]") -{ - // Create a column with alternating nulls - - auto col = cudf::make_numeric_column( - cudf::data_type{cudf::type_id::INT32}, 100, cudf::mask_state::ALL_VALID, shared_stream()); - - // Set every other bit to null using cudf utilities - auto const num_rows = 100; - std::vector host_mask(cudf::bitmask_allocation_size_bytes(num_rows), 0xFF); - // Clear every other bit (set elements 0, 2, 4, ... to null) - for (int i = 0; i < num_rows; i += 2) { - auto byte_idx = static_cast(i / 8); - auto bit_idx = static_cast(i % 8); - host_mask[byte_idx] &= static_cast(~(1u << bit_idx)); - } - rmm::device_buffer dev_mask(host_mask.data(), host_mask.size(), shared_stream()); - col->set_null_mask(std::move(dev_mask), 50); // 50 nulls out of 100 - - std::vector> cols; - cols.push_back(std::move(col)); - round_trip_test(std::make_unique(std::move(cols))); -} - -// ============================================================================= -// STRING column round-trip tests (TEST-04) -// ============================================================================= - -TEST_CASE("host_data disk round-trip string column", "[disk][converter][string]") -{ - std::vector host_strings = {"hello", "", "world", "test123", "", "a"}; - auto const num_strings = static_cast(host_strings.size()); - - // Compute offsets - std::vector host_offsets(static_cast(num_strings) + 1); - host_offsets[0] = 0; - for (std::size_t i = 0; i < host_strings.size(); ++i) { - host_offsets[i + 1] = host_offsets[i] + static_cast(host_strings[i].size()); - } - auto const total_chars = static_cast(host_offsets.back()); - - // Concatenate all chars - std::vector host_chars; - host_chars.reserve(total_chars); - for (const auto& s : host_strings) { - host_chars.insert(host_chars.end(), s.begin(), s.end()); - } - - // Create device buffers - rmm::device_buffer dev_chars(host_chars.data(), host_chars.size(), shared_stream()); - auto offsets_col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_strings + 1, - cudf::mask_state::UNALLOCATED, - shared_stream()); - CUCASCADE_CUDA_TRY(cudaMemcpyAsync(offsets_col->mutable_view().data(), - host_offsets.data(), - host_offsets.size() * sizeof(int32_t), - cudaMemcpyHostToDevice, - shared_stream().value())); - shared_stream().synchronize(); - - auto str_col = cudf::make_strings_column( - num_strings, std::move(offsets_col), std::move(dev_chars), 0, rmm::device_buffer{}); - - std::vector> cols; - cols.push_back(std::move(str_col)); - round_trip_test(std::make_unique(std::move(cols))); -} - -// Regression: cudf::make_empty_column(STRING) returns a 0-row strings column with no children. -// plan_column_copy faithfully records 0 children, and reconstruct_column previously rejected -// that metadata. Empty-string batches show up in real pipelines (e.g. probe-side partition -// outputs after a filter), get downgraded to host under memory pressure, and crash on -// upgrade back to GPU. -TEST_CASE("host_data disk round-trip empty strings column", "[disk][converter][string][empty]") -{ - std::vector> cols; - cols.push_back(cudf::make_empty_column(cudf::data_type{cudf::type_id::STRING})); - round_trip_test(std::make_unique(std::move(cols))); -} - -TEST_CASE("host_data disk round-trip empty strings column mixed with non-empty column", - "[disk][converter][string][empty]") -{ - // Mimic the in-pipeline shape: a 0-row table where one of the columns is an empty STRING. - std::vector> cols; - cols.push_back(cudf::make_empty_column(cudf::data_type{cudf::type_id::INT32})); - cols.push_back(cudf::make_empty_column(cudf::data_type{cudf::type_id::STRING})); - cols.push_back(cudf::make_empty_column(cudf::data_type{cudf::type_id::INT64})); - round_trip_test(std::make_unique(std::move(cols))); -} - -TEST_CASE("host_data disk round-trip string column with nulls", "[disk][converter][string][null]") -{ - std::vector host_strings = {"alpha", "beta", "gamma", "delta"}; - auto const num_strings = static_cast(host_strings.size()); - - std::vector host_offsets(static_cast(num_strings) + 1); - host_offsets[0] = 0; - for (std::size_t i = 0; i < host_strings.size(); ++i) { - host_offsets[i + 1] = host_offsets[i] + static_cast(host_strings[i].size()); - } - - std::vector host_chars; - for (const auto& s : host_strings) { - host_chars.insert(host_chars.end(), s.begin(), s.end()); - } - - rmm::device_buffer dev_chars(host_chars.data(), host_chars.size(), shared_stream()); - auto offsets_col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_strings + 1, - cudf::mask_state::UNALLOCATED, - shared_stream()); - CUCASCADE_CUDA_TRY(cudaMemcpyAsync(offsets_col->mutable_view().data(), - host_offsets.data(), - host_offsets.size() * sizeof(int32_t), - cudaMemcpyHostToDevice, - shared_stream().value())); - - // Create null mask: elements 1 and 3 are null - auto null_mask_size = cudf::bitmask_allocation_size_bytes(num_strings); - std::vector host_mask(null_mask_size, 0xFF); - host_mask[0] &= static_cast(~(1u << 1)); // null at index 1 - host_mask[0] &= static_cast(~(1u << 3)); // null at index 3 - rmm::device_buffer dev_mask(host_mask.data(), host_mask.size(), shared_stream()); - shared_stream().synchronize(); - - auto str_col = cudf::make_strings_column( - num_strings, std::move(offsets_col), std::move(dev_chars), 2, std::move(dev_mask)); - - std::vector> cols; - cols.push_back(std::move(str_col)); - round_trip_test(std::make_unique(std::move(cols))); -} - -// ============================================================================= -// LIST column round-trip tests (TEST-05) -// ============================================================================= - -TEST_CASE("host_data disk round-trip list column", "[disk][converter][list]") -{ - // LIST column - - // Create offsets: [0, 2, 5, 5, 8] -> 4 lists with lengths [2, 3, 0, 3] - std::vector host_offsets = {0, 2, 5, 5, 8}; - auto const num_lists = static_cast(host_offsets.size() - 1); - - auto offsets_col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_lists + 1, - cudf::mask_state::UNALLOCATED, - shared_stream()); - CUCASCADE_CUDA_TRY(cudaMemcpyAsync(offsets_col->mutable_view().data(), - host_offsets.data(), - host_offsets.size() * sizeof(int32_t), - cudaMemcpyHostToDevice, - shared_stream().value())); - - // Create values child: 8 INT32 values - auto values_col = cudf::make_numeric_column( - cudf::data_type{cudf::type_id::INT32}, 8, cudf::mask_state::UNALLOCATED, shared_stream()); - - shared_stream().synchronize(); - - auto list_col = cudf::make_lists_column( - num_lists, std::move(offsets_col), std::move(values_col), 0, rmm::device_buffer{}); - - std::vector> cols; - cols.push_back(std::move(list_col)); - round_trip_test(std::make_unique(std::move(cols))); -} - -TEST_CASE("host_data disk round-trip nested list column", "[disk][converter][list][nested]") -{ - // LIST> column - - // Inner lists: offsets [0, 2, 3] -> 2 inner lists - std::vector inner_offsets = {0, 2, 3}; - auto inner_offsets_col = cudf::make_numeric_column( - cudf::data_type{cudf::type_id::INT32}, 3, cudf::mask_state::UNALLOCATED, shared_stream()); - CUCASCADE_CUDA_TRY(cudaMemcpyAsync(inner_offsets_col->mutable_view().data(), - inner_offsets.data(), - inner_offsets.size() * sizeof(int32_t), - cudaMemcpyHostToDevice, - shared_stream().value())); - - auto inner_values = cudf::make_numeric_column( - cudf::data_type{cudf::type_id::INT32}, 3, cudf::mask_state::UNALLOCATED, shared_stream()); - - auto inner_list = cudf::make_lists_column( - 2, std::move(inner_offsets_col), std::move(inner_values), 0, rmm::device_buffer{}); - - // Outer lists: offsets [0, 1, 2] -> 2 outer lists, each containing 1 inner list - std::vector outer_offsets = {0, 1, 2}; - auto outer_offsets_col = cudf::make_numeric_column( - cudf::data_type{cudf::type_id::INT32}, 3, cudf::mask_state::UNALLOCATED, shared_stream()); - CUCASCADE_CUDA_TRY(cudaMemcpyAsync(outer_offsets_col->mutable_view().data(), - outer_offsets.data(), - outer_offsets.size() * sizeof(int32_t), - cudaMemcpyHostToDevice, - shared_stream().value())); - - shared_stream().synchronize(); - - auto outer_list = cudf::make_lists_column( - 2, std::move(outer_offsets_col), std::move(inner_list), 0, rmm::device_buffer{}); - - std::vector> cols; - cols.push_back(std::move(outer_list)); - round_trip_test(std::make_unique(std::move(cols))); -} - -// ============================================================================= -// STRUCT column round-trip tests (TEST-06) -// ============================================================================= - -TEST_CASE("host_data disk round-trip struct column", "[disk][converter][struct]") -{ - // STRUCT - - auto const num_rows = cudf::size_type{50}; - - auto int_child = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_rows, - cudf::mask_state::UNALLOCATED, - shared_stream()); - auto float_child = cudf::make_numeric_column(cudf::data_type{cudf::type_id::FLOAT64}, - num_rows, - cudf::mask_state::UNALLOCATED, - shared_stream()); - - std::vector> children; - children.push_back(std::move(int_child)); - children.push_back(std::move(float_child)); - - auto struct_col = - cudf::make_structs_column(num_rows, std::move(children), 0, rmm::device_buffer{}); - - std::vector> cols; - cols.push_back(std::move(struct_col)); - round_trip_test(std::make_unique(std::move(cols))); -} - -TEST_CASE("host_data disk round-trip nested struct column", "[disk][converter][struct][nested]") -{ - // STRUCT> - - auto const num_rows = cudf::size_type{30}; - - auto inner_child = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_rows, - cudf::mask_state::UNALLOCATED, - shared_stream()); - - std::vector> inner_children; - inner_children.push_back(std::move(inner_child)); - auto inner_struct = - cudf::make_structs_column(num_rows, std::move(inner_children), 0, rmm::device_buffer{}); - - std::vector> outer_children; - outer_children.push_back(std::move(inner_struct)); - auto outer_struct = - cudf::make_structs_column(num_rows, std::move(outer_children), 0, rmm::device_buffer{}); - - std::vector> cols; - cols.push_back(std::move(outer_struct)); - round_trip_test(std::make_unique(std::move(cols))); -} - -TEST_CASE("host_data disk round-trip struct with null mask", "[disk][converter][struct][null]") -{ - // STRUCT with struct-level null mask - - auto const num_rows = cudf::size_type{40}; - - auto child = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_rows, - cudf::mask_state::UNALLOCATED, - shared_stream()); - - // Create struct-level null mask (every 3rd element is null) - auto null_mask_size = cudf::bitmask_allocation_size_bytes(num_rows); - std::vector host_mask(null_mask_size, 0xFF); - cudf::size_type null_count = 0; - for (int i = 0; i < num_rows; i += 3) { - auto byte_idx = static_cast(i / 8); - auto bit_idx = static_cast(i % 8); - host_mask[byte_idx] &= static_cast(~(1u << bit_idx)); - null_count++; - } - rmm::device_buffer dev_mask(host_mask.data(), host_mask.size(), shared_stream()); - - std::vector> children; - children.push_back(std::move(child)); - - shared_stream().synchronize(); - auto struct_col = - cudf::make_structs_column(num_rows, std::move(children), null_count, std::move(dev_mask)); - - std::vector> cols; - cols.push_back(std::move(struct_col)); - round_trip_test(std::make_unique(std::move(cols))); -} - -// ============================================================================= -// DICTIONARY column round-trip tests (TEST-07) -// ============================================================================= - -TEST_CASE("host_data disk round-trip dictionary column", "[disk][converter][dictionary]") -{ - // DICTIONARY32 with INT32 keys - - auto const num_rows = cudf::size_type{50}; - auto const num_keys = cudf::size_type{5}; - - // Keys: 5 unique INT32 values - auto keys = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_keys, - cudf::mask_state::UNALLOCATED, - shared_stream()); - - // Indices: INT32 values in [0, num_keys) - auto indices = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_rows, - cudf::mask_state::UNALLOCATED, - shared_stream()); - - shared_stream().synchronize(); - - auto dict_col = cudf::make_dictionary_column(std::move(keys), std::move(indices)); - - std::vector> cols; - cols.push_back(std::move(dict_col)); - round_trip_test(std::make_unique(std::move(cols))); -} - -// ============================================================================= -// Sliced column round-trip tests (TEST-09) -// ============================================================================= - -TEST_CASE("host_data disk round-trip sliced column", "[disk][converter][sliced]") -{ - // Create a column, slice it, compact it, then round-trip - - auto const total_rows = cudf::size_type{200}; - auto const slice_start = cudf::size_type{50}; - auto const slice_end = cudf::size_type{150}; - - auto col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - total_rows, - cudf::mask_state::ALL_VALID, - shared_stream()); - - std::vector> table_cols; - table_cols.push_back(std::move(col)); - auto table = std::make_unique(std::move(table_cols)); - - // Slice to get a view with non-zero offset - auto sliced_views = cudf::slice(table->view(), {slice_start, slice_end}); - REQUIRE(sliced_views.size() == 1); - REQUIRE(sliced_views[0].num_rows() == slice_end - slice_start); - - // Compact the slice into a new table (removes offset) - auto compacted = std::make_unique(sliced_views[0], shared_stream()); - REQUIRE(compacted->num_rows() == slice_end - slice_start); - - round_trip_test(std::move(compacted)); -} - -TEST_CASE("host_data disk round-trip sliced column with nulls", "[disk][converter][sliced][null]") -{ - auto const total_rows = cudf::size_type{200}; - - auto col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - total_rows, - cudf::mask_state::ALL_VALID, - shared_stream()); - - // Set some nulls - auto null_mask_size = cudf::bitmask_allocation_size_bytes(total_rows); - std::vector host_mask(null_mask_size, 0xFF); - cudf::size_type null_count = 0; - for (int i = 0; i < total_rows; i += 5) { - host_mask[static_cast(i / 8)] &= static_cast(~(1u << (i % 8))); - null_count++; - } - rmm::device_buffer dev_mask(host_mask.data(), host_mask.size(), shared_stream()); - col->set_null_mask(std::move(dev_mask), null_count); - - std::vector> table_cols; - table_cols.push_back(std::move(col)); - auto table = std::make_unique(std::move(table_cols)); - - // Slice middle portion - auto sliced_views = cudf::slice(table->view(), {cudf::size_type{50}, cudf::size_type{150}}); - shared_stream().synchronize(); - auto compacted = std::make_unique(sliced_views[0], shared_stream()); - - round_trip_test(std::move(compacted)); -} diff --git a/test/data/test_gpu_disk_converters.cpp b/test/data/test_gpu_disk_converters.cpp deleted file mode 100644 index 3035973..0000000 --- a/test/data/test_gpu_disk_converters.cpp +++ /dev/null @@ -1,702 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "utils/cudf_test_utils.hpp" -#include "utils/mock_test_utils.hpp" - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include - -#include -#include -#include - -using namespace cucascade; - -namespace { - -/// Shared memory spaces — reused across all tests to avoid cumulative CUDA context -/// degradation from repeated creation/destruction of CUDA stream pools and pinned memory. -auto& shared_gpu_space() -{ - static auto s = test::make_mock_memory_space(memory::Tier::GPU, 0); - return s; -} -auto& shared_disk_space() -{ - static auto s = test::make_mock_memory_space(memory::Tier::DISK, 0); - return s; -} -rmm::cuda_stream_view shared_stream() -{ - static rmm::cuda_stream s; - return s.view(); -} - -/// Round-trip test helper: GPU -> disk -> GPU (direct 3-step), compare tables. -void gpu_disk_round_trip_test(std::unique_ptr original_table) -{ - auto stream = shared_stream(); - auto& gpu_space = shared_gpu_space(); - auto& disk_space = shared_disk_space(); - - representation_converter_registry registry; - register_builtin_converters(registry); - - // Create GPU representation from the original table - auto gpu_rep = std::make_unique( - std::move(original_table), *gpu_space, rmm::cuda_stream_view{}); - - // GPU -> disk (direct via write/write_batch) - auto disk_rep = - registry.convert(*gpu_rep, disk_space.get(), shared_stream()); - - // disk -> GPU (direct via read) - auto gpu_rep2 = - registry.convert(*disk_rep, gpu_space.get(), shared_stream()); - - // Compare original (from gpu_rep) with round-tripped - test::expect_cudf_tables_equal_on_stream( - gpu_rep->get_table_view(), gpu_rep2->get_table_view(), shared_stream()); -} - -/// Create a simple column of the given type with uninitialized data. -/// Returns a single-column cudf::table. -std::unique_ptr make_typed_table(cudf::type_id type, cudf::size_type num_rows) -{ - auto const dtype = cudf::data_type{type}; - std::vector> cols; - if (cudf::is_numeric(dtype)) { - // cudf::is_numeric includes BOOL8 - auto col = - cudf::make_numeric_column(dtype, num_rows, cudf::mask_state::UNALLOCATED, shared_stream()); - cols.push_back(std::move(col)); - } else if (cudf::is_timestamp(dtype)) { - auto col = - cudf::make_timestamp_column(dtype, num_rows, cudf::mask_state::UNALLOCATED, shared_stream()); - cols.push_back(std::move(col)); - } else if (cudf::is_duration(dtype)) { - auto col = - cudf::make_duration_column(dtype, num_rows, cudf::mask_state::UNALLOCATED, shared_stream()); - cols.push_back(std::move(col)); - } else if (cudf::is_fixed_point(dtype)) { - auto const fp_dtype = cudf::data_type{type, -2}; // scale = -2 - auto col = cudf::make_fixed_point_column( - fp_dtype, num_rows, cudf::mask_state::UNALLOCATED, shared_stream()); - cols.push_back(std::move(col)); - } - return std::make_unique(std::move(cols)); -} - -/// Create a table with ALL_NULL mask. -std::unique_ptr make_typed_table_with_nulls(cudf::type_id type, - cudf::size_type num_rows) -{ - auto const dtype = cudf::data_type{type}; - std::vector> cols; - if (cudf::is_numeric(dtype)) { - auto col = - cudf::make_numeric_column(dtype, num_rows, cudf::mask_state::ALL_NULL, shared_stream()); - cols.push_back(std::move(col)); - } else if (cudf::is_timestamp(dtype)) { - auto col = - cudf::make_timestamp_column(dtype, num_rows, cudf::mask_state::ALL_NULL, shared_stream()); - cols.push_back(std::move(col)); - } else if (cudf::is_duration(dtype)) { - auto col = - cudf::make_duration_column(dtype, num_rows, cudf::mask_state::ALL_NULL, shared_stream()); - cols.push_back(std::move(col)); - } else if (cudf::is_fixed_point(dtype)) { - auto const fp_dtype = cudf::data_type{type, -2}; - auto col = cudf::make_fixed_point_column( - fp_dtype, num_rows, cudf::mask_state::ALL_NULL, shared_stream()); - cols.push_back(std::move(col)); - } - return std::make_unique(std::move(cols)); -} - -} // namespace - -// ============================================================================= -// Numeric type round-trip tests -// ============================================================================= - -TEST_CASE("gpu disk round-trip numeric types", "[disk][gpu-converter][numeric]") -{ - auto type = GENERATE(cudf::type_id::INT8, - cudf::type_id::INT16, - cudf::type_id::INT32, - cudf::type_id::INT64, - cudf::type_id::UINT8, - cudf::type_id::UINT16, - cudf::type_id::UINT32, - cudf::type_id::UINT64, - cudf::type_id::FLOAT32, - cudf::type_id::FLOAT64); - - SECTION("100 rows") { gpu_disk_round_trip_test(make_typed_table(type, 100)); } -} - -// ============================================================================= -// BOOL8 round-trip tests -// ============================================================================= - -TEST_CASE("gpu disk round-trip bool type", "[disk][gpu-converter][bool]") -{ - gpu_disk_round_trip_test(make_typed_table(cudf::type_id::BOOL8, 100)); -} - -// ============================================================================= -// Timestamp round-trip tests -// ============================================================================= - -TEST_CASE("gpu disk round-trip timestamp types", "[disk][gpu-converter][timestamp]") -{ - auto type = GENERATE(cudf::type_id::TIMESTAMP_DAYS, - cudf::type_id::TIMESTAMP_SECONDS, - cudf::type_id::TIMESTAMP_MILLISECONDS, - cudf::type_id::TIMESTAMP_MICROSECONDS, - cudf::type_id::TIMESTAMP_NANOSECONDS); - - gpu_disk_round_trip_test(make_typed_table(type, 100)); -} - -// ============================================================================= -// Duration round-trip tests -// ============================================================================= - -TEST_CASE("gpu disk round-trip duration types", "[disk][gpu-converter][duration]") -{ - auto type = GENERATE(cudf::type_id::DURATION_DAYS, - cudf::type_id::DURATION_SECONDS, - cudf::type_id::DURATION_MILLISECONDS, - cudf::type_id::DURATION_MICROSECONDS, - cudf::type_id::DURATION_NANOSECONDS); - - gpu_disk_round_trip_test(make_typed_table(type, 100)); -} - -// ============================================================================= -// Decimal round-trip tests -// ============================================================================= - -TEST_CASE("gpu disk round-trip decimal types", "[disk][gpu-converter][decimal]") -{ - auto type = - GENERATE(cudf::type_id::DECIMAL32, cudf::type_id::DECIMAL64, cudf::type_id::DECIMAL128); - - gpu_disk_round_trip_test(make_typed_table(type, 100)); -} - -// ============================================================================= -// Null mask round-trip tests -// ============================================================================= - -TEST_CASE("gpu disk round-trip all-null column", "[disk][gpu-converter][null]") -{ - gpu_disk_round_trip_test(make_typed_table_with_nulls(cudf::type_id::INT32, 100)); -} - -TEST_CASE("gpu disk round-trip no-null column", "[disk][gpu-converter][null]") -{ - gpu_disk_round_trip_test(make_typed_table(cudf::type_id::INT32, 100)); -} - -TEST_CASE("gpu disk round-trip mixed-null column", "[disk][gpu-converter][null]") -{ - // Create a column with alternating nulls - - auto col = cudf::make_numeric_column( - cudf::data_type{cudf::type_id::INT32}, 100, cudf::mask_state::ALL_VALID, shared_stream()); - - // Set every other bit to null using cudf utilities - auto const num_rows = 100; - std::vector host_mask(cudf::bitmask_allocation_size_bytes(num_rows), 0xFF); - // Clear every other bit (set elements 0, 2, 4, ... to null) - for (int i = 0; i < num_rows; i += 2) { - auto byte_idx = static_cast(i / 8); - auto bit_idx = static_cast(i % 8); - host_mask[byte_idx] &= static_cast(~(1u << bit_idx)); - } - rmm::device_buffer dev_mask(host_mask.data(), host_mask.size(), shared_stream()); - col->set_null_mask(std::move(dev_mask), 50); // 50 nulls out of 100 - - std::vector> cols; - cols.push_back(std::move(col)); - gpu_disk_round_trip_test(std::make_unique(std::move(cols))); -} - -// ============================================================================= -// STRING column round-trip tests -// ============================================================================= - -TEST_CASE("gpu disk round-trip string column", "[disk][gpu-converter][string]") -{ - std::vector host_strings = {"hello", "", "world", "test123", "", "a"}; - auto const num_strings = static_cast(host_strings.size()); - - // Compute offsets - std::vector host_offsets(static_cast(num_strings) + 1); - host_offsets[0] = 0; - for (std::size_t i = 0; i < host_strings.size(); ++i) { - host_offsets[i + 1] = host_offsets[i] + static_cast(host_strings[i].size()); - } - auto const total_chars = static_cast(host_offsets.back()); - - // Concatenate all chars - std::vector host_chars; - host_chars.reserve(total_chars); - for (const auto& s : host_strings) { - host_chars.insert(host_chars.end(), s.begin(), s.end()); - } - - // Create device buffers - rmm::device_buffer dev_chars(host_chars.data(), host_chars.size(), shared_stream()); - auto offsets_col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_strings + 1, - cudf::mask_state::UNALLOCATED, - shared_stream()); - CUCASCADE_CUDA_TRY(cudaMemcpyAsync(offsets_col->mutable_view().data(), - host_offsets.data(), - host_offsets.size() * sizeof(int32_t), - cudaMemcpyHostToDevice, - shared_stream().value())); - shared_stream().synchronize(); - - auto str_col = cudf::make_strings_column( - num_strings, std::move(offsets_col), std::move(dev_chars), 0, rmm::device_buffer{}); - - std::vector> cols; - cols.push_back(std::move(str_col)); - gpu_disk_round_trip_test(std::make_unique(std::move(cols))); -} - -// Regression: see test_disk_host_converters.cpp — same root cause, but here we exercise the -// direct disk→GPU path through reconstruct_column_from_disk. -TEST_CASE("gpu disk round-trip empty strings column", "[disk][gpu-converter][string][empty]") -{ - std::vector> cols; - cols.push_back(cudf::make_empty_column(cudf::data_type{cudf::type_id::STRING})); - gpu_disk_round_trip_test(std::make_unique(std::move(cols))); -} - -TEST_CASE("gpu disk round-trip empty strings column mixed with non-empty column", - "[disk][gpu-converter][string][empty]") -{ - std::vector> cols; - cols.push_back(cudf::make_empty_column(cudf::data_type{cudf::type_id::INT32})); - cols.push_back(cudf::make_empty_column(cudf::data_type{cudf::type_id::STRING})); - cols.push_back(cudf::make_empty_column(cudf::data_type{cudf::type_id::INT64})); - gpu_disk_round_trip_test(std::make_unique(std::move(cols))); -} - -TEST_CASE("gpu disk round-trip string column with nulls", "[disk][gpu-converter][string][null]") -{ - std::vector host_strings = {"alpha", "beta", "gamma", "delta"}; - auto const num_strings = static_cast(host_strings.size()); - - std::vector host_offsets(static_cast(num_strings) + 1); - host_offsets[0] = 0; - for (std::size_t i = 0; i < host_strings.size(); ++i) { - host_offsets[i + 1] = host_offsets[i] + static_cast(host_strings[i].size()); - } - - std::vector host_chars; - for (const auto& s : host_strings) { - host_chars.insert(host_chars.end(), s.begin(), s.end()); - } - - rmm::device_buffer dev_chars(host_chars.data(), host_chars.size(), shared_stream()); - auto offsets_col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_strings + 1, - cudf::mask_state::UNALLOCATED, - shared_stream()); - CUCASCADE_CUDA_TRY(cudaMemcpyAsync(offsets_col->mutable_view().data(), - host_offsets.data(), - host_offsets.size() * sizeof(int32_t), - cudaMemcpyHostToDevice, - shared_stream().value())); - - // Create null mask: elements 1 and 3 are null - auto null_mask_size = cudf::bitmask_allocation_size_bytes(num_strings); - std::vector host_mask(null_mask_size, 0xFF); - host_mask[0] &= static_cast(~(1u << 1)); // null at index 1 - host_mask[0] &= static_cast(~(1u << 3)); // null at index 3 - rmm::device_buffer dev_mask(host_mask.data(), host_mask.size(), shared_stream()); - shared_stream().synchronize(); - - auto str_col = cudf::make_strings_column( - num_strings, std::move(offsets_col), std::move(dev_chars), 2, std::move(dev_mask)); - - std::vector> cols; - cols.push_back(std::move(str_col)); - gpu_disk_round_trip_test(std::make_unique(std::move(cols))); -} - -// ============================================================================= -// LIST column round-trip tests -// ============================================================================= - -TEST_CASE("gpu disk round-trip list column", "[disk][gpu-converter][list]") -{ - // LIST column - - // Create offsets: [0, 2, 5, 5, 8] -> 4 lists with lengths [2, 3, 0, 3] - std::vector host_offsets = {0, 2, 5, 5, 8}; - auto const num_lists = static_cast(host_offsets.size() - 1); - - auto offsets_col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_lists + 1, - cudf::mask_state::UNALLOCATED, - shared_stream()); - CUCASCADE_CUDA_TRY(cudaMemcpyAsync(offsets_col->mutable_view().data(), - host_offsets.data(), - host_offsets.size() * sizeof(int32_t), - cudaMemcpyHostToDevice, - shared_stream().value())); - - // Create values child: 8 INT32 values - auto values_col = cudf::make_numeric_column( - cudf::data_type{cudf::type_id::INT32}, 8, cudf::mask_state::UNALLOCATED, shared_stream()); - - shared_stream().synchronize(); - - auto list_col = cudf::make_lists_column( - num_lists, std::move(offsets_col), std::move(values_col), 0, rmm::device_buffer{}); - - std::vector> cols; - cols.push_back(std::move(list_col)); - gpu_disk_round_trip_test(std::make_unique(std::move(cols))); -} - -TEST_CASE("gpu disk round-trip nested list column", "[disk][gpu-converter][list][nested]") -{ - // LIST> column - - // Inner lists: offsets [0, 2, 3] -> 2 inner lists - std::vector inner_offsets = {0, 2, 3}; - auto inner_offsets_col = cudf::make_numeric_column( - cudf::data_type{cudf::type_id::INT32}, 3, cudf::mask_state::UNALLOCATED, shared_stream()); - CUCASCADE_CUDA_TRY(cudaMemcpyAsync(inner_offsets_col->mutable_view().data(), - inner_offsets.data(), - inner_offsets.size() * sizeof(int32_t), - cudaMemcpyHostToDevice, - shared_stream().value())); - - auto inner_values = cudf::make_numeric_column( - cudf::data_type{cudf::type_id::INT32}, 3, cudf::mask_state::UNALLOCATED, shared_stream()); - - auto inner_list = cudf::make_lists_column( - 2, std::move(inner_offsets_col), std::move(inner_values), 0, rmm::device_buffer{}); - - // Outer lists: offsets [0, 1, 2] -> 2 outer lists, each containing 1 inner list - std::vector outer_offsets = {0, 1, 2}; - auto outer_offsets_col = cudf::make_numeric_column( - cudf::data_type{cudf::type_id::INT32}, 3, cudf::mask_state::UNALLOCATED, shared_stream()); - CUCASCADE_CUDA_TRY(cudaMemcpyAsync(outer_offsets_col->mutable_view().data(), - outer_offsets.data(), - outer_offsets.size() * sizeof(int32_t), - cudaMemcpyHostToDevice, - shared_stream().value())); - - shared_stream().synchronize(); - - auto outer_list = cudf::make_lists_column( - 2, std::move(outer_offsets_col), std::move(inner_list), 0, rmm::device_buffer{}); - - std::vector> cols; - cols.push_back(std::move(outer_list)); - gpu_disk_round_trip_test(std::make_unique(std::move(cols))); -} - -// ============================================================================= -// STRUCT column round-trip tests -// ============================================================================= - -TEST_CASE("gpu disk round-trip struct column", "[disk][gpu-converter][struct]") -{ - // STRUCT - - auto const num_rows = cudf::size_type{50}; - - auto int_child = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_rows, - cudf::mask_state::UNALLOCATED, - shared_stream()); - auto float_child = cudf::make_numeric_column(cudf::data_type{cudf::type_id::FLOAT64}, - num_rows, - cudf::mask_state::UNALLOCATED, - shared_stream()); - - std::vector> children; - children.push_back(std::move(int_child)); - children.push_back(std::move(float_child)); - - auto struct_col = - cudf::make_structs_column(num_rows, std::move(children), 0, rmm::device_buffer{}); - - std::vector> cols; - cols.push_back(std::move(struct_col)); - gpu_disk_round_trip_test(std::make_unique(std::move(cols))); -} - -TEST_CASE("gpu disk round-trip nested struct column", "[disk][gpu-converter][struct][nested]") -{ - // STRUCT> - - auto const num_rows = cudf::size_type{30}; - - auto inner_child = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_rows, - cudf::mask_state::UNALLOCATED, - shared_stream()); - - std::vector> inner_children; - inner_children.push_back(std::move(inner_child)); - auto inner_struct = - cudf::make_structs_column(num_rows, std::move(inner_children), 0, rmm::device_buffer{}); - - std::vector> outer_children; - outer_children.push_back(std::move(inner_struct)); - auto outer_struct = - cudf::make_structs_column(num_rows, std::move(outer_children), 0, rmm::device_buffer{}); - - std::vector> cols; - cols.push_back(std::move(outer_struct)); - gpu_disk_round_trip_test(std::make_unique(std::move(cols))); -} - -TEST_CASE("gpu disk round-trip struct with null mask", "[disk][gpu-converter][struct][null]") -{ - // STRUCT with struct-level null mask - - auto const num_rows = cudf::size_type{40}; - - auto child = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_rows, - cudf::mask_state::UNALLOCATED, - shared_stream()); - - // Create struct-level null mask (every 3rd element is null) - auto null_mask_size = cudf::bitmask_allocation_size_bytes(num_rows); - std::vector host_mask(null_mask_size, 0xFF); - cudf::size_type null_count = 0; - for (int i = 0; i < num_rows; i += 3) { - auto byte_idx = static_cast(i / 8); - auto bit_idx = static_cast(i % 8); - host_mask[byte_idx] &= static_cast(~(1u << bit_idx)); - null_count++; - } - rmm::device_buffer dev_mask(host_mask.data(), host_mask.size(), shared_stream()); - - std::vector> children; - children.push_back(std::move(child)); - - shared_stream().synchronize(); - auto struct_col = - cudf::make_structs_column(num_rows, std::move(children), null_count, std::move(dev_mask)); - - std::vector> cols; - cols.push_back(std::move(struct_col)); - gpu_disk_round_trip_test(std::make_unique(std::move(cols))); -} - -// ============================================================================= -// DICTIONARY column round-trip tests -// ============================================================================= - -TEST_CASE("gpu disk round-trip dictionary column", "[disk][gpu-converter][dictionary]") -{ - // DICTIONARY32 with INT32 keys - - auto const num_rows = cudf::size_type{50}; - auto const num_keys = cudf::size_type{5}; - - // Keys: 5 unique INT32 values - auto keys = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_keys, - cudf::mask_state::UNALLOCATED, - shared_stream()); - - // Indices: INT32 values in [0, num_keys) - auto indices = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - num_rows, - cudf::mask_state::UNALLOCATED, - shared_stream()); - - shared_stream().synchronize(); - - auto dict_col = cudf::make_dictionary_column(std::move(keys), std::move(indices)); - - std::vector> cols; - cols.push_back(std::move(dict_col)); - gpu_disk_round_trip_test(std::make_unique(std::move(cols))); -} - -// ============================================================================= -// Sliced column round-trip tests -// ============================================================================= - -TEST_CASE("gpu disk round-trip sliced column", "[disk][gpu-converter][sliced]") -{ - // Create a column, slice it, compact it, then round-trip - - auto const total_rows = cudf::size_type{200}; - auto const slice_start = cudf::size_type{50}; - auto const slice_end = cudf::size_type{150}; - - auto col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - total_rows, - cudf::mask_state::ALL_VALID, - shared_stream()); - - std::vector> table_cols; - table_cols.push_back(std::move(col)); - auto table = std::make_unique(std::move(table_cols)); - - // Slice to get a view with non-zero offset - auto sliced_views = cudf::slice(table->view(), {slice_start, slice_end}); - REQUIRE(sliced_views.size() == 1); - REQUIRE(sliced_views[0].num_rows() == slice_end - slice_start); - - // Compact the slice into a new table (removes offset) - auto compacted = std::make_unique(sliced_views[0], shared_stream()); - REQUIRE(compacted->num_rows() == slice_end - slice_start); - - gpu_disk_round_trip_test(std::move(compacted)); -} - -TEST_CASE("gpu disk round-trip sliced column with nulls", "[disk][gpu-converter][sliced][null]") -{ - auto const total_rows = cudf::size_type{200}; - - auto col = cudf::make_numeric_column(cudf::data_type{cudf::type_id::INT32}, - total_rows, - cudf::mask_state::ALL_VALID, - shared_stream()); - - // Set some nulls - auto null_mask_size = cudf::bitmask_allocation_size_bytes(total_rows); - std::vector host_mask(null_mask_size, 0xFF); - cudf::size_type null_count = 0; - for (int i = 0; i < total_rows; i += 5) { - host_mask[static_cast(i / 8)] &= static_cast(~(1u << (i % 8))); - null_count++; - } - rmm::device_buffer dev_mask(host_mask.data(), host_mask.size(), shared_stream()); - col->set_null_mask(std::move(dev_mask), null_count); - - std::vector> table_cols; - table_cols.push_back(std::move(col)); - auto table = std::make_unique(std::move(table_cols)); - - // Slice middle portion - auto sliced_views = cudf::slice(table->view(), {cudf::size_type{50}, cudf::size_type{150}}); - shared_stream().synchronize(); - auto compacted = std::make_unique(sliced_views[0], shared_stream()); - - gpu_disk_round_trip_test(std::move(compacted)); -} - -// ============================================================================= -// I/O backend selection tests (TEST-10) -// ============================================================================= - -TEST_CASE("gpu disk round-trip with explicit pipeline backend", - "[disk][gpu-converter][backend][pipeline]") -{ - auto gpu_space = test::make_mock_memory_space(memory::Tier::GPU, 0); - auto disk_space = test::make_mock_memory_space(memory::Tier::DISK, 0); - - representation_converter_registry registry; - register_builtin_converters(registry); - - // Simple INT32 column, 100 rows - auto table = make_typed_table(cudf::type_id::INT32, 100); - auto gpu_rep = std::make_unique( - std::move(table), *gpu_space, rmm::cuda_stream_view{}); - - auto disk_rep = - registry.convert(*gpu_rep, disk_space.get(), shared_stream()); - auto gpu_rep2 = - registry.convert(*disk_rep, gpu_space.get(), shared_stream()); - - test::expect_cudf_tables_equal_on_stream( - gpu_rep->get_table_view(), gpu_rep2->get_table_view(), shared_stream()); -} - -TEST_CASE("gpu disk round-trip pipeline with multiple types", - "[disk][gpu-converter][backend][pipeline]") -{ - auto gpu_space = test::make_mock_memory_space(memory::Tier::GPU, 0); - auto disk_space = test::make_mock_memory_space(memory::Tier::DISK, 0); - - representation_converter_registry registry; - register_builtin_converters(registry); - - // Test with multiple numeric types - auto type_id = GENERATE(cudf::type_id::INT32, cudf::type_id::INT64, cudf::type_id::FLOAT64); - auto table = make_typed_table(type_id, 1000); - auto gpu_rep = std::make_unique( - std::move(table), *gpu_space, rmm::cuda_stream_view{}); - - auto disk_rep = - registry.convert(*gpu_rep, disk_space.get(), shared_stream()); - - REQUIRE(disk_rep->get_size_in_bytes() > 0); - REQUIRE(disk_rep->get_uncompressed_data_size_in_bytes() == disk_rep->get_size_in_bytes()); - - auto gpu_rep2 = - registry.convert(*disk_rep, gpu_space.get(), shared_stream()); - - test::expect_cudf_tables_equal_on_stream( - gpu_rep->get_table_view(), gpu_rep2->get_table_view(), shared_stream()); -} - -TEST_CASE("disk_data_representation get_uncompressed_data_size_in_bytes", "[disk][representation]") -{ - auto gpu_space = test::make_mock_memory_space(memory::Tier::GPU, 0); - auto disk_space = test::make_mock_memory_space(memory::Tier::DISK, 0); - - representation_converter_registry registry; - register_builtin_converters(registry); - - auto table = make_typed_table(cudf::type_id::INT64, 1000); - auto gpu_rep = std::make_unique( - std::move(table), *gpu_space, rmm::cuda_stream_view{}); - - auto disk_rep = - registry.convert(*gpu_rep, disk_space.get(), shared_stream()); - - REQUIRE(disk_rep->get_size_in_bytes() > 0); - REQUIRE(disk_rep->get_uncompressed_data_size_in_bytes() == disk_rep->get_size_in_bytes()); -} diff --git a/test/data/test_representation_converter.cpp b/test/data/test_representation_converter.cpp index 7d5222b..e753942 100644 --- a/test/data/test_representation_converter.cpp +++ b/test/data/test_representation_converter.cpp @@ -15,11 +15,8 @@ * limitations under the License. */ -#include "utils/cudf_test_utils.hpp" #include "utils/mock_test_utils.hpp" -#include -#include #include #include @@ -31,8 +28,6 @@ #include using namespace cucascade; -using cucascade::test::create_conversion_test_configs; -using cucascade::test::create_simple_cudf_table; using cucascade::test::make_mock_memory_space; // ============================================================================= @@ -251,12 +246,6 @@ TEST_CASE("representation_converter_registry unregister_converter", "[representa } } -// Note: We don't test clear() directly on the global registry because: -// 1. It removes builtin converters (GPU<->HOST) -// 2. The static flag in register_builtin_converters() prevents re-registration -// 3. This would break other tests that depend on builtin converters -// The unregister_converter tests above verify the removal functionality safely. - // ============================================================================= // representation_converter_registry Conversion Tests // ============================================================================= @@ -332,131 +321,6 @@ TEST_CASE("representation_converter_registry convert with type_index", "[represe } } -// ============================================================================= -// Built-in Converter Registration Tests -// ============================================================================= - -TEST_CASE("register_builtin_converters registers all expected converters", - "[representation_converter][builtin]") -{ - representation_converter_registry registry; - register_builtin_converters(registry); - - SECTION("GPU to HOST converter is registered") - { - REQUIRE(registry.has_converter()); - } - - SECTION("HOST to GPU converter is registered") - { - REQUIRE(registry.has_converter()); - } - - SECTION("GPU to GPU converter is registered") - { - REQUIRE(registry.has_converter()); - } - - SECTION("HOST to HOST converter is registered") - { - REQUIRE( - registry.has_converter()); - } -} - -// ============================================================================= -// Built-in Converter Functional Tests -// ============================================================================= - -TEST_CASE("Built-in GPU to HOST conversion works", "[representation_converter][builtin][gpu_host]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const memory::memory_space* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const memory::memory_space* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - - // Use the same stream for table creation and conversion to avoid stream-ordered races - rmm::cuda_stream stream; - auto table = create_simple_cudf_table(50, 2, gpu_space->get_default_allocator(), stream.view()); - gpu_table_representation gpu_repr(std::make_unique(std::move(table)), - *const_cast(gpu_space), - rmm::cuda_stream_view{}); - - auto host_result = - registry.convert(gpu_repr, host_space, stream); - stream.synchronize(); - - REQUIRE(host_result != nullptr); - REQUIRE(host_result->get_current_tier() == memory::Tier::HOST); - REQUIRE(host_result->get_size_in_bytes() > 0); -} - -TEST_CASE("Built-in HOST to GPU conversion works", "[representation_converter][builtin][gpu_host]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const memory::memory_space* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const memory::memory_space* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - - // Use the same stream for table creation and conversions to avoid stream-ordered races - rmm::cuda_stream stream; - - // First create a GPU repr then convert to host to get a valid host_data_packed_representation - auto table = create_simple_cudf_table(50, 2, gpu_space->get_default_allocator(), stream.view()); - gpu_table_representation gpu_repr(std::make_unique(std::move(table)), - *const_cast(gpu_space), - rmm::cuda_stream_view{}); - - auto host_repr = registry.convert(gpu_repr, host_space, stream); - stream.synchronize(); - - // Now convert back to GPU - auto gpu_result = registry.convert(*host_repr, gpu_space, stream); - stream.synchronize(); - - REQUIRE(gpu_result != nullptr); - REQUIRE(gpu_result->get_current_tier() == memory::Tier::GPU); - REQUIRE(gpu_result->get_table_view().num_rows() == 50); -} - -TEST_CASE("Built-in roundtrip GPU->HOST->GPU preserves data", - "[representation_converter][builtin][roundtrip]") -{ - memory::memory_reservation_manager mgr(create_conversion_test_configs()); - representation_converter_registry registry; - register_builtin_converters(registry); - - const memory::memory_space* gpu_space = mgr.get_memory_space(memory::Tier::GPU, 0); - const memory::memory_space* host_space = mgr.get_memory_space(memory::Tier::HOST, 0); - - // Use the same stream for table creation and conversions to avoid stream-ordered races - rmm::cuda_stream stream; - - auto original_table = - create_simple_cudf_table(100, 2, gpu_space->get_default_allocator(), stream.view()); - gpu_table_representation original_repr(std::make_unique(std::move(original_table)), - *const_cast(gpu_space), - rmm::cuda_stream_view{}); - - // GPU -> HOST - auto host_repr = - registry.convert(original_repr, host_space, stream); - stream.synchronize(); - - // HOST -> GPU - auto result_repr = registry.convert(*host_repr, gpu_space, stream); - stream.synchronize(); - - // Verify data integrity - REQUIRE(result_repr != nullptr); - cucascade::test::expect_cudf_tables_equal_on_stream( - original_repr.get_table_view(), result_repr->get_table_view(), stream); -} - // ============================================================================= // Edge Cases and Error Handling Tests // ============================================================================= diff --git a/test/utils/cudf_test_utils.cpp b/test/utils/cudf_test_utils.cpp deleted file mode 100644 index de9b659..0000000 --- a/test/utils/cudf_test_utils.cpp +++ /dev/null @@ -1,418 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace cucascade { -namespace test { - -// Forward declarations for helpers defined later in this file -static void install_rmm_logging_resource_once(); -static inline bool host_mem_equal(const uint8_t* a, const uint8_t* b, size_t n); - -static void dump_hex_context(const uint8_t* data, - size_t size, - size_t center, - size_t context_len = 64) -{ - size_t start = (center > context_len / 2) ? (center - context_len / 2) : 0; - if (start + context_len > size) { context_len = (size > start) ? (size - start) : 0; } - std::ostringstream oss; - oss << std::hex << std::setfill('0'); - for (size_t i = 0; i < context_len; ++i) { - size_t idx = start + i; - if (i && (i % 16 == 0)) oss << " | "; - if (idx < size) { oss << std::setw(2) << static_cast(data[idx]) << ' '; } - } - std::cout << " hex@" << std::dec << start << " (" << context_len << "B): " << oss.str() - << std::endl - << std::flush; -} - -/** - * @brief Copy device data to a host vector. - */ -static std::vector device_to_host(const void* dev_ptr, std::size_t size) -{ - std::vector host_data(size); - if (size > 0 && dev_ptr != nullptr) { - CUCASCADE_CUDA_TRY(cudaMemcpy(host_data.data(), dev_ptr, size, cudaMemcpyDeviceToHost)); - } - return host_data; -} - -/** - * @brief Read an offsets column (INT32 or INT64) into a host vector of int64_t values. - * - * This normalizes INT32 and INT64 offsets to a common int64_t type so both can be compared - * regardless of the internal offset width. - */ -static std::vector read_offsets_to_host(const cudf::column_view& col) -{ - auto const n = static_cast(col.size()); - std::vector result(n); - if (n == 0) { return result; } - - if (col.type().id() == cudf::type_id::INT64) { - auto host_data = device_to_host(col.head(), n * sizeof(int64_t)); - std::memcpy(result.data(), host_data.data(), n * sizeof(int64_t)); - } else { - // INT32 offsets -- widen to INT64 - auto host_data = device_to_host(col.head(), n * sizeof(int32_t)); - std::vector i32(n); - std::memcpy(i32.data(), host_data.data(), n * sizeof(int32_t)); - for (std::size_t i = 0; i < n; ++i) { - result[i] = static_cast(i32[i]); - } - } - return result; -} - -/** - * @brief Recursively compare two column_views for logical equality. - * - * Handles all column types including compound types (STRING, LIST, STRUCT, DICTIONARY). - * Allows INT32 vs INT64 offsets to differ (compares values after widening to INT64). - */ -static bool columns_equal_recursive(const cudf::column_view& left, - const cudf::column_view& right, - const std::string& path) -{ - // Type check: must match at the logical level - if (left.type().id() != right.type().id()) { - std::cout << "[cudf-equal] " << path - << " type mismatch: left=" << static_cast(left.type().id()) - << " right=" << static_cast(right.type().id()) << std::endl - << std::flush; - return false; - } - - // Size check - if (left.size() != right.size()) { - std::cout << "[cudf-equal] " << path << " size mismatch: left=" << left.size() - << " right=" << right.size() << std::endl - << std::flush; - return false; - } - - // Null count check - if (left.null_count() != right.null_count()) { - std::cout << "[cudf-equal] " << path << " null_count mismatch: left=" << left.null_count() - << " right=" << right.null_count() << std::endl - << std::flush; - return false; - } - - // Compare null masks (only significant bits, ignoring padding beyond num_rows) - if (left.nullable() && right.nullable() && left.size() > 0) { - auto const num_bits = static_cast(left.size()); - auto const num_full_bytes = num_bits / 8; - auto const remaining_bits = num_bits % 8; - auto const bytes_to_read = num_full_bytes + (remaining_bits > 0 ? 1 : 0); - auto const alloc_bytes = cudf::bitmask_allocation_size_bytes(left.size()); - auto left_mask = device_to_host(left.null_mask(), alloc_bytes); - auto right_mask = device_to_host(right.null_mask(), alloc_bytes); - - // Compare full bytes - if (num_full_bytes > 0 && - !host_mem_equal(left_mask.data(), right_mask.data(), num_full_bytes)) { - std::cout << "[cudf-equal] " << path << " null mask differs (full bytes)" << std::endl - << std::flush; - return false; - } - // Compare the last partial byte using a mask for significant bits only - if (remaining_bits > 0) { - auto const last_byte_mask = - static_cast((1u << remaining_bits) - 1u); // e.g., 0b00011111 for 5 bits - if ((left_mask[num_full_bytes] & last_byte_mask) != - (right_mask[num_full_bytes] & last_byte_mask)) { - std::cout << "[cudf-equal] " << path << " null mask differs (last byte)" << std::endl - << std::flush; - return false; - } - } - } - - auto const type_id = left.type().id(); - - // STRING: compare chars data + offsets (normalized to INT64) - if (type_id == cudf::type_id::STRING) { - // Compare chars data via strings_column_view - if (left.size() > 0 && left.num_children() > 0) { - auto left_scv = cudf::strings_column_view(left); - auto right_scv = cudf::strings_column_view(right); - - // Compare offsets (child 0), normalizing INT32/INT64 - auto left_offsets = read_offsets_to_host(left_scv.offsets()); - auto right_offsets = read_offsets_to_host(right_scv.offsets()); - if (left_offsets != right_offsets) { - std::cout << "[cudf-equal] " << path << " STRING offsets differ" << std::endl << std::flush; - return false; - } - - // Compare chars data - rmm::cuda_stream stream; - auto left_chars_size = left_scv.chars_size(stream.view()); - auto right_chars_size = right_scv.chars_size(stream.view()); - if (left_chars_size != right_chars_size) { - std::cout << "[cudf-equal] " << path - << " STRING chars_size mismatch: left=" << left_chars_size - << " right=" << right_chars_size << std::endl - << std::flush; - return false; - } - if (left_chars_size > 0) { - auto left_chars = device_to_host(left.data(), left_chars_size); - auto right_chars = device_to_host(right.data(), right_chars_size); - if (!host_mem_equal(left_chars.data(), right_chars.data(), left_chars_size)) { - std::cout << "[cudf-equal] " << path << " STRING chars data differs" << std::endl - << std::flush; - return false; - } - } - } - return true; - } - - // LIST: compare offsets (normalized) + recurse into values child - if (type_id == cudf::type_id::LIST) { - if (left.num_children() >= 2 && left.size() > 0) { - auto left_offsets = read_offsets_to_host(left.child(0)); - auto right_offsets = read_offsets_to_host(right.child(0)); - if (left_offsets != right_offsets) { - std::cout << "[cudf-equal] " << path << " LIST offsets differ" << std::endl << std::flush; - return false; - } - if (!columns_equal_recursive(left.child(1), right.child(1), path + ".values")) { - return false; - } - } - return true; - } - - // STRUCT: recurse into each child field - if (type_id == cudf::type_id::STRUCT) { - if (left.num_children() != right.num_children()) { - std::cout << "[cudf-equal] " << path - << " STRUCT children count mismatch: left=" << left.num_children() - << " right=" << right.num_children() << std::endl - << std::flush; - return false; - } - for (int i = 0; i < left.num_children(); ++i) { - if (!columns_equal_recursive( - left.child(i), right.child(i), path + ".field" + std::to_string(i))) { - return false; - } - } - return true; - } - - // DICTIONARY: compare keys and indices children - if (type_id == cudf::type_id::DICTIONARY32) { - if (left.num_children() != right.num_children()) { - std::cout << "[cudf-equal] " << path - << " DICTIONARY children count mismatch: left=" << left.num_children() - << " right=" << right.num_children() << std::endl - << std::flush; - return false; - } - for (int i = 0; i < left.num_children(); ++i) { - if (!columns_equal_recursive( - left.child(i), right.child(i), path + ".child" + std::to_string(i))) { - return false; - } - } - return true; - } - - // Fixed-width types: compare data bytes directly - if (left.size() > 0 && cudf::is_fixed_width(left.type())) { - auto data_bytes = static_cast(left.size()) * cudf::size_of(left.type()); - auto left_data = device_to_host(left.head(), data_bytes); - auto right_data = device_to_host(right.head(), data_bytes); - if (!host_mem_equal(left_data.data(), right_data.data(), data_bytes)) { - size_t diff_idx = 0; - for (; diff_idx < data_bytes; ++diff_idx) { - if (left_data[diff_idx] != right_data[diff_idx]) break; - } - std::cout << "[cudf-equal] " << path << " data differs at byte " << diff_idx - << " left=" << static_cast(left_data[diff_idx]) - << " right=" << static_cast(right_data[diff_idx]) << std::endl - << std::flush; - dump_hex_context(left_data.data(), data_bytes, diff_idx); - dump_hex_context(right_data.data(), data_bytes, diff_idx); - return false; - } - } - - return true; -} - -// Stream-aware comparison that recursively compares column content. -// Handles all column types including compound types (STRING, LIST, STRUCT, DICTIONARY) -// and tolerates INT32 vs INT64 offset differences in STRING/LIST columns. -bool cudf_tables_have_equal_contents_on_stream(const cudf::table_view& left, - const cudf::table_view& right, - rmm::cuda_stream_view stream_view) -{ - if (left.num_rows() != right.num_rows()) { - std::cout << "[cudf-equal] row count mismatch: left=" << left.num_rows() - << " right=" << right.num_rows() << std::endl - << std::flush; - return false; - } - if (left.num_columns() != right.num_columns()) { - std::cout << "[cudf-equal] column count mismatch: left=" << left.num_columns() - << " right=" << right.num_columns() << std::endl - << std::flush; - return false; - } - - stream_view.synchronize(); - - for (int col_idx = 0; col_idx < left.num_columns(); ++col_idx) { - if (!columns_equal_recursive( - left.column(col_idx), right.column(col_idx), "col[" + std::to_string(col_idx) + "]")) { - return false; - } - } - - return true; -} - -void expect_cudf_tables_equal_on_stream(const cudf::table_view& left, - const cudf::table_view& right, - rmm::cuda_stream_view stream_view) -{ - REQUIRE(cudf_tables_have_equal_contents_on_stream(left, right, stream_view)); -} - -// Simple logging adaptor to print all RMM device allocations/frees with pointers/sizes/stream/tid -class logging_device_resource { - public: - explicit logging_device_resource(rmm::device_async_resource_ref upstream) : _upstream(upstream) {} - - ~logging_device_resource() = default; - - void* allocate(cuda::stream_ref stream, - std::size_t bytes, - std::size_t alignment = alignof(std::max_align_t)) - { - void* ptr = _upstream.allocate(stream, bytes, alignment); - std::ostringstream oss; - oss << "[rmm-alloc] ptr=" << ptr << " size=" << bytes << " stream=" << stream.get() - << " tid=" << std::this_thread::get_id(); - std::cout << oss.str() << std::endl << std::flush; - return ptr; - } - - void deallocate(cuda::stream_ref stream, - void* ptr, - std::size_t bytes, - std::size_t alignment = alignof(std::max_align_t)) noexcept - { - std::ostringstream oss; - oss << "[rmm-free ] ptr=" << ptr << " size=" << bytes << " stream=" << stream.get() - << " tid=" << std::this_thread::get_id(); - std::cout << oss.str() << std::endl << std::flush; - _upstream.deallocate(stream, ptr, bytes, alignment); - } - - void* allocate_sync(std::size_t bytes, std::size_t alignment = alignof(std::max_align_t)) - { - return allocate(cuda::stream_ref{cudaStream_t{nullptr}}, bytes, alignment); - } - - void deallocate_sync(void* ptr, - std::size_t bytes, - std::size_t alignment = alignof(std::max_align_t)) noexcept - { - deallocate(cuda::stream_ref{cudaStream_t{nullptr}}, ptr, bytes, alignment); - } - - bool operator==(logging_device_resource const& other) const noexcept { return this == &other; } - - friend void get_property(logging_device_resource const&, cuda::mr::device_accessible) noexcept {} - - private: - rmm::device_async_resource_ref _upstream; -}; - -// Install the logging resource once per process (wraps whatever the current device resource is) -static void install_rmm_logging_resource_once() -{ - static bool installed = false; -#if !(RMM_VERSION_MAJOR > 26 || (RMM_VERSION_MAJOR == 26 && RMM_VERSION_MINOR >= 6)) - static std::optional> logging_resource; -#endif - if (!installed) { - auto prev = rmm::mr::get_current_device_resource_ref(); -#if RMM_VERSION_MAJOR > 26 || (RMM_VERSION_MAJOR == 26 && RMM_VERSION_MINOR >= 6) - rmm::mr::set_current_device_resource( - cuda::mr::any_resource{logging_device_resource{prev}}); -#else - logging_resource = - cuda::mr::any_resource{logging_device_resource{prev}}; - rmm::mr::set_current_device_resource_ref(*logging_resource); -#endif - installed = true; - std::cout << "[rmm-log ] installed logging device resource adaptor" << std::endl << std::flush; - } -} - -static inline bool host_mem_equal(const uint8_t* a, const uint8_t* b, size_t n) -{ - if (a == b) return true; - if ((a == nullptr) || (b == nullptr)) return false; - return std::memcmp(a, b, n) == 0; -} - -// Removed non-stream variants to enforce explicit stream usage in tests - -} // namespace test -} // namespace cucascade diff --git a/test/utils/cudf_test_utils.hpp b/test/utils/cudf_test_utils.hpp deleted file mode 100644 index 4b21918..0000000 --- a/test/utils/cudf_test_utils.hpp +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include - -#include - -namespace cucascade { -namespace test { - -// Stream-aware variants to enforce stream ordering with async allocations -bool cudf_tables_have_equal_contents_on_stream(const cudf::table_view& left, - const cudf::table_view& right, - rmm::cuda_stream_view stream_view); -void expect_cudf_tables_equal_on_stream(const cudf::table_view& left, - const cudf::table_view& right, - rmm::cuda_stream_view stream_view); - -} // namespace test -} // namespace cucascade diff --git a/test/utils/mock_test_utils.hpp b/test/utils/mock_test_utils.hpp index 18f8ccc..b495df6 100644 --- a/test/utils/mock_test_utils.hpp +++ b/test/utils/mock_test_utils.hpp @@ -29,17 +29,10 @@ #include #include -#include -#include -#include - #include #include -#include -#include #include -#include #include #include @@ -151,64 +144,5 @@ inline std::vector create_conversion_test_configs() return builder.build(); } -/** - * @brief Create a simple cuDF table for testing. - * - * @param num_rows Number of rows in the table - * @param num_columns Number of columns (1 or 2 supported) - * @return cudf::table A simple table with numeric columns - * - * When num_columns == 1: Creates a single INT32 column filled with 0x42 - * When num_columns == 2: Creates INT32 (0x11) and INT64 (0x22) columns - */ -inline cudf::table create_simple_cudf_table( - int num_rows, - int num_columns, - rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource_ref(), - rmm::cuda_stream_view stream = rmm::cuda_stream_default) -{ - std::vector> columns; - - // First column: INT32 - auto col1 = cudf::make_numeric_column( - cudf::data_type{cudf::type_id::INT32}, num_rows, cudf::mask_state::UNALLOCATED, stream, mr); - if (num_rows > 0) { - auto view = col1->mutable_view(); - auto bytes = static_cast(num_rows) * sizeof(int32_t); - CUCASCADE_CUDA_TRY( - cudaMemset(const_cast(view.head()), (num_columns == 1) ? 0x42 : 0x11, bytes)); - } - columns.push_back(std::move(col1)); - - // Second column: INT64 (only if num_columns >= 2) - if (num_columns >= 2) { - auto col2 = cudf::make_numeric_column( - cudf::data_type{cudf::type_id::INT64}, num_rows, cudf::mask_state::UNALLOCATED, stream, mr); - if (num_rows > 0) { - auto view = col2->mutable_view(); - auto bytes = static_cast(num_rows) * sizeof(int64_t); - CUCASCADE_CUDA_TRY(cudaMemset(const_cast(view.head()), 0x22, bytes)); - } - columns.push_back(std::move(col2)); - } - - return cudf::table(std::move(columns)); -} - -inline cudf::table create_simple_cudf_table( - int num_rows, - rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource_ref(), - rmm::cuda_stream_view stream = rmm::cuda_stream_default) -{ - return create_simple_cudf_table(num_rows, 2, mr, stream); -} - -inline cudf::table create_simple_cudf_table( - rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource_ref(), - rmm::cuda_stream_view stream = rmm::cuda_stream_default) -{ - return create_simple_cudf_table(100, 2, mr, stream); -} - } // namespace test } // namespace cucascade