From 694f9dc02edfae492127954a1059a9d1eb773800 Mon Sep 17 00:00:00 2001 From: TroyHernandez Date: Mon, 20 Jul 2026 14:10:25 -0500 Subject: [PATCH 1/5] Make the LTX gc-gate options actually take effect ltx23_tune_gc set the four allocator gate options one call AFTER torch::cuda_is_available() started torch - and start_torch reads them exactly once at init, so every option it set was inert by construction. Measured cost of the inert gates: ~5% of render wall and a ~550-gc / ~12 s storm across pipeline load. Options now land before the first torch call in the function, the three CUDA gates are additionally pushed into the live allocator (the .flux_gc_gates pattern, so the function works however late it runs), and .onLoad defaults torch.threshold_call_gc - which has no live setter - so torch reads it at init whenever diffuseR loads before the first torch op. User-set options win everywhere; tinytest covers defaults, user override, and the [0.20, 0.92] reserved-rate clamps. --- R/memory_ltx23.R | 49 +++++++++++++++++++++++-------- R/zzz.R | 13 ++++++++ inst/tinytest/test_memory_ltx23.R | 49 +++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 13 deletions(-) create mode 100644 inst/tinytest/test_memory_ltx23.R diff --git a/R/memory_ltx23.R b/R/memory_ltx23.R index 5f92a48..476ef65 100644 --- a/R/memory_ltx23.R +++ b/R/memory_ltx23.R @@ -106,8 +106,17 @@ ltx23_memory_profile <- function(vram_gb = NULL) { #' actual footprint is safe here because the LTX hot loops compute into #' persistent scratch buffers (near-zero per-step garbage). Also raises #' the host-allocation GC threshold and defaults -#' \code{PYTORCH_CUDA_ALLOC_CONF} to expandable segments. Must run before -#' the first CUDA op; user-set options win. +#' \code{PYTORCH_CUDA_ALLOC_CONF} to expandable segments. User-set +#' options win. +#' +#' \code{start_torch()} reads the gate options exactly once, so setting +#' them after torch has started is inert on its own. The three CUDA +#' gates are therefore also pushed into the live allocator here (the +#' \code{.flux_gc_gates} pattern), which makes this function effective +#' whenever it runs. The host-side \code{torch.threshold_call_gc} has +#' no live setter; the package defaults it in \code{.onLoad} so torch +#' reads it at init in any session that loads diffuseR before running +#' torch ops. #' #' @param footprint_gb Numeric. Expected resident GPU footprint in GB #' (NF4 transformer: ~12). @@ -120,15 +129,10 @@ ltx23_tune_gc <- function(footprint_gb = 12, total_gb = NULL) { if (!nzchar(Sys.getenv("PYTORCH_CUDA_ALLOC_CONF"))) { Sys.setenv(PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True") } - if (!torch::cuda_is_available()) { - return(invisible(NULL)) - } - if (is.null(total_gb)) { - total_gb <- .detect_vram(use_free = FALSE) - if (!isTRUE(total_gb > 0)) { - return(invisible(NULL)) - } - } + # Options are set before the first torch:: call in this function: + # torch::cuda_is_available() below starts torch, and start_torch + # reads these once. (The old order set them one call after init + # read them - inert by construction.) if (is.null(getOption("torch.threshold_call_gc"))) { options(torch.threshold_call_gc = 16000) } @@ -144,8 +148,27 @@ ltx23_tune_gc <- function(footprint_gb = 12, total_gb = NULL) { } rate <- NULL if (is.null(getOption("torch.cuda_allocator_reserved_rate"))) { - rate <- min(0.92, max(0.20, footprint_gb / total_gb)) - options(torch.cuda_allocator_reserved_rate = rate) + if (is.null(total_gb)) { + total_gb <- .detect_vram(use_free = FALSE) + } + if (isTRUE(total_gb > 0)) { + rate <- min(0.92, max(0.20, footprint_gb / total_gb)) + options(torch.cuda_allocator_reserved_rate = rate) + } + } + if (!torch::cuda_is_available()) { + return(invisible(rate)) + } + # Torch is usually long started by the time a loader calls this: + # push the CUDA gates into the live allocator directly. + push <- get0("cpp_set_cuda_allocator_allocator_thresholds", + envir = asNamespace("torch")) + if (is.function(push)) { + try(push( + getOption("torch.cuda_allocator_reserved_rate", 0.2), + getOption("torch.cuda_allocator_allocated_rate", 0.8), + getOption("torch.cuda_allocator_allocated_reserved_rate", 0.8) + ), silent = TRUE) } invisible(rate) } diff --git a/R/zzz.R b/R/zzz.R index fde3513..a52e45d 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -3,3 +3,16 @@ NULL # torch indexing uses `..` as a Python-style ellipsis utils::globalVariables("..") + +.onLoad <- function(libname, pkgname) { + # start_torch() reads torch.threshold_call_gc exactly once at torch + # init and there is no live setter (unlike the CUDA gates, which + # ltx23_tune_gc pushes via cpp). The 4000 MB default fires an R gc + # for every few GB of host allocation - measured ~550 gcs / ~12 s + # across an LTX pipeline load. Defaulting it here lands before + # torch starts in any session that loads diffuseR first; a + # user-set option wins. + if (is.null(getOption("torch.threshold_call_gc"))) { + options(torch.threshold_call_gc = 16000) + } +} diff --git a/inst/tinytest/test_memory_ltx23.R b/inst/tinytest/test_memory_ltx23.R new file mode 100644 index 0000000..204cc0a --- /dev/null +++ b/inst/tinytest/test_memory_ltx23.R @@ -0,0 +1,49 @@ +# ltx23_tune_gc option semantics: defaults land when unset, user-set +# values win, and the reserved rate clamps to [0.20, 0.92]. The cpp +# gate push itself is CUDA-only and exercised implicitly wherever a +# GPU is present. + +if (!requireNamespace("torch", quietly = TRUE) || !torch::torch_is_installed()) { + exit_file("torch not fully installed") +} + +library(diffuseR) + +gate_opts <- c("torch.threshold_call_gc", + "torch.cuda_allocator_allocated_rate", + "torch.cuda_allocator_allocated_reserved_rate", + "torch.cuda_allocator_reserved_rate") +old <- options()[gate_opts] +names(old) <- gate_opts +clear <- function() { + for (o in gate_opts) { + opt <- list(NULL) + names(opt) <- o + options(opt) + } +} + +clear() +rate <- ltx23_tune_gc(footprint_gb = 12, total_gb = 16) +expect_equal(rate, 0.75) +expect_equal(getOption("torch.cuda_allocator_reserved_rate"), 0.75) +expect_equal(getOption("torch.threshold_call_gc"), 16000) +expect_equal(getOption("torch.cuda_allocator_allocated_rate"), 0.95) +expect_equal(getOption("torch.cuda_allocator_allocated_reserved_rate"), 0.95) + +# User-set values win; the reserved rate is then not recomputed +clear() +options(torch.cuda_allocator_reserved_rate = 0.5, + torch.threshold_call_gc = 4000) +rate <- ltx23_tune_gc(footprint_gb = 12, total_gb = 16) +expect_null(rate) +expect_equal(getOption("torch.cuda_allocator_reserved_rate"), 0.5) +expect_equal(getOption("torch.threshold_call_gc"), 4000) + +# Clamps: floor 0.20 for small footprints, ceiling 0.92 for tight fits +clear() +expect_equal(ltx23_tune_gc(footprint_gb = 12, total_gb = 100), 0.20) +clear() +expect_equal(ltx23_tune_gc(footprint_gb = 12, total_gb = 12), 0.92) + +options(old) From b76bc626572d9d2cbde143d6bdc4c8f04f111248 Mon Sep 17 00:00:00 2001 From: TroyHernandez Date: Mon, 20 Jul 2026 14:15:03 -0500 Subject: [PATCH 2/5] Default pin_staging on for LTX phase offload Measured post byte-LUT (768x512x49 NF4, 3-render A/B): warm renders 64-66 s pageable vs 57-59 s pinned (~7 s/render, all of it transfer time - denoise and decode identical), page-locking +9 s at load. Break-even on the second render, ~2 s net cost for a single-render session. The docstring's stale numbers (2.7 s/render, break-even 11) predate the byte-LUT denoise and the current phase-transition behavior. Existing per-component silent fallback covers page-locking failure; options(diffuseR.pin_staging = FALSE) opts out. --- R/staging_ltx23.R | 21 +++++++++++---------- R/txt2vid_ltx23.R | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/R/staging_ltx23.R b/R/staging_ltx23.R index c1999d5..12f857d 100644 --- a/R/staging_ltx23.R +++ b/R/staging_ltx23.R @@ -10,16 +10,17 @@ #' re-points the tensors at the still-valid pinned copies — weights #' are immutable during inference, so offload moves no bytes at all. #' -#' Costs: the pinned copies are non-swappable host RAM for the life -#' of the pipeline (about the model's CPU footprint), and page-locking -#' ~20GB adds ~30s to pipeline load. Measured per render the win is -#' small (~2.7s: warm pinned onload 0.54s vs pageable 0.99s, offload -#' 0.06s vs 2.36s) because the real phase-transition cost was -#' allocator pool regrowth, fixed separately by the loader's pool -#' pre-warm and by not emptying the CUDA cache between phases. Off by -#' default; enable for long multi-render sessions with -#' \code{options(diffuseR.pin_staging = TRUE)} before -#' \code{ltx23_load_pipeline} (breaks even after ~11 renders). +#' Costs: the model's host copies become non-swappable for the life +#' of the pipeline (no extra RAM - set_data repoints the same +#' tensors), and page-locking adds ~9s to pipeline load. Measured +#' post byte-LUT (768x512x49, NF4, RTX 5060 Ti): ~7s saved per render +#' (warm renders 64-66s pageable vs 57-59s pinned; denoise and decode +#' identical, the delta is pure transfer), so pinning breaks even on +#' the second render and costs a single-render session ~2s net. On by +#' default; page-locking failure falls back silently per component, +#' and \code{options(diffuseR.pin_staging = FALSE)} before +#' \code{ltx23_load_pipeline} opts out (e.g. under host memory +#' pressure, where unswappable pages turn thrashing into OOM). #' #' @name staging_ltx23 NULL diff --git a/R/txt2vid_ltx23.R b/R/txt2vid_ltx23.R index 0879335..5829d0b 100644 --- a/R/txt2vid_ltx23.R +++ b/R/txt2vid_ltx23.R @@ -349,7 +349,7 @@ ltx23_load_pipeline <- function(checkpoint_path, device = "cuda", } if (phase_offload && device == "cuda" && - isTRUE(getOption("diffuseR.pin_staging", FALSE))) { + isTRUE(getOption("diffuseR.pin_staging", TRUE))) { # Page-lock every phase-offloaded component once so the # per-render CPU<->GPU moves run at full PCIe rate (offload # becomes a pointer swap; see staging_ltx23.R). Falls back From a6ba5067390348c3c7a79fd87e328a53128b7a20 Mon Sep 17 00:00:00 2001 From: TroyHernandez Date: Mon, 20 Jul 2026 14:20:54 -0500 Subject: [PATCH 3/5] Correct the .onLoad comment: load-time A/B showed no wall change --- R/zzz.R | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/R/zzz.R b/R/zzz.R index a52e45d..8bcb20e 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -7,11 +7,11 @@ utils::globalVariables("..") .onLoad <- function(libname, pkgname) { # start_torch() reads torch.threshold_call_gc exactly once at torch # init and there is no live setter (unlike the CUDA gates, which - # ltx23_tune_gc pushes via cpp). The 4000 MB default fires an R gc - # for every few GB of host allocation - measured ~550 gcs / ~12 s - # across an LTX pipeline load. Defaulting it here lands before - # torch starts in any session that loads diffuseR first; a - # user-set option wins. + # ltx23_tune_gc pushes via cpp), so this is the only place the + # option can land in time. The 4000 MB default fires an R gc for + # every few GB of host allocation; raising it measurably cuts gc + # counts, though an LTX load-time A/B showed no wall-clock change + # (those gcs were cheap). A user-set option wins. if (is.null(getOption("torch.threshold_call_gc"))) { options(torch.threshold_call_gc = 16000) } From 2dd03431c935ca370b923f2fe95351c032b3e5bc Mon Sep 17 00:00:00 2001 From: TroyHernandez Date: Mon, 20 Jul 2026 14:21:21 -0500 Subject: [PATCH 4/5] rformat + document --- man/ltx23_tune_gc.Rd | 15 +++++++++++++-- man/staging_ltx23.Rd | 21 +++++++++++---------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/man/ltx23_tune_gc.Rd b/man/ltx23_tune_gc.Rd index 8c280bf..ba91ec6 100644 --- a/man/ltx23_tune_gc.Rd +++ b/man/ltx23_tune_gc.Rd @@ -23,6 +23,17 @@ weights that fires on nearly every allocation. Raising the rate to the actual footprint is safe here because the LTX hot loops compute into persistent scratch buffers (near-zero per-step garbage). Also raises the host-allocation GC threshold and defaults -\code{PYTORCH_CUDA_ALLOC_CONF} to expandable segments. Must run before -the first CUDA op; user-set options win. +\code{PYTORCH_CUDA_ALLOC_CONF} to expandable segments. User-set +options win. +} +\details{ +\code{start_torch()} reads the gate options exactly once, so setting +them after torch has started is inert on its own. The three CUDA +gates are therefore also pushed into the live allocator here (the +\code{.flux_gc_gates} pattern), which makes this function effective +whenever it runs. The host-side \code{torch.threshold_call_gc} has +no live setter; the package defaults it in \code{.onLoad} so torch +reads it at init in any session that loads diffuseR before running +torch ops. + } diff --git a/man/staging_ltx23.Rd b/man/staging_ltx23.Rd index 7b2e245..d7fd3a5 100644 --- a/man/staging_ltx23.Rd +++ b/man/staging_ltx23.Rd @@ -14,15 +14,16 @@ re-points the tensors at the still-valid pinned copies — weights are immutable during inference, so offload moves no bytes at all. } \details{ -Costs: the pinned copies are non-swappable host RAM for the life -of the pipeline (about the model's CPU footprint), and page-locking -~20GB adds ~30s to pipeline load. Measured per render the win is -small (~2.7s: warm pinned onload 0.54s vs pageable 0.99s, offload -0.06s vs 2.36s) because the real phase-transition cost was -allocator pool regrowth, fixed separately by the loader's pool -pre-warm and by not emptying the CUDA cache between phases. Off by -default; enable for long multi-render sessions with -\code{options(diffuseR.pin_staging = TRUE)} before -\code{ltx23_load_pipeline} (breaks even after ~11 renders). +Costs: the model's host copies become non-swappable for the life +of the pipeline (no extra RAM - set_data repoints the same +tensors), and page-locking adds ~9s to pipeline load. Measured +post byte-LUT (768x512x49, NF4, RTX 5060 Ti): ~7s saved per render +(warm renders 64-66s pageable vs 57-59s pinned; denoise and decode +identical, the delta is pure transfer), so pinning breaks even on +the second render and costs a single-render session ~2s net. On by +default; page-locking failure falls back silently per component, +and \code{options(diffuseR.pin_staging = FALSE)} before +\code{ltx23_load_pipeline} opts out (e.g. under host memory +pressure, where unswappable pages turn thrashing into OOM). } From 7383ee81c5fb61fdc97bc3102df9613f1e368a64 Mon Sep 17 00:00:00 2001 From: TroyHernandez Date: Mon, 20 Jul 2026 14:21:37 -0500 Subject: [PATCH 5/5] Bump version to 0.1.0.11 --- DESCRIPTION | 2 +- NEWS.md | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 5a83857..50df0c4 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: diffuseR Title: Functional Interface to Diffusion Models in R -Version: 0.1.0.10 +Version: 0.1.0.11 Authors@R: c( person("Troy", "Hernandez", email = "troy@cornball.ai", role = c("aut", "cre"), comment = c(ORCID = "0009-0005-4248-604X")), diff --git a/NEWS.md b/NEWS.md index 92b58b3..83e3cd4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,20 @@ +# diffuseR 0.1.0.11 (development) + +* `ltx23_tune_gc()` now takes effect: it used to set the allocator gate + options one call after `torch::cuda_is_available()` had started torch + (which reads them exactly once at init), so every option it set was + inert. Options now land before the first torch call, the three CUDA + gates are pushed into the live allocator, and `.onLoad` defaults + `torch.threshold_call_gc` (which has no live setter). Measured: ~4-5% + off LTX render walls (63.7/62.2/61.9 s vs 67.9/64.2/65.6 s at + 768x512x49 NF4). +* `diffuseR.pin_staging` now defaults to TRUE for LTX phase offload: + page-locked host copies make onload a DMA transfer and offload a + pointer swap, saving ~7 s per render for +9 s of one-time + page-locking at pipeline load (break-even on the second render). + Opt out with `options(diffuseR.pin_staging = FALSE)` under host + memory pressure. + # diffuseR 0.1.0.10 (development) * The generators accept a three-level `verbose`: "silent", "progress",