diff --git a/.typos.toml b/.typos.toml index 141848dc3..e7642038d 100644 --- a/.typos.toml +++ b/.typos.toml @@ -1,6 +1,8 @@ [default.extend-words] ratatui = "ratatui" PUNICODE = "PUNICODE" +# Win32 spells `ERROR_FILENAME_EXCED_RANGE` this way. +EXCED = "EXCED" [files] extend-exclude = [ diff --git a/crates/fspy/src/ipc.rs b/crates/fspy/src/ipc.rs index 51d498600..a804fa8c3 100644 --- a/crates/fspy/src/ipc.rs +++ b/crates/fspy/src/ipc.rs @@ -6,10 +6,32 @@ use fspy_shared::ipc::{ }; use tokio::task::spawn_blocking; -// Shared memory size for storing path accesses. -// 4 GiB is large enough to store path accesses in almost any realistic scenario. -// This doesn't allocate physical memory until it's actually used. -pub const SHM_CAPACITY: usize = 4 * 1024 * 1024 * 1024; +/// Shared memory for one tracked run's file-access records. +/// +/// 4 GiB of sparse address space: none of it becomes real memory until +/// records land in it, and it leaves room for tens of millions of +/// accesses. +const DEFAULT_SHM_CAPACITY: usize = 4 * 1024 * 1024 * 1024; + +/// Overrides [`DEFAULT_SHM_CAPACITY`] with a byte count. Internal: it +/// exists so a test can shrink the region until a run overruns it, and +/// nothing outside this repository should set it. +const SHM_CAPACITY_ENV: &str = "VP_RUN_INTERNAL_FSPY_SHM_CAPACITY"; + +/// How much shared memory to give the next tracked run. +/// +/// # Panics +/// +/// When the override is set to something that is not a byte count. It is +/// ours to set, so a value we cannot read is a mistake worth stopping for +/// rather than quietly ignoring. +pub fn shm_capacity() -> usize { + std::env::var_os(SHM_CAPACITY_ENV).map_or(DEFAULT_SHM_CAPACITY, |value| { + value.to_str().and_then(|value| value.parse().ok()).unwrap_or_else(|| { + panic!("{SHM_CAPACITY_ENV} is not a byte count: {}", value.display()) + }) + }) +} #[ouroboros::self_referencing] pub struct OwnedReceiverLockGuard { diff --git a/crates/fspy/src/unix/mod.rs b/crates/fspy/src/unix/mod.rs index f1d657436..c612d00b7 100644 --- a/crates/fspy/src/unix/mod.rs +++ b/crates/fspy/src/unix/mod.rs @@ -25,7 +25,7 @@ use tokio::task::spawn_blocking; use tokio_util::sync::CancellationToken; #[cfg(not(target_env = "musl"))] -use crate::ipc::{OwnedReceiverLockGuard, SHM_CAPACITY}; +use crate::ipc::OwnedReceiverLockGuard; use crate::{ChildTermination, Command, TrackedChild, arena::PathAccessArena, error::SpawnError}; #[derive(Debug)] @@ -80,7 +80,7 @@ impl SpyImpl { #[cfg(not(target_env = "musl"))] let (ipc_channel_conf, ipc_receiver) = - channel(SHM_CAPACITY).map_err(SpawnError::ChannelCreation)?; + channel(crate::ipc::shm_capacity()).map_err(SpawnError::ChannelCreation)?; let payload = Payload { #[cfg(not(target_env = "musl"))] diff --git a/crates/fspy/src/windows/mod.rs b/crates/fspy/src/windows/mod.rs index c468888a6..2e6470d47 100644 --- a/crates/fspy/src/windows/mod.rs +++ b/crates/fspy/src/windows/mod.rs @@ -21,10 +21,8 @@ use winapi::{ use winsafe::co::{CP, WC}; use crate::{ - ChildTermination, TrackedChild, - command::Command, - error::SpawnError, - ipc::{OwnedReceiverLockGuard, SHM_CAPACITY}, + ChildTermination, TrackedChild, command::Command, error::SpawnError, + ipc::OwnedReceiverLockGuard, }; const INTERPOSE_CDYLIB: Artifact = @@ -87,7 +85,7 @@ impl SpyImpl { command.creation_flags(CREATE_SUSPENDED); let (channel_conf, receiver) = - channel(SHM_CAPACITY).map_err(SpawnError::ChannelCreation)?; + channel(crate::ipc::shm_capacity()).map_err(SpawnError::ChannelCreation)?; let mut spawn_success = false; let spawn_success = &mut spawn_success; diff --git a/crates/vt_bin/src/vtt/main.rs b/crates/vt_bin/src/vtt/main.rs index 65a12f8b9..dd36cadba 100644 --- a/crates/vt_bin/src/vtt/main.rs +++ b/crates/vt_bin/src/vtt/main.rs @@ -26,7 +26,7 @@ mod rm; #[cfg(target_os = "linux")] mod small_dev_shm; mod stat_file; -mod stat_long_filename; +mod stat_many; mod touch_file; mod write_file; @@ -35,7 +35,7 @@ fn main() { if args.len() < 2 { eprintln!("Usage: vtt [args...]"); eprintln!( - "Subcommands: barrier, check-tty, cp, exit, exit-on-ctrlc, grep-file, list-dir, mkdir, pipe-stdin, print, print-color, print-cwd, print-env, print-file, read-stdin, replace-file-content, rm, small_dev_shm, stat-file, stat_long_filename, touch-file, write-file" + "Subcommands: barrier, check-tty, cp, exit, exit-on-ctrlc, grep-file, list-dir, mkdir, pipe-stdin, print, print-color, print-cwd, print-env, print-file, read-stdin, replace-file-content, rm, small_dev_shm, stat-file, stat-many, touch-file, write-file" ); std::process::exit(1); } @@ -75,7 +75,7 @@ fn main() { stat_file::run(&args[2..]); Ok(()) } - "stat_long_filename" => stat_long_filename::run(&args[2..]), + "stat-many" => stat_many::run(&args[2..]), "touch-file" => touch_file::run(&args[2..]), "write-file" => write_file::run(&args[2..]), other => { diff --git a/crates/vt_bin/src/vtt/stat_long_filename.rs b/crates/vt_bin/src/vtt/stat_long_filename.rs deleted file mode 100644 index 1b44b1453..000000000 --- a/crates/vt_bin/src/vtt/stat_long_filename.rs +++ /dev/null @@ -1,39 +0,0 @@ -use std::{error::Error, io}; - -const USAGE: &str = "Usage: vtt stat_long_filename "; - -pub fn run(args: &[String]) -> Result<(), Box> { - let count = parse_count(args)?; - access_generated_path(count, metadata)?; - Ok(()) -} - -fn parse_count(args: &[String]) -> Result { - let [count] = args else { return Err(USAGE.to_owned()) }; - count.parse().map_err(|_| USAGE.to_owned()) -} - -fn generated_path(count: usize) -> String { - "x".repeat(count) -} - -fn access_generated_path( - count: usize, - mut metadata: impl FnMut(&str) -> io::Result<()>, -) -> io::Result<()> { - let path = generated_path(count); - match metadata(&path) { - Ok(()) => Ok(()), - Err(error) - if error.kind() == io::ErrorKind::NotFound - || error.raw_os_error() == Some(libc::ENAMETOOLONG) => - { - Ok(()) - } - Err(error) => Err(error), - } -} - -fn metadata(path: &str) -> io::Result<()> { - std::fs::metadata(path).map(|_| ()) -} diff --git a/crates/vt_bin/src/vtt/stat_many.rs b/crates/vt_bin/src/vtt/stat_many.rs new file mode 100644 index 000000000..777d71d90 --- /dev/null +++ b/crates/vt_bin/src/vtt/stat_many.rs @@ -0,0 +1,76 @@ +//! Stats generated names, to make a known number of tracked file accesses. +//! +//! The names are missing on purpose: an access is recorded whether or not +//! the file is there, and this exists to have the access attempted, not to +//! find a file. Each name differs from the last, so no two can fold into +//! one record. +//! +//! Both knobs give tracking more than it can hold, and only one of them +//! travels. A count works everywhere. A long name does not: on Windows a +//! path reaches the tracer through a `UNICODE_STRING`, whose length is a +//! `u16`, so however long a name this asks for, no single record there +//! exceeds 64 KiB. + +use std::{error::Error, io}; + +const USAGE: &str = "Usage: vtt stat-many [name-length]"; + +pub fn run(args: &[String]) -> Result<(), Box> { + let (count, name_length) = parse_args(args)?; + for index in 0..count { + access_generated_path(index, name_length, metadata)?; + } + // Printing last proves the process survived every one of them, which a + // channel that has stopped accepting records must not disturb. + println!("stat {count}"); + Ok(()) +} + +fn parse_args(args: &[String]) -> Result<(usize, usize), String> { + let (count, name_length) = match args { + [count] => (count, None), + [count, name_length] => (count, Some(name_length)), + _ => return Err(USAGE.to_owned()), + }; + let count = count.parse().map_err(|_| USAGE.to_owned())?; + let name_length = + name_length.map(|length| length.parse()).transpose().map_err(|_| USAGE.to_owned())?; + Ok((count, name_length.unwrap_or(0))) +} + +/// A name unique to `index`, padded out to `name_length` when that leaves +/// room for padding. A length short enough to truncate the index would +/// hand two accesses the same name, so the index always survives. +fn generated_path(index: usize, name_length: usize) -> String { + let name = std::format!("vtt-stat-many-{index}"); + let padding = name_length.saturating_sub(name.len()); + name + &"x".repeat(padding) +} + +fn access_generated_path( + index: usize, + name_length: usize, + mut metadata: impl FnMut(&str) -> io::Result<()>, +) -> io::Result<()> { + let path = generated_path(index, name_length); + match metadata(&path) { + Ok(()) => Ok(()), + Err(error) if is_absent_or_too_long(&error) => Ok(()), + Err(error) => Err(error), + } +} + +/// Whether the platform said the file is not there, or that the name is +/// longer than it accepts. Either is the expected answer. +/// +/// Windows reports an over-long name as `ERROR_FILENAME_EXCED_RANGE`, +/// which reaches here as [`io::ErrorKind::InvalidFilename`] rather than as +/// the `ENAMETOOLONG` unix returns. +fn is_absent_or_too_long(error: &io::Error) -> bool { + matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::InvalidFilename) + || error.raw_os_error() == Some(libc::ENAMETOOLONG) +} + +fn metadata(path: &str) -> io::Result<()> { + std::fs::metadata(path).map(|_| ()) +} diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/constrained_dev_shm/snapshots/constrained_dev_shm.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/constrained_dev_shm/snapshots/constrained_dev_shm.md index 4dd59fa3b..183835957 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/constrained_dev_shm/snapshots/constrained_dev_shm.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/constrained_dev_shm/snapshots/constrained_dev_shm.md @@ -5,5 +5,6 @@ With fspy's shared-memory backing moved to memfd, file-access tracking succeeds ## `vtt small_dev_shm vt run stress` ``` -$ vtt stat_long_filename 1048576 +$ vtt stat-many 1 1048576 +stat 1 ``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/constrained_dev_shm/vite-task.json b/crates/vt_bin/tests/e2e_snapshots/fixtures/constrained_dev_shm/vite-task.json index 0415ae320..aa51997cc 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/constrained_dev_shm/vite-task.json +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/constrained_dev_shm/vite-task.json @@ -1,7 +1,7 @@ { "tasks": { "stress": { - "command": "vtt stat_long_filename 1048576", + "command": "vtt stat-many 1 1048576", "cache": true } } diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/fspy_shm_capacity/package.json b/crates/vt_bin/tests/e2e_snapshots/fixtures/fspy_shm_capacity/package.json new file mode 100644 index 000000000..5fe27cb1c --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/fspy_shm_capacity/package.json @@ -0,0 +1,4 @@ +{ + "name": "fspy-shm-capacity", + "private": true +} diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/fspy_shm_capacity/snapshots.toml b/crates/vt_bin/tests/e2e_snapshots/fixtures/fspy_shm_capacity/snapshots.toml new file mode 100644 index 000000000..6ce6bb943 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/fspy_shm_capacity/snapshots.toml @@ -0,0 +1,29 @@ +[[e2e]] +name = "shm_capacity_env_sizes_the_tracking_channel" +comment = """ +`VP_RUN_INTERNAL_FSPY_SHM_CAPACITY` sizes the shared memory a tracked task reports its file accesses through. The task makes twenty thousand of them, and 64 MiB holds every one, so the run caches like any other. + +Setting the capacity below what the task needs is what the knob exists for. That case has to wait: a channel with no room for a record currently aborts the task process, and the panic it prints carries a thread id, a toolchain path, a backtrace and a platform's own abort code, none of which snapshot the same way twice. + +Not on musl, which has no preload: those builds collect through the seccomp supervisor, on the runner's own side of the boundary, so they have no shared-memory channel to fill. +""" +cfg = 'not(target_env = "musl")' +steps = [ + { argv = [ + "vt", + "run", + "-v", + "stat", + ], envs = [ + [ + "VP_RUN_INTERNAL_FSPY_SHM_CAPACITY", + "67108864", + ], + ], comment = "64 MiB, room for every access" }, + { argv = [ + "vt", + "run", + "-v", + "stat", + ], comment = "replayed from the entry the first run stored" }, +] diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/fspy_shm_capacity/snapshots/shm_capacity_env_sizes_the_tracking_channel.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/fspy_shm_capacity/snapshots/shm_capacity_env_sizes_the_tracking_channel.md new file mode 100644 index 000000000..f28d778bc --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/fspy_shm_capacity/snapshots/shm_capacity_env_sizes_the_tracking_channel.md @@ -0,0 +1,53 @@ +# shm_capacity_env_sizes_the_tracking_channel + +`VP_RUN_INTERNAL_FSPY_SHM_CAPACITY` sizes the shared memory a tracked task reports its file accesses through. The task makes twenty thousand of them, and 64 MiB holds every one, so the run caches like any other. + +Setting the capacity below what the task needs is what the knob exists for. That case has to wait: a channel with no room for a record currently aborts the task process, and the panic it prints carries a thread id, a toolchain path, a backtrace and a platform's own abort code, none of which snapshot the same way twice. + +Not on musl, which has no preload: those builds collect through the seccomp supervisor, on the runner's own side of the boundary, so they have no shared-memory channel to fill. + +## `VP_RUN_INTERNAL_FSPY_SHM_CAPACITY=67108864 vt run -v stat` + +64 MiB, room for every access + +``` +$ vtt stat-many 20000 +stat 20000 + + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Vite+ Task Runner • Execution Summary +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Statistics: 1 tasks • 0 cache hits • 1 cache misses +Performance: 0% cache hit rate + +Task Details: +──────────────────────────────────────────────── + [1] fspy-shm-capacity#stat: $ vtt stat-many 20000 ✓ + → Cache miss: no previous cache entry found +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +``` + +## `vt run -v stat` + +replayed from the entry the first run stored + +``` +$ vtt stat-many 20000 ◉ cache hit, replaying +stat 20000 + + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Vite+ Task Runner • Execution Summary +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Statistics: 1 tasks • 1 cache hits • 0 cache misses +Performance: 100% cache hit rate + +Task Details: +──────────────────────────────────────────────── + [1] fspy-shm-capacity#stat: $ vtt stat-many 20000 ✓ + → Cache hit - output replayed - +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/fspy_shm_capacity/vite-task.json b/crates/vt_bin/tests/e2e_snapshots/fixtures/fspy_shm_capacity/vite-task.json new file mode 100644 index 000000000..fce05d408 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/fspy_shm_capacity/vite-task.json @@ -0,0 +1,8 @@ +{ + "tasks": { + "stat": { + "command": "vtt stat-many 20000", + "cache": true + } + } +}