diff --git a/CHANGELOG.md b/CHANGELOG.md index 080d8764..b484165f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,13 @@ behavior-preserving moves are not listed individually. - `vfs seed` records its resolved pin in `fs_session_metadata` as `seed_pin`, giving packed artifacts the base provenance `vfs adopt` verifies on the receiving machine. +- Artifact wire contract for daemon-to-daemon handoff: `vfs version --json` + reports `artifactVersion` and `minSupportedArtifactVersion` for + version-floor negotiation, and the `vfs pack` manifest adds + `artifactVersion`, `chunkSizeBytes`, and a `chunks` list of + `{index, sizeBytes, sha256}` digests over consecutive `--chunk-size`-byte + ranges (default 4194304) of the published artifact, alongside the existing + whole-file `dbSha256`. ### Removed diff --git a/crates/vfs-cli/src/cmd/pack.rs b/crates/vfs-cli/src/cmd/pack.rs index b26b0237..fec2574e 100644 --- a/crates/vfs-cli/src/cmd/pack.rs +++ b/crates/vfs-cli/src/cmd/pack.rs @@ -19,6 +19,9 @@ use super::safety::{ pub const SESSION_STILL_RUNNING_EXIT_CODE: i32 = 3; +/// Default byte size of the per-chunk verification digests in the manifest. +pub const DEFAULT_MANIFEST_CHUNK_SIZE: u64 = 4 * 1024 * 1024; + const DEFAULT_PRUNES: &[&str] = &[ "**/node_modules/**", "**/target/**", @@ -49,6 +52,9 @@ pub struct PackManifest { db_path: PathBuf, db_sha256: String, db_size_bytes: u64, + artifact_version: String, + chunk_size_bytes: u64, + chunks: Vec, base_repo: Option, base_pin: Option, base_path: PathBuf, @@ -58,6 +64,16 @@ pub struct PackManifest { generation: u64, } +/// Digest of one consecutive `chunk_size_bytes` range of the published +/// artifact, so a consumer can stream and verify the transfer chunk by chunk. +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ArtifactChunk { + index: u64, + size_bytes: u64, + sha256: String, +} + struct TempDatabase { path: PathBuf, } @@ -85,6 +101,7 @@ pub async fn handle_pack_command( extra_prunes: Vec, no_default_prunes: bool, output: Option, + chunk_size: u64, _json: bool, ) -> Result<()> { let home = dirs::home_dir().context("Failed to get home directory")?; @@ -95,6 +112,7 @@ pub async fn handle_pack_command( extra_prunes, no_default_prunes, output, + chunk_size, ) .await } @@ -106,10 +124,14 @@ async fn pack_session( extra_prunes: Vec, no_default_prunes: bool, output: Option, + chunk_size: u64, ) -> Result<()> { if !VfsOptions::validate_agent_id(&session_id) { anyhow::bail!("invalid session ID: {session_id}"); } + if chunk_size == 0 { + anyhow::bail!("--chunk-size must be at least 1 byte"); + } let session_dir = home.join(".vfs").join("run").join(&session_id); if !session_dir.is_dir() { @@ -180,7 +202,7 @@ async fn pack_session( .context("Staged metadata verification after compaction failed")?; remove_sqlite_sidecars_after_checkpoint(staging.path())?; - let (db_sha256, db_size_bytes) = hash_file(staging.path())?; + let (db_sha256, db_size_bytes, chunks) = hash_file(staging.path(), chunk_size)?; let output_temp = output .as_deref() .map(|path| create_output_temp(staging.path(), path)) @@ -211,6 +233,9 @@ async fn pack_session( db_path: manifest_path, db_sha256, db_size_bytes, + artifact_version: schema::CURRENT.as_str().to_string(), + chunk_size_bytes: chunk_size, + chunks, base_repo, base_pin, base_path, @@ -406,10 +431,15 @@ async fn prune_delta_paths(vfs: &Vfs, prune_set: &GlobSet) -> Result Ok(paths) } -fn hash_file(path: &Path) -> Result<(String, u64)> { +/// Hash a published artifact whole-file and over consecutive `chunk_size` +/// byte ranges in one streaming pass. +fn hash_file(path: &Path, chunk_size: u64) -> Result<(String, u64, Vec)> { let mut file = fs::File::open(path).with_context(|| format!("Failed to open {}", path.display()))?; let mut hasher = Sha256::new(); + let mut chunk_hasher = Sha256::new(); + let mut chunks = Vec::new(); + let mut chunk_len = 0_u64; let mut buffer = [0_u8; 64 * 1024]; let mut size = 0_u64; loop { @@ -421,8 +451,31 @@ fn hash_file(path: &Path) -> Result<(String, u64)> { } hasher.update(&buffer[..read]); size += read as u64; + + let mut slice = &buffer[..read]; + while !slice.is_empty() { + let take = slice.len().min((chunk_size - chunk_len) as usize); + chunk_hasher.update(&slice[..take]); + chunk_len += take as u64; + slice = &slice[take..]; + if chunk_len == chunk_size { + push_chunk(&mut chunks, &mut chunk_hasher, &mut chunk_len); + } + } } - Ok((hex::encode(hasher.finalize()), size)) + if chunk_len > 0 { + push_chunk(&mut chunks, &mut chunk_hasher, &mut chunk_len); + } + Ok((hex::encode(hasher.finalize()), size, chunks)) +} + +fn push_chunk(chunks: &mut Vec, chunk_hasher: &mut Sha256, chunk_len: &mut u64) { + chunks.push(ArtifactChunk { + index: chunks.len() as u64, + size_bytes: *chunk_len, + sha256: hex::encode(std::mem::take(chunk_hasher).finalize()), + }); + *chunk_len = 0; } fn create_output_temp(staging_path: &Path, output: &Path) -> Result { @@ -704,10 +757,14 @@ mod tests { Vec::new(), false, Some(output_path.clone()), + 4096, ) .await?; let first: PackManifest = serde_json::from_slice(&first_stdout)?; assert_eq!(first.manifest_version, 1); + assert_eq!(first.artifact_version, schema::CURRENT.as_str()); + assert_eq!(first.chunk_size_bytes, 4096); + assert_chunks_cover_artifact(&first.chunks, &fs::read(&output_path)?, 4096); assert_eq!(first.session_id, "pack-test"); assert_eq!(first.db_path, output_path); assert_eq!(first.generation, 1); @@ -754,11 +811,67 @@ mod tests { Vec::new(), false, None, + DEFAULT_MANIFEST_CHUNK_SIZE, ) .await?; let second: PackManifest = serde_json::from_slice(&second_stdout)?; assert_eq!(second.generation, 2); assert_eq!(second.db_path, db_path); + assert_eq!(second.chunk_size_bytes, DEFAULT_MANIFEST_CHUNK_SIZE); + assert_chunks_cover_artifact( + &second.chunks, + &fs::read(&db_path)?, + DEFAULT_MANIFEST_CHUNK_SIZE, + ); + Ok(()) + } + + fn assert_chunks_cover_artifact(chunks: &[ArtifactChunk], bytes: &[u8], chunk_size: u64) { + let expected: Vec<_> = bytes.chunks(chunk_size as usize).collect(); + assert_eq!(chunks.len(), expected.len()); + for (index, (chunk, expected_bytes)) in chunks.iter().zip(expected).enumerate() { + assert_eq!(chunk.index, index as u64); + assert_eq!(chunk.size_bytes, expected_bytes.len() as u64); + assert_eq!(chunk.sha256, hex::encode(Sha256::digest(expected_bytes))); + } + assert_eq!( + chunks.iter().map(|chunk| chunk.size_bytes).sum::(), + bytes.len() as u64 + ); + } + + #[test] + fn hash_file_chunks_exact_multiples_small_and_empty_files() -> Result<()> { + let dir = tempdir()?; + + let exact = dir.path().join("exact.bin"); + let exact_bytes = (0..8192_u32).map(|value| value as u8).collect::>(); + fs::write(&exact, &exact_bytes)?; + let (sha256, size, chunks) = hash_file(&exact, 4096)?; + assert_eq!(sha256, hex::encode(Sha256::digest(&exact_bytes))); + assert_eq!(size, 8192); + assert_eq!(chunks.len(), 2); + assert!(chunks.iter().all(|chunk| chunk.size_bytes == 4096)); + assert_chunks_cover_artifact(&chunks, &exact_bytes, 4096); + + let small = dir.path().join("small.bin"); + fs::write(&small, b"tiny artifact")?; + let (sha256, size, chunks) = hash_file(&small, 4096)?; + assert_eq!(sha256, hex::encode(Sha256::digest(b"tiny artifact"))); + assert_eq!(size, 13); + assert_eq!(chunks.len(), 1); + assert_eq!(chunks[0].index, 0); + assert_eq!(chunks[0].size_bytes, 13); + assert_eq!( + chunks[0].sha256, + hex::encode(Sha256::digest(b"tiny artifact")) + ); + + let empty = dir.path().join("empty.bin"); + fs::write(&empty, b"")?; + let (_, size, chunks) = hash_file(&empty, 4096)?; + assert_eq!(size, 0); + assert!(chunks.is_empty()); Ok(()) } @@ -792,6 +905,7 @@ mod tests { Vec::new(), false, None, + DEFAULT_MANIFEST_CHUNK_SIZE, ) .await .expect_err("live session must be rejected"); diff --git a/crates/vfs-cli/src/cmd/version.rs b/crates/vfs-cli/src/cmd/version.rs index 9db64fc5..57d5551d 100644 --- a/crates/vfs-cli/src/cmd/version.rs +++ b/crates/vfs-cli/src/cmd/version.rs @@ -4,13 +4,17 @@ use std::io::Write; use anyhow::Result; use serde::Serialize; +use vfs_core::schema; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); #[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] pub struct VersionInfo { version: &'static str, commit: Option<&'static str>, + artifact_version: &'static str, + min_supported_artifact_version: &'static str, features: VersionFeatures, } @@ -29,6 +33,8 @@ pub fn handle_version_command(stdout: &mut impl Write, json: bool) -> Result<()> let info = VersionInfo { version: VERSION, commit, + artifact_version: schema::CURRENT.as_str(), + min_supported_artifact_version: schema::MIN_SUPPORTED.as_str(), features: VersionFeatures { uid_squash_run: cfg!(target_os = "linux"), pack: true, @@ -58,6 +64,11 @@ mod tests { handle_version_command(&mut output, true).unwrap(); let value: serde_json::Value = serde_json::from_slice(&output).unwrap(); assert_eq!(value["version"], env!("CARGO_PKG_VERSION")); + assert_eq!(value["artifactVersion"], schema::CURRENT.as_str()); + assert_eq!( + value["minSupportedArtifactVersion"], + schema::MIN_SUPPORTED.as_str() + ); assert_eq!(value["features"]["uidSquashRun"], cfg!(target_os = "linux")); assert_eq!(value["features"]["pack"], true); assert_eq!(value["features"]["seed"], true); diff --git a/crates/vfs-cli/src/main.rs b/crates/vfs-cli/src/main.rs index 2b3d5523..ed791832 100644 --- a/crates/vfs-cli/src/main.rs +++ b/crates/vfs-cli/src/main.rs @@ -347,6 +347,7 @@ fn dispatch(args: Args) -> anyhow::Result<()> { prune, no_default_prunes, output, + chunk_size, json, } => { let rt = get_runtime(); @@ -356,6 +357,7 @@ fn dispatch(args: Args) -> anyhow::Result<()> { prune, no_default_prunes, output, + chunk_size, json, )) } diff --git a/crates/vfs-cli/src/opts.rs b/crates/vfs-cli/src/opts.rs index 14684bf0..ad6fcac9 100644 --- a/crates/vfs-cli/src/opts.rs +++ b/crates/vfs-cli/src/opts.rs @@ -456,6 +456,11 @@ pub enum Command { #[arg(long, value_name = "PATH")] output: Option, + /// Byte size of the per-chunk verification digests reported in the + /// manifest's `chunks` list + #[arg(long = "chunk-size", value_name = "BYTES", default_value_t = 4_194_304)] + chunk_size: u64, + /// Emit machine-readable JSON (pack output is always JSON) #[arg(long)] json: bool, @@ -890,16 +895,25 @@ mod tests { prune, no_default_prunes, output, + chunk_size, json, } => { assert_eq!(session_id, "session-1"); assert_eq!(prune, vec!["**/.cache/**"]); assert!(no_default_prunes); assert_eq!(output, Some(PathBuf::from("/tmp/packed.db"))); + assert_eq!(chunk_size, 4_194_304); assert!(json); } other => panic!("expected pack command, got {other:?}"), } + + let args = + Args::try_parse_from(["vfs", "pack", "session-1", "--chunk-size", "65536"]).unwrap(); + match args.command { + Command::Pack { chunk_size, .. } => assert_eq!(chunk_size, 65_536), + other => panic!("expected pack command, got {other:?}"), + } } #[test] diff --git a/crates/vfs-core/src/schema/mod.rs b/crates/vfs-core/src/schema/mod.rs index 0c745128..38e18c18 100644 --- a/crates/vfs-core/src/schema/mod.rs +++ b/crates/vfs-core/src/schema/mod.rs @@ -13,6 +13,10 @@ use turso::{Connection, Value}; /// Current schema version. pub const CURRENT: SchemaVersion = SchemaVersion::V0_6; +/// Oldest schema version with a migration path to [`CURRENT`]; the artifact +/// version floor for `vfs adopt` and `vfs migrate`. +pub const MIN_SUPPORTED: SchemaVersion = SchemaVersion::V0_0; + /// Compatibility string for callers that still surface the historical version. pub const VFS_SCHEMA_VERSION: &str = CURRENT.as_str(); pub const CONFIG_SCHEMA_VERSION_KEY: &str = "schema_version"; diff --git a/docs/MANUAL.md b/docs/MANUAL.md index e84eacde..e3ec368a 100644 --- a/docs/MANUAL.md +++ b/docs/MANUAL.md @@ -411,6 +411,7 @@ vfs pack [OPTIONS] - `--prune ` — Additional delta path glob to prune (can be specified multiple times) - `--no-default-prunes` — Disable the default generated-artifact prune globs - `--output ` — Copy the packed database to this path +- `--chunk-size ` — Byte size of the per-chunk verification digests reported in the manifest's `chunks` list [default: 4194304] - `--json` — Emit machine-readable JSON (pack output is always JSON) ### vfs seed @@ -663,6 +664,24 @@ The packed transfer artifact is only `delta.db`; `procs/`, `mnt/`, `.session.lock`, `base_path`, and other session-directory files are never included in that artifact. +Pack prints a one-line JSON manifest describing the published artifact. +`dbSha256` and `dbSizeBytes` cover the whole file; `artifactVersion` is the +schema version the artifact was migrated to during staging; and `chunks` +lists `{index, sizeBytes, sha256}` digests over consecutive +`--chunk-size`-byte ranges (default 4194304) of the exact published bytes. +A consumer streams the artifact in precisely those chunks and verifies each +one, with the whole-file `dbSha256` as the final check. + +### Artifact version negotiation + +`vfs version --json` reports `artifactVersion` (the schema version this +binary writes when packing) and `minSupportedArtifactVersion` (the oldest +artifact schema it can still migrate forward during adopt), so two daemons +can agree on an artifact-version floor before transferring anything. The +sender's `artifactVersion` must be within the receiver's supported range; +`vfs adopt` enforces the ceiling by refusing an artifact newer than its own +`artifactVersion` with an error naming both versions. + ### Adopting run sessions `vfs adopt --db --base ` installs a transferred