Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,13 @@ behavior-preserving moves are not listed individually.
- `vfs seed` records its resolved pin in `fs_session_metadata` as
`seed_pin`, giving packed artifacts the base provenance `vfs adopt`
verifies on the receiving machine.
- Artifact wire contract for daemon-to-daemon handoff: `vfs version --json`
reports `artifactVersion` and `minSupportedArtifactVersion` for
version-floor negotiation, and the `vfs pack` manifest adds
`artifactVersion`, `chunkSizeBytes`, and a `chunks` list of
`{index, sizeBytes, sha256}` digests over consecutive `--chunk-size`-byte
ranges (default 4194304) of the published artifact, alongside the existing
whole-file `dbSha256`.

### Removed

Expand Down
120 changes: 117 additions & 3 deletions crates/vfs-cli/src/cmd/pack.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ use super::safety::{

pub const SESSION_STILL_RUNNING_EXIT_CODE: i32 = 3;

/// Default byte size of the per-chunk verification digests in the manifest.
pub const DEFAULT_MANIFEST_CHUNK_SIZE: u64 = 4 * 1024 * 1024;

const DEFAULT_PRUNES: &[&str] = &[
"**/node_modules/**",
"**/target/**",
Expand Down Expand Up @@ -49,6 +52,9 @@ pub struct PackManifest {
db_path: PathBuf,
db_sha256: String,
db_size_bytes: u64,
artifact_version: String,
chunk_size_bytes: u64,
chunks: Vec<ArtifactChunk>,
base_repo: Option<String>,
base_pin: Option<String>,
base_path: PathBuf,
Expand All @@ -58,6 +64,16 @@ pub struct PackManifest {
generation: u64,
}

/// Digest of one consecutive `chunk_size_bytes` range of the published
/// artifact, so a consumer can stream and verify the transfer chunk by chunk.
#[derive(Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ArtifactChunk {
index: u64,
size_bytes: u64,
sha256: String,
}

struct TempDatabase {
path: PathBuf,
}
Expand Down Expand Up @@ -85,6 +101,7 @@ pub async fn handle_pack_command(
extra_prunes: Vec<String>,
no_default_prunes: bool,
output: Option<PathBuf>,
chunk_size: u64,
_json: bool,
) -> Result<()> {
let home = dirs::home_dir().context("Failed to get home directory")?;
Expand All @@ -95,6 +112,7 @@ pub async fn handle_pack_command(
extra_prunes,
no_default_prunes,
output,
chunk_size,
)
.await
}
Expand All @@ -106,10 +124,14 @@ async fn pack_session(
extra_prunes: Vec<String>,
no_default_prunes: bool,
output: Option<PathBuf>,
chunk_size: u64,
) -> Result<()> {
if !VfsOptions::validate_agent_id(&session_id) {
anyhow::bail!("invalid session ID: {session_id}");
}
if chunk_size == 0 {
anyhow::bail!("--chunk-size must be at least 1 byte");
}

let session_dir = home.join(".vfs").join("run").join(&session_id);
if !session_dir.is_dir() {
Expand Down Expand Up @@ -180,7 +202,7 @@ async fn pack_session(
.context("Staged metadata verification after compaction failed")?;
remove_sqlite_sidecars_after_checkpoint(staging.path())?;

let (db_sha256, db_size_bytes) = hash_file(staging.path())?;
let (db_sha256, db_size_bytes, chunks) = hash_file(staging.path(), chunk_size)?;
let output_temp = output
.as_deref()
.map(|path| create_output_temp(staging.path(), path))
Expand Down Expand Up @@ -211,6 +233,9 @@ async fn pack_session(
db_path: manifest_path,
db_sha256,
db_size_bytes,
artifact_version: schema::CURRENT.as_str().to_string(),
chunk_size_bytes: chunk_size,
chunks,
base_repo,
base_pin,
base_path,
Expand Down Expand Up @@ -406,10 +431,15 @@ async fn prune_delta_paths(vfs: &Vfs, prune_set: &GlobSet) -> Result<Vec<String>
Ok(paths)
}

fn hash_file(path: &Path) -> Result<(String, u64)> {
/// Hash a published artifact whole-file and over consecutive `chunk_size`
/// byte ranges in one streaming pass.
fn hash_file(path: &Path, chunk_size: u64) -> Result<(String, u64, Vec<ArtifactChunk>)> {
let mut file =
fs::File::open(path).with_context(|| format!("Failed to open {}", path.display()))?;
let mut hasher = Sha256::new();
let mut chunk_hasher = Sha256::new();
let mut chunks = Vec::new();
let mut chunk_len = 0_u64;
let mut buffer = [0_u8; 64 * 1024];
let mut size = 0_u64;
loop {
Expand All @@ -421,8 +451,31 @@ fn hash_file(path: &Path) -> Result<(String, u64)> {
}
hasher.update(&buffer[..read]);
size += read as u64;

let mut slice = &buffer[..read];
while !slice.is_empty() {
let take = slice.len().min((chunk_size - chunk_len) as usize);
chunk_hasher.update(&slice[..take]);
chunk_len += take as u64;
slice = &slice[take..];
if chunk_len == chunk_size {
push_chunk(&mut chunks, &mut chunk_hasher, &mut chunk_len);
}
}
}
Ok((hex::encode(hasher.finalize()), size))
if chunk_len > 0 {
push_chunk(&mut chunks, &mut chunk_hasher, &mut chunk_len);
}
Ok((hex::encode(hasher.finalize()), size, chunks))
}

fn push_chunk(chunks: &mut Vec<ArtifactChunk>, chunk_hasher: &mut Sha256, chunk_len: &mut u64) {
chunks.push(ArtifactChunk {
index: chunks.len() as u64,
size_bytes: *chunk_len,
sha256: hex::encode(std::mem::take(chunk_hasher).finalize()),
});
*chunk_len = 0;
}

fn create_output_temp(staging_path: &Path, output: &Path) -> Result<TempDatabase> {
Expand Down Expand Up @@ -704,10 +757,14 @@ mod tests {
Vec::new(),
false,
Some(output_path.clone()),
4096,
)
.await?;
let first: PackManifest = serde_json::from_slice(&first_stdout)?;
assert_eq!(first.manifest_version, 1);
assert_eq!(first.artifact_version, schema::CURRENT.as_str());
assert_eq!(first.chunk_size_bytes, 4096);
assert_chunks_cover_artifact(&first.chunks, &fs::read(&output_path)?, 4096);
assert_eq!(first.session_id, "pack-test");
assert_eq!(first.db_path, output_path);
assert_eq!(first.generation, 1);
Expand Down Expand Up @@ -754,11 +811,67 @@ mod tests {
Vec::new(),
false,
None,
DEFAULT_MANIFEST_CHUNK_SIZE,
)
.await?;
let second: PackManifest = serde_json::from_slice(&second_stdout)?;
assert_eq!(second.generation, 2);
assert_eq!(second.db_path, db_path);
assert_eq!(second.chunk_size_bytes, DEFAULT_MANIFEST_CHUNK_SIZE);
assert_chunks_cover_artifact(
&second.chunks,
&fs::read(&db_path)?,
DEFAULT_MANIFEST_CHUNK_SIZE,
);
Ok(())
}

fn assert_chunks_cover_artifact(chunks: &[ArtifactChunk], bytes: &[u8], chunk_size: u64) {
let expected: Vec<_> = bytes.chunks(chunk_size as usize).collect();
assert_eq!(chunks.len(), expected.len());
for (index, (chunk, expected_bytes)) in chunks.iter().zip(expected).enumerate() {
assert_eq!(chunk.index, index as u64);
assert_eq!(chunk.size_bytes, expected_bytes.len() as u64);
assert_eq!(chunk.sha256, hex::encode(Sha256::digest(expected_bytes)));
}
assert_eq!(
chunks.iter().map(|chunk| chunk.size_bytes).sum::<u64>(),
bytes.len() as u64
);
}

#[test]
fn hash_file_chunks_exact_multiples_small_and_empty_files() -> Result<()> {
let dir = tempdir()?;

let exact = dir.path().join("exact.bin");
let exact_bytes = (0..8192_u32).map(|value| value as u8).collect::<Vec<_>>();
fs::write(&exact, &exact_bytes)?;
let (sha256, size, chunks) = hash_file(&exact, 4096)?;
assert_eq!(sha256, hex::encode(Sha256::digest(&exact_bytes)));
assert_eq!(size, 8192);
assert_eq!(chunks.len(), 2);
assert!(chunks.iter().all(|chunk| chunk.size_bytes == 4096));
assert_chunks_cover_artifact(&chunks, &exact_bytes, 4096);

let small = dir.path().join("small.bin");
fs::write(&small, b"tiny artifact")?;
let (sha256, size, chunks) = hash_file(&small, 4096)?;
assert_eq!(sha256, hex::encode(Sha256::digest(b"tiny artifact")));
assert_eq!(size, 13);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].index, 0);
assert_eq!(chunks[0].size_bytes, 13);
assert_eq!(
chunks[0].sha256,
hex::encode(Sha256::digest(b"tiny artifact"))
);

let empty = dir.path().join("empty.bin");
fs::write(&empty, b"")?;
let (_, size, chunks) = hash_file(&empty, 4096)?;
assert_eq!(size, 0);
assert!(chunks.is_empty());
Ok(())
}

Expand Down Expand Up @@ -792,6 +905,7 @@ mod tests {
Vec::new(),
false,
None,
DEFAULT_MANIFEST_CHUNK_SIZE,
)
.await
.expect_err("live session must be rejected");
Expand Down
11 changes: 11 additions & 0 deletions crates/vfs-cli/src/cmd/version.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,17 @@ use std::io::Write;

use anyhow::Result;
use serde::Serialize;
use vfs_core::schema;

pub const VERSION: &str = env!("CARGO_PKG_VERSION");

#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct VersionInfo {
version: &'static str,
commit: Option<&'static str>,
artifact_version: &'static str,
min_supported_artifact_version: &'static str,
features: VersionFeatures,
}

Expand All @@ -29,6 +33,8 @@ pub fn handle_version_command(stdout: &mut impl Write, json: bool) -> Result<()>
let info = VersionInfo {
version: VERSION,
commit,
artifact_version: schema::CURRENT.as_str(),
min_supported_artifact_version: schema::MIN_SUPPORTED.as_str(),
features: VersionFeatures {
uid_squash_run: cfg!(target_os = "linux"),
pack: true,
Expand Down Expand Up @@ -58,6 +64,11 @@ mod tests {
handle_version_command(&mut output, true).unwrap();
let value: serde_json::Value = serde_json::from_slice(&output).unwrap();
assert_eq!(value["version"], env!("CARGO_PKG_VERSION"));
assert_eq!(value["artifactVersion"], schema::CURRENT.as_str());
assert_eq!(
value["minSupportedArtifactVersion"],
schema::MIN_SUPPORTED.as_str()
);
assert_eq!(value["features"]["uidSquashRun"], cfg!(target_os = "linux"));
assert_eq!(value["features"]["pack"], true);
assert_eq!(value["features"]["seed"], true);
Expand Down
2 changes: 2 additions & 0 deletions crates/vfs-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,7 @@ fn dispatch(args: Args) -> anyhow::Result<()> {
prune,
no_default_prunes,
output,
chunk_size,
json,
} => {
let rt = get_runtime();
Expand All @@ -356,6 +357,7 @@ fn dispatch(args: Args) -> anyhow::Result<()> {
prune,
no_default_prunes,
output,
chunk_size,
json,
))
}
Expand Down
14 changes: 14 additions & 0 deletions crates/vfs-cli/src/opts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -456,6 +456,11 @@ pub enum Command {
#[arg(long, value_name = "PATH")]
output: Option<PathBuf>,

/// Byte size of the per-chunk verification digests reported in the
/// manifest's `chunks` list
#[arg(long = "chunk-size", value_name = "BYTES", default_value_t = 4_194_304)]
chunk_size: u64,

/// Emit machine-readable JSON (pack output is always JSON)
#[arg(long)]
json: bool,
Expand Down Expand Up @@ -890,16 +895,25 @@ mod tests {
prune,
no_default_prunes,
output,
chunk_size,
json,
} => {
assert_eq!(session_id, "session-1");
assert_eq!(prune, vec!["**/.cache/**"]);
assert!(no_default_prunes);
assert_eq!(output, Some(PathBuf::from("/tmp/packed.db")));
assert_eq!(chunk_size, 4_194_304);
assert!(json);
}
other => panic!("expected pack command, got {other:?}"),
}

let args =
Args::try_parse_from(["vfs", "pack", "session-1", "--chunk-size", "65536"]).unwrap();
match args.command {
Command::Pack { chunk_size, .. } => assert_eq!(chunk_size, 65_536),
other => panic!("expected pack command, got {other:?}"),
}
}

#[test]
Expand Down
4 changes: 4 additions & 0 deletions crates/vfs-core/src/schema/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ use turso::{Connection, Value};
/// Current schema version.
pub const CURRENT: SchemaVersion = SchemaVersion::V0_6;

/// Oldest schema version with a migration path to [`CURRENT`]; the artifact
/// version floor for `vfs adopt` and `vfs migrate`.
pub const MIN_SUPPORTED: SchemaVersion = SchemaVersion::V0_0;

/// Compatibility string for callers that still surface the historical version.
pub const VFS_SCHEMA_VERSION: &str = CURRENT.as_str();
pub const CONFIG_SCHEMA_VERSION_KEY: &str = "schema_version";
Expand Down
19 changes: 19 additions & 0 deletions docs/MANUAL.md
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,7 @@ vfs pack [OPTIONS] <SESSION_ID>
- `--prune <GLOB>` — Additional delta path glob to prune (can be specified multiple times)
- `--no-default-prunes` — Disable the default generated-artifact prune globs
- `--output <PATH>` — Copy the packed database to this path
- `--chunk-size <BYTES>` — Byte size of the per-chunk verification digests reported in the manifest's `chunks` list [default: 4194304]
- `--json` — Emit machine-readable JSON (pack output is always JSON)

### vfs seed
Expand Down Expand Up @@ -663,6 +664,24 @@ The packed transfer artifact is only `delta.db`; `procs/`, `mnt/`,
`.session.lock`, `base_path`, and other session-directory files are never
included in that artifact.

Pack prints a one-line JSON manifest describing the published artifact.
`dbSha256` and `dbSizeBytes` cover the whole file; `artifactVersion` is the
schema version the artifact was migrated to during staging; and `chunks`
lists `{index, sizeBytes, sha256}` digests over consecutive
`--chunk-size`-byte ranges (default 4194304) of the exact published bytes.
A consumer streams the artifact in precisely those chunks and verifies each
one, with the whole-file `dbSha256` as the final check.

### Artifact version negotiation

`vfs version --json` reports `artifactVersion` (the schema version this
binary writes when packing) and `minSupportedArtifactVersion` (the oldest
artifact schema it can still migrate forward during adopt), so two daemons
can agree on an artifact-version floor before transferring anything. The
sender's `artifactVersion` must be within the receiver's supported range;
`vfs adopt` enforces the ceiling by refusing an artifact newer than its own
`artifactVersion` with an error naming both versions.

### Adopting run sessions

`vfs adopt <session-id> --db <path> --base <path>` installs a transferred
Expand Down
Loading