diff --git a/Cargo.lock b/Cargo.lock index d3ad535a..cb0e5369 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -815,7 +815,7 @@ dependencies = [ [[package]] name = "ant-cli" -version = "0.2.7" +version = "0.2.8" dependencies = [ "ant-core", "anyhow", @@ -835,7 +835,7 @@ dependencies = [ [[package]] name = "ant-core" -version = "0.2.7" +version = "0.2.8" dependencies = [ "alloy", "ant-node", @@ -858,7 +858,6 @@ dependencies = [ "rand 0.8.6", "reqwest 0.12.28", "rmp-serde", - "saorsa-core", "self-replace", "self_encryption", "semver 1.0.28", @@ -893,9 +892,9 @@ dependencies = [ [[package]] name = "ant-node" -version = "0.11.6" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b9db385f7dc01a18dd6921f54999e03de4d11fcbb1493493e86a062ab75b5e" +checksum = "b7e57398bdba4060e26a0114f1af4fc3ec8f401a1676b4899aa047d1924d075d" dependencies = [ "ant-protocol", "blake3", @@ -943,9 +942,9 @@ dependencies = [ [[package]] name = "ant-protocol" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e950d12c9f6d08d0ea560573729d93f15e105d53b669defa682f5e6f92da4b1" +checksum = "f61260c89bbc0039e0643f3e2ec79b1c17aee9d81b58d7edbc52314689489f39" dependencies = [ "blake3", "bytes", @@ -1710,9 +1709,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -3262,7 +3261,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core 0.58.0", ] [[package]] @@ -3765,9 +3764,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.31" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "113b30b4cd05f7c06868fdb2854f66a7b9fece9a48425351cd532e810d74024f" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "lru" @@ -4611,7 +4610,7 @@ dependencies = [ "once_cell", "socket2 0.6.4", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -5251,9 +5250,9 @@ dependencies = [ [[package]] name = "saorsa-core" -version = "0.24.5" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c0f8952fc5a4d37eb0bca7de0740830f40347f9da663effde3ddd6b68bcd2fb" +checksum = "fa8cc1b7f59f97d018760ff150bbb4f217197c41622b83f7085c9cf0424b736e" dependencies = [ "anyhow", "async-trait", @@ -5366,9 +5365,9 @@ dependencies = [ [[package]] name = "saorsa-transport" -version = "0.34.2" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852400712537856ab6fec5293be4290daf0130df0dbcb249a6e8280f9257665f" +checksum = "621d0a207914a8fd6453f25e4bcc369914cbfaf59a2857e898c079b95f52f5bb" dependencies = [ "anyhow", "async-trait", @@ -5685,9 +5684,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ "base64", "bs58", @@ -5705,9 +5704,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ "darling", "proc-macro2", @@ -6991,19 +6990,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement 0.60.2", - "windows-interface 0.59.3", - "windows-link", - "windows-result 0.4.1", - "windows-strings 0.5.1", -] - [[package]] name = "windows-implement" version = "0.57.0" @@ -7026,17 +7012,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "windows-implement" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "windows-interface" version = "0.57.0" @@ -7059,17 +7034,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "windows-interface" -version = "0.59.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "windows-link" version = "0.2.1" @@ -7169,15 +7133,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -7226,30 +7181,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -7268,12 +7206,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -7292,12 +7224,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -7316,24 +7242,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -7352,12 +7266,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -7376,12 +7284,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -7400,12 +7302,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -7424,12 +7320,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "0.7.15" diff --git a/Cargo.toml b/Cargo.toml index ec8f5ce5..f3978b22 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,3 @@ [workspace] members = ["ant-core", "ant-cli"] resolver = "2" - diff --git a/ant-cli/Cargo.toml b/ant-cli/Cargo.toml index 2c6d8285..0ee1b56f 100644 --- a/ant-cli/Cargo.toml +++ b/ant-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ant-cli" -version = "0.2.7" +version = "0.2.8" edition = "2021" description = "Unified CLI (`ant`) for the Autonomi network: store and retrieve data, and manage local nodes." license = "MIT OR Apache-2.0" diff --git a/ant-cli/src/commands/data/file.rs b/ant-cli/src/commands/data/file.rs index d6464f6a..613af9b6 100644 --- a/ant-cli/src/commands/data/file.rs +++ b/ant-cli/src/commands/data/file.rs @@ -1,3 +1,4 @@ +use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; @@ -8,7 +9,8 @@ use tokio::sync::mpsc; use tracing::info; use ant_core::data::{ - Client, CollisionPolicy, DownloadEvent, Error as DataError, PaymentMode, UploadEvent, + Client, CollisionPolicy, CostEstimateConfidence, DownloadEvent, Error as DataError, + PaymentMode, UploadEvent, }; use ant_core::datamap_file::{original_name_from_datamap, read_datamap, write_datamap}; @@ -68,6 +70,9 @@ pub enum FileAction { /// written to the current directory). #[arg(short, long)] output: Option, + /// Number of closest peers to try for each chunk fetch. + #[arg(long, alias = "peer-count", value_name = "COUNT")] + peers: Option, }, /// Estimate the cost of uploading a file without uploading. /// @@ -153,6 +158,7 @@ impl FileAction { address, datamap, output, + peers, } => { let resolved_output = resolve_download_output(output, datamap.as_deref())?; handle_file_download( @@ -161,6 +167,7 @@ impl FileAction { datamap.as_deref(), resolved_output, json, + peers, ) .await } @@ -202,8 +209,12 @@ async fn handle_file_upload( ); let upload_outcome = if json_output { - // No progress bars in JSON mode - client.file_upload_with_mode(path, mode).await + // No progress bars in JSON mode. + if public { + client.file_upload_public_with_mode(path, mode).await + } else { + client.file_upload_with_mode(path, mode).await + } } else { // Set up progress channel and drive progress bars let (tx, rx) = mpsc::channel(64); @@ -213,7 +224,13 @@ async fn handle_file_upload( file_size, )); - let upload_result = client.file_upload_with_progress(path, mode, Some(tx)).await; + let upload_result = if public { + client + .file_upload_public_with_progress(path, mode, Some(tx)) + .await + } else { + client.file_upload_with_progress(path, mode, Some(tx)).await + }; // Wait for progress display to finish (sender dropped → receiver exits) let _ = pb_handle.await; @@ -227,6 +244,7 @@ async fn handle_file_upload( stored_count, failed_count, total_chunks, + spend, reason, .. }) => { @@ -236,12 +254,18 @@ async fn handle_file_upload( total_chunks, chunks_stored: stored_count, chunks_failed: failed_count, + storage_cost_atto: spend.storage_cost_atto.clone(), + gas_cost_wei: spend.gas_cost_wei.to_string(), reason: &reason, }; println!("{}", serde_json::to_string(&out)?); } + // The partial upload still spent money on-chain for the chunks it + // paid for; report it so the user knows what the failed attempt cost. + let cost_display = format_cost(&spend.storage_cost_atto, spend.gas_cost_wei); anyhow::bail!( - "Upload failed: {stored_count}/{total_chunks} stored, {failed_count} failed: {reason}" + "Upload failed: {stored_count}/{total_chunks} stored, {failed_count} failed \ + (spent {cost_display}): {reason}" ); } Err(e) => anyhow::bail!("File upload failed: {e}"), @@ -250,41 +274,13 @@ async fn handle_file_upload( let elapsed = start.elapsed(); if public { - let spinner = if !json_output { - Some(progress::new_spinner("Storing public data map...")) - } else { - None - }; - let dm_result = client.data_map_store(&result.data_map).await; - if let Some(s) = &spinner { - s.finish_and_clear(); - } - let dm_address = match dm_result { - Ok(addr) => addr, - Err(e) => { - // The file body is fully stored and paid for at this point — - // only the public DataMap chunk failed. In JSON mode emit a - // parseable failure record (like the PartialUpload arm above) - // so callers don't report 0/0 chunks for an upload that is one - // chunk away from being retrievable. - if json_output { - let reason = format!("failed to store public DataMap: {e}"); - let out = UploadFailureJson { - error: "datamap_store_failed", - total_chunks: result.chunks_stored + 1, - chunks_stored: result.chunks_stored, - chunks_failed: 1, - reason: &reason, - }; - println!("{}", serde_json::to_string(&out)?); - } - anyhow::bail!("Failed to store public DataMap: {e}"); - } - }; - + let dm_address = result + .data_map_address + .ok_or_else(|| anyhow::anyhow!("Public upload completed without a DataMap address"))?; let hex_addr = hex::encode(dm_address); let cost_display = format_cost(&result.storage_cost_atto, result.gas_cost_wei); - let total_chunks = result.chunks_stored + 1; // +1 for the public data map chunk + let total_chunks = result.total_chunks; + let data_chunks = total_chunks.saturating_sub(1); if json_output { let out = UploadJsonResult { @@ -293,8 +289,8 @@ async fn handle_file_upload( mode: "public".into(), chunks: total_chunks, total_chunks, - chunks_stored: total_chunks, - chunks_failed: 0, + chunks_stored: result.chunks_stored, + chunks_failed: result.chunks_failed, size: file_size, storage_cost_atto: result.storage_cost_atto.clone(), gas_cost_wei: result.gas_cost_wei.to_string(), @@ -308,10 +304,7 @@ async fn handle_file_upload( println!(); println!("Upload complete!"); println!(" Address: {hex_addr}"); - println!( - " Chunks: {total_chunks} ({} + 1 data map)", - result.chunks_stored - ); + println!(" Chunks: {total_chunks} ({} + 1 data map)", data_chunks); println!(" Size: {}", format_size(file_size)); println!(" Cost: {cost_display}"); println!(" Time: {:.1}s", elapsed.as_secs_f64()); @@ -322,7 +315,7 @@ async fn handle_file_upload( info!( "Public upload complete: address={hex_addr}, chunks={}", - result.chunks_stored + result.total_chunks ); } else { let parent = path @@ -445,22 +438,34 @@ async fn handle_file_download( datamap_path: Option<&Path>, output: PathBuf, json_output: bool, + peer_count: Option, ) -> anyhow::Result<()> { let output_path = output; let start = Instant::now(); let data_map = if let Some(addr_hex) = address { info!("Downloading public file from address {addr_hex}"); + let address = parse_address(addr_hex)?; if !json_output { let spinner = progress::new_spinner("Fetching data map..."); - let result = client.data_map_fetch(&parse_address(addr_hex)?).await; + let result = if let Some(peer_count) = peer_count { + client + .data_map_fetch_from_closest_peers(&address, peer_count) + .await + } else { + client.data_map_fetch(&address).await + }; spinner.finish_and_clear(); result.map_err(|e| anyhow::anyhow!("Failed to fetch public DataMap: {e}"))? } else { - client - .data_map_fetch(&parse_address(addr_hex)?) - .await - .map_err(|e| anyhow::anyhow!("Failed to fetch public DataMap: {e}"))? + if let Some(peer_count) = peer_count { + client + .data_map_fetch_from_closest_peers(&address, peer_count) + .await + } else { + client.data_map_fetch(&address).await + } + .map_err(|e| anyhow::anyhow!("Failed to fetch public DataMap: {e}"))? } } else { let dm_path = datamap_path @@ -470,10 +475,15 @@ async fn handle_file_download( }; if json_output { - client - .file_download(&data_map, &output_path) - .await - .map_err(|e| anyhow::anyhow!("Download failed: {e}"))?; + let download_result = if let Some(peer_count) = peer_count { + client + .file_download_from_closest_peers(&data_map, &output_path, peer_count) + .await + } else { + client.file_download(&data_map, &output_path).await + }; + + download_result.map_err(|e| anyhow::anyhow!("Download failed: {e}"))?; } else { let (tx, mut rx) = mpsc::channel(64); @@ -512,9 +522,20 @@ async fn handle_file_download( pb.finish_and_clear(); }); - let download_result = client - .file_download_with_progress(&data_map, &output_path, Some(tx)) - .await; + let download_result = if let Some(peer_count) = peer_count { + client + .file_download_with_progress_from_closest_peers( + &data_map, + &output_path, + Some(tx), + peer_count, + ) + .await + } else { + client + .file_download_with_progress(&data_map, &output_path, Some(tx)) + .await + }; // Wait for progress bar cleanup (sender dropped → receiver exits) let _ = progress_handle.await; @@ -565,24 +586,26 @@ async fn handle_file_cost( result }; - let estimate = match raw_result { - Ok(e) => e, - Err(DataError::CostEstimationInconclusive(msg)) => { - anyhow::bail!( - "Cost estimation inconclusive: {msg}. The sampled chunks are \ - already stored on the network, so we can't sample a representative \ - price for the rest of the file. Try again later or upload a file \ - that contains some new data." - ); - } - Err(e) => anyhow::bail!("Cost estimation failed: {e}"), - }; + let estimate = raw_result.map_err(|e| anyhow::anyhow!("Cost estimation failed: {e}"))?; if json_output { println!("{}", serde_json::to_string(&estimate)?); } else { - let gas_wei: u128 = estimate.estimated_gas_cost_wei.parse().unwrap_or(0); - let cost_display = format_cost(&estimate.storage_cost_atto, gas_wei); + // The estimate is display-only; the real upload reconciles the true + // cost at payment time. When every sampled chunk is already stored we + // say so rather than print a misleading priced number. + let cost_display = match estimate.confidence { + CostEstimateConfidence::VerifiedAllAlreadyStored => { + "already stored on the network — free".to_string() + } + CostEstimateConfidence::AllSamplesAlreadyStoredIncomplete => { + "likely already stored — free (confirmed at payment)".to_string() + } + CostEstimateConfidence::PricedSample => { + let gas_wei: u128 = estimate.estimated_gas_cost_wei.parse().unwrap_or(0); + format_cost(&estimate.storage_cost_atto, gas_wei) + } + }; println!(); println!("Estimated upload cost for {}", path.display()); @@ -625,6 +648,11 @@ struct UploadFailureJson<'a> { total_chunks: usize, chunks_stored: usize, chunks_failed: usize, + /// Storage cost paid on-chain so far, in atto-tokens. A partial upload + /// still spends money for the chunks it paid for. + storage_cost_atto: String, + /// Gas cost paid on-chain so far, in wei. + gas_cost_wei: String, reason: &'a str, } @@ -697,6 +725,21 @@ fn format_cost(storage_cost_atto: &str, gas_cost_wei: u128) -> String { #[cfg(test)] mod tests { use super::*; + use clap::Parser; + + #[derive(Debug, Parser)] + struct TestFileCli { + #[command(subcommand)] + action: FileAction, + } + + const TEST_ADDRESS_BYTE_LEN: usize = 32; + const PUBLIC_DOWNLOAD_PEERS: usize = 12; + const PRIVATE_DOWNLOAD_PEERS: usize = 9; + + fn test_address() -> String { + "00".repeat(TEST_ADDRESS_BYTE_LEN) + } #[test] fn resolve_download_output_returns_explicit_output_unchanged() { @@ -754,4 +797,71 @@ mod tests { let err = resolve_download_output(None, Some(datamap.as_path())).unwrap_err(); assert!(err.to_string().contains("Cannot derive")); } + + #[test] + fn download_peers_is_accepted_for_public_download() { + let address = test_address(); + let peer_count = PUBLIC_DOWNLOAD_PEERS.to_string(); + let cli = TestFileCli::try_parse_from([ + "test", + "download", + address.as_str(), + "--peers", + peer_count.as_str(), + "--output", + "out.bin", + ]) + .expect("--peers must parse for address downloads"); + + match cli.action { + FileAction::Download { peers, address, .. } => { + assert!(address.is_some()); + assert_eq!(peers.map(NonZeroUsize::get), Some(PUBLIC_DOWNLOAD_PEERS)); + } + FileAction::Upload { .. } | FileAction::Cost { .. } => { + panic!("expected file download action") + } + } + } + + #[test] + fn download_peers_is_accepted_for_private_download() { + let peer_count = PRIVATE_DOWNLOAD_PEERS.to_string(); + let cli = TestFileCli::try_parse_from([ + "test", + "download", + "--datamap", + "photo.jpg.datamap", + "--peers", + peer_count.as_str(), + ]) + .expect("--peers must parse for datamap downloads"); + + match cli.action { + FileAction::Download { peers, datamap, .. } => { + assert!(datamap.is_some()); + assert_eq!(peers.map(NonZeroUsize::get), Some(PRIVATE_DOWNLOAD_PEERS)); + } + FileAction::Upload { .. } | FileAction::Cost { .. } => { + panic!("expected file download action") + } + } + } + + #[test] + fn download_peers_rejects_zero() { + let address = test_address(); + let err = TestFileCli::try_parse_from([ + "test", + "download", + address.as_str(), + "--peers", + "0", + "--output", + "out.bin", + ]) + .expect_err("--peers=0 must fail"); + + assert_eq!(err.kind(), clap::error::ErrorKind::ValueValidation); + } } diff --git a/ant-cli/src/main.rs b/ant-cli/src/main.rs index 905df7a2..a4bf5dd9 100644 --- a/ant-cli/src/main.rs +++ b/ant-cli/src/main.rs @@ -12,6 +12,7 @@ use tracing::info; use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; use ant_core::data::{ + peer_cache::{self, BootstrapAddressFilter}, Client, ClientConfig, CoreNodeConfig, CustomNetwork, DevnetManifest, EvmAddress, EvmNetwork, IPDiversityConfig, MultiAddr, NodeMode, P2PNode, Wallet, MAX_WIRE_MESSAGE_SIZE, }; @@ -147,6 +148,7 @@ async fn run() -> anyhow::Result<()> { // Persist whatever the controller learned this run, even // on error — partial signal is still better than cold next // time. Drop will also fire as a backstop. + client.save_peer_cache().await; client.save_adaptive_snapshot(); result?; } @@ -154,6 +156,7 @@ async fn run() -> anyhow::Result<()> { let needs_wallet = matches!(action, commands::data::ChunkAction::Put { .. }); let client = build_data_client(&data_ctx, needs_wallet, json, None, None).await?; let result = action.execute(&client).await; + client.save_peer_cache().await; client.save_adaptive_snapshot(); result?; } @@ -204,17 +207,34 @@ async fn build_data_client( let manifest = load_manifest(ctx)?; let bootstrap = resolve_bootstrap_from(ctx, manifest.as_ref())?; + // Explicit network selectors should be isolated from the general client + // peer cache. `--bootstrap` and `--devnet-manifest` both mean "use exactly + // this network entrypoint", so cached public-network peers must not be + // mixed in or saved back from that run. + let use_peer_cache = ctx.devnet_manifest.is_none() && ctx.bootstrap.is_empty(); // Connection phase with animated spinner showing peer discovery in real-time. // The spinner is the user-facing UI; tracing::info! provides log-level visibility // when `-v` is set. info!("Connecting to autonomi network"); let node = if quiet { - create_client_node(bootstrap, ctx.allow_loopback, ctx.ipv4_only).await? + create_client_node( + &bootstrap, + ctx.allow_loopback, + ctx.ipv4_only, + use_peer_cache, + ) + .await? } else { let spinner = progress::new_spinner("Connecting to autonomi network..."); - let node = match create_client_node_raw(bootstrap, ctx.allow_loopback, ctx.ipv4_only).await + let node = match create_client_node_raw( + &bootstrap, + ctx.allow_loopback, + ctx.ipv4_only, + use_peer_cache, + ) + .await { Ok(n) => n, Err(e) => { @@ -252,6 +272,9 @@ async fn build_data_client( start_result.map_err(|e| anyhow::anyhow!("Failed to start P2P node: {e}"))?; let peers = node.connected_peers().await.len(); + if use_peer_cache { + promote_client_peer_cache(&node).await; + } info!("Connected to autonomi network ({peers} peers)"); eprintln!("Connected to autonomi network (found {peers} peers)"); node @@ -324,7 +347,8 @@ async fn build_data_client( config.store_concurrency = c; } - let mut client = Client::from_node(node, config); + let peer_cache_path = use_peer_cache.then(peer_cache::cache_path).flatten(); + let mut client = Client::from_node_with_peer_cache(node, config, peer_cache_path); if needs_wallet { let key = private_key @@ -461,22 +485,27 @@ fn resolve_bootstrap_from( } async fn create_client_node( - bootstrap: Vec, + bootstrap: &[SocketAddr], allow_loopback: bool, ipv4_only: bool, + use_peer_cache: bool, ) -> anyhow::Result> { - let node = create_client_node_raw(bootstrap, allow_loopback, ipv4_only).await?; + let node = create_client_node_raw(bootstrap, allow_loopback, ipv4_only, use_peer_cache).await?; node.start() .await .map_err(|e| anyhow::anyhow!("Failed to start P2P node: {e}"))?; + if use_peer_cache { + promote_client_peer_cache(&node).await; + } Ok(node) } /// Create a P2P node without starting it (for spinner polling during start). async fn create_client_node_raw( - bootstrap: Vec, + bootstrap: &[SocketAddr], allow_loopback: bool, ipv4_only: bool, + use_peer_cache: bool, ) -> anyhow::Result> { let mut core_config = CoreNodeConfig::builder() .port(0) @@ -493,10 +522,24 @@ async fn create_client_node_raw( // silently drop legitimate testnet peers that share an IP or /24. core_config.diversity_config = Some(IPDiversityConfig::permissive()); - core_config.bootstrap_peers = bootstrap - .iter() - .map(|addr| MultiAddr::quic(*addr)) - .collect(); + let dht_k_value = core_config.dht_config.k_value; + let cache_path = use_peer_cache.then(peer_cache::cache_path).flatten(); + let cache_address_filter = if ipv4_only { + BootstrapAddressFilter::Ipv4Only + } else { + BootstrapAddressFilter::All + }; + let cached_bootstrap_peers = cache_path + .as_deref() + .map(|path| { + peer_cache::cached_bootstrap_peers_with_filter(path, dht_k_value, cache_address_filter) + }) + .unwrap_or_default(); + + core_config.bootstrap_peers = peer_cache::select_bootstrap_peers( + cached_bootstrap_peers, + bootstrap.iter().map(|addr| MultiAddr::quic(*addr)), + ); let node = P2PNode::new(core_config) .await @@ -504,3 +547,10 @@ async fn create_client_node_raw( Ok(Arc::new(node)) } + +async fn promote_client_peer_cache(node: &P2PNode) { + let Some(cache_path) = peer_cache::cache_path() else { + return; + }; + peer_cache::promote_connected_direct_peers(node, &cache_path, node.dht().k_value()).await; +} diff --git a/ant-core/Cargo.toml b/ant-core/Cargo.toml index 4ec9baa0..c5a1f215 100644 --- a/ant-core/Cargo.toml +++ b/ant-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ant-core" -version = "0.2.7" +version = "0.2.8" edition = "2021" description = "Headless Rust library for the Autonomi network: data storage and retrieval with self-encryption and EVM payments, plus node lifecycle management." license = "MIT OR Apache-2.0" @@ -37,7 +37,7 @@ tower-http = { version = "0.6.8", features = ["cors"] } # under `ant_protocol::{evm, transport, pqc}`. This is the ONE pin for # those three deps — do not add direct evmlib/saorsa-core/saorsa-pqc # deps here or the version can skew between ant-client and ant-node. -ant-protocol = "2.1.2" +ant-protocol = "2.2.0" xor_name = "5" self_encryption = "0.36" futures = "0.3" @@ -61,9 +61,11 @@ sysinfo = { version = "0.32", default-features = false, features = ["system"] } # Must track the same `saorsa-core` / `ant-protocol` line as the # `ant-protocol` pin above — a version skew pulls a second copy of # `saorsa-core` into the graph and makes `ant_node`'s and `ant_protocol`'s -# `MultiAddr` mutually incompatible in `node/devnet.rs`. ant-node 0.11.6 -# tracks saorsa-core 0.24.5 / ant-protocol 2.1.2, matching the pins here. -ant-node = { version = "0.11.6", optional = true } +# `MultiAddr` mutually incompatible in `node/devnet.rs`. While the runtime +# `ant-protocol` pin above points at a git branch, this ant-node must point at +# the matching ant-node branch carrying the same saorsa-core / ant-protocol +# lineage rather than a released version. +ant-node = { version = "0.13.0", optional = true } tracing-subscriber = { version = "0.3", features = ["env-filter"] } [target.'cfg(unix)'.dependencies] @@ -91,19 +93,12 @@ devnet = ["dep:ant-node"] # always compile even without the `devnet` feature. Pinned to the same # version as the runtime dep so there is a single ant-node / # saorsa-core version across the whole graph. -ant-node = "0.11.6" +ant-node = { version = "0.13.0", features = ["test-utils"] } serial_test = "3" anyhow = "1" alloy = { version = "1.6", features = ["node-bindings"] } tokio-test = "0.4" rmp-serde = "1" -# Direct access to BootstrapManager used by the cold-start-from-disk test, -# which populates a cache via `add_peer_trusted` (bypasses Sybil rate limits) -# and then verifies reload after save. Test-only — no runtime version-pin -# concern. Tracks ant-node's transitive saorsa-core dep, so it must match -# the version ant-node 0.11.6 pins to avoid a duplicate saorsa-core in -# the graph. -saorsa-core = "0.24.5" [[example]] name = "start-local-devnet" diff --git a/ant-core/examples/bench-quoting.rs b/ant-core/examples/bench-quoting.rs index e09cbda7..54bf27b3 100644 --- a/ant-core/examples/bench-quoting.rs +++ b/ant-core/examples/bench-quoting.rs @@ -309,11 +309,11 @@ async fn bench_normal_once(client: &Client, rep: usize) -> Rep { rand::thread_rng().fill(&mut content[..]); let address = compute_address(&content); - // 2. find_closest_peers (same call single-node quoting uses). + // 2. find_closest_peers (same strict close-group call single-node quoting uses). let t0 = Instant::now(); let peers = match client .network() - .find_closest_peers(&address, CLOSE_GROUP_SIZE * 2) + .find_closest_peers(&address, CLOSE_GROUP_SIZE) .await { Ok(p) => p, @@ -417,7 +417,7 @@ async fn bench_normal_once(client: &Client, rep: usize) -> Rep { stages.push(("quote_rpc_max".into(), *s.last().unwrap_or(&0))); } - let ok = collect_res.is_ok() && successes >= CLOSE_GROUP_SIZE; + let ok = collect_res.is_ok() && successes == CLOSE_GROUP_SIZE; Rep { rep, stages_ms: stages, diff --git a/ant-core/src/data/client/adaptive.rs b/ant-core/src/data/client/adaptive.rs index 2e983b28..ab586799 100644 --- a/ant-core/src/data/client/adaptive.rs +++ b/ant-core/src/data/client/adaptive.rs @@ -43,8 +43,9 @@ //! //! - Not a payment-batching controller. Wave / batch sizes are //! orthogonal (gas-economics tradeoff, not throughput). -//! - Not a peer-quality scorer. That lives in `peer_cache` and feeds -//! `BootstrapManager`. Outcomes flow into both, separately. +//! - Not a persistent peer-quality scorer. Bootstrap cache scoring was +//! removed from saorsa-core; this controller only tunes client +//! concurrency. use futures::stream::{self, FuturesUnordered, StreamExt}; use serde::{Deserialize, Serialize}; @@ -1059,7 +1060,31 @@ impl AdaptiveController { let mut config = config; config.sanitize(); let quote_cfg = LimiterConfig::from_adaptive(&config, config.max.quote); - let store_cfg = LimiterConfig::from_adaptive(&config, config.max.store); + let mut store_cfg = LimiterConfig::from_adaptive(&config, config.max.store); + // Store-channel growth/decision tuning (V2-468). The store limiter + // starts at 8 (correct — deliberately low for low-bandwidth uplinks) + // but on the merkle upload path its health signals are polluted by two + // things that are NOT local-capacity signals, so it never ramps and + // gets crushed to a +1-per-window crawl. Both are the structural twin + // of the fetch-channel overrides below (verification variance instead + // of retry variance); the cold-start floor is deliberately untouched. + // + // - Disable the p95-latency Decrease. Node-side PUT latency is + // dominated by the ~28s synchronous merkle closeness lookup, giving a + // client-observed p95/median of ~3-6x that straddles + // `latency_inflation_factor` (4.0) and trips Decrease even though + // nothing about it is local congestion. Genuine store congestion + // still surfaces via the timeout-rate ceiling. + // - Never exit slow-start. With the default threshold 0, any single + // Decrease at any cap permanently drops the store cap to additive + // +1-per-healthy-window growth, which cannot reach a useful cap + // before a file finishes (843 chunks stuck at effective ~5-9 in the + // PROD-UL-01 incident). `usize::MAX` keeps slow-start armed at every + // cap, so a transient Decrease still halves but the next healthy + // window doubles it back instead of condemning the rest of the file + // to a crawl. See the fetch override and `LimiterConfig` field docs. + store_cfg.latency_decrease_enabled = false; + store_cfg.slow_start_ramp_threshold = usize::MAX; let mut fetch_cfg = LimiterConfig::from_adaptive(&config, config.max.fetch); // Lift the fetch channel's floor above the global // `min_concurrency`. Reasoning is specific to download: on @@ -1823,8 +1848,9 @@ mod tests { #[test] fn controller_sets_fetch_channel_download_tuning() { - // AdaptiveController::new must apply the download-specific - // tuning to fetch only, leaving quote/store on classic AIMD. + // AdaptiveController::new must apply the slow-start / + // latency-decrease tuning to fetch AND store (V2-468), leaving + // quote on classic AIMD. let c = AdaptiveController::new(ChannelStart::default(), AdaptiveConfig::default()); assert!( !c.fetch.config.latency_decrease_enabled, @@ -1843,8 +1869,116 @@ mod tests { c.quote.config.slow_start_ramp_threshold, 0, "quote must keep classic AIMD slow-start exit", ); - assert!(c.store.config.latency_decrease_enabled); - assert_eq!(c.store.config.slow_start_ramp_threshold, 0); + // Store now mirrors fetch on these two knobs: node-side merkle + // verification latency is not local congestion, and a transient + // Decrease must not condemn the cap to a +1-per-window crawl. + assert!( + !c.store.config.latency_decrease_enabled, + "store latency-decrease must be disabled (verification variance is not congestion)", + ); + assert_eq!( + c.store.config.slow_start_ramp_threshold, + usize::MAX, + "store slow-start must never exit so a transient Decrease re-doubles", + ); + // The store floor must stay at the cold-start value — V2-468 does + // NOT change the floor, only the polluted ramp/decrease signals. + assert_eq!( + c.store.current(), + ChannelStart::default().store, + "store cold-start floor must remain unchanged at 8", + ); + } + + #[test] + fn store_channel_ramps_and_recovers_under_v2_468_tuning() { + // End-to-end on the real `controller.store` limiter: with the + // V2-468 tuning, (a) verification-latency p95 inflation alone must + // not shrink the cap, (b) a genuine timeout burst still cuts it, + // and (c) the cap re-doubles on the next healthy window instead of + // crawling +1 (slow-start stays armed). + let mut adaptive = adaptive_cfg_for_tests(); + // Give the store channel real headroom to ramp. + adaptive.max.store = 256; + let c = AdaptiveController::new( + ChannelStart { + quote: 8, + store: 8, + fetch: 8, + }, + adaptive, + ); + let store = &c.store; + let win = c.config().window_ops; + + // (a) Establish a fast baseline, then a window of slow successes + // (the ~28s verification tail). The cap must not drop. + for _ in 0..win { + store.observe(Outcome::Success, Duration::from_millis(5)); + } + let after_baseline = store.current(); + assert!(after_baseline >= 8, "store should ramp on healthy windows"); + for _ in 0..win { + store.observe(Outcome::Success, Duration::from_secs(30)); + } + assert!( + store.current() >= after_baseline, + "verification-latency p95 must not shrink store cap: {} < {}", + store.current(), + after_baseline, + ); + + // (b) A genuine local-congestion timeout burst must still cut it. + let before_stress = store.current(); + for _ in 0..win { + store.observe(Outcome::Timeout, Duration::from_millis(50)); + } + let after_stress = store.current(); + assert!( + after_stress < before_stress, + "timeout-rate breach must still cut the store cap: {after_stress} !< {before_stress}", + ); + + // (c) Slow-start stays armed, so healthy windows re-DOUBLE the cap + // back to where it was instead of crawling +1 per window. Over this + // many windows additive +1 recovery could not climb back to + // `before_stress` from the stressed floor — only multiplicative + // doubling can — so reaching it proves the crawl pathology is gone. + for _ in 0..(win * 8) { + store.observe(Outcome::Success, Duration::from_millis(5)); + } + assert!( + store.current() >= before_stress, + "store must re-double back to {before_stress} after a transient Decrease, got {}", + store.current(), + ); + } + + #[test] + fn store_application_rejections_do_not_move_cap() { + // The merkle incident's 397 remote app-rejections (now classified + // ApplicationError via `Error::RemotePut`) must not push the store + // cap down — they are not capacity signals. + let mut adaptive = adaptive_cfg_for_tests(); + adaptive.max.store = 256; + let c = AdaptiveController::new( + ChannelStart { + quote: 8, + store: 8, + fetch: 8, + }, + adaptive, + ); + let store = &c.store; + let start = store.current(); + for _ in 0..(c.config().window_ops * 5) { + store.observe(Outcome::ApplicationError, Duration::from_secs(30)); + } + assert_eq!( + store.current(), + start, + "remote app-rejections must not move the store cap", + ); } #[test] @@ -2731,6 +2865,10 @@ mod tests { failed: vec![], failed_count: 0, total_chunks: 0, + spend: Box::new(crate::data::error::PartialUploadSpend { + storage_cost_atto: "0".to_string(), + gas_cost_wei: 0, + }), reason: "r".to_string(), }), ), @@ -2876,7 +3014,7 @@ mod tests { } #[tokio::test] - async fn fetch_hill_accepts_constant_size_upward_probe_from_timed_ops() { + async fn fetch_hill_records_constant_size_timed_ops_without_stress() { let cfg = hill_cfg_for_tests(); let l = fetch_hill_for_tests(HILL_TEST_START_CAP, cfg.clone()); let total_ops = hill_epoch_target_samples(HILL_TEST_START_CAP, &cfg) @@ -2903,15 +3041,19 @@ mod tests { .await; result.unwrap(); - assert_eq!( - l.snapshot(), - HILL_TEST_UP_PROBE_CAP, - "timed constant-size chunks should prove the higher cap improves goodput" + // The timed wrapper records real wall-clock latency. Loaded runners can make the + // wider probe miss the deterministic gain covered by + // `fetch_hill_accepts_upward_probe_with_goodput_gain`, so this test constrains + // the async observation path to a non-stress outcome. + let snapshot = l.snapshot(); + assert!( + matches!(snapshot, HILL_TEST_START_CAP | HILL_TEST_UP_PROBE_CAP), + "timed successes should finish at the existing or accepted best cap, got {snapshot}" ); - assert_eq!( - l.current(), - HILL_TEST_NEXT_UP_PROBE_CAP, - "accepted upward probe should immediately test the next cap" + let current = l.current(); + assert!( + matches!(current, HILL_TEST_START_CAP | HILL_TEST_NEXT_UP_PROBE_CAP), + "timed successes should leave the controller unstressed, got {current}" ); } diff --git a/ant-core/src/data/client/batch.rs b/ant-core/src/data/client/batch.rs index 6221f258..3470b80d 100644 --- a/ant-core/src/data/client/batch.rs +++ b/ant-core/src/data/client/batch.rs @@ -9,7 +9,7 @@ use crate::data::client::classify_error; use crate::data::client::file::UploadEvent; use crate::data::client::payment::peer_id_to_encoded; use crate::data::client::Client; -use crate::data::error::{Error, Result}; +use crate::data::error::{Error, PartialUploadSpend, Result}; use ant_protocol::evm::{ Amount, EncodedPeerId, PayForQuotesError, PaymentQuote, ProofOfPayment, QuoteHash, RewardsAddress, TxHash, @@ -29,6 +29,13 @@ use tracing::{debug, info, warn}; /// Number of chunks per payment wave. const PAYMENT_WAVE_SIZE: usize = 64; +/// Soft ceiling on the combined body size of chunks stored concurrently in a +/// single wave. Caps store concurrency for large chunks so the send path's +/// per-peer body buffers can't pin multiple GB at once (see V2-461). At ~4 MB +/// chunks this permits ~16 concurrent stores; small chunks hit the chunk-count +/// / adaptive limits instead and are unaffected. +const STORE_INFLIGHT_BYTE_BUDGET: usize = 64 * 1024 * 1024; + /// Chunk quoted but not yet paid. Produced by [`Client::prepare_chunk_payment`]. #[derive(Debug)] pub struct PreparedChunk { @@ -236,23 +243,21 @@ impl Client { let data_size = u64::try_from(content.len()) .map_err(|e| Error::InvalidData(format!("content size too large: {e}")))?; - let quotes_with_peers = match self - .get_store_quotes(&address, data_size, DATA_TYPE_CHUNK) + let quote_plan = match self + .get_store_quote_plan(&address, data_size, DATA_TYPE_CHUNK) .await { - Ok(quotes) => quotes, + Ok(plan) => plan, Err(Error::AlreadyStored) => { debug!("Chunk {} already stored, skipping", hex::encode(address)); return Ok(None); } Err(e) => return Err(e), }; + let quotes_with_peers = quote_plan.quotes; // Capture all quoted peers for close-group replication. - let quoted_peers: Vec<(PeerId, Vec)> = quotes_with_peers - .iter() - .map(|(peer_id, addrs, _, _)| (*peer_id, addrs.clone())) - .collect(); + let quoted_peers = quote_plan.put_peers; // Build peer_quotes for ProofOfPayment + quotes for SingleNodePayment. // Use node-reported prices directly — no contract price fetch needed. @@ -413,35 +418,28 @@ impl Client { // is decision-pure: we never hand a doomed proof to a storer, // and the cache is updated under our own lock with no remote // text involved. - // `cached_cost` carries the cumulative cost from waves paid in - // a previous run so the returned tally reflects total spend on - // this file, not just freshly-paid chunks. Without this the - // user's "this upload cost X" message under-reports by the - // resumed waves' cost. - let (cached_proofs, cached_storage, cached_gas): (HashMap>, Amount, u128) = - match resume_key { - Some(key) => match crate::data::client::cached_single::try_load_for_file(key) { - Some((_, receipt)) => { - let prior_storage = receipt - .storage_cost_atto - .parse::() - .unwrap_or(Amount::ZERO); - let prior_gas = receipt.gas_cost_wei; - let kept = prune_locally_expired_proofs(key, receipt.proofs); - (kept, prior_storage, prior_gas) - } - None => (HashMap::new(), Amount::ZERO, 0u128), - }, - None => (HashMap::new(), Amount::ZERO, 0u128), - }; + // Load only the cached PROOFS (for reuse). The cost this function + // returns is a per-call DELTA — what was freshly paid in THIS call — + // not the cache's cumulative. The single-node wave driver + // (`upload_spill_addresses_single`) calls this once per wave and SUMS + // the per-call costs, so seeding the return with the cumulative cache + // (which grows as each wave appends to it) double-counts: + // A + (A+B) + (A+B+C) instead of A+B+C. + let cached_proofs: HashMap> = match resume_key { + Some(key) => match crate::data::client::cached_single::try_load_for_file(key) { + Some((_, receipt)) => prune_locally_expired_proofs(key, receipt.proofs), + None => HashMap::new(), + }, + None => HashMap::new(), + }; let mut all_addresses = Vec::with_capacity(total_chunks); let mut seen_addresses: HashSet = HashSet::new(); - // Accumulate costs across waves, seeded with cumulative from - // any cached receipt loaded above. - let mut total_storage = cached_storage; - let mut total_gas: u128 = cached_gas; + // Accumulate only THIS call's freshly-paid cost (per-call delta; see + // the proof-load comment above for why this must not include the cache). + let mut total_storage = Amount::ZERO; + let mut total_gas: u128 = 0; let mut agg_stats = WaveAggregateStats::default(); // Deduplicate chunks by content address. @@ -520,6 +518,10 @@ impl Client { failed: wave_result.failed, failed_count, total_chunks: file_total, + spend: Box::new(PartialUploadSpend { + storage_cost_atto: total_storage.to_string(), + gas_cost_wei: total_gas, + }), reason: "wave store failed after retries".into(), }); } @@ -618,6 +620,10 @@ impl Client { failed: wave_result.failed, failed_count, total_chunks: file_total, + spend: Box::new(PartialUploadSpend { + storage_cost_atto: total_storage.to_string(), + gas_cost_wei: total_gas, + }), reason: "final wave store failed after retries".into(), }); } @@ -735,6 +741,22 @@ impl Client { first_seen.entry(chunk.address).or_insert_with(Instant::now); } + // Bound concurrency by IN-FLIGHT BYTES, not just chunk count. Each + // concurrently-stored chunk is held in memory while it is sent to its + // close group, and the send path re-serializes the body once per peer, + // so a wave of large (~4 MB) chunks at full store concurrency can pin + // multiple GB and OOM a small host. Cap how many chunks store at once + // so their combined body size stays under the budget; small chunks are + // unaffected (the byte bound exceeds the chunk-count bound). The budget + // is deliberately conservative for the current per-peer send + // amplification and can be raised once that is reduced upstream. + let max_chunk_bytes = to_retry.iter().map(|c| c.content.len()).max().unwrap_or(0); + // `checked_div` yields `None` only when `max_chunk_bytes == 0` (an + // empty/zero-length wave), in which case there is no byte limit. + let byte_bound = STORE_INFLIGHT_BYTE_BUDGET + .checked_div(max_chunk_bytes) + .map_or(usize::MAX, |n| n.max(1)); + let mut chunk_attempts_total: usize = 0; let mut store_durations_ms: Vec = Vec::new(); let mut retries_per_chunk: Vec = Vec::new(); @@ -753,7 +775,10 @@ impl Client { chunk_attempts_total = chunk_attempts_total.saturating_add(to_retry.len()); let store_limiter = self.controller().store.clone(); - let store_concurrency = store_limiter.current().min(to_retry.len().max(1)); + let store_concurrency = store_limiter + .current() + .min(to_retry.len().max(1)) + .min(byte_bound); let mut upload_stream = stream::iter(to_retry) .map(|chunk| { let chunk_clone = chunk.clone(); diff --git a/ant-core/src/data/client/chunk.rs b/ant-core/src/data/client/chunk.rs index 434c1074..8f7ac9f6 100644 --- a/ant-core/src/data/client/chunk.rs +++ b/ant-core/src/data/client/chunk.rs @@ -5,7 +5,6 @@ use crate::data::client::adaptive::Outcome; use crate::data::client::batch::{finalize_batch_payment, PreparedChunk}; -use crate::data::client::peer_cache::record_peer_outcome; use crate::data::client::peer_xor_distance; use crate::data::client::Client; use crate::data::error::{Error, Result}; @@ -178,8 +177,17 @@ impl Client { /// sustained run of close-group exhaustions correctly drives the /// cap down rather than silently inflating it. pub(crate) async fn chunk_get_observed(&self, address: &XorName) -> Result> { + self.chunk_get_observed_from_closest_peers(address, self.config().close_group_size) + .await + } + + pub(crate) async fn chunk_get_observed_from_closest_peers( + &self, + address: &XorName, + peer_count: usize, + ) -> Result> { let started = Instant::now(); - let result = self.chunk_get(address).await; + let result = self.chunk_get_from_closest_peers(address, peer_count).await; let latency = started.elapsed(); let bytes = result .as_ref() @@ -280,6 +288,17 @@ impl Client { let mut success_count = 0usize; let mut failures: Vec = Vec::new(); + // Distinguish the *cause* of a quorum shortfall so it feeds the + // store AIMD limiter correctly (V2-468). If every failure was a + // structured remote application rejection (`Error::RemotePut` — the + // node responded and declined: pool-rejected / quote-stale / + // disk-full), the shortfall is not evidence the client is sending + // too fast and must not push the limiter down. Anything else + // (transport failure, or a different error) keeps it a real + // capacity signal. Hold the first remote rejection as the + // representative reason to surface when the shortfall is app-only. + let mut had_non_rejection_failure = false; + let mut first_remote_rejection: Option = None; let mut fallback_iter = fallback_peers.iter(); while let Some((peer_id, result)) = put_futures.next().await { @@ -297,6 +316,13 @@ impl Client { Err(e) => { warn!("Failed to store chunk on {peer_id}: {e}"); failures.push(format!("{peer_id}: {e}")); + if matches!(e, Error::RemotePut { .. }) { + if first_remote_rejection.is_none() { + first_remote_rejection = Some(e); + } + } else { + had_non_rejection_failure = true; + } if let Some((fb_peer, fb_addrs)) = fallback_iter.next() { debug!( @@ -314,6 +340,17 @@ impl Client { } } + // Quorum not reached. If the only failures were structured remote + // rejections, surface a representative `RemotePut` (classifies + // `ApplicationError`, still recoverable in the merkle retry path) + // so the shortfall doesn't suppress the store limiter. Otherwise + // it's a real capacity shortfall. + if !had_non_rejection_failure { + if let Some(remote_rejection) = first_remote_rejection { + return Err(remote_rejection); + } + } + Err(Error::InsufficientPeers(format!( "Stored on {success_count} peers, need {CLOSE_GROUP_MAJORITY}. Failures: [{}]", failures.join("; ") @@ -394,9 +431,17 @@ impl Client { ChunkMessageBody::PutResponse(ChunkPutResponse::PaymentRequired { message }) => { Some(Err(Error::Payment(format!("Payment required: {message}")))) } - ChunkMessageBody::PutResponse(ChunkPutResponse::Error(e)) => Some(Err( - Error::Protocol(format!("Remote PUT error for {addr_hex}: {e}")), - )), + ChunkMessageBody::PutResponse(ChunkPutResponse::Error(e)) => { + // Preserve the structured remote reason instead of + // flattening it into `Error::Protocol`. The node + // responded, so the transport round-trip succeeded — + // this is an application-level rejection and must not + // suppress the store AIMD limiter (V2-468). + Some(Err(Error::RemotePut { + address: addr_hex.clone(), + source: e, + })) + } _ => None, }, |e| Error::Network(format!("Failed to send PUT to peer: {e}")), @@ -408,12 +453,6 @@ impl Client { ) .await; - // No RTT recorded on the PUT path: the wall-clock is dominated by - // the ~4 MB payload upload, which reflects the uploader's uplink - // rather than the peer's responsiveness. Quote-path and GET-path - // RTTs still feed quality scoring. - record_peer_outcome(node, *target_peer, peer_addrs, result.is_ok(), None).await; - result } @@ -763,7 +802,6 @@ impl Client { let addr_hex = hex::encode(address); let timeout_secs = self.config().chunk_get_timeout_secs; - let start = Instant::now(); let result = send_and_await_chunk_response( node, peer, @@ -813,10 +851,6 @@ impl Client { ) .await; - let success = result.is_ok(); - let rtt_ms = success.then(|| start.elapsed().as_millis() as u64); - record_peer_outcome(node, *peer, peer_addrs, success, rtt_ms).await; - result } diff --git a/ant-core/src/data/client/data.rs b/ant-core/src/data/client/data.rs index fc951fa8..2446f6fb 100644 --- a/ant-core/src/data/client/data.rs +++ b/ant-core/src/data/client/data.rs @@ -17,6 +17,7 @@ use ant_protocol::{compute_address, DATA_TYPE_CHUNK}; use bytes::Bytes; use futures::stream::StreamExt; use self_encryption::{decrypt, encrypt, DataMap, EncryptedChunk}; +use std::num::NonZeroUsize; use tracing::{debug, info}; /// Result of an in-memory data upload: the `DataMap` needed to retrieve the data. @@ -401,8 +402,31 @@ impl Client { )) })?; - rmp_serde::from_slice(&chunk.content) - .map_err(|e| Error::Serialization(format!("Failed to deserialize DataMap: {e}"))) + decode_data_map_chunk(&chunk.content) + } + + /// Fetch a `DataMap` from the network by trying the requested number + /// of closest peers for the DataMap chunk. + /// + /// # Errors + /// + /// Returns an error if the chunk is not found or deserialization fails. + pub async fn data_map_fetch_from_closest_peers( + &self, + address: &[u8; 32], + peer_count: NonZeroUsize, + ) -> Result { + let chunk = self + .chunk_get_from_closest_peers(address, peer_count.get()) + .await? + .ok_or_else(|| { + Error::InvalidData(format!( + "DataMap chunk not found at {}", + hex::encode(address) + )) + })?; + + decode_data_map_chunk(&chunk.content) } /// Download and decrypt data from the network using its `DataMap`. @@ -469,6 +493,11 @@ impl Client { } } +fn decode_data_map_chunk(content: &[u8]) -> Result { + rmp_serde::from_slice(content) + .map_err(|e| Error::Serialization(format!("Failed to deserialize DataMap: {e}"))) +} + /// Compile-time assertions that Client method futures are Send. /// /// These methods are called from axum handlers and tokio::spawn contexts diff --git a/ant-core/src/data/client/file.rs b/ant-core/src/data/client/file.rs index 4d190fac..de9cfbab 100644 --- a/ant-core/src/data/client/file.rs +++ b/ant-core/src/data/client/file.rs @@ -16,12 +16,12 @@ use crate::data::client::batch::{ }; use crate::data::client::classify_error; use crate::data::client::merkle::{ - chunk_contents_for_upload_addresses, finalize_merkle_batch, merkle_store_with_retry, - should_use_merkle, MerkleBatchPaymentResult, PaymentMode, PreparedMerkleBatch, - MERKLE_RETRY_BACKOFF, MERKLE_STORE_MAX_ATTEMPTS, + chunk_contents_for_upload_addresses, finalize_merkle_batch, merkle_deferred_retry, + merkle_store_with_retry, should_use_merkle, MerkleBatchPaymentResult, PaymentMode, + PreparedMerkleBatch, DEFERRED_ROUND_DELAYS_SECS, }; use crate::data::client::Client; -use crate::data::error::{Error, Result}; +use crate::data::error::{Error, PartialUploadSpend, Result}; use ant_protocol::evm::{Amount, PaymentQuote, QuoteHash, TxHash, MAX_LEAVES}; use ant_protocol::transport::{MultiAddr, PeerId}; use ant_protocol::{compute_address, DATA_TYPE_CHUNK}; @@ -34,6 +34,7 @@ use self_encryption::{ }; use std::collections::{HashMap, HashSet}; use std::io::Write; +use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use tokio::runtime::Handle; @@ -110,6 +111,33 @@ const DOWNLOAD_STREAM_BATCH_BYTES_PER_CHUNK_MULTIPLIER: u64 = 3; /// of a file already live on the network. const ESTIMATE_SAMPLE_CAP: usize = 5; +/// Pick up to `cap` chunk indices spread evenly across `[0, total)`, always +/// including the first and last chunk. +/// +/// Sampling the *first* N chunks biases the probe: a file sharing a leading +/// prefix with a prior upload (compressed archives, similar headers) reports +/// those chunks as `AlreadyStored` even when the tail is new, so a positional +/// sample looks in the worst possible place. Spreading the sample means a +/// single new chunk anywhere in the file yields a real price. +/// +/// Returns `[0]` for a single chunk and every index when `total <= cap`, so +/// [`Client::estimate_upload_cost`] can still detect the "whole file sampled" +/// case. Indices are strictly increasing. +fn distributed_sample_indices(total: usize, cap: usize) -> Vec { + if total == 0 { + return Vec::new(); + } + let sample_limit = total.min(cap); + if sample_limit <= 1 { + return vec![0]; + } + let mut indices: Vec = (0..sample_limit) + .map(|i| i * (total - 1) / (sample_limit - 1)) + .collect(); + indices.dedup(); // defensive: already strictly increasing for cap >= 2 + indices +} + /// Gas used by one `pay_for_quotes` transaction that packs up to /// `UPLOAD_WAVE_SIZE` (quote_hash, rewards_address, amount) entries. /// @@ -444,6 +472,107 @@ fn partition_addresses_by_proof( .partition(|addr| proofs.contains_key(addr)) } +/// Build a `PartialUpload` after a fatal merkle store error, with accurate +/// counts. +/// +/// A fatal abort can leave chunks in three states: confirmed stored (in +/// `stored_addresses`), known-failed (in `known_failed` — missing proofs, the +/// quorum shortfalls and the fatal chunk seen so far), and "in flight when the +/// abort hit" (neither). Rather than trust the helpers to enumerate the last +/// group, this derives the failed set authoritatively as *every* `addresses` +/// entry not in `stored_addresses`, preferring a known per-chunk message and +/// falling back to the fatal `reason`. That guarantees +/// `stored_count + failed_count` accounts for the whole file — fixing the +/// under-reporting where a fatal wave could surface `failed_count = 0` and omit +/// same-pass successes. +fn partial_upload_after_fatal( + addresses: &[[u8; 32]], + stored_addresses: Vec<[u8; 32]>, + stored_count: usize, + total_chunks: usize, + known_failed: Vec<([u8; 32], String)>, + spend: PartialUploadSpend, + reason: String, +) -> Error { + let stored_set: HashSet<[u8; 32]> = stored_addresses.iter().copied().collect(); + let mut failed_map: HashMap<[u8; 32], String> = HashMap::new(); + for (addr, msg) in known_failed { + if !stored_set.contains(&addr) { + failed_map.entry(addr).or_insert(msg); + } + } + for addr in addresses { + if !stored_set.contains(addr) { + failed_map.entry(*addr).or_insert_with(|| reason.clone()); + } + } + let failed: Vec<([u8; 32], String)> = failed_map.into_iter().collect(); + let failed_count = failed.len(); + Error::PartialUpload { + stored: stored_addresses, + stored_count, + failed, + failed_count, + total_chunks, + spend: Box::new(spend), + reason, + } +} + +/// One wave's contribution to a single-node upload, distilled from its +/// `batch_upload_chunks_with_events` result. +#[derive(Debug)] +struct SingleWaveOutcome { + /// Addresses confirmed stored in this wave. + stored: Vec<[u8; 32]>, + /// Chunks that failed after retries in this wave. + failed: Vec<([u8; 32], String)>, + /// Storage cost paid on-chain for this wave, in atto-tokens. + storage_atto: Amount, + /// Gas paid on-chain for this wave, in wei. + gas_wei: u128, + /// Per-wave store/retry statistics. Empty for a quorum-short wave, whose + /// `PartialUpload` carries no stats. + stats: WaveAggregateStats, +} + +/// Fold one wave's batch-upload result for the single-node path. +/// +/// A `PartialUpload` (chunks short of quorum after retries) is **recoverable**: +/// its stored/failed chunks and on-chain spend are returned so the caller +/// records them and continues to the next wave, making the file make maximum +/// progress exactly like `upload_waves_merkle`. Every other error is **fatal** +/// (wallet/payment-infrastructure failures, missing proofs, spill reads) and is +/// returned via `Err` to abort the file. Because `UPLOAD_WAVE_SIZE == +/// PAYMENT_WAVE_SIZE`, each batch call is exactly one payment wave, so folding a +/// `PartialUpload` leaves nothing un-attempted within the wave. +fn fold_single_wave( + result: Result<(Vec<[u8; 32]>, String, u128, WaveAggregateStats)>, +) -> Result { + match result { + Ok((stored, storage, gas, stats)) => Ok(SingleWaveOutcome { + stored, + failed: Vec::new(), + storage_atto: storage.parse().unwrap_or(Amount::ZERO), + gas_wei: gas, + stats, + }), + Err(Error::PartialUpload { + stored, + failed, + spend, + .. + }) => Ok(SingleWaveOutcome { + stored, + failed, + storage_atto: spend.storage_cost_atto.parse().unwrap_or(Amount::ZERO), + gas_wei: spend.gas_cost_wei, + stats: WaveAggregateStats::default(), + }), + Err(e) => Err(e), + } +} + /// Check that the spill directory has enough free space for the spilled chunks. /// /// `file_size` is the source file's byte count. We require @@ -570,9 +699,38 @@ pub enum Visibility { Public, } +/// Confidence attached to an [`UploadCostEstimate`]'s `storage_cost_atto`. +/// +/// `estimate_upload_cost` prices a file by sampling a few of its chunk +/// addresses and extrapolating. When every sampled chunk is already stored +/// there is no live price to extrapolate from, so a `"0"` cost can mean either +/// "provably free" (the whole file was sampled) or only "probably free" (the +/// tail was unsampled). This lets callers tell those apart instead of treating +/// every `"0"` as unconditionally free. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CostEstimateConfidence { + /// At least one sampled chunk returned a live quote; `storage_cost_atto` + /// is extrapolated from a real per-chunk price. The normal case. + #[default] + PricedSample, + /// Every chunk in the file was sampled and every one was already stored. + /// `storage_cost_atto` is exactly `"0"` — the upload is genuinely free. + VerifiedAllAlreadyStored, + /// Every *sampled* chunk was already stored, but not all chunks were + /// sampled. `storage_cost_atto` is `"0"` as a best-effort guess; the real + /// upload reconciles the true cost at payment time. Render this as "likely + /// already stored", not a guaranteed-free price. + AllSamplesAlreadyStoredIncomplete, +} + /// Estimated cost of uploading a file, returned by /// [`Client::estimate_upload_cost`]. +/// +/// Marked `#[non_exhaustive]` so adding a field later is not a breaking change +/// for downstream consumers that construct or pattern-match on this struct. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[non_exhaustive] pub struct UploadCostEstimate { /// Original file size in bytes. pub file_size: u64, @@ -586,6 +744,9 @@ pub struct UploadCostEstimate { pub estimated_gas_cost_wei: String, /// Payment mode that would be used. pub payment_mode: PaymentMode, + /// How much to trust `storage_cost_atto`. See [`CostEstimateConfidence`]. + #[serde(default)] + pub confidence: CostEstimateConfidence, } /// Result of a file upload: the `DataMap` needed to retrieve the file. @@ -778,6 +939,55 @@ fn spawn_file_encryption(path: PathBuf) -> Result { Ok((chunk_rx, datamap_rx, handle)) } +/// RAII guard for the staging temp file used during a disk download. +/// +/// Removes the file on drop — including a panic unwind out of the +/// `block_in_place` decrypt loop — unless [`commit`](Self::commit) has +/// promoted it to its final path. Centralizes the cleanup the explicit error +/// arms used to repeat. +struct TempDownload { + /// `Some` while the staging file may need cleanup; `None` once committed. + path: Option, +} + +impl TempDownload { + fn new(path: PathBuf) -> Self { + Self { path: Some(path) } + } + + /// Path of the staging file (valid until `commit`). + fn path(&self) -> &Path { + self.path + .as_deref() + .expect("TempDownload::path called after commit") + } + + /// Rename the staged file to `dest`. On success the guard is defused so + /// `Drop` is a no-op; on failure the guard stays armed and `Drop` removes + /// the orphaned temp file. + fn commit(mut self, dest: &Path) -> std::io::Result<()> { + std::fs::rename(self.path(), dest)?; // err → guard armed → Drop cleans up + self.path = None; // success → nothing left to clean + Ok(()) + } +} + +impl Drop for TempDownload { + fn drop(&mut self) { + if let Some(path) = self.path.take() { + if let Err(e) = std::fs::remove_file(&path) { + // Absent file is fine (never created / already gone). + if e.kind() != std::io::ErrorKind::NotFound { + warn!( + "Failed to remove temp download file {}: {e}", + path.display() + ); + } + } + } + } +} + impl Client { /// Upload a file to the network using streaming self-encryption. /// @@ -807,19 +1017,22 @@ impl Client { /// `GAS_PER_MERKLE_TX`) priced at `ARBITRUM_GAS_PRICE_WEI`. Real gas /// varies with network conditions. /// - /// If the first sampled chunk is already stored on the network, the - /// function retries with subsequent chunk addresses (up to - /// `ESTIMATE_SAMPLE_CAP`). If every sampled address reports stored, - /// a [`Error::CostEstimationInconclusive`] is returned so callers can - /// decide how to react rather than trust a bogus "free" estimate. Only - /// when every address in the file is stored do we return a zero-cost - /// estimate. + /// Sampled chunk addresses are spread across the whole file (not the first + /// N) so a shared leading prefix doesn't bias the sample. When a sample + /// returns a live quote the per-chunk price is extrapolated and the result + /// is tagged [`CostEstimateConfidence::PricedSample`]. + /// + /// When every sampled chunk is already stored the result is still `Ok` + /// with `storage_cost_atto: "0"`, tagged either + /// [`CostEstimateConfidence::VerifiedAllAlreadyStored`] when the whole file + /// was sampled (exactly free) or + /// [`CostEstimateConfidence::AllSamplesAlreadyStoredIncomplete`] when the + /// tail was unsampled (a best-effort guess that payment reconciles). /// /// # Errors /// - /// Returns an error if the file cannot be read, encryption fails, - /// the network cannot provide a quote, or every sampled chunk is - /// already stored ([`Error::CostEstimationInconclusive`]). + /// Returns an error if the file cannot be read, encryption fails, or the + /// network cannot provide a quote. pub async fn estimate_upload_cost( &self, path: &Path, @@ -853,26 +1066,33 @@ impl Client { } info!("Encrypted into {chunk_count} chunks, requesting quote"); + let uses_merkle = should_use_merkle(chunk_count, mode); - // Sample up to ESTIMATE_SAMPLE_CAP distinct chunk addresses. A single - // AlreadyStored result says nothing about the rest of the file — the - // first chunk is often a DataMap-adjacent chunk that collides with - // prior uploads even when 99% of the file is new. Only treat the - // whole file as "fully stored" when every sample comes back stored. - let sample_limit = spill.addresses.len().min(ESTIMATE_SAMPLE_CAP); + // Sample chunk addresses spread evenly across the file (see + // `distributed_sample_indices`) rather than the first N. A single + // AlreadyStored result says nothing about the rest of the file, and a + // positional sample lands on a shared leading prefix in the worst case, + // so we spread the probe and only treat the whole file as "fully + // stored" when every sample comes back stored. + let sample_indices = distributed_sample_indices(spill.addresses.len(), ESTIMATE_SAMPLE_CAP); let mut sampled = 0usize; let mut all_already_stored = true; let mut quotes_opt: Option> = None; - for addr in spill.addresses.iter().take(sample_limit) { + for &idx in &sample_indices { + let addr = &spill.addresses[idx]; sampled += 1; let chunk_bytes = spill.read_chunk(addr)?; let data_size = u64::try_from(chunk_bytes.len()) .map_err(|e| Error::InvalidData(format!("chunk size too large: {e}")))?; - match self - .get_store_quotes(addr, data_size, DATA_TYPE_CHUNK) - .await - { + let result = if uses_merkle { + self.get_store_quotes_with_fault_tolerance(addr, data_size, DATA_TYPE_CHUNK) + .await + } else { + self.get_store_quotes(addr, data_size, DATA_TYPE_CHUNK) + .await + }; + match result { Ok(q) => { quotes_opt = Some(q); all_already_stored = false; @@ -880,8 +1100,9 @@ impl Client { } Err(Error::AlreadyStored) => { debug!( - "Sample chunk {} already stored; trying next address ({sampled}/{sample_limit})", - hex::encode(addr) + "Sample chunk {} already stored; trying next address ({sampled}/{})", + hex::encode(addr), + sample_indices.len() ); continue; } @@ -889,14 +1110,11 @@ impl Client { } } - let uses_merkle = should_use_merkle(chunk_count, mode); - let quotes = match quotes_opt { Some(q) => q, None if all_already_stored && sampled == chunk_count => { // Every address in the file was sampled and every one is - // already on the network — returning a zero-cost estimate is - // accurate in this case. + // already on the network — a zero-cost estimate is exact here. info!("All {chunk_count} chunks already stored; returning zero-cost estimate"); return Ok(UploadCostEstimate { file_size, @@ -908,14 +1126,31 @@ impl Client { } else { PaymentMode::Single }, + confidence: CostEstimateConfidence::VerifiedAllAlreadyStored, }); } None => { - return Err(Error::CostEstimationInconclusive(format!( - "sampled {sampled} chunk addresses out of {chunk_count} and every \ - one reported AlreadyStored; cannot infer a representative price \ - for the remaining chunks" - ))); + // Every sampled chunk was already stored but the tail was not + // sampled, so there is no live price to extrapolate. The + // estimate is display-only and payment reconciles the true + // cost, so return an optimistic zero flagged as incomplete + // rather than erroring — callers still get a value to show. + info!( + "All {sampled}/{chunk_count} sampled chunks already stored; \ + returning incomplete zero-cost estimate" + ); + return Ok(UploadCostEstimate { + file_size, + chunk_count, + storage_cost_atto: "0".into(), + estimated_gas_cost_wei: "0".into(), + payment_mode: if uses_merkle { + PaymentMode::Merkle + } else { + PaymentMode::Single + }, + confidence: CostEstimateConfidence::AllSamplesAlreadyStoredIncomplete, + }); } }; @@ -973,6 +1208,7 @@ impl Client { } else { PaymentMode::Single }, + confidence: CostEstimateConfidence::PricedSample, }) } @@ -1334,7 +1570,7 @@ impl Client { match prepared.payment_info { ExternalPaymentInfo::WaveBatch { prepared_chunks, - payment_intent: _, + payment_intent, } => { let paid_chunks = finalize_batch_payment(prepared_chunks, tx_hash_map)?; let wave_result = self @@ -1356,6 +1592,13 @@ impl Client { failed: wave_result.failed, failed_count, total_chunks, + // Report the storage spend known from the payment intent + // the external signer was handed. Gas is paid by the + // signer out-of-band, so it stays unknown (0). + spend: Box::new(PartialUploadSpend { + storage_cost_atto: payment_intent.total_amount.to_string(), + gas_cost_wei: 0, + }), reason: "finalize_upload: chunk storage failed after retries".into(), }); } @@ -1372,7 +1615,9 @@ impl Client { chunks_failed: 0, total_chunks, payment_mode_used: PaymentMode::Single, - storage_cost_atto: "0".into(), + // Storage spend is known from the payment intent; gas is + // paid by the external signer out-of-band (unknown here). + storage_cost_atto: payment_intent.total_amount.to_string(), gas_cost_wei: 0, data_map_address, chunk_attempts_total: stats.chunk_attempts_total, @@ -1492,6 +1737,21 @@ impl Client { self.file_upload_with_progress(path, mode, None).await } + /// Upload a file publicly, storing the serialized [`DataMap`] as part of + /// the same upload payment batch. + /// + /// The returned [`FileUploadResult::data_map_address`] can be shared for + /// public downloads via [`Client::data_map_fetch`]. + #[allow(clippy::too_many_lines)] + pub async fn file_upload_public_with_mode( + &self, + path: &Path, + mode: PaymentMode, + ) -> Result { + self.file_upload_with_visibility_and_progress(path, mode, Visibility::Public, None) + .await + } + /// Upload a file with progress events sent to the given channel. /// /// Same as [`Client::file_upload_with_mode`] but sends [`UploadEvent`]s to the @@ -1502,9 +1762,36 @@ impl Client { path: &Path, mode: PaymentMode, progress: Option>, + ) -> Result { + self.file_upload_with_visibility_and_progress(path, mode, Visibility::Private, progress) + .await + } + + /// Public file upload with progress events. + /// + /// Same as [`Client::file_upload_public_with_mode`] but sends + /// [`UploadEvent`]s to the provided channel for UI progress feedback. + #[allow(clippy::too_many_lines)] + pub async fn file_upload_public_with_progress( + &self, + path: &Path, + mode: PaymentMode, + progress: Option>, + ) -> Result { + self.file_upload_with_visibility_and_progress(path, mode, Visibility::Public, progress) + .await + } + + #[allow(clippy::too_many_lines)] + async fn file_upload_with_visibility_and_progress( + &self, + path: &Path, + mode: PaymentMode, + visibility: Visibility, + progress: Option>, ) -> Result { debug!( - "Streaming file upload with mode {mode:?}: {}", + "Streaming file upload with mode {mode:?}, visibility {visibility:?}: {}", path.display() ); @@ -1514,7 +1801,24 @@ impl Client { // Phase 1: Encrypt file and spill chunks to temp directory. // Only 32-byte addresses stay in memory — chunk data lives on disk. - let (spill, data_map) = self.encrypt_file_to_spill(path, progress.as_ref()).await?; + let (mut spill, data_map) = self.encrypt_file_to_spill(path, progress.as_ref()).await?; + + let data_map_address = match visibility { + Visibility::Private => None, + Visibility::Public => { + let serialized = rmp_serde::to_vec(&data_map).map_err(|e| { + Error::Serialization(format!("Failed to serialize DataMap: {e}")) + })?; + let address = compute_address(&serialized); + info!( + "Public upload: adding DataMap chunk ({} bytes) at address {} to payment batch", + serialized.len(), + hex::encode(address) + ); + spill.push(&serialized)?; + Some(address) + } + }; let chunk_count = spill.len(); info!( @@ -1585,7 +1889,7 @@ impl Client { payment_mode_used: PaymentMode::Merkle, storage_cost_atto: sc, gas_cost_wei: gc, - data_map_address: None, + data_map_address, chunk_attempts_total: stats.chunk_attempts_total, store_durations_ms: stats.store_durations_ms, retries_histogram: stats.retries_histogram, @@ -1613,7 +1917,7 @@ impl Client { payment_mode_used: PaymentMode::Single, storage_cost_atto: sc, gas_cost_wei: gc, - data_map_address: None, + data_map_address, chunk_attempts_total: fb_stats.chunk_attempts_total, store_durations_ms: fb_stats.store_durations_ms, retries_histogram: fb_stats.retries_histogram, @@ -1675,7 +1979,7 @@ impl Client { &spill, &merkle_plan.to_upload, progress.as_ref(), - merkle_plan.already_stored.len(), + &merkle_plan.already_stored, chunk_count, Some(&file_path_key), ) @@ -1737,7 +2041,7 @@ impl Client { &spill, &merkle_plan.to_upload, progress.as_ref(), - merkle_plan.already_stored.len(), + &merkle_plan.already_stored, chunk_count, Some(&file_path_key), ) @@ -1751,7 +2055,7 @@ impl Client { payment_mode_used: PaymentMode::Single, storage_cost_atto: sc, gas_cost_wei: gc, - data_map_address: None, + data_map_address, chunk_attempts_total: fb_stats.chunk_attempts_total, store_durations_ms: fb_stats.store_durations_ms, retries_histogram: fb_stats.retries_histogram, @@ -1796,7 +2100,7 @@ impl Client { payment_mode_used: actual_mode, storage_cost_atto, gas_cost_wei, - data_map_address: None, + data_map_address, chunk_attempts_total: stats.chunk_attempts_total, store_durations_ms: stats.store_durations_ms, retries_histogram: stats.retries_histogram, @@ -1863,7 +2167,7 @@ impl Client { spill, &spill.addresses, progress, - 0, + &[], spill.len(), resume_key, ) @@ -1875,17 +2179,38 @@ impl Client { spill: &ChunkSpill, addresses: &[[u8; 32]], progress: Option<&mpsc::Sender>, - stored_offset: usize, + already_stored_addresses: &[[u8; 32]], total_chunks: usize, resume_key: Option<&str>, ) -> Result<(usize, String, u128, WaveAggregateStats)> { - let mut total_stored = stored_offset; + let mut total_stored = already_stored_addresses.len(); let mut total_storage = Amount::ZERO; let mut total_gas: u128 = 0; let mut agg_stats = WaveAggregateStats::default(); + // A wave whose chunks fall short of quorum after retries must not abort + // the file: its failures are accumulated here and surfaced as a single + // `PartialUpload` only after every wave has been attempted, mirroring + // `upload_waves_merkle`. Aborting on the first failed wave (the old `?`) + // discarded all later waves' progress — already self-encrypted, spilled, + // and in some cases already paid for — converting high per-chunk success + // into 0% per-file success. + // Seed with the addresses a preflight already confirmed stored (e.g. + // the merkle-fallback path passes `merkle_plan.already_stored`), so a + // returned `PartialUpload.stored` lists every stored chunk and + // `stored_count == stored.len()` holds for programmatic callers. + let mut stored_addresses: Vec<[u8; 32]> = already_stored_addresses.to_vec(); + let mut failed: Vec<([u8; 32], String)> = Vec::new(); let waves: Vec<&[[u8; 32]]> = addresses.chunks(UPLOAD_WAVE_SIZE).collect(); let wave_count = waves.len(); + // Unconditional breadcrumb: lets a clean run confirm the continue-on- + // partial single-node path is in effect (the old path aborted the file + // on the first failed wave instead of continuing across all waves). + info!( + "single-node upload: {} chunk(s) in {wave_count} wave(s) (continue-on-partial)", + addresses.len() + ); + for (wave_idx, wave_addrs) in waves.into_iter().enumerate() { let wave_num = wave_idx + 1; let wave_data: Vec = wave_addrs @@ -1906,35 +2231,50 @@ impl Client { }) .await; } - let (addresses, wave_storage, wave_gas, wave_stats) = self - .batch_upload_chunks_with_events( + // Fold this wave's result. A quorum shortfall (`PartialUpload`) is + // recoverable and its parts are returned to be recorded here; + // genuinely fatal errors propagate via `?` and abort the file, as in + // `upload_waves_merkle`. + let outcome = fold_single_wave( + self.batch_upload_chunks_with_events( wave_data, progress, total_stored, total_chunks, resume_key, ) - .await?; - total_stored += addresses.len(); - if let Ok(cost) = wave_storage.parse::() { - total_storage += cost; + .await, + )?; + + if !outcome.failed.is_empty() { + warn!( + "Wave {wave_num}/{wave_count}: {} chunk(s) failed to store after retries; \ + continuing with remaining waves", + outcome.failed.len() + ); } - total_gas = total_gas.saturating_add(wave_gas); - // Merge per-call stats (each call already aggregates across the - // waves it ran internally, so a simple sum/extend is correct). + + total_stored += outcome.stored.len(); + stored_addresses.extend(outcome.stored); + failed.extend(outcome.failed); + total_storage += outcome.storage_atto; + total_gas = total_gas.saturating_add(outcome.gas_wei); + // Merge per-wave stats (a quorum-short wave contributes none, since + // `PartialUpload` carries no stats). agg_stats.chunk_attempts_total = agg_stats .chunk_attempts_total - .saturating_add(wave_stats.chunk_attempts_total); + .saturating_add(outcome.stats.chunk_attempts_total); agg_stats .store_durations_ms - .extend(wave_stats.store_durations_ms); + .extend(outcome.stats.store_durations_ms); for (slot, count) in agg_stats .retries_histogram .iter_mut() - .zip(wave_stats.retries_histogram.iter()) + .zip(outcome.stats.retries_histogram.iter()) { *slot = slot.saturating_add(*count); } + if let Some(tx) = progress { let _ = tx .send(UploadEvent::WaveComplete { @@ -1947,6 +2287,28 @@ impl Client { } } + // Any chunk still failed after every wave was attempted means the file + // is not fully stored — surface it as `PartialUpload` (never silently + // succeed with missing chunks), carrying the real on-chain spend. + if !failed.is_empty() { + let failed_count = failed.len(); + warn!( + "single-node upload incomplete: {failed_count}/{total_chunks} chunks failed after retries" + ); + return Err(Error::PartialUpload { + stored: stored_addresses, + stored_count: total_stored, + failed, + failed_count, + total_chunks, + spend: Box::new(PartialUploadSpend { + storage_cost_atto: total_storage.to_string(), + gas_cost_wei: total_gas, + }), + reason: format!("{failed_count} chunk(s) failed to store after retries"), + }); + } + Ok(( total_stored, total_storage.to_string(), @@ -1961,12 +2323,16 @@ impl Client { /// and uploads concurrently. Peak memory: ~`UPLOAD_WAVE_SIZE × MAX_CHUNK_SIZE`. /// /// A chunk that is transiently short of quorum (`InsufficientPeers`) does - /// **not** abort the file: each wave is driven through - /// [`merkle_store_with_retry`], which collects such chunks and retries them - /// — re-collecting their close group and reusing the same proof — for up to - /// [`MERKLE_STORE_MAX_ATTEMPTS`] rounds with a [`MERKLE_RETRY_BACKOFF`] wait - /// between rounds. Retry is per-wave to preserve the streaming memory bound. - /// Non-quorum errors (e.g. a missing proof) stay fatal and abort immediately. + /// **not** abort the file, nor does it block the pipeline: each wave is + /// stored in a **single pass** (no in-wave backoff barrier), and chunks + /// short of quorum are collected into a file-level deferred set rather than + /// retried in place. After the last wave, [`merkle_deferred_retry`] retries + /// the whole deferred set in concurrent rounds ([`DEFERRED_ROUND_DELAYS_SECS`] + /// delays), re-reading each chunk's body from the spill and reusing its + /// proof. This keeps every wave running at full fan-out instead of parking + /// idle slots behind one slow chunk's backoff, while peak memory stays + /// bounded (bodies are re-read from disk, never pinned). Non-quorum errors + /// (e.g. a missing proof) stay fatal and abort immediately. /// /// Returns `(chunks_stored, storage_cost_atto, gas_cost_wei)` on success. /// Costs come from the `batch_result` which was populated during payment. @@ -1988,6 +2354,10 @@ impl Client { let total_chunks = total_stored + addresses.len(); let mut stored_addresses: Vec<[u8; 32]> = already_stored_addresses.to_vec(); let mut failed: Vec<([u8; 32], String)> = Vec::new(); + // Chunks short of quorum on their single wave pass are collected here and + // retried after the last wave (see `merkle_deferred_retry`), so a slow + // chunk never holds its wave's other slots idle behind a backoff. + let mut deferred: Vec<([u8; 32], String)> = Vec::new(); let mut agg_stats = WaveAggregateStats::default(); // Chunks without a merkle proof were never paid for: a partial @@ -2061,57 +2431,30 @@ impl Client { .map(|(content, addr)| (addr, content)) .collect(); - // Retry quorum-short chunks instead of aborting on the first miss. - // `stored_offset` is the running cumulative count so the progress - // events the driver emits stay correctly numbered across waves. - let outcome = match merkle_store_with_retry( + // Store the wave in a SINGLE pass (`max_attempts = 1`, no backoff): + // quorum-short chunks are collected and deferred to a post-wave + // concurrent retry rather than parking this wave's other slots + // behind a backoff. `stored_offset` is the running cumulative count + // so the progress events the driver emits stay correctly numbered + // across waves. + let outcome = merkle_store_with_retry( chunks, store_concurrency, - MERKLE_STORE_MAX_ATTEMPTS, - MERKLE_RETRY_BACKOFF, + 1, + std::time::Duration::ZERO, progress, total_stored, total_chunks, &store_one, ) - .await - { - Ok(outcome) => outcome, - Err(e) => { - // A non-quorum store error is fatal for the retry helper - // (missing proofs were filtered out above, so this is a - // genuine network/store failure, e.g. a DHT lookup error). - // Surface it at the file boundary as `PartialUpload` so the - // chunks already stored in earlier waves — and any - // missing-proof chunks already recorded — are preserved for - // resume, rather than discarded with a generic error. - warn!("merkle wave {wave_num}/{wave_count} aborted: {e}"); - let failed_count = failed.len(); - return Err(Error::PartialUpload { - stored: stored_addresses, - stored_count: total_stored, - failed, - failed_count, - total_chunks, - reason: format!("merkle chunk store aborted: {e}"), - }); - } - }; - - // Record which of this wave's chunks landed and which exhausted - // their retries, so a permanently-failed chunk can surface as - // `PartialUpload` once the whole file has been attempted. - let wave_failed: HashSet<[u8; 32]> = outcome - .failed_addresses - .iter() - .map(|(addr, _)| *addr) - .collect(); - for addr in wave_addrs { - if !wave_failed.contains(addr) { - stored_addresses.push(*addr); - } - } - failed.extend(outcome.failed_addresses); + .await?; + + // Record this wave's confirmed stores from the explicit set the + // store helper reports. Using that set (rather than inferring + // "wave chunks minus failed") keeps `stored_addresses` correct even + // when a fatal abort leaves some of the wave neither stored nor + // reported short of quorum. + stored_addresses.extend(&outcome.stored_addresses); total_stored = outcome.stored; // Merge per-wave stats (durations, attempts, per-round histogram). @@ -2129,6 +2472,38 @@ impl Client { *slot = slot.saturating_add(*count); } + if let Some(e) = outcome.fatal { + // A non-quorum store error is fatal (missing proofs were + // filtered out above, so this is a genuine network/store + // failure). Preserve every chunk stored so far — including this + // wave's same-pass successes — and report every not-stored chunk + // as failed, so the `PartialUpload` counts are accurate rather + // than omitting same-wave stores and under-counting failures. + warn!("merkle wave {wave_num}/{wave_count} aborted: {e}"); + // Best per-chunk messages we already have: missing-proof, this + // wave's shortfalls, and earlier waves' deferred shortfalls. + let mut known_failed = failed; + known_failed.extend(outcome.failed_addresses); + known_failed.extend(std::mem::take(&mut deferred)); + return Err(partial_upload_after_fatal( + addresses, + stored_addresses, + total_stored, + total_chunks, + known_failed, + PartialUploadSpend { + storage_cost_atto: batch_result.storage_cost_atto.clone(), + gas_cost_wei: batch_result.gas_cost_wei, + }, + format!("merkle chunk store aborted: {e}"), + )); + } + + // Non-fatal: this wave's quorum-short chunks are deferred (not failed + // yet) for the post-wave concurrent retry. A deferred chunk joins + // `stored_addresses` only if/when a later round stores it. + deferred.extend(outcome.failed_addresses); + if let Some(tx) = progress { let _ = tx .send(UploadEvent::WaveComplete { @@ -2141,11 +2516,87 @@ impl Client { } } + // The wave passes never blocked on backoff; now retry the whole + // file-level deferred set in concurrent rounds. Bodies are re-read from + // the spill at retry time (peak RAM unchanged) and proofs are re-attached + // by `store_one`. Chunks still short after the final round become + // `failed`; a non-quorum error aborts as `PartialUpload`. + if !deferred.is_empty() { + info!( + "Deferring {} merkle chunk(s) short of quorum for concurrent retry after final wave", + deferred.len() + ); + let dr = merkle_deferred_retry( + deferred, + &DEFERRED_ROUND_DELAYS_SECS, + // Read and store at most one wave's worth of bodies at a time so + // the deferred path keeps the wave path's ~256 MB peak-memory + // bound regardless of how many chunks were deferred file-wide. + UPLOAD_WAVE_SIZE, + |addrs: &[[u8; 32]]| { + spill.read_wave(addrs).map(|wave| { + wave.into_iter() + .map(|(content, addr)| (addr, content)) + .collect() + }) + }, + |n: usize| store_limiter.current().min(n.max(1)), + progress, + total_stored, + total_chunks, + &store_one, + ) + .await?; + + stored_addresses.extend(dr.stored_addresses); + total_stored = dr.stored; + + // Merge the deferred pass's stats — its histogram is already mapped + // to the right per-round slots — into the file aggregate. + agg_stats.chunk_attempts_total = agg_stats + .chunk_attempts_total + .saturating_add(dr.stats.chunk_attempts_total); + agg_stats + .store_durations_ms + .extend(dr.stats.store_durations_ms); + for (slot, count) in agg_stats + .retries_histogram + .iter_mut() + .zip(dr.stats.retries_histogram.iter()) + { + *slot = slot.saturating_add(*count); + } + + if let Some(reason) = dr.fatal { + // A non-quorum store error during a deferred round is fatal, the + // same as in the wave path: preserve everything stored so far and + // report every not-stored chunk as failed. + warn!("merkle deferred retry aborted: {reason}"); + let mut known_failed = failed; + known_failed.extend(dr.failed_addresses); + return Err(partial_upload_after_fatal( + addresses, + stored_addresses, + total_stored, + total_chunks, + known_failed, + PartialUploadSpend { + storage_cost_atto: batch_result.storage_cost_atto.clone(), + gas_cost_wei: batch_result.gas_cost_wei, + }, + format!("merkle chunk store aborted: {reason}"), + )); + } + failed.extend(dr.failed_addresses); + } + // A file with any permanently-failed chunk is not fully stored — surface - // it as `PartialUpload`, but only after retries across all waves are - // exhausted (never silently succeed with missing chunks). + // it as `PartialUpload`, but only after the single wave pass and every + // deferred retry round are exhausted (never silently succeed with + // missing chunks). if !failed.is_empty() { let failed_count = failed.len(); + let total_attempts = 1 + DEFERRED_ROUND_DELAYS_SECS.len(); warn!( "merkle upload incomplete: {failed_count}/{total_chunks} chunks short of quorum after retries" ); @@ -2155,8 +2606,12 @@ impl Client { failed, failed_count, total_chunks, + spend: Box::new(PartialUploadSpend { + storage_cost_atto: batch_result.storage_cost_atto.clone(), + gas_cost_wei: batch_result.gas_cost_wei, + }), reason: format!( - "{failed_count} chunk(s) short of quorum after {MERKLE_STORE_MAX_ATTEMPTS} attempts" + "{failed_count} chunk(s) short of quorum after {total_attempts} attempts" ), }); } @@ -2189,30 +2644,83 @@ impl Client { /// /// Returns an error if any chunk cannot be retrieved, decryption fails, /// or the file cannot be written. - #[allow(clippy::unused_async)] pub async fn file_download(&self, data_map: &DataMap, output: &Path) -> Result { self.file_download_with_progress(data_map, output, None) .await } - /// Download and decrypt a file with progress events. + /// Download and decrypt a file, trying the requested number of + /// closest peers for every chunk fetch. /// - /// Same as [`Client::file_download`] but sends [`DownloadEvent`]s for UI feedback. + /// Returns the number of bytes written. /// - /// Progress reporting: + /// # Errors + /// + /// Returns an error if any chunk cannot be retrieved, decryption fails, + /// or the file cannot be written. + pub async fn file_download_from_closest_peers( + &self, + data_map: &DataMap, + output: &Path, + peer_count: NonZeroUsize, + ) -> Result { + self.file_download_with_progress_using_peer_count(data_map, output, None, peer_count.get()) + .await + } + + /// Download and decrypt a file with progress events, trying the + /// requested number of closest peers for every chunk fetch. + /// + /// Same as [`Client::file_download_from_closest_peers`] but sends + /// [`DownloadEvent`]s for UI feedback. + /// + /// # Errors + /// + /// Returns an error if any chunk cannot be retrieved, decryption fails, + /// or the file cannot be written. + pub async fn file_download_with_progress_from_closest_peers( + &self, + data_map: &DataMap, + output: &Path, + progress: Option>, + peer_count: NonZeroUsize, + ) -> Result { + self.file_download_with_progress_using_peer_count( + data_map, + output, + progress, + peer_count.get(), + ) + .await + } + + /// Shared download core: resolve the DataMap, then fetch + streaming-decrypt + /// the file one batch at a time, handing each decrypted plaintext segment + /// (in order) to `on_chunk`. Constant memory — only one decrypt batch is + /// resident at a time. Returns the total plaintext bytes produced. + /// + /// `on_chunk` is async so a sink can apply backpressure (e.g. a bounded + /// channel). Driving the decrypt iterator runs the batched chunk fetch via + /// `block_in_place`, so this requires a multi-threaded Tokio runtime. + /// + /// Every chunk fetch tries `peer_count` closest peers. + /// + /// Progress reporting (via `progress`): /// 1. Resolves hierarchical DataMaps to the root level first (reports as /// `ChunksFetched` with `total: 0` during resolution) /// 2. Once the root DataMap is known, sends `total_chunks` with accurate count /// 3. Fetches data chunks with accurate `fetched/total` progress - #[allow(clippy::unused_async)] - pub async fn file_download_with_progress( + async fn download_decrypted_chunks( &self, data_map: &DataMap, - output: &Path, progress: Option>, - ) -> Result { - debug!("Downloading file to {}", output.display()); - + peer_count: usize, + mut on_chunk: F, + ) -> Result + where + F: FnMut(Bytes) -> Fut, + Fut: std::future::Future>, + { let handle = Handle::current(); // Phase 1: Resolve hierarchical DataMap to root level. @@ -2260,7 +2768,7 @@ impl Client { // load-shedding signal for // sustained close-group exhaustion). let chunk = self - .chunk_get_observed(&addr) + .chunk_get_observed_from_closest_peers(&addr, peer_count) .await .map_err(|e| { self_encryption::Error::Generic(format!( @@ -2372,7 +2880,10 @@ impl Client { async move { let addr = hash.0; let addr_hex = hex::encode(addr); - match self.chunk_get_observed(&addr).await { + match self + .chunk_get_observed_from_closest_peers(&addr, peer_count) + .await + { Ok(Some(chunk)) => { let fetched = fetched_ref.fetch_add( 1, @@ -2484,7 +2995,12 @@ impl Client { // next round rather than // aborting; only the final // round's leftovers are fatal. - match self.chunk_get_observed(&addr).await { + match self + .chunk_get_observed_from_closest_peers( + &addr, peer_count, + ) + .await + { Ok(Some(chunk)) => { let fetched = fetched_ref.fetch_add( 1, @@ -2550,53 +3066,118 @@ impl Client { ) .map_err(|e| Error::Encryption(format!("streaming decrypt failed: {e}")))?; - // Write decrypted chunks to a temp file, then rename atomically. + // Drive the iterator (each `next()` runs the batched fetch via + // block_in_place) and hand each decrypted segment to the sink in + // order. Awaiting the sink between items yields back to the runtime so + // a bounded sink can apply backpressure. + let mut bytes_total = 0u64; + for chunk_result in stream { + let chunk: Bytes = + chunk_result.map_err(|e| Error::Encryption(format!("decryption failed: {e}")))?; + bytes_total += chunk.len() as u64; + on_chunk(chunk).await?; + } + Ok(bytes_total) + } + + /// Download and decrypt a file to disk, with optional progress events. + /// + /// Same as [`Client::file_download`] but sends [`DownloadEvent`]s for UI + /// feedback. Streams to a temp file (one decrypt batch resident at a time) + /// and renames atomically on success. A `TempDownload` guard removes the + /// staging file on any error path, including a panic. + pub async fn file_download_with_progress( + &self, + data_map: &DataMap, + output: &Path, + progress: Option>, + ) -> Result { + self.file_download_with_progress_using_peer_count( + data_map, + output, + progress, + self.config().close_group_size, + ) + .await + } + + /// Download and decrypt a file to disk with progress events, trying + /// `peer_count` closest peers for every chunk fetch. + /// + /// Streams to a temp file (one decrypt batch resident at a time) and + /// renames atomically on success. + async fn file_download_with_progress_using_peer_count( + &self, + data_map: &DataMap, + output: &Path, + progress: Option>, + peer_count: usize, + ) -> Result { + debug!("Downloading file to {}", output.display()); + let parent = output.parent().unwrap_or_else(|| Path::new(".")); let unique: u64 = rand::random(); let tmp_path = parent.join(format!(".ant_download_{}_{unique}.tmp", std::process::id())); - let write_result = (|| -> Result { - let mut file = std::fs::File::create(&tmp_path)?; - let mut bytes_written = 0u64; - for chunk_result in stream { - let chunk_bytes = chunk_result - .map_err(|e| Error::Encryption(format!("decryption failed: {e}")))?; - file.write_all(&chunk_bytes)?; - bytes_written += chunk_bytes.len() as u64; - } - file.flush()?; - Ok(bytes_written) - })(); + // Guard removes the staging file on any early return OR a panic unwind + // out of the `block_in_place` decrypt loop; defused only by a + // successful commit(). Centralizes what used to be three duplicated + // cleanup arms. + let tmp = TempDownload::new(tmp_path); + let mut file = std::fs::File::create(tmp.path())?; + + let bytes_written = self + .download_decrypted_chunks(data_map, progress, peer_count, |bytes| { + let r = file.write_all(&bytes).map_err(Error::from); + std::future::ready(r) + }) + .await?; + file.flush()?; + drop(file); // close the handle before rename (Windows won't rename an open file) - match write_result { - Ok(bytes_written) => match std::fs::rename(&tmp_path, output) { - Ok(()) => { - info!( - "File downloaded: {bytes_written} bytes written to {}", - output.display() - ); - Ok(bytes_written) - } - Err(rename_err) => { - if let Err(cleanup_err) = std::fs::remove_file(&tmp_path) { - warn!( - "Failed to remove temp download file {}: {cleanup_err}", - tmp_path.display() - ); - } - Err(rename_err.into()) - } - }, - Err(e) => { - if let Err(cleanup_err) = std::fs::remove_file(&tmp_path) { - warn!( - "Failed to remove temp download file {}: {cleanup_err}", - tmp_path.display() - ); - } - Err(e) + tmp.commit(output)?; + info!( + "File downloaded: {bytes_written} bytes written to {}", + output.display() + ); + Ok(bytes_written) + } + + /// Download and decrypt a file, streaming the plaintext to `sink` instead + /// of writing to disk. + /// + /// Constant memory (one decrypt batch resident at a time); the caller + /// receives bytes progressively as each batch decrypts, suitable for + /// forwarding to an HTTP chunked body or a gRPC response stream. The + /// bounded `sink` applies backpressure. If the receiver is dropped (e.g. + /// the client disconnected) the download stops early and returns + /// [`Error::Cancelled`]. + /// + /// The channel item type is `Result`, so the caller sets up: + /// + /// ```ignore + /// let (tx, rx) = tokio::sync::mpsc::channel::>(8); + /// ``` + /// + /// Typically the caller `tokio::spawn`s this and converts the matching + /// `Receiver` into its response stream. Requires a multi-threaded Tokio + /// runtime (the decrypt iterator uses `block_in_place`). + pub async fn file_download_to_sender( + &self, + data_map: &DataMap, + sink: mpsc::Sender>, + progress: Option>, + ) -> Result { + let peer_count = self.config().close_group_size; + self.download_decrypted_chunks(data_map, progress, peer_count, |bytes| { + let sink = sink.clone(); + async move { + sink.send(Ok(bytes)) + .await + .map_err(|_| Error::Cancelled("download stream receiver dropped".into())) } - } + }) + .await } } @@ -2605,6 +3186,33 @@ impl Client { mod tests { use super::*; + #[test] + fn distributed_sample_indices_spreads_across_large_file() { + // cap 5 over 100 chunks: first and last included, evenly spread. + assert_eq!(distributed_sample_indices(100, 5), vec![0, 24, 49, 74, 99]); + } + + #[test] + fn distributed_sample_indices_covers_whole_small_file() { + // total <= cap returns every index, preserving the exact + // "whole file sampled" detection in estimate_upload_cost. + assert_eq!(distributed_sample_indices(3, 5), vec![0, 1, 2]); + assert_eq!(distributed_sample_indices(5, 5), vec![0, 1, 2, 3, 4]); + } + + #[test] + fn distributed_sample_indices_is_in_range_and_increasing() { + assert!(distributed_sample_indices(0, 5).is_empty()); + assert_eq!(distributed_sample_indices(1, 5), vec![0]); + for total in 1..200usize { + let idx = distributed_sample_indices(total, 5); + assert_eq!(*idx.first().unwrap(), 0); + assert_eq!(*idx.last().unwrap(), total - 1); + assert!(idx.iter().all(|&i| i < total)); + assert!(idx.windows(2).all(|w| w[0] < w[1])); + } + } + #[test] fn disk_space_check_passes_for_small_file() { // A 1 KB file should always pass the disk space check @@ -2712,6 +3320,68 @@ mod tests { assert_eq!(missing, vec![unpaid_b, unpaid_d]); } + /// A wave that returns `Ok` contributes its stored chunks, parsed cost, and + /// stats; nothing is recorded as failed. + #[test] + fn fold_single_wave_keeps_ok_wave() { + let stored = vec![[1u8; 32], [2u8; 32]]; + let stats = WaveAggregateStats { + chunk_attempts_total: 7, + ..Default::default() + }; + + let outcome = fold_single_wave(Ok((stored.clone(), "100".to_string(), 9, stats))).unwrap(); + + assert_eq!(outcome.stored, stored); + assert!(outcome.failed.is_empty()); + assert_eq!(outcome.storage_atto.to_string(), "100"); + assert_eq!(outcome.gas_wei, 9); + assert_eq!(outcome.stats.chunk_attempts_total, 7); + } + + /// The core V2-461 semantic: a wave short of quorum (`PartialUpload`) is + /// recoverable — its stored chunks, failed chunks, and on-chain spend are + /// folded so the caller can continue to the next wave rather than aborting + /// the whole file. + #[test] + fn fold_single_wave_folds_partial_upload() { + let stored = vec![[3u8; 32]]; + let failed = vec![([4u8; 32], "short of quorum".to_string())]; + let err = Error::PartialUpload { + stored: stored.clone(), + stored_count: 1, + failed: failed.clone(), + failed_count: 1, + total_chunks: 2, + spend: Box::new(PartialUploadSpend { + storage_cost_atto: "250".to_string(), + gas_cost_wei: 11, + }), + reason: "wave store failed after retries".to_string(), + }; + + let outcome = fold_single_wave(Err(err)).unwrap(); + + assert_eq!(outcome.stored, stored); + assert_eq!(outcome.failed, failed); + assert_eq!(outcome.storage_atto.to_string(), "250"); + assert_eq!(outcome.gas_wei, 11); + // `PartialUpload` carries no stats, so the failed wave contributes none. + assert_eq!(outcome.stats.chunk_attempts_total, 0); + } + + /// A non-`PartialUpload` error (wallet/payment-infrastructure failure) is + /// fatal and must abort the file, not be folded into the failed set. + #[test] + fn fold_single_wave_propagates_fatal_error() { + let result = fold_single_wave(Err(Error::Payment("wallet unavailable".to_string()))); + + assert!( + matches!(result, Err(Error::Payment(_))), + "fatal payment error must propagate, got: {result:?}" + ); + } + #[test] fn partition_addresses_by_proof_handles_all_or_nothing() { let a = [5u8; 32]; diff --git a/ant-core/src/data/client/merkle.rs b/ant-core/src/data/client/merkle.rs index 532d37c5..450537e0 100644 --- a/ant-core/src/data/client/merkle.rs +++ b/ant-core/src/data/client/merkle.rs @@ -390,7 +390,9 @@ impl Client { data_type: u32, data_size: u64, ) -> Result { - let result = self.get_store_quotes(address, data_size, data_type).await; + let result = self + .get_store_quotes_with_fault_tolerance(address, data_size, data_type) + .await; if let Err(e) = &result { if matches!(classify_error(e), Outcome::Timeout | Outcome::NetworkError) { debug!( @@ -863,7 +865,7 @@ impl Client { } }; - merkle_store_with_retry( + let outcome = merkle_store_with_retry( chunks, store_concurrency, MERKLE_STORE_MAX_ATTEMPTS, @@ -873,7 +875,17 @@ impl Client { total_chunks, store_one, ) - .await + .await?; + + // The external-signer path treats a non-quorum error as terminal (it + // returns a single all-or-nothing `FileUploadResult`), so re-raise the + // fatal that `merkle_store_with_retry` now carries in the outcome. The + // CLI/spill paths, which can surface `PartialUpload`, read `fatal` + // directly instead. + if let Some(e) = outcome.fatal { + return Err(e); + } + Ok(outcome) } } @@ -908,6 +920,13 @@ pub(crate) struct MerkleStoreOutcome { /// Chunks that reached quorum, including any `stored_offset` carried in /// from a preflight (counted once, even if they needed retries). pub stored: usize, + /// Addresses confirmed stored by this call (excludes the `stored_offset` + /// preflight carry-in — those have no address here). The caller appends + /// these to the file's stored set; using the explicit set (rather than + /// inferring "input minus failed") keeps accounting correct even when a + /// `fatal` error aborts the pass mid-flight, leaving some input chunks + /// neither stored nor in `failed_addresses`. + pub stored_addresses: Vec<[u8; 32]>, /// Chunks still short of quorum after [`MERKLE_STORE_MAX_ATTEMPTS`]. pub failed: usize, /// Addresses (and the last error message) of chunks still short of quorum @@ -915,6 +934,13 @@ pub(crate) struct MerkleStoreOutcome { /// build [`crate::data::Error::PartialUpload`]; the external-signer path /// only reads the counts. pub failed_addresses: Vec<([u8; 32], String)>, + /// Set when a non-quorum (fatal) store error aborted the pass. Successes + /// completed before the abort are still recorded in `stored`/ + /// `stored_addresses`; the chunks that had already failed quorum are in + /// `failed_addresses`; chunks still in flight when the abort hit are in + /// neither (the caller treats input-minus-stored as failed). Callers that + /// want the old "fatal aborts everything" contract re-raise this as `Err`. + pub fatal: Option, /// Aggregate store stats (durations, attempts, per-round retry histogram). pub stats: crate::data::client::batch::WaveAggregateStats, } @@ -929,7 +955,14 @@ pub(crate) struct MerkleStoreOutcome { /// chunk's success is counted once and recorded in the retry round it landed on /// (`retries_histogram[round]`). `stored_offset` seeds the returned `stored` /// count and the progress numbering; `total` is the whole-file total reported -/// in progress events. Non-quorum errors abort immediately. +/// in progress events. +/// +/// A non-quorum error stops the pass but does **not** discard progress: the +/// successes already completed this pass stay in `stored`/`stored_addresses`, +/// the quorum shortfalls so far stay in `failed_addresses`, and the error is +/// returned in [`MerkleStoreOutcome::fatal`] (as `Ok(outcome)`, not `Err`). +/// Callers that want the old abort-everything behaviour re-raise `fatal` as +/// `Err`; CLI callers fold it into `PartialUpload` while keeping the stores. #[allow(clippy::too_many_arguments)] pub(crate) async fn merkle_store_with_retry( chunks: Vec<([u8; 32], Bytes)>, @@ -977,6 +1010,7 @@ where outcome.stats.retries_histogram[idx] = outcome.stats.retries_histogram[idx].saturating_add(1); outcome.stored += 1; + outcome.stored_addresses.push(addr); if let Some(tx) = progress { let _ = tx.try_send(UploadEvent::ChunkStored { stored: outcome.stored, @@ -984,13 +1018,38 @@ where }); } } - Err(e @ Error::InsufficientPeers(_)) => { + // A quorum shortfall — whether reported as a transport + // shortfall (`InsufficientPeers`) or an app-only rejection + // (`RemotePut`, e.g. pool-rejected / quote-stale / disk-full, + // which are transient) — is recoverable: defer and retry the + // chunk rather than aborting the whole upload (V2-468). + Err(e @ (Error::InsufficientPeers(_) | Error::RemotePut { .. })) => { + next_failed.push((addr, content, e.to_string())); + } + Err(e) => { + // Non-quorum error: fatal. Stop consuming the stream but do + // NOT discard the outcome — successes already completed this + // pass stay recorded in `stored`/`stored_addresses`. Record + // the fatal chunk itself (and any quorum shortfalls seen so + // far) as failed; anything still in flight is left for the + // caller to treat as not-stored (input minus + // `stored_addresses`). next_failed.push((addr, content, e.to_string())); + outcome.fatal = Some(e); + break; } - Err(e) => return Err(e), } } + if outcome.fatal.is_some() { + outcome.failed = next_failed.len(); + outcome.failed_addresses = next_failed + .into_iter() + .map(|(addr, _content, msg)| (addr, msg)) + .collect(); + return Ok(outcome); + } + if next_failed.is_empty() { break; } @@ -1030,6 +1089,184 @@ where Ok(outcome) } +/// Round delays (seconds) for the merkle upload deferred-retry pass. Round 0 +/// fires immediately — most quorum shortfalls on a healthy network are +/// momentary close-group divergence that clears in well under a second, and +/// serializing them behind mandatory sleeps was the single biggest throughput +/// sink in the wave path (one bad chunk parked the other 63 slots for minutes). +/// Only chunks that survive a round get a longer back-off before the next, so a +/// genuinely saturated/diverged group still gets time to settle. Mirrors the +/// download path's `DEFERRED_ROUND_DELAYS_SECS`. +pub(crate) const DEFERRED_ROUND_DELAYS_SECS: [u64; 3] = [0, 15, 45]; + +/// Histogram slot for a deferred-retry round's successes. +/// +/// The wave first pass lands in slot 0; deferred round `r` (0-indexed) lands in +/// slot `r + 1`, clamped to the last slot so the four-slot +/// [`WaveAggregateStats::retries_histogram`] keeps recording "which round a +/// chunk landed on" under the post-wave deferred structure. +pub(crate) fn deferred_round_histogram_slot(round: usize, hist_len: usize) -> usize { + (round + 1).min(hist_len.saturating_sub(1)) +} + +/// Outcome of the post-wave deferred-retry pass. +#[derive(Debug, Default)] +pub(crate) struct DeferredRetryOutcome { + /// Running total of stored chunks, seeded with the `stored_offset` passed in + /// (i.e. everything the wave passes already stored) and advanced by each + /// deferred round's successes. + pub stored: usize, + /// Addresses that reached quorum during the deferred rounds (to be appended + /// to the file's `stored` set). + pub stored_addresses: Vec<[u8; 32]>, + /// Count of chunks still short of quorum after the final deferred round. + pub failed: usize, + /// Addresses (and last quorum-shortfall message) still short after the final + /// round, or — when `fatal` is set — the chunks that were still pending when + /// a non-quorum error aborted the pass. + pub failed_addresses: Vec<([u8; 32], String)>, + /// Set when a deferred round hit a non-quorum (fatal) store error. The + /// caller surfaces this as `PartialUpload` preserving everything stored so + /// far, mirroring the wave path's fatal handling. + pub fatal: Option, + /// Aggregate store stats merged across rounds, with each round's successes + /// already mapped into its [`deferred_round_histogram_slot`]. + pub stats: crate::data::client::batch::WaveAggregateStats, +} + +/// Retry a file-level set of quorum-short merkle chunks in concurrent rounds. +/// +/// This is the upload analogue of the download path's deferred-retry loop. The +/// wave passes store each wave in a single pass (no in-wave backoff barrier) and +/// hand their quorum-short chunks here. Each round processes the still-pending +/// chunks in **bounded batches of `batch_size`**: it re-reads only one batch of +/// bodies at a time via `read_bodies` (from the spill file), so peak resident +/// memory stays at the wave path's `batch_size × MAX_CHUNK_SIZE` bound rather +/// than scaling with the whole file's deferred-chunk count. Each batch is stored +/// concurrently at `concurrency_for(len)` via the single-pass +/// [`merkle_store_with_retry`] primitive, and survivors carry to the next round +/// after a `round_delays_secs` sleep. Chunks still short after the final round +/// become `failed_addresses`; a non-quorum store error stops the pass and is +/// reported via `fatal` (with every not-yet-stored chunk recorded as +/// `failed_addresses`) so the caller can surface `PartialUpload` without +/// discarding earlier progress. +/// +/// `store_one`, `progress`, `stored_offset` and `total` mirror +/// [`merkle_store_with_retry`]. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn merkle_deferred_retry( + deferred: Vec<([u8; 32], String)>, + round_delays_secs: &[u64], + batch_size: usize, + read_bodies: RB, + concurrency_for: CF, + progress: Option<&mpsc::Sender>, + stored_offset: usize, + total: usize, + store_one: SF, +) -> Result +where + RB: Fn(&[[u8; 32]]) -> Result>, + CF: Fn(usize) -> usize, + SF: Fn([u8; 32], Bytes) -> Fut, + Fut: std::future::Future>, +{ + let batch_size = batch_size.max(1); + let mut outcome = DeferredRetryOutcome { + stored: stored_offset, + ..DeferredRetryOutcome::default() + }; + let mut remaining = deferred; + let rounds = round_delays_secs.len(); + + for (round, &delay_secs) in round_delays_secs.iter().enumerate() { + if remaining.is_empty() { + break; + } + if delay_secs > 0 { + tokio::time::sleep(Duration::from_secs(delay_secs)).await; + } + info!( + "Deferred merkle retry round {}/{}: {} chunk(s) short of quorum", + round + 1, + rounds, + remaining.len(), + ); + + // Drain this round's input; survivors accumulate back into `remaining` + // for the next round. A single-pass batch records its successes in + // histogram slot 0, so all of this round's successes redirect to one + // slot. + let slot = deferred_round_histogram_slot(round, outcome.stats.retries_histogram.len()); + let round_input = std::mem::take(&mut remaining); + let mut input_iter = round_input.into_iter(); + + loop { + let batch: Vec<([u8; 32], String)> = input_iter.by_ref().take(batch_size).collect(); + if batch.is_empty() { + break; + } + let batch_addrs: Vec<[u8; 32]> = batch.iter().map(|(addr, _)| *addr).collect(); + // Re-read only this batch's bodies from the spill (≤ batch_size + // resident at a time), so the deferred path keeps the wave path's + // memory bound regardless of how many chunks were deferred. + let chunks = read_bodies(&batch_addrs)?; + let concurrency = concurrency_for(batch_addrs.len()); + + let batch_outcome = merkle_store_with_retry( + chunks, + concurrency, + 1, + Duration::ZERO, + progress, + outcome.stored, + total, + &store_one, + ) + .await?; + + outcome.stored = batch_outcome.stored; + outcome + .stored_addresses + .extend(batch_outcome.stored_addresses); + + // Merge stats, redirecting this round's successes to its slot. + outcome.stats.chunk_attempts_total = outcome + .stats + .chunk_attempts_total + .saturating_add(batch_outcome.stats.chunk_attempts_total); + outcome + .stats + .store_durations_ms + .extend(batch_outcome.stats.store_durations_ms); + let landed: usize = batch_outcome.stats.retries_histogram.iter().sum(); + outcome.stats.retries_histogram[slot] = + outcome.stats.retries_histogram[slot].saturating_add(landed); + + if let Some(fatal) = batch_outcome.fatal { + // Fatal mid-pass: confirmed stores are preserved above. Report + // everything not stored as failed — this batch's quorum + // shortfalls, the remaining unprocessed batches in this round, + // and any survivors already carried from earlier batches. + outcome.fatal = Some(fatal.to_string()); + let mut failed = batch_outcome.failed_addresses; + failed.extend(input_iter); + failed.extend(std::mem::take(&mut remaining)); + outcome.failed = failed.len(); + outcome.failed_addresses = failed; + return Ok(outcome); + } + + // Quorum-short chunks from this batch survive to the next round. + remaining.extend(batch_outcome.failed_addresses); + } + } + + outcome.failed = remaining.len(); + outcome.failed_addresses = remaining; + Ok(outcome) +} + /// Phase 2 of external-signer merkle payment: generate proofs from winner. /// /// Takes the prepared batch and the winner pool hash returned by the @@ -1572,17 +1809,83 @@ mod tests { assert_eq!(outcome.stats.chunk_attempts_total, 6); } - /// A non-quorum error (e.g. a missing proof) stays fatal and aborts. + /// V2-468: an app-only quorum shortfall surfaces as `Error::RemotePut` + /// (pool-rejected / quote-stale / disk-full — transient), which must be + /// treated as recoverable just like `InsufficientPeers`: collected and + /// retried, never aborting the whole batch. + #[tokio::test] + async fn store_with_retry_treats_remote_put_as_recoverable() { + let chunks = make_chunks(6); + let failing: std::collections::HashSet<[u8; 32]> = + chunks.iter().take(2).map(|(a, _)| *a).collect(); + let failing_for_closure = failing.clone(); + + let store_one = move |addr: [u8; 32], _content: Bytes| { + let fail = failing_for_closure.contains(&addr); + async move { + if fail { + Err(Error::RemotePut { + address: hex::encode(addr), + source: ant_protocol::ProtocolError::StorageFailed( + "insufficient disk space".into(), + ), + }) + } else { + Ok(std::time::Instant::now()) + } + } + }; + + let outcome = merkle_store_with_retry(chunks, 8, 1, Duration::ZERO, None, 0, 6, store_one) + .await + .expect("remote app-rejections must not abort the batch"); + + assert_eq!(outcome.stored, 4); + assert_eq!(outcome.failed, 2); + } + + /// A non-quorum error (e.g. a missing proof) is captured in `fatal` rather + /// than discarded — the call returns `Ok(outcome)` so the caller can decide + /// whether to re-raise it or fold it into `PartialUpload`. #[tokio::test] - async fn store_with_retry_propagates_non_quorum_errors() { + async fn store_with_retry_reports_non_quorum_errors_as_fatal() { let chunks = make_chunks(3); let store_one = |_addr: [u8; 32], _content: Bytes| async move { Err::(Error::Payment("missing proof".into())) }; - let result = - merkle_store_with_retry(chunks, 8, 3, Duration::ZERO, None, 0, 3, store_one).await; - assert!(matches!(result, Err(Error::Payment(_)))); + let outcome = merkle_store_with_retry(chunks, 8, 3, Duration::ZERO, None, 0, 3, store_one) + .await + .expect("fatal is carried in the outcome, not returned as Err"); + assert!(matches!(outcome.fatal, Some(Error::Payment(_)))); + } + + /// A fatal error mid-pass preserves the successes that already completed in + /// the same pass — they are not discarded with the abort. Concurrency 1 + /// makes ordering deterministic: the first five chunks store, then the sixth + /// aborts fatally. + #[tokio::test] + async fn store_with_retry_fatal_preserves_same_pass_successes() { + let chunks = make_chunks(6); + let bad = chunks[5].0; + let store_one = move |addr: [u8; 32], _content: Bytes| async move { + if addr == bad { + Err(Error::Payment("fatal".into())) + } else { + Ok(std::time::Instant::now()) + } + }; + + let outcome = merkle_store_with_retry(chunks, 1, 1, Duration::ZERO, None, 0, 6, store_one) + .await + .expect("fatal carried in outcome, not returned as Err"); + assert!(matches!(outcome.fatal, Some(Error::Payment(_)))); + // The five chunks stored before the abort are preserved, not lost. + assert_eq!(outcome.stored, 5); + assert_eq!(outcome.stored_addresses.len(), 5); + assert!(!outcome.stored_addresses.contains(&bad)); + // The fatal chunk is reported as failed (not silently dropped). + assert!(outcome.failed_addresses.iter().any(|(a, _)| *a == bad)); } /// C2.2: only the chunks that failed the previous round are retried. @@ -1785,4 +2088,251 @@ mod tests { assert_eq!(outcome.failed, 0); assert!(outcome.failed_addresses.is_empty()); } + + // ========================================================================= + // merkle_deferred_retry: download-style concurrent post-wave retry (V2-466) + // ========================================================================= + + /// The histogram slot mapping: the wave first pass is slot 0; deferred + /// round `r` is slot `r + 1`, clamped to the last slot. + #[test] + fn deferred_round_histogram_slot_maps_and_clamps() { + assert_eq!(deferred_round_histogram_slot(0, 4), 1); + assert_eq!(deferred_round_histogram_slot(1, 4), 2); + assert_eq!(deferred_round_histogram_slot(2, 4), 3); + // Beyond the histogram width, clamp to the final slot. + assert_eq!(deferred_round_histogram_slot(3, 4), 3); + assert_eq!(deferred_round_histogram_slot(9, 4), 3); + } + + /// Re-read bodies for a deferred set from a fake "spill": every requested + /// address is returned paired with a stub body. Zero delays so tests do not + /// actually sleep between rounds. + fn fake_read_bodies(addrs: &[[u8; 32]]) -> Result> { + Ok(addrs + .iter() + .map(|a| (*a, Bytes::from_static(b"deferred-body"))) + .collect()) + } + + fn deferred_set(count: usize) -> Vec<([u8; 32], String)> { + make_test_addresses(count) + .into_iter() + .map(|addr| (addr, "short of quorum".to_string())) + .collect() + } + + /// A chunk that is quorum-short on early rounds but succeeds on a later + /// round is stored exactly once, recorded in that round's histogram slot, + /// and reported with no failures. + #[tokio::test] + async fn deferred_retry_succeeds_on_a_later_round() { + let deferred = deferred_set(3); + // Each chunk fails its first attempt (round 0) and succeeds the second + // (round 1 → histogram slot 2). + let attempts = Arc::new(Mutex::new(HashMap::<[u8; 32], usize>::new())); + let attempts_for_closure = attempts.clone(); + let store_one = move |addr: [u8; 32], _content: Bytes| { + let attempts = attempts_for_closure.clone(); + async move { + let n = { + let mut map = attempts.lock().unwrap(); + let e = map.entry(addr).or_insert(0); + *e += 1; + *e + }; + if n < 2 { + Err(Error::InsufficientPeers("still short".into())) + } else { + Ok(std::time::Instant::now()) + } + } + }; + + let outcome = merkle_deferred_retry( + deferred, + &[0, 0, 0], + 64, + fake_read_bodies, + |n: usize| n.max(1), + None, + 0, + 3, + store_one, + ) + .await + .expect("deferred retry must not abort on quorum shortfalls"); + + assert_eq!(outcome.stored, 3, "all three land by round 1"); + assert_eq!(outcome.stored_addresses.len(), 3); + assert_eq!(outcome.failed, 0); + assert!(outcome.failed_addresses.is_empty()); + assert!(outcome.fatal.is_none()); + // Round 1 → slot 2; round 0 (slot 1) saw zero successes. + assert_eq!(outcome.stats.retries_histogram[1], 0); + assert_eq!(outcome.stats.retries_histogram[2], 3); + // Each chunk attempted twice: one failed round + one success round. + assert_eq!(outcome.stats.chunk_attempts_total, 6); + } + + /// Chunks still short of quorum after the final deferred round become + /// `failed`, not silently dropped, and no fatal error is set. + #[tokio::test] + async fn deferred_retry_leftovers_become_failed() { + let deferred = deferred_set(2); + let store_one = |_addr: [u8; 32], _content: Bytes| async move { + Err::(Error::InsufficientPeers("always short".into())) + }; + + let outcome = merkle_deferred_retry( + deferred, + &[0, 0, 0], + 64, + fake_read_bodies, + |n: usize| n.max(1), + None, + 0, + 2, + store_one, + ) + .await + .expect("exhausted retries report failures, not an error"); + + assert_eq!(outcome.stored, 0); + assert!(outcome.stored_addresses.is_empty()); + assert_eq!(outcome.failed, 2); + assert_eq!(outcome.failed_addresses.len(), 2); + assert!(outcome.fatal.is_none()); + // Three rounds × two chunks, all failing. + assert_eq!(outcome.stats.chunk_attempts_total, 6); + } + + /// A non-quorum (fatal) error during a deferred round stops the pass, is + /// surfaced via `fatal`, and preserves an earlier round's success in + /// `stored`/`stored_addresses` while the still-pending chunk is reported as + /// failed. + #[tokio::test] + async fn deferred_retry_fatal_error_preserves_prior_progress() { + let addrs = make_test_addresses(2); + let good = addrs[0]; + let bad = addrs[1]; + let deferred = vec![(good, "short".to_string()), (bad, "short".to_string())]; + + // `good` succeeds on round 0; `bad` is quorum-short on round 0, then + // hits a fatal Payment error on round 1. + let attempts = Arc::new(Mutex::new(HashMap::<[u8; 32], usize>::new())); + let attempts_for_closure = attempts.clone(); + let store_one = move |addr: [u8; 32], _content: Bytes| { + let attempts = attempts_for_closure.clone(); + async move { + let n = { + let mut map = attempts.lock().unwrap(); + let e = map.entry(addr).or_insert(0); + *e += 1; + *e + }; + if addr == good { + Ok(std::time::Instant::now()) + } else if n == 1 { + Err(Error::InsufficientPeers("short".into())) + } else { + Err(Error::Payment("fatal on retry".into())) + } + } + }; + + let outcome = merkle_deferred_retry( + deferred, + &[0, 0, 0], + 64, + fake_read_bodies, + |n: usize| n.max(1), + None, + 0, + 2, + store_one, + ) + .await + .expect("a fatal round error is reported via `fatal`, not as Err"); + + assert!(outcome.fatal.is_some(), "fatal error must be captured"); + assert_eq!(outcome.stored, 1, "round-0 success preserved"); + assert_eq!(outcome.stored_addresses, vec![good]); + assert_eq!(outcome.failed, 1); + assert_eq!(outcome.failed_addresses.len(), 1); + assert_eq!(outcome.failed_addresses[0].0, bad); + } + + /// An empty deferred set is a no-op: no rounds run, nothing stored or failed. + #[tokio::test] + async fn deferred_retry_empty_set_is_a_noop() { + let store_one = |_addr: [u8; 32], _content: Bytes| async move { + Err::(Error::InsufficientPeers("unused".into())) + }; + + let outcome = merkle_deferred_retry( + Vec::new(), + &DEFERRED_ROUND_DELAYS_SECS, + 64, + fake_read_bodies, + |n: usize| n.max(1), + None, + 7, + 7, + store_one, + ) + .await + .expect("empty deferred set is a no-op"); + + assert_eq!(outcome.stored, 7, "stored_offset carried through unchanged"); + assert_eq!(outcome.failed, 0); + assert!(outcome.stored_addresses.is_empty()); + assert!(outcome.failed_addresses.is_empty()); + assert!(outcome.fatal.is_none()); + } + + /// The memory-bound guard (V2-466 review finding 1): a deferred set far + /// larger than `batch_size` is read from the spill in batches of at most + /// `batch_size`, so peak resident bodies never scale with the file-wide + /// deferred count. All chunks still store. + #[tokio::test] + async fn deferred_retry_reads_bodies_in_bounded_batches() { + let deferred = deferred_set(10); + let batch_size = 4; + // Record the largest single read_bodies request. + let max_batch = Arc::new(Mutex::new(0usize)); + let max_batch_for_closure = max_batch.clone(); + let read_bodies = move |addrs: &[[u8; 32]]| { + let mut m = max_batch_for_closure.lock().unwrap(); + *m = (*m).max(addrs.len()); + Ok(addrs + .iter() + .map(|a| (*a, Bytes::from_static(b"body"))) + .collect()) + }; + let store_one = + |_addr: [u8; 32], _content: Bytes| async move { Ok(std::time::Instant::now()) }; + + let outcome = merkle_deferred_retry( + deferred, + &[0, 0, 0], + batch_size, + read_bodies, + |n: usize| n.max(1), + None, + 0, + 10, + store_one, + ) + .await + .expect("bounded-batch deferred retry stores everything"); + + assert_eq!(outcome.stored, 10); + assert_eq!(outcome.stored_addresses.len(), 10); + assert_eq!(outcome.failed, 0); + assert!( + *max_batch.lock().unwrap() <= batch_size, + "read_bodies must never be handed more than batch_size addresses at once" + ); + } } diff --git a/ant-core/src/data/client/mod.rs b/ant-core/src/data/client/mod.rs index 2775ad83..807e28b5 100644 --- a/ant-core/src/data/client/mod.rs +++ b/ant-core/src/data/client/mod.rs @@ -13,13 +13,13 @@ pub mod data; pub mod file; pub mod merkle; pub mod payment; -pub(crate) mod peer_cache; pub mod quote; use crate::data::client::adaptive::{AdaptiveConfig, AdaptiveController, ChannelStart, Outcome}; use crate::data::client::cache::ChunkCache; use crate::data::error::{Error, Result}; use crate::data::network::Network; +use crate::data::peer_cache; use ant_protocol::evm::Wallet; use ant_protocol::transport::{MultiAddr, P2PNode, PeerId}; use ant_protocol::{XorName, CLOSE_GROUP_SIZE}; @@ -47,8 +47,13 @@ use tracing::debug; /// chunks could not be stored) /// - `AlreadyStored`, `Encryption`, `Crypto`, `Payment`, /// `Serialization`, `InvalidData`, `SignatureVerification`, -/// `Config`, `InsufficientDiskSpace`, `CostEstimationInconclusive` -/// -> `ApplicationError` (would happen on a perfectly healthy link) +/// `Config`, `InsufficientDiskSpace`, `CostEstimationInconclusive`, +/// `Cancelled` -> `ApplicationError` (would happen on a perfectly +/// healthy link; `Cancelled` is caller-initiated and must not be retried +/// as a transport failure) +/// - `RemotePut` -> `ApplicationError` (the remote node responded with a +/// structured rejection — the transport succeeded, so the node declined +/// at the application layer; not a local capacity signal) pub(crate) fn classify_error(err: &Error) -> Outcome { match err { Error::Timeout(_) => Outcome::Timeout, @@ -68,7 +73,13 @@ pub(crate) fn classify_error(err: &Error) -> Outcome { | Error::Config(_) | Error::InsufficientDiskSpace(_) | Error::CostEstimationInconclusive(_) - | Error::BadQuoteBinding { .. } => Outcome::ApplicationError, + | Error::Cancelled(_) + | Error::BadQuoteBinding { .. } + // A remote node responded with a structured rejection — the + // transport round-trip succeeded, so the node declined at the + // application layer (payment/disk/quote/pool). Not a local + // capacity signal; recorded but must not push the limiter down. + | Error::RemotePut { .. } => Outcome::ApplicationError, } } @@ -325,12 +336,25 @@ pub struct Client { /// Path the controller persists its snapshot to. `None` disables /// persistence (useful for tests / non-disk environments). persist_path: Option, + /// Path for the persistent client peer cache. `None` disables the cache. + peer_cache_path: Option, } impl Client { /// Create a client connected to the given P2P node. #[must_use] pub fn from_node(node: Arc, config: ClientConfig) -> Self { + Self::from_node_with_peer_cache(node, config, None) + } + + /// Create a client connected to the given P2P node and attach an optional + /// persistent peer cache path. + #[must_use] + pub fn from_node_with_peer_cache( + node: Arc, + config: ClientConfig, + peer_cache_path: Option, + ) -> Self { let network = Network::from_node(node); let (controller, persist_path) = build_controller(&config); Self { @@ -342,6 +366,7 @@ impl Client { next_request_id: AtomicU64::new(1), controller, persist_path, + peer_cache_path, } } @@ -376,6 +401,7 @@ impl Client { next_request_id: AtomicU64::new(1), controller, persist_path, + peer_cache_path: None, }) } @@ -466,6 +492,17 @@ impl Client { } } + /// Persist currently connected peers that have Direct-tagged addresses in + /// the DHT. Best effort; failures are logged and do not affect the client + /// operation that just completed. + pub async fn save_peer_cache(&self) { + if let Some(ref path) = self.peer_cache_path { + let node = self.network().node(); + peer_cache::promote_connected_direct_peers(node.as_ref(), path, node.dht().k_value()) + .await; + } + } + /// Get the next request ID for protocol messages. pub(crate) fn next_request_id(&self) -> u64 { self.next_request_id.fetch_add(1, Ordering::Relaxed) @@ -597,10 +634,31 @@ mod tests { failed: vec![], failed_count: 0, total_chunks: 0, + spend: Box::new(crate::data::error::PartialUploadSpend { + storage_cost_atto: "0".to_string(), + gas_cost_wei: 0, + }), reason: "r".to_string(), }, Outcome::NetworkError, ), + ( + Error::BadQuoteBinding { + peer_id: "peer".to_string(), + detail: "mismatch".to_string(), + }, + Outcome::ApplicationError, + ), + // A remote application rejection: the node responded with a + // structured `ProtocolError`, so the transport succeeded and + // this must NOT register as a capacity signal (V2-468). + ( + Error::RemotePut { + address: "abcd".to_string(), + source: ant_protocol::ProtocolError::PaymentFailed("stale quote".to_string()), + }, + Outcome::ApplicationError, + ), ]; for (err, expected) in &cases { let got = classify_error(err); @@ -679,8 +737,10 @@ mod tests { | Error::AlreadyStored | Error::InsufficientDiskSpace(_) | Error::CostEstimationInconclusive(_) + | Error::Cancelled(_) | Error::PartialUpload { .. } - | Error::BadQuoteBinding { .. } => (), + | Error::BadQuoteBinding { .. } + | Error::RemotePut { .. } => (), }; } } diff --git a/ant-core/src/data/client/payment.rs b/ant-core/src/data/client/payment.rs index 014f8bd8..3452d599 100644 --- a/ant-core/src/data/client/payment.rs +++ b/ant-core/src/data/client/payment.rs @@ -3,6 +3,7 @@ //! Connects quote collection, on-chain EVM payment, and proof serialization. //! Every PUT to the network requires a valid payment proof. +use crate::data::client::quote::median_paid_quote_issuer; use crate::data::client::Client; use crate::data::error::{Error, Result}; use ant_protocol::evm::{EncodedPeerId, ProofOfPayment, Wallet}; @@ -22,7 +23,7 @@ impl Client { /// Pay for storage and return the serialized payment proof bytes. /// /// This orchestrates the full payment flow: - /// 1. Collect `CLOSE_GROUP_SIZE` quotes from closest peers + /// 1. Collect `CLOSE_GROUP_SIZE` quotes from the witnessed close group /// 2. Build `SingleNodePayment` using node-reported prices (median 3x, others 0) /// 3. Pay on-chain via the wallet /// 4. Serialize `PaymentProof` with transaction hashes @@ -47,13 +48,19 @@ impl Client { debug!("Collecting quotes for address {}", hex::encode(address)); // 1. Collect quotes from network - let quotes_with_peers = self.get_store_quotes(address, data_size, data_type).await?; + let quote_plan = self + .get_store_quote_plan(address, data_size, data_type) + .await?; + let quotes_with_peers = quote_plan.quotes; + let median_quote_issuer = + median_paid_quote_issuer("es_with_peers).ok_or_else(|| { + Error::Payment( + "Failed to select median quote issuer from witnessed quotes".to_string(), + ) + })?; // Capture all quoted peers for replication by the caller. - let quoted_peers: Vec<(PeerId, Vec)> = quotes_with_peers - .iter() - .map(|(peer_id, addrs, _, _)| (*peer_id, addrs.clone())) - .collect(); + let quoted_peers = quote_plan.put_peers; // 2. Build peer_quotes for ProofOfPayment + quotes for SingleNodePayment. // Use node-reported prices directly — no contract price fetch needed. @@ -70,6 +77,12 @@ impl Client { let payment = SingleNodePayment::from_quotes(quotes_for_payment) .map_err(|e| Error::Payment(format!("Failed to create payment: {e}")))?; + info!( + "Selected SNP median paid quote issuer {} for address {} (median price: {})", + median_quote_issuer.0, + hex::encode(address), + median_quote_issuer.1 + ); info!("Payment total: {} atto", payment.total_amount()); // 4. Pay on-chain diff --git a/ant-core/src/data/client/peer_cache.rs b/ant-core/src/data/client/peer_cache.rs deleted file mode 100644 index 0673f182..00000000 --- a/ant-core/src/data/client/peer_cache.rs +++ /dev/null @@ -1,137 +0,0 @@ -//! Bootstrap-cache population helpers. -//! -//! Wires client-side peer contacts into saorsa-core's `BootstrapManager` -//! so the persistent cache reflects real peer quality across sessions. - -use ant_protocol::transport::{MultiAddr, P2PNode, PeerId}; -use std::net::{IpAddr, SocketAddr}; -use std::sync::Arc; -use tracing::debug; - -/// Feed a peer contact outcome into the `BootstrapManager` cache so future -/// cold-starts can rank peers by observed latency and success. -/// -/// `success = true`: upserts the peer via `add_discovered_peer` (subject to -/// saorsa-core Sybil checks — rate limit + IP diversity) and records RTT via -/// `update_peer_metrics`. -/// -/// `success = false`: only updates the quality score of peers already in -/// the cache. Unreachable peers are never inserted. -/// -/// Both upstream calls silently discard errors — peer-cache bookkeeping -/// must never abort a user operation. Enable the `saorsa_core::bootstrap` -/// tracing target to see rejection reasons. -pub(crate) async fn record_peer_outcome( - node: &Arc, - peer_id: PeerId, - addrs: &[MultiAddr], - success: bool, - rtt_ms: Option, -) { - if success { - let before = node.cached_peer_count().await; - let _ = node.add_discovered_peer(peer_id, addrs.to_vec()).await; - let after = node.cached_peer_count().await; - if after > before { - debug!("Bootstrap cache grew: {before} -> {after} peers"); - } - } - if let Some(primary) = select_primary_multiaddr(addrs) { - let _ = node - .update_peer_metrics(primary, success, rtt_ms, None) - .await; - } -} - -/// Pick the `MultiAddr` to use as the peer's cache key. -/// -/// Prefers a globally routable socket address over RFC1918 / link-local / -/// loopback. Without this, a peer advertising `[10.0.0.5, 203.0.113.1]` -/// would be keyed under the RFC1918 address, so metrics recorded during -/// a contact over the public address would land on a stale cache entry. -/// Falls back to any socket-addressable `MultiAddr` if none look global. -fn select_primary_multiaddr(addrs: &[MultiAddr]) -> Option<&MultiAddr> { - addrs - .iter() - .find(|a| a.socket_addr().is_some_and(|sa| is_globally_routable(&sa))) - .or_else(|| addrs.iter().find(|a| a.socket_addr().is_some())) -} - -fn is_globally_routable(addr: &SocketAddr) -> bool { - match addr.ip() { - IpAddr::V4(v4) => { - !v4.is_private() - && !v4.is_loopback() - && !v4.is_link_local() - && !v4.is_broadcast() - && !v4.is_documentation() - && !v4.is_unspecified() - } - IpAddr::V6(v6) => { - // Full Ipv6Addr::is_global is unstable; this is the practical - // subset that mirrors the IPv4 checks above. - !v6.is_loopback() - && !v6.is_unspecified() - && !v6.is_multicast() - && !v6.segments()[0].eq(&0xfe80) // link-local fe80::/10 (approx) - && !matches!(v6.segments()[0] & 0xfe00, 0xfc00) // unique-local fc00::/7 - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::net::{Ipv4Addr, Ipv6Addr}; - - #[test] - fn globally_routable_v4() { - // 8.8.8.8 (Google DNS) — genuinely public, not in any reserved range. - assert!(is_globally_routable(&SocketAddr::new( - IpAddr::V4(Ipv4Addr::new(8, 8, 8, 8)), - 80 - ))); - assert!(!is_globally_routable(&SocketAddr::new( - IpAddr::V4(Ipv4Addr::new(10, 0, 0, 5)), - 80 - ))); - assert!(!is_globally_routable(&SocketAddr::new( - IpAddr::V4(Ipv4Addr::LOCALHOST), - 80 - ))); - assert!(!is_globally_routable(&SocketAddr::new( - IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1)), - 80 - ))); - // 203.0.113.0/24 is TEST-NET-3 documentation — rejected by - // `is_documentation()`, which is the behaviour we want: quality - // metrics should not land on addresses that are never dialed in - // production by spec. - assert!(!is_globally_routable(&SocketAddr::new( - IpAddr::V4(Ipv4Addr::new(203, 0, 113, 1)), - 80 - ))); - } - - #[test] - fn globally_routable_v6() { - // 2606:4700:4700::1111 (Cloudflare DNS) — a real public v6 outside - // the `2001:db8::/32` documentation prefix. - assert!(is_globally_routable(&SocketAddr::new( - IpAddr::V6(Ipv6Addr::new(0x2606, 0x4700, 0x4700, 0, 0, 0, 0, 0x1111)), - 80 - ))); - assert!(!is_globally_routable(&SocketAddr::new( - IpAddr::V6(Ipv6Addr::LOCALHOST), - 80 - ))); - assert!(!is_globally_routable(&SocketAddr::new( - IpAddr::V6(Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1)), - 80 - ))); - assert!(!is_globally_routable(&SocketAddr::new( - IpAddr::V6(Ipv6Addr::new(0xfc00, 0, 0, 0, 0, 0, 0, 1)), - 80 - ))); - } -} diff --git a/ant-core/src/data/client/quote.rs b/ant-core/src/data/client/quote.rs index e03621f8..f4bc38ea 100644 --- a/ant-core/src/data/client/quote.rs +++ b/ant-core/src/data/client/quote.rs @@ -3,20 +3,44 @@ //! Handles requesting storage quotes from network nodes and //! managing payment for data storage. -use crate::data::client::peer_cache::record_peer_outcome; use crate::data::client::peer_xor_distance; use crate::data::client::Client; use crate::data::error::{Error, Result}; use ant_protocol::evm::{Amount, PaymentQuote}; -use ant_protocol::transport::{MultiAddr, PeerId}; +use ant_protocol::transport::{DHTNode, MultiAddr, P2PNode, PeerId, WitnessedCloseGroup}; use ant_protocol::{ compute_address, send_and_await_chunk_response, ChunkMessage, ChunkMessageBody, ChunkQuoteRequest, ChunkQuoteResponse, CLOSE_GROUP_MAJORITY, CLOSE_GROUP_SIZE, }; use futures::stream::{FuturesUnordered, StreamExt}; -use std::time::{Duration, Instant}; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::time::Duration; use tracing::{debug, info, warn}; +/// Fault-tolerant quote collection asks one extra close group of peers and +/// keeps the closest successful `CLOSE_GROUP_SIZE` responders. This remains +/// useful for merkle preflight probes, but single-node payments deliberately +/// ask only the actual close group. +const FAULT_TOLERANT_QUOTE_QUERY_MULTIPLIER: usize = 2; + +/// Witnessed close-group quorum as a fraction of the initial close group. +/// For today's `CLOSE_GROUP_SIZE = 7`, this yields the requested 5-of-7 +/// quorum. +const WITNESSED_QUORUM_NUMERATOR: usize = 2; +const WITNESSED_QUORUM_DENOMINATOR: usize = 3; + +/// Number of closest nodes each initial witnessed responder contributes. +const SINGLE_NODE_WITNESSED_VIEW_COUNT: usize = 20; + +/// Index of the paid median quote after sorting by quoted price. +const MEDIAN_QUOTE_INDEX: usize = CLOSE_GROUP_SIZE / 2; + +/// Overall timeout for collecting quote responses. Must accommodate +/// connect_with_fallback cascade (direct 5s + hole-punch 15s×3 + relay 30s ≈ +/// 80s) plus the per-peer quote timeout. +const QUOTE_COLLECTION_TIMEOUT_SECS: u64 = 120; + /// ML-DSA-65 public key length in bytes. Mirrors the same value defined as /// `pub const ML_DSA_65_PUBLIC_KEY_SIZE` in `saorsa-pqc::pqc::types`, which /// the storer's `peer_id_from_public_key_bytes` enforces. We keep a local @@ -39,9 +63,9 @@ const ML_DSA_PUB_KEY_LEN: usize = 1952; /// /// We mirror the cheap structural check here. The storer also runs /// `verify_quote_content` and `verify_quote_signature`; those are ML-DSA -/// verifications (~1 ms × 14 quotes × every chunk) and are deliberately NOT -/// mirrored on the client to keep upload latency unchanged. They are tracked -/// as a follow-up if a real attack surfaces them. +/// verifications (~1 ms per requested quote) and are deliberately NOT mirrored +/// on the client to keep upload latency unchanged. They are tracked as a +/// follow-up if a real attack surfaces them. fn quote_binding_is_valid(peer_id: &PeerId, quote: &PaymentQuote) -> bool { if quote.pub_key.len() != ML_DSA_PUB_KEY_LEN { return false; @@ -72,9 +96,8 @@ fn quote_binding_is_valid(peer_id: &PeerId, quote: &PaymentQuote) -> bool { /// - `Err(Error::AlreadyStored)` — the peer claims the chunk is already /// present AND the quote it provided binds to its peer ID. Vote counts. /// - `Err(Error::BadQuoteBinding { .. })` — bad binding (mirrors the -/// storer-side rejection); the peer is treated as a failure so the -/// AIMD cache learns to deprioritize it. Outer collector counts these -/// via the typed variant (no string matching). +/// storer-side rejection). Outer collector counts these via the typed +/// variant (no string matching). /// - `Err(Error::Serialization(...))` — the quote bytes did not deserialize. fn classify_quote_response( peer_id: &PeerId, @@ -115,20 +138,6 @@ fn classify_quote_response( Ok((payment_quote, price)) } -/// Map a per-peer quote-collection outcome to the AIMD-cache success flag. -/// -/// `Ok(_)` and `AlreadyStored` are both *benign* outcomes — the peer is -/// reachable and well-behaved — so we record them as successes (recording -/// a smooth RTT). Every other variant (network/timeout/protocol/ -/// serialization, plus `BadQuoteBinding`) records as a failure so the -/// local AIMD bootstrap cache learns to deprioritize peers that don't -/// help us upload. -/// -/// Pulled out of the per-peer closure for unit-testing. -fn quote_outcome_is_success(result: &std::result::Result<(PaymentQuote, Amount), Error>) -> bool { - matches!(result, Ok(_) | Err(Error::AlreadyStored)) -} - /// Drop quotes whose `pub_key` does not BLAKE3-hash to the peer that supplied /// them. Logs each dropped quote at WARN. fn drop_quotes_with_bad_bindings( @@ -150,12 +159,541 @@ fn drop_quotes_with_bad_bindings( before - quotes.len() } +#[allow(clippy::too_many_arguments)] +async fn request_store_quote_from_peer( + node: Arc, + peer_id: PeerId, + peer_addrs: Vec, + request_id: u64, + address: [u8; 32], + data_size: u64, + data_type: u32, + per_peer_timeout: Duration, +) -> StoreQuoteRequestResult { + let request = ChunkQuoteRequest { + address, + data_size, + data_type, + }; + let message = ChunkMessage { + request_id, + body: ChunkMessageBody::QuoteRequest(request), + }; + + let message_bytes = match message.encode() { + Ok(bytes) => bytes, + Err(e) => { + return ( + peer_id, + peer_addrs, + Err(Error::Protocol(format!( + "Failed to encode quote request for {peer_id}: {e}" + ))), + ); + } + }; + + let result = send_and_await_chunk_response( + &node, + &peer_id, + message_bytes, + request_id, + per_peer_timeout, + &peer_addrs, + |body| match body { + ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Success { + quote, + already_stored, + }) => Some(classify_quote_response(&peer_id, "e, already_stored)), + ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Error(e)) => Some(Err( + Error::Protocol(format!("Quote error from {peer_id}: {e}")), + )), + _ => None, + }, + |e| Error::Network(format!("Failed to send quote request to {peer_id}: {e}")), + || Error::Timeout(format!("Timeout waiting for quote from {peer_id}")), + ) + .await; + + (peer_id, peer_addrs, result) +} + +#[allow(clippy::too_many_arguments)] +fn record_store_quote_result( + peer_id: PeerId, + addrs: Vec, + quote_result: Result<(PaymentQuote, Amount)>, + address: &[u8; 32], + quotes: &mut Vec, + already_stored_peers: &mut Vec<(PeerId, [u8; 32])>, + failures: &mut Vec, + bad_quote_count: &mut usize, +) { + match quote_result { + Ok((quote, price)) => { + quotes.push((peer_id, addrs, quote, price)); + } + Err(Error::AlreadyStored) => { + info!("Peer {peer_id} reports chunk already stored"); + let dist = peer_xor_distance(&peer_id, address); + already_stored_peers.push((peer_id, dist)); + } + Err(e) => { + if matches!(&e, Error::BadQuoteBinding { .. }) { + *bad_quote_count += 1; + } + warn!("Failed to get quote from {peer_id}: {e}"); + failures.push(format!("{peer_id}: {e}")); + } + } +} + +fn witnessed_quote_launch_budget( + successful_quotes: usize, + in_flight: usize, + remaining_peers: usize, +) -> usize { + CLOSE_GROUP_SIZE + .saturating_sub(successful_quotes.saturating_add(in_flight)) + .min(remaining_peers) +} + +fn single_node_quote_query_count() -> usize { + CLOSE_GROUP_SIZE +} + +fn fault_tolerant_quote_query_count() -> usize { + CLOSE_GROUP_SIZE * FAULT_TOLERANT_QUOTE_QUERY_MULTIPLIER +} + +fn witnessed_close_group_quorum() -> usize { + (CLOSE_GROUP_SIZE * WITNESSED_QUORUM_NUMERATOR).div_ceil(WITNESSED_QUORUM_DENOMINATOR) +} + +fn witnessed_close_group_quorum_for_missing_views(missing_views: usize) -> usize { + witnessed_close_group_quorum() + .saturating_sub(missing_views) + .max(1) +} + +fn missing_witnessed_responder_views(witnessed: &WitnessedCloseGroup) -> usize { + witnessed + .initial_closest + .len() + .saturating_sub(witnessed.responder_views.len()) +} + +fn witnessed_close_group_quorum_for_transcript(witnessed: &WitnessedCloseGroup) -> usize { + witnessed_close_group_quorum_for_missing_views(missing_witnessed_responder_views(witnessed)) +} + +fn peer_list(peers: &[PeerId]) -> Vec { + peers.iter().map(ToString::to_string).collect() +} + +pub(crate) type StoreQuote = (PeerId, Vec, PaymentQuote, Amount); +type StoreQuoteRequestResult = (PeerId, Vec, Result<(PaymentQuote, Amount)>); +type VotersByPeer = HashMap>; +type WitnessedVoteData = (HashMap, VotersByPeer, Vec<(PeerId, usize)>); + +pub(crate) struct StoreQuotePlan { + pub(crate) quotes: Vec, + pub(crate) put_peers: Vec<(PeerId, Vec)>, +} + +#[derive(Debug, Clone)] +struct WitnessedQuoteCandidate { + node: DHTNode, + votes: usize, + voters: HashSet, +} + +#[derive(Debug, Clone)] +struct WitnessedQuotePeer { + peer_id: PeerId, + addrs: Vec, + voters: HashSet, +} + +#[derive(Debug, Clone)] +struct WitnessedQuoteSelection { + quote_peers: Vec, + initial_put_peers: Vec<(PeerId, Vec)>, + quorum: usize, +} + +enum QuoteSelectionPolicy { + ClosestByDistance, + WitnessedMedianVoters { + voters_by_peer: VotersByPeer, + quorum: usize, + }, +} + +fn witnessed_initial_peers(witnessed: &WitnessedCloseGroup) -> Vec { + witnessed + .initial_closest + .iter() + .map(|node| node.peer_id.to_string()) + .collect() +} + +fn witnessed_responder_views(witnessed: &WitnessedCloseGroup) -> Vec { + witnessed + .responder_views + .iter() + .map(|view| { + let peers = view + .closest + .iter() + .map(|node| node.peer_id) + .collect::>(); + format!("{}=>{:?}", view.responder, peer_list(&peers)) + }) + .collect() +} + +fn merge_witnessed_node(nodes: &mut HashMap, node: DHTNode) { + match nodes.entry(node.peer_id) { + std::collections::hash_map::Entry::Occupied(mut entry) => { + entry.get_mut().merge_from(node); + } + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert(node); + } + } +} + +fn sort_vote_counts_by_distance(vote_counts: &mut [(PeerId, usize)], address: &[u8; 32]) { + vote_counts.sort_by(|left, right| { + peer_xor_distance(&left.0, address) + .cmp(&peer_xor_distance(&right.0, address)) + .then_with(|| left.0.as_bytes().cmp(right.0.as_bytes())) + }); +} + +fn witnessed_vote_counts_and_nodes( + witnessed: &WitnessedCloseGroup, + address: &[u8; 32], +) -> WitnessedVoteData { + let mut known_nodes = HashMap::new(); + for node in &witnessed.initial_closest { + merge_witnessed_node(&mut known_nodes, node.clone()); + } + + let mut voters_by_peer: HashMap> = HashMap::new(); + for view in &witnessed.responder_views { + let mut voted = HashSet::new(); + for node in &view.closest { + merge_witnessed_node(&mut known_nodes, node.clone()); + if voted.insert(node.peer_id) { + voters_by_peer + .entry(node.peer_id) + .or_default() + .insert(view.responder); + } + } + } + + let mut vote_counts: Vec<(PeerId, usize)> = voters_by_peer + .iter() + .map(|(peer_id, voters)| (*peer_id, voters.len())) + .collect(); + sort_vote_counts_by_distance(&mut vote_counts, address); + (known_nodes, voters_by_peer, vote_counts) +} + +fn witnessed_consensus_candidates( + witnessed: &WitnessedCloseGroup, + address: &[u8; 32], + quorum: usize, +) -> Vec { + let (known_nodes, voters_by_peer, vote_counts) = + witnessed_vote_counts_and_nodes(witnessed, address); + let mut candidates = vote_counts + .iter() + .filter_map(|(peer_id, votes)| { + if *votes < quorum { + return None; + } + known_nodes.get(peer_id).cloned().and_then(|node| { + voters_by_peer + .get(peer_id) + .cloned() + .map(|voters| WitnessedQuoteCandidate { + node, + votes: *votes, + voters, + }) + }) + }) + .collect::>(); + + candidates.sort_by(|left, right| { + peer_xor_distance(&left.node.peer_id, address) + .cmp(&peer_xor_distance(&right.node.peer_id, address)) + .then_with(|| right.votes.cmp(&left.votes)) + .then_with(|| { + left.node + .peer_id + .as_bytes() + .cmp(right.node.peer_id.as_bytes()) + }) + }); + candidates +} + +fn witnessed_vote_counts(witnessed: &WitnessedCloseGroup, address: &[u8; 32]) -> Vec { + let (_, _, vote_counts) = witnessed_vote_counts_and_nodes(witnessed, address); + vote_counts + .iter() + .map(|(peer_id, votes)| format!("{peer_id}:{votes}")) + .collect() +} + +fn witnessed_consensus( + witnessed: &WitnessedCloseGroup, + address: &[u8; 32], + quorum: usize, +) -> Vec { + witnessed_consensus_candidates(witnessed, address, quorum) + .iter() + .map(|candidate| format!("{}:{}", candidate.node.peer_id, candidate.votes)) + .collect() +} + +fn witnessed_close_group_diagnostics( + address: &[u8; 32], + witnessed: &WitnessedCloseGroup, + quorum: usize, +) -> String { + format!( + "target={}, initial={:?}, responder_views={:?}, vote_counts={:?}, quorum={}, final={:?}", + hex::encode(address), + witnessed_initial_peers(witnessed), + witnessed_responder_views(witnessed), + witnessed_vote_counts(witnessed, address), + quorum, + witnessed_consensus(witnessed, address, quorum) + ) +} + +fn witnessed_quote_selection_or_error( + address: &[u8; 32], + witnessed: &WitnessedCloseGroup, + required: usize, + quorum: usize, +) -> Result { + let candidates = witnessed_consensus_candidates(witnessed, address, quorum); + if candidates.len() < required { + return Err(Error::InsufficientPeers(format!( + "Witnessed close group inconclusive before payment: got {}/{} quorum-recognised peers. {}", + candidates.len(), + required, + witnessed_close_group_diagnostics(address, witnessed, quorum) + ))); + } + + let initial_put_peers = witnessed + .initial_closest + .iter() + .take(CLOSE_GROUP_SIZE) + .map(|node| (node.peer_id, node.addresses_by_priority())) + .collect::>(); + + if initial_put_peers.len() < CLOSE_GROUP_SIZE { + return Err(Error::InsufficientPeers(format!( + "Witnessed close group returned only {}/{} initial PUT peers before payment. {}", + initial_put_peers.len(), + CLOSE_GROUP_SIZE, + witnessed_close_group_diagnostics(address, witnessed, quorum) + ))); + } + + let quote_peers = candidates + .into_iter() + .map(|candidate| WitnessedQuotePeer { + peer_id: candidate.node.peer_id, + addrs: candidate.node.addresses_by_priority(), + voters: candidate.voters, + }) + .collect(); + + Ok(WitnessedQuoteSelection { + quote_peers, + initial_put_peers, + quorum, + }) +} + +pub(crate) fn median_paid_quote_issuer( + quotes: &[(PeerId, Vec, PaymentQuote, Amount)], +) -> Option<(PeerId, Amount)> { + if quotes.len() <= MEDIAN_QUOTE_INDEX { + return None; + } + + let mut by_price: Vec<(usize, PeerId, Amount)> = quotes + .iter() + .enumerate() + .map(|(index, (peer_id, _, _, price))| (index, *peer_id, *price)) + .collect(); + by_price.sort_by_key(|(index, _, price)| (*price, *index)); + by_price + .get(MEDIAN_QUOTE_INDEX) + .map(|(_, peer_id, price)| (*peer_id, *price)) +} + +fn sort_quotes_by_distance(quotes: &mut [StoreQuote], address: &[u8; 32]) { + quotes.sort_by(|left, right| { + peer_xor_distance(&left.0, address) + .cmp(&peer_xor_distance(&right.0, address)) + .then_with(|| left.0.as_bytes().cmp(right.0.as_bytes())) + }); +} + +fn median_paid_quote_issuer_for_indices( + quotes: &[StoreQuote], + indices: &[usize], +) -> Option<(PeerId, Amount)> { + if indices.len() <= MEDIAN_QUOTE_INDEX { + return None; + } + + let mut by_price: Vec<(usize, PeerId, Amount)> = indices + .iter() + .enumerate() + .map(|(selected_index, quote_index)| { + let (peer_id, _, _, price) = "es[*quote_index]; + (selected_index, *peer_id, *price) + }) + .collect(); + by_price.sort_by_key(|(selected_index, _, price)| (*price, *selected_index)); + by_price + .get(MEDIAN_QUOTE_INDEX) + .map(|(_, peer_id, price)| (*peer_id, *price)) +} + +fn median_issuer_voter_support( + quotes: &[StoreQuote], + indices: &[usize], + voters_by_peer: &VotersByPeer, +) -> Option<(PeerId, usize)> { + let (median_peer_id, _) = median_paid_quote_issuer_for_indices(quotes, indices)?; + let voters = voters_by_peer.get(&median_peer_id)?; + Some((median_peer_id, voters.len())) +} + +fn visit_quote_subsets( + quote_count: usize, + subset_size: usize, + start_index: usize, + current: &mut Vec, + visit: &mut F, +) where + F: FnMut(&[usize]), +{ + if current.len() == subset_size { + visit(current); + return; + } + + let remaining = subset_size - current.len(); + let last_start = quote_count - remaining; + for index in start_index..=last_start { + current.push(index); + visit_quote_subsets(quote_count, subset_size, index + 1, current, visit); + current.pop(); + } +} + +fn select_closest_quotes(mut quotes: Vec, address: &[u8; 32]) -> Vec { + sort_quotes_by_distance(&mut quotes, address); + quotes.truncate(CLOSE_GROUP_SIZE); + quotes +} + +fn select_witnessed_median_voter_quotes( + mut quotes: Vec, + address: &[u8; 32], + voters_by_peer: &VotersByPeer, + required_support: usize, +) -> Option> { + if quotes.len() < CLOSE_GROUP_SIZE { + return None; + } + + sort_quotes_by_distance(&mut quotes, address); + + let mut best_indices: Option<(usize, Vec)> = None; + let mut current_indices = Vec::with_capacity(CLOSE_GROUP_SIZE); + visit_quote_subsets( + quotes.len(), + CLOSE_GROUP_SIZE, + 0, + &mut current_indices, + &mut |indices| { + let Some((_, support)) = median_issuer_voter_support("es, indices, voters_by_peer) + else { + return; + }; + if support < required_support { + return; + } + match &best_indices { + Some((best_support, best)) if *best_support > support => {} + Some((best_support, best)) + if *best_support == support && best.as_slice() <= indices => {} + _ => best_indices = Some((support, indices.to_vec())), + } + }, + ); + + best_indices.map(|(_, indices)| { + indices + .into_iter() + .map(|index| quotes[index].clone()) + .collect() + }) +} + +fn put_peers_with_median_voters_first( + quotes: &[StoreQuote], + put_peers: &[(PeerId, Vec)], + voters_by_peer: &VotersByPeer, + required_support: usize, +) -> Option)>> { + let (median_peer_id, _) = median_paid_quote_issuer(quotes)?; + let voters = voters_by_peer.get(&median_peer_id)?; + + let mut supporting_peers = Vec::new(); + let mut fallback_peers = Vec::new(); + for (peer_id, addrs) in put_peers { + let peer = (*peer_id, addrs.clone()); + if voters.contains(peer_id) { + supporting_peers.push(peer); + } else { + fallback_peers.push(peer); + } + } + + if supporting_peers.len() < required_support { + return None; + } + + supporting_peers.extend(fallback_peers); + Some(supporting_peers) +} + impl Client { /// Get storage quotes from the closest peers for a given address. /// - /// Queries 2x `CLOSE_GROUP_SIZE` peers from the DHT for fault tolerance, - /// requests quotes from all of them concurrently, and returns the - /// `CLOSE_GROUP_SIZE` closest successful responders sorted by XOR distance. + /// Builds a quorum-witnessed candidate set with at least + /// `CLOSE_GROUP_SIZE` peers, requests quotes from all of them concurrently, + /// and returns the closest supported `CLOSE_GROUP_SIZE` successful + /// responders. When multiple sets are possible, the client prefers the + /// one with the strongest paid-median voter support, then the closest + /// peers by XOR distance. /// /// Returns `Error::AlreadyStored` early if `CLOSE_GROUP_MAJORITY` peers /// report the chunk is already stored. @@ -163,115 +701,184 @@ impl Client { /// # Errors /// /// Returns an error if insufficient quotes can be collected. - #[allow(clippy::too_many_lines)] pub async fn get_store_quotes( &self, address: &[u8; 32], data_size: u64, data_type: u32, ) -> Result, PaymentQuote, Amount)>> { - let node = self.network().node(); + Ok(self + .get_store_quote_plan(address, data_size, data_type) + .await? + .quotes) + } - // Over-query for fault tolerance: ask 2x peers, keep closest successful ones. - let over_query_count = CLOSE_GROUP_SIZE * 2; - debug!( - "Requesting quotes from up to {over_query_count} peers for address {} (size: {data_size})", - hex::encode(address) - ); + /// Get storage quotes plus PUT targets ordered for paid-median acceptance. + /// + /// Quote order is preserved for proof construction because tied quote + /// prices rely on stable median selection. PUT target order is separate: + /// peers that voted for the paid median issuer are placed first so the + /// initial write wave is locally acceptable to a storage majority. + pub(crate) async fn get_store_quote_plan( + &self, + address: &[u8; 32], + data_size: u64, + data_type: u32, + ) -> Result { + let witnessed_selection = self.select_witnessed_quote_selection(address).await?; + let voters_by_peer: VotersByPeer = witnessed_selection + .quote_peers + .iter() + .map(|peer| (peer.peer_id, peer.voters.clone())) + .collect(); + let remote_peers = witnessed_selection + .quote_peers + .into_iter() + .map(|peer| (peer.peer_id, peer.addrs)) + .collect(); + let initial_put_peers = witnessed_selection.initial_put_peers; + let quorum = witnessed_selection.quorum; + let quotes = self + .collect_store_quotes_from_remote_peers( + address, + data_size, + data_type, + remote_peers, + QuoteSelectionPolicy::WitnessedMedianVoters { + voters_by_peer: voters_by_peer.clone(), + quorum, + }, + ) + .await?; + let put_peers = put_peers_with_median_voters_first( + "es, + &initial_put_peers, + &voters_by_peer, + quorum, + ) + .ok_or_else(|| { + Error::InsufficientPeers(format!( + "Collected {} witnessed quotes, but fewer than {} initial witness PUT peers \ + voted for the paid median issuer for {}", + quotes.len(), + quorum, + hex::encode(address) + )) + })?; + Ok(StoreQuotePlan { quotes, put_peers }) + } + + /// Get storage quotes with the previous over-query behaviour. + /// + /// Merkle preflight uses quote responses only as an already-stored probe; + /// the actual payment still happens through merkle candidate pools. Keep + /// the extra peer buffer there so merkle upload behaviour remains + /// unchanged when a few peers are slow or return unusable quote bindings. + pub(crate) async fn get_store_quotes_with_fault_tolerance( + &self, + address: &[u8; 32], + data_size: u64, + data_type: u32, + ) -> Result, PaymentQuote, Amount)>> { + let peer_query_count = fault_tolerant_quote_query_count(); let remote_peers = self .network() - .find_closest_peers(address, over_query_count) + .find_closest_peers(address, peer_query_count) .await?; - if remote_peers.len() < CLOSE_GROUP_SIZE { - return Err(Error::InsufficientPeers(format!( - "Found {} peers, need {CLOSE_GROUP_SIZE}", - remote_peers.len() - ))); + self.collect_store_quotes_from_remote_peers( + address, + data_size, + data_type, + remote_peers, + QuoteSelectionPolicy::ClosestByDistance, + ) + .await + } + + async fn select_witnessed_quote_selection( + &self, + address: &[u8; 32], + ) -> Result { + let required = single_node_quote_query_count(); + let witnessed = self + .network() + .find_witnessed_close_group_with_view_count( + address, + required, + SINGLE_NODE_WITNESSED_VIEW_COUNT, + ) + .await + .map_err(|e| { + Error::InsufficientPeers(format!( + "Witnessed close group lookup failed before payment for target {}: {e}", + hex::encode(address) + )) + })?; + let base_quorum = witnessed_close_group_quorum(); + let missing_views = missing_witnessed_responder_views(&witnessed); + let quorum = witnessed_close_group_quorum_for_transcript(&witnessed); + + if missing_views > 0 { + warn!( + target = %hex::encode(address), + initial = witnessed.initial_closest.len(), + responder_views = witnessed.responder_views.len(), + missing_views = missing_views, + base_quorum = base_quorum, + adjusted_quorum = quorum, + "Witnessed close group transcript is missing responder views; lowering SNP witness quorum" + ); } - let per_peer_timeout = Duration::from_secs(self.config().quote_timeout_secs); - // Overall timeout for collecting all quotes. Must accommodate - // connect_with_fallback cascade (direct 5s + hole-punch 15s×3 + relay 30s ≈ 80s) - // plus the per-peer quote timeout. 120s is generous. - let overall_timeout = Duration::from_secs(120); - - // Request quotes from all peers concurrently - let mut quote_futures = FuturesUnordered::new(); - - for (peer_id, peer_addrs) in &remote_peers { - let request_id = self.next_request_id(); - let request = ChunkQuoteRequest { - address: *address, - data_size, - data_type, - }; - let message = ChunkMessage { - request_id, - body: ChunkMessageBody::QuoteRequest(request), - }; + debug!( + target = %hex::encode(address), + quorum = quorum, + view_count = SINGLE_NODE_WITNESSED_VIEW_COUNT, + initial = ?witnessed_initial_peers(&witnessed), + responder_views = ?witnessed_responder_views(&witnessed), + vote_counts = ?witnessed_vote_counts(&witnessed, address), + final_witnessed_set = ?witnessed_consensus(&witnessed, address, quorum), + "Witnessed close group selected for SNP quote collection" + ); - let message_bytes = match message.encode() { - Ok(bytes) => bytes, - Err(e) => { - warn!("Failed to encode quote request for {peer_id}: {e}"); - continue; - } - }; + witnessed_quote_selection_or_error(address, &witnessed, required, quorum) + } - let peer_id_clone = *peer_id; - let addrs_clone = peer_addrs.clone(); - let node_clone = node.clone(); - - let quote_future = async move { - let start = Instant::now(); - let result = send_and_await_chunk_response( - &node_clone, - &peer_id_clone, - message_bytes, - request_id, - per_peer_timeout, - &addrs_clone, - |body| match body { - ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Success { - quote, - already_stored, - }) => Some(classify_quote_response( - &peer_id_clone, - "e, - already_stored, - )), - ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Error(e)) => Some(Err( - Error::Protocol(format!("Quote error from {peer_id_clone}: {e}")), - )), - _ => None, - }, - |e| { - Error::Network(format!( - "Failed to send quote request to {peer_id_clone}: {e}" - )) - }, - || Error::Timeout(format!("Timeout waiting for quote from {peer_id_clone}")), - ) - .await; + #[allow(clippy::too_many_lines)] + async fn collect_store_quotes_from_remote_peers( + &self, + address: &[u8; 32], + data_size: u64, + data_type: u32, + remote_peers: Vec<(PeerId, Vec)>, + quote_selection_policy: QuoteSelectionPolicy, + ) -> Result, PaymentQuote, Amount)>> { + let peer_query_count = remote_peers.len(); - // Record the per-peer outcome for the AIMD bootstrap cache. - // See `quote_outcome_is_success` for the full classification. - let success = quote_outcome_is_success(&result); - let rtt_ms = success.then(|| start.elapsed().as_millis() as u64); - record_peer_outcome(&node_clone, peer_id_clone, &addrs_clone, success, rtt_ms) - .await; + let node = self.network().node(); - (peer_id_clone, addrs_clone, result) - }; + debug!( + "Requesting quotes from up to {peer_query_count} peers for address {} (size: {data_size})", + hex::encode(address) + ); - quote_futures.push(quote_future); + if remote_peers.len() < CLOSE_GROUP_SIZE { + return Err(Error::InsufficientPeers(format!( + "Found {} peers, need {CLOSE_GROUP_SIZE}", + remote_peers.len() + ))); } + debug_assert!(peer_query_count >= CLOSE_GROUP_SIZE); - // Collect all responses with an overall timeout to prevent indefinite stalls. - // Over-query means we have 2x peers, so we can tolerate failures. - let mut quotes = Vec::with_capacity(over_query_count); + let per_peer_timeout = Duration::from_secs(self.config().quote_timeout_secs); + let overall_timeout = Duration::from_secs(QUOTE_COLLECTION_TIMEOUT_SECS); + + // Collect quote responses. SNP/witnessed collection deliberately tries + // the closest witnessed peers first and only falls back to further + // witnessed peers when a closer peer fails to produce a usable quote. + let mut quotes = Vec::with_capacity(peer_query_count); let mut already_stored_peers: Vec<(PeerId, [u8; 32])> = Vec::new(); let mut failures: Vec = Vec::new(); @@ -281,46 +888,120 @@ impl Client { // network-broken) and the user benefits from seeing them called out. let mut bad_quote_count = 0usize; - let collect_result: std::result::Result, _> = - tokio::time::timeout(overall_timeout, async { - while let Some((peer_id, addrs, quote_result)) = quote_futures.next().await { - match quote_result { - Ok((quote, price)) => { - quotes.push((peer_id, addrs, quote, price)); - } - Err(Error::AlreadyStored) => { - info!("Peer {peer_id} reports chunk already stored"); - let dist = peer_xor_distance(&peer_id, address); - already_stored_peers.push((peer_id, dist)); + let staged_witnessed_collection = matches!( + "e_selection_policy, + QuoteSelectionPolicy::WitnessedMedianVoters { .. } + ); + + if staged_witnessed_collection { + let mut quote_futures = FuturesUnordered::new(); + let mut next_peer_index = 0usize; + let collect_result: std::result::Result, _> = + tokio::time::timeout(overall_timeout, async { + loop { + let launch_count = witnessed_quote_launch_budget( + quotes.len(), + quote_futures.len(), + remote_peers.len().saturating_sub(next_peer_index), + ); + for _ in 0..launch_count { + let (peer_id, peer_addrs) = &remote_peers[next_peer_index]; + next_peer_index += 1; + quote_futures.push(request_store_quote_from_peer( + node.clone(), + *peer_id, + peer_addrs.clone(), + self.next_request_id(), + *address, + data_size, + data_type, + per_peer_timeout, + )); } - Err(e) => { - // Count bad-binding peers separately (typed - // variant — no string sniffing). Treat as a - // normal failure for InsufficientPeers reporting. - if matches!(&e, Error::BadQuoteBinding { .. }) { - bad_quote_count += 1; - } - warn!("Failed to get quote from {peer_id}: {e}"); - failures.push(format!("{peer_id}: {e}")); + + if quotes.len() >= CLOSE_GROUP_SIZE || quote_futures.is_empty() { + break; } + + let Some((peer_id, addrs, quote_result)) = quote_futures.next().await + else { + break; + }; + record_store_quote_result( + peer_id, + addrs, + quote_result, + address, + &mut quotes, + &mut already_stored_peers, + &mut failures, + &mut bad_quote_count, + ); } + Ok(()) + }) + .await; + + match collect_result { + Err(_elapsed) => { + warn!( + "Quote collection timed out after {overall_timeout:?} for address {}", + hex::encode(address) + ); } - Ok(()) - }) - .await; + Ok(Err(e)) => return Err(e), + Ok(Ok(())) => {} + } + } else { + // Merkle preflight keeps the previous behaviour: query the full + // over-query set concurrently because those quote responses are + // only used as an already-stored probe. + let mut quote_futures = FuturesUnordered::new(); + + for (peer_id, peer_addrs) in &remote_peers { + quote_futures.push(request_store_quote_from_peer( + node.clone(), + *peer_id, + peer_addrs.clone(), + self.next_request_id(), + *address, + data_size, + data_type, + per_peer_timeout, + )); + } - match collect_result { - Err(_elapsed) => { - warn!( - "Quote collection timed out after {overall_timeout:?} for address {}", - hex::encode(address) - ); - // Fall through to check if we have enough quotes despite timeout. - // The timeout fires when slow peers haven't responded yet, but we - // may already have enough successful quotes from fast peers. + let collect_result: std::result::Result, _> = + tokio::time::timeout(overall_timeout, async { + while let Some((peer_id, addrs, quote_result)) = quote_futures.next().await { + record_store_quote_result( + peer_id, + addrs, + quote_result, + address, + &mut quotes, + &mut already_stored_peers, + &mut failures, + &mut bad_quote_count, + ); + } + Ok(()) + }) + .await; + + match collect_result { + Err(_elapsed) => { + warn!( + "Quote collection timed out after {overall_timeout:?} for address {}", + hex::encode(address) + ); + // Fall through to check if we have enough quotes despite timeout. + // The timeout fires when slow peers haven't responded yet, but we + // may already have enough successful quotes from fast peers. + } + Ok(Err(e)) => return Err(e), + Ok(Ok(())) => {} } - Ok(Err(e)) => return Err(e), - Ok(Ok(())) => {} } // Defensive double-check: the per-peer handler already filters @@ -371,22 +1052,34 @@ impl Client { let total_responses = quote_count + failure_count + already_stored_count; if quotes.len() >= CLOSE_GROUP_SIZE { - // Sort by XOR distance to target, keep the closest CLOSE_GROUP_SIZE. - quotes.sort_by(|a, b| { - let dist_a = peer_xor_distance(&a.0, address); - let dist_b = peer_xor_distance(&b.0, address); - dist_a.cmp(&dist_b) - }); - quotes.truncate(CLOSE_GROUP_SIZE); + let selected_quotes = match quote_selection_policy { + QuoteSelectionPolicy::ClosestByDistance => select_closest_quotes(quotes, address), + QuoteSelectionPolicy::WitnessedMedianVoters { + voters_by_peer, + quorum, + } => select_witnessed_median_voter_quotes(quotes, address, &voters_by_peer, quorum) + .ok_or_else(|| { + Error::InsufficientPeers(format!( + "Got {quote_count} quotes, need {CLOSE_GROUP_SIZE} whose paid \ + median issuer is recognised by at least {} \ + selected witness peers ({total_responses} responses: \ + {already_stored_count} already_stored, {failure_count} failed \ + including {bad_quote_count} with mismatched peer bindings). \ + Failures: [{}]", + quorum, + failures.join("; ") + )) + })?, + }; info!( "Collected {} quotes for address {} ({total_responses} responses: \ {quote_count} ok, {already_stored_count} already_stored, {failure_count} failed, \ {bad_quote_count} bad-binding)", - quotes.len(), + selected_quotes.len(), hex::encode(address), ); - return Ok(quotes); + return Ok(selected_quotes); } Err(Error::InsufficientPeers(format!( @@ -415,7 +1108,7 @@ mod tests { use super::*; use ant_protocol::evm::RewardsAddress; use ant_protocol::pqc::ops::{MlDsaOperations, MlDsaPublicKey}; - use ant_protocol::transport::MlDsa65; + use ant_protocol::transport::{DHTNode, MlDsa65, ResponderView, WitnessedCloseGroup}; use std::time::SystemTime; use xor_name::XorName; @@ -471,6 +1164,70 @@ mod tests { (claimed.peer_id, Vec::new(), quote, Amount::ZERO) } + fn witnessed_test_node(seed: u8) -> DHTNode { + DHTNode { + peer_id: PeerId::from_bytes([seed; 32]), + addresses: Vec::new(), + address_types: Vec::new(), + distance: None, + reliability: 1.0, + } + } + + fn witnessed_test_nodes(seeds: &[u8]) -> Vec { + seeds.iter().copied().map(witnessed_test_node).collect() + } + + fn witnessed_test_view(responder: u8, closest: &[u8]) -> ResponderView { + ResponderView { + responder: PeerId::from_bytes([responder; 32]), + closest: witnessed_test_nodes(closest), + } + } + + fn synthetic_peer(seed: u8) -> PeerId { + PeerId::from_bytes([seed; 32]) + } + + fn synthetic_quote(seed: u8, price: u64) -> (PeerId, Vec, PaymentQuote, Amount) { + let amount = Amount::from(price); + let quote = PaymentQuote { + content: XorName([0u8; 32]), + timestamp: SystemTime::UNIX_EPOCH, + price: amount, + rewards_address: RewardsAddress::new([0u8; 20]), + pub_key: Vec::new(), + signature: Vec::new(), + }; + (synthetic_peer(seed), Vec::new(), quote, amount) + } + + fn synthetic_voters(seeds: &[u8]) -> HashSet { + seeds.iter().copied().map(synthetic_peer).collect() + } + + fn quote_peer_seeds(quotes: &[(PeerId, Vec, PaymentQuote, Amount)]) -> Vec { + quotes + .iter() + .map(|(peer_id, _, _, _)| peer_id.as_bytes()[0]) + .collect() + } + + fn put_peer_seeds(peers: &[(PeerId, Vec)]) -> Vec { + peers + .iter() + .map(|(peer_id, _)| peer_id.as_bytes()[0]) + .collect() + } + + fn put_peers_from_seeds(seeds: &[u8]) -> Vec<(PeerId, Vec)> { + seeds + .iter() + .copied() + .map(|seed| (synthetic_peer(seed), Vec::new())) + .collect() + } + /// Independent re-implementation of the storer-side binding spec /// (`ant-node/src/payment/verifier.rs::validate_peer_bindings` + /// `peer_id_from_public_key_bytes`): @@ -552,6 +1309,351 @@ mod tests { // Tests for the filter (`drop_quotes_with_bad_bindings`) // ============================================================ + #[test] + fn quote_query_counts_keep_single_node_close_group_only() { + assert_eq!(single_node_quote_query_count(), CLOSE_GROUP_SIZE); + assert_eq!(SINGLE_NODE_WITNESSED_VIEW_COUNT, 20); + assert!(SINGLE_NODE_WITNESSED_VIEW_COUNT > single_node_quote_query_count()); + assert_eq!(witnessed_close_group_quorum(), 5); + assert_eq!(witnessed_close_group_quorum_for_missing_views(0), 5); + assert_eq!(witnessed_close_group_quorum_for_missing_views(1), 4); + assert_eq!(witnessed_close_group_quorum_for_missing_views(2), 3); + assert_eq!( + fault_tolerant_quote_query_count(), + CLOSE_GROUP_SIZE * FAULT_TOLERANT_QUOTE_QUERY_MULTIPLIER + ); + assert!(fault_tolerant_quote_query_count() > single_node_quote_query_count()); + } + + #[test] + fn witnessed_quote_launch_budget_keeps_exact_quote_window() { + assert_eq!( + witnessed_quote_launch_budget(0, 0, CLOSE_GROUP_SIZE * 2), + CLOSE_GROUP_SIZE, + "initial SNP quote fetch should launch the closest seven peers" + ); + assert_eq!( + witnessed_quote_launch_budget(1, CLOSE_GROUP_SIZE - 1, CLOSE_GROUP_SIZE), + 0, + "a successful quote should not launch an extra fallback" + ); + assert_eq!( + witnessed_quote_launch_budget(0, CLOSE_GROUP_SIZE - 1, CLOSE_GROUP_SIZE), + 1, + "a failed in-flight quote should launch the next closest fallback" + ); + assert_eq!( + witnessed_quote_launch_budget(CLOSE_GROUP_SIZE - 1, 0, 3), + 1, + "only one more peer is needed for the seventh quote" + ); + assert_eq!( + witnessed_quote_launch_budget(0, 0, CLOSE_GROUP_SIZE - 1), + CLOSE_GROUP_SIZE - 1, + "launch budget is capped by remaining candidates" + ); + } + + #[test] + fn witnessed_candidates_sort_by_xor_distance_then_votes() { + let address = [0u8; 32]; + let witnessed = WitnessedCloseGroup { + target: address, + k: CLOSE_GROUP_SIZE, + initial_closest: witnessed_test_nodes(&[1, 2, 3, 4, 5, 6, 7]), + responder_views: vec![ + witnessed_test_view(1, &[1, 9]), + witnessed_test_view(2, &[1, 9]), + witnessed_test_view(3, &[1, 9]), + witnessed_test_view(4, &[1, 9]), + witnessed_test_view(5, &[1, 9]), + witnessed_test_view(6, &[9]), + witnessed_test_view(7, &[9]), + ], + }; + + let candidates = + witnessed_consensus_candidates(&witnessed, &address, witnessed_close_group_quorum()); + + assert_eq!( + candidates + .iter() + .map(|candidate| candidate.node.peer_id.as_bytes()[0]) + .collect::>(), + vec![1, 9], + "XOR closeness must be the primary sort before quote collection" + ); + } + + #[test] + fn witnessed_quote_peers_error_is_typed_and_pre_payment_when_consensus_is_short() { + let address = [0u8; 32]; + let responder_views = (1..=7) + .map(|responder| witnessed_test_view(responder, &[1, 2, 3, 4])) + .collect(); + let witnessed = WitnessedCloseGroup { + target: address, + k: CLOSE_GROUP_SIZE, + initial_closest: witnessed_test_nodes(&[1, 2, 3, 4, 5, 6, 7]), + responder_views, + }; + + let err = witnessed_quote_selection_or_error( + &address, + &witnessed, + CLOSE_GROUP_SIZE, + witnessed_close_group_quorum(), + ) + .expect_err("short witnessed consensus must fail before payment"); + + match err { + Error::InsufficientPeers(message) => { + assert!(message.contains("before payment")); + assert!(message.contains("vote_counts")); + assert!(message.contains("quorum")); + } + other => panic!("expected typed InsufficientPeers error, got {other:?}"), + } + } + + #[test] + fn witnessed_quote_peers_include_quorum_fallback_candidates() { + const EXTRA_QUORUM_CANDIDATES: usize = 1; + + let address = [0u8; 32]; + let witnessed = WitnessedCloseGroup { + target: address, + k: CLOSE_GROUP_SIZE, + initial_closest: witnessed_test_nodes(&[1, 2, 3, 4, 5, 6, 7]), + responder_views: vec![ + witnessed_test_view(1, &[1, 2, 3, 4, 5, 6, 7]), + witnessed_test_view(2, &[1, 2, 3, 4, 5, 6, 8]), + witnessed_test_view(3, &[1, 2, 3, 4, 5, 7, 8]), + witnessed_test_view(4, &[1, 2, 3, 4, 6, 7, 8]), + witnessed_test_view(5, &[1, 2, 3, 5, 6, 7, 8]), + witnessed_test_view(6, &[1, 2, 4, 5, 6, 7, 8]), + witnessed_test_view(7, &[1, 3, 4, 5, 6, 7, 8]), + ], + }; + + let selection = witnessed_quote_selection_or_error( + &address, + &witnessed, + CLOSE_GROUP_SIZE, + witnessed_close_group_quorum(), + ) + .expect("fallback candidates should be retained for quote collection"); + + assert_eq!( + selection.quote_peers.len(), + CLOSE_GROUP_SIZE + EXTRA_QUORUM_CANDIDATES + ); + assert_eq!( + selection + .quote_peers + .iter() + .map(|peer| peer.peer_id.as_bytes()[0]) + .collect::>(), + vec![1, 2, 3, 4, 5, 6, 7, 8] + ); + assert_eq!( + put_peer_seeds(&selection.initial_put_peers), + vec![1, 2, 3, 4, 5, 6, 7] + ); + } + + #[test] + fn witnessed_quote_peers_lower_quorum_for_missing_responder_views() { + let address = [0u8; 32]; + let witnessed = WitnessedCloseGroup { + target: address, + k: CLOSE_GROUP_SIZE, + initial_closest: witnessed_test_nodes(&[1, 2, 3, 4, 5, 6, 7]), + responder_views: vec![ + witnessed_test_view(1, &[1, 2, 3, 4, 5, 6, 7]), + witnessed_test_view(2, &[1, 2, 3, 4, 5, 6, 8]), + witnessed_test_view(3, &[1, 2, 3, 4, 5, 7, 8]), + witnessed_test_view(4, &[1, 2, 3, 4, 6, 7, 8]), + witnessed_test_view(5, &[1, 2, 3, 5, 6, 7, 8]), + witnessed_test_view(6, &[1, 2, 4, 5, 6, 7, 8]), + ], + }; + let quorum = witnessed_close_group_quorum_for_transcript(&witnessed); + + assert_eq!(missing_witnessed_responder_views(&witnessed), 1); + assert_eq!(quorum, 4); + + let selection = + witnessed_quote_selection_or_error(&address, &witnessed, CLOSE_GROUP_SIZE, quorum) + .expect( + "one missing responder view should lower quorum and still select candidates", + ); + + assert_eq!( + selection + .quote_peers + .iter() + .map(|peer| peer.peer_id.as_bytes()[0]) + .collect::>(), + vec![1, 2, 3, 4, 5, 6, 7, 8] + ); + assert_eq!(selection.quorum, quorum); + } + + #[test] + fn witnessed_quote_selection_keeps_closest_set_with_median_voter_quorum() { + const MEDIAN_ISSUER_SEED: u8 = 7; + const FAR_SUPPORTING_VOTER_SEED: u8 = 20; + const UNSUCCESSFUL_SUPPORTING_VOTER_SEED: u8 = 21; + + let address = [0u8; 32]; + let quotes = vec![ + synthetic_quote(1, 10), + synthetic_quote(2, 20), + synthetic_quote(3, 30), + synthetic_quote(6, 50), + synthetic_quote(MEDIAN_ISSUER_SEED, 40), + synthetic_quote(8, 60), + synthetic_quote(9, 70), + synthetic_quote(FAR_SUPPORTING_VOTER_SEED, 80), + ]; + let mut voters_by_peer = HashMap::new(); + voters_by_peer.insert( + synthetic_peer(MEDIAN_ISSUER_SEED), + synthetic_voters(&[ + 1, + 2, + 3, + MEDIAN_ISSUER_SEED, + FAR_SUPPORTING_VOTER_SEED, + UNSUCCESSFUL_SUPPORTING_VOTER_SEED, + ]), + ); + + let quorum = witnessed_close_group_quorum(); + let selected = + select_witnessed_median_voter_quotes(quotes, &address, &voters_by_peer, quorum) + .expect("a supported close-group quote set should be selected"); + + assert_eq!(quote_peer_seeds(&selected), vec![1, 2, 3, 6, 7, 8, 9]); + let (median_peer_id, _) = + median_paid_quote_issuer(&selected).expect("selected quotes have a median"); + assert_eq!(median_peer_id, synthetic_peer(MEDIAN_ISSUER_SEED)); + assert!(voters_by_peer[&median_peer_id].len() >= quorum); + } + + #[test] + fn witnessed_quote_selection_uses_direct_median_witness_recognition() { + const MEDIAN_ISSUER_SEED: u8 = 7; + + let address = [0u8; 32]; + let quotes = vec![ + synthetic_quote(1, 10), + synthetic_quote(2, 20), + synthetic_quote(3, 30), + synthetic_quote(4, 50), + synthetic_quote(MEDIAN_ISSUER_SEED, 40), + synthetic_quote(8, 60), + synthetic_quote(9, 70), + ]; + let mut voters_by_peer = HashMap::new(); + voters_by_peer.insert( + synthetic_peer(MEDIAN_ISSUER_SEED), + synthetic_voters(&[20, 21, 22, 23, 24]), + ); + + let quorum = witnessed_close_group_quorum(); + let selected = + select_witnessed_median_voter_quotes(quotes, &address, &voters_by_peer, quorum) + .expect("direct witness recognition should support the paid median issuer"); + + let (median_peer_id, _) = + median_paid_quote_issuer(&selected).expect("selected quotes have a median"); + let selected_peers = selected + .iter() + .map(|(peer_id, _, _, _)| *peer_id) + .collect::>(); + assert_eq!(median_peer_id, synthetic_peer(MEDIAN_ISSUER_SEED)); + assert_eq!( + voters_by_peer[&median_peer_id] + .intersection(&selected_peers) + .count(), + 0, + "recognising witnesses need not also be selected quote issuers" + ); + assert_eq!(voters_by_peer[&median_peer_id].len(), quorum); + } + + #[test] + fn witnessed_quote_selection_rejects_median_without_witness_quorum() { + const MEDIAN_ISSUER_SEED: u8 = 7; + + let address = [0u8; 32]; + let quotes = vec![ + synthetic_quote(1, 10), + synthetic_quote(2, 20), + synthetic_quote(3, 30), + synthetic_quote(6, 50), + synthetic_quote(MEDIAN_ISSUER_SEED, 40), + synthetic_quote(8, 60), + synthetic_quote(9, 70), + synthetic_quote(10, 80), + ]; + let mut voters_by_peer = HashMap::new(); + voters_by_peer.insert( + synthetic_peer(MEDIAN_ISSUER_SEED), + synthetic_voters(&[1, 2, 3, 20]), + ); + + let selected = select_witnessed_median_voter_quotes( + quotes, + &address, + &voters_by_peer, + witnessed_close_group_quorum(), + ); + + assert!( + selected.is_none(), + "the selector must not return a paid quote set when fewer than the \ + witnessed median voter quorum recognised the paid median issuer" + ); + } + + #[test] + fn put_peers_prioritise_median_voters_without_reordering_quotes() { + const MEDIAN_ISSUER_SEED: u8 = 7; + + let quotes = vec![ + synthetic_quote(1, 10), + synthetic_quote(2, 20), + synthetic_quote(3, 30), + synthetic_quote(4, 50), + synthetic_quote(5, 60), + synthetic_quote(6, 70), + synthetic_quote(MEDIAN_ISSUER_SEED, 40), + ]; + let mut voters_by_peer = HashMap::new(); + voters_by_peer.insert( + synthetic_peer(MEDIAN_ISSUER_SEED), + synthetic_voters(&[3, 4, 5, 6, MEDIAN_ISSUER_SEED]), + ); + + let put_candidates = put_peers_from_seeds(&[1, 2, 3, 4, 5, 6, 7]); + let put_peers = put_peers_with_median_voters_first( + "es, + &put_candidates, + &voters_by_peer, + witnessed_close_group_quorum(), + ) + .expect("median voters should produce an ordered PUT set"); + + assert_eq!(quote_peer_seeds("es), vec![1, 2, 3, 4, 5, 6, 7]); + let (median_peer_id, _) = + median_paid_quote_issuer("es).expect("selected quotes have a median"); + assert_eq!(median_peer_id, synthetic_peer(MEDIAN_ISSUER_SEED)); + assert_eq!(put_peer_seeds(&put_peers), vec![3, 4, 5, 6, 7, 1, 2]); + } + #[test] fn filter_drops_only_bad_bindings_and_leaves_storer_acceptable_quotes() { let mut quotes = vec![ @@ -594,15 +1696,15 @@ mod tests { #[test] fn filter_drops_all_when_every_responder_is_bad() { - // The "all hostile" case: every over-queried peer returned a bad - // binding. The patch should leave us with zero quotes (not panic, - // not skip the filter, not return malformed quotes). The caller in - // get_store_quotes then surfaces InsufficientPeers. - let mut quotes: Vec<_> = (0..CLOSE_GROUP_SIZE * 2) + // The "all hostile" case: every peer returned a bad binding. The + // patch should leave us with zero quotes (not panic, not skip the + // filter, not return malformed quotes). The caller then surfaces + // InsufficientPeers. + let mut quotes: Vec<_> = (0..fault_tolerant_quote_query_count()) .map(|_| bad_quote_real()) .collect(); let dropped = drop_quotes_with_bad_bindings(&mut quotes); - assert_eq!(dropped, CLOSE_GROUP_SIZE * 2); + assert_eq!(dropped, fault_tolerant_quote_query_count()); assert!(quotes.is_empty()); } @@ -642,10 +1744,11 @@ mod tests { /// quote, and the storer's `validate_peer_bindings` rejected the /// entire close-group proof — burning the chunk's payment. /// - /// This test is the strongest proof the patch fixes that failure shape: + /// This test proves the fault-tolerant quote path still fixes that failure + /// shape: /// /// 1. We assemble `2x CLOSE_GROUP_SIZE` real ML-DSA-65 quotes — the same - /// over-query buffer the production code uses (line 93 of this file). + /// buffer merkle preflight and merkle-mode estimates retain for probes. /// 2. One of them is a *crossed-key* quote — the production failure shape. /// 3. We run an independent `storer_would_accept` check (re-derived from /// the storer spec, not from `quote_binding_is_valid`) over the @@ -653,14 +1756,14 @@ mod tests { /// storer **would** burn the chunk's payment if we proceeded unfiltered. /// 4. We run `drop_quotes_with_bad_bindings`. /// 5. We re-run `storer_would_accept` over the post-filter set; we confirm - /// EVERY remaining quote would be accepted, proving the patched - /// `ProofOfPayment` will not trigger the `validate_peer_bindings` - /// rejection that caused the Apr 30 outage. + /// EVERY remaining quote would be accepted, proving the filtered set + /// will not trigger the `validate_peer_bindings` rejection that caused + /// the Apr 30 outage. /// 6. We confirm the post-filter set has at least `CLOSE_GROUP_SIZE` /// quotes — the over-query buffer (2x) is sufficient. #[test] fn repro_apr_30_storer_would_have_rejected_pre_filter_and_accepts_post_filter() { - let over_query_count = CLOSE_GROUP_SIZE * 2; + let over_query_count = fault_tolerant_quote_query_count(); let mut quotes: Vec<_> = (0..over_query_count - 1) .map(|_| good_quote_real()) .collect(); @@ -688,7 +1791,7 @@ mod tests { assert!( storer_binding_would_accept(peer_id, quote), "every post-filter quote must be accepted by the storer spec — \ - this is what the patch guarantees: no more burned payments" + this is what the filter guarantees before any quote set is used" ); } @@ -696,7 +1799,7 @@ mod tests { assert!( quotes.len() >= CLOSE_GROUP_SIZE, "after filtering, at least CLOSE_GROUP_SIZE good quotes must remain \ - so we can build a non-rejected ProofOfPayment" + so a fault-tolerant probe can still return a full close group" ); } @@ -706,9 +1809,8 @@ mod tests { /// and return `InsufficientPeers`. #[test] fn filter_leaves_short_set_when_too_many_bad_peers() { - // Buffer is 2x; if more than half are bad, there's no way to refill. - let bad_count = CLOSE_GROUP_SIZE + 1; let good_count = CLOSE_GROUP_SIZE - 1; + let bad_count = fault_tolerant_quote_query_count() - good_count; let mut quotes: Vec<_> = std::iter::repeat_with(bad_quote_real) .take(bad_count) .chain(std::iter::repeat_with(good_quote_real).take(good_count)) @@ -825,61 +1927,6 @@ mod tests { ); } - // ============================================================ - // AIMD attribution: every error variant is classified correctly - // for `record_peer_outcome` so misbehaving peers are deprioritized - // and reachable-but-already-storing peers stay reputable. - // ============================================================ - - #[test] - fn aimd_success_for_ok_result() { - let (_, _, quote, _) = good_quote_real(); - let result: std::result::Result<(PaymentQuote, Amount), Error> = - Ok((quote.clone(), quote.price)); - assert!(quote_outcome_is_success(&result)); - } - - #[test] - fn aimd_success_for_already_stored() { - let result: std::result::Result<(PaymentQuote, Amount), Error> = Err(Error::AlreadyStored); - assert!( - quote_outcome_is_success(&result), - "an honest peer reporting already_stored is a benign outcome — \ - the peer is reachable and well-behaved, so the AIMD cache must \ - keep them at high reputation" - ); - } - - #[test] - fn aimd_failure_for_bad_quote_binding() { - let result: std::result::Result<(PaymentQuote, Amount), Error> = - Err(Error::BadQuoteBinding { - peer_id: "abc123".to_string(), - detail: "test".to_string(), - }); - assert!( - !quote_outcome_is_success(&result), - "BadQuoteBinding peers must be marked as failures so the AIMD \ - bootstrap cache learns to stop asking them on every upload" - ); - } - - #[test] - fn aimd_failure_for_network_and_timeout_and_protocol_and_serialization() { - for err in [ - Error::Network("net".to_string()), - Error::Timeout("to".to_string()), - Error::Protocol("proto".to_string()), - Error::Serialization("ser".to_string()), - ] { - let result: std::result::Result<(PaymentQuote, Amount), Error> = Err(err); - assert!( - !quote_outcome_is_success(&result), - "network-class errors must be classified as failures: {result:?}" - ); - } - } - /// Cross-validate the classifier's binding verdict against the /// independent storer-spec re-derivation across mixed responders. #[test] diff --git a/ant-core/src/data/error.rs b/ant-core/src/data/error.rs index 6212d823..e3ad8706 100644 --- a/ant-core/src/data/error.rs +++ b/ant-core/src/data/error.rs @@ -24,6 +24,25 @@ pub enum Error { #[error("protocol error: {0}")] Protocol(String), + /// A remote node rejected a chunk PUT at the application layer. + /// + /// The node responded with a structured `ProtocolError`, so the + /// transport round-trip succeeded — this is an application-level + /// rejection (payment-failed, storage/disk-full, quote-stale, + /// merkle-pool-rejected), NOT evidence the client is sending too + /// fast. It therefore classifies as `Outcome::ApplicationError` + /// (see `classify_error`) and does not push the adaptive store + /// limiter down. The structured `source` is preserved (rather than + /// flattened into `Protocol`) so the controller — and a future + /// full-node skip-list (V2-469) — can key on the reason. + #[error("remote PUT rejected for {address}: {source}")] + RemotePut { + /// Hex-encoded chunk address the rejection was for. + address: String, + /// The structured remote rejection reason. + source: ant_protocol::ProtocolError, + }, + /// Invalid data received. #[error("invalid data: {0}")] InvalidData(String), @@ -60,6 +79,14 @@ pub enum Error { #[error("encryption error: {0}")] Encryption(String), + /// The operation was cancelled by the caller rather than failing. + /// + /// Returned, for example, by streaming downloads when the consumer drops + /// its receiver (a client disconnect) — distinct from a transport + /// [`Error::Network`] failure, since nothing went wrong on the wire. + #[error("operation cancelled: {0}")] + Cancelled(String), + /// Data already exists on the network — no payment needed. #[error("already stored on network")] AlreadyStored, @@ -105,11 +132,29 @@ pub enum Error { failed_count: usize, /// Total number of chunks the upload was attempting to store. total_chunks: usize, + /// On-chain spend incurred so far. Boxed to keep the `Error` enum small + /// (the variant is returned in `Result` across the crate; without the + /// box the two cost fields would trip `clippy::result_large_err`). + spend: Box, /// Root cause description. reason: String, }, } +/// On-chain spend recorded on a [`Error::PartialUpload`]. +/// +/// A partial upload still spends money for the chunks it paid for. In the +/// single-node path payment precedes store, so this includes a failed wave's +/// chunks; surfacing it lets the caller report real spend rather than silently +/// dropping it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PartialUploadSpend { + /// Storage cost paid on-chain so far, in atto-tokens. + pub storage_cost_atto: String, + /// Gas cost paid on-chain so far, in wei. + pub gas_cost_wei: u128, +} + // ant-node is only linked when the `devnet` feature is on, so the // blanket `From` impl follows that gate. LocalDevnet maps node errors // to `Error::Network` via this conversion; default builds never see it. @@ -207,6 +252,15 @@ mod tests { assert_eq!(err.to_string(), "encryption error: decrypt failed"); } + #[test] + fn test_display_cancelled() { + let err = Error::Cancelled("download stream receiver dropped".to_string()); + assert_eq!( + err.to_string(), + "operation cancelled: download stream receiver dropped" + ); + } + #[test] fn test_display_insufficient_disk_space() { let err = Error::InsufficientDiskSpace("need 100 MB but only 10 MB available".to_string()); diff --git a/ant-core/src/data/mod.rs b/ant-core/src/data/mod.rs index cda3e31c..29f16e86 100644 --- a/ant-core/src/data/mod.rs +++ b/ant-core/src/data/mod.rs @@ -6,6 +6,7 @@ pub mod client; pub mod error; pub mod network; +pub mod peer_cache; pub use client::cache::ChunkCache; pub use client::{Client, ClientConfig}; @@ -24,8 +25,8 @@ pub use ant_protocol::{compute_address, DataChunk, XorName}; pub use client::batch::{finalize_batch_payment, PaidChunk, PaymentIntent, PreparedChunk}; pub use client::data::DataUploadResult; pub use client::file::{ - DownloadEvent, ExternalPaymentInfo, FileUploadResult, PreparedUpload, UploadCostEstimate, - UploadEvent, Visibility, + CostEstimateConfidence, DownloadEvent, ExternalPaymentInfo, FileUploadResult, PreparedUpload, + UploadCostEstimate, UploadEvent, Visibility, }; pub use client::merkle::{ finalize_merkle_batch, MerkleBatchPaymentResult, PaymentMode, PreparedMerkleBatch, diff --git a/ant-core/src/data/network.rs b/ant-core/src/data/network.rs index dc370ef0..42275452 100644 --- a/ant-core/src/data/network.rs +++ b/ant-core/src/data/network.rs @@ -5,7 +5,7 @@ use crate::data::error::{Error, Result}; use ant_protocol::transport::{ - CoreNodeConfig, IPDiversityConfig, MultiAddr, NodeMode, P2PNode, PeerId, + CoreNodeConfig, IPDiversityConfig, MultiAddr, NodeMode, P2PNode, PeerId, WitnessedCloseGroup, }; use ant_protocol::MAX_WIRE_MESSAGE_SIZE; use std::net::SocketAddr; @@ -131,6 +131,47 @@ impl Network { .collect()) } + /// Find a witnessed close-group transcript for a target address. + /// + /// The underlying DHT method returns the initial client K, each responder's + /// self-inclusive closest-K node view, and enough trusted node records for + /// callers to apply their own quorum and fallback policy. + /// + /// # Errors + /// + /// Returns an error if the DHT lookup itself fails. The returned transcript + /// may still be inconclusive; callers should evaluate it before payment. + pub async fn find_witnessed_close_group( + &self, + target: &[u8; 32], + count: usize, + ) -> Result { + self.find_witnessed_close_group_with_view_count(target, count, count) + .await + } + + /// Find a witnessed close-group transcript with wider responder views. + /// + /// `count` is the initial responder set size. `view_count` is the number + /// of closest nodes each responder view may contribute. + /// + /// # Errors + /// + /// Returns an error if the DHT lookup itself fails. The returned transcript + /// may still be inconclusive; callers should evaluate it before payment. + pub async fn find_witnessed_close_group_with_view_count( + &self, + target: &[u8; 32], + count: usize, + view_count: usize, + ) -> Result { + self.node + .dht() + .find_witnessed_close_group_with_view_count(target, count, view_count) + .await + .map_err(|e| Error::Network(format!("DHT witnessed close-group lookup failed: {e}"))) + } + /// Get all currently connected peers. pub async fn connected_peers(&self) -> Vec { self.node.connected_peers().await diff --git a/ant-core/src/data/peer_cache.rs b/ant-core/src/data/peer_cache.rs new file mode 100644 index 00000000..88bf5b92 --- /dev/null +++ b/ant-core/src/data/peer_cache.rs @@ -0,0 +1,924 @@ +//! Persistent client bootstrap peer cache. +//! +//! Client peer IDs are ephemeral, so this cache is not keyed by distance from +//! the local client. It remembers authenticated node peers that we have already +//! connected to directly during client runs, stores their dialable channel +//! addresses, and prefers retaining peers that are spread across the peer-id +//! keyspace. + +use crate::config; +use ant_protocol::transport::{IPDiversityConfig, MultiAddr, P2PNode, PeerId}; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::net::IpAddr; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; +use tracing::{debug, info, warn}; + +pub const CLIENT_PEER_CACHE_MAX_PEERS: usize = 50; + +/// Address families allowed when materializing cached startup candidates. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BootstrapAddressFilter { + /// Allow every dialable cached address. + All, + /// Allow only IPv4 cached addresses. + Ipv4Only, +} + +const CLIENT_PEER_CACHE_SCHEMA_VERSION: u32 = 1; +const CLIENT_PEER_CACHE_FILE_NAME: &str = "client_peer_cache.json"; +const CLIENT_PEER_CACHE_TEMP_SUFFIX: &str = "tmp"; +const DEFAULT_MAX_PER_EXACT_IP: usize = 2; +const SUBNET_LIMIT_K_DIVISOR: usize = 4; +const IPV4_SUBNET_PREFIX_OCTETS: usize = 3; +const IPV6_SUBNET_PREFIX_SEGMENTS: usize = 3; +const BITS_PER_BYTE: u8 = 8; +const PEER_ID_SECTOR_BITS: u8 = 4; +const PEER_ID_SECTOR_COUNT: usize = 1 << PEER_ID_SECTOR_BITS; +const PEER_ID_XOR_DISTANCE_BYTES: usize = 32; + +static TEMP_FILE_COUNTER: AtomicU64 = AtomicU64::new(0); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct ClientPeerCacheFile { + schema_version: u32, + peers: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct CachedPeer { + peer_id: PeerId, + direct_addresses: Vec, + first_connected_epoch_secs: u64, + last_connected_epoch_secs: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum SubnetKey { + V4([u8; IPV4_SUBNET_PREFIX_OCTETS]), + V6([u16; IPV6_SUBNET_PREFIX_SEGMENTS]), +} + +struct DiversityTracker { + exact_ip_counts: HashMap, + subnet_counts: HashMap, + max_per_ip: usize, + max_per_subnet: usize, +} + +/// Build the on-disk cache path for the client peer cache. +#[must_use] +pub fn cache_path() -> Option { + match config::data_dir() { + Ok(data_dir) => Some(data_dir.join(CLIENT_PEER_CACHE_FILE_NAME)), + Err(err) => { + warn!("client peer cache disabled: failed to resolve data dir: {err}"); + None + } + } +} + +/// Load cache addresses to try before configured bootstrap peers. +/// +/// Returns at most one direct address per cached peer. saorsa-core stops client +/// bootstrap after the client bootstrap target is reached, so every usable +/// cached peer is ordered before the configured fallback peers without forcing +/// all cached peers to be dialed on a healthy warm start. +#[must_use] +pub fn cached_bootstrap_peers(cache_path: &Path, k_value: usize) -> Vec { + cached_bootstrap_peers_with_filter(cache_path, k_value, BootstrapAddressFilter::All) +} + +/// Load cache addresses to try before configured bootstrap peers, applying an +/// address-family filter before choosing the first address for each peer. +#[must_use] +pub fn cached_bootstrap_peers_with_filter( + cache_path: &Path, + k_value: usize, + address_filter: BootstrapAddressFilter, +) -> Vec { + let Some(mut cache) = ClientPeerCacheFile::load_existing(cache_path) else { + return Vec::new(); + }; + let loaded_peer_count = cache.peers.len(); + let loaded_direct_address_count = cache.direct_address_count(); + let diversity_config = cache_diversity_config(); + let normalized = cache.normalize(&diversity_config, k_value); + if normalized { + cache.save(cache_path); + } + let bootstrap_addresses = + cache.bootstrap_addresses(CLIENT_PEER_CACHE_MAX_PEERS, address_filter); + info!( + path = %cache_path.display(), + cached_peers = loaded_peer_count, + direct_addresses = loaded_direct_address_count, + usable_cached_peers = cache.peers.len(), + bootstrap_candidates = bootstrap_addresses.len(), + "client peer bootstrap cache file found and loaded; cached peers available", + ); + bootstrap_addresses +} + +/// Select startup bootstrap peers. +/// +/// Cached peers are ordered first and configured bootstrap peers are appended +/// behind them. saorsa-core stops client bootstrap after the client bootstrap +/// target is reached, so configured peers are only reached when the cached +/// candidates do not produce enough successful connections. +#[must_use] +pub fn select_bootstrap_peers( + cached: impl IntoIterator, + configured: impl IntoIterator, +) -> Vec { + dedupe_bootstrap_peers(cached.into_iter().chain(configured)) +} + +fn dedupe_bootstrap_peers(addrs: impl IntoIterator) -> Vec { + let mut seen = HashSet::new(); + let mut deduped = Vec::new(); + + for addr in addrs { + if seen.insert(bootstrap_address_key(&addr)) { + deduped.push(addr); + } + } + + deduped +} + +/// Persist authenticated peers reached directly during this client run. +/// +/// A DHT Direct tag is not required here. The cache records dialable addresses +/// from currently live peer connections so the next client run can try peers it +/// actually reached. +pub async fn promote_connected_direct_peers(node: &P2PNode, cache_path: &Path, k_value: usize) { + let connected_peers = node.connected_peers().await; + if connected_peers.is_empty() { + return; + } + + let connected_peer_count = connected_peers.len(); + let mut cache = ClientPeerCacheFile::load(cache_path); + let diversity_config = cache_diversity_config(); + let now = now_epoch_secs(); + let mut changed = false; + let mut cacheable_peer_count = 0usize; + let mut cacheable_address_count = 0usize; + + for peer_id in connected_peers { + let Some(peer_info) = node.peer_info(&peer_id).await else { + continue; + }; + + let channel_addresses = peer_info + .addresses + .into_iter() + .filter(|addr| addr.dialable_socket_addr().is_some()) + .collect::>(); + if channel_addresses.is_empty() { + continue; + } + + cacheable_peer_count += 1; + cacheable_address_count += channel_addresses.len(); + + changed |= cache.upsert_connected_peer( + peer_id, + channel_addresses, + now, + &diversity_config, + k_value, + ); + } + + if changed { + info!( + path = %cache_path.display(), + connected_peers = connected_peer_count, + cacheable_peers = cacheable_peer_count, + cacheable_addresses = cacheable_address_count, + cached_peers = cache.peers.len(), + direct_addresses = cache.direct_address_count(), + "client peer bootstrap cache updated from live connected peers", + ); + cache.save(cache_path); + } +} + +/// The cache applies the default k-bucket IP diversity policy rather than the +/// client's permissive routing-table setting. This keeps the persisted +/// bootstrap surface from collapsing onto one IP or subnet. +#[must_use] +fn cache_diversity_config() -> IPDiversityConfig { + IPDiversityConfig::default() +} + +impl BootstrapAddressFilter { + fn allows(self, addr: &MultiAddr) -> bool { + match self { + Self::All => addr.dialable_socket_addr().is_some(), + Self::Ipv4Only => addr + .dialable_socket_addr() + .is_some_and(|socket| socket.is_ipv4()), + } + } +} + +impl ClientPeerCacheFile { + fn empty() -> Self { + Self { + schema_version: CLIENT_PEER_CACHE_SCHEMA_VERSION, + peers: Vec::new(), + } + } + + fn load(path: &Path) -> Self { + Self::load_existing(path).unwrap_or_else(Self::empty) + } + + fn load_existing(path: &Path) -> Option { + let Ok(data) = std::fs::read_to_string(path) else { + return None; + }; + + match serde_json::from_str::(&data) { + Ok(cache) if cache.schema_version == CLIENT_PEER_CACHE_SCHEMA_VERSION => Some(cache), + Ok(cache) => { + debug!( + path = %path.display(), + schema_version = cache.schema_version, + "ignoring client peer cache with unsupported schema version", + ); + None + } + Err(err) => { + warn!( + path = %path.display(), + "ignoring unreadable client peer cache: {err}", + ); + None + } + } + } + + fn direct_address_count(&self) -> usize { + self.peers + .iter() + .map(|peer| peer.direct_addresses.len()) + .sum() + } + + fn save(&self, path: &Path) { + if let Some(parent) = path.parent() { + if let Err(err) = std::fs::create_dir_all(parent) { + warn!( + path = %path.display(), + "failed to create client peer cache directory: {err}", + ); + return; + } + } + + let data = match serde_json::to_vec_pretty(self) { + Ok(data) => data, + Err(err) => { + warn!("failed to serialize client peer cache: {err}"); + return; + } + }; + + let temp_path = temp_path_for(path); + if let Err(err) = std::fs::write(&temp_path, data) { + warn!( + path = %temp_path.display(), + "failed to write client peer cache temp file: {err}", + ); + return; + } + + #[cfg(windows)] + if path.exists() { + if let Err(err) = std::fs::remove_file(path) { + warn!( + path = %path.display(), + "failed to replace existing client peer cache: {err}", + ); + let _ = std::fs::remove_file(&temp_path); + return; + } + } + + if let Err(err) = std::fs::rename(&temp_path, path) { + warn!( + from = %temp_path.display(), + to = %path.display(), + "failed to commit client peer cache: {err}", + ); + let _ = std::fs::remove_file(temp_path); + } + } + + fn upsert_connected_peer( + &mut self, + peer_id: PeerId, + direct_addresses: Vec, + now: u64, + diversity_config: &IPDiversityConfig, + k_value: usize, + ) -> bool { + let direct_addresses = sanitize_direct_addresses(peer_id, direct_addresses); + if direct_addresses.is_empty() { + return false; + } + + let before = self.peers.clone(); + if let Some(existing) = self.peers.iter_mut().find(|peer| peer.peer_id == peer_id) { + existing.direct_addresses = direct_addresses; + existing.last_connected_epoch_secs = now; + } else { + self.peers.push(CachedPeer { + peer_id, + direct_addresses, + first_connected_epoch_secs: now, + last_connected_epoch_secs: now, + }); + } + + self.normalize(diversity_config, k_value); + self.peers != before + } + + fn normalize(&mut self, diversity_config: &IPDiversityConfig, k_value: usize) -> bool { + let before = self.peers.clone(); + self.peers.retain(|peer| !peer.direct_addresses.is_empty()); + self.peers.sort_by(|left, right| { + right + .last_connected_epoch_secs + .cmp(&left.last_connected_epoch_secs) + .then_with(|| left.peer_id.to_hex().cmp(&right.peer_id.to_hex())) + }); + + let mut candidates = Vec::with_capacity(self.peers.len()); + let mut seen_peers = HashSet::new(); + for peer in self.peers.drain(..) { + if seen_peers.insert(peer.peer_id) { + candidates.push(peer); + } + } + + let mut tracker = DiversityTracker::new(diversity_config, k_value); + let mut normalized = Vec::with_capacity(CLIENT_PEER_CACHE_MAX_PEERS); + + while normalized.len() < CLIENT_PEER_CACHE_MAX_PEERS { + let Some(best_index) = + select_peer_id_diverse_candidate(&candidates, &normalized, &tracker) + else { + break; + }; + let peer = candidates.swap_remove(best_index); + tracker.record_peer(&peer); + normalized.push(peer); + } + + self.peers = normalized; + self.peers != before + } + + fn bootstrap_addresses( + &self, + limit: usize, + address_filter: BootstrapAddressFilter, + ) -> Vec { + let mut sectors = (0..PEER_ID_SECTOR_COUNT) + .map(|_| Vec::new()) + .collect::>>(); + + for peer in &self.peers { + sectors[peer_id_sector(peer.peer_id)].push(peer); + } + + let mut positions = [0usize; PEER_ID_SECTOR_COUNT]; + let mut addresses = Vec::with_capacity(self.peers.len().min(limit)); + + loop { + let mut advanced_this_round = false; + for sector in 0..PEER_ID_SECTOR_COUNT { + let position = positions[sector]; + let Some(peer) = sectors[sector].get(position) else { + continue; + }; + positions[sector] += 1; + advanced_this_round = true; + if let Some(addr) = peer + .direct_addresses + .iter() + .find(|addr| address_filter.allows(addr)) + { + addresses.push(addr.clone()); + } + if addresses.len() >= limit { + return addresses; + } + } + if !advanced_this_round { + return addresses; + } + } + } +} + +fn select_peer_id_diverse_candidate( + candidates: &[CachedPeer], + selected: &[CachedPeer], + tracker: &DiversityTracker, +) -> Option { + let mut best_index = None; + + for (candidate_index, candidate) in candidates.iter().enumerate() { + if !tracker.can_admit_peer(candidate) { + continue; + } + let Some(current_best_index) = best_index else { + best_index = Some(candidate_index); + continue; + }; + let current_best = &candidates[current_best_index]; + if prefer_peer_id_candidate(candidate, current_best, selected) { + best_index = Some(candidate_index); + } + } + + best_index +} + +fn prefer_peer_id_candidate( + candidate: &CachedPeer, + current_best: &CachedPeer, + selected: &[CachedPeer], +) -> bool { + peer_id_spread_score(candidate, selected) + .cmp(&peer_id_spread_score(current_best, selected)) + .then_with(|| { + candidate + .last_connected_epoch_secs + .cmp(¤t_best.last_connected_epoch_secs) + }) + .then_with(|| { + current_best + .peer_id + .to_hex() + .cmp(&candidate.peer_id.to_hex()) + }) + .is_gt() +} + +fn peer_id_spread_score( + candidate: &CachedPeer, + selected: &[CachedPeer], +) -> Option<[u8; PEER_ID_XOR_DISTANCE_BYTES]> { + selected + .iter() + .map(|peer| peer_id_xor_distance(candidate.peer_id, peer.peer_id)) + .min() +} + +fn peer_id_xor_distance(left: PeerId, right: PeerId) -> [u8; PEER_ID_XOR_DISTANCE_BYTES] { + let left_bytes = left.as_bytes(); + let right_bytes = right.as_bytes(); + let mut distance = [0u8; PEER_ID_XOR_DISTANCE_BYTES]; + for (index, byte) in distance.iter_mut().enumerate() { + *byte = left_bytes[index] ^ right_bytes[index]; + } + distance +} + +impl DiversityTracker { + fn new(config: &IPDiversityConfig, k_value: usize) -> Self { + Self { + exact_ip_counts: HashMap::new(), + subnet_counts: HashMap::new(), + max_per_ip: config.max_per_ip.unwrap_or(DEFAULT_MAX_PER_EXACT_IP), + max_per_subnet: config + .max_per_subnet + .unwrap_or_else(|| default_subnet_limit(k_value)), + } + } + + fn can_admit_peer(&self, peer: &CachedPeer) -> bool { + let Some((ip_set, subnet_set)) = peer_diversity_sets(peer) else { + return false; + }; + + for ip in &ip_set { + if self.exact_ip_counts.get(ip).copied().unwrap_or_default() >= self.max_per_ip { + return false; + } + } + + for subnet in &subnet_set { + if self.subnet_counts.get(subnet).copied().unwrap_or_default() >= self.max_per_subnet { + return false; + } + } + + true + } + + fn record_peer(&mut self, peer: &CachedPeer) { + let Some((ip_set, subnet_set)) = peer_diversity_sets(peer) else { + return; + }; + + for ip in ip_set { + *self.exact_ip_counts.entry(ip).or_default() += 1; + } + for subnet in subnet_set { + *self.subnet_counts.entry(subnet).or_default() += 1; + } + } +} + +fn peer_diversity_sets(peer: &CachedPeer) -> Option<(HashSet, HashSet)> { + let ip_set = peer + .direct_addresses + .iter() + .filter_map(|addr| { + addr.dialable_socket_addr() + .map(|socket| canonical_ip(socket.ip())) + }) + .collect::>(); + + if ip_set.is_empty() { + return None; + } + + let subnet_set = ip_set + .iter() + .map(|ip| subnet_key(*ip)) + .collect::>(); + + Some((ip_set, subnet_set)) +} + +fn sanitize_direct_addresses(peer_id: PeerId, direct_addresses: Vec) -> Vec { + let mut seen = HashSet::new(); + let mut sanitized = Vec::new(); + + for addr in direct_addresses { + if addr.dialable_socket_addr().is_none() { + continue; + } + let addr = addr.with_peer_id(peer_id); + if seen.insert(addr.to_string()) { + sanitized.push(addr); + } + } + + sanitized +} + +fn bootstrap_address_key(addr: &MultiAddr) -> String { + addr.dialable_socket_addr() + .map(|socket| socket.to_string()) + .unwrap_or_else(|| addr.to_string()) +} + +fn default_subnet_limit(k_value: usize) -> usize { + std::cmp::max(k_value / SUBNET_LIMIT_K_DIVISOR, 1) +} + +fn subnet_key(ip: IpAddr) -> SubnetKey { + match ip { + IpAddr::V4(ip) => { + let octets = ip.octets(); + SubnetKey::V4([octets[0], octets[1], octets[IPV4_SUBNET_PREFIX_OCTETS - 1]]) + } + IpAddr::V6(ip) => { + let segments = ip.segments(); + SubnetKey::V6([ + segments[0], + segments[1], + segments[IPV6_SUBNET_PREFIX_SEGMENTS - 1], + ]) + } + } +} + +fn canonical_ip(ip: IpAddr) -> IpAddr { + match ip { + IpAddr::V4(ip) => IpAddr::V4(ip), + IpAddr::V6(ip) => ip + .to_ipv4_mapped() + .map(IpAddr::V4) + .unwrap_or(IpAddr::V6(ip)), + } +} + +fn peer_id_sector(peer_id: PeerId) -> usize { + let sector_shift = BITS_PER_BYTE - PEER_ID_SECTOR_BITS; + usize::from(peer_id.as_bytes()[0] >> sector_shift) +} + +fn temp_path_for(path: &Path) -> PathBuf { + let counter = TEMP_FILE_COUNTER.fetch_add(1, Ordering::Relaxed); + let process_id = std::process::id(); + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(CLIENT_PEER_CACHE_FILE_NAME); + path.with_file_name(format!( + ".{file_name}.{process_id}.{counter}.{CLIENT_PEER_CACHE_TEMP_SUFFIX}" + )) +} + +fn now_epoch_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .unwrap_or_default() +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr}; + + const TEST_PEER_ID_LEN: usize = 32; + const TEST_K_VALUE: usize = 20; + const FIRST_PORT: u16 = 10_000; + const TEST_NOW: u64 = 1_000_000; + const EXACT_IP_ATTEMPTS: u8 = 3; + const SUBNET_ATTEMPTS: u8 = 6; + const BOOTSTRAP_ROUND_ROBIN_TEST_LIMIT: usize = 6; + + fn peer_id(byte: u8) -> PeerId { + peer_id_with_prefix(byte, 0) + } + + fn peer_id_with_prefix(first_byte: u8, second_byte: u8) -> PeerId { + let mut bytes = [0u8; TEST_PEER_ID_LEN]; + bytes[0] = first_byte; + bytes[1] = second_byte; + PeerId::from_bytes(bytes) + } + + fn direct_addr(ip: IpAddr, port: u16) -> MultiAddr { + MultiAddr::quic(SocketAddr::new(ip, port)) + } + + fn v4(a: u8, b: u8, c: u8, d: u8) -> IpAddr { + IpAddr::V4(Ipv4Addr::new(a, b, c, d)) + } + + fn v6(first_segment: u16, host: u16) -> IpAddr { + IpAddr::V6(Ipv6Addr::new(first_segment, 0, 0, 0, 0, 0, 0, host)) + } + + #[test] + fn cache_prefers_peer_id_spread_over_recency_when_full() { + let mut cache = ClientPeerCacheFile::empty(); + let diversity = IPDiversityConfig::permissive(); + + let old_distant_peer = peer_id_with_prefix(u8::MAX, 0); + cache.peers.push(CachedPeer { + peer_id: old_distant_peer, + direct_addresses: vec![direct_addr(v4(203, 0, 113, 1), FIRST_PORT)], + first_connected_epoch_secs: TEST_NOW, + last_connected_epoch_secs: TEST_NOW, + }); + + for idx in 0..CLIENT_PEER_CACHE_MAX_PEERS { + let peer = peer_id_with_prefix(0, idx as u8); + let addr = direct_addr( + v4(1, 0, idx as u8, 1), + FIRST_PORT + u16::try_from(idx).unwrap(), + ); + let connected_epoch_secs = TEST_NOW + u64::try_from(idx).unwrap() + 1; + cache.peers.push(CachedPeer { + peer_id: peer, + direct_addresses: vec![addr.with_peer_id(peer)], + first_connected_epoch_secs: connected_epoch_secs, + last_connected_epoch_secs: connected_epoch_secs, + }); + } + + cache.normalize(&diversity, TEST_K_VALUE); + + assert_eq!(cache.peers.len(), CLIENT_PEER_CACHE_MAX_PEERS); + assert!( + cache + .peers + .iter() + .any(|peer| peer.peer_id == old_distant_peer), + "old distant peer must be retained ahead of one newer clustered peer" + ); + assert_eq!( + cache + .peers + .iter() + .filter(|peer| peer.peer_id.as_bytes()[0] == 0) + .count(), + CLIENT_PEER_CACHE_MAX_PEERS - 1 + ); + } + + #[test] + fn cache_applies_exact_ip_limit() { + let mut cache = ClientPeerCacheFile::empty(); + let diversity = IPDiversityConfig::default(); + + for idx in 0..EXACT_IP_ATTEMPTS { + cache.upsert_connected_peer( + peer_id(idx), + vec![direct_addr(v4(203, 0, 113, 1), FIRST_PORT + u16::from(idx))], + TEST_NOW + u64::from(idx), + &diversity, + TEST_K_VALUE, + ); + } + + assert_eq!(cache.peers.len(), DEFAULT_MAX_PER_EXACT_IP); + assert!(cache.peers.iter().any(|peer| peer.peer_id == peer_id(2))); + assert!(cache.peers.iter().any(|peer| peer.peer_id == peer_id(1))); + assert!(!cache.peers.iter().any(|peer| peer.peer_id == peer_id(0))); + } + + #[test] + fn cache_applies_subnet_limit() { + let mut cache = ClientPeerCacheFile::empty(); + let diversity = IPDiversityConfig::default(); + + for idx in 0..SUBNET_ATTEMPTS { + cache.upsert_connected_peer( + peer_id(idx), + vec![direct_addr( + v4(198, 51, 100, idx), + FIRST_PORT + u16::from(idx), + )], + TEST_NOW + u64::from(idx), + &diversity, + TEST_K_VALUE, + ); + } + + assert_eq!(cache.peers.len(), default_subnet_limit(TEST_K_VALUE)); + assert!(cache.peers.iter().any(|peer| peer.peer_id == peer_id(5))); + assert!(!cache.peers.iter().any(|peer| peer.peer_id == peer_id(0))); + } + + #[test] + fn cache_rejects_peers_without_dialable_direct_addresses() { + let mut cache = ClientPeerCacheFile::empty(); + let diversity = IPDiversityConfig::permissive(); + + let changed = + cache.upsert_connected_peer(peer_id(1), Vec::new(), TEST_NOW, &diversity, TEST_K_VALUE); + + assert!(!changed); + assert!(cache.peers.is_empty()); + } + + #[test] + fn cached_bootstrap_addresses_round_robin_peer_id_sectors() { + let mut cache = ClientPeerCacheFile::empty(); + let diversity = IPDiversityConfig::permissive(); + + cache.upsert_connected_peer( + peer_id(0x01), + vec![direct_addr(v4(1, 0, 0, 1), FIRST_PORT)], + TEST_NOW, + &diversity, + TEST_K_VALUE, + ); + cache.upsert_connected_peer( + peer_id(0x02), + vec![direct_addr(v4(1, 0, 0, 2), FIRST_PORT + 1)], + TEST_NOW + 1, + &diversity, + TEST_K_VALUE, + ); + cache.upsert_connected_peer( + peer_id(0xf0), + vec![direct_addr(v6(0x2001, 1), FIRST_PORT + 2)], + TEST_NOW + 2, + &diversity, + TEST_K_VALUE, + ); + + let addresses = cache.bootstrap_addresses( + BOOTSTRAP_ROUND_ROBIN_TEST_LIMIT, + BootstrapAddressFilter::All, + ); + + assert_eq!(addresses.len(), 3); + assert_eq!( + addresses[0].dialable_socket_addr().unwrap().ip(), + v4(1, 0, 0, 2) + ); + assert_eq!( + addresses[1].dialable_socket_addr().unwrap().ip(), + v6(0x2001, 1) + ); + assert_eq!( + addresses[2].dialable_socket_addr().unwrap().ip(), + v4(1, 0, 0, 1) + ); + } + + #[test] + fn cached_addresses_are_stored_with_peer_id_suffix() { + let mut cache = ClientPeerCacheFile::empty(); + let diversity = IPDiversityConfig::permissive(); + + cache.upsert_connected_peer( + peer_id(1), + vec![direct_addr(v4(203, 0, 113, 10), FIRST_PORT)], + TEST_NOW, + &diversity, + TEST_K_VALUE, + ); + + let addr = cache.peers[0].direct_addresses[0].clone(); + assert_eq!(addr.peer_id(), Some(&peer_id(1))); + } + + #[test] + fn cached_bootstrap_addresses_respect_ipv4_only_filter() { + let mut cache = ClientPeerCacheFile::empty(); + let diversity = IPDiversityConfig::permissive(); + + let peer = peer_id(1); + let ipv6_addr = direct_addr(v6(0x2001, 1), FIRST_PORT); + let ipv4_addr = direct_addr(v4(203, 0, 113, 10), FIRST_PORT + 1); + cache.upsert_connected_peer( + peer, + vec![ipv6_addr.clone(), ipv4_addr.clone()], + TEST_NOW, + &diversity, + TEST_K_VALUE, + ); + + let all_addresses = + cache.bootstrap_addresses(CLIENT_PEER_CACHE_MAX_PEERS, BootstrapAddressFilter::All); + assert_eq!(all_addresses, vec![ipv6_addr.with_peer_id(peer)]); + + let ipv4_addresses = cache.bootstrap_addresses( + CLIENT_PEER_CACHE_MAX_PEERS, + BootstrapAddressFilter::Ipv4Only, + ); + assert_eq!(ipv4_addresses, vec![ipv4_addr.with_peer_id(peer)]); + } + + #[test] + fn select_bootstrap_peers_orders_configured_after_cached_fallback() { + let first_cached = MultiAddr::quic(SocketAddr::new(v4(203, 0, 113, 20), FIRST_PORT)) + .with_peer_id(peer_id(1)); + let second_cached = MultiAddr::quic(SocketAddr::new(v4(203, 0, 113, 21), FIRST_PORT)) + .with_peer_id(peer_id(2)); + let configured = MultiAddr::quic(SocketAddr::new(v4(203, 0, 113, 22), FIRST_PORT)); + + let selected = select_bootstrap_peers( + vec![first_cached.clone(), second_cached.clone()], + vec![configured.clone()], + ); + + assert_eq!(selected, vec![first_cached, second_cached, configured]); + } + + #[test] + fn select_bootstrap_peers_uses_configured_when_cache_empty() { + let configured = MultiAddr::quic(SocketAddr::new(v4(203, 0, 113, 21), FIRST_PORT)); + + let selected = select_bootstrap_peers(Vec::new(), vec![configured.clone()]); + + assert_eq!(selected, vec![configured]); + } + + #[test] + fn cached_bootstrap_peers_include_all_usable_cached_peers() { + let mut cache = ClientPeerCacheFile::empty(); + let diversity = IPDiversityConfig::permissive(); + + for idx in 0..BOOTSTRAP_ROUND_ROBIN_TEST_LIMIT + 1 { + cache.upsert_connected_peer( + peer_id(idx as u8), + vec![direct_addr( + v4(1, 0, idx as u8, 1), + FIRST_PORT + u16::try_from(idx).unwrap(), + )], + TEST_NOW + u64::try_from(idx).unwrap(), + &diversity, + TEST_K_VALUE, + ); + } + + let addresses = + cache.bootstrap_addresses(CLIENT_PEER_CACHE_MAX_PEERS, BootstrapAddressFilter::All); + + assert_eq!(addresses.len(), BOOTSTRAP_ROUND_ROBIN_TEST_LIMIT + 1); + } +} diff --git a/ant-core/src/node/devnet.rs b/ant-core/src/node/devnet.rs index 969fc262..e5d87f52 100644 --- a/ant-core/src/node/devnet.rs +++ b/ant-core/src/node/devnet.rs @@ -6,6 +6,7 @@ use crate::data::client::ClientConfig; use crate::data::error::{Error, Result}; use crate::data::Client; +use ant_node::core::MultiAddr as NodeMultiAddr; use ant_node::devnet::{Devnet, DevnetConfig}; use ant_protocol::evm::testnet::Testnet; use ant_protocol::evm::{Network as EvmNetwork, Wallet}; @@ -63,7 +64,7 @@ impl LocalDevnet { .await .map_err(|e| Error::Network(format!("devnet start failed: {e}")))?; - let bootstrap = devnet.bootstrap_addrs(); + let bootstrap = convert_bootstrap_addrs(devnet.bootstrap_addrs())?; let evm_info = DevnetEvmInfo { rpc_url, @@ -208,6 +209,20 @@ fn extract_custom_network_info(network: &EvmNetwork) -> Result<(String, String, } } +fn convert_bootstrap_addrs(addrs: Vec) -> Result> { + addrs + .into_iter() + .map(|addr| { + let addr_text = addr.to_string(); + addr_text.parse::().map_err(|e| { + Error::Config(format!( + "failed to convert devnet bootstrap address {addr_text}: {e}" + )) + }) + }) + .collect() +} + /// Get a simple ISO-8601 timestamp string. fn current_timestamp() -> String { let duration = SystemTime::now() diff --git a/ant-core/tests/e2e_bootstrap_cache.rs b/ant-core/tests/e2e_bootstrap_cache.rs deleted file mode 100644 index 80303ef9..00000000 --- a/ant-core/tests/e2e_bootstrap_cache.rs +++ /dev/null @@ -1,143 +0,0 @@ -//! E2E tests for BootstrapManager cache population from real peer interactions. -//! -//! Proves that client-side uploads and downloads feed the BootstrapManager -//! cache via `add_discovered_peer` + `update_peer_metrics`, so that subsequent -//! cold-starts can load quality-scored peers beyond the bundled bootstrap set. -//! -//! ## Why the assertion is "cache grew", not "cache >= 10" -//! -//! saorsa-core gates `add_peer` through two independent Sybil mechanisms: -//! -//! 1. `BootstrapIpLimiter::can_accept` — the IP-diversity limiter. When the -//! node is built with `allow_loopback = true` (as `MiniTestnet` does), -//! this returns early for loopback IPs, so it is NOT the bottleneck here. -//! 2. `JoinRateLimiter::check_join_allowed` — the temporal rate limiter. -//! Defaults cap inserts at 3 per /24 subnet per hour and are NOT exempt -//! for loopback (`saorsa-core/src/rate_limit.rs:254` has no `is_loopback` -//! branch). All testnet nodes bind to `127.0.0.1`, so all ~11 available -//! peers fall in the single `127.0.0.0/24` bucket — the first 3 land in -//! the cache, the rest are rejected with `Subnet24LimitExceeded`. -//! -//! In production, peers span many /24s (typically one per ASN), so the /24 -//! rate limit is never the binding constraint and crossing -//! `min_peers_to_save = 10` is straightforward. -//! -//! Asserting `after > before` is sufficient proof that the client library -//! correctly wires `add_discovered_peer` and `update_peer_metrics` into the -//! upload (and, transitively, download) paths. The threshold-crossing + -//! persistence behavior is an upstream contract covered by saorsa-transport's -//! own tests. - -#![allow(clippy::unwrap_used, clippy::expect_used)] - -mod support; - -use ant_core::data::{Client, ClientConfig}; -use bytes::Bytes; -use serial_test::serial; -use std::sync::Arc; -use support::MiniTestnet; - -const BOOTSTRAP_CACHE_TEST_NODES: usize = 12; - -#[tokio::test(flavor = "multi_thread")] -#[serial] -async fn test_bootstrap_cache_grows_after_client_activity() { - let testnet = MiniTestnet::start(BOOTSTRAP_CACHE_TEST_NODES).await; - let node = testnet.node(3).expect("Node 3 should exist"); - - let client = Client::from_node(Arc::clone(&node), ClientConfig::default()) - .with_wallet(testnet.wallet().clone()); - - let before = node.cached_peer_count().await; - - let content = Bytes::from("bootstrap-cache e2e payload"); - let address = client - .chunk_put(content.clone()) - .await - .expect("chunk_put should succeed with payment"); - - // The GET exercises the download-side hook (chunk_get_from_peer), which - // would silently break if record_peer_outcome's signature drifted from - // what chunk.rs expects. The assertion here is just that the round-trip - // works — cache growth from the GET itself is capped by the /24 rate - // limiter which saturated during the PUT. - let retrieved = client - .chunk_get(&address) - .await - .expect("chunk_get should succeed") - .expect("chunk should be retrievable"); - assert_eq!(retrieved.content.as_ref(), content.as_ref()); - - let after = node.cached_peer_count().await; - assert!( - after > before, - "cache should grow after peer interactions: before={before} after={after}" - ); - - drop(client); - testnet.teardown().await; -} - -/// Cold-start-from-disk round-trip. -/// -/// ## What this proves -/// -/// - A populated `BootstrapManager` cache with ≥ `min_peers_to_save` peers -/// is persisted to disk on `save()`. -/// - A *fresh* `BootstrapManager` constructed against the same `cache_dir` -/// reloads the persisted peers on startup. -/// -/// Together with `test_bootstrap_cache_grows_after_client_activity` above -/// (which exercises the add-during-activity hook), this closes the loop on -/// the V2-202 value prop: cold-start clients reload real peers from disk. -/// -/// ## Why `add_peer_trusted` and not `add_discovered_peer` -/// -/// `add_discovered_peer` goes through `BootstrapManager::add_peer`, which -/// runs both the IP-diversity limiter and the temporal `JoinRateLimiter`. -/// The latter caps inserts at 3 per /24 subnet per hour and has no -/// loopback exemption. A real test that populates 15 peers through that -/// path would need peers on distinct /24s — not practical on a single-host -/// testnet. `add_peer_trusted` skips both limiters and talks to the same -/// underlying `BootstrapCache::add_seed` that our hooks ultimately feed, -/// so the persistence path exercised is identical to production's. -#[tokio::test] -async fn test_bootstrap_cache_roundtrip_through_disk() { - use saorsa_core::{BootstrapConfig, BootstrapManager}; - use std::net::{IpAddr, Ipv4Addr, SocketAddr}; - - let cache_dir = tempfile::TempDir::new().expect("create temp cache dir"); - let config = BootstrapConfig { - cache_dir: cache_dir.path().to_path_buf(), - ..BootstrapConfig::default() - }; - - // Populate with peers on distinct /24s (cosmetic — add_peer_trusted - // skips rate limits — but keeps the data realistic if saorsa-transport - // ever tightens its invariants). - let peer_count = 15; - { - let mgr = BootstrapManager::with_config(config.clone()) - .await - .expect("construct populating BootstrapManager"); - for i in 0..peer_count { - let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(203, 0, 113, i as u8 + 1)), 9000); - mgr.add_peer_trusted(&addr, vec![addr]).await; - } - assert_eq!(mgr.peer_count().await, peer_count, "in-memory populate"); - mgr.save() - .await - .expect("save should succeed above threshold"); - } - - // Fresh manager, same cache_dir: peers should be reloaded. - let reloaded = BootstrapManager::with_config(config) - .await - .expect("construct reloading BootstrapManager"); - let reloaded_count = reloaded.peer_count().await; - assert_eq!( - reloaded_count, peer_count, - "all {peer_count} peers should reload from disk, got {reloaded_count}" - ); -} diff --git a/ant-core/tests/e2e_cost_estimate.rs b/ant-core/tests/e2e_cost_estimate.rs index 2c5a3766..40cb299b 100644 --- a/ant-core/tests/e2e_cost_estimate.rs +++ b/ant-core/tests/e2e_cost_estimate.rs @@ -11,7 +11,7 @@ mod support; use ant_core::data::client::merkle::PaymentMode; -use ant_core::data::{Client, ClientConfig}; +use ant_core::data::{Client, ClientConfig, CostEstimateConfidence}; use serial_test::serial; use std::io::Write; use std::path::{Path, PathBuf}; @@ -257,3 +257,87 @@ async fn test_estimate_rejects_tiny_files() { .await; assert!(result.is_err(), "Estimate should fail for files < 3 bytes"); } + +/// Regression for the partial-sample case (issue #114): re-estimating a +/// fully-stored file with more chunks than the sample cap must return `Ok` +/// flagged `AllSamplesAlreadyStoredIncomplete`, not `CostEstimationInconclusive`. +/// +/// Every sampled chunk is already stored, but the sample cannot cover the whole +/// file, so the old code errored and left consumers (the GUI) with no estimate. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn test_estimate_all_stored_partial_sample_is_incomplete() { + let testnet = MiniTestnet::start(10).await; + let node = testnet.node(3).expect("Node 3 should exist"); + let client = Client::from_node(Arc::clone(&node), ClientConfig::default()) + .with_wallet(testnet.wallet().clone()); + + let work_dir = TempDir::new().expect("create work dir"); + // ~30 MB -> ~8 chunks at MAX_CHUNK_SIZE (4,190,208 B), comfortably above the + // 5-address sample cap so the sample cannot cover every chunk. + let path = create_test_file( + work_dir.path(), + 30 * 1024 * 1024, + "partial.bin", + 0xCAFE_0001, + ); + + // Upload so every chunk is stored on the network. + client + .file_upload_with_mode(&path, PaymentMode::Auto) + .await + .expect("upload should succeed"); + + // Re-estimate the same file: every sampled chunk is now AlreadyStored. + let estimate = client + .estimate_upload_cost(&path, PaymentMode::Auto, None) + .await + .expect("estimate must return Ok for a partially-sampled all-stored file"); + + assert!( + estimate.chunk_count > 5, + "test file must exceed the sample cap to exercise the partial-sample path, got {} chunks", + estimate.chunk_count + ); + assert_eq!(estimate.storage_cost_atto, "0"); + assert_eq!( + estimate.confidence, + CostEstimateConfidence::AllSamplesAlreadyStoredIncomplete + ); +} + +/// A fully-stored file small enough to be sampled in full returns the exact +/// zero-cost estimate tagged `VerifiedAllAlreadyStored` (the provably-free case). +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn test_estimate_all_stored_full_sample_is_verified() { + let testnet = MiniTestnet::start(10).await; + let node = testnet.node(3).expect("Node 3 should exist"); + let client = Client::from_node(Arc::clone(&node), ClientConfig::default()) + .with_wallet(testnet.wallet().clone()); + + let work_dir = TempDir::new().expect("create work dir"); + // ~4 KB -> 3 chunks, within the sample cap so every chunk is sampled. + let path = create_test_file(work_dir.path(), 4096, "fully_stored.bin", 0xCAFE_0002); + + client + .file_upload_with_mode(&path, PaymentMode::Auto) + .await + .expect("upload should succeed"); + + let estimate = client + .estimate_upload_cost(&path, PaymentMode::Auto, None) + .await + .expect("estimate should succeed"); + + assert!( + estimate.chunk_count <= 5, + "small file should be within the sample cap, got {} chunks", + estimate.chunk_count + ); + assert_eq!(estimate.storage_cost_atto, "0"); + assert_eq!( + estimate.confidence, + CostEstimateConfidence::VerifiedAllAlreadyStored + ); +} diff --git a/ant-core/tests/e2e_file.rs b/ant-core/tests/e2e_file.rs index 7106de82..3b3202a4 100644 --- a/ant-core/tests/e2e_file.rs +++ b/ant-core/tests/e2e_file.rs @@ -4,7 +4,7 @@ mod support; -use ant_core::data::{compute_address, Client, ExternalPaymentInfo, Visibility}; +use ant_core::data::{compute_address, Client, ExternalPaymentInfo, PaymentMode, Visibility}; use ant_protocol::evm::{QuoteHash, TxHash}; use serial_test::serial; use std::collections::HashMap; @@ -64,6 +64,65 @@ async fn test_file_upload_download_round_trip() { testnet.teardown().await; } +/// Streaming download: `file_download_to_sender` must yield exactly the bytes, +/// in order, that the file contained — without buffering the whole file. Uses +/// a multi-batch payload so the streaming-decrypt path runs more than one +/// batch, then reassembles the stream and asserts equality with the source. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn test_file_download_to_sender_multibatch_round_trip() { + use tokio::sync::mpsc; + + let (client, testnet) = setup().await; + + let mut input_file = NamedTempFile::new().expect("create temp file"); + // ~1 MiB of varied bytes → many self-encryption chunks (multiple batches). + let data: Vec = (0..1_048_576u32).map(|i| (i % 251) as u8).collect(); + input_file.write_all(&data).expect("write temp file"); + input_file.flush().expect("flush temp file"); + + let result = client + .file_upload(input_file.path()) + .await + .expect("file_upload should succeed"); + + // Channel item type is inferred from `file_download_to_sender`'s signature. + let (tx, mut rx) = mpsc::channel(8); + let data_map = result.data_map.clone(); + let dl = tokio::spawn(async move { client.file_download_to_sender(&data_map, tx, None).await }); + + let mut streamed: Vec = Vec::with_capacity(data.len()); + let mut chunk_count = 0usize; + while let Some(item) = rx.recv().await { + let chunk = item.expect("stream chunk should be Ok"); + // A buggy "send one empty/sentinel then drop" producer would still + // close the channel; assert each delivered chunk carries real bytes. + assert!(!chunk.is_empty(), "streamed chunk should be non-empty"); + chunk_count += 1; + streamed.extend_from_slice(&chunk); + } + + let bytes_streamed = dl + .await + .expect("download task should join") + .expect("file_download_to_sender should succeed"); + + // The whole point of the streaming path: a multi-batch payload must arrive + // as more than one segment, not buffered and emitted in one shot. + assert!( + chunk_count >= 2, + "multi-batch payload should stream as ≥2 segments, got {chunk_count}" + ); + assert_eq!(streamed, data, "streamed content should match original"); + assert_eq!( + bytes_streamed, + data.len() as u64, + "bytes_streamed should match original size" + ); + + testnet.teardown().await; +} + #[tokio::test(flavor = "multi_thread")] #[serial] async fn test_file_large_content() { @@ -300,3 +359,58 @@ async fn test_public_upload_round_trip_wave_batch() { drop(client); testnet.teardown().await; } + +/// Full wallet-backed public upload round-trip (direct CLI-style path). +/// +/// This covers the non-external-signer path used by `ant file upload --public`: +/// the serialized DataMap must be appended to the upload chunk set before +/// payment, so the returned address is immediately retrievable without a +/// second `data_map_store` payment. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn test_public_file_upload_direct_batches_datamap() { + let (client, testnet) = setup().await; + + let original = vec![0x6bu8; 4096]; + let mut input_file = NamedTempFile::new().expect("create temp file"); + input_file.write_all(&original).expect("write temp file"); + input_file.flush().expect("flush temp file"); + + let result = client + .file_upload_public_with_mode(input_file.path(), PaymentMode::Single) + .await + .expect("public upload should succeed"); + + let data_map_address = result + .data_map_address + .expect("public upload must return a DataMap address"); + let expected_bytes = rmp_serde::to_vec(&result.data_map).expect("serialize DataMap"); + assert_eq!( + data_map_address, + compute_address(&expected_bytes), + "data_map_address must point to the serialized DataMap chunk" + ); + assert_eq!( + result.chunks_stored, result.total_chunks, + "public upload should store every chunk, including the DataMap" + ); + + let fetched_data_map = client + .data_map_fetch(&data_map_address) + .await + .expect("public DataMap should be retrievable by returned address"); + + let output_dir = TempDir::new().expect("create output temp dir"); + let output_path = output_dir.path().join("direct_public_out.bin"); + let bytes_written = client + .file_download(&fetched_data_map, &output_path) + .await + .expect("file_download should succeed"); + + assert_eq!(bytes_written, original.len() as u64); + let downloaded = std::fs::read(&output_path).expect("read downloaded file"); + assert_eq!(downloaded, original); + + drop(client); + testnet.teardown().await; +} diff --git a/ant-core/tests/support/mod.rs b/ant-core/tests/support/mod.rs index a5842fe1..4a168c81 100644 --- a/ant-core/tests/support/mod.rs +++ b/ant-core/tests/support/mod.rs @@ -27,6 +27,7 @@ use ant_node::storage::{AntProtocol, LmdbStorage, LmdbStorageConfig}; // Wire / transport / EVM types: route through ant-protocol so the test // harness exercises the same surface the client does. use ant_protocol::evm::{testnet::Testnet, Network as EvmNetwork, RewardsAddress, Wallet}; +use ant_protocol::pqc::ops::{MlDsaOperations, MlDsaSecretKey}; use ant_protocol::transport::{ CoreNodeConfig, IPDiversityConfig, MlDsa65, MultiAddr, NodeIdentity, P2PEvent, P2PNode, }; @@ -46,19 +47,19 @@ const STABILIZATION_TIMEOUT_SECS: u64 = 180; /// Default node count for standard E2E tests. /// /// `CLOSE_GROUP_SIZE` (7) is the quorum the client needs for a quote to -/// succeed, so spawning exactly that many — or `+ 1` — leaves zero slack: -/// a single slow peer drops the count to 6 and fails the whole test with -/// `InsufficientPeers("Got 6 quotes, need 7. ...")`. +/// succeed. Spawning only that many nodes leaves the DHT and direct +/// connection set too thin during startup, especially while every test node is +/// still stabilising. /// /// This is systematic on macOS CI runners, which are heavily virtualised /// (nested virt) and roughly half the CPU throughput of Linux runners. -/// The 8-node QUIC handshake burst saturates the CPU and at least one -/// peer consistently can't complete its handshake within the 10 s default -/// per-peer timeout. Linux runners finish all 8 handshakes comfortably. +/// The QUIC handshake burst saturates the CPU and can leave too few peers +/// ready for a `CLOSE_GROUP_SIZE` quote attempt. Linux runners finish those +/// handshakes more comfortably. /// -/// Spawning `CLOSE_GROUP_SIZE * 2` gives us one full group of slack — if -/// up to 7 peers are slow, quote collection still reaches quorum. Each -/// extra node is cheap (~200 ms spawn delay) compared to a flaky suite. +/// Spawning `CLOSE_GROUP_SIZE * 2` gives the lookup layer enough nearby peers +/// to return a full close group reliably. Each extra node is cheap (~200 ms +/// spawn delay) compared to a flaky suite. pub const DEFAULT_NODE_COUNT: usize = CLOSE_GROUP_SIZE * 2; /// Index of the median quote in a `SingleNodePayment` quotes array. @@ -77,7 +78,7 @@ const TEST_MAX_RECORDS: usize = 1280; /// DHT lookups, and payment round-trips compete for the same cores. On /// heavily-virtualised runners (macOS GitHub Actions in particular), the /// 10 s per-peer timeout fires before the slowest peer can finish its -/// handshake, which surfaces as `InsufficientPeers("Got 6 quotes, need 7")`. +/// handshake, which can surface as `InsufficientPeers`. /// /// 60 s is deliberately conservative: in the happy path everything completes /// in well under a second, so the larger budget only shows up on flakes. @@ -197,6 +198,32 @@ impl MiniTestnet { sleep(Duration::from_millis(500)).await; } + // The in-process E2E harness builds clients from one of the storage + // nodes. Saorsa's witnessed client lookup filters that local peer out, + // while node-side payment verification uses a self-inclusive local + // close-group view. In tiny random testnets that can make an otherwise + // valid paid median issuer fall just outside a storer's local top-7. + // Keep the production live-DHT check in normal builds, but use + // ant-node's test-only override here so these client E2Es exercise + // payment/proof/storage behaviour without depending on that topology + // artifact. + let paid_quote_close_group_override: Vec<[u8; 32]> = nodes + .iter() + .filter_map(|test_node| { + test_node + .p2p_node + .as_ref() + .map(|p2p_node| *p2p_node.peer_id().as_bytes()) + }) + .collect(); + for test_node in &nodes { + if let Some(protocol) = &test_node.protocol { + protocol + .payment_verifier_arc() + .set_paid_quote_close_group_for_tests(paid_quote_close_group_override.clone()); + } + } + // Approve token spend for the unified payment vault contract let vault_address = evm_network.payment_vault_address(); wallet @@ -286,25 +313,19 @@ impl MiniTestnet { network: evm_network.clone(), }, cache_capacity: 1000, + close_group_size: CLOSE_GROUP_SIZE, local_rewards_address: rewards_address, }; let payment_verifier = Arc::new(PaymentVerifier::new(payment_config)); - // Wire the P2P node into the verifier so the merkle pay-yourself - // closeness check can do its DHT lookup. Without this, the - // verifier fail-closes on every merkle payment (PR #77 defense). - payment_verifier.attach_p2p_node(Arc::clone(&node)); let metrics_tracker = QuotingMetricsTracker::new(TEST_MAX_RECORDS); let mut quote_generator = QuoteGenerator::new(rewards_address, metrics_tracker); // Wire ML-DSA-65 signing so quotes are properly signed and verifiable let pub_key_bytes = identity.public_key().as_bytes().to_vec(); let sk_bytes = identity.secret_key_bytes().to_vec(); - let sk = { - use ant_protocol::pqc::ops::MlDsaSecretKey; - MlDsaSecretKey::from_bytes(&sk_bytes).expect("deserialize ML-DSA-65 secret key") - }; + let sk = + { MlDsaSecretKey::from_bytes(&sk_bytes).expect("deserialize ML-DSA-65 secret key") }; quote_generator.set_signer(pub_key_bytes, move |msg| { - use ant_protocol::pqc::ops::MlDsaOperations; let ml_dsa = MlDsa65::new(); ml_dsa .sign(&sk, msg) @@ -317,6 +338,9 @@ impl MiniTestnet { payment_verifier, Arc::new(quote_generator), )); + // Wire the P2P node into the protocol so direct PUT storage-admission + // and payment closeness checks use the node's live DHT view. + protocol.attach_p2p_node(Arc::clone(&node)); // Start message handler loop let handler_node = Arc::clone(&node);