diff --git a/Cargo.lock b/Cargo.lock index 88c1c02b..1e29bb25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10403,6 +10403,7 @@ dependencies = [ "frame-support", "frame-system", "hex", + "itertools 0.12.1", "native", "native-api", "on-chain-table", @@ -10418,13 +10419,16 @@ dependencies = [ "pallet-zkpay", "parity-scale-codec", "postcard", + "proof-of-sql", "proof-of-sql-commitment-map", "proof-of-sql-static-setups", + "proptest", "scale-info", "sp-core", "sp-io", "sp-runtime", "sp-staking 36.0.0", + "sqlparser", "sxt-core", ] diff --git a/pallets/indexing/Cargo.toml b/pallets/indexing/Cargo.toml index 97c9bc57..e4f5d175 100644 --- a/pallets/indexing/Cargo.toml +++ b/pallets/indexing/Cargo.toml @@ -37,6 +37,7 @@ postcard.workspace = true proof-of-sql-static-setups = { workspace = true, features = ["io"], optional = true } on-chain-table = { workspace = true, default-features = false } proof-of-sql-commitment-map = { workspace = true, optional = true } +itertools.workspace = true [dev-dependencies] sp-staking.workspace = true @@ -53,6 +54,11 @@ native.workspace = true arrow.workspace = true proof-of-sql-commitment-map.workspace = true pallet-zkpay.workspace = true +proptest.workspace = true +proof-of-sql.workspace = true +on-chain-table = { workspace = true, features = ["proptest"] } +sqlparser.workspace = true + [features] default = ["std"] diff --git a/pallets/indexing/src/lib.rs b/pallets/indexing/src/lib.rs index 34f6dcce..f9491d3f 100644 --- a/pallets/indexing/src/lib.rs +++ b/pallets/indexing/src/lib.rs @@ -34,22 +34,25 @@ pub mod native_pallet; #[allow(clippy::manual_inspect)] #[frame_support::pallet] pub mod pallet { + use alloc::collections::{BTreeMap, BTreeSet}; use alloc::string::String; use alloc::vec::Vec; - use codec::Decode; + use codec::{Decode, EncodeLike}; use commitment_sql::InsertAndCommitmentMetadata; use frame_support::dispatch::RawOrigin; use frame_support::pallet_prelude::*; use frame_support::{Blake2_128, Blake2_128Concat}; use frame_system::pallet_prelude::*; use hex::FromHex; + use itertools::Itertools; use native_api::NativeApi; use on_chain_table::OnChainTable; use sp_core::crypto::Ss58Codec; use sp_core::{H256, U256}; use sp_runtime::traits::{Bounded, Hash, StaticLookup, UniqueSaturatedInto}; - use sp_runtime::{BoundedVec, SaturatedConversion}; + use sp_runtime::{BoundedVec, Either, SaturatedConversion}; + use sxt_core::heavy::Heavy; use sxt_core::permissions::{IndexingPalletPermission, PermissionLevel}; use sxt_core::tables::{ InsertQuorumSize, @@ -79,11 +82,19 @@ pub mod pallet { + IsType<::RuntimeEvent>; /// The weight info to be used with the extrinsics provided by the pallet type WeightInfo: WeightInfo; + + /// The maximum batches finding quorum at any given time. + #[pallet::constant] + type MaxBatchesFindingQuorum: Get; + + /// The maximum batches pruned per transaction from submissions storage when it exceeds + /// `MaxBatchesPruned`. + #[pallet::constant] + type MaxBatchesPruned: Get; } /// Storage map of `BatchId` and data hash to submitters that have agreed to the batch/hash. #[pallet::storage] - #[pallet::getter(fn submissions)] pub type Submissions, I: 'static = ()> = StorageDoubleMap< _, Blake2_128Concat, @@ -94,6 +105,37 @@ pub mod pallet { ValueQuery, // Allows us to receive a default instead of None >; + /// Storage map of `(BatchId, QuorumScope, AccountId)` to data hash for batches that are still + /// finding quorum. + /// + /// Will get pruned once `Config::MaxBatchesFindingQuorum` is reached. + #[pallet::storage] + #[pallet::getter(fn submissions_v1)] + pub type SubmissionsV1, I: 'static = ()> = StorageNMap< + _, + ( + NMapKey, + NMapKey, + NMapKey, + ), + ::Hash, + >; + + /// Storage map of `BatchId`s by batch index, in the order of first submission. + /// + /// Will get pruned once `Config::MaxBatchesFindingQuorum` is reached. + #[pallet::storage] + #[pallet::getter(fn batch_queue_get)] + pub type BatchQueue, I: 'static = ()> = + CountedStorageMap<_, Blake2_128Concat, u32, BatchId>; + + /// The lowest index currently in the `BatchQueue`. + /// + /// Will increment as the `BatchQueue` is pruned. + #[pallet::storage] + #[pallet::getter(fn batch_queue_bottom)] + pub type BatchQueueBottom, I: 'static = ()> = StorageValue<_, u32, ValueQuery>; + /// Storage map of `BatchId`s to `DataQuorum`s for batches that have reached quorum. #[pallet::storage] #[pallet::getter(fn final_data)] @@ -157,6 +199,12 @@ pub mod pallet { /// Voters against this quorum dissents: BoundedBTreeSet>, }, + + /// The `BatchQueue` and submissions storage have been pruned. + BatchQueuePruned { + /// The number of batches removed from storage. + num_pruned: u32, + }, } #[pallet::error] @@ -199,6 +247,8 @@ pub mod pallet { TableSerializationError, /// Submitter Injection Failed SubmitterInjectionFailed, + /// Maximum submissions already reached for this batch id + MaxSubmittersReached, } #[pallet::call] @@ -402,22 +452,26 @@ pub mod pallet { T: Config, I: NativeApi, { - // We don't need to save the full data. We just need a count associated with each submission - let match_submissions = Submissions::::get(&batch_id, data_hash); - - // Check if this user has already submitted this data - let current_num_matching_submissions = match_submissions.len_of_scope(quorum_scope); + // There is no `StorageNMap::contains_key_prefix` + if SubmissionsV1::::iter_prefix((&batch_id,)) + .next() + .is_none() + { + let batch_index = Pallet::::batch_queue_bottom() + BatchQueue::::count(); + BatchQueue::::insert(batch_index, &batch_id); + } - let new_match_submissions = match_submissions - .with_submitter(who.clone(), quorum_scope) - // Just return the unchanged submissions if maximum is exceeded. - .unwrap_or_else(|(submitters, _)| submitters); + let submission_map_with_this = + SubmissionsV1::::iter_prefix((&batch_id, quorum_scope)) + .take(MAX_SUBMITTERS as usize) + .chain(core::iter::once((who.clone(), data_hash))) + .collect::>(); - if new_match_submissions.len_of_scope(quorum_scope) == current_num_matching_submissions { - Err(Error::::AlreadySubmitted)? + if submission_map_with_this.len() > MAX_SUBMITTERS as usize { + Err(Error::MaxSubmittersReached::)?; } - Submissions::::insert(&batch_id, data_hash, &new_match_submissions); + SubmissionsV1::::insert((&batch_id, quorum_scope, &who), data_hash); let submission = DataSubmission { table: table.clone(), @@ -425,38 +479,41 @@ pub mod pallet { data_hash, quorum_scope: *quorum_scope, }; - // Emit an event noting who submitted what Pallet::::deposit_event(Event::DataSubmitted { who, submission }); - match table_insert_quorum.of_scope(quorum_scope) { - Some(quorum_size) - if new_match_submissions.len_of_scope(quorum_scope) as u8 > *quorum_size => - { - // Iterate over the submitters who submitted differing data and collect - // their account ids - let dissenters = Submissions::::iter_prefix(&batch_id) - .filter(|(hash, _)| hash != &data_hash) - .flat_map(|(_, submitters)| submitters.into_iter_scope(quorum_scope)) - // de-dup collection - .collect::>() - .into_iter() - // resulting set should contain up to MAX_SUBMITTERS items *after* de-dup - .take(MAX_SUBMITTERS as usize) - .collect::>() - .try_into() - .expect("source Vec is constructed to not exceed maximum submitter list size"); + let (agreements_unbounded, dissents_unbounded): (BTreeSet<_>, BTreeSet<_>) = + submission_map_with_this + .into_iter() + .partition_map(|(account_id, hash)| { + if hash == data_hash { + Either::Left(account_id) + } else { + Either::Right(account_id) + } + }); + match table_insert_quorum.of_scope(quorum_scope) { + Some(quorum_size) if agreements_unbounded.len() as u8 > *quorum_size => { let block_number = >::block_number(); + // Technically we don't need to check this, we know at this point that both lists + // sizes will sum up to the size of submission_map_with_this, which we already + // checked is below the number of max submitters. We still avoid the panic out of + // an abundance of caution. + let (agreements, dissents) = agreements_unbounded + .try_into() + .and_then(|agreements| Ok((agreements, dissents_unbounded.try_into()?))) + .map_err(|_| Error::MaxSubmittersReached::)?; + // Decide on the quorum let data_quorum = DataQuorum { table, batch_id, data_hash, block_number: block_number.into(), - agreements: new_match_submissions.of_scope(quorum_scope).clone(), - dissents: dissenters, + agreements, + dissents, quorum_scope: *quorum_scope, }; @@ -482,8 +539,7 @@ pub mod pallet { I: NativeApi, { // Clean up submissions for this batch - Submissions::::iter_key_prefix(&quorum.batch_id) - .for_each(|key| Submissions::::remove(&quorum.batch_id, key)); + let _ = remove_batch_id_from_submissions_v1::(&quorum.batch_id); // Record final decision FinalData::::insert(&quorum.batch_id, &quorum); @@ -625,4 +681,167 @@ pub mod pallet { ); Ok(()) } + + /// Returns the count of `BatchQueue` and the weight of the get. + fn batch_queue_count_heavy() -> Heavy + where + T: Config, + I: NativeApi, + { + let out = BatchQueue::::count(); + let weight = T::DbWeight::get().reads(1); + Heavy { out, weight } + } + + /// Returns the value of `BatchQueueBottom` and the weight of the get. + fn batch_queue_bottom_heavy() -> Heavy + where + T: Config, + I: NativeApi, + { + let out = Pallet::::batch_queue_bottom(); + let weight = T::DbWeight::get().reads(1); + Heavy { out, weight } + } + + /// Sets the value of `BatchQueueBottom` and returns the weight of the set. + fn batch_queue_bottom_set_heavy(bottom: u32) -> Heavy<()> + where + T: Config, + I: NativeApi, + { + BatchQueueBottom::::set(bottom); + T::DbWeight::get().writes(1).into() + } + + /// Removes and returns the `BatchId` at the given index in the `BatchQueue`, along with the + /// weight of the take. + fn batch_queue_take_heavy(batch_index: u32) -> Heavy> + where + T: Config, + I: NativeApi, + { + let out = BatchQueue::::take(batch_index); + + let weight = if out.is_some() { + T::DbWeight::get().reads_writes(1, 1) + } else { + T::DbWeight::get().reads(1) + }; + + Heavy { out, weight } + } + + /// Removes up to `prune_limit` entries from the v0 `Submissions` storage. + /// + /// Returns what remains of the prune limit, i.e., `prune_limit - num_pruned`. + fn prune_submissions_v0(prune_limit: u32) -> Heavy + where + T: Config, + I: NativeApi, + { + // In testing, `StorageDoubleMap::clear` didn't obey the limits, removing all entries + // instead. So, this does a manual iter-keys-take-n-remove instead. + + // Technically, since this is a double map, this clears n (batch_id, data_hash) pairs, not + // n batch_ids. These won't be 1-to-1 in the case that there was a controversial batch_id. + // However, any partially-removed batch_id will be cleaned up in future calls. + let keys_to_remove = Submissions::::iter_keys() + .take(prune_limit.try_into().unwrap_or_default()) + .collect::>(); + let removal_count = keys_to_remove.len().try_into().unwrap_or_default(); + + keys_to_remove + .into_iter() + .for_each(|(batch_id, data_hash)| { + Submissions::::remove(batch_id, data_hash); + }); + + let remaining_prunes = prune_limit.saturating_sub(removal_count); + let weight = T::DbWeight::get().reads_writes(removal_count.into(), removal_count.into()); + + Heavy { + out: remaining_prunes, + weight, + } + } + + /// Removes the given `batch_id` from the `SubmissionsV1` storage and returns the weight of the + /// clear_prefix. + fn remove_batch_id_from_submissions_v1(batch_id: impl EncodeLike) -> Heavy<()> + where + T: Config, + I: NativeApi, + { + let removal_limit = MAX_SUBMITTERS + .saturating_mul(QuorumScope::VARIANT_COUNT.try_into().unwrap_or_default()); + + let removal_results = SubmissionsV1::::clear_prefix((batch_id,), removal_limit, None); + + T::DbWeight::get() + .reads_writes(removal_results.loops.into(), removal_results.unique.into()) + .into() + } + + /// Removes up to `prune_limit` batches from the `SubmissionsV1` storage. + /// + /// Returns what remains of the prune limit, i.e., `prune_limit - num_pruned`. + fn prune_batch_queue(prune_limit: u32) -> Heavy + where + T: Config, + I: NativeApi, + { + batch_queue_count_heavy::().and_then(|batch_queue_size| { + if batch_queue_size <= T::MaxBatchesFindingQuorum::get() { + // nothing to prune + return prune_limit.into(); + } + + batch_queue_bottom_heavy::().and_then(|batch_queue_bottom| { + let num_batches_to_prune = + batch_queue_size.saturating_sub(T::MaxBatchesFindingQuorum::get()); + let clamped_num_batches_to_prune = num_batches_to_prune.min(prune_limit); + + let new_batch_queue_bottom = batch_queue_bottom + clamped_num_batches_to_prune; + + (batch_queue_bottom..new_batch_queue_bottom) + .map(|batch_index| { + batch_queue_take_heavy::(batch_index).and_then(|batch_id| { + if let Some(batch_id) = batch_id { + remove_batch_id_from_submissions_v1::(batch_id) + } else { + ().into() + } + }) + }) + .sum::>() + .and_then(|_| batch_queue_bottom_set_heavy::(new_batch_queue_bottom)) + .map(|_| prune_limit.saturating_sub(clamped_num_batches_to_prune)) + }) + }) + } + + #[pallet::hooks] + impl, I: 'static> Hooks> for Pallet + where + I: NativeApi, + { + fn on_initialize(_: BlockNumberFor) -> Weight { + let max_batches_pruned = T::MaxBatchesPruned::get(); + + let Heavy { + out: remaining_prunes, + weight, + } = prune_submissions_v0::(max_batches_pruned) + .and_then(prune_batch_queue::); + + let num_pruned = max_batches_pruned.saturating_sub(remaining_prunes); + + if num_pruned > 0 { + Pallet::::deposit_event(Event::::BatchQueuePruned { num_pruned }); + } + + weight + } + } } diff --git a/pallets/indexing/src/mock.rs b/pallets/indexing/src/mock.rs index 223eedcd..b1d0594a 100644 --- a/pallets/indexing/src/mock.rs +++ b/pallets/indexing/src/mock.rs @@ -121,6 +121,8 @@ impl pallet_staking::Config for Test { impl pallet_indexing::pallet::Config for Test { type RuntimeEvent = RuntimeEvent; type WeightInfo = pallet_indexing::weights::SubstrateWeight; + type MaxBatchesFindingQuorum = ConstU32<5>; + type MaxBatchesPruned = ConstU32<3>; } pub type BlockNumber = u64; diff --git a/pallets/indexing/src/tests.rs b/pallets/indexing/src/tests.rs index c3bf3e35..edf23dc0 100644 --- a/pallets/indexing/src/tests.rs +++ b/pallets/indexing/src/tests.rs @@ -1,4 +1,5 @@ use alloc::boxed::Box; +use std::collections::{HashMap, HashSet}; use std::convert::Into; use std::io::Cursor; use std::sync::Arc; @@ -10,13 +11,21 @@ use codec::{Decode, Encode, MaxEncodedLen}; use frame_support::__private::RuntimeDebug; use frame_support::dispatch::DispatchResult; use frame_support::pallet_prelude::TypeInfo; -use frame_support::{assert_err, assert_ok}; -use frame_system::ensure_signed; -use native_api::Api; +use frame_support::traits::{Get, Hooks}; +use frame_support::{assert_err, assert_noop, assert_ok}; +use frame_system::{ensure_signed, RawOrigin}; +use native_api::{Api, NativeApi}; +use on_chain_table::proptest::{on_chain_table, ProofOfSqlSchema}; +use on_chain_table::{IndexSet, OnChainTable}; use pallet_tables::{CommitmentCreationCmd, UpdateTable}; +use proof_of_sql::base::database::ColumnType; use proof_of_sql_commitment_map::CommitmentSchemeFlags; +use proptest::prelude::*; +use proptest::sample::SizeRange; use sp_core::Hasher; use sp_runtime::BoundedVec; +use sqlparser::ast::Ident; +use sxt_core::indexing::{SubmittersByScope, ID_LEN, MAX_SUBMITTERS}; use sxt_core::permissions::{IndexingPalletPermission, PermissionLevel, PermissionList}; use sxt_core::tables::{ CommitmentScheme, @@ -31,16 +40,26 @@ use sxt_core::tables::{ }; use crate::mock::*; -use crate::{build_inner_batch_id, BatchId, Event, RowData}; +use crate::{build_inner_batch_id, BatchId, Config, Event, RowData}; /// Used as a convenience wrapper for data we need to submit -#[derive(Clone, Encode, Decode, Eq, PartialEq, RuntimeDebug, TypeInfo, MaxEncodedLen)] +#[derive(Clone, Encode, Decode, Eq, PartialEq, TypeInfo, MaxEncodedLen, Hash)] struct TestSubmission { table: TableIdentifier, batch_id: BatchId, data: RowData, } +impl core::fmt::Debug for TestSubmission { + fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result { + fmt.debug_struct("TestSubmission") + .field("table", &String::try_from(&self.table).unwrap()) + .field("batch_id", &hex::encode(&self.batch_id)) + .field("data", &hex::encode(&self.data)) + .finish() + } +} + /// Helper function to streamline data submission fn submit_test_data(signer: RuntimeOrigin, submission: TestSubmission) -> DispatchResult { Indexing::submit_data( @@ -62,7 +81,7 @@ fn row_data_with_count(rows: i32) -> RowData { let batch = RecordBatch::try_new(schema.clone(), vec![int_data]).unwrap(); - record_batch_to_row_data(batch, schema) + record_batch_to_row_data(batch) } fn row_data() -> RowData { @@ -80,14 +99,14 @@ fn diff_row_data() -> RowData { let batch = RecordBatch::try_new(schema.clone(), vec![int_data]).unwrap(); - record_batch_to_row_data(batch, schema) + record_batch_to_row_data(batch) } -fn record_batch_to_row_data(batch: RecordBatch, schema: Arc) -> RowData { +fn record_batch_to_row_data(batch: RecordBatch) -> RowData { let buffer: Vec = Vec::new(); let mut cursor = Cursor::new(buffer); - let mut writer = StreamWriter::try_new(&mut cursor, &schema).unwrap(); + let mut writer = StreamWriter::try_new(&mut cursor, batch.schema().as_ref()).unwrap(); writer.write(&batch).unwrap(); writer.finish().unwrap(); @@ -125,6 +144,96 @@ fn sample_table_definition() -> (TableIdentifier, CreateStatement) { (table_id, create_statement) } +/// Returns a `Strategy` for row data compatible with [`sample_table_definition`]. +fn row_data_for_sample_table(num_rows: NR) -> impl Strategy +where + NR: Strategy, +{ + let schema = + ProofOfSqlSchema::try_from_iter([(Ident::new("INT_COLUMN"), ColumnType::Int)]).unwrap(); + + on_chain_table(Just(schema), num_rows).prop_map(|on_chain_table| { + let record_batch = on_chain_table.into(); + record_batch_to_row_data(record_batch) + }) +} + +/// Returns a `Strategy` for test submissions compatible with [`sample_table_definition`]. +fn submission_for_sample_table( + num_rows: NR, + batch_id: BI, +) -> impl Strategy +where + NR: Strategy, + BI: Strategy, +{ + (row_data_for_sample_table(num_rows), batch_id).prop_map(|(data, batch_id)| { + let table = TableIdentifier::from_str_unchecked("TEST_TABLE", "TEST_NAMESPACE"); + TestSubmission { + table, + batch_id, + data, + } + }) +} + +/// Returns a `Strategy` for `BatchId`s. +fn batch_id_strategy() -> impl Strategy { + proptest::collection::vec(any::(), 1..ID_LEN as usize) + .prop_map(|batch_id_bytes| batch_id_bytes.try_into().unwrap()) +} + +/// Returns a `Strategy` for a set of test submissions for [`sample_table_definition`]. +fn submissions_for_sample_table( + num_submissions: NS, + num_rows_per_submission: NR, + batch_id: BI, +) -> impl Strategy> +where + NS: Into + Clone, + NR: Strategy + Clone, + BI: Strategy, +{ + proptest::collection::hash_set( + submission_for_sample_table(num_rows_per_submission, batch_id), + num_submissions, + ) +} + +/// Returns a `Strategy` for a mapping of `BatchId`s to a set of test submissions for [`sample_table_definition`]. +fn submissions_for_sample_table_by_batch_id( + num_batches: NB, + num_submissions_per_batch: NS, + num_rows_per_submission: NR, + batch_id: BI, +) -> impl Strategy>> +where + NB: Into, + NS: Into + Clone, + NR: Strategy + Clone, + BI: Strategy, +{ + proptest::collection::hash_set(batch_id, num_batches) + .prop_flat_map(move |batch_ids| { + let num_submissions_per_batch = num_submissions_per_batch.clone(); + let num_rows_per_submission = num_rows_per_submission.clone(); + batch_ids + .into_iter() + .map(move |batch_id| { + ( + Just(batch_id.clone()), + submissions_for_sample_table( + num_submissions_per_batch.clone(), + num_rows_per_submission.clone(), + Just(batch_id), + ), + ) + }) + .collect::>() + }) + .prop_map(HashMap::from_iter) +} + fn empty_row_data() -> RowData { let schema = Arc::new(Schema::new(vec![Field::new( "int_column", @@ -134,7 +243,7 @@ fn empty_row_data() -> RowData { let empty_batch = RecordBatch::new_empty(schema.clone()); - record_batch_to_row_data(empty_batch, schema) + record_batch_to_row_data(empty_batch) } fn row_data_w_block_number() -> RowData { @@ -148,7 +257,7 @@ fn row_data_w_block_number() -> RowData { let batch = RecordBatch::try_new(schema.clone(), vec![int_data, block_data]).unwrap(); - record_batch_to_row_data(batch, schema) + record_batch_to_row_data(batch) } fn sample_table_definition_with_block_number() -> (TableIdentifier, CreateStatement) { @@ -218,7 +327,7 @@ fn inserting_data_succeeds_when_data_is_good() { IndexingPalletPermission::SubmitDataForPublicQuorum, )]) .unwrap(); - pallet_permissions::Permissions::::insert(who, permissions.clone()); + pallet_permissions::Permissions::::insert(&who, permissions.clone()); let test_batch = BatchId::try_from(b"test_batch".to_vec()).unwrap(); let test_data = row_data(); @@ -236,74 +345,8 @@ fn inserting_data_succeeds_when_data_is_good() { // Verify that the submission was stored as expected // and the hash was generated from the submitted data assert_eq!( - Indexing::submissions(internal_batch_id, hash).len_of_scope(&QuorumScope::Public), - 1 - ); - }) -} - -#[test] -fn submission_fails_when_data_is_already_submitted() { - new_test_ext().execute_with(|| { - System::set_block_number(1); - let (table_id, test_create) = sample_table_definition(); - Tables::create_tables( - RuntimeOrigin::root(), - vec![UpdateTable { - ident: table_id.clone(), - create_statement: test_create, - table_type: TableType::CoreBlockchain, - commitment: CommitmentCreationCmd::Empty(CommitmentSchemeFlags { - hyper_kzg: false, - dynamic_dory: true, - }), - source: sxt_core::tables::Source::Ethereum, - }] - .try_into() - .unwrap(), - ) - .unwrap(); - let signer = RuntimeOrigin::signed(sp_runtime::AccountId32::new([1; 32])); - let who = ensure_signed(signer.clone()).unwrap(); - let permissions = PermissionList::try_from(vec![PermissionLevel::IndexingPallet( - IndexingPalletPermission::SubmitDataForPublicQuorum, - )]) - .unwrap(); - pallet_permissions::Permissions::::insert(who, permissions.clone()); - - let test_batch = BatchId::try_from(b"test_batch".to_vec()).unwrap(); - let test_data = row_data(); - - assert_ok!(Indexing::submit_data( - signer.clone(), - table_id.clone(), - test_batch.clone(), - test_data.clone(), - ),); - - let mut hash_input = test_data.encode(); - hash_input.extend(None::.encode()); - let hash = <::Hashing as Hasher>::hash(&hash_input); - - let internal_batch_id = build_inner_batch_id::(&test_batch, &table_id); - - // Verify that the submission was stored as expected - // and the hash was generated from the submitted data - assert_eq!( - Indexing::submissions(internal_batch_id.clone(), hash) - .len_of_scope(&QuorumScope::Public), - 1 - ); - - // Verify that submitting the same thing again returns the expected error - assert_err!( - Indexing::submit_data( - signer.clone(), - table_id.clone(), - test_batch.clone(), - test_data.clone(), - ), - crate::Error::::AlreadySubmitted + Indexing::submissions_v1((internal_batch_id.clone(), QuorumScope::Public, who)), + Some(hash) ); }) } @@ -319,7 +362,8 @@ fn data_submission_fails_if_no_permissions() { let test_data = RowData::try_from(b"some arbitrary row data".to_vec()).unwrap(); // Create a non permissioned signer - let signer = RuntimeOrigin::signed(sp_runtime::AccountId32::new([1; 32])); + let account = sp_runtime::AccountId32::new([1; 32]); + let signer = RuntimeOrigin::signed(account.clone()); assert_err!( Indexing::submit_data( signer.clone(), @@ -330,13 +374,13 @@ fn data_submission_fails_if_no_permissions() { crate::Error::::UnauthorizedSubmitter, ); - let hash = <::Hashing as Hasher>::hash(&test_data); + let _ = <::Hashing as Hasher>::hash(&test_data); // Verify that the submission was not stored - assert_eq!( - Indexing::submissions(test_batch.clone(), hash).len_of_scope(&QuorumScope::Public), - 0 - ); + let internal_batch_id = build_inner_batch_id::(&test_batch, &test_identifier); + let submitters_count = + crate::SubmissionsV1::::iter_prefix((internal_batch_id.clone(),)).count(); + assert_eq!(submitters_count, 0); }) } @@ -419,9 +463,9 @@ fn data_is_decided_on_after_required_submissions() { assert_eq!(fd.quorum_scope, QuorumScope::Public); // Verify that the old data was successfully removed for this batch - let submitters = Indexing::submissions(&internal_batch_id, test_data_hash); - assert!(submitters.scope_is_empty(&QuorumScope::Public)); - assert!(submitters.scope_is_empty(&QuorumScope::Privileged)); + let submitters_count = + crate::SubmissionsV1::::iter_prefix((internal_batch_id.clone(),)).count(); + assert_eq!(submitters_count, 0); }) } @@ -518,10 +562,9 @@ fn correct_data_is_decided_on_after_required_submissions() { assert_eq!(final_data.unwrap().data_hash, data_hash); // Verify that the old data was successfully removed for this batch - for _i in 1..4 { - assert!(Indexing::submissions(&internal_batch_id, data_hash) - .scope_is_empty(&QuorumScope::Public)) - } + let submitters_count = + crate::SubmissionsV1::::iter_prefix((internal_batch_id.clone(),)).count(); + assert_eq!(submitters_count, 0); }) } @@ -807,7 +850,7 @@ fn submit_data_with_mothership_key_work() { PermissionLevel::IndexingPallet(IndexingPalletPermission::SubmitDataForPublicQuorum); assert_ok!(pallet_permissions::Pallet::::add_proxy_permission( RuntimeOrigin::signed(admin), - signer_key, + signer_key.clone(), permission, )); @@ -827,8 +870,8 @@ fn submit_data_with_mothership_key_work() { // Verify that the submission was stored as expected // and the hash was generated from the submitted data assert_eq!( - Indexing::submissions(internal_batch_id, hash).len_of_scope(&QuorumScope::Public), - 1 + Indexing::submissions_v1((internal_batch_id, QuorumScope::Public, signer_key)), + Some(hash) ); }) } @@ -889,9 +932,11 @@ fn we_can_reach_privileged_quorum() { assert_eq!(fd.quorum_scope, QuorumScope::Privileged); // Verify that the old data was successfully removed for this batch - let submitters = Indexing::submissions(&internal_batch_id, test_data_hash); - assert!(submitters.scope_is_empty(&QuorumScope::Public)); - assert!(submitters.scope_is_empty(&QuorumScope::Privileged)); + let internal_batch_id = + build_inner_batch_id::(&test_submission.batch_id, &test_submission.table); + let submitters_count = + crate::SubmissionsV1::::iter_prefix((internal_batch_id.clone(),)).count(); + assert_eq!(submitters_count, 0); }) } @@ -961,17 +1006,43 @@ fn we_can_manage_quorum_state_for_both_scopes() { let internal_batch_id = build_inner_batch_id::(&test_submission.batch_id, &table_id); - let submissions = Indexing::submissions(&internal_batch_id, test_data_hash); - assert_eq!(submissions.len_of_scope(&QuorumScope::Public), 1); - assert!(submissions.scope_is_empty(&QuorumScope::Privileged)); + assert_eq!( + crate::SubmissionsV1::::iter_prefix(( + &internal_batch_id, + QuorumScope::Public + )) + .count(), + 1 + ); + assert_eq!( + crate::SubmissionsV1::::iter_prefix(( + &internal_batch_id, + QuorumScope::Privileged + )) + .count(), + 0 + ); assert!(Indexing::final_data(&internal_batch_id).is_none()); // both submission assert_ok!(submit_test_data(both_submitter, test_submission.clone())); - let submissions = Indexing::submissions(&internal_batch_id, test_data_hash); - assert_eq!(submissions.len_of_scope(&QuorumScope::Public), 1); - assert_eq!(submissions.len_of_scope(&QuorumScope::Privileged), 1); + assert_eq!( + crate::SubmissionsV1::::iter_prefix(( + &internal_batch_id, + QuorumScope::Public + )) + .count(), + 1 + ); + assert_eq!( + crate::SubmissionsV1::::iter_prefix(( + &internal_batch_id, + QuorumScope::Privileged + )) + .count(), + 1 + ); assert!(Indexing::final_data(&internal_batch_id).is_none()); // privileged submission @@ -986,9 +1057,10 @@ fn we_can_manage_quorum_state_for_both_scopes() { assert_eq!(final_data.quorum_scope, QuorumScope::Privileged); // Verify that the old data was successfully removed for this batch - let submitters = Indexing::submissions(&internal_batch_id, test_data_hash); - assert!(submitters.scope_is_empty(&QuorumScope::Public)); - assert!(submitters.scope_is_empty(&QuorumScope::Privileged)); + assert_eq!( + crate::SubmissionsV1::::iter_prefix((&internal_batch_id,)).count(), + 0 + ); assert_eq!( System::read_events_for_pallet::>() @@ -1060,9 +1132,10 @@ fn reaching_quorum_for_both_scopes_simultaneously_produces_privileged_quorum_rea assert_eq!(final_data.quorum_scope, QuorumScope::Privileged); // Verify that the old data was successfully removed for this batch - let submitters = Indexing::submissions(test_submission.batch_id.clone(), test_data_hash); - assert!(submitters.scope_is_empty(&QuorumScope::Public)); - assert!(submitters.scope_is_empty(&QuorumScope::Privileged)); + assert_eq!( + crate::SubmissionsV1::::iter_prefix((&internal_batch_id,)).count(), + 0 + ); assert_eq!( System::read_events_for_pallet::>() @@ -1104,8 +1177,6 @@ fn we_cannot_submit_for_table_disabled_quorum_scope() { batch_id: BatchId::try_from(b"test_batch".to_vec()).unwrap(), data: row_data(), }; - let test_data_hash = - <::Hashing as Hasher>::hash(&test_submission.data); let public_permission = PermissionLevel::IndexingPallet(IndexingPalletPermission::SubmitDataForPublicQuorum); @@ -1122,10 +1193,13 @@ fn we_cannot_submit_for_table_disabled_quorum_scope() { submit_test_data(public_submitter, test_submission.clone()), crate::Error::::UnauthorizedSubmitter ); - let submissions = Indexing::submissions(&test_submission.batch_id, test_data_hash); - assert!(submissions.scope_is_empty(&QuorumScope::Public)); - assert!(submissions.scope_is_empty(&QuorumScope::Privileged)); - assert!(Indexing::final_data(&test_submission.batch_id).is_none()); + let internal_batch_id = + build_inner_batch_id::(&test_submission.batch_id, &test_submission.table); + assert_eq!( + crate::SubmissionsV1::::iter_prefix((&internal_batch_id,)).count(), + 0 + ); + assert!(Indexing::final_data(&internal_batch_id).is_none()); }) } @@ -1178,9 +1252,12 @@ fn we_cannot_submit_with_privilege_to_different_table() { submit_test_data(privileged_submitter, test_submission.clone()), crate::Error::::UnauthorizedSubmitter ); - let submissions = Indexing::submissions(&test_submission.batch_id, test_data_hash); - assert!(submissions.scope_is_empty(&QuorumScope::Public)); - assert!(submissions.scope_is_empty(&QuorumScope::Privileged)); + let internal_batch_id = + build_inner_batch_id::(&test_submission.batch_id, &test_submission.table); + assert_eq!( + crate::SubmissionsV1::::iter_prefix((&internal_batch_id,)).count(), + 0 + ); assert!(Indexing::final_data(&test_submission.batch_id).is_none()); }) } @@ -1464,9 +1541,9 @@ fn we_can_reach_quorum_before_and_after_changing_quorum_size() { build_inner_batch_id::(&test_submission.batch_id, &table_id); // Verify that the old data was successfully removed for this batch - let submitters = Indexing::submissions(&internal_batch_id, test_data_hash); - assert!(submitters.scope_is_empty(&QuorumScope::Public)); - assert!(submitters.scope_is_empty(&QuorumScope::Privileged)); + let submitters_count = + crate::SubmissionsV1::::iter_prefix((internal_batch_id.clone(),)).count(); + assert_eq!(submitters_count, 0); // Now update the quorum to make this table public let new_quorum = InsertQuorumSize { @@ -1523,9 +1600,9 @@ fn we_can_reach_quorum_before_and_after_changing_quorum_size() { assert_eq!(fd.quorum_scope, QuorumScope::Public); // Verify that the old data was successfully removed for this batch - let submitters = Indexing::submissions(&internal_batch_id_2, test_data_hash); - assert!(submitters.scope_is_empty(&QuorumScope::Public)); - assert!(submitters.scope_is_empty(&QuorumScope::Privileged)); + let submitters_count = + crate::SubmissionsV1::::iter_prefix((internal_batch_id_2.clone(),)).count(); + assert_eq!(submitters_count, 0); }); } @@ -1570,3 +1647,401 @@ fn we_can_submit_to_permissionless_table_with_no_permissions() { assert!(Indexing::final_data(&internal_batch_id).is_some()); }) } + +#[test] +fn submitters_can_overwrite_their_submission() { + new_test_ext().execute_with(|| { + System::set_block_number(1); + let (table_id, create_statement) = sample_table_definition(); + Tables::create_tables( + RuntimeOrigin::root(), + vec![UpdateTable { + ident: table_id.clone(), + create_statement, + table_type: TableType::CoreBlockchain, + commitment: CommitmentCreationCmd::Empty(CommitmentSchemeFlags::all()), + source: sxt_core::tables::Source::Ethereum, + }] + .try_into() + .unwrap(), + ) + .unwrap(); + + let permissions = PermissionList::try_from(vec![PermissionLevel::IndexingPallet( + IndexingPalletPermission::SubmitDataForPublicQuorum, + )]) + .unwrap(); + let signer = RuntimeOrigin::signed(sp_runtime::AccountId32::new([1; 32])); + let who = ensure_signed(signer.clone()).unwrap(); + pallet_permissions::Permissions::::insert(&who, permissions); + + let batch_id = BatchId::try_from(b"test_batch".to_vec()).unwrap(); + let data = row_data(); + let data_hash = hash_row_data_with_block_number::(&data, None); + + Indexing::submit_data( + signer.clone(), + table_id.clone(), + batch_id.clone(), + data.clone(), + ) + .unwrap(); + let internal_batch_id = build_inner_batch_id::(&batch_id, &table_id); + assert_eq!( + crate::SubmissionsV1::::get((&internal_batch_id, QuorumScope::Public, &who)) + .unwrap(), + data_hash + ); + + let different_data = diff_row_data(); + let different_data_hash = hash_row_data_with_block_number::(&different_data, None); + Indexing::submit_data(signer, table_id, batch_id.clone(), different_data.clone()).unwrap(); + assert_eq!( + crate::SubmissionsV1::::get((&internal_batch_id, QuorumScope::Public, &who)) + .unwrap(), + different_data_hash + ); + }); +} + +#[test] +fn submitters_cannot_exceed_maximum() { + new_test_ext().execute_with(|| { + System::set_block_number(1); + let (table_id, create_statement) = sample_table_definition(); + Tables::create_tables( + RuntimeOrigin::root(), + vec![UpdateTable { + ident: table_id.clone(), + create_statement, + table_type: TableType::CoreBlockchain, + commitment: CommitmentCreationCmd::Empty(CommitmentSchemeFlags::all()), + source: sxt_core::tables::Source::Ethereum, + }] + .try_into() + .unwrap(), + ) + .unwrap(); + + let batch_id = BatchId::try_from(b"test_batch".to_vec()).unwrap(); + + let internal_batch_id = build_inner_batch_id::(&batch_id, &table_id); + // artificially fill submissions for batch + (1..=MAX_SUBMITTERS).for_each(|submitter_num| { + let signer = + RuntimeOrigin::signed(sp_runtime::AccountId32::new([submitter_num as u8; 32])); + let who = ensure_signed(signer.clone()).unwrap(); + let artificial_data_hash = <::Hashing as Hasher>::hash( + &submitter_num.to_le_bytes(), + ); + + crate::SubmissionsV1::::insert( + (internal_batch_id.clone(), QuorumScope::Public, who), + artificial_data_hash, + ); + }); + + // we cannot insert one more + let permissions = PermissionList::try_from(vec![PermissionLevel::IndexingPallet( + IndexingPalletPermission::SubmitDataForPublicQuorum, + )]) + .unwrap(); + + let signer = RuntimeOrigin::signed(sp_runtime::AccountId32::new( + [(MAX_SUBMITTERS + 1) as u8; 32], + )); + let who = ensure_signed(signer.clone()).unwrap(); + pallet_permissions::Permissions::::insert(who, permissions.clone()); + let data = row_data(); + + assert_noop!( + Indexing::submit_data(signer.clone(), table_id.clone(), batch_id.clone(), data,), + crate::Error::::MaxSubmittersReached + ); + + // submitters can still re-submit new hashes + let signer = RuntimeOrigin::signed(sp_runtime::AccountId32::new([1; 32])); + let who = ensure_signed(signer.clone()).unwrap(); + pallet_permissions::Permissions::::insert(&who, permissions); + let data = row_data(); + let data_hash = hash_row_data_with_block_number::(&data, None); + + Indexing::submit_data( + signer.clone(), + table_id.clone(), + batch_id.clone(), + data.clone(), + ) + .unwrap(); + assert_eq!( + crate::SubmissionsV1::::get((&internal_batch_id, QuorumScope::Public, &who)) + .unwrap(), + data_hash + ); + }); +} + +/// Returns an `AccountId32` seeded by a `usize`. +fn account_from_num(index: usize) -> sp_runtime::AccountId32 { + let bytes = index + .to_le_bytes() + .into_iter() + .chain(std::iter::repeat(0)) + .take(32) + .collect::>() + .try_into() + .unwrap(); + sp_runtime::AccountId32::new(bytes) +} + +/// Returns a `QuorumScope` seeded by a `usize` +fn quorum_scope_public_if_even(index: usize) -> QuorumScope { + if index % 2 == 0 { + QuorumScope::Public + } else { + QuorumScope::Privileged + } +} + +/// Populates the `Submissions` (v0) storage with the given test submissions. +/// +/// The account and quorum scope used for the storage is determined by seeding the `account_fn` and +/// `quorum_scope_fn`. +fn populate_submissions_v0( + mut account_fn: impl FnMut(usize) -> T::AccountId, + mut quorum_scope_fn: impl FnMut(usize) -> QuorumScope, + submissions: impl IntoIterator, +) where + T: Config, + I: NativeApi, +{ + submissions + .into_iter() + .enumerate() + .for_each(|(i, test_submission)| { + let account = account_fn(i); + let internal_batch = + build_inner_batch_id::(&test_submission.batch_id, &test_submission.table); + + let data_hash = hash_row_data_with_block_number::(&test_submission.data, None); + + let quorum_scope = quorum_scope_fn(i); + + crate::Submissions::::insert( + internal_batch, + data_hash, + SubmittersByScope::::default() + .with_submitter(account, &quorum_scope) + .unwrap(), + ) + }) +} + +/// Submits the given test submissions. +/// +/// The account and their quorum scope is determined by seeding the `account_fn` and +/// `quorum_scope_fn`. +fn submit_submissions_v1( + mut account_fn: impl FnMut(usize) -> T::AccountId, + mut quorum_scope_fn: impl FnMut(usize) -> QuorumScope, + submissions: impl IntoIterator, +) where + T: Config, + I: NativeApi, +{ + submissions + .into_iter() + .enumerate() + .for_each(|(i, test_submission)| { + let account = account_fn(i); + let quorum_scope = quorum_scope_fn(i); + + let origin = RawOrigin::::Signed(account.clone()).into(); + + match quorum_scope { + QuorumScope::Public => { + let permission = PermissionLevel::IndexingPallet( + IndexingPalletPermission::SubmitDataForPublicQuorum, + ); + if !pallet_permissions::Pallet::::has_permissions(&account, &permission) { + pallet_permissions::Pallet::::add_proxy_permission( + RawOrigin::Root.into(), + account, + permission, + ) + .unwrap(); + } + } + QuorumScope::Privileged => { + let permission = PermissionLevel::IndexingPallet( + IndexingPalletPermission::SubmitDataForPrivilegedQuorum( + test_submission.table.clone(), + ), + ); + if !pallet_permissions::Pallet::::has_permissions(&account, &permission) { + pallet_permissions::Pallet::::add_proxy_permission( + RawOrigin::Root.into(), + account, + permission, + ) + .unwrap(); + } + } + } + + crate::Pallet::::submit_data( + origin, + test_submission.table, + test_submission.batch_id, + test_submission.data, + ) + .unwrap(); + }) +} + +/// Creates the [`sample_table_definition`] table and submits the given `TestSubmission`s for it. +/// +/// Used primarily for pruning tests, hence both versions of the submission storage can be +/// parameterized. +fn setup_sample_table_with_submissions( + submissions_v0: impl IntoIterator, + submissions_v1: impl IntoIterator, +) { + let (table_id, create_statement) = sample_table_definition(); + Tables::create_tables( + RuntimeOrigin::root(), + vec![UpdateTable { + ident: table_id.clone(), + create_statement, + table_type: TableType::Testing(InsertQuorumSize { + public: Some(2), + privileged: Some(2), + }), + commitment: CommitmentCreationCmd::Empty(CommitmentSchemeFlags::all()), + source: sxt_core::tables::Source::Ethereum, + }] + .try_into() + .unwrap(), + ) + .unwrap(); + + populate_submissions_v0::( + account_from_num, + quorum_scope_public_if_even, + submissions_v0, + ); + submit_submissions_v1::( + account_from_num, + quorum_scope_public_if_even, + submissions_v1, + ); +} + +/// Returns the number of `BatchId`s in `SubmissionsV1` storage. +fn count_submissions_v1_batch_ids() -> usize +where + T: Config, + I: NativeApi, +{ + crate::SubmissionsV1::::iter_keys() + .map(|(batch_id, ..)| batch_id) + .collect::>() + .len() +} + +proptest! { + #[test] + fn submissions_v0_are_always_pruned( + submissions_v0 in { + let max_batches_finding_quorum = + <>::MaxBatchesFindingQuorum as Get>::get() as usize; + submissions_for_sample_table(0..max_batches_finding_quorum, 0..4usize, batch_id_strategy()) + }, + // won't trigger v1 submission pruning + submissions_v1 in { + let max_batches_finding_quorum = + <>::MaxBatchesFindingQuorum as Get>::get() as usize; + submissions_for_sample_table_by_batch_id(0..max_batches_finding_quorum, 1..=32usize, 0..4usize, batch_id_strategy()) + }, + ) { + new_test_ext().execute_with(|| { + System::set_block_number(1); + setup_sample_table_with_submissions(submissions_v0.clone(), submissions_v1.clone().into_values().flatten()); + + assert_eq!(crate::Submissions::::iter().count(), submissions_v0.len()); + assert_eq!(count_submissions_v1_batch_ids::(), submissions_v1.len()); + assert_eq!(crate::BatchQueueBottom::::get(), 0); + assert_eq!(crate::BatchQueue::::count(), submissions_v1.len() as u32); + + crate::Pallet::::on_initialize(2); + + let max_batches_pruned = <>::MaxBatchesPruned as Get>::get(); + assert_eq!(crate::Submissions::::iter().count(), submissions_v0.len().saturating_sub(max_batches_pruned as usize)); + assert_eq!(count_submissions_v1_batch_ids::(), submissions_v1.len()); + assert_eq!(crate::BatchQueueBottom::::get(), 0); + assert_eq!(crate::BatchQueue::::count(), submissions_v1.len() as u32); + + let expected_num_pruned_total = (submissions_v0.len() as u32).min(max_batches_pruned); + if expected_num_pruned_total > 0 { + let events = System::read_events_for_pallet::>(); + assert!(events.iter().any( + |event| matches!(event, Event::BatchQueuePruned { num_pruned } + if *num_pruned == expected_num_pruned_total) + )); + } + }); + } + + #[test] + fn submissions_v1_are_pruned_after_submissions_v0( + // Usually pruning of v0 won't satisfy the max batches pruned + submissions_v0 in { + let max_batches_pruned = + <>::MaxBatchesPruned as Get>::get() as usize; + submissions_for_sample_table(0..max_batches_pruned, 0..4usize, batch_id_strategy()) + }, + // Pruning of v1 will begin because max_batches_finding_quorum is exceeded + submissions_v1 in { + let max_batches_finding_quorum = + <>::MaxBatchesFindingQuorum as Get>::get() as usize; + submissions_for_sample_table_by_batch_id(max_batches_finding_quorum..max_batches_finding_quorum*2, 1..=32usize, 0..4usize, batch_id_strategy()) + }, + ) { + new_test_ext().execute_with(|| { + System::set_block_number(1); + setup_sample_table_with_submissions(submissions_v0.clone(), submissions_v1.clone().into_values().flatten()); + + assert_eq!(crate::Submissions::::iter().count(), submissions_v0.len()); + assert_eq!(count_submissions_v1_batch_ids::(), submissions_v1.len()); + assert_eq!(crate::BatchQueueBottom::::get(), 0); + assert_eq!(crate::BatchQueue::::count(), submissions_v1.len() as u32); + + crate::Pallet::::on_initialize(2); + + assert_eq!(crate::Submissions::::iter().count(), 0); + + let max_batches_finding_quorum = + <>::MaxBatchesFindingQuorum as Get>::get(); + let max_batches_pruned = <>::MaxBatchesPruned as Get>::get(); + + let expected_prune_limit = max_batches_pruned - submissions_v0.len() as u32; + let excessive_batches = submissions_v1.len() as u32 - max_batches_finding_quorum; + + let expected_num_pruned = excessive_batches.min(expected_prune_limit); + + assert_eq!(count_submissions_v1_batch_ids::(), submissions_v1.len() - expected_num_pruned as usize); + assert_eq!(crate::BatchQueueBottom::::get(), expected_num_pruned); + assert_eq!(crate::BatchQueue::::count(), submissions_v1.len() as u32 - expected_num_pruned); + + let expected_num_pruned_total = expected_num_pruned + submissions_v0.len() as u32; + + if expected_num_pruned_total > 0 { + let events = System::read_events_for_pallet::>(); + assert!(events.iter().any( + |event| matches!(event, Event::BatchQueuePruned { num_pruned } + if *num_pruned == expected_num_pruned_total) + )); + } + }); + } +} diff --git a/pallets/indexing/src/weights.rs b/pallets/indexing/src/weights.rs index cfa5f1fe..3c51cf6f 100644 --- a/pallets/indexing/src/weights.rs +++ b/pallets/indexing/src/weights.rs @@ -2,7 +2,7 @@ //! Autogenerated weights for `pallet_indexing` //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 43.0.0 -//! DATE: 2025-10-31, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2025-12-03, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` //! WORST CASE MAP SIZE: `1000000` //! HOSTNAME: `trevor-benchmark`, CPU: `AMD EPYC 7763 64-Core Processor` //! WASM-EXECUTION: `Compiled`, CHAIN: `None`, DB CACHE: `1024` @@ -50,16 +50,22 @@ impl WeightInfo for SubstrateWeight { /// Proof: `Indexing::FinalData` (`max_values`: None, `max_size`: Some(2337), added: 4812, mode: `MaxEncodedLen`) /// Storage: `Tables::Schemas` (r:1 w:0) /// Proof: `Tables::Schemas` (`max_values`: None, `max_size`: Some(8358), added: 10833, mode: `MaxEncodedLen`) - /// Storage: `Indexing::Submissions` (r:1 w:1) - /// Proof: `Indexing::Submissions` (`max_values`: None, `max_size`: Some(2151), added: 4626, mode: `MaxEncodedLen`) + /// Storage: `Indexing::SubmissionsV1` (r:2 w:1) + /// Proof: `Indexing::SubmissionsV1` (`max_values`: None, `max_size`: Some(150), added: 2625, mode: `MaxEncodedLen`) + /// Storage: `Indexing::BatchQueueBottom` (r:1 w:0) + /// Proof: `Indexing::BatchQueueBottom` (`max_values`: Some(1), `max_size`: Some(4), added: 499, mode: `MaxEncodedLen`) + /// Storage: `Indexing::CounterForBatchQueue` (r:1 w:1) + /// Proof: `Indexing::CounterForBatchQueue` (`max_values`: Some(1), `max_size`: Some(4), added: 499, mode: `MaxEncodedLen`) + /// Storage: `Indexing::BatchQueue` (r:1 w:1) + /// Proof: `Indexing::BatchQueue` (`max_values`: None, `max_size`: Some(57), added: 2532, mode: `MaxEncodedLen`) fn submit_data_quorum_not_reached() -> Weight { // Proof Size summary in bytes: - // Measured: `611` + // Measured: `1086` // Estimated: `276827554` - // Minimum execution time: 362_637_000 picoseconds. - Weight::from_parts(365_193_000, 276827554) - .saturating_add(T::DbWeight::get().reads(7_u64)) - .saturating_add(T::DbWeight::get().writes(1_u64)) + // Minimum execution time: 393_946_000 picoseconds. + Weight::from_parts(396_661_000, 276827554) + .saturating_add(T::DbWeight::get().reads(11_u64)) + .saturating_add(T::DbWeight::get().writes(3_u64)) } /// Storage: `Tables::TableInsertQuorums` (r:1 w:0) /// Proof: `Tables::TableInsertQuorums` (`max_values`: None, `max_size`: Some(152), added: 2627, mode: `MaxEncodedLen`) @@ -71,18 +77,18 @@ impl WeightInfo for SubstrateWeight { /// Proof: `Indexing::FinalData` (`max_values`: None, `max_size`: Some(2337), added: 4812, mode: `MaxEncodedLen`) /// Storage: `Tables::Schemas` (r:1 w:0) /// Proof: `Tables::Schemas` (`max_values`: None, `max_size`: Some(8358), added: 10833, mode: `MaxEncodedLen`) - /// Storage: `Indexing::Submissions` (r:2 w:1) - /// Proof: `Indexing::Submissions` (`max_values`: None, `max_size`: Some(2151), added: 4626, mode: `MaxEncodedLen`) + /// Storage: `Indexing::SubmissionsV1` (r:5 w:4) + /// Proof: `Indexing::SubmissionsV1` (`max_values`: None, `max_size`: Some(150), added: 2625, mode: `MaxEncodedLen`) /// Storage: `Commitments::CommitmentStorageMap` (r:2 w:2) /// Proof: `Commitments::CommitmentStorageMap` (`max_values`: None, `max_size`: Some(45497), added: 47972, mode: `MaxEncodedLen`) fn submit_data_quorum_reached() -> Weight { // Proof Size summary in bytes: - // Measured: `46053` + // Measured: `46691` // Estimated: `276827554` - // Minimum execution time: 121_426_138_000 picoseconds. - Weight::from_parts(125_614_953_000, 276827554) - .saturating_add(T::DbWeight::get().reads(10_u64)) - .saturating_add(T::DbWeight::get().writes(4_u64)) + // Minimum execution time: 121_523_547_000 picoseconds. + Weight::from_parts(125_296_478_000, 276827554) + .saturating_add(T::DbWeight::get().reads(13_u64)) + .saturating_add(T::DbWeight::get().writes(7_u64)) } } @@ -98,16 +104,22 @@ impl WeightInfo for () { /// Proof: `Indexing::FinalData` (`max_values`: None, `max_size`: Some(2337), added: 4812, mode: `MaxEncodedLen`) /// Storage: `Tables::Schemas` (r:1 w:0) /// Proof: `Tables::Schemas` (`max_values`: None, `max_size`: Some(8358), added: 10833, mode: `MaxEncodedLen`) - /// Storage: `Indexing::Submissions` (r:1 w:1) - /// Proof: `Indexing::Submissions` (`max_values`: None, `max_size`: Some(2151), added: 4626, mode: `MaxEncodedLen`) + /// Storage: `Indexing::SubmissionsV1` (r:2 w:1) + /// Proof: `Indexing::SubmissionsV1` (`max_values`: None, `max_size`: Some(150), added: 2625, mode: `MaxEncodedLen`) + /// Storage: `Indexing::BatchQueueBottom` (r:1 w:0) + /// Proof: `Indexing::BatchQueueBottom` (`max_values`: Some(1), `max_size`: Some(4), added: 499, mode: `MaxEncodedLen`) + /// Storage: `Indexing::CounterForBatchQueue` (r:1 w:1) + /// Proof: `Indexing::CounterForBatchQueue` (`max_values`: Some(1), `max_size`: Some(4), added: 499, mode: `MaxEncodedLen`) + /// Storage: `Indexing::BatchQueue` (r:1 w:1) + /// Proof: `Indexing::BatchQueue` (`max_values`: None, `max_size`: Some(57), added: 2532, mode: `MaxEncodedLen`) fn submit_data_quorum_not_reached() -> Weight { // Proof Size summary in bytes: - // Measured: `611` + // Measured: `1086` // Estimated: `276827554` - // Minimum execution time: 362_637_000 picoseconds. - Weight::from_parts(365_193_000, 276827554) - .saturating_add(RocksDbWeight::get().reads(7_u64)) - .saturating_add(RocksDbWeight::get().writes(1_u64)) + // Minimum execution time: 393_946_000 picoseconds. + Weight::from_parts(396_661_000, 276827554) + .saturating_add(RocksDbWeight::get().reads(11_u64)) + .saturating_add(RocksDbWeight::get().writes(3_u64)) } /// Storage: `Tables::TableInsertQuorums` (r:1 w:0) /// Proof: `Tables::TableInsertQuorums` (`max_values`: None, `max_size`: Some(152), added: 2627, mode: `MaxEncodedLen`) @@ -119,17 +131,17 @@ impl WeightInfo for () { /// Proof: `Indexing::FinalData` (`max_values`: None, `max_size`: Some(2337), added: 4812, mode: `MaxEncodedLen`) /// Storage: `Tables::Schemas` (r:1 w:0) /// Proof: `Tables::Schemas` (`max_values`: None, `max_size`: Some(8358), added: 10833, mode: `MaxEncodedLen`) - /// Storage: `Indexing::Submissions` (r:2 w:1) - /// Proof: `Indexing::Submissions` (`max_values`: None, `max_size`: Some(2151), added: 4626, mode: `MaxEncodedLen`) + /// Storage: `Indexing::SubmissionsV1` (r:5 w:4) + /// Proof: `Indexing::SubmissionsV1` (`max_values`: None, `max_size`: Some(150), added: 2625, mode: `MaxEncodedLen`) /// Storage: `Commitments::CommitmentStorageMap` (r:2 w:2) /// Proof: `Commitments::CommitmentStorageMap` (`max_values`: None, `max_size`: Some(45497), added: 47972, mode: `MaxEncodedLen`) fn submit_data_quorum_reached() -> Weight { // Proof Size summary in bytes: - // Measured: `46053` + // Measured: `46691` // Estimated: `276827554` - // Minimum execution time: 121_426_138_000 picoseconds. - Weight::from_parts(125_614_953_000, 276827554) - .saturating_add(RocksDbWeight::get().reads(10_u64)) - .saturating_add(RocksDbWeight::get().writes(4_u64)) + // Minimum execution time: 121_523_547_000 picoseconds. + Weight::from_parts(125_296_478_000, 276827554) + .saturating_add(RocksDbWeight::get().reads(13_u64)) + .saturating_add(RocksDbWeight::get().writes(7_u64)) } } diff --git a/pallets/smartcontracts/src/mock.rs b/pallets/smartcontracts/src/mock.rs index f1d5d8d6..a81832c7 100644 --- a/pallets/smartcontracts/src/mock.rs +++ b/pallets/smartcontracts/src/mock.rs @@ -191,6 +191,8 @@ impl pallet_commitments::Config for Test { impl pallet_indexing::pallet::Config for Test { type RuntimeEvent = RuntimeEvent; type WeightInfo = pallet_indexing::weights::SubstrateWeight; + type MaxBatchesPruned = ConstU32<3>; + type MaxBatchesFindingQuorum = ConstU32<10>; } impl pallet_system_tables::Config for Test { diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index ae202909..4198b473 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -366,7 +366,7 @@ pub const AVERAGE_INSERT_TARGET_COST: u128 = MILLICENTS pub const TARGET_BYTE_FEE: u128 = AVERAGE_INSERT_TARGET_COST.saturating_div(AVERAGE_INSERT_SIZE_BYTES); /// Approximated Average Insert Weight from actual transactions on testnet -pub const AVERAGE_INSERT_CALL_WEIGHT: u128 = 125_614_953_000; +pub const AVERAGE_INSERT_CALL_WEIGHT: u128 = 125_296_478_000; pub const WEIGHT_FEE: u128 = AVERAGE_INSERT_TARGET_COST.saturating_div(AVERAGE_INSERT_CALL_WEIGHT); parameter_types! { @@ -802,6 +802,8 @@ impl pallet_commitments::Config for Runtime { impl pallet_indexing::Config for Runtime { type RuntimeEvent = RuntimeEvent; type WeightInfo = pallet_indexing::weights::SubstrateWeight; + type MaxBatchesFindingQuorum = ConstU32<500_000>; + type MaxBatchesPruned = ConstU32<100>; } impl pallet_attestation::Config for Runtime { diff --git a/sxt-core/Cargo.toml b/sxt-core/Cargo.toml index a92b9cd7..fbaff126 100644 --- a/sxt-core/Cargo.toml +++ b/sxt-core/Cargo.toml @@ -46,6 +46,7 @@ proptest = { workspace = true, features = ["std"], optional = true } [dev-dependencies] rand = "0.8.5" +proptest = { workspace = true, features = ["std"] } [features] default = ["std"] diff --git a/sxt-core/src/heavy.rs b/sxt-core/src/heavy.rs new file mode 100644 index 00000000..b6712efd --- /dev/null +++ b/sxt-core/src/heavy.rs @@ -0,0 +1,164 @@ +//! Contains [`Heavy`], a monad for composing functions that incur weight. + +use core::iter::Sum; + +use frame_support::weights::Weight; + +/// A simple monad for composing functions that incur polkadot weight. +#[derive(Copy, Clone, PartialEq, Eq, Debug, Default)] +pub struct Heavy { + /// The output of the operation. + pub out: T, + /// The weight of the operation. + pub weight: Weight, +} + +impl Heavy { + /// Maps a `Heavy` to a `Heavy` by applying a function to the contained `out` field. + /// + /// # Examples + /// ``` + /// use frame_support::weights::Weight; + /// use sxt_core::heavy::Heavy; + /// + /// let weight = Weight::zero().set_ref_time(1).set_proof_size(2); + /// + /// let initial = Heavy { + /// out: 4, + /// weight, + /// }; + /// let mapped = initial.map(|n| n.to_string()); + /// + /// assert_eq!(mapped, Heavy { out: "4".to_string(), weight }); + /// ``` + pub fn map(self, f: impl FnOnce(T) -> U) -> Heavy { + let Heavy { out, weight } = self; + Heavy { + out: f(out), + weight, + } + } + + /// Apply a function `f` to the `out` and return a `Heavy` that sums the weights. + /// + /// `f` takes `out: T` and returns `Heavy`. Then, this returns a `Heavy` whose weight is + /// the sum of `self`'s weight and the weight returned by `f`. + /// + /// # Examples + /// ``` + /// use frame_support::weights::Weight; + /// use sxt_core::heavy::Heavy; + /// + /// let weight = Weight::zero().set_ref_time(1).set_proof_size(2); + /// + /// let initial = Heavy { + /// out: 4, + /// weight, + /// }; + /// let composed = initial.and_then(|n| { + /// let weight = Weight::zero().set_ref_time(3).set_proof_size(4); + /// Heavy { out: n.to_string(), weight } + /// }); + /// + /// let expected_weight = Weight::zero().set_ref_time(4).set_proof_size(6); + /// + /// assert_eq!(composed, Heavy { out: "4".to_string(), weight: expected_weight }); + /// ``` + pub fn and_then(self, f: impl FnOnce(T) -> Heavy) -> Heavy { + self.map(f).flatten() + } +} + +impl Heavy> { + /// Flattens a nested `Heavy` by summing the weights. + /// + /// # Examples + /// ``` + /// use frame_support::weights::Weight; + /// use sxt_core::heavy::Heavy; + /// + /// let inner_weight = Weight::zero().set_ref_time(1).set_proof_size(2); + /// let outer_weight = Weight::zero().set_ref_time(3).set_proof_size(4); + /// + /// let nested = Heavy { + /// out: Heavy { + /// out: 4, + /// weight: inner_weight, + /// }, + /// weight: outer_weight, + /// }; + /// let flattened = nested.flatten(); + /// + /// let expected_weight = Weight::zero().set_ref_time(4).set_proof_size(6); + /// + /// assert_eq!(flattened, Heavy { out: 4, weight: expected_weight }); + /// ``` + pub fn flatten(self) -> Heavy { + let Heavy { + out: Heavy { + out, + weight: weight_a, + }, + weight: weight_b, + } = self; + + Heavy { + out, + weight: weight_a.saturating_add(weight_b), + } + } +} + +impl From for Heavy { + fn from(out: T) -> Self { + Heavy { + out, + weight: Weight::zero(), + } + } +} + +impl From for Heavy<()> { + fn from(weight: Weight) -> Self { + Heavy { out: (), weight } + } +} + +impl From> for Weight { + fn from(Heavy { weight, .. }: Heavy<()>) -> Weight { + weight + } +} + +impl Sum> for Heavy<()> { + fn sum>>(iter: I) -> Self { + let weight = iter.fold(Weight::zero(), |weight, heavy| { + weight.saturating_add(heavy.weight) + }); + Heavy { out: (), weight } + } +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use super::*; + + proptest! { + #[test] + fn we_can_sum_heavys(weight_values in proptest::collection::vec((0..1000u64, 0..1000u64), 0..1000)) { + let heavys = weight_values.iter().map(|(ref_time, proof_size)| { + Heavy::from(Weight::zero().set_ref_time(*ref_time).set_proof_size(*proof_size)) + }); + + let sum: Heavy<()> = heavys.sum(); + + let (expected_ref_time, expected_proof_size) = weight_values.iter().fold((0, 0), |acc, values| (acc.0 + values.0, acc.1 + values.1)); + let expected_weight = + Weight::zero().set_ref_time(expected_ref_time).set_proof_size(expected_proof_size); + + assert_eq!(sum, Heavy { out: (), weight: expected_weight}); + } + } +} diff --git a/sxt-core/src/lib.rs b/sxt-core/src/lib.rs index 2722f2ca..876f30c0 100644 --- a/sxt-core/src/lib.rs +++ b/sxt-core/src/lib.rs @@ -52,5 +52,7 @@ pub mod system_tables; /// Utility functions for handling Runtime types pub mod utils; +pub mod heavy; + #[cfg(feature = "proptest")] pub mod proptest; diff --git a/sxt-core/src/sxt_chain_runtime.rs b/sxt-core/src/sxt_chain_runtime.rs index 10419dc0..afc0a0b1 100644 --- a/sxt-core/src/sxt_chain_runtime.rs +++ b/sxt-core/src/sxt_chain_runtime.rs @@ -2560,6 +2560,9 @@ pub mod api { pub fn multi_block_migrations(&self) -> multi_block_migrations::constants::ConstantsApi { multi_block_migrations::constants::ConstantsApi } + pub fn indexing(&self) -> indexing::constants::ConstantsApi { + indexing::constants::ConstantsApi + } } pub struct StorageApi; impl StorageApi { @@ -2739,9 +2742,9 @@ pub mod api { .hash(); runtime_metadata_hash == [ - 78u8, 89u8, 213u8, 113u8, 97u8, 54u8, 134u8, 89u8, 105u8, 190u8, 88u8, 112u8, - 199u8, 151u8, 19u8, 97u8, 185u8, 155u8, 180u8, 99u8, 143u8, 79u8, 236u8, 164u8, - 156u8, 192u8, 190u8, 120u8, 126u8, 37u8, 212u8, 129u8, + 213u8, 207u8, 67u8, 76u8, 14u8, 134u8, 16u8, 0u8, 26u8, 226u8, 103u8, 192u8, 80u8, + 171u8, 62u8, 229u8, 196u8, 6u8, 120u8, 148u8, 179u8, 240u8, 194u8, 5u8, 172u8, + 129u8, 5u8, 66u8, 54u8, 22u8, 129u8, 61u8, ] } pub mod system { @@ -3898,10 +3901,10 @@ pub mod api { "Events", (), [ - 112u8, 30u8, 128u8, 176u8, 162u8, 232u8, 158u8, 104u8, 88u8, 132u8, - 119u8, 162u8, 28u8, 146u8, 225u8, 47u8, 141u8, 212u8, 26u8, 209u8, - 249u8, 45u8, 234u8, 203u8, 214u8, 60u8, 86u8, 41u8, 135u8, 195u8, - 137u8, 235u8, + 238u8, 151u8, 62u8, 82u8, 150u8, 100u8, 246u8, 110u8, 206u8, 176u8, + 195u8, 241u8, 39u8, 161u8, 140u8, 205u8, 101u8, 195u8, 93u8, 87u8, + 118u8, 13u8, 233u8, 167u8, 226u8, 140u8, 172u8, 192u8, 247u8, 223u8, + 248u8, 82u8, ], ) } @@ -20170,6 +20173,29 @@ pub mod api { const PALLET: &'static str = "Indexing"; const EVENT: &'static str = "QuorumEmptyBlock"; } + #[derive( + :: subxt :: ext :: subxt_core :: ext :: codec :: Decode, + :: subxt :: ext :: subxt_core :: ext :: codec :: Encode, + :: subxt :: ext :: subxt_core :: ext :: scale_decode :: DecodeAsType, + :: subxt :: ext :: subxt_core :: ext :: scale_encode :: EncodeAsType, + Debug, + )] + # [codec (crate = :: subxt :: ext :: subxt_core :: ext :: codec)] + #[codec(dumb_trait_bound)] + #[decode_as_type(crate_path = ":: subxt :: ext :: subxt_core :: ext :: scale_decode")] + #[encode_as_type(crate_path = ":: subxt :: ext :: subxt_core :: ext :: scale_encode")] + #[doc = "The `BatchQueue` and submissions storage have been pruned."] + pub struct BatchQueuePruned { + pub num_pruned: batch_queue_pruned::NumPruned, + } + pub mod batch_queue_pruned { + use super::runtime_types; + pub type NumPruned = ::core::primitive::u32; + } + impl ::subxt::ext::subxt_core::events::StaticEvent for BatchQueuePruned { + const PALLET: &'static str = "Indexing"; + const EVENT: &'static str = "BatchQueuePruned"; + } } pub mod storage { use super::runtime_types; @@ -20185,6 +20211,31 @@ pub mod api { >; pub type Param1 = ::subxt::ext::subxt_core::utils::H256; } + pub mod submissions_v1 { + use super::runtime_types; + pub type SubmissionsV1 = ::subxt::ext::subxt_core::utils::H256; + pub type Param0 = runtime_types::bounded_collections::bounded_vec::BoundedVec< + ::core::primitive::u8, + >; + pub type Param1 = runtime_types::sxt_core::tables::QuorumScope; + pub type Param2 = ::subxt::ext::subxt_core::utils::AccountId32; + } + pub mod batch_queue { + use super::runtime_types; + pub type BatchQueue = + runtime_types::bounded_collections::bounded_vec::BoundedVec< + ::core::primitive::u8, + >; + pub type Param0 = ::core::primitive::u32; + } + pub mod counter_for_batch_queue { + use super::runtime_types; + pub type CounterForBatchQueue = ::core::primitive::u32; + } + pub mod batch_queue_bottom { + use super::runtime_types; + pub type BatchQueueBottom = ::core::primitive::u32; + } pub mod final_data { use super::runtime_types; pub type FinalData = runtime_types::sxt_core::indexing::DataQuorum< @@ -20287,6 +20338,244 @@ pub mod api { ], ) } + #[doc = " Storage map of `(BatchId, QuorumScope, AccountId)` to data hash for batches that are still"] + #[doc = " finding quorum."] + #[doc = ""] + #[doc = " Will get pruned once `Config::MaxBatchesFindingQuorum` is reached."] + pub fn submissions_v1_iter( + &self, + ) -> ::subxt::ext::subxt_core::storage::address::StaticAddress< + (), + types::submissions_v1::SubmissionsV1, + (), + (), + ::subxt::ext::subxt_core::utils::Yes, + > { + ::subxt::ext::subxt_core::storage::address::StaticAddress::new_static( + "Indexing", + "SubmissionsV1", + (), + [ + 203u8, 99u8, 162u8, 223u8, 54u8, 150u8, 101u8, 19u8, 163u8, 5u8, 3u8, + 186u8, 65u8, 194u8, 49u8, 221u8, 105u8, 104u8, 27u8, 141u8, 109u8, + 42u8, 144u8, 42u8, 119u8, 72u8, 50u8, 236u8, 124u8, 6u8, 45u8, 49u8, + ], + ) + } + #[doc = " Storage map of `(BatchId, QuorumScope, AccountId)` to data hash for batches that are still"] + #[doc = " finding quorum."] + #[doc = ""] + #[doc = " Will get pruned once `Config::MaxBatchesFindingQuorum` is reached."] + pub fn submissions_v1_iter1( + &self, + _0: impl ::core::borrow::Borrow, + ) -> ::subxt::ext::subxt_core::storage::address::StaticAddress< + ::subxt::ext::subxt_core::storage::address::StaticStorageKey< + types::submissions_v1::Param0, + >, + types::submissions_v1::SubmissionsV1, + (), + (), + ::subxt::ext::subxt_core::utils::Yes, + > { + ::subxt::ext::subxt_core::storage::address::StaticAddress::new_static( + "Indexing", + "SubmissionsV1", + ::subxt::ext::subxt_core::storage::address::StaticStorageKey::new( + _0.borrow(), + ), + [ + 203u8, 99u8, 162u8, 223u8, 54u8, 150u8, 101u8, 19u8, 163u8, 5u8, 3u8, + 186u8, 65u8, 194u8, 49u8, 221u8, 105u8, 104u8, 27u8, 141u8, 109u8, + 42u8, 144u8, 42u8, 119u8, 72u8, 50u8, 236u8, 124u8, 6u8, 45u8, 49u8, + ], + ) + } + #[doc = " Storage map of `(BatchId, QuorumScope, AccountId)` to data hash for batches that are still"] + #[doc = " finding quorum."] + #[doc = ""] + #[doc = " Will get pruned once `Config::MaxBatchesFindingQuorum` is reached."] + pub fn submissions_v1_iter2( + &self, + _0: impl ::core::borrow::Borrow, + _1: impl ::core::borrow::Borrow, + ) -> ::subxt::ext::subxt_core::storage::address::StaticAddress< + ( + ::subxt::ext::subxt_core::storage::address::StaticStorageKey< + types::submissions_v1::Param0, + >, + ::subxt::ext::subxt_core::storage::address::StaticStorageKey< + types::submissions_v1::Param1, + >, + ), + types::submissions_v1::SubmissionsV1, + (), + (), + ::subxt::ext::subxt_core::utils::Yes, + > { + ::subxt::ext::subxt_core::storage::address::StaticAddress::new_static( + "Indexing", + "SubmissionsV1", + ( + ::subxt::ext::subxt_core::storage::address::StaticStorageKey::new( + _0.borrow(), + ), + ::subxt::ext::subxt_core::storage::address::StaticStorageKey::new( + _1.borrow(), + ), + ), + [ + 203u8, 99u8, 162u8, 223u8, 54u8, 150u8, 101u8, 19u8, 163u8, 5u8, 3u8, + 186u8, 65u8, 194u8, 49u8, 221u8, 105u8, 104u8, 27u8, 141u8, 109u8, + 42u8, 144u8, 42u8, 119u8, 72u8, 50u8, 236u8, 124u8, 6u8, 45u8, 49u8, + ], + ) + } + #[doc = " Storage map of `(BatchId, QuorumScope, AccountId)` to data hash for batches that are still"] + #[doc = " finding quorum."] + #[doc = ""] + #[doc = " Will get pruned once `Config::MaxBatchesFindingQuorum` is reached."] + pub fn submissions_v1( + &self, + _0: impl ::core::borrow::Borrow, + _1: impl ::core::borrow::Borrow, + _2: impl ::core::borrow::Borrow, + ) -> ::subxt::ext::subxt_core::storage::address::StaticAddress< + ( + ::subxt::ext::subxt_core::storage::address::StaticStorageKey< + types::submissions_v1::Param0, + >, + ::subxt::ext::subxt_core::storage::address::StaticStorageKey< + types::submissions_v1::Param1, + >, + ::subxt::ext::subxt_core::storage::address::StaticStorageKey< + types::submissions_v1::Param2, + >, + ), + types::submissions_v1::SubmissionsV1, + ::subxt::ext::subxt_core::utils::Yes, + (), + (), + > { + ::subxt::ext::subxt_core::storage::address::StaticAddress::new_static( + "Indexing", + "SubmissionsV1", + ( + ::subxt::ext::subxt_core::storage::address::StaticStorageKey::new( + _0.borrow(), + ), + ::subxt::ext::subxt_core::storage::address::StaticStorageKey::new( + _1.borrow(), + ), + ::subxt::ext::subxt_core::storage::address::StaticStorageKey::new( + _2.borrow(), + ), + ), + [ + 203u8, 99u8, 162u8, 223u8, 54u8, 150u8, 101u8, 19u8, 163u8, 5u8, 3u8, + 186u8, 65u8, 194u8, 49u8, 221u8, 105u8, 104u8, 27u8, 141u8, 109u8, + 42u8, 144u8, 42u8, 119u8, 72u8, 50u8, 236u8, 124u8, 6u8, 45u8, 49u8, + ], + ) + } + #[doc = " Storage map of `BatchId`s by batch index, in the order of first submission."] + #[doc = ""] + #[doc = " Will get pruned once `Config::MaxBatchesFindingQuorum` is reached."] + pub fn batch_queue_iter( + &self, + ) -> ::subxt::ext::subxt_core::storage::address::StaticAddress< + (), + types::batch_queue::BatchQueue, + (), + (), + ::subxt::ext::subxt_core::utils::Yes, + > { + ::subxt::ext::subxt_core::storage::address::StaticAddress::new_static( + "Indexing", + "BatchQueue", + (), + [ + 236u8, 125u8, 175u8, 198u8, 10u8, 130u8, 27u8, 232u8, 51u8, 32u8, + 243u8, 213u8, 181u8, 196u8, 218u8, 197u8, 35u8, 49u8, 58u8, 92u8, + 150u8, 197u8, 255u8, 138u8, 235u8, 128u8, 112u8, 66u8, 11u8, 152u8, + 21u8, 124u8, + ], + ) + } + #[doc = " Storage map of `BatchId`s by batch index, in the order of first submission."] + #[doc = ""] + #[doc = " Will get pruned once `Config::MaxBatchesFindingQuorum` is reached."] + pub fn batch_queue( + &self, + _0: impl ::core::borrow::Borrow, + ) -> ::subxt::ext::subxt_core::storage::address::StaticAddress< + ::subxt::ext::subxt_core::storage::address::StaticStorageKey< + types::batch_queue::Param0, + >, + types::batch_queue::BatchQueue, + ::subxt::ext::subxt_core::utils::Yes, + (), + (), + > { + ::subxt::ext::subxt_core::storage::address::StaticAddress::new_static( + "Indexing", + "BatchQueue", + ::subxt::ext::subxt_core::storage::address::StaticStorageKey::new( + _0.borrow(), + ), + [ + 236u8, 125u8, 175u8, 198u8, 10u8, 130u8, 27u8, 232u8, 51u8, 32u8, + 243u8, 213u8, 181u8, 196u8, 218u8, 197u8, 35u8, 49u8, 58u8, 92u8, + 150u8, 197u8, 255u8, 138u8, 235u8, 128u8, 112u8, 66u8, 11u8, 152u8, + 21u8, 124u8, + ], + ) + } + #[doc = "Counter for the related counted storage map"] + pub fn counter_for_batch_queue( + &self, + ) -> ::subxt::ext::subxt_core::storage::address::StaticAddress< + (), + types::counter_for_batch_queue::CounterForBatchQueue, + ::subxt::ext::subxt_core::utils::Yes, + ::subxt::ext::subxt_core::utils::Yes, + (), + > { + ::subxt::ext::subxt_core::storage::address::StaticAddress::new_static( + "Indexing", + "CounterForBatchQueue", + (), + [ + 157u8, 209u8, 245u8, 148u8, 196u8, 247u8, 155u8, 87u8, 52u8, 105u8, + 9u8, 68u8, 7u8, 83u8, 69u8, 242u8, 149u8, 21u8, 147u8, 251u8, 16u8, + 198u8, 3u8, 252u8, 88u8, 89u8, 214u8, 115u8, 212u8, 66u8, 207u8, 153u8, + ], + ) + } + #[doc = " The lowest index currently in the `BatchQueue`."] + #[doc = ""] + #[doc = " Will increment as the `BatchQueue` is pruned."] + pub fn batch_queue_bottom( + &self, + ) -> ::subxt::ext::subxt_core::storage::address::StaticAddress< + (), + types::batch_queue_bottom::BatchQueueBottom, + ::subxt::ext::subxt_core::utils::Yes, + ::subxt::ext::subxt_core::utils::Yes, + (), + > { + ::subxt::ext::subxt_core::storage::address::StaticAddress::new_static( + "Indexing", + "BatchQueueBottom", + (), + [ + 125u8, 144u8, 49u8, 157u8, 239u8, 173u8, 129u8, 44u8, 110u8, 41u8, + 201u8, 209u8, 228u8, 241u8, 24u8, 29u8, 117u8, 209u8, 18u8, 210u8, + 126u8, 168u8, 136u8, 124u8, 140u8, 1u8, 96u8, 125u8, 234u8, 50u8, 90u8, + 148u8, + ], + ) + } #[doc = " Storage map of `BatchId`s to `DataQuorum`s for batches that have reached quorum."] pub fn final_data_iter( &self, @@ -20393,6 +20682,47 @@ pub mod api { } } } + pub mod constants { + use super::runtime_types; + pub struct ConstantsApi; + impl ConstantsApi { + #[doc = " The maximum batches finding quorum at any given time."] + pub fn max_batches_finding_quorum( + &self, + ) -> ::subxt::ext::subxt_core::constants::address::StaticAddress< + ::core::primitive::u32, + > { + ::subxt::ext::subxt_core::constants::address::StaticAddress::new_static( + "Indexing", + "MaxBatchesFindingQuorum", + [ + 98u8, 252u8, 116u8, 72u8, 26u8, 180u8, 225u8, 83u8, 200u8, 157u8, + 125u8, 151u8, 53u8, 76u8, 168u8, 26u8, 10u8, 9u8, 98u8, 68u8, 9u8, + 178u8, 197u8, 113u8, 31u8, 79u8, 200u8, 90u8, 203u8, 100u8, 41u8, + 145u8, + ], + ) + } + #[doc = " The maximum batches pruned per transaction from submissions storage when it exceeds"] + #[doc = " `MaxBatchesPruned`."] + pub fn max_batches_pruned( + &self, + ) -> ::subxt::ext::subxt_core::constants::address::StaticAddress< + ::core::primitive::u32, + > { + ::subxt::ext::subxt_core::constants::address::StaticAddress::new_static( + "Indexing", + "MaxBatchesPruned", + [ + 98u8, 252u8, 116u8, 72u8, 26u8, 180u8, 225u8, 83u8, 200u8, 157u8, + 125u8, 151u8, 53u8, 76u8, 168u8, 26u8, 10u8, 9u8, 98u8, 68u8, 9u8, + 178u8, 197u8, 113u8, 31u8, 79u8, 200u8, 90u8, 203u8, 100u8, 41u8, + 145u8, + ], + ) + } + } + } } pub mod commitments { use super::{root_mod, runtime_types}; @@ -25423,6 +25753,9 @@ pub mod api { #[codec(index = 18)] #[doc = "Submitter Injection Failed"] SubmitterInjectionFailed, + #[codec(index = 19)] + #[doc = "Maximum submissions already reached for this batch id"] + MaxSubmittersReached, } #[derive( :: subxt :: ext :: subxt_core :: ext :: codec :: Decode, @@ -25494,6 +25827,9 @@ pub mod api { ::subxt::ext::subxt_core::utils::AccountId32, >, }, + #[codec(index = 5)] + #[doc = "The `BatchQueue` and submissions storage have been pruned."] + BatchQueuePruned { num_pruned: ::core::primitive::u32 }, } } } diff --git a/sxt-core/src/tables.rs b/sxt-core/src/tables.rs index 163bfd2a..1c64862b 100644 --- a/sxt-core/src/tables.rs +++ b/sxt-core/src/tables.rs @@ -158,6 +158,7 @@ pub type TableUuid = BoundedVec>; Serialize, Deserialize, )] +#[cfg_attr(feature = "std", derive(Hash))] pub struct TableIdentifier { /// The name of the table, utf8-encoded pub name: TableName, @@ -374,6 +375,13 @@ pub enum QuorumScope { Privileged, } +impl QuorumScope { + /// Number of scopes. + /// + /// Replace with core::mem::variant_count when it is stable/no_std. + pub const VARIANT_COUNT: usize = 2; +} + /// Quorum sizes to exceed to insert to a table for all [`QuorumScope`]s. #[derive( Copy,