From 7e4aab01d4317ba4f610df6a5adbc17461a3cd0c Mon Sep 17 00:00:00 2001 From: Michael Assaf Date: Tue, 19 Mar 2024 17:58:31 -0400 Subject: [PATCH 01/18] bsp stop storing, storage request refactor --- Cargo.lock | 1 + pallets/file-system/Cargo.toml | 3 +- pallets/file-system/src/lib.rs | 139 +++++++++++++++++++++------- pallets/file-system/src/mock.rs | 5 +- pallets/file-system/src/tests.rs | 52 ++++++++--- pallets/file-system/src/types.rs | 51 ++++++++--- pallets/file-system/src/utils.rs | 151 ++++++++++++++++++++++++------- runtime/src/lib.rs | 11 ++- 8 files changed, 314 insertions(+), 99 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cdd611518..5cdc98af4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6425,6 +6425,7 @@ dependencies = [ "sp-core", "sp-io", "sp-runtime", + "sp-std 8.0.0 (git+https://github.com/paritytech/polkadot-sdk.git?tag=polkadot-v1.5.0)", ] [[package]] diff --git a/pallets/file-system/Cargo.toml b/pallets/file-system/Cargo.toml index ea0f63f63..d286f1e31 100644 --- a/pallets/file-system/Cargo.toml +++ b/pallets/file-system/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-file-system" -description = "FRAME pallet template for defining custom runtime logic." +description = "Pallet exposing storage related actions actors can execute within Storage Hub." version = "0.1.0" homepage = { workspace = true } license = { workspace = true } @@ -28,6 +28,7 @@ frame-support = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = frame-system = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false } sp-runtime = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false } +sp-std = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false } [dev-dependencies] serde = { version = "1.0.193" } diff --git a/pallets/file-system/src/lib.rs b/pallets/file-system/src/lib.rs index 9165f5c91..4206f1476 100644 --- a/pallets/file-system/src/lib.rs +++ b/pallets/file-system/src/lib.rs @@ -51,7 +51,7 @@ pub mod pallet { sp_runtime::traits::{AtLeast32Bit, CheckEqual, MaybeDisplay, SimpleBitOps}, }; use frame_system::pallet_prelude::{BlockNumberFor, *}; - use sp_runtime::traits::CheckedAdd; + use sp_runtime::traits::{CheckedAdd, Zero}; use sp_runtime::BoundedVec; #[pallet::config] @@ -84,6 +84,26 @@ pub mod pallet { + MaxEncodedLen + HasCompact; + /// Unit representing the size of a file. + type StorageRequestBspsRequiredType: Parameter + + Member + + MaybeSerializeDeserialize + + Default + + MaybeDisplay + + AtLeast32Bit + + Copy + + MaxEncodedLen + + HasCompact + + Copy + + Default + + scale_info::TypeInfo + + MaybeSerializeDeserialize + + Zero; + + /// Minimum number of BSPs required to store a file. + #[pallet::constant] + type DefaultBspsRequired: Get; + /// Maximum number of BSPs that can store a file. #[pallet::constant] type MaxBspsPerStorageRequest: Get; @@ -96,6 +116,10 @@ pub mod pallet { #[pallet::constant] type MaxMultiAddressSize: Get; + /// Maximum number of multiaddresses for a storage request. + #[pallet::constant] + type MaxMultiAddresses: Get; + /// Time-to-live for a storage request. #[pallet::constant] type StorageRequestTtl: Get; @@ -113,10 +137,25 @@ pub mod pallet { pub type StorageRequests = StorageMap<_, Blake2_128Concat, FileLocation, StorageRequestMetadata>; - /// A map of storage requests to their expiration block. + /// A double map of [`storage request`](FileLocation) to [`BSPs`](StorageProviderId) that volunteered to store data. + /// + /// Any BSP under a storage request is considered to be a volunteer and can be removed at any time. + /// Once a BSP submits a valid proof to the `pallet-proofs-dealer-trie`, the `confirmed` field in [`StorageRequestBsps`] should be set to `true`. /// - /// The key is the block number at which the storage request will expire. - /// The value is a list of file locations that will expire at the given block number. (file locations map to storage requests) + /// When a storage request is expired or removed, the corresponding storage request key in this map should be removed. + #[pallet::storage] + #[pallet::getter(fn storage_request_bsps)] + pub type StorageRequestBsps = StorageDoubleMap< + _, + Blake2_128Concat, + FileLocation, + Blake2_128Concat, + StorageProviderId, + StorageRequestBspsMetadata, + OptionQuery, + >; + + /// A map of blocks to expired storage requests. #[pallet::storage] #[pallet::getter(fn storage_request_expirations)] pub type StorageRequestExpirations = StorageMap< @@ -129,23 +168,17 @@ pub mod pallet { /// A pointer to the earliest available block to insert a new storage request expiration. /// - /// This should always be equal or greater than `current_block` + [`Config::StorageRequestTtl`]. - /// - /// In the event when this value is smaller than `current_block` + `StorageRequestTtl` value, the - /// storage request expiration will be inserted in the block `StorageRequestTtl` ahead, and then - /// this value will be reset to block number a `current_block` + `StorageRequestTtl`. + /// This should always be greater or equal than current block + [`Config::StorageRequestTtl`]. #[pallet::storage] #[pallet::getter(fn next_available_expiration_insertion_block)] pub type NextAvailableExpirationInsertionBlock = StorageValue<_, BlockNumberFor, ValueQuery>; - /// A pointer to the latest block at which the storage requests were cleaned up. + /// A pointer to the starting block to clean up expired storage requests. /// - /// This value keeps track of the last block at which the storage requests were cleaned up, and - /// it is needed because the clean-up process is not guaranteed to happen in every block, since - /// it is executed in the `on_idle` hook. If a given block doesn't have enough remaining weight - /// to perform the clean-up, the clean-up will be postponed to the next block, and this value - /// avoids skipping blocks when the clean-up is postponed. + /// If this block is behind the current block number, the cleanup algorithm in `on_idle` will + /// attempt to accelerate this block pointer as close to or up to the current block number if there is + /// enough remaining weight to do so. #[pallet::storage] #[pallet::getter(fn next_starting_block_to_clean_up)] pub type NextStartingBlockToCleanUp = StorageValue<_, BlockNumberFor, ValueQuery>; @@ -159,7 +192,7 @@ pub mod pallet { location: FileLocation, fingerprint: Fingerprint, size: StorageUnit, - user_multiaddr: MultiAddress, + multiaddresses: BoundedVec, T::MaxMultiAddresses>, }, /// Notifies that a BSP has been accepted to store a given file. @@ -167,7 +200,7 @@ pub mod pallet { who: T::AccountId, location: FileLocation, fingerprint: Fingerprint, - bsp_multiaddress: MultiAddress, + multiaddresses: MultiAddresses, }, /// Notifies the expiration of a storage request. @@ -175,6 +208,12 @@ pub mod pallet { /// Notifies that a storage request has been revoked by the user who initiated it. StorageRequestRevoked { location: FileLocation }, + + /// Notifies that a BSP has stopped storing a file. + BspStoppedStoring { + bsp: T::AccountId, + location: FileLocation, + }, } // Errors inform users that something went wrong. @@ -183,18 +222,23 @@ pub mod pallet { /// Storage request already registered for the given file. StorageRequestAlreadyRegistered, /// Storage request not registered for the given file. - StorageRequestNotRegistered, + StorageRequestNotFound, + /// BSPs required for storage request cannot be 0. + BspsRequiredCannotBeZero, + /// BSPs required for storage request cannot exceed the maximum allowed. + BspsRequiredExceedsMax, /// BSP already volunteered to store the given file. BspVolunteerFailed, - /// BSP already confirmed to store the given file. - BspAlreadyConfirmed, + /// Number of BSPs required for storage request has been reached. + StorageRequestBspsRequiredFullfilled, + /// BSP already volunteered to store the given file. + BspAlreadyVolunteered, /// No slot available found in blocks to insert storage request expiration time. StorageRequestExpiredNoSlotAvailable, - /// The current expiration block has overflowed (i.e. it is larger than the maximum block number). - StorageRequestExpirationBlockOverflow, /// Not authorized to delete the storage request. StorageRequestNotAuthorized, - /// Reached maximum block number :O + /// Error created in 2024. If you see this, you are well beyond the singularity and should + /// probably stop using this pallet. MaxBlockNumberReached, } @@ -214,7 +258,7 @@ pub mod pallet { location: FileLocation, fingerprint: Fingerprint, size: StorageUnit, - user_multiaddr: MultiAddress, + multiaddresses: MultiAddresses, ) -> DispatchResult { // Check that the extrinsic was signed and get the signer let who = ensure_signed(origin)?; @@ -225,7 +269,9 @@ pub mod pallet { location.clone(), fingerprint, size, - user_multiaddr.clone(), + None, + Some(multiaddresses.clone()), + Default::default(), )?; // BSPs listen to this event and volunteer to store the file @@ -234,7 +280,7 @@ pub mod pallet { location, fingerprint, size, - user_multiaddr, + multiaddresses, }); Ok(()) @@ -271,7 +317,7 @@ pub mod pallet { origin: OriginFor, location: FileLocation, fingerprint: Fingerprint, - bsp_multiaddress: MultiAddress, + multiaddresses: MultiAddresses, ) -> DispatchResult { // Check that the extrinsic was signed and get the signer. let who = ensure_signed(origin)?; @@ -284,7 +330,7 @@ pub mod pallet { who, location, fingerprint, - bsp_multiaddress, + multiaddresses, }); Ok(()) @@ -292,11 +338,37 @@ pub mod pallet { /// Executed by a BSP to stop storing a file. /// - /// A compensation should be provided for the user, to deter this behaviour. A successful execution of this extrinsic automatically generates a storage request for that file with one remaining_bsps_slot left, and if a storage request for that file already exists, the slots left are incremented in one. It also automatically registers a challenge for this file, for the next round of storage proofs, so that the other BSPs and MSP who are storing it would be forced to disclose themselves then. + /// In the event when a storage request no longer exists for the data the BSP no longer stores, + /// it is required that the BSP still has access to the metadata of the initial storage request. + /// If they do not, they will at least need the necessary data to reconstruct the bucket ID and + /// request the metadata from an MSP. This metadata is necessary since it is needed to reconstruct + /// the leaf node key in the storage provider's merkle trie. #[pallet::call_index(5)] #[pallet::weight(10_000 + T::DbWeight::get().reads_writes(1,1).ref_time())] - pub fn bsp_stop_storing(_origin: OriginFor) -> DispatchResult { - todo!() + pub fn bsp_stop_storing( + origin: OriginFor, + location: FileLocation, + owner: T::AccountId, + fingerprint: Fingerprint, + size: StorageUnit, + can_serve: bool, + ) -> DispatchResult { + let who = ensure_signed(origin)?; + + // Perform validations and stop storing the file. + Self::do_bsp_stop_storing( + who.clone(), + location.clone(), + owner, + fingerprint, + size, + can_serve, + )?; + + // Emit event. + Self::deposit_event(Event::BspStoppedStoring { bsp: who, location }); + + Ok(()) } } @@ -313,8 +385,9 @@ pub mod pallet { let start_block = NextStartingBlockToCleanUp::::get(); let mut block_to_clean = start_block; - // Total weight used to avoid exceeding the remaining weight - let mut total_used_weight = Weight::zero(); + // Total weight used to avoid exceeding the remaining weight. + // We count one write for the `NextBlockToCleanup` storage item updated at the end. + let mut total_used_weight = db_weight.writes(1); let required_weight_for_iteration = db_weight.reads_writes(1, T::MaxExpiredStorageRequests::get().into()); diff --git a/pallets/file-system/src/mock.rs b/pallets/file-system/src/mock.rs index d5615bd2f..4c5cd8e06 100644 --- a/pallets/file-system/src/mock.rs +++ b/pallets/file-system/src/mock.rs @@ -76,7 +76,10 @@ impl crate::Config for Test { type RuntimeEvent = RuntimeEvent; type Fingerprint = H256; type StorageUnit = u128; - type MaxBspsPerStorageRequest = ConstU32<5u32>; + type StorageRequestBspsRequiredType = u32; + type DefaultBspsRequired = ConstU32<1>; + type MaxBspsPerStorageRequest = ConstU32<5>; + type MaxMultiAddresses = ConstU32<5>; // TODO: this should probably be a multiplier of the number of maximum multiaddresses per storage provider type MaxFilePathSize = ConstU32<512u32>; type MaxMultiAddressSize = ConstU32<512u32>; type StorageRequestTtl = ConstU32<40u32>; diff --git a/pallets/file-system/src/tests.rs b/pallets/file-system/src/tests.rs index bdaeae0c9..de81c6771 100644 --- a/pallets/file-system/src/tests.rs +++ b/pallets/file-system/src/tests.rs @@ -1,4 +1,8 @@ -use crate::{mock::*, types::FileLocation, Config, Event, StorageRequestExpirations}; +use crate::{ + mock::*, + types::{FileLocation, MultiAddress}, + Config, Event, StorageRequestExpirations, +}; use frame_support::{assert_ok, traits::Hooks, weights::Weight}; use sp_runtime::{ traits::{BlakeTwo256, Get, Hash}, @@ -15,6 +19,9 @@ fn request_storage_success() { let location = FileLocation::::try_from(b"test".to_vec()).unwrap(); let file_content = b"test".to_vec(); let fingerprint = BlakeTwo256::hash(&file_content); + let multiaddr = BoundedVec::try_from(vec![1]).unwrap(); + let multiaddresses: BoundedVec, ::MaxMultiAddresses> = + BoundedVec::try_from(vec![multiaddr]).unwrap(); // Dispatch a signed extrinsic. assert_ok!(FileSystem::issue_storage_request( @@ -22,7 +29,7 @@ fn request_storage_success() { location.clone(), fingerprint, 4, - BoundedVec::try_from(vec![1]).unwrap(), + multiaddresses.clone(), )); // Assert that the correct event was deposited @@ -32,7 +39,7 @@ fn request_storage_success() { location: location.clone(), fingerprint, size: 4, - user_multiaddr: BoundedVec::try_from(vec![1]).unwrap(), + multiaddresses, } .into(), ); @@ -49,6 +56,9 @@ fn request_storage_expiration_clear_success() { let location = FileLocation::::try_from(b"test".to_vec()).unwrap(); let file_content = b"test".to_vec(); let fingerprint = BlakeTwo256::hash(&file_content); + let multiaddr = BoundedVec::try_from(vec![1]).unwrap(); + let multiaddresses: BoundedVec, ::MaxMultiAddresses> = + BoundedVec::try_from(vec![multiaddr]).unwrap(); // Dispatch a signed extrinsic. assert_ok!(FileSystem::issue_storage_request( @@ -56,7 +66,7 @@ fn request_storage_expiration_clear_success() { location.clone(), fingerprint, 4, - BoundedVec::try_from(vec![1]).unwrap(), + multiaddresses, )); let expected_expiration_inserted_at_block_number: BlockNumber = @@ -88,6 +98,9 @@ fn request_storage_expiration_current_block_increment_success() { let location = FileLocation::::try_from(b"test".to_vec()).unwrap(); let file_content = b"test".to_vec(); let fingerprint = BlakeTwo256::hash(&file_content); + let multiaddr = BoundedVec::try_from(vec![1]).unwrap(); + let multiaddresses: BoundedVec, ::MaxMultiAddresses> = + BoundedVec::try_from(vec![multiaddr]).unwrap(); let mut expected_expiration_block_number: BlockNumber = FileSystem::next_expiration_insertion_block_number().into(); @@ -107,7 +120,7 @@ fn request_storage_expiration_current_block_increment_success() { location.clone(), fingerprint, 4, - BoundedVec::try_from(vec![1]).unwrap(), + multiaddresses, )); // Assert that the storage request expirations storage is at max capacity @@ -137,7 +150,7 @@ fn request_storage_expiration_current_block_increment_success() { } #[test] -fn request_storage_expiration_current_block_increment_when_on_idle_skips_success() { +fn request_storage_clear_old_expirations_success() { new_test_ext().execute_with(|| { // Go past genesis block so events get deposited System::set_block_number(1); @@ -146,6 +159,9 @@ fn request_storage_expiration_current_block_increment_when_on_idle_skips_success let location = FileLocation::::try_from(b"test".to_vec()).unwrap(); let file_content = b"test".to_vec(); let fingerprint = BlakeTwo256::hash(&file_content); + let multiaddr = BoundedVec::try_from(vec![1]).unwrap(); + let multiaddresses: BoundedVec, ::MaxMultiAddresses> = + BoundedVec::try_from(vec![multiaddr]).unwrap(); let mut expected_expiration_block_number: BlockNumber = FileSystem::next_expiration_insertion_block_number().into(); @@ -165,7 +181,7 @@ fn request_storage_expiration_current_block_increment_when_on_idle_skips_success location.clone(), fingerprint, 4, - BoundedVec::try_from(vec![1]).unwrap(), + multiaddresses, )); // Assert that the storage request expirations storage is at max capacity @@ -188,12 +204,19 @@ fn request_storage_expiration_current_block_increment_when_on_idle_skips_success // Assert that the `NextExpirationInsertionBlockNumber` storage is set to 0 initially assert_eq!(FileSystem::next_starting_block_to_clean_up(), 0); + // Assert that the storage request expirations storage is at max capacity + assert_eq!( + FileSystem::storage_request_expirations(expected_expiration_block_number).len(), + max_storage_request_expiry as usize + ); + let used_weight = FileSystem::on_idle(System::block_number(), Weight::zero()); // Assert that the weight used is zero assert_eq!(used_weight, Weight::zero()); // Assert that the storage request expirations storage is at max capacity + // TODO: Fix this test... assert_eq!( FileSystem::storage_request_expirations(expected_expiration_block_number).len(), max_storage_request_expiry as usize @@ -236,7 +259,7 @@ fn revoke_request_storage_success() { location.clone(), fingerprint, 4, - BoundedVec::try_from(vec![1]).unwrap(), + Default::default() )); // Assert that the storage request expiration was appended to the list at `StorageRequestTtl` @@ -269,10 +292,9 @@ fn bsp_volunteer_success() { let location = FileLocation::::try_from(b"test".to_vec()).unwrap(); let file_content = b"test".to_vec(); let fingerprint = BlakeTwo256::hash(&file_content); - - // TODO add this after adding identity pallet - // Register BSP in Identity Pallet. - // assert_ok!(Identity::register_user(RuntimeOrigin::root(), 2)); + let multiaddr = BoundedVec::try_from(vec![1]).unwrap(); + let multiaddresses: BoundedVec, ::MaxMultiAddresses> = + BoundedVec::try_from(vec![multiaddr]).unwrap(); // Dispatch storage request. assert_ok!(FileSystem::issue_storage_request( @@ -280,7 +302,7 @@ fn bsp_volunteer_success() { location.clone(), fingerprint, 4, - BoundedVec::try_from(vec![1]).unwrap(), + multiaddresses.clone(), )); // Dispatch BSP volunteer. @@ -288,7 +310,7 @@ fn bsp_volunteer_success() { bsp.clone(), location.clone(), fingerprint, - BoundedVec::try_from(vec![2]).unwrap() + multiaddresses.clone() )); // Assert that the correct event was deposited @@ -297,7 +319,7 @@ fn bsp_volunteer_success() { who: 2, location, fingerprint, - bsp_multiaddress: BoundedVec::try_from(vec![2]).unwrap(), + multiaddresses, } .into(), ); diff --git a/pallets/file-system/src/types.rs b/pallets/file-system/src/types.rs index 753842de7..b33876735 100644 --- a/pallets/file-system/src/types.rs +++ b/pallets/file-system/src/types.rs @@ -5,19 +5,18 @@ use scale_info::TypeInfo; use crate::Config; -/// Metadata for a storage request. -/// -/// This is used to track the status of a storage request +/// Ephemeral metadata of a storage request. #[derive(Encode, Decode, MaxEncodedLen, TypeInfo, Debug, PartialEq, Eq, Clone)] #[scale_info(skip_type_params(T))] pub struct StorageRequestMetadata { + // TODO: Add MSP /// Block number at which the storage request was made. /// /// Used primarily for tracking the age of the request which is useful for /// cleaning up old requests. pub requested_at: BlockNumberFor, - /// AccountId of the user who requested the storage. - pub requested_by: T::AccountId, + /// AccountId of the user who owns the data being stored. + pub owner: T::AccountId, /// Identifier of the data being stored. pub fingerprint: Fingerprint, /// Size of the data being stored. @@ -28,14 +27,34 @@ pub struct StorageRequestMetadata { /// Multiaddress of the user who requested the storage. /// /// SPs will expect a connection request to be initiated by the user with this multiaddress. - pub user_multiaddr: MultiAddress, - /// List of BSPs that have volunteered to store the data. - pub bsps_volunteered: BoundedVec, MaxBspsPerStorageRequest>, - /// List of BSPs that have proven they are storing the data. + pub user_multiaddresses: BoundedVec, MaxMultiAddresses>, + /// List of storage providers that can serve the data that is requested to be stored. + /// + /// This is useful when a BSP stops serving data and automatically creates a new storage request with no user multiaddresses, since + /// SPs can prove and serve the data to be replicated to other BSPs without the user having this stored on their local machine. + pub data_server_sps: BoundedVec, MaxBspsPerStorageRequest>, // TODO: Change the Maximum data servers to be the maximum SPs allowed + /// Number of BSPs requested to store the data. + /// /// /// The storage request will be dropped/complete once all the minimum required BSPs have /// submitted a proof of storage after volunteering to store the data. - pub bsps_confirmed: BoundedVec, MaxBspsPerStorageRequest>, + pub bsps_required: T::StorageRequestBspsRequiredType, + /// Number of BSPs that have successfully volunteered AND confirmed that they are storing the data. + /// + /// This starts at 0 and increases up to `bsps_required`. Once this reaches `bsps_required`, the + /// storage request is considered complete and will be deleted.. + pub bsps_confirmed: T::StorageRequestBspsRequiredType, +} + +/// Ephemeral BSP storage request tracking metadata. +#[derive(Encode, Decode, MaxEncodedLen, TypeInfo, Debug, PartialEq, Eq, Clone)] +#[scale_info(skip_type_params(T))] +pub struct StorageRequestBspsMetadata { + /// Confirmed that the data is being stored. + /// + /// This is normally when the BSP submits a proof of storage to the `pallet-proofs-dealer-trie`. + pub confirmed: bool, + pub _phantom: core::marker::PhantomData, } /// Alias for the `AccountId` type used in the FileSystem pallet. @@ -47,9 +66,6 @@ pub type MaxBspsPerStorageRequest = ::MaxBspsPerStorageRe /// Alias for the `MaxFilePathSize` type used in the FileSystem pallet. pub type MaxFilePathSize = ::MaxFilePathSize; -/// Alias for the `MaxMultiAddressSize` type used in the FileSystem pallet. -pub type MaxMultiAddressSize = ::MaxMultiAddressSize; - /// Alias for the `Fingerprint` type used in the FileSystem pallet. pub type Fingerprint = ::Fingerprint; @@ -59,5 +75,14 @@ pub type StorageUnit = ::StorageUnit; /// Byte array representing the file path. pub type FileLocation = BoundedVec>; +/// Alias for the `MaxMultiAddressSize` type used in the FileSystem pallet. +pub type MaxMultiAddressSize = ::MaxMultiAddressSize; + /// Byte array representing the libp2p multiaddress. pub type MultiAddress = BoundedVec>; + +/// Alias for the `MaxMultiAddresses` type used in the FileSystem pallet. +pub type MaxMultiAddresses = ::MaxMultiAddresses; + +/// Alias for a bounded vector of [`MultiAddress`]. +pub type MultiAddresses = BoundedVec, MaxMultiAddresses>; diff --git a/pallets/file-system/src/utils.rs b/pallets/file-system/src/utils.rs index 125d1845d..2614caa25 100644 --- a/pallets/file-system/src/utils.rs +++ b/pallets/file-system/src/utils.rs @@ -1,14 +1,21 @@ use core::cmp::max; -use frame_support::{ensure, pallet_prelude::DispatchResult, sp_runtime::BoundedVec, traits::Get}; +use frame_support::{ensure, pallet_prelude::DispatchResult, traits::Get}; use frame_system::pallet_prelude::BlockNumberFor; -use sp_runtime::traits::CheckedAdd; +use sp_runtime::Saturating; +use sp_runtime::{ + traits::{CheckedAdd, Zero}, + BoundedVec, +}; use crate::{ pallet, - types::{FileLocation, Fingerprint, MultiAddress, StorageRequestMetadata, StorageUnit}, - Error, NextAvailableExpirationInsertionBlock, Pallet, StorageRequestExpirations, - StorageRequests, + types::{ + FileLocation, Fingerprint, MaxBspsPerStorageRequest, MultiAddresses, StorageProviderId, + StorageRequestBspsMetadata, StorageRequestMetadata, StorageUnit, + }, + Error, NextAvailableExpirationInsertionBlock, Pallet, StorageRequestBsps, + StorageRequestExpirations, StorageRequests, }; macro_rules! expect_or_err { @@ -32,24 +39,43 @@ impl Pallet where T: pallet::Config, { + /// Request storage for a file. + /// + /// In the event that a storage request is created without any user multiaddresses (checkout `do_bsp_stop_storing`), + /// it is expected that storage providers that do have this file in storage already, will be able to send a + /// transaction to the chain to add themselves as a data server for the storage request. pub(crate) fn do_request_storage( - requested_by: T::AccountId, + owner: T::AccountId, location: FileLocation, fingerprint: Fingerprint, size: StorageUnit, - user_multiaddr: MultiAddress, + bsps_required: Option, + user_multiaddresses: Option>, + data_server_sps: BoundedVec, MaxBspsPerStorageRequest>, ) -> DispatchResult { // TODO: Check user funds and lock them for the storage request. // TODO: Check storage capacity of chosen MSP (when we support MSPs) // TODO: Return error if the file is already stored and overwrite is false. + + let bsps_required = bsps_required.unwrap_or(T::DefaultBspsRequired::get()); + + if bsps_required.is_zero() { + return Err(Error::::BspsRequiredCannotBeZero)?; + } + + if bsps_required > MaxBspsPerStorageRequest::::get().into() { + return Err(Error::::BspsRequiredExceedsMax)?; + } + let file_metadata = StorageRequestMetadata:: { requested_at: >::block_number(), - requested_by, + owner, fingerprint, size, - user_multiaddr, - bsps_volunteered: BoundedVec::default(), - bsps_confirmed: BoundedVec::default(), + user_multiaddresses: user_multiaddresses.unwrap_or_default(), + data_server_sps, + bsps_required, + bsps_confirmed: T::StorageRequestBspsRequiredType::zero(), }; // TODO: if we add the overwrite flag, this would only fail if the overwrite flag is false. @@ -65,15 +91,15 @@ where let mut block_to_insert_expiration = Self::next_expiration_insertion_block_number(); // Get current storage request expirations vec. - let curr_storage_request_expirations = + let storage_request_expirations = >::get(block_to_insert_expiration); // Check size of storage request expirations vec. - if curr_storage_request_expirations.len() >= T::MaxExpiredStorageRequests::get() as usize { + if storage_request_expirations.len() >= T::MaxExpiredStorageRequests::get() as usize { block_to_insert_expiration = match block_to_insert_expiration.checked_add(&1u8.into()) { Some(block) => block, None => { - return Err(Error::::StorageRequestExpirationBlockOverflow.into()); + return Err(Error::::MaxBlockNumberReached.into()); } }; @@ -99,31 +125,34 @@ where // TODO: Check that sender is a registered storage provider // Check that the storage request exists. - ensure!( - >::contains_key(&location), - Error::::StorageRequestNotRegistered - ); - - // Get storage request metadata. - let mut file_metadata = expect_or_err!( + let file_metadata = expect_or_err!( >::get(&location), "Storage request should exist", - Error::::StorageRequestNotRegistered + Error::::StorageRequestNotFound + ); + + // Check that the storage request did not reach the required bsps. + ensure!( + file_metadata.bsps_confirmed < file_metadata.bsps_required, + Error::::StorageRequestBspsRequiredFullfilled ); - // Check if the BSP is not already registered for this storage request. + // Check if the BSP is already volunteered for this storage request. ensure!( - !file_metadata.bsps_volunteered.contains(&who), - Error::::BspAlreadyConfirmed + !>::contains_key(&location, &who), + Error::::BspAlreadyVolunteered ); // TODO: Check that the BSP XOR is lower than the threshold // Add BSP to storage request metadata. - expect_or_err!( - file_metadata.bsps_volunteered.try_push(who.clone()).ok(), - "BSP volunteer failed", - Error::::BspVolunteerFailed + >::insert( + &location, + &who, + StorageRequestBspsMetadata:: { + confirmed: false, + _phantom: Default::default(), + }, ); >::set(&location, Some(file_metadata.clone())); @@ -139,19 +168,19 @@ where // Check that the storage request exists. ensure!( >::contains_key(&location), - Error::::StorageRequestNotRegistered + Error::::StorageRequestNotFound ); // Get storage request metadata. let file_metadata = expect_or_err!( >::get(&location), "Storage request should exist", - Error::::StorageRequestNotRegistered + Error::::StorageRequestNotFound ); // Check that the sender is the same as the one who requested the storage. ensure!( - file_metadata.requested_by == who, + file_metadata.owner == who, Error::::StorageRequestNotAuthorized ); @@ -163,6 +192,64 @@ where Ok(()) } + /// BSP stops storing a file. + /// + /// `can_serve` is a flag that indicates if the BSP can serve the file. This is useful for + /// the case where the BSP lost the file somehow and cannot send it to other BSPs. When it is `true`, + /// the multiaddresses of the BSP are fetched from the `pallet-storage-providers` and added to the storage request. + /// When it is `false`, the storage request will be created without any multiaddresses and `do_request_storage` + /// will handle triggering the appropriate event and pending storage request. + pub(crate) fn do_bsp_stop_storing( + who: StorageProviderId, + location: FileLocation, + owner: T::AccountId, + fingerprint: Fingerprint, + size: StorageUnit, + can_serve: bool, + ) -> DispatchResult { + // Check that the storage request exists. + ensure!( + >::contains_key(&location), + Error::::StorageRequestNotFound + ); + + // If the storage request exists, then we should reduce the number of bsps confirmed and + match >::get(&location) { + Some(mut metadata) => { + // Remove BSP from storage request and challenge if has confirmed having stored this file. + if let Some(bsp) = >::get(&location, &who) { + // TODO: challenge the BSP to force update its storage + + if bsp.confirmed { + metadata.bsps_confirmed = + metadata.bsps_confirmed.saturating_sub(1u32.into()); + + >::set(&location, Some(metadata)); + } + + >::remove(&location, &who); + } + } + None => { + Self::do_request_storage( + owner, + location.clone(), + fingerprint, + size, + Some(1u32.into()), + None, + if can_serve { + BoundedVec::try_from(vec![who]).unwrap() + } else { + BoundedVec::default() + }, + )?; + } + }; + + Ok(()) + } + /// Get the block number at which the storage request will expire. /// /// This will also update the [`CurrentExpirationBlock`] if the current expiration block pointer is lower then the [`crate::Config::StorageRequestTtl`]. diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index d7f3a58c0..447150ae3 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -493,11 +493,14 @@ impl pallet_file_system::Config for Runtime { type RuntimeEvent = RuntimeEvent; type Fingerprint = Hash; type StorageUnit = u128; - type MaxBspsPerStorageRequest = ConstU32<5u32>; + type StorageRequestBspsRequiredType = u32; + type DefaultBspsRequired = ConstU32<1>; + type MaxBspsPerStorageRequest = ConstU32<5>; type MaxFilePathSize = ConstU32<512u32>; - type MaxMultiAddressSize = ConstU32<512u32>; - type StorageRequestTtl = ConstU32<40u32>; - type MaxExpiredStorageRequests = ConstU32<100u32>; + type MaxMultiAddresses = ConstU32<10>; + type MaxMultiAddressSize = ConstU32<512>; + type StorageRequestTtl = ConstU32<40>; + type MaxExpiredStorageRequests = ConstU32<100>; } // Create the runtime by composing the FRAME pallets that were previously configured. From 354d8e77a2d3a01ca88062ccca554b4ad4745ea2 Mon Sep 17 00:00:00 2001 From: Michael Assaf Date: Wed, 20 Mar 2024 10:16:28 -0400 Subject: [PATCH 02/18] fix benchmarking, add todo challenge bsp delete storage --- pallets/file-system/src/benchmarking.rs | 6 +++--- pallets/file-system/src/utils.rs | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pallets/file-system/src/benchmarking.rs b/pallets/file-system/src/benchmarking.rs index 1c3747782..dfe2e6773 100644 --- a/pallets/file-system/src/benchmarking.rs +++ b/pallets/file-system/src/benchmarking.rs @@ -7,7 +7,7 @@ use crate::Pallet as FileSystem; use frame_benchmarking::{benchmarks, impl_benchmark_test_suite, whitelisted_caller}; use frame_system::RawOrigin; -use crate::types::{FileLocation, Fingerprint, MultiAddress, StorageUnit}; +use crate::types::{FileLocation, Fingerprint, MultiAddresses, StorageUnit}; benchmarks! { issue_storage_request { @@ -16,8 +16,8 @@ benchmarks! { let location: FileLocation = Default::default(); let fingerprint: Fingerprint = Default::default(); let size: StorageUnit = Default::default(); - let user_multiaddr: MultiAddress = Default::default(); - }: _(RawOrigin::Signed(caller), location.clone(), fingerprint, size, user_multiaddr) + let multiaddresses: MultiAddresses = Default::default(); + }: _(RawOrigin::Signed(caller), location.clone(), fingerprint, size, multiaddresses) verify { assert!(FileSystem::::storage_requests(location).is_some()); } diff --git a/pallets/file-system/src/utils.rs b/pallets/file-system/src/utils.rs index 2614caa25..f431563dd 100644 --- a/pallets/file-system/src/utils.rs +++ b/pallets/file-system/src/utils.rs @@ -247,6 +247,8 @@ where } }; + // TODO: Challenge the BSP to force update its root. + Ok(()) } From 67f7bb3e89a986732ae017913af224d7e1d0e73a Mon Sep 17 00:00:00 2001 From: Michael Assaf Date: Wed, 20 Mar 2024 11:07:35 -0400 Subject: [PATCH 03/18] Fix weights for mock testing --- pallets/file-system/src/lib.rs | 16 +++++++++++++++- pallets/file-system/src/mock.rs | 4 ++-- pallets/file-system/src/tests.rs | 17 +---------------- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/pallets/file-system/src/lib.rs b/pallets/file-system/src/lib.rs index 4206f1476..61a14f6ae 100644 --- a/pallets/file-system/src/lib.rs +++ b/pallets/file-system/src/lib.rs @@ -387,11 +387,25 @@ pub mod pallet { // Total weight used to avoid exceeding the remaining weight. // We count one write for the `NextBlockToCleanup` storage item updated at the end. - let mut total_used_weight = db_weight.writes(1); + let mut total_used_weight = Weight::zero(); let required_weight_for_iteration = db_weight.reads_writes(1, T::MaxExpiredStorageRequests::get().into()); + // Check if we can process a single iteration and update the `NextStartingBlockToCleanUp` storage item + if !remaining_weight.all_gte( + match total_used_weight.checked_add(&required_weight_for_iteration) { + Some(weight) => { + // Update `total_used_weight` to take into account the final write operation + total_used_weight = db_weight.writes(1); + weight + } + None => return Weight::zero(), + }, + ) { + return Weight::zero(); + } + // Iterate over blocks from the start block to the current block, // cleaning up storage requests until the remaining weight is insufficient while block_to_clean <= current_block diff --git a/pallets/file-system/src/mock.rs b/pallets/file-system/src/mock.rs index 4c5cd8e06..4eb280375 100644 --- a/pallets/file-system/src/mock.rs +++ b/pallets/file-system/src/mock.rs @@ -1,7 +1,7 @@ use frame_support::{ derive_impl, parameter_types, traits::{Everything, Hooks}, - weights::Weight, + weights::{constants::RocksDbWeight, Weight}, }; use frame_system as system; use sp_core::{ConstU32, H256}; @@ -50,7 +50,7 @@ impl system::Config for Test { type BaseCallFilter = Everything; type BlockWeights = (); type BlockLength = (); - type DbWeight = (); + type DbWeight = RocksDbWeight; type RuntimeOrigin = RuntimeOrigin; type RuntimeCall = RuntimeCall; type Nonce = u64; diff --git a/pallets/file-system/src/tests.rs b/pallets/file-system/src/tests.rs index de81c6771..5670a8dc9 100644 --- a/pallets/file-system/src/tests.rs +++ b/pallets/file-system/src/tests.rs @@ -163,7 +163,7 @@ fn request_storage_clear_old_expirations_success() { let multiaddresses: BoundedVec, ::MaxMultiAddresses> = BoundedVec::try_from(vec![multiaddr]).unwrap(); - let mut expected_expiration_block_number: BlockNumber = + let expected_expiration_block_number: BlockNumber = FileSystem::next_expiration_insertion_block_number().into(); // Append storage request expiration to the list at `StorageRequestTtl` @@ -184,21 +184,6 @@ fn request_storage_clear_old_expirations_success() { multiaddresses, )); - // Assert that the storage request expirations storage is at max capacity - assert_eq!( - FileSystem::storage_request_expirations(expected_expiration_block_number).len(), - max_storage_request_expiry as usize - ); - - expected_expiration_block_number = - FileSystem::next_expiration_insertion_block_number().into(); - - // Assert that the `CurrentExpirationBlock` storage is incremented by 1 - assert_eq!( - FileSystem::next_available_expiration_insertion_block(), - expected_expiration_block_number - ); - System::set_block_number(expected_expiration_block_number); // Assert that the `NextExpirationInsertionBlockNumber` storage is set to 0 initially From 0e447058254e1220fa3c48e0eb4b79c5518b3fd2 Mon Sep 17 00:00:00 2001 From: Michael Assaf Date: Wed, 20 Mar 2024 11:29:12 -0400 Subject: [PATCH 04/18] add bsp_stop_storing test --- pallets/file-system/src/lib.rs | 9 +++- pallets/file-system/src/tests.rs | 81 +++++++++++++++++++++++++++++++- 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/pallets/file-system/src/lib.rs b/pallets/file-system/src/lib.rs index 61a14f6ae..376fc7c5c 100644 --- a/pallets/file-system/src/lib.rs +++ b/pallets/file-system/src/lib.rs @@ -212,6 +212,7 @@ pub mod pallet { /// Notifies that a BSP has stopped storing a file. BspStoppedStoring { bsp: T::AccountId, + owner: T::AccountId, location: FileLocation, }, } @@ -359,14 +360,18 @@ pub mod pallet { Self::do_bsp_stop_storing( who.clone(), location.clone(), - owner, + owner.clone(), fingerprint, size, can_serve, )?; // Emit event. - Self::deposit_event(Event::BspStoppedStoring { bsp: who, location }); + Self::deposit_event(Event::BspStoppedStoring { + bsp: who, + owner, + location, + }); Ok(()) } diff --git a/pallets/file-system/src/tests.rs b/pallets/file-system/src/tests.rs index 5670a8dc9..04ebe0ddf 100644 --- a/pallets/file-system/src/tests.rs +++ b/pallets/file-system/src/tests.rs @@ -1,6 +1,6 @@ use crate::{ mock::*, - types::{FileLocation, MultiAddress}, + types::{FileLocation, MultiAddress, StorageRequestBspsMetadata}, Config, Event, StorageRequestExpirations, }; use frame_support::{assert_ok, traits::Hooks, weights::Weight}; @@ -298,6 +298,16 @@ fn bsp_volunteer_success() { multiaddresses.clone() )); + // Assert that the RequestStorageBsps has the correct value + assert_eq!( + FileSystem::storage_request_bsps(location.clone(), 2) + .expect("BSP should exist in storage"), + StorageRequestBspsMetadata:: { + confirmed: false, + _phantom: Default::default() + } + ); + // Assert that the correct event was deposited System::assert_last_event( Event::AcceptedBspVolunteer { @@ -310,3 +320,72 @@ fn bsp_volunteer_success() { ); }); } + +#[test] +fn bsp_stop_storing_success() { + new_test_ext().execute_with(|| { + // Go past genesis block so events get deposited + System::set_block_number(1); + + let owner = 1; + let user = RuntimeOrigin::signed(owner); + let bsp = RuntimeOrigin::signed(2); + let location = FileLocation::::try_from(b"test".to_vec()).unwrap(); + let size = 4; + let file_content = b"test".to_vec(); + let fingerprint = BlakeTwo256::hash(&file_content); + let multiaddr = BoundedVec::try_from(vec![1]).unwrap(); + let multiaddresses: BoundedVec, ::MaxMultiAddresses> = + BoundedVec::try_from(vec![multiaddr]).unwrap(); + + // Dispatch storage request. + assert_ok!(FileSystem::issue_storage_request( + user.clone(), + location.clone(), + fingerprint, + size, + multiaddresses.clone(), + )); + + // Dispatch BSP volunteer. + assert_ok!(FileSystem::bsp_volunteer( + bsp.clone(), + location.clone(), + fingerprint, + multiaddresses.clone() + )); + + // Assert that the RequestStorageBsps has the correct value + assert_eq!( + FileSystem::storage_request_bsps(location.clone(), 2) + .expect("BSP should exist in storage"), + StorageRequestBspsMetadata:: { + confirmed: false, + _phantom: Default::default() + } + ); + + // Dispatch BSP stop storing. + assert_ok!(FileSystem::bsp_stop_storing( + bsp.clone(), + location.clone(), + owner, + fingerprint, + size, + false + )); + + // Assert that the RequestStorageBsps has the correct value + assert!(FileSystem::storage_request_bsps(location.clone(), 2).is_none()); + + // Assert that the correct event was deposited + System::assert_last_event( + Event::BspStoppedStoring { + bsp: 2, + owner, + location, + } + .into(), + ); + }); +} From d9699e93e4ecfe179eb7340eebf560a748afc9ac Mon Sep 17 00:00:00 2001 From: Michael Assaf Date: Wed, 20 Mar 2024 11:37:35 -0400 Subject: [PATCH 05/18] assert storage requests unit tests --- pallets/file-system/src/tests.rs | 40 +++++++++++++++++++++++++++++--- pallets/file-system/src/types.rs | 3 +++ pallets/file-system/src/utils.rs | 3 ++- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/pallets/file-system/src/tests.rs b/pallets/file-system/src/tests.rs index 04ebe0ddf..ee178e5f6 100644 --- a/pallets/file-system/src/tests.rs +++ b/pallets/file-system/src/tests.rs @@ -1,6 +1,9 @@ use crate::{ mock::*, - types::{FileLocation, MultiAddress, StorageRequestBspsMetadata}, + types::{ + DefaultBspsRequired, FileLocation, MultiAddress, StorageRequestBspsMetadata, + StorageRequestMetadata, + }, Config, Event, StorageRequestExpirations, }; use frame_support::{assert_ok, traits::Hooks, weights::Weight}; @@ -15,8 +18,10 @@ fn request_storage_success() { // Go past genesis block so events get deposited System::set_block_number(1); - let user = RuntimeOrigin::signed(1); + let owner = 1; + let user = RuntimeOrigin::signed(owner); let location = FileLocation::::try_from(b"test".to_vec()).unwrap(); + let size = 4; let file_content = b"test".to_vec(); let fingerprint = BlakeTwo256::hash(&file_content); let multiaddr = BoundedVec::try_from(vec![1]).unwrap(); @@ -28,10 +33,25 @@ fn request_storage_success() { user.clone(), location.clone(), fingerprint, - 4, + size, multiaddresses.clone(), )); + // Assert that the storage was updated + assert_eq!( + FileSystem::storage_requests(location.clone()), + Some(StorageRequestMetadata { + requested_at: 1, + owner, + fingerprint, + size, + user_multiaddresses: multiaddresses.clone(), + data_server_sps: BoundedVec::default(), + bsps_required: DefaultBspsRequired::::get(), + bsps_confirmed: 0, + }) + ); + // Assert that the correct event was deposited System::assert_last_event( Event::NewStorageRequest { @@ -378,6 +398,20 @@ fn bsp_stop_storing_success() { // Assert that the RequestStorageBsps has the correct value assert!(FileSystem::storage_request_bsps(location.clone(), 2).is_none()); + // Assert that the storage was updated + assert_eq!( + FileSystem::storage_requests(location.clone()), + Some(StorageRequestMetadata { + requested_at: 1, + owner, + fingerprint, + size, + user_multiaddresses: multiaddresses.clone(), + data_server_sps: BoundedVec::default(), + bsps_required: DefaultBspsRequired::::get(), + bsps_confirmed: 0, + }) + ); // Assert that the correct event was deposited System::assert_last_event( Event::BspStoppedStoring { diff --git a/pallets/file-system/src/types.rs b/pallets/file-system/src/types.rs index b33876735..65f4be071 100644 --- a/pallets/file-system/src/types.rs +++ b/pallets/file-system/src/types.rs @@ -72,6 +72,9 @@ pub type Fingerprint = ::Fingerprint; /// Alias for the `StorageCount` type used in the FileSystem pallet. pub type StorageUnit = ::StorageUnit; +/// Alias for the `DefaultBspsRequired` type used in the FileSystem pallet. +pub type DefaultBspsRequired = ::DefaultBspsRequired; + /// Byte array representing the file path. pub type FileLocation = BoundedVec>; diff --git a/pallets/file-system/src/utils.rs b/pallets/file-system/src/utils.rs index f431563dd..2df93c3b4 100644 --- a/pallets/file-system/src/utils.rs +++ b/pallets/file-system/src/utils.rs @@ -8,6 +8,7 @@ use sp_runtime::{ BoundedVec, }; +use crate::types::DefaultBspsRequired; use crate::{ pallet, types::{ @@ -57,7 +58,7 @@ where // TODO: Check storage capacity of chosen MSP (when we support MSPs) // TODO: Return error if the file is already stored and overwrite is false. - let bsps_required = bsps_required.unwrap_or(T::DefaultBspsRequired::get()); + let bsps_required = bsps_required.unwrap_or(DefaultBspsRequired::::get()); if bsps_required.is_zero() { return Err(Error::::BspsRequiredCannotBeZero)?; From cf3dfaa69360461163639ebb94b0de83fd2cb6f7 Mon Sep 17 00:00:00 2001 From: Michael Assaf Date: Wed, 20 Mar 2024 11:54:47 -0400 Subject: [PATCH 06/18] add vec import --- pallets/file-system/src/utils.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/pallets/file-system/src/utils.rs b/pallets/file-system/src/utils.rs index 2df93c3b4..65583f1dd 100644 --- a/pallets/file-system/src/utils.rs +++ b/pallets/file-system/src/utils.rs @@ -7,6 +7,7 @@ use sp_runtime::{ traits::{CheckedAdd, Zero}, BoundedVec, }; +use sp_std::vec; use crate::types::DefaultBspsRequired; use crate::{ From aa9f173bcc827ce62dbc83925308be9d7af0c539 Mon Sep 17 00:00:00 2001 From: Michael Assaf Date: Wed, 20 Mar 2024 12:57:23 -0400 Subject: [PATCH 07/18] add storage providers and proofs dealer to runtime and file system mock runtime, call do_challenge for bsp stop storing --- Cargo.lock | 5 +++ Cargo.toml | 17 +++++++++- pallets/file-system/Cargo.toml | 6 ++++ pallets/file-system/src/lib.rs | 6 +++- pallets/file-system/src/mock.rs | 57 ++++++++++++++++++++++++++++++-- pallets/file-system/src/tests.rs | 4 +++ pallets/file-system/src/types.rs | 3 ++ pallets/file-system/src/utils.rs | 10 ++++-- pallets/proofs-dealer/src/lib.rs | 6 ++-- pallets/providers/Cargo.toml | 38 +++++++++++---------- pallets/providers/src/lib.rs | 11 ++++++ runtime/Cargo.toml | 33 ++++-------------- runtime/src/lib.rs | 38 +++++++++++++++++++-- 13 files changed, 177 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d7ad54da9..819dcad94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6419,6 +6419,9 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", + "pallet-balances", + "pallet-proofs-dealer", + "pallet-storage-providers", "parity-scale-codec", "scale-info", "serde", @@ -6950,6 +6953,7 @@ dependencies = [ "frame-support", "frame-system", "pallet-balances", + "pallet-proofs-dealer", "parity-scale-codec", "scale-info", "serde", @@ -12441,6 +12445,7 @@ dependencies = [ "pallet-message-queue", "pallet-proofs-dealer", "pallet-session", + "pallet-storage-providers", "pallet-sudo", "pallet-timestamp", "pallet-transaction-payment", diff --git a/Cargo.toml b/Cargo.toml index 4d1df6e85..34ccc6724 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,9 +9,24 @@ homepage = "https://moonsonglabs.com/" panic = "unwind" [workspace] -members = ["runtime", "pallets/*", "node", "bsp", "msp", "storage-kit", "support/*"] +members = [ + "runtime", + "pallets/*", + "node", + "bsp", + "msp", + "storage-kit", + "support/*", +] resolver = "2" +[workspace.dependencies] +storage-hub-traits = { path = "support/traits", default-features = false } + +pallet-storage-providers = { path = "pallets/providers", default-features = false } +pallet-file-system = { path = "pallets/file-system", default-features = false } +pallet-proofs-dealer = { path = "pallets/proofs-dealer", default-features = false } + [workspace.lints.rust] suspicious_double_ref_op = { level = "allow", priority = 2 } diff --git a/pallets/file-system/Cargo.toml b/pallets/file-system/Cargo.toml index d286f1e31..5f380a53b 100644 --- a/pallets/file-system/Cargo.toml +++ b/pallets/file-system/Cargo.toml @@ -30,6 +30,10 @@ frame-system = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = " sp-runtime = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false } sp-std = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false } +# Local +pallet-storage-providers = { workspace = true } +pallet-proofs-dealer = { workspace = true } + [dev-dependencies] serde = { version = "1.0.193" } @@ -37,6 +41,8 @@ serde = { version = "1.0.193" } sp-core = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false } sp-io = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false } +pallet-balances = { version = "4.0.0-dev", git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0" } + [features] default = ["std"] runtime-benchmarks = [ diff --git a/pallets/file-system/src/lib.rs b/pallets/file-system/src/lib.rs index 376fc7c5c..407c13abd 100644 --- a/pallets/file-system/src/lib.rs +++ b/pallets/file-system/src/lib.rs @@ -55,7 +55,7 @@ pub mod pallet { use sp_runtime::BoundedVec; #[pallet::config] - pub trait Config: frame_system::Config { + pub trait Config: frame_system::Config + pallet_proofs_dealer::Config { /// Because this pallet emits events, it depends on the runtime's definition of an event. type RuntimeEvent: From> + IsType<::RuntimeEvent>; @@ -212,6 +212,7 @@ pub mod pallet { /// Notifies that a BSP has stopped storing a file. BspStoppedStoring { bsp: T::AccountId, + file_key: FileKey, owner: T::AccountId, location: FileLocation, }, @@ -348,6 +349,7 @@ pub mod pallet { #[pallet::weight(10_000 + T::DbWeight::get().reads_writes(1,1).ref_time())] pub fn bsp_stop_storing( origin: OriginFor, + file_key: FileKey, location: FileLocation, owner: T::AccountId, fingerprint: Fingerprint, @@ -359,6 +361,7 @@ pub mod pallet { // Perform validations and stop storing the file. Self::do_bsp_stop_storing( who.clone(), + file_key, location.clone(), owner.clone(), fingerprint, @@ -369,6 +372,7 @@ pub mod pallet { // Emit event. Self::deposit_event(Event::BspStoppedStoring { bsp: who, + file_key, owner, location, }); diff --git a/pallets/file-system/src/mock.rs b/pallets/file-system/src/mock.rs index 4eb280375..588926bb8 100644 --- a/pallets/file-system/src/mock.rs +++ b/pallets/file-system/src/mock.rs @@ -4,7 +4,7 @@ use frame_support::{ weights::{constants::RocksDbWeight, Weight}, }; use frame_system as system; -use sp_core::{ConstU32, H256}; +use sp_core::{ConstU128, ConstU32, H256}; use sp_runtime::{ traits::{BlakeTwo256, IdentityLookup}, BuildStorage, @@ -12,6 +12,7 @@ use sp_runtime::{ type Block = frame_system::mocking::MockBlock; pub(crate) type BlockNumber = u64; +type Balance = u128; /// Rolls to the desired block. Returns the number of blocks played. pub(crate) fn roll_to(n: BlockNumber) -> BlockNumber { @@ -36,7 +37,10 @@ frame_support::construct_runtime!( pub enum Test { System: frame_system::{Pallet, Call, Config, Storage, Event}, + Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, FileSystem: crate::{Pallet, Call, Storage, Event}, + Providers: pallet_storage_providers::{Pallet, Call, Storage, Event}, + ProofsDealer: pallet_proofs_dealer::{Pallet, Call, Storage, Event}, } ); @@ -63,7 +67,7 @@ impl system::Config for Test { type BlockHashCount = BlockHashCount; type Version = (); type PalletInfo = PalletInfo; - type AccountData = (); + type AccountData = pallet_balances::AccountData; type OnNewAccount = (); type OnKilledAccount = (); type SystemWeightInfo = (); @@ -72,6 +76,55 @@ impl system::Config for Test { type MaxConsumers = frame_support::traits::ConstU32<16>; } +impl pallet_balances::Config for Test { + type Balance = Balance; + type DustRemoval = (); + type RuntimeEvent = RuntimeEvent; + type ExistentialDeposit = ConstU128<1>; + type AccountStore = System; + type WeightInfo = (); + type MaxLocks = ConstU32<10>; + type MaxReserves = (); + type ReserveIdentifier = [u8; 8]; + type RuntimeHoldReason = (); + type RuntimeFreezeReason = (); + type FreezeIdentifier = (); + type MaxHolds = ConstU32<10>; + type MaxFreezes = ConstU32<10>; +} + +impl pallet_storage_providers::Config for Test { + type RuntimeEvent = RuntimeEvent; + type NativeBalance = Balances; + type StorageData = u32; + type SpCount = u32; + type HashId = H256; + type MerklePatriciaRoot = H256; + type ValuePropId = H256; + type MaxMultiAddressSize = ConstU32<100>; + type MaxMultiAddressAmount = ConstU32<5>; + type MaxProtocols = ConstU32<100>; + type MaxBsps = ConstU32<100>; + type MaxMsps = ConstU32<100>; + type MaxBuckets = ConstU32<10000>; + type SpMinDeposit = ConstU128<10>; + type SpMinCapacity = ConstU32<1>; + type DepositPerData = ConstU128<2>; +} + +impl pallet_proofs_dealer::Config for Test { + type RuntimeEvent = RuntimeEvent; + type ProvidersPallet = Providers; + type NativeBalance = Balances; + type MerkleHash = H256; + type TrieVerifier = Providers; + type MaxChallengesPerBlock = ConstU32<10>; + type MaxProvidersChallengedPerBlock = ConstU32<10>; + type ChallengeHistoryLength = ConstU32<10>; + type ChallengesQueueLength = ConstU32<10>; + type CheckpointChallengePeriod = ConstU32<10>; +} + impl crate::Config for Test { type RuntimeEvent = RuntimeEvent; type Fingerprint = H256; diff --git a/pallets/file-system/src/tests.rs b/pallets/file-system/src/tests.rs index ee178e5f6..9d6400521 100644 --- a/pallets/file-system/src/tests.rs +++ b/pallets/file-system/src/tests.rs @@ -7,6 +7,7 @@ use crate::{ Config, Event, StorageRequestExpirations, }; use frame_support::{assert_ok, traits::Hooks, weights::Weight}; +use sp_core::H256; use sp_runtime::{ traits::{BlakeTwo256, Get, Hash}, BoundedVec, @@ -350,6 +351,7 @@ fn bsp_stop_storing_success() { let owner = 1; let user = RuntimeOrigin::signed(owner); let bsp = RuntimeOrigin::signed(2); + let file_key = H256::from_slice(&[1; 32]); let location = FileLocation::::try_from(b"test".to_vec()).unwrap(); let size = 4; let file_content = b"test".to_vec(); @@ -388,6 +390,7 @@ fn bsp_stop_storing_success() { // Dispatch BSP stop storing. assert_ok!(FileSystem::bsp_stop_storing( bsp.clone(), + file_key, location.clone(), owner, fingerprint, @@ -416,6 +419,7 @@ fn bsp_stop_storing_success() { System::assert_last_event( Event::BspStoppedStoring { bsp: 2, + file_key, owner, location, } diff --git a/pallets/file-system/src/types.rs b/pallets/file-system/src/types.rs index 65f4be071..dd1466c19 100644 --- a/pallets/file-system/src/types.rs +++ b/pallets/file-system/src/types.rs @@ -89,3 +89,6 @@ pub type MaxMultiAddresses = ::MaxMultiAddresses; /// Alias for a bounded vector of [`MultiAddress`]. pub type MultiAddresses = BoundedVec, MaxMultiAddresses>; + +/// Alias for the `FileKey` type used in the proofs-dealer pallet. +pub type FileKey = ::MerkleHash; diff --git a/pallets/file-system/src/utils.rs b/pallets/file-system/src/utils.rs index 65583f1dd..b725c85c2 100644 --- a/pallets/file-system/src/utils.rs +++ b/pallets/file-system/src/utils.rs @@ -9,7 +9,7 @@ use sp_runtime::{ }; use sp_std::vec; -use crate::types::DefaultBspsRequired; +use crate::types::{DefaultBspsRequired, FileKey}; use crate::{ pallet, types::{ @@ -203,6 +203,7 @@ where /// will handle triggering the appropriate event and pending storage request. pub(crate) fn do_bsp_stop_storing( who: StorageProviderId, + file_key: FileKey, location: FileLocation, owner: T::AccountId, fingerprint: Fingerprint, @@ -215,6 +216,8 @@ where Error::::StorageRequestNotFound ); + // TODO: Check that the hash of all the metadata is equal to the `file_key` hash. + // If the storage request exists, then we should reduce the number of bsps confirmed and match >::get(&location) { Some(mut metadata) => { @@ -241,7 +244,7 @@ where Some(1u32.into()), None, if can_serve { - BoundedVec::try_from(vec![who]).unwrap() + BoundedVec::try_from(vec![who.clone()]).unwrap() } else { BoundedVec::default() }, @@ -249,7 +252,8 @@ where } }; - // TODO: Challenge the BSP to force update its root. + // Challenge BSP to force update its storage root to uninclude the file. + pallet_proofs_dealer::Pallet::::do_challenge(&who, &file_key)?; Ok(()) } diff --git a/pallets/proofs-dealer/src/lib.rs b/pallets/proofs-dealer/src/lib.rs index b1dcfe23b..34bc13f28 100644 --- a/pallets/proofs-dealer/src/lib.rs +++ b/pallets/proofs-dealer/src/lib.rs @@ -19,7 +19,7 @@ pub mod utils; use frame_support::{inherent::IsFatalError, pallet_prelude::*, sp_runtime::RuntimeString}; use scale_info::prelude::fmt::Debug; -use sp_trie::CompactProof; +pub use sp_trie::CompactProof; // TODO: Define this. const INHERENT_IDENTIFIER: InherentIdentifier = *b"todo____"; @@ -80,11 +80,11 @@ pub mod pallet { /// The maximum number of challenges that can be made in a single block. #[pallet::constant] - type MaxChallengesPerBlock: Get + FullCodec; + type MaxChallengesPerBlock: Get; /// The maximum number of Providers that can be challenged in block. #[pallet::constant] - type MaxProvidersChallengedPerBlock: Get + FullCodec; + type MaxProvidersChallengedPerBlock: Get; /// The number of blocks that challenges history is kept for. /// After this many blocks, challenges are removed from `Challenges` StorageMap. diff --git a/pallets/providers/Cargo.toml b/pallets/providers/Cargo.toml index 6735086e7..983208d22 100644 --- a/pallets/providers/Cargo.toml +++ b/pallets/providers/Cargo.toml @@ -16,14 +16,16 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0", features = [ - "derive", + "derive", ], default-features = false } scale-info = { version = "2.10.0", default-features = false, features = [ - "derive", + "derive", ] } # Local -storage-hub-traits = { path = "../../support/traits", version = "0.1.0", default-features = false} +storage-hub-traits = { workspace = true, default-features = false } + +pallet-proofs-dealer = { workspace = true } # Substrate frame-benchmarking = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false, optional = true } @@ -43,23 +45,23 @@ sp-runtime = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "po [features] default = ["std"] runtime-benchmarks = [ - "frame-benchmarking/runtime-benchmarks", - "frame-support/runtime-benchmarks", - "frame-system/runtime-benchmarks", - "sp-runtime/runtime-benchmarks", + "frame-benchmarking/runtime-benchmarks", + "frame-support/runtime-benchmarks", + "frame-system/runtime-benchmarks", + "sp-runtime/runtime-benchmarks", ] std = [ - "codec/std", - "frame-benchmarking/std", - "frame-support/std", - "frame-system/std", - "scale-info/std", - "sp-core/std", - "sp-io/std", - "sp-runtime/std", + "codec/std", + "frame-benchmarking/std", + "frame-support/std", + "frame-system/std", + "scale-info/std", + "sp-core/std", + "sp-io/std", + "sp-runtime/std", ] try-runtime = [ - "frame-support/try-runtime", - "frame-system/try-runtime", - "sp-runtime/try-runtime", + "frame-support/try-runtime", + "frame-system/try-runtime", + "sp-runtime/try-runtime", ] diff --git a/pallets/providers/src/lib.rs b/pallets/providers/src/lib.rs index ef24393fc..0d3b3f583 100644 --- a/pallets/providers/src/lib.rs +++ b/pallets/providers/src/lib.rs @@ -511,6 +511,17 @@ impl Pallet { } } +impl pallet_proofs_dealer::TrieVerifier for Pallet { + fn verify_proof( + _root: &[u8; 32], + _challenges: &[u8; 32], + _proof: &pallet_proofs_dealer::CompactProof, + ) -> bool { + // TODO: implement this + todo!() + } +} + // Trait definitions: use frame_support::pallet_prelude::DispatchResult; diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 4ebe4937d..e91a20293 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -29,8 +29,9 @@ scale-info = { version = "2.10.0", default-features = false, features = [ smallvec = "1.11.0" # Local -pallet-file-system = { path = "../pallets/file-system", default-features = false } -pallet-proofs-dealer = { path = "../pallets/proofs-dealer", default-features = false } +pallet-file-system = { workspace = true } +pallet-storage-providers = { workspace = true } +pallet-proofs-dealer = { workspace = true } # Substrate frame-benchmarking = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false, optional = true } @@ -114,6 +115,7 @@ std = [ "pallet-session/std", "pallet-proofs-dealer/std", "pallet-sudo/std", + "pallet-storage-providers/std", "pallet-timestamp/std", "pallet-transaction-payment-rpc-runtime-api/std", "pallet-transaction-payment/std", @@ -159,6 +161,7 @@ runtime-benchmarks = [ "pallet-file-system/runtime-benchmarks", "pallet-proofs-dealer/runtime-benchmarks", "pallet-sudo/runtime-benchmarks", + "pallet-storage-providers/runtime-benchmarks", "pallet-timestamp/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", "parachains-common/runtime-benchmarks", @@ -187,31 +190,7 @@ try-runtime = [ "pallet-file-system/try-runtime", "pallet-session/try-runtime", "pallet-sudo/try-runtime", - "pallet-timestamp/try-runtime", - "pallet-transaction-payment/try-runtime", - "pallet-xcm/try-runtime", - "parachain-info/try-runtime", - "polkadot-runtime-common/try-runtime", - "sp-runtime/try-runtime", - - "cumulus-pallet-aura-ext/try-runtime", - "cumulus-pallet-dmp-queue/try-runtime", - "cumulus-pallet-parachain-system/try-runtime", - "cumulus-pallet-xcm/try-runtime", - "cumulus-pallet-xcmp-queue/try-runtime", - "frame-executive/try-runtime", - "frame-support/try-runtime", - "frame-system/try-runtime", - "frame-try-runtime/try-runtime", - "pallet-aura/try-runtime", - "pallet-authorship/try-runtime", - "pallet-balances/try-runtime", - "pallet-collator-selection/try-runtime", - "pallet-message-queue/try-runtime", - "pallet-file-system/try-runtime", - "pallet-session/try-runtime", - "pallet-proofs-dealer/try-runtime", - "pallet-sudo/try-runtime", + "pallet-storage-providers/try-runtime", "pallet-timestamp/try-runtime", "pallet-transaction-payment/try-runtime", "pallet-xcm/try-runtime", diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 447150ae3..76f000841 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -32,7 +32,7 @@ use frame_support::{ dispatch::DispatchClass, genesis_builder_helper::{build_config, create_default_config}, parameter_types, - traits::{ConstBool, ConstU32, ConstU64, ConstU8, EitherOfDiverse, TransformOrigin}, + traits::{ConstBool, ConstU128, ConstU32, ConstU64, ConstU8, EitherOfDiverse, TransformOrigin}, weights::{ constants::WEIGHT_REF_TIME_PER_SECOND, ConstantMultiplier, Weight, WeightToFeeCoefficient, WeightToFeeCoefficients, WeightToFeePolynomial, @@ -488,6 +488,38 @@ impl pallet_collator_selection::Config for Runtime { type WeightInfo = (); } +impl pallet_storage_providers::Config for Runtime { + type RuntimeEvent = RuntimeEvent; + type NativeBalance = Balances; + type StorageData = u32; + type SpCount = u32; + type HashId = Hash; + type MerklePatriciaRoot = Hash; + type ValuePropId = Hash; + type MaxMultiAddressSize = ConstU32<100>; + type MaxMultiAddressAmount = ConstU32<5>; + type MaxProtocols = ConstU32<100>; + type MaxBsps = ConstU32<100>; + type MaxMsps = ConstU32<100>; + type MaxBuckets = ConstU32<10000>; + type SpMinDeposit = ConstU128<10>; + type SpMinCapacity = ConstU32<1>; + type DepositPerData = ConstU128<2>; +} + +impl pallet_proofs_dealer::Config for Runtime { + type RuntimeEvent = RuntimeEvent; + type ProvidersPallet = Providers; + type NativeBalance = Balances; + type MerkleHash = Hash; + type TrieVerifier = Providers; + type MaxChallengesPerBlock = ConstU32<10>; + type MaxProvidersChallengedPerBlock = ConstU32<10>; + type ChallengeHistoryLength = ConstU32<10>; + type ChallengesQueueLength = ConstU32<10>; + type CheckpointChallengePeriod = ConstU32<10>; +} + /// Configure the pallet template in pallets/template. impl pallet_file_system::Config for Runtime { type RuntimeEvent = RuntimeEvent; @@ -532,7 +564,9 @@ construct_runtime!( CumulusXcm: cumulus_pallet_xcm = 32, MessageQueue: pallet_message_queue = 33, - FileSystem: pallet_file_system = 40, + Providers: pallet_storage_providers = 40, + FileSystem: pallet_file_system = 41, + ProofsDealer: pallet_proofs_dealer = 42, } ); From 8f6ee3d8be013e59d67d8ee7f755a2c5b894e8a6 Mon Sep 17 00:00:00 2001 From: Michael Assaf Date: Wed, 20 Mar 2024 13:24:59 -0400 Subject: [PATCH 08/18] use default bounded vec --- pallets/providers/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pallets/providers/src/lib.rs b/pallets/providers/src/lib.rs index 0d3b3f583..2974ace8c 100644 --- a/pallets/providers/src/lib.rs +++ b/pallets/providers/src/lib.rs @@ -19,7 +19,6 @@ mod benchmarking; pub mod pallet { use super::types::*; use codec::{FullCodec, HasCompact}; - use frame_support::testing_prelude::bounded_vec; use frame_support::{ dispatch::DispatchResultWithPostInfo, pallet_prelude::*, @@ -312,7 +311,7 @@ pub mod pallet { let who = ensure_signed(origin)?; let msp_info = MainStorageProvider { - buckets: bounded_vec![], + buckets: BoundedVec::default(), total_data, data_used: StorageData::::default(), multiaddresses: multiaddress.clone(), From 45112906c5abbc979d96a0ca3d58a742cff19ac9 Mon Sep 17 00:00:00 2001 From: Michael Assaf Date: Wed, 20 Mar 2024 14:00:00 -0400 Subject: [PATCH 09/18] have proofs dealer implement trie verifier --- Cargo.lock | 1 - pallets/file-system/src/mock.rs | 2 +- pallets/proofs-dealer/src/lib.rs | 7 +++++++ pallets/providers/Cargo.toml | 2 -- pallets/providers/src/lib.rs | 11 ----------- runtime/src/lib.rs | 2 +- 6 files changed, 9 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 819dcad94..c1935d3c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6953,7 +6953,6 @@ dependencies = [ "frame-support", "frame-system", "pallet-balances", - "pallet-proofs-dealer", "parity-scale-codec", "scale-info", "serde", diff --git a/pallets/file-system/src/mock.rs b/pallets/file-system/src/mock.rs index 588926bb8..ad87eb3d6 100644 --- a/pallets/file-system/src/mock.rs +++ b/pallets/file-system/src/mock.rs @@ -117,7 +117,7 @@ impl pallet_proofs_dealer::Config for Test { type ProvidersPallet = Providers; type NativeBalance = Balances; type MerkleHash = H256; - type TrieVerifier = Providers; + type TrieVerifier = ProofsDealer; type MaxChallengesPerBlock = ConstU32<10>; type MaxProvidersChallengedPerBlock = ConstU32<10>; type ChallengeHistoryLength = ConstU32<10>; diff --git a/pallets/proofs-dealer/src/lib.rs b/pallets/proofs-dealer/src/lib.rs index 34bc13f28..9fe5ee82c 100644 --- a/pallets/proofs-dealer/src/lib.rs +++ b/pallets/proofs-dealer/src/lib.rs @@ -388,3 +388,10 @@ impl InherentError { pub trait TrieVerifier { fn verify_proof(root: &[u8; 32], challenges: &[u8; 32], proof: &CompactProof) -> bool; } + +impl TrieVerifier for Pallet { + fn verify_proof(_root: &[u8; 32], _challenges: &[u8; 32], _proof: &CompactProof) -> bool { + // TODO: implement this + todo!() + } +} diff --git a/pallets/providers/Cargo.toml b/pallets/providers/Cargo.toml index 983208d22..23ccdba2d 100644 --- a/pallets/providers/Cargo.toml +++ b/pallets/providers/Cargo.toml @@ -25,8 +25,6 @@ scale-info = { version = "2.10.0", default-features = false, features = [ # Local storage-hub-traits = { workspace = true, default-features = false } -pallet-proofs-dealer = { workspace = true } - # Substrate frame-benchmarking = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false, optional = true } frame-support = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false } diff --git a/pallets/providers/src/lib.rs b/pallets/providers/src/lib.rs index 2974ace8c..9f20a889d 100644 --- a/pallets/providers/src/lib.rs +++ b/pallets/providers/src/lib.rs @@ -510,17 +510,6 @@ impl Pallet { } } -impl pallet_proofs_dealer::TrieVerifier for Pallet { - fn verify_proof( - _root: &[u8; 32], - _challenges: &[u8; 32], - _proof: &pallet_proofs_dealer::CompactProof, - ) -> bool { - // TODO: implement this - todo!() - } -} - // Trait definitions: use frame_support::pallet_prelude::DispatchResult; diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 76f000841..a734dfd2c 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -512,7 +512,7 @@ impl pallet_proofs_dealer::Config for Runtime { type ProvidersPallet = Providers; type NativeBalance = Balances; type MerkleHash = Hash; - type TrieVerifier = Providers; + type TrieVerifier = ProofsDealer; type MaxChallengesPerBlock = ConstU32<10>; type MaxProvidersChallengedPerBlock = ConstU32<10>; type ChallengeHistoryLength = ConstU32<10>; From fca649017d7ba953893b98f3c084ea3884e4f8b8 Mon Sep 17 00:00:00 2001 From: Michael Assaf Date: Wed, 20 Mar 2024 14:44:10 -0400 Subject: [PATCH 10/18] remove unused import --- pallets/proofs-dealer/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/pallets/proofs-dealer/src/lib.rs b/pallets/proofs-dealer/src/lib.rs index 9fe5ee82c..9315d9d28 100644 --- a/pallets/proofs-dealer/src/lib.rs +++ b/pallets/proofs-dealer/src/lib.rs @@ -34,7 +34,6 @@ pub mod pallet { traits::fungible, }; use frame_system::pallet_prelude::*; - use scale_info::prelude::fmt::Debug; use sp_trie::CompactProof; use storage_hub_traits::ProvidersInterface; use types::ProviderFor; From 18d278839fbeecc540a68205e0e9c35174e77de3 Mon Sep 17 00:00:00 2001 From: Michael Assaf Date: Wed, 20 Mar 2024 15:03:19 -0400 Subject: [PATCH 11/18] add pallets to features --- pallets/file-system/Cargo.toml | 6 +++++ pallets/proofs-dealer/Cargo.toml | 39 ++++++++++++++++---------------- runtime/Cargo.toml | 1 + 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/pallets/file-system/Cargo.toml b/pallets/file-system/Cargo.toml index 5f380a53b..ce9f589f2 100644 --- a/pallets/file-system/Cargo.toml +++ b/pallets/file-system/Cargo.toml @@ -49,6 +49,8 @@ runtime-benchmarks = [ "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", + "pallet-storage-providers/runtime-benchmarks", + "pallet-proofs-dealer/runtime-benchmarks", "sp-runtime/runtime-benchmarks", ] std = [ @@ -56,6 +58,8 @@ std = [ "frame-benchmarking/std", "frame-support/std", "frame-system/std", + "pallet-storage-providers/std", + "pallet-proofs-dealer/std", "scale-info/std", "sp-core/std", "sp-io/std", @@ -64,5 +68,7 @@ std = [ try-runtime = [ "frame-support/try-runtime", "frame-system/try-runtime", + "pallet-storage-providers/try-runtime", + "pallet-proofs-dealer/try-runtime", "sp-runtime/try-runtime", ] diff --git a/pallets/proofs-dealer/Cargo.toml b/pallets/proofs-dealer/Cargo.toml index 2bef2497c..cbb3adca1 100644 --- a/pallets/proofs-dealer/Cargo.toml +++ b/pallets/proofs-dealer/Cargo.toml @@ -16,19 +16,20 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0", features = [ - "derive", + "derive", ], default-features = false } scale-info = { version = "2.10.0", default-features = false, features = [ - "derive", + "derive", ] } # Local -storage-hub-traits = { path = "../../support/traits", version = "0.1.0", default-features = false } +storage-hub-traits = { workspace = true } # Substrate frame-benchmarking = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false, optional = true } frame-support = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false } frame-system = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false } + sp-trie = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false } [dev-dependencies] @@ -42,24 +43,24 @@ sp-runtime = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "po [features] default = ["std"] runtime-benchmarks = [ - "frame-benchmarking/runtime-benchmarks", - "frame-support/runtime-benchmarks", - "frame-system/runtime-benchmarks", - "sp-runtime/runtime-benchmarks", + "frame-benchmarking/runtime-benchmarks", + "frame-support/runtime-benchmarks", + "frame-system/runtime-benchmarks", + "sp-runtime/runtime-benchmarks", ] std = [ - "codec/std", - "frame-benchmarking/std", - "frame-support/std", - "frame-system/std", - "scale-info/std", - "sp-core/std", - "sp-io/std", - "sp-runtime/std", - "sp-trie/std", + "codec/std", + "frame-benchmarking/std", + "frame-support/std", + "frame-system/std", + "scale-info/std", + "sp-core/std", + "sp-io/std", + "sp-runtime/std", + "sp-trie/std", ] try-runtime = [ - "frame-support/try-runtime", - "frame-system/try-runtime", - "sp-runtime/try-runtime", + "frame-support/try-runtime", + "frame-system/try-runtime", + "sp-runtime/try-runtime", ] diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index e91a20293..bdc9f4f29 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -188,6 +188,7 @@ try-runtime = [ "pallet-collator-selection/try-runtime", "pallet-message-queue/try-runtime", "pallet-file-system/try-runtime", + "pallet-proofs-dealer/try-runtime", "pallet-session/try-runtime", "pallet-sudo/try-runtime", "pallet-storage-providers/try-runtime", From a03efd26952e11cb2127f459984e6e0da4a353e9 Mon Sep 17 00:00:00 2001 From: Michael Assaf Date: Thu, 21 Mar 2024 09:27:45 -0400 Subject: [PATCH 12/18] amend review; use mock verifier, remove proofs dealer impl and extract it into its own struct, revert whitespace change, --- pallets/file-system/src/lib.rs | 2 +- pallets/file-system/src/mock.rs | 14 +++++++++++++- pallets/proofs-dealer/Cargo.toml | 1 - pallets/proofs-dealer/src/lib.rs | 7 ------- runtime/src/lib.rs | 14 +++++++++++++- 5 files changed, 27 insertions(+), 11 deletions(-) diff --git a/pallets/file-system/src/lib.rs b/pallets/file-system/src/lib.rs index 407c13abd..c45dd8bbf 100644 --- a/pallets/file-system/src/lib.rs +++ b/pallets/file-system/src/lib.rs @@ -84,7 +84,7 @@ pub mod pallet { + MaxEncodedLen + HasCompact; - /// Unit representing the size of a file. + /// Type representing the storage request bsps size type. type StorageRequestBspsRequiredType: Parameter + Member + MaybeSerializeDeserialize diff --git a/pallets/file-system/src/mock.rs b/pallets/file-system/src/mock.rs index ad87eb3d6..c564dac1b 100644 --- a/pallets/file-system/src/mock.rs +++ b/pallets/file-system/src/mock.rs @@ -4,6 +4,7 @@ use frame_support::{ weights::{constants::RocksDbWeight, Weight}, }; use frame_system as system; +use pallet_proofs_dealer::{CompactProof, TrieVerifier}; use sp_core::{ConstU128, ConstU32, H256}; use sp_runtime::{ traits::{BlakeTwo256, IdentityLookup}, @@ -117,7 +118,7 @@ impl pallet_proofs_dealer::Config for Test { type ProvidersPallet = Providers; type NativeBalance = Balances; type MerkleHash = H256; - type TrieVerifier = ProofsDealer; + type TrieVerifier = MockVerifier; type MaxChallengesPerBlock = ConstU32<10>; type MaxProvidersChallengedPerBlock = ConstU32<10>; type ChallengeHistoryLength = ConstU32<10>; @@ -125,6 +126,17 @@ impl pallet_proofs_dealer::Config for Test { type CheckpointChallengePeriod = ConstU32<10>; } +/// Structure to mock a verifier that returns `true` when `proof` is not empty +/// and `false` otherwise. +pub struct MockVerifier; + +/// Implement the `TrieVerifier` trait for the `MockVerifier` struct. +impl TrieVerifier for MockVerifier { + fn verify_proof(_root: &[u8; 32], _challenges: &[u8; 32], proof: &CompactProof) -> bool { + proof.encoded_nodes.len() > 0 + } +} + impl crate::Config for Test { type RuntimeEvent = RuntimeEvent; type Fingerprint = H256; diff --git a/pallets/proofs-dealer/Cargo.toml b/pallets/proofs-dealer/Cargo.toml index cbb3adca1..3025b6c84 100644 --- a/pallets/proofs-dealer/Cargo.toml +++ b/pallets/proofs-dealer/Cargo.toml @@ -29,7 +29,6 @@ storage-hub-traits = { workspace = true } frame-benchmarking = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false, optional = true } frame-support = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false } frame-system = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false } - sp-trie = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-v1.5.0", default-features = false } [dev-dependencies] diff --git a/pallets/proofs-dealer/src/lib.rs b/pallets/proofs-dealer/src/lib.rs index 9315d9d28..612917b23 100644 --- a/pallets/proofs-dealer/src/lib.rs +++ b/pallets/proofs-dealer/src/lib.rs @@ -387,10 +387,3 @@ impl InherentError { pub trait TrieVerifier { fn verify_proof(root: &[u8; 32], challenges: &[u8; 32], proof: &CompactProof) -> bool; } - -impl TrieVerifier for Pallet { - fn verify_proof(_root: &[u8; 32], _challenges: &[u8; 32], _proof: &CompactProof) -> bool { - // TODO: implement this - todo!() - } -} diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index a734dfd2c..3a21be9ab 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -10,6 +10,7 @@ mod weights; pub mod xcm_config; use cumulus_pallet_parachain_system::RelayNumberStrictlyIncreases; +use pallet_proofs_dealer::{CompactProof, TrieVerifier}; use polkadot_runtime_common::xcm_sender::NoPriceForMessageDelivery; use smallvec::smallvec; use sp_api::impl_runtime_apis; @@ -512,7 +513,7 @@ impl pallet_proofs_dealer::Config for Runtime { type ProvidersPallet = Providers; type NativeBalance = Balances; type MerkleHash = Hash; - type TrieVerifier = ProofsDealer; + type TrieVerifier = ProofTrieVerifier; type MaxChallengesPerBlock = ConstU32<10>; type MaxProvidersChallengedPerBlock = ConstU32<10>; type ChallengeHistoryLength = ConstU32<10>; @@ -520,6 +521,17 @@ impl pallet_proofs_dealer::Config for Runtime { type CheckpointChallengePeriod = ConstU32<10>; } +/// Structure to mock a verifier that returns `true` when `proof` is not empty +/// and `false` otherwise. +pub struct ProofTrieVerifier; + +/// Implement the `TrieVerifier` trait for the `MockVerifier` struct. +impl TrieVerifier for ProofTrieVerifier { + fn verify_proof(_root: &[u8; 32], _challenges: &[u8; 32], proof: &CompactProof) -> bool { + proof.encoded_nodes.len() > 0 + } +} + /// Configure the pallet template in pallets/template. impl pallet_file_system::Config for Runtime { type RuntimeEvent = RuntimeEvent; From f9b879838db72fa2e83436e8553b7a6dbc891f62 Mon Sep 17 00:00:00 2001 From: Facundo Farall <37149322+ffarall@users.noreply.github.com> Date: Thu, 21 Mar 2024 13:47:04 -0300 Subject: [PATCH 13/18] fix: :pencil2: Mend some typos --- pallets/file-system/Cargo.toml | 2 +- pallets/file-system/src/lib.rs | 15 ++++++++------- pallets/file-system/src/utils.rs | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/pallets/file-system/Cargo.toml b/pallets/file-system/Cargo.toml index ce9f589f2..32b3e8ab4 100644 --- a/pallets/file-system/Cargo.toml +++ b/pallets/file-system/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-file-system" -description = "Pallet exposing storage related actions actors can execute within Storage Hub." +description = "Pallet exposing storage related actions actors can execute within StorageHub." version = "0.1.0" homepage = { workspace = true } license = { workspace = true } diff --git a/pallets/file-system/src/lib.rs b/pallets/file-system/src/lib.rs index c45dd8bbf..cbb02602b 100644 --- a/pallets/file-system/src/lib.rs +++ b/pallets/file-system/src/lib.rs @@ -140,7 +140,7 @@ pub mod pallet { /// A double map of [`storage request`](FileLocation) to [`BSPs`](StorageProviderId) that volunteered to store data. /// /// Any BSP under a storage request is considered to be a volunteer and can be removed at any time. - /// Once a BSP submits a valid proof to the `pallet-proofs-dealer-trie`, the `confirmed` field in [`StorageRequestBsps`] should be set to `true`. + /// Once a BSP submits a valid proof to the `pallet-proofs-dealer`, the `confirmed` field in [`StorageRequestBsps`] should be set to `true`. /// /// When a storage request is expired or removed, the corresponding storage request key in this map should be removed. #[pallet::storage] @@ -177,8 +177,8 @@ pub mod pallet { /// A pointer to the starting block to clean up expired storage requests. /// /// If this block is behind the current block number, the cleanup algorithm in `on_idle` will - /// attempt to accelerate this block pointer as close to or up to the current block number if there is - /// enough remaining weight to do so. + /// attempt to accelerate this block pointer as close to or up to the current block number. This + /// will execute provided that there is enough remaining weight to do so. #[pallet::storage] #[pallet::getter(fn next_starting_block_to_clean_up)] pub type NextStartingBlockToCleanUp = StorageValue<_, BlockNumberFor, ValueQuery>; @@ -232,7 +232,7 @@ pub mod pallet { /// BSP already volunteered to store the given file. BspVolunteerFailed, /// Number of BSPs required for storage request has been reached. - StorageRequestBspsRequiredFullfilled, + StorageRequestBspsRequiredFulfilled, /// BSP already volunteered to store the given file. BspAlreadyVolunteered, /// No slot available found in blocks to insert storage request expiration time. @@ -342,9 +342,10 @@ pub mod pallet { /// /// In the event when a storage request no longer exists for the data the BSP no longer stores, /// it is required that the BSP still has access to the metadata of the initial storage request. - /// If they do not, they will at least need the necessary data to reconstruct the bucket ID and - /// request the metadata from an MSP. This metadata is necessary since it is needed to reconstruct - /// the leaf node key in the storage provider's merkle trie. + /// If they do not, they will at least need that metadata to reconstruct the File ID and. Wherever + /// the BSP gets the data it needs is up to it, but one example could be the assigned MSP. + /// This metadata is necessary since it is needed to reconstruct the leaf node key in the storage + /// provider's Merkle Forest. #[pallet::call_index(5)] #[pallet::weight(10_000 + T::DbWeight::get().reads_writes(1,1).ref_time())] pub fn bsp_stop_storing( diff --git a/pallets/file-system/src/utils.rs b/pallets/file-system/src/utils.rs index b725c85c2..ec92995d0 100644 --- a/pallets/file-system/src/utils.rs +++ b/pallets/file-system/src/utils.rs @@ -136,7 +136,7 @@ where // Check that the storage request did not reach the required bsps. ensure!( file_metadata.bsps_confirmed < file_metadata.bsps_required, - Error::::StorageRequestBspsRequiredFullfilled + Error::::StorageRequestBspsRequiredFulfilled ); // Check if the BSP is already volunteered for this storage request. From 646ba541b216992190d1de81480d962e730a7a95 Mon Sep 17 00:00:00 2001 From: Michael Assaf Date: Thu, 21 Mar 2024 13:33:55 -0400 Subject: [PATCH 14/18] amend review; db_write check in iteration, target bsps required rename --- pallets/file-system/src/lib.rs | 24 ++++++++---------------- pallets/file-system/src/mock.rs | 2 +- pallets/file-system/src/tests.rs | 8 ++++---- pallets/file-system/src/types.rs | 4 ++-- pallets/file-system/src/utils.rs | 4 ++-- runtime/src/lib.rs | 2 +- 6 files changed, 18 insertions(+), 26 deletions(-) diff --git a/pallets/file-system/src/lib.rs b/pallets/file-system/src/lib.rs index cbb02602b..00f111dbd 100644 --- a/pallets/file-system/src/lib.rs +++ b/pallets/file-system/src/lib.rs @@ -101,10 +101,15 @@ pub mod pallet { + Zero; /// Minimum number of BSPs required to store a file. + /// + /// This is also used as a default value if the BSPs required are not specified when creating a storage request. #[pallet::constant] - type DefaultBspsRequired: Get; + type TargetBspsRequired: Get; /// Maximum number of BSPs that can store a file. + /// + /// This is used to limit the number of BSPs storing a file and claiming rewards for it. + /// If this number is to high, then the reward for storing a file might be to diluted and pointless to store. #[pallet::constant] type MaxBspsPerStorageRequest: Get; @@ -400,21 +405,7 @@ pub mod pallet { let mut total_used_weight = Weight::zero(); let required_weight_for_iteration = - db_weight.reads_writes(1, T::MaxExpiredStorageRequests::get().into()); - - // Check if we can process a single iteration and update the `NextStartingBlockToCleanUp` storage item - if !remaining_weight.all_gte( - match total_used_weight.checked_add(&required_weight_for_iteration) { - Some(weight) => { - // Update `total_used_weight` to take into account the final write operation - total_used_weight = db_weight.writes(1); - weight - } - None => return Weight::zero(), - }, - ) { - return Weight::zero(); - } + db_weight.reads_writes(1, (T::MaxExpiredStorageRequests::get() + 1).into()); // Iterate over blocks from the start block to the current block, // cleaning up storage requests until the remaining weight is insufficient @@ -446,6 +437,7 @@ pub mod pallet { // `NextStartingBlockToCleanUp` is always updated to start from the block we reached in the current `on_idle` call. if block_to_clean > start_block { NextStartingBlockToCleanUp::::put(block_to_clean); + total_used_weight += db_weight.writes(1); } total_used_weight diff --git a/pallets/file-system/src/mock.rs b/pallets/file-system/src/mock.rs index c564dac1b..b9a195423 100644 --- a/pallets/file-system/src/mock.rs +++ b/pallets/file-system/src/mock.rs @@ -142,7 +142,7 @@ impl crate::Config for Test { type Fingerprint = H256; type StorageUnit = u128; type StorageRequestBspsRequiredType = u32; - type DefaultBspsRequired = ConstU32<1>; + type TargetBspsRequired = ConstU32<1>; type MaxBspsPerStorageRequest = ConstU32<5>; type MaxMultiAddresses = ConstU32<5>; // TODO: this should probably be a multiplier of the number of maximum multiaddresses per storage provider type MaxFilePathSize = ConstU32<512u32>; diff --git a/pallets/file-system/src/tests.rs b/pallets/file-system/src/tests.rs index 9d6400521..3544cbae0 100644 --- a/pallets/file-system/src/tests.rs +++ b/pallets/file-system/src/tests.rs @@ -1,8 +1,8 @@ use crate::{ mock::*, types::{ - DefaultBspsRequired, FileLocation, MultiAddress, StorageRequestBspsMetadata, - StorageRequestMetadata, + FileLocation, MultiAddress, StorageRequestBspsMetadata, StorageRequestMetadata, + TargetBspsRequired, }, Config, Event, StorageRequestExpirations, }; @@ -48,7 +48,7 @@ fn request_storage_success() { size, user_multiaddresses: multiaddresses.clone(), data_server_sps: BoundedVec::default(), - bsps_required: DefaultBspsRequired::::get(), + bsps_required: TargetBspsRequired::::get(), bsps_confirmed: 0, }) ); @@ -411,7 +411,7 @@ fn bsp_stop_storing_success() { size, user_multiaddresses: multiaddresses.clone(), data_server_sps: BoundedVec::default(), - bsps_required: DefaultBspsRequired::::get(), + bsps_required: TargetBspsRequired::::get(), bsps_confirmed: 0, }) ); diff --git a/pallets/file-system/src/types.rs b/pallets/file-system/src/types.rs index dd1466c19..dabf26944 100644 --- a/pallets/file-system/src/types.rs +++ b/pallets/file-system/src/types.rs @@ -72,8 +72,8 @@ pub type Fingerprint = ::Fingerprint; /// Alias for the `StorageCount` type used in the FileSystem pallet. pub type StorageUnit = ::StorageUnit; -/// Alias for the `DefaultBspsRequired` type used in the FileSystem pallet. -pub type DefaultBspsRequired = ::DefaultBspsRequired; +/// Alias for the `TargetBspsRequired` type used in the FileSystem pallet. +pub type TargetBspsRequired = ::TargetBspsRequired; /// Byte array representing the file path. pub type FileLocation = BoundedVec>; diff --git a/pallets/file-system/src/utils.rs b/pallets/file-system/src/utils.rs index ec92995d0..d5dedfd59 100644 --- a/pallets/file-system/src/utils.rs +++ b/pallets/file-system/src/utils.rs @@ -9,7 +9,7 @@ use sp_runtime::{ }; use sp_std::vec; -use crate::types::{DefaultBspsRequired, FileKey}; +use crate::types::{FileKey, TargetBspsRequired}; use crate::{ pallet, types::{ @@ -59,7 +59,7 @@ where // TODO: Check storage capacity of chosen MSP (when we support MSPs) // TODO: Return error if the file is already stored and overwrite is false. - let bsps_required = bsps_required.unwrap_or(DefaultBspsRequired::::get()); + let bsps_required = bsps_required.unwrap_or(TargetBspsRequired::::get()); if bsps_required.is_zero() { return Err(Error::::BspsRequiredCannotBeZero)?; diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 3a21be9ab..4500808bc 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -538,7 +538,7 @@ impl pallet_file_system::Config for Runtime { type Fingerprint = Hash; type StorageUnit = u128; type StorageRequestBspsRequiredType = u32; - type DefaultBspsRequired = ConstU32<1>; + type TargetBspsRequired = ConstU32<1>; type MaxBspsPerStorageRequest = ConstU32<5>; type MaxFilePathSize = ConstU32<512u32>; type MaxMultiAddresses = ConstU32<10>; From 0d3e16482d14f4c60aa66b77c165dd8719a50849 Mon Sep 17 00:00:00 2001 From: Michael Assaf Date: Thu, 21 Mar 2024 15:53:37 -0400 Subject: [PATCH 15/18] amend review; expect_or_err bsps_confirmed on volunteer, checked_add write --- pallets/file-system/src/lib.rs | 8 ++++++-- pallets/file-system/src/utils.rs | 27 +++++++++++++++++++++------ 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/pallets/file-system/src/lib.rs b/pallets/file-system/src/lib.rs index 00f111dbd..7f240c9f7 100644 --- a/pallets/file-system/src/lib.rs +++ b/pallets/file-system/src/lib.rs @@ -400,12 +400,16 @@ pub mod pallet { let start_block = NextStartingBlockToCleanUp::::get(); let mut block_to_clean = start_block; + let writes = match T::MaxExpiredStorageRequests::get().checked_add(1) { + Some(writes) => writes, + None => return Weight::zero(), + }; + // Total weight used to avoid exceeding the remaining weight. // We count one write for the `NextBlockToCleanup` storage item updated at the end. let mut total_used_weight = Weight::zero(); - let required_weight_for_iteration = - db_weight.reads_writes(1, (T::MaxExpiredStorageRequests::get() + 1).into()); + let required_weight_for_iteration = db_weight.reads_writes(1, writes.into()); // Iterate over blocks from the start block to the current block, // cleaning up storage requests until the remaining weight is insufficient diff --git a/pallets/file-system/src/utils.rs b/pallets/file-system/src/utils.rs index d5dedfd59..29769d967 100644 --- a/pallets/file-system/src/utils.rs +++ b/pallets/file-system/src/utils.rs @@ -21,7 +21,8 @@ use crate::{ }; macro_rules! expect_or_err { - ($optional:expr, $error_msg:expr, $error_type:path) => { + // Handle Option type + ($optional:expr, $error_msg:expr, $error_type:path) => {{ match $optional { Some(value) => value, None => { @@ -30,11 +31,23 @@ macro_rules! expect_or_err { #[allow(unreachable_code)] { - Err($error_type)? + return Err($error_type.into()); } } } - }; + }}; + // Handle boolean type + ($condition:expr, $error_msg:expr, $error_type:path, bool) => {{ + if !$condition { + #[cfg(test)] + unreachable!($error_msg); + + #[allow(unreachable_code)] + { + return Err($error_type.into()); + } + } + }}; } impl Pallet @@ -133,10 +146,11 @@ where Error::::StorageRequestNotFound ); - // Check that the storage request did not reach the required bsps. - ensure!( + expect_or_err!( file_metadata.bsps_confirmed < file_metadata.bsps_required, - Error::::StorageRequestBspsRequiredFulfilled + "Storage request should never have confirmed bsps equal to or greater than required bsps, since they are deleted when it is reached.", + Error::::StorageRequestBspsRequiredFulfilled, + bool ); // Check if the BSP is already volunteered for this storage request. @@ -252,6 +266,7 @@ where } }; + // TODO: loose couple this with a trait // Challenge BSP to force update its storage root to uninclude the file. pallet_proofs_dealer::Pallet::::do_challenge(&who, &file_key)?; From 66acd035a67911e75bcd91999a228e5ded516531 Mon Sep 17 00:00:00 2001 From: Michael Assaf Date: Thu, 21 Mar 2024 17:18:29 -0400 Subject: [PATCH 16/18] proof dealer pallet treasury account placeholder --- pallets/file-system/src/mock.rs | 14 ++++++++++++-- pallets/file-system/src/utils.rs | 8 ++++---- runtime/src/lib.rs | 13 ++++++++++++- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/pallets/file-system/src/mock.rs b/pallets/file-system/src/mock.rs index b9a195423..64c2cd8ca 100644 --- a/pallets/file-system/src/mock.rs +++ b/pallets/file-system/src/mock.rs @@ -5,10 +5,10 @@ use frame_support::{ }; use frame_system as system; use pallet_proofs_dealer::{CompactProof, TrieVerifier}; -use sp_core::{ConstU128, ConstU32, H256}; +use sp_core::{ConstU128, ConstU32, Get, H256}; use sp_runtime::{ traits::{BlakeTwo256, IdentityLookup}, - BuildStorage, + AccountId32, BuildStorage, }; type Block = frame_system::mocking::MockBlock; @@ -113,6 +113,14 @@ impl pallet_storage_providers::Config for Test { type DepositPerData = ConstU128<2>; } +// TODO: remove this and replace with pallet treasury +pub struct TreasuryAccount; +impl Get for TreasuryAccount { + fn get() -> AccountId32 { + AccountId32::from([0; 32]) + } +} + impl pallet_proofs_dealer::Config for Test { type RuntimeEvent = RuntimeEvent; type ProvidersPallet = Providers; @@ -124,6 +132,8 @@ impl pallet_proofs_dealer::Config for Test { type ChallengeHistoryLength = ConstU32<10>; type ChallengesQueueLength = ConstU32<10>; type CheckpointChallengePeriod = ConstU32<10>; + type ChallengesFee = ConstU128<1_000_000>; + type Treasury = TreasuryAccount; } /// Structure to mock a verifier that returns `true` when `proof` is not empty diff --git a/pallets/file-system/src/utils.rs b/pallets/file-system/src/utils.rs index 29769d967..4a726988c 100644 --- a/pallets/file-system/src/utils.rs +++ b/pallets/file-system/src/utils.rs @@ -31,7 +31,7 @@ macro_rules! expect_or_err { #[allow(unreachable_code)] { - return Err($error_type.into()); + Err($error_type)? } } } @@ -44,7 +44,7 @@ macro_rules! expect_or_err { #[allow(unreachable_code)] { - return Err($error_type.into()); + Err($error_type)? } } }}; @@ -217,7 +217,7 @@ where /// will handle triggering the appropriate event and pending storage request. pub(crate) fn do_bsp_stop_storing( who: StorageProviderId, - file_key: FileKey, + _file_key: FileKey, location: FileLocation, owner: T::AccountId, fingerprint: Fingerprint, @@ -268,7 +268,7 @@ where // TODO: loose couple this with a trait // Challenge BSP to force update its storage root to uninclude the file. - pallet_proofs_dealer::Pallet::::do_challenge(&who, &file_key)?; + // pallet_proofs_dealer::Pallet::::do_challenge(&who, &file_key)?; Ok(()) } diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 4500808bc..87c13b529 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -14,7 +14,7 @@ use pallet_proofs_dealer::{CompactProof, TrieVerifier}; use polkadot_runtime_common::xcm_sender::NoPriceForMessageDelivery; use smallvec::smallvec; use sp_api::impl_runtime_apis; -use sp_core::{crypto::KeyTypeId, OpaqueMetadata}; +use sp_core::{crypto::KeyTypeId, Get, OpaqueMetadata}; use sp_runtime::{ create_runtime_str, generic, impl_opaque_keys, traits::{BlakeTwo256, Block as BlockT, IdentifyAccount, Verify}, @@ -47,6 +47,7 @@ use frame_system::{ use pallet_xcm::{EnsureXcm, IsVoiceOfBody}; use parachains_common::message_queue::{NarrowOriginToSibling, ParaIdToSibling}; pub use sp_consensus_aura::sr25519::AuthorityId as AuraId; +use sp_runtime::AccountId32; pub use sp_runtime::{MultiAddress, Perbill, Permill}; use xcm_config::{RelayLocation, XcmOriginToTransactDispatchOrigin}; @@ -508,6 +509,14 @@ impl pallet_storage_providers::Config for Runtime { type DepositPerData = ConstU128<2>; } +// TODO: remove this and replace with pallet treasury +pub struct TreasuryAccount; +impl Get for TreasuryAccount { + fn get() -> AccountId32 { + AccountId32::from([0; 32]) + } +} + impl pallet_proofs_dealer::Config for Runtime { type RuntimeEvent = RuntimeEvent; type ProvidersPallet = Providers; @@ -519,6 +528,8 @@ impl pallet_proofs_dealer::Config for Runtime { type ChallengeHistoryLength = ConstU32<10>; type ChallengesQueueLength = ConstU32<10>; type CheckpointChallengePeriod = ConstU32<10>; + type ChallengesFee = ConstU128<1_000_000>; + type Treasury = TreasuryAccount; } /// Structure to mock a verifier that returns `true` when `proof` is not empty From d0e5eea8d794ae805d025154196dd9cc19df2434 Mon Sep 17 00:00:00 2001 From: Michael Assaf Date: Thu, 21 Mar 2024 17:36:24 -0400 Subject: [PATCH 17/18] fix treasury account mock --- pallets/file-system/src/mock.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pallets/file-system/src/mock.rs b/pallets/file-system/src/mock.rs index 64c2cd8ca..dbaf20fa1 100644 --- a/pallets/file-system/src/mock.rs +++ b/pallets/file-system/src/mock.rs @@ -8,12 +8,13 @@ use pallet_proofs_dealer::{CompactProof, TrieVerifier}; use sp_core::{ConstU128, ConstU32, Get, H256}; use sp_runtime::{ traits::{BlakeTwo256, IdentityLookup}, - AccountId32, BuildStorage, + BuildStorage, }; type Block = frame_system::mocking::MockBlock; pub(crate) type BlockNumber = u64; type Balance = u128; +type AccountId = u64; /// Rolls to the desired block. Returns the number of blocks played. pub(crate) fn roll_to(n: BlockNumber) -> BlockNumber { @@ -61,7 +62,7 @@ impl system::Config for Test { type Nonce = u64; type Hash = H256; type Hashing = BlakeTwo256; - type AccountId = u64; + type AccountId = AccountId; type Lookup = IdentityLookup; type Block = Block; type RuntimeEvent = RuntimeEvent; @@ -115,9 +116,9 @@ impl pallet_storage_providers::Config for Test { // TODO: remove this and replace with pallet treasury pub struct TreasuryAccount; -impl Get for TreasuryAccount { - fn get() -> AccountId32 { - AccountId32::from([0; 32]) +impl Get for TreasuryAccount { + fn get() -> AccountId { + 1 } } From a8f7f059921a81c6f9a33a403297202af56e425a Mon Sep 17 00:00:00 2001 From: Michael Assaf <94772640+snowmead@users.noreply.github.com> Date: Thu, 21 Mar 2024 21:00:31 -0400 Subject: [PATCH 18/18] Update pallets/file-system/src/lib.rs Co-authored-by: Facundo Farall <37149322+ffarall@users.noreply.github.com> --- pallets/file-system/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pallets/file-system/src/lib.rs b/pallets/file-system/src/lib.rs index 7f240c9f7..0753e4859 100644 --- a/pallets/file-system/src/lib.rs +++ b/pallets/file-system/src/lib.rs @@ -402,6 +402,8 @@ pub mod pallet { let writes = match T::MaxExpiredStorageRequests::get().checked_add(1) { Some(writes) => writes, + // This should never happen. It would mean that MaxExpiredStorageRequests is u32::MAX, + // which is an irrational number to set as a limit. None => return Weight::zero(), };