diff --git a/apps/commune-worker/src/db/type-transformations.ts b/apps/commune-worker/src/db/type-transformations.ts index d81a75d..2d002dd 100644 --- a/apps/commune-worker/src/db/type-transformations.ts +++ b/apps/commune-worker/src/db/type-transformations.ts @@ -19,9 +19,8 @@ export function SubspaceModuleToDatabase( incentive: module.incentive ?? null, dividend: module.dividends ?? null, delegationFee: module.delegationFee ?? null, - totalStaked: null, - totalStakers: null, - totalRewards: null, + totalStaked: module.totalStaked, + totalStakers: module.totalStakers, moduleId: module.uid, isWhitelisted: whitelisted, }; diff --git a/apps/commune-worker/src/workers/module-fetcher.ts b/apps/commune-worker/src/workers/module-fetcher.ts index d5a8381..6155c2c 100644 --- a/apps/commune-worker/src/workers/module-fetcher.ts +++ b/apps/commune-worker/src/workers/module-fetcher.ts @@ -31,6 +31,7 @@ export async function moduleFetcherWorker(props: WorkerProps) { const modules = await queryRegisteredModulesInfo( lastBlock.apiAtBlock, NETUID_ZERO, + props.lastBlock.blockNumber, ); const modulesData = modules.map((module) => SubspaceModuleToDatabase( diff --git a/packages/subspace/queries/index.ts b/packages/subspace/queries/index.ts index 7180a5f..e746f1d 100644 --- a/packages/subspace/queries/index.ts +++ b/packages/subspace/queries/index.ts @@ -23,6 +23,7 @@ import { checkSS58, GOVERNANCE_CONFIG_SCHEMA, isSS58, + STAKE_FROM_SCHEMA, MODULE_BURN_CONFIG_SCHEMA, NetworkSubnetConfigSchema, SUBSPACE_MODULE_SCHEMA, @@ -34,6 +35,7 @@ import { standardizeUidToSS58address, } from "@commune-ts/utils"; + export { ApiPromise }; // == chain == @@ -557,9 +559,26 @@ export async function querySubnetParams(api: Api): Promise> +) { + const stakerMap = stakeFromStorage.get(targetKey); + let totalStake = 0n; + if (stakerMap === undefined) { + console.log("could not find map for key: ", targetKey); + return totalStake; + } + for (const stake of stakerMap.values()) { + totalStake += stake; + } + return totalStake; +} + export async function queryRegisteredModulesInfo( api: Api, netuid: number, + blockNumber: number, ): Promise { console.log("Fetching module keys from the chain..."); const keyQuery: { subspaceModule: SubspaceStorageName[] } = { @@ -576,13 +595,15 @@ export async function queryRegisteredModulesInfo( "emission", "incentive", "dividends", - ]; - const extraPropsQuery: { subspaceModule: SubspaceStorageName[] } = { - subspaceModule: moduleProps, - }; + "delegationFee", + "stakeFrom", + ] + + const extraPropsQuery: { subspaceModule: SubspaceStorageName[] } = { subspaceModule: moduleProps } const modulesInfo = await queryChain(api, extraPropsQuery, netuid); const processedModules = standardizeUidToSS58address(modulesInfo, uidToSS58); const moduleMap: SubspaceModule[] = []; + const parsedStakeFromStorage = STAKE_FROM_SCHEMA.parse({ stakeFromStorage: processedModules.stakeFrom }); for (const uid of Object.keys(uidToSS58)) { const moduleKey = uidToSS58[uid]; @@ -599,13 +620,15 @@ export async function queryRegisteredModulesInfo( registrationBlock: processedModules.registrationBlock[moduleKey], metadata: processedModules.metadata[moduleKey], lastUpdate: processedModules.lastUpdate[moduleKey], - atBlock: 0, + atBlock: blockNumber, emission: processedModules.emission[moduleKey], incentive: processedModules.incentive[moduleKey], dividends: processedModules.dividends[moduleKey], - delegationFee: 0, - stakeFrom: 0n, - }); + delegationFee: processedModules.delegationFee[moduleKey], + totalStaked: keyStakeFrom(moduleKey, parsedStakeFromStorage.stakeFromStorage), + totalStakers: parsedStakeFromStorage.stakeFromStorage.get(moduleKey)?.size ?? 0, + + }) moduleMap.push(module); } return moduleMap; diff --git a/packages/types/validations.ts b/packages/types/validations.ts index 18ad236..c4a709e 100644 --- a/packages/types/validations.ts +++ b/packages/types/validations.ts @@ -4,12 +4,9 @@ import { assert } from "tsafe"; import { z } from "zod"; import type { - Codec, DaoApplicationStatus, - OptionalProperties, Proposal, SS58Address, - SubspaceModule, } from "./types"; export function checkSS58(value: string | SS58Address): SS58Address { @@ -145,6 +142,26 @@ export const NUMBER_SCHEMA = z.coerce.number(); export const SUBSPACE_MODULE_REGISTRATION_BLOCK_SCHEMA = z.coerce.number(); export const SUBSPACE_MODULE_METADATA_SCHEMA = z.string(); // TODO: validate it's a valid ipfs hash or something (?) export const SUBSPACE_MODULE_LAST_UPDATE_SCHEMA = z.any(); +export const STAKE_FROM_SCHEMA = z.object({ + stakeFromStorage: z.record( + ADDRESS_SCHEMA, + z.record(ADDRESS_SCHEMA, z.coerce.bigint()) + ).transform( + (val) => { + const map = new Map>(); + const stakeMapEntries = Object.entries(val) as [SS58Address, Record][]; + for (const [stakedInto, stakerMap] of stakeMapEntries) { + const innerMap = new Map(); + const stakers = Object.entries(stakerMap) as [SS58Address, bigint][]; + for (const [staker, stake] of stakers) { + innerMap.set(staker, BigInt(stake)); + } + map.set(stakedInto, innerMap); + } + return map; + }, + ), +}); export const SUBSPACE_MODULE_SCHEMA = z.object({ netuid: z.coerce.number(), @@ -162,5 +179,6 @@ export const SUBSPACE_MODULE_SCHEMA = z.object({ dividends: z.coerce.bigint().optional(), delegationFee: z.coerce.number().optional(), - stakeFrom: z.bigint().optional(), + totalStaked: z.coerce.bigint(), + totalStakers: z.coerce.number(), }); diff --git a/packages/utils/index.ts b/packages/utils/index.ts index b037276..06c8745 100644 --- a/packages/utils/index.ts +++ b/packages/utils/index.ts @@ -2,7 +2,6 @@ import { BN, stringToHex } from "@polkadot/util"; import { CID } from "multiformats/cid"; import { match } from "rustie"; import { AssertionError } from "tsafe"; - import type { AnyTuple, Api, @@ -269,7 +268,8 @@ export function getExpirationTime( } export interface ChainEntry { - getMapModules(netuid?: number): Record; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + getMapModules(netuid?: number): Record; } export type SubspacePalletName = @@ -278,49 +278,27 @@ export type SubspacePalletName = | "subnetEmissionModule"; export type SubspaceStorageName = - | "emission" - | "incentive" - | "dividends" - | "lastUpdate" - | "metadata" - | "registrationBlock" - | "name" - | "address" - | "keys" - | "subnetNames" - | "immunityPeriod" - | "minAllowedWeights" - | "maxAllowedWeights" - | "tempo" - | "maxAllowedUids" - | "founder" - | "founderShare" - | "incentiveRatio" - | "trustRatio" - | "maxWeightAge" - | "bondsMovingAverage" - | "maximumSetWeightCallsPerEpoch" - | "minValidatorStake" - | "maxAllowedValidators" - | "moduleBurnConfig" - | "subnetMetadata" - | "subnetGovernanceConfig" - | "subnetEmission"; + | "emission" | "incentive" | "dividends" | "lastUpdate" + | "metadata" | "registrationBlock" | "name" | "address" + | "keys" | "subnetNames" | "immunityPeriod" | "minAllowedWeights" + | "maxAllowedWeights" | "tempo" | "maxAllowedUids" | "founder" + | "founderShare" | "incentiveRatio" | "trustRatio" | "maxWeightAge" + | "bondsMovingAverage" | "maximumSetWeightCallsPerEpoch" | "minValidatorStake" + | "maxAllowedValidators" | "moduleBurnConfig" | "subnetMetadata" + | "subnetGovernanceConfig" | "subnetEmission" | "delegationFee" + | "stakeFrom"; // TODO: add MinimumAllowedStake, stakeFrom -export function standardizeUidToSS58address( - outerRecord: Record>, +export function standardizeUidToSS58address( + outerRecord: Record>, uidToKey: Record, -): Record> { - const processedRecord: Record< - T, - Record - > = {} as Record>; +): Record> { + const processedRecord: Record> = {} as Record>; - const entries = Object.entries(outerRecord) as [T, Record][]; + const entries = Object.entries(outerRecord) as [T, Record][]; for (const [outerKey, innerRecord] of entries) { - const processedInnerRecord: Record = {}; + const processedInnerRecord: Record = {}; for (const [innerKey, value] of Object.entries(innerRecord)) { if (!isNaN(Number(innerKey))) { @@ -339,53 +317,39 @@ export function standardizeUidToSS58address( return processedRecord; } -type StorageTypes = "VecMapping" | "DoubleMap" | "SimpleMap"; + +type StorageTypes = "VecMapping" | "NetuidMap" | "SimpleMap" | "DoubleMap"; export function getSubspaceStorageMappingKind( prop: SubspaceStorageName, ): StorageTypes | null { const vecProps: SubspaceStorageName[] = [ - "emission", - "incentive", - "dividends", - "lastUpdate", + "emission", "incentive", "dividends", "lastUpdate", ]; - const doubleMapProps: SubspaceStorageName[] = [ - "metadata", - "registrationBlock", - "name", - "address", - "keys", + const netuidMapProps: SubspaceStorageName[] = [ + "metadata", "registrationBlock", "name", "address", "keys" ]; const simpleMapProps: SubspaceStorageName[] = [ - "minAllowedWeights", - "maxWeightAge", - "maxAllowedWeights", - "trustRatio", - "tempo", - "founderShare", - "subnetNames", - "immunityPeriod", - "maxAllowedUids", - "founder", - "incentiveRatio", - "bondsMovingAverage", - "maximumSetWeightCallsPerEpoch", - "minValidatorStake", - "maxAllowedValidators", - "moduleBurnConfig", - "subnetMetadata", - "subnetGovernanceConfig", - "subnetEmission", + "minAllowedWeights", "maxWeightAge", "maxAllowedWeights", "trustRatio", + "tempo", "founderShare", "subnetNames", "immunityPeriod", "maxAllowedUids", + "founder", "incentiveRatio", "bondsMovingAverage", + "maximumSetWeightCallsPerEpoch", "minValidatorStake", "maxAllowedValidators", + "moduleBurnConfig", "subnetMetadata", "subnetGovernanceConfig", "subnetEmission", + "delegationFee" + ] + const doubleMapProps: SubspaceStorageName[] = [ + "stakeFrom" ]; const mapping = { VecMapping: vecProps, - DoubleMap: doubleMapProps, + NetuidMap: netuidMapProps, SimpleMap: simpleMapProps, + DoubleMap: doubleMapProps, }; if (mapping.VecMapping.includes(prop)) return "VecMapping"; - else if (mapping.DoubleMap.includes(prop)) return "DoubleMap"; + else if (mapping.NetuidMap.includes(prop)) return "NetuidMap"; else if (mapping.SimpleMap.includes(prop)) return "SimpleMap"; + else if (mapping.DoubleMap.includes(prop)) return "DoubleMap"; else return null; } @@ -401,34 +365,33 @@ export async function getPropsToMap( const asyncOperations = storageNames.map(async (storageName) => { const value = getSubspaceStorageMappingKind(storageName); - if (value === "VecMapping") { - const entries = await api.query[palletName]?.[storageName]?.entries(); - - if (entries === undefined) { - console.log(`No entries for ${palletName}.${storageName}`); - // TODO: panic - process.exit(1); - } - mapped_prop_entries[storageName] = new StorageVecMap(entries); - } else if (value === "DoubleMap") { - const entries = - await api.query[palletName]?.[storageName]?.entries(netuid); - + if (value === "NetuidMap") { + const entries = await api.query[palletName]?.[storageName]?.entries(netuid); if (entries === undefined) { - console.log(`No entries for ${palletName}.${storageName}`); - // TODO: panic - process.exit(1); + throw new Error(`No entries for ${palletName}.${storageName}`); } - mapped_prop_entries[storageName] = new DoubleMapEntries(entries); - } else if (value === "SimpleMap") { - const entries = await api.query[palletName]?.[storageName]?.entries(); - - if (entries === undefined) { - console.log(`No entries for ${palletName}.${storageName}`); - // TODO: panic - process.exit(1); - } - mapped_prop_entries[storageName] = new SimpleMap(entries); + } + const storageQuery = api.query[palletName]?.[storageName]?.entries; + if (storageQuery === undefined) { + throw new Error(`${palletName}.${storageName} doesn't exist`); + } + const entries = value === "NetuidMap" ? await storageQuery(netuid) : await storageQuery(); + + switch (value) { + case "VecMapping": + mapped_prop_entries[storageName] = new StorageVecMap(entries); + break; + case "NetuidMap": + mapped_prop_entries[storageName] = new NetuidMapEntries(entries); + break; + case "SimpleMap": + mapped_prop_entries[storageName] = new SimpleMap(entries); + break; + case "DoubleMap": + mapped_prop_entries[storageName] = new DoubleMapEntries(entries); + break; + default: + throw new Error(`Unknown storage type ${value}`); } }); @@ -442,9 +405,6 @@ export class StorageVecMap implements ChainEntry { constructor(private readonly entry: [StorageKey, Codec][]) {} getMapModules(netuid: number) { - if (netuid === undefined) { - throw new Error("netuid is required for StorageVecMap"); - } const subnet_values = this.entry[netuid]; if (subnet_values != undefined) { const values = subnet_values[1].toPrimitive() as string[]; @@ -467,12 +427,11 @@ export class SimpleMap implements ChainEntry { } } -export class DoubleMapEntries implements ChainEntry { - constructor(private readonly entries: [StorageKey, Codec][]) {} +export class NetuidMapEntries implements ChainEntry { + constructor(private readonly entries: [StorageKey, Codec][]) { } getMapModules() { const moduleIdToPropValue: Record = {}; - - this.entries.forEach((entry) => { + this.entries.forEach(entry => { const moduleCodec = entry[1]; const moduleId = entry[0].args[1]?.toPrimitive() as number; moduleIdToPropValue[moduleId] = moduleCodec.toPrimitive() as string; @@ -481,6 +440,27 @@ export class DoubleMapEntries implements ChainEntry { } } + +export class DoubleMapEntries implements ChainEntry { + constructor(private readonly entries: [StorageKey, Codec][]) { } + getMapModules() { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const moduleIdToPropValue: Record> = {}; + + this.entries.forEach(entry => { + const keyFrom = entry[0].args[0]?.toPrimitive() as string; + const keyTo = entry[0].args[1]?.toPrimitive() as string; + if (moduleIdToPropValue[keyFrom] === undefined) { + moduleIdToPropValue[keyFrom] = {}; + } + moduleIdToPropValue[keyFrom][keyTo] = entry[1].toPrimitive() as string; + + }); + return moduleIdToPropValue; + } +} + + export function parseAddress(valueRaw: Codec): DaoApplications | null { const value = valueRaw.toPrimitive(); const validated = DAO_APPLICATIONS_SCHEMA.safeParse(value);