From 7a8c6d79e6ab69d66d018b46b41ee304379f4b98 Mon Sep 17 00:00:00 2001 From: g1nt0ki <99907941+g1nt0ki@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:09:28 +0100 Subject: [PATCH 1/4] refactor sol token supply query --- defi/l2/utils.ts | 113 ++++++++++++++++++++++++++--------------- defi/package-lock.json | 54 ++++++++++++++++++++ defi/package.json | 1 + 3 files changed, 126 insertions(+), 42 deletions(-) diff --git a/defi/l2/utils.ts b/defi/l2/utils.ts index 3484fd65e2..ebd7047d75 100644 --- a/defi/l2/utils.ts +++ b/defi/l2/utils.ts @@ -13,6 +13,10 @@ import { storeNotTokens } from "./layer2pg"; import { getBlock } from "@defillama/sdk/build/util/blocks"; import { readFromPGCache, writeToPGCache } from "../src/api2/db"; import { unixTimestampNow } from "../emissions-adapters/utils/time"; +import { Connection, PublicKey } from "@solana/web3.js" +import * as sdk from "@defillama/sdk"; + +import { struct, u8, u64 } from 'buffer-layout'; type CachedSupplies = { timestamp: number; data: { [token: string]: number } }; export function aggregateChainTokenBalances(usdTokenBalances: AllProtocols): TokenTvlData { @@ -64,8 +68,7 @@ export async function getPrices( restCallWrapper( () => fetch( - `https://coins.llama.fi/prices?source=internal${ - process.env.COINS_KEY ? `?apikey=${process.env.COINS_KEY}` : "" + `https://coins.llama.fi/prices?source=internal${process.env.COINS_KEY ? `?apikey=${process.env.COINS_KEY}` : "" }`, { method: "POST", @@ -190,55 +193,81 @@ async function getAptosSupplies(tokens: string[], timestamp?: number): Promise<{ await storeNotTokens(notTokens); return supplies; } + +let connection: any = {} + +const endpoint = (isClient = false) => { + if (isClient) return process.env.SOLANA_RPC_CLIENT ?? process.env.SOLANA_RPC + return process.env.SOLANA_RPC +} + +const endpointMap: any = { + solana: endpoint, +} + +function getConnection(chain = 'solana') { + if (!connection[chain]) connection[chain] = new Connection(endpointMap[chain](true)) + return connection[chain] +} + + + async function getSolanaTokenSupply(tokens: string[], timestamp?: number): Promise<{ [token: string]: number }> { if (timestamp) throw new Error(`timestamp incompatible with Solana adapter!`); - const now = unixTimestampNow(); - const margin = 12 * 60 * 60; // 12hrs - const cachedSupplies: CachedSupplies = await readFromPGCache("L2-solanaTokenSupplies"); + const solanaMintLayout = struct([ + u8('mintAuthorityOption'), + u8('mintAuthority'), + u64('supply'), + u8('decimals'), + u8('isInitialized'), + u8('freezeAuthorityOption'), + u8('freezeAuthority'), + ]) + + const sleepTime = tokens.length > 2000 ? 2000 : 200 + const tokensPK = tokens.map(i => typeof i === 'string' ? new PublicKey(i) : i) + const connection = getConnection('solana') + const res = await runInChunks(tokensPK, (chunk: any) => connection.getMultipleAccountsInfo(chunk), { sleepTime }) + const supplies: { [token: string]: number } = {}; - if (cachedSupplies && "timestamp" in cachedSupplies && now - cachedSupplies.timestamp < margin) - return cachedSupplies.data; + res.forEach((data, idx) => { + if (!data) { + sdk.log(`Invalid account: ${tokens[idx]}`) + return; + } + try { + const buffer = Buffer.from(data.data) + const decoded = solanaMintLayout.decode(buffer) + supplies['solana:'+tokens[idx]] = decoded.supply.toString() + } catch (e) { + sdk.log(`Error decoding account: ${tokens[idx]}`) + } + }) - const supplies: { [token: string]: number } = {}; - let i = 0; - let j = 0; - const notTokens: string[] = []; - console.log(`length: ${tokens.length}`); - if (!process.env.SOLANA_RPC) throw new Error(`no Solana RPC supplied`); - await PromisePool.withConcurrency(50) - .for(tokens) - .process(async (token) => { - tokens; - i; - try { - const res = await fetch(process.env.SOLANA_RPC ?? "", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - jsonrpc: "2.0", - id: 1, - method: "getTokenSupply", - params: [token], - }), - }).then((r) => r.json()); - i++; + return supplies; - if (res.error) { - notTokens.push(`solana:${token}`); - res.error; - } else supplies[`solana:${token}`] = res.result.value.amount; - } catch (e) { - j++; - } - }); + async function runInChunks(inputs: any, fn: any, { chunkSize = 99, sleepTime }: any = {}) { + const chunks = sliceIntoChunks(inputs, chunkSize) + const results = [] + for (const chunk of chunks) { + results.push(...(await fn(chunk) ?? [])) + if (sleepTime) await sleep(sleepTime) + } - let a = Object.keys(supplies).length; - const cacheData: CachedSupplies = { data: supplies, timestamp: now }; - await Promise.all([storeNotTokens(notTokens), writeToPGCache("L2-solanaTokenSupplies", cacheData)]); + return results.flat() - return supplies; + function sliceIntoChunks(arr: any, chunkSize = 100) { + const res = []; + for (let i = 0; i < arr.length; i += chunkSize) { + const chunk = arr.slice(i, i + chunkSize); + res.push(chunk); + } + return res; + } + } } + async function getSuiSupplies(tokens: Address[], timestamp?: number): Promise<{ [token: string]: number }> { if (timestamp) throw new Error(`timestamp incompatible with Sui adapter!`); const supplies: { [token: string]: number } = {}; diff --git a/defi/package-lock.json b/defi/package-lock.json index 7029bc7f2b..de17a099d9 100644 --- a/defi/package-lock.json +++ b/defi/package-lock.json @@ -26,6 +26,7 @@ "@types/promise.allsettled": "^1.0.6", "axios": "^1.6.5", "bignumber.js": "^9.0.1", + "buffer-layout": "^1.2.2", "dotenv": "^16.0.1", "ethers": "^6.0.0", "hyper-express": "^6.8.7", @@ -72,6 +73,7 @@ "hasInstallScript": true, "license": "ISC", "dependencies": { + "@coral-xyz/anchor": "^0.30.1", "@defillama/sdk": "latest", "@project-serum/anchor": "^0.26.0", "@solana/web3.js": "^1.87.6", @@ -103,6 +105,48 @@ "ts-node": "^10.8.1" } }, + "DefiLlama-Adapters/node_modules/@coral-xyz/anchor": { + "version": "0.30.1", + "resolved": "https://registry.npmjs.org/@coral-xyz/anchor/-/anchor-0.30.1.tgz", + "integrity": "sha512-gDXFoF5oHgpriXAaLpxyWBHdCs8Awgf/gLHIo6crv7Aqm937CNdY+x+6hoj7QR5vaJV7MxWSQ0NGFzL3kPbWEQ==", + "license": "(MIT OR Apache-2.0)", + "dependencies": { + "@coral-xyz/anchor-errors": "^0.30.1", + "@coral-xyz/borsh": "^0.30.1", + "@noble/hashes": "^1.3.1", + "@solana/web3.js": "^1.68.0", + "bn.js": "^5.1.2", + "bs58": "^4.0.1", + "buffer-layout": "^1.2.2", + "camelcase": "^6.3.0", + "cross-fetch": "^3.1.5", + "crypto-hash": "^1.3.0", + "eventemitter3": "^4.0.7", + "pako": "^2.0.3", + "snake-case": "^3.0.4", + "superstruct": "^0.15.4", + "toml": "^3.0.0" + }, + "engines": { + "node": ">=11" + } + }, + "DefiLlama-Adapters/node_modules/@coral-xyz/borsh": { + "version": "0.30.1", + "resolved": "https://registry.npmjs.org/@coral-xyz/borsh/-/borsh-0.30.1.tgz", + "integrity": "sha512-aaxswpPrCFKl8vZTbxLssA2RvwX2zmKLlRCIktJOwW+VpVwYtXRtlWiIP+c2pPRKneiTiWCN2GEMSH9j1zTlWQ==", + "license": "Apache-2.0", + "dependencies": { + "bn.js": "^5.1.2", + "buffer-layout": "^1.2.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@solana/web3.js": "^1.68.0" + } + }, "DefiLlama-Adapters/node_modules/dotenv": { "version": "8.6.0", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.6.0.tgz", @@ -3109,6 +3153,15 @@ "node": ">=11" } }, + "node_modules/@coral-xyz/anchor-errors": { + "version": "0.30.1", + "resolved": "https://registry.npmjs.org/@coral-xyz/anchor-errors/-/anchor-errors-0.30.1.tgz", + "integrity": "sha512-9Mkradf5yS5xiLWrl9WrpjqOrAV+/W2RQHDlbnAZBivoGpOs1ECjoDCkVk4aRG8ZdiFiB8zQEVlxf+8fKkmSfQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=10" + } + }, "node_modules/@coral-xyz/anchor/node_modules/@coral-xyz/borsh": { "version": "0.29.0", "resolved": "https://registry.npmjs.org/@coral-xyz/borsh/-/borsh-0.29.0.tgz", @@ -8922,6 +8975,7 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/buffer-layout/-/buffer-layout-1.2.2.tgz", "integrity": "sha512-kWSuLN694+KTk8SrYvCqwP2WcgQjoRCiF5b4QDvkkz8EmgD+aWAIceGFKMIAdmF/pH+vpgNV3d3kAKorcdAmWA==", + "license": "MIT", "engines": { "node": ">=4.5" } diff --git a/defi/package.json b/defi/package.json index b4f1dadf7c..fbfb5c1c1a 100644 --- a/defi/package.json +++ b/defi/package.json @@ -97,6 +97,7 @@ "@types/promise.allsettled": "^1.0.6", "axios": "^1.6.5", "bignumber.js": "^9.0.1", + "buffer-layout": "^1.2.2", "dotenv": "^16.0.1", "ethers": "^6.0.0", "hyper-express": "^6.8.7", From 243ad9131b1e39a2d1f3f6f250cdac25404d47a3 Mon Sep 17 00:00:00 2001 From: waynebruce0x Date: Thu, 31 Oct 2024 15:29:17 +0000 Subject: [PATCH 2/4] sol patch --- defi/l2/utils.ts | 84 +++++++++++++++++++----------------- defi/src/storeChainAssets.ts | 10 ++++- 2 files changed, 53 insertions(+), 41 deletions(-) diff --git a/defi/l2/utils.ts b/defi/l2/utils.ts index ebd7047d75..e3059fe999 100644 --- a/defi/l2/utils.ts +++ b/defi/l2/utils.ts @@ -13,10 +13,9 @@ import { storeNotTokens } from "./layer2pg"; import { getBlock } from "@defillama/sdk/build/util/blocks"; import { readFromPGCache, writeToPGCache } from "../src/api2/db"; import { unixTimestampNow } from "../emissions-adapters/utils/time"; -import { Connection, PublicKey } from "@solana/web3.js" +import { Connection, PublicKey } from "@solana/web3.js"; import * as sdk from "@defillama/sdk"; - -import { struct, u8, u64 } from 'buffer-layout'; +import { struct, u8, u64 } from "../DefiLlama-Adapters/projects/helper/utils/solana/layouts/layout-base.js"; type CachedSupplies = { timestamp: number; data: { [token: string]: number } }; export function aggregateChainTokenBalances(usdTokenBalances: AllProtocols): TokenTvlData { @@ -68,7 +67,8 @@ export async function getPrices( restCallWrapper( () => fetch( - `https://coins.llama.fi/prices?source=internal${process.env.COINS_KEY ? `?apikey=${process.env.COINS_KEY}` : "" + `https://coins.llama.fi/prices?source=internal${ + process.env.COINS_KEY ? `?apikey=${process.env.COINS_KEY}` : "" }`, { method: "POST", @@ -194,68 +194,74 @@ async function getAptosSupplies(tokens: string[], timestamp?: number): Promise<{ return supplies; } -let connection: any = {} +let connection: any = {}; const endpoint = (isClient = false) => { - if (isClient) return process.env.SOLANA_RPC_CLIENT ?? process.env.SOLANA_RPC - return process.env.SOLANA_RPC -} + if (isClient) return process.env.SOLANA_RPC_CLIENT ?? process.env.SOLANA_RPC; + return process.env.SOLANA_RPC; +}; const endpointMap: any = { solana: endpoint, -} +}; -function getConnection(chain = 'solana') { - if (!connection[chain]) connection[chain] = new Connection(endpointMap[chain](true)) - return connection[chain] +function getConnection(chain = "solana") { + let a = endpointMap[chain](true); + if (!connection[chain]) connection[chain] = new Connection(endpointMap[chain](true)); + return connection[chain]; } - - - async function getSolanaTokenSupply(tokens: string[], timestamp?: number): Promise<{ [token: string]: number }> { if (timestamp) throw new Error(`timestamp incompatible with Solana adapter!`); const solanaMintLayout = struct([ - u8('mintAuthorityOption'), - u8('mintAuthority'), - u64('supply'), - u8('decimals'), - u8('isInitialized'), - u8('freezeAuthorityOption'), - u8('freezeAuthority'), - ]) - - const sleepTime = tokens.length > 2000 ? 2000 : 200 - const tokensPK = tokens.map(i => typeof i === 'string' ? new PublicKey(i) : i) - const connection = getConnection('solana') - const res = await runInChunks(tokensPK, (chunk: any) => connection.getMultipleAccountsInfo(chunk), { sleepTime }) + u8("mintAuthorityOption"), + u8("mintAuthority"), + u64("supply"), + u8("decimals"), + u8("isInitialized"), + u8("freezeAuthorityOption"), + u8("freezeAuthority"), + ]); + + const sleepTime = tokens.length > 2000 ? 2000 : 200; + const tokensPK: PublicKey[] = []; + const filteredTokens: string[] = []; + tokens.map((i) => { + try { + const key = new PublicKey(i); + tokensPK.push(key); + filteredTokens.push(i); + } catch (e) {} + }); + const connection = getConnection("solana"); + const res = await runInChunks(tokensPK, (chunk: any) => connection.getMultipleAccountsInfo(chunk), { sleepTime }); const supplies: { [token: string]: number } = {}; res.forEach((data, idx) => { if (!data) { - sdk.log(`Invalid account: ${tokens[idx]}`) + sdk.log(`Invalid account: ${tokens[idx]}`); return; } try { - const buffer = Buffer.from(data.data) - const decoded = solanaMintLayout.decode(buffer) - supplies['solana:'+tokens[idx]] = decoded.supply.toString() + const buffer = Buffer.from(data.data); + const decoded = solanaMintLayout.decode(buffer); + supplies["solana:" + filteredTokens[idx]] = decoded.supply.toString(); // 10 ** decoded.decimals.toString(); } catch (e) { - sdk.log(`Error decoding account: ${tokens[idx]}`) + sdk.log(`Error decoding account: ${filteredTokens[idx]}`); } - }) + }); return supplies; async function runInChunks(inputs: any, fn: any, { chunkSize = 99, sleepTime }: any = {}) { - const chunks = sliceIntoChunks(inputs, chunkSize) - const results = [] + const chunks = sliceIntoChunks(inputs, chunkSize); + const results = []; for (const chunk of chunks) { - results.push(...(await fn(chunk) ?? [])) - if (sleepTime) await sleep(sleepTime) + results.push(...((await fn(chunk)) ?? [])); + if (sleepTime) await sleep(sleepTime); } - return results.flat() + return results.flat(); function sliceIntoChunks(arr: any, chunkSize = 100) { const res = []; diff --git a/defi/src/storeChainAssets.ts b/defi/src/storeChainAssets.ts index 7c965348f4..c195ed7ea1 100644 --- a/defi/src/storeChainAssets.ts +++ b/defi/src/storeChainAssets.ts @@ -4,12 +4,16 @@ import { withTimeout } from "./utils/shared/withTimeout"; import { storeR2JSONString } from "./utils/r2"; import { getCurrentUnixTimestamp } from "./utils/date"; import storeHistorical from "../l2/storeToDb"; +import setEnvSecrets from "./utils/shared/setEnvSecrets"; async function getChainAssets() { + await setEnvSecrets(); const res: any = await chainAssets(); res.timestamp = getCurrentUnixTimestamp(); - await storeR2JSONString("chainAssets", JSON.stringify(res)); - await storeHistorical(res); + let a = JSON.stringify(res); + let b = JSON.parse(a); + // await storeR2JSONString("chainAssets", JSON.stringify(res)); + // await storeHistorical(res); console.log("chain assets stored"); process.exit(); } @@ -23,3 +27,5 @@ export async function handler() { } handler(); // ts-node defi/src/storeChainAssets.ts +("1 231 905 171.69984650060294015202"); // con decimal => WRONG BECAUSE OF USDT +("471 787 548 264.79288990485587611621787169929275"); // sin decimal => TOO BIG From 3e8ce3b25a106ddf01fb8df69538d2fb9acec0f6 Mon Sep 17 00:00:00 2001 From: waynebruce0x Date: Wed, 6 Nov 2024 15:30:40 +0000 Subject: [PATCH 3/4] patch struct --- defi/l2/utils.ts | 18 ++++-------------- defi/src/storeChainAssets.ts | 10 ++++------ 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/defi/l2/utils.ts b/defi/l2/utils.ts index e3059fe999..335aedaa5f 100644 --- a/defi/l2/utils.ts +++ b/defi/l2/utils.ts @@ -206,22 +206,13 @@ const endpointMap: any = { }; function getConnection(chain = "solana") { - let a = endpointMap[chain](true); if (!connection[chain]) connection[chain] = new Connection(endpointMap[chain](true)); return connection[chain]; } async function getSolanaTokenSupply(tokens: string[], timestamp?: number): Promise<{ [token: string]: number }> { if (timestamp) throw new Error(`timestamp incompatible with Solana adapter!`); - const solanaMintLayout = struct([ - u8("mintAuthorityOption"), - u8("mintAuthority"), - u64("supply"), - u8("decimals"), - u8("isInitialized"), - u8("freezeAuthorityOption"), - u8("freezeAuthority"), - ]); + const solanaMintLayout = struct([u64("supply")]); const sleepTime = tokens.length > 2000 ? 2000 : 200; const tokensPK: PublicKey[] = []; @@ -243,9 +234,9 @@ async function getSolanaTokenSupply(tokens: string[], timestamp?: number): Promi return; } try { - const buffer = Buffer.from(data.data); - const decoded = solanaMintLayout.decode(buffer); - supplies["solana:" + filteredTokens[idx]] = decoded.supply.toString(); // 10 ** decoded.decimals.toString(); + const buffer = data.data.slice(36, 44); + const supply = solanaMintLayout.decode(buffer).supply.toString(); + supplies["solana:" + filteredTokens[idx]] = supply; } catch (e) { sdk.log(`Error decoding account: ${filteredTokens[idx]}`); } @@ -273,7 +264,6 @@ async function getSolanaTokenSupply(tokens: string[], timestamp?: number): Promi } } } - async function getSuiSupplies(tokens: Address[], timestamp?: number): Promise<{ [token: string]: number }> { if (timestamp) throw new Error(`timestamp incompatible with Sui adapter!`); const supplies: { [token: string]: number } = {}; diff --git a/defi/src/storeChainAssets.ts b/defi/src/storeChainAssets.ts index c195ed7ea1..ec27a3f5a9 100644 --- a/defi/src/storeChainAssets.ts +++ b/defi/src/storeChainAssets.ts @@ -10,10 +10,10 @@ async function getChainAssets() { await setEnvSecrets(); const res: any = await chainAssets(); res.timestamp = getCurrentUnixTimestamp(); - let a = JSON.stringify(res); - let b = JSON.parse(a); - // await storeR2JSONString("chainAssets", JSON.stringify(res)); - // await storeHistorical(res); + // let a = JSON.stringify(res); + // let b = JSON.parse(a); + await storeR2JSONString("chainAssets", JSON.stringify(res)); + await storeHistorical(res); console.log("chain assets stored"); process.exit(); } @@ -27,5 +27,3 @@ export async function handler() { } handler(); // ts-node defi/src/storeChainAssets.ts -("1 231 905 171.69984650060294015202"); // con decimal => WRONG BECAUSE OF USDT -("471 787 548 264.79288990485587611621787169929275"); // sin decimal => TOO BIG From ae96c35fefb9fcc961cd12b9c0201a6f1aec644f Mon Sep 17 00:00:00 2001 From: waynebruce0x Date: Wed, 6 Nov 2024 15:35:01 +0000 Subject: [PATCH 4/4] debug --- defi/src/storeChainAssets.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/defi/src/storeChainAssets.ts b/defi/src/storeChainAssets.ts index ec27a3f5a9..4639060c86 100644 --- a/defi/src/storeChainAssets.ts +++ b/defi/src/storeChainAssets.ts @@ -4,10 +4,10 @@ import { withTimeout } from "./utils/shared/withTimeout"; import { storeR2JSONString } from "./utils/r2"; import { getCurrentUnixTimestamp } from "./utils/date"; import storeHistorical from "../l2/storeToDb"; -import setEnvSecrets from "./utils/shared/setEnvSecrets"; +// import setEnvSecrets from "./utils/shared/setEnvSecrets"; async function getChainAssets() { - await setEnvSecrets(); + // await setEnvSecrets(); const res: any = await chainAssets(); res.timestamp = getCurrentUnixTimestamp(); // let a = JSON.stringify(res);