import axios from "axios"; import { dailyPeggedBalances, getLastRecord, hourlyPeggedBalances, hourlyPeggedPrices } from "../../src/peggedAssets/utils/getLastRecord"; import peggedAssets from "../../src/peggedData/peggedData"; import { getClosestDayStartTimestamp, secondsInDay, secondsInHour, } from "../../src/utils/date"; import * as sdk from "@defillama/sdk"; import { normalizeChain } from "../../src/utils/normalizeChain"; import { getHistoricalValues } from "../../src/utils/shared/dynamodb"; import { buildFxRateMap, lookupFxRate, pegTypeFxTicker } from "../../src/utils/fxRates"; import { cache } from "../cache"; import { storeRouteData } from "../file-cache"; import { chainCacheSlug } from "../utils/cachePath"; type TokenBalance = { [token: string]: number | undefined; }; export default async function handler() { const timeKey = "chart-update-init"; console.time(timeKey); const historicalRates = (await axios.get(`https://llama-stablecoins-data.s3.eu-central-1.amazonaws.com/rates/full`))?.data; const lastPrices = await getLastRecord(hourlyPeggedPrices); const priceTimestamps = cache.historicalPrices?.map((item: any) => item.SK); await getPeggedAssetsData() cache.fxRateMap = buildFxRateMap(historicalRates) cache.lastPrices = lastPrices cache.priceTimestamps = priceTimestamps console.timeEnd(timeKey) } export async function storeChartsPart2(assetChainMap: any) { const { lastPrices, fxRateMap, priceTimestamps, } = cache const commonOptions = { assetChainMap, lastPrices, historicalPrices: cache.historicalPrices, fxRateMap, priceTimestamps, peggedAssetsData: cache.peggedAssetsData, } // store overall chart const allData = await craftChartsResponse({ ...commonOptions, chain: "all" }); await storeRouteData('charts/all/all', allData) // store chain charts const chainKeys = new Set( Object.values(sdk.chainUtils.chainLabelsToKeyMap as Record), ); for (const normalizedChain of chainKeys) { const chainData = await craftChartsResponse({ ...commonOptions, chain: normalizedChain, }); await storeRouteData(`charts/${chainCacheSlug(normalizedChain)}`, chainData) } // store pegged asset charts for (const pegged of peggedAssets) { const id = pegged.id; const chart = await craftChartsResponse({ ...commonOptions, peggedID: id }); await storeRouteData(`charts/all/${id}`, chart) } } async function getPeggedAssetsData() { if (!cache.peggedAssetsData) cache.peggedAssetsData = {} await Promise.all(peggedAssets.map(async (pegged) => { const lastBalance = await getLastRecord(hourlyPeggedBalances(pegged.id)); const allBalances = cache.peggedAssetsData[pegged.id]?.balances || [] const highestSK = allBalances.reduce((highest: any, item: any) => Math.max(highest, item.SK), -1) const pullData = !lastBalance || (lastBalance.SK - highestSK > secondsInDay) // if last item on the table is more than 1 day older than the last balance, pull data if (pullData) { console.info('fetching new data for', pegged.id) const newItems = await getHistoricalValues(dailyPeggedBalances(pegged.id), highestSK) allBalances.push(...newItems) } cache.peggedAssetsData[pegged.id] = { balances: allBalances, lastBalance } })) // remove all Terra data after the depeg timestamp cache.peggedAssetsData['3'].balances = cache.peggedAssetsData['3'].balances .filter((item: any) => item.SK <= 1655891865) replaceAvalanceAvax(cache.peggedAssetsData) // convert all 'avalanche' keys to 'avax' return cache.peggedAssetsData } function replaceAvalanceAvax(obj: any) { if (typeof obj !== 'object' || obj === null) { return; // Not an object or is null, do nothing } if (obj.hasOwnProperty('avalanche')) { obj['avax'] = obj['avalanche']; // Create 'avax' key with 'avalanche' value delete obj['avalanche']; // Remove 'avalanche' key } // Recursively apply to all object values Object.values(obj).forEach(value => { if (typeof value === 'object') { replaceAvalanceAvax(value); // Recursive call } }); } const formatTokenBalance = (tokenBalance: TokenBalance) => { let formattedTokenBalance = {} as TokenBalance; for (const token in tokenBalance) { const balance = tokenBalance[token]; formattedTokenBalance[token] = balance ? parseFloat(balance.toFixed(2)) : 0; } return formattedTokenBalance; }; // this should not get prices from the previous day function pricesCompareFn(a: number, b: number) { if (Math.abs(a - b) < secondsInDay) return 0; return a - b; } // returns index of element or -n-1 where n is index of closest element // uses compare_fn to consider timestamps the same if they are within 1 day range function timestampsBinarySearch( ar: number[], el: number, compare_fn: (a: number, b: number) => number ) { var m = 0; var n = ar.length - 1; while (m <= n) { var k = (n + m) >> 1; var cmp = compare_fn(el, ar[k]); if (cmp > 0) { m = k + 1; } else if (cmp < 0) { n = k - 1; } else { return k; } } return -m - 1; } function extractResultOfBinarySearch(ar: any[], binarySearchResult: number) { if (binarySearchResult < 0) { if (binarySearchResult === -1 || ar.length < -binarySearchResult) { return null; } return ar[-binarySearchResult - 1]; } return ar[binarySearchResult]; } const _assetCache: any = {}; let lastDailyTimestamp = 0; export function craftChartsResponse( { chain = 'all', peggedID, startTimestamp, assetChainMap, excludeDoublecounted, }: { chain?: string, peggedID?: string, startTimestamp?: string | number, assetChainMap: { [asset: string]: Set }, excludeDoublecounted?: boolean, } ) { if (startTimestamp && typeof startTimestamp === 'string') startTimestamp = parseInt(startTimestamp) const filterChart = (chart: any) => { if (startTimestamp) chart = chart.filter((entry: any) => entry?.date >= startTimestamp) return chart.filter((entry: any) => entry) } const { historicalPrices, fxRateMap, lastPrices, priceTimestamps, peggedAssetsData, } = cache as any const sumDailyBalances = {} as { [timestamp: number]: { circulating: TokenBalance; unreleased: TokenBalance; totalCirculatingUSD: TokenBalance; totalMintedUSD: TokenBalance; totalBridgedToUSD: TokenBalance; }; }; const normalizedChain = normalizeChain(chain!); /* * whenever "chain" and "peggedAsset", and peggedAsset has no entry in lastBalance for that chain, * historicalPeggedBalances is empty. Not sure exactly where that's happening. */ const historicalPeggedBalances = peggedAssets.map((pegged) => { if (peggedID && pegged.id !== peggedID) return; if (excludeDoublecounted && pegged.doublecounted) return; const chainMap = assetChainMap[pegged.id]; if (!chainMap || chain !== "all" && !chainMap?.has(chain)) return; // if the coin is not found an given chain or coin has no data, dont process it if (!_assetCache[pegged.id]) addToAssetCache(pegged); return _assetCache[pegged.id]; }).filter((i) => i); function addToAssetCache(pegged: any) { const { balance: lastBalance, balances } = peggedAssetsData[pegged.id] let historicalBalance = { Items: balances } as any; if (!historicalBalance.Items || historicalBalance.Items.length < 1) return undefined; const lastDailyItem = historicalBalance.Items[historicalBalance.Items.length - 1]; if (lastBalance && lastBalance.SK > lastDailyItem.SK && lastDailyItem.SK + secondsInHour * 25 > lastBalance.SK) { lastBalance.SK = lastDailyItem.SK; historicalBalance.Items[historicalBalance.Items.length - 1] = lastBalance; } const lastTimestamp = getClosestDayStartTimestamp(historicalBalance.Items[historicalBalance.Items.length - 1].SK); lastDailyTimestamp = Math.max(lastDailyTimestamp, lastTimestamp); _assetCache[pegged.id] = { pegged, historicalBalance: [...historicalBalance.Items], lastTimestamp, }; } const lastDailyItem = historicalPrices[historicalPrices.length - 1]; if (lastPrices && lastPrices.SK > lastDailyItem.SK && lastDailyItem.SK + secondsInHour * 25 > lastPrices.SK) { lastPrices.SK = lastDailyItem.SK; historicalPrices[historicalPrices.length - 1] = lastPrices; } historicalPeggedBalances.forEach((peggedBalance) => { let { historicalBalance: _historicalBalance, pegged, lastTimestamp } = peggedBalance; const historicalBalance = [..._historicalBalance]; const pegType = pegged.pegType; const peggedGeckoID = pegged.gecko_id; const lastBalance = historicalBalance[historicalBalance.length - 1]; // fill missing data with last available data while (lastTimestamp < lastDailyTimestamp) { if (pegged.deadFrom || pegged.delisted) break; // don't extend inactive assets beyond their last real data point lastTimestamp = getClosestDayStartTimestamp(lastTimestamp + 24 * secondsInHour); const syntheticBalance = { ...lastBalance, SK: lastTimestamp }; historicalBalance.push(syntheticBalance); peggedBalance.historicalBalance.push(syntheticBalance); peggedBalance.lastTimestamp = lastTimestamp; } historicalBalance.map((item: any) => { const timestamp = getClosestDayStartTimestamp(item.SK) let itemBalance: any = {}; const closestPriceIndex = timestampsBinarySearch(priceTimestamps, timestamp, pricesCompareFn); const closestPrices = extractResultOfBinarySearch(historicalPrices, closestPriceIndex); let fallbackPrice = 1; const historicalPrice = closestPrices?.prices[peggedGeckoID]; if (pegType === "peggedVAR") { fallbackPrice = 0; } else if (pegType !== "peggedUSD" && !historicalPrice) { const ticker = pegTypeFxTicker(pegType); const rate = fxRateMap ? lookupFxRate(fxRateMap, ticker, timestamp) : null; fallbackPrice = rate ? 1 / rate : 0; if (!isFinite(fallbackPrice)) fallbackPrice = 0; } const price = historicalPrice ? historicalPrice : fallbackPrice; const isAllChain = (chain === "all"); if (isAllChain) { if (!item.totalCirculating.circulating) { throw new Error( `missing totalCirculating for ${peggedGeckoID} at timestamp ${timestamp}` ); } const itemPegType = Object.keys(item.totalCirculating.circulating)?.[0]; if (item.totalCirculating.circulating && itemPegType !== pegType) return; itemBalance.circulating = item.totalCirculating.circulating ?? { [pegType]: 0 }; if (item.totalCirculating.unreleased) { itemBalance.unreleased = item.totalCirculating.unreleased; } itemBalance.bridgedTo = { [pegType]: 0 }; itemBalance.minted = { [pegType]: 0 }; } else { let chainData: any = null; for (const [itemChain, data] of Object.entries(item)) { if (itemChain === 'SK' || itemChain === 'totalCirculating') continue; const itemChainNormalized = normalizeChain(itemChain); if (itemChainNormalized === normalizedChain) { chainData = data; break; } } if (chainData?.circulating) { const itemPegType = Object.keys(chainData.circulating)?.[0]; if (chainData.circulating && itemPegType !== pegType) return; } itemBalance.circulating = chainData?.circulating ?? { [pegType]: 0 }; itemBalance.unreleased = chainData?.unreleased ?? { [pegType]: 0 }; itemBalance.bridgedTo = chainData?.bridgedTo ?? { [pegType]: 0 }; itemBalance.minted = chainData?.minted ?? { [pegType]: 0 }; if (itemBalance.circulating === undefined) { return; } } // need stricter checks here if (itemBalance !== null) { sumDailyBalances[timestamp] = sumDailyBalances[timestamp] || {}; sumDailyBalances[timestamp].circulating = sumDailyBalances[timestamp].circulating || {}; sumDailyBalances[timestamp].circulating[pegType] = (sumDailyBalances[timestamp].circulating[pegType] ?? 0) + itemBalance.circulating[pegType]; sumDailyBalances[timestamp].unreleased = sumDailyBalances[timestamp].unreleased || {}; sumDailyBalances[timestamp].unreleased[pegType] = (sumDailyBalances[timestamp].unreleased[pegType] ?? 0) + itemBalance.unreleased[pegType]; sumDailyBalances[timestamp].totalCirculatingUSD = sumDailyBalances[timestamp].totalCirculatingUSD || {}; sumDailyBalances[timestamp].totalCirculatingUSD[pegType] = (sumDailyBalances[timestamp].totalCirculatingUSD[pegType] ?? 0) + itemBalance.circulating[pegType] * price; sumDailyBalances[timestamp].totalMintedUSD = sumDailyBalances[timestamp].totalMintedUSD || {}; sumDailyBalances[timestamp].totalMintedUSD[pegType] = (sumDailyBalances[timestamp].totalMintedUSD[pegType] ?? 0) + (itemBalance.minted[pegType] - itemBalance.unreleased[pegType]) * price; sumDailyBalances[timestamp].totalBridgedToUSD = sumDailyBalances[timestamp].totalBridgedToUSD || {}; sumDailyBalances[timestamp].totalBridgedToUSD[pegType] = (sumDailyBalances[timestamp].totalBridgedToUSD[pegType] ?? 0) + itemBalance.bridgedTo[pegType] * price; if (chain === "all") { sumDailyBalances[timestamp].totalMintedUSD[pegType] = 0; sumDailyBalances[timestamp].totalBridgedToUSD[pegType] = 0; } } else { console.log( "itemBalance is invalid", itemBalance, item, pegged, lastTimestamp, historicalBalance ); } }) }) const response = Object.entries(sumDailyBalances).map( ([timestamp, balance]) => ({ date: timestamp, totalCirculating: formatTokenBalance(balance.circulating), totalUnreleased: formatTokenBalance(balance.unreleased), totalCirculatingUSD: formatTokenBalance(balance.totalCirculatingUSD), totalMintedUSD: formatTokenBalance(balance.totalMintedUSD), totalBridgedToUSD: formatTokenBalance(balance.totalBridgedToUSD), }) ); return filterChart(response) }