diff --git a/backend/jest.config.ts b/backend/jest.config.ts index 5576bfe80..14f932f98 100644 --- a/backend/jest.config.ts +++ b/backend/jest.config.ts @@ -7,11 +7,14 @@ const config: Config.InitialOptions = { automock: false, collectCoverage: true, collectCoverageFrom: ["./src/**/**.ts"], - coverageProvider: "v8", + coverageProvider: "babel", coverageThreshold: { global: { lines: 1 } - } + }, + setupFiles: [ + "./testSetup.ts", + ], } export default config; diff --git a/backend/src/__tests__/api/difficulty-adjustment.test.ts b/backend/src/__tests__/api/difficulty-adjustment.test.ts new file mode 100644 index 000000000..eb774d445 --- /dev/null +++ b/backend/src/__tests__/api/difficulty-adjustment.test.ts @@ -0,0 +1,62 @@ +import { calcDifficultyAdjustment, DifficultyAdjustment } from '../../api/difficulty-adjustment'; + +describe('Mempool Difficulty Adjustment', () => { + test('should calculate Difficulty Adjustments properly', () => { + const dt = (dtString) => { + return Math.floor(new Date(dtString).getTime() / 1000); + }; + + const vectors = [ + [ // Vector 1 + [ // Inputs + dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds) + dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds) + 750134, // Current block height + 0.6280047707459726, // Previous retarget % (Passed through) + 'mainnet', // Network (if testnet, next value is non-zero) + 0, // If not testnet, not used + ], + { // Expected Result + progressPercent: 9.027777777777777, + difficultyChange: 12.562233927411782, + estimatedRetargetDate: 1661895424692, + remainingBlocks: 1834, + remainingTime: 977591692, + previousRetarget: 0.6280047707459726, + nextRetargetHeight: 751968, + timeAvg: 533038, + timeOffset: 0, + }, + ], + [ // Vector 2 (testnet) + [ // Inputs + dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds) + dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds) + 750134, // Current block height + 0.6280047707459726, // Previous retarget % (Passed through) + 'testnet', // Network + dt('2022-08-19T13:52:46.000Z'), // Latest block timestamp in seconds + ], + { // Expected Result is same other than timeOffset + progressPercent: 9.027777777777777, + difficultyChange: 12.562233927411782, + estimatedRetargetDate: 1661895424692, + remainingBlocks: 1834, + remainingTime: 977591692, + previousRetarget: 0.6280047707459726, + nextRetargetHeight: 751968, + timeAvg: 533038, + timeOffset: -667000, // 11 min 7 seconds since last block (testnet only) + // If we add time avg to abs(timeOffset) it makes exactly 1200000 ms, or 20 minutes + }, + ], + ] as [[number, number, number, number, string, number], DifficultyAdjustment][]; + + for (const vector of vectors) { + const result = calcDifficultyAdjustment(...vector[0]); + // previousRetarget is passed through untouched + expect(result.previousRetarget).toStrictEqual(vector[0][3]); + expect(result).toStrictEqual(vector[1]); + } + }); +}); diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 9453505f4..7314fde6f 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -136,5 +136,4 @@ describe('Mempool Backend Config', () => { expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER); }); }); - }); diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index 66bcb2569..2c3fd9467 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -510,7 +510,12 @@ class BitcoinRoutes { private getDifficultyChange(req: Request, res: Response) { try { - res.json(difficultyAdjustment.getDifficultyAdjustment()); + const da = difficultyAdjustment.getDifficultyAdjustment(); + if (da) { + res.json(da); + } else { + res.status(503).send(`Service Temporarily Unavailable`); + } } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); } diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 8635ee96f..b9cc1453c 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -228,34 +228,75 @@ export class Common { return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0]; } - static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket { + static findSocketNetwork(addr: string): {network: string | null, url: string} { let network: string | null = null; + let url = addr.split('://')[1]; - if (config.LIGHTNING.BACKEND === 'cln') { - network = socket.network; - } else if (config.LIGHTNING.BACKEND === 'lnd') { - if (socket.addr.indexOf('onion') !== -1) { - if (socket.addr.split('.')[0].length >= 56) { - network = 'torv3'; - } else { - network = 'torv2'; - } - } else if (socket.addr.indexOf('i2p') !== -1) { - network = 'i2p'; + if (!url) { + return { + network: null, + url: addr, + }; + } + + if (addr.indexOf('onion') !== -1) { + if (url.split('.')[0].length >= 56) { + network = 'torv3'; } else { - const ipv = isIP(socket.addr.split(':')[0]); - if (ipv === 4) { - network = 'ipv4'; - } else if (ipv === 6) { - network = 'ipv6'; - } + network = 'torv2'; } + } else if (addr.indexOf('i2p') !== -1) { + network = 'i2p'; + } else if (addr.indexOf('ipv4') !== -1) { + const ipv = isIP(url.split(':')[0]); + if (ipv === 4) { + network = 'ipv4'; + } else { + return { + network: null, + url: addr, + }; + } + } else if (addr.indexOf('ipv6') !== -1) { + url = url.split('[')[1].split(']')[0]; + const ipv = isIP(url); + if (ipv === 6) { + const parts = addr.split(':'); + network = 'ipv6'; + url = `[${url}]:${parts[parts.length - 1]}`; + } else { + return { + network: null, + url: addr, + }; + } + } else { + return { + network: null, + url: addr, + }; } return { - publicKey: publicKey, network: network, - addr: socket.addr, + url: url, }; } + + static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket { + if (config.LIGHTNING.BACKEND === 'cln') { + return { + publicKey: publicKey, + network: socket.network, + addr: socket.addr, + }; + } else /* if (config.LIGHTNING.BACKEND === 'lnd') */ { + const formatted = this.findSocketNetwork(socket.addr); + return { + publicKey: publicKey, + network: formatted.network, + addr: formatted.url, + }; + } + } } diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 1d6f10a2c..1dc0b9704 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 39; + private static currentVersion = 40; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -342,6 +342,12 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `nodes` ADD alias_search TEXT NULL DEFAULT NULL AFTER `alias`'); await this.$executeQuery('ALTER TABLE nodes ADD FULLTEXT(alias_search)'); } + + if (databaseSchemaVersion < 40 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `nodes` ADD capacity bigint(20) unsigned DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `nodes` ADD channels int(11) unsigned DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `nodes` ADD INDEX `capacity` (`capacity`);'); + } } /** diff --git a/backend/src/api/difficulty-adjustment.ts b/backend/src/api/difficulty-adjustment.ts index 1f85fdb80..a1b6ab70e 100644 --- a/backend/src/api/difficulty-adjustment.ts +++ b/backend/src/api/difficulty-adjustment.ts @@ -2,65 +2,100 @@ import config from '../config'; import { IDifficultyAdjustment } from '../mempool.interfaces'; import blocks from './blocks'; -class DifficultyAdjustmentApi { - constructor() { } +export interface DifficultyAdjustment { + progressPercent: number; // Percent: 0 to 100 + difficultyChange: number; // Percent: -75 to 300 + estimatedRetargetDate: number; // Unix time in ms + remainingBlocks: number; // Block count + remainingTime: number; // Duration of time in ms + previousRetarget: number; // Percent: -75 to 300 + nextRetargetHeight: number; // Block Height + timeAvg: number; // Duration of time in ms + timeOffset: number; // (Testnet) Time since last block (cap @ 20min) in ms +} - public getDifficultyAdjustment(): IDifficultyAdjustment { +export function calcDifficultyAdjustment( + DATime: number, + nowSeconds: number, + blockHeight: number, + previousRetarget: number, + network: string, + latestBlockTimestamp: number, +): DifficultyAdjustment { + const ESTIMATE_LAG_BLOCKS = 146; // For first 7.2% of epoch, don't estimate. + const EPOCH_BLOCK_LENGTH = 2016; // Bitcoin mainnet + const BLOCK_SECONDS_TARGET = 600; // Bitcoin mainnet + const TESTNET_MAX_BLOCK_SECONDS = 1200; // Bitcoin testnet + + const diffSeconds = nowSeconds - DATime; + const blocksInEpoch = (blockHeight >= 0) ? blockHeight % EPOCH_BLOCK_LENGTH : 0; + const progressPercent = (blockHeight >= 0) ? blocksInEpoch / EPOCH_BLOCK_LENGTH * 100 : 100; + const remainingBlocks = EPOCH_BLOCK_LENGTH - blocksInEpoch; + const nextRetargetHeight = (blockHeight >= 0) ? blockHeight + remainingBlocks : 0; + + let difficultyChange = 0; + let timeAvgSecs = BLOCK_SECONDS_TARGET; + // Only calculate the estimate once we have 7.2% of blocks in current epoch + if (blocksInEpoch >= ESTIMATE_LAG_BLOCKS) { + timeAvgSecs = diffSeconds / blocksInEpoch; + difficultyChange = (BLOCK_SECONDS_TARGET / timeAvgSecs - 1) * 100; + // Max increase is x4 (+300%) + if (difficultyChange > 300) { + difficultyChange = 300; + } + // Max decrease is /4 (-75%) + if (difficultyChange < -75) { + difficultyChange = -75; + } + } + + // Testnet difficulty is set to 1 after 20 minutes of no blocks, + // therefore the time between blocks will always be below 20 minutes (1200s). + let timeOffset = 0; + if (network === 'testnet') { + if (timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) { + timeAvgSecs = TESTNET_MAX_BLOCK_SECONDS; + } + + const secondsSinceLastBlock = nowSeconds - latestBlockTimestamp; + if (secondsSinceLastBlock + timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) { + timeOffset = -Math.min(secondsSinceLastBlock, TESTNET_MAX_BLOCK_SECONDS) * 1000; + } + } + + const timeAvg = Math.floor(timeAvgSecs * 1000); + const remainingTime = remainingBlocks * timeAvg; + const estimatedRetargetDate = remainingTime + nowSeconds * 1000; + + return { + progressPercent, + difficultyChange, + estimatedRetargetDate, + remainingBlocks, + remainingTime, + previousRetarget, + nextRetargetHeight, + timeAvg, + timeOffset, + }; +} + +class DifficultyAdjustmentApi { + public getDifficultyAdjustment(): IDifficultyAdjustment | null { const DATime = blocks.getLastDifficultyAdjustmentTime(); const previousRetarget = blocks.getPreviousDifficultyRetarget(); const blockHeight = blocks.getCurrentBlockHeight(); const blocksCache = blocks.getBlocks(); const latestBlock = blocksCache[blocksCache.length - 1]; - - const now = new Date().getTime() / 1000; - const diff = now - DATime; - const blocksInEpoch = blockHeight % 2016; - const progressPercent = (blocksInEpoch >= 0) ? blocksInEpoch / 2016 * 100 : 100; - const remainingBlocks = 2016 - blocksInEpoch; - const nextRetargetHeight = blockHeight + remainingBlocks; - - let difficultyChange = 0; - if (remainingBlocks < 1870) { - if (blocksInEpoch > 0) { - difficultyChange = (600 / (diff / blocksInEpoch) - 1) * 100; - } - if (difficultyChange > 300) { - difficultyChange = 300; - } - if (difficultyChange < -75) { - difficultyChange = -75; - } + if (!latestBlock) { + return null; } + const nowSeconds = Math.floor(new Date().getTime() / 1000); - let timeAvgMins = blocksInEpoch && blocksInEpoch > 146 ? diff / blocksInEpoch / 60 : 10; - - // Testnet difficulty is set to 1 after 20 minutes of no blocks, - // therefore the time between blocks will always be below 20 minutes (1200s). - let timeOffset = 0; - if (config.MEMPOOL.NETWORK === 'testnet') { - if (timeAvgMins > 20) { - timeAvgMins = 20; - } - if (now - latestBlock.timestamp + timeAvgMins * 60 > 1200) { - timeOffset = -Math.min(now - latestBlock.timestamp, 1200) * 1000; - } - } - - const timeAvg = timeAvgMins * 60 * 1000 ; - const remainingTime = (remainingBlocks * timeAvg) + (now * 1000); - const estimatedRetargetDate = remainingTime + now; - - return { - progressPercent, - difficultyChange, - estimatedRetargetDate, - remainingBlocks, - remainingTime, - previousRetarget, - nextRetargetHeight, - timeAvg, - timeOffset, - }; + return calcDifficultyAdjustment( + DATime, nowSeconds, blockHeight, previousRetarget, + config.MEMPOOL.NETWORK, latestBlock.timestamp + ); } } diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index a2db61f78..b396e4808 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -80,7 +80,7 @@ class ChannelsApi { public async $searchChannelsById(search: string): Promise { try { const searchStripped = search.replace('%', '') + '%'; - const query = `SELECT id, short_id, capacity FROM channels WHERE id LIKE ? OR short_id LIKE ? LIMIT 10`; + const query = `SELECT id, short_id, capacity, status FROM channels WHERE id LIKE ? OR short_id LIKE ? LIMIT 10`; const [rows]: any = await DB.query(query, [searchStripped, searchStripped]); return rows; } catch (e) { @@ -229,9 +229,14 @@ class ChannelsApi { public async $getChannelsByTransactionId(transactionIds: string[]): Promise { try { - transactionIds = transactionIds.map((id) => '\'' + id + '\''); - const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.* FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key WHERE channels.transaction_id IN (${transactionIds.join(', ')}) OR channels.closing_transaction_id IN (${transactionIds.join(', ')})`; - const [rows]: any = await DB.query(query); + const query = ` + SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.* + FROM channels + LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key + LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key + WHERE channels.transaction_id IN ? OR channels.closing_transaction_id IN ? + `; + const [rows]: any = await DB.query(query, [[transactionIds], [transactionIds]]); const channels = rows.map((row) => this.convertChannel(row)); return channels; } catch (e) { @@ -245,15 +250,20 @@ class ChannelsApi { let channelStatusFilter; if (status === 'open') { channelStatusFilter = '< 2'; + } else if (status === 'active') { + channelStatusFilter = '= 1'; } else if (status === 'closed') { channelStatusFilter = '= 2'; + } else { + throw new Error('getChannelsForNode: Invalid status requested'); } // Channels originating from node let query = ` SELECT COALESCE(node2.alias, SUBSTRING(node2_public_key, 0, 20)) AS alias, COALESCE(node2.public_key, node2_public_key) AS public_key, channels.status, channels.node1_fee_rate, - channels.capacity, channels.short_id, channels.id, channels.closing_reason + channels.capacity, channels.short_id, channels.id, channels.closing_reason, + UNIX_TIMESTAMP(closing_date) as closing_date, UNIX_TIMESTAMP(channels.updated_at) as updated_at FROM channels LEFT JOIN nodes AS node2 ON node2.public_key = channels.node2_public_key WHERE node1_public_key = ? AND channels.status ${channelStatusFilter} @@ -264,7 +274,8 @@ class ChannelsApi { query = ` SELECT COALESCE(node1.alias, SUBSTRING(node1_public_key, 0, 20)) AS alias, COALESCE(node1.public_key, node1_public_key) AS public_key, channels.status, channels.node2_fee_rate, - channels.capacity, channels.short_id, channels.id, channels.closing_reason + channels.capacity, channels.short_id, channels.id, channels.closing_reason, + UNIX_TIMESTAMP(closing_date) as closing_date, UNIX_TIMESTAMP(channels.updated_at) as updated_at FROM channels LEFT JOIN nodes AS node1 ON node1.public_key = channels.node1_public_key WHERE node2_public_key = ? AND channels.status ${channelStatusFilter} @@ -273,27 +284,56 @@ class ChannelsApi { let allChannels = channelsFromNode.concat(channelsToNode); allChannels.sort((a, b) => { - return b.capacity - a.capacity; + if (status === 'closed') { + if (!b.closing_date && !a.closing_date) { + return (b.updated_at ?? 0) - (a.updated_at ?? 0); + } else { + return (b.closing_date ?? 0) - (a.closing_date ?? 0); + } + } else { + return b.capacity - a.capacity; + } }); - allChannels = allChannels.slice(index, index + length); + + if (index >= 0) { + allChannels = allChannels.slice(index, index + length); + } else if (index === -1) { // Node channels tree chart + allChannels = allChannels.slice(0, 1000); + } const channels: any[] = [] for (const row of allChannels) { - const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key); - channels.push({ - status: row.status, - closing_reason: row.closing_reason, - capacity: row.capacity ?? 0, - short_id: row.short_id, - id: row.id, - fee_rate: row.node1_fee_rate ?? row.node2_fee_rate ?? 0, - node: { - alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20), - public_key: row.public_key, - channels: activeChannelsStats.active_channel_count ?? 0, - capacity: activeChannelsStats.capacity ?? 0, - } - }); + let channel; + if (index >= 0) { + const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key); + channel = { + status: row.status, + closing_reason: row.closing_reason, + closing_date: row.closing_date, + capacity: row.capacity ?? 0, + short_id: row.short_id, + id: row.id, + fee_rate: row.node1_fee_rate ?? row.node2_fee_rate ?? 0, + node: { + alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20), + public_key: row.public_key, + channels: activeChannelsStats.active_channel_count ?? 0, + capacity: activeChannelsStats.capacity ?? 0, + } + }; + } else if (index === -1) { + channel = { + capacity: row.capacity ?? 0, + short_id: row.short_id, + id: row.id, + node: { + alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20), + public_key: row.public_key, + } + }; + } + + channels.push(channel); } return channels; @@ -498,6 +538,23 @@ class ChannelsApi { logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e)); } } + + public async $getLatestChannelUpdateForNode(publicKey: string): Promise { + try { + const query = ` + SELECT MAX(UNIX_TIMESTAMP(updated_at)) as updated_at + FROM channels + WHERE node1_public_key = ? + `; + const [rows]: any[] = await DB.query(query, [publicKey]); + if (rows.length > 0) { + return rows[0].updated_at; + } + } catch (e) { + logger.err(`Can't getLatestChannelUpdateForNode for ${publicKey}. Reason ${e instanceof Error ? e.message : e}`); + } + return 0; + } } export default new ChannelsApi(); diff --git a/backend/src/api/explorer/channels.routes.ts b/backend/src/api/explorer/channels.routes.ts index 0fa91db92..2b7f3fa6d 100644 --- a/backend/src/api/explorer/channels.routes.ts +++ b/backend/src/api/explorer/channels.routes.ts @@ -47,8 +47,17 @@ class ChannelsRoutes { res.status(400).send('Missing parameter: public_key'); return; } + const index = parseInt(typeof req.query.index === 'string' ? req.query.index : '0', 10) || 0; const status: string = typeof req.query.status === 'string' ? req.query.status : ''; + + if (index < -1) { + res.status(400).send('Invalid index'); + } + if (['open', 'active', 'closed'].includes(status) === false) { + res.status(400).send('Invalid status'); + } + const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, 10, status); const channelsCount = await channelsApi.$getChannelsCountForNode(req.query.public_key, status); res.header('Pragma', 'public'); @@ -61,7 +70,7 @@ class ChannelsRoutes { } } - private async $getChannelsByTransactionIds(req: Request, res: Response) { + private async $getChannelsByTransactionIds(req: Request, res: Response): Promise { try { if (!Array.isArray(req.query.txId)) { res.status(400).send('Not an array'); @@ -74,27 +83,26 @@ class ChannelsRoutes { } } const channels = await channelsApi.$getChannelsByTransactionId(txIds); - const inputs: any[] = []; - const outputs: any[] = []; + const result: any[] = []; for (const txid of txIds) { - const foundChannelInputs = channels.find((channel) => channel.closing_transaction_id === txid); - if (foundChannelInputs) { - inputs.push(foundChannelInputs); - } else { - inputs.push(null); + const inputs: any = {}; + const outputs: any = {}; + // Assuming that we only have one lightning close input in each transaction. This may not be true in the future + const foundChannelsFromInput = channels.find((channel) => channel.closing_transaction_id === txid); + if (foundChannelsFromInput) { + inputs[0] = foundChannelsFromInput; } - const foundChannelOutputs = channels.find((channel) => channel.transaction_id === txid); - if (foundChannelOutputs) { - outputs.push(foundChannelOutputs); - } else { - outputs.push(null); + const foundChannelsFromOutputs = channels.filter((channel) => channel.transaction_id === txid); + for (const output of foundChannelsFromOutputs) { + outputs[output.transaction_vout] = output; } + result.push({ + inputs, + outputs, + }); } - res.json({ - inputs: inputs, - outputs: outputs, - }); + res.json(result); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); } diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 379df4213..128405ffd 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -115,17 +115,13 @@ class NodesApi { public async $getTopCapacityNodes(full: boolean): Promise { try { - let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats'); - const latestDate = rows[0].maxAdded; - + let rows: any; let query: string; if (full === false) { query = ` SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, - node_stats.capacity - FROM node_stats - JOIN nodes ON nodes.public_key = node_stats.public_key - WHERE added = FROM_UNIXTIME(${latestDate}) + nodes.capacity + FROM nodes ORDER BY capacity DESC LIMIT 100 `; @@ -133,16 +129,14 @@ class NodesApi { [rows] = await DB.query(query); } else { query = ` - SELECT node_stats.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(node_stats.public_key, 1, 20), alias) as alias, - CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, - CAST(COALESCE(node_stats.channels, 0) as INT) as channels, + SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, + CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, + CAST(COALESCE(nodes.channels, 0) as INT) as channels, UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt, geo_names_city.names as city, geo_names_country.names as country - FROM node_stats - RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key + FROM nodes LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' - WHERE added = FROM_UNIXTIME(${latestDate}) ORDER BY capacity DESC LIMIT 100 `; @@ -163,17 +157,13 @@ class NodesApi { public async $getTopChannelsNodes(full: boolean): Promise { try { - let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats'); - const latestDate = rows[0].maxAdded; - + let rows: any; let query: string; if (full === false) { query = ` SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, - node_stats.channels - FROM node_stats - JOIN nodes ON nodes.public_key = node_stats.public_key - WHERE added = FROM_UNIXTIME(${latestDate}) + nodes.channels + FROM nodes ORDER BY channels DESC LIMIT 100; `; @@ -181,16 +171,14 @@ class NodesApi { [rows] = await DB.query(query); } else { query = ` - SELECT node_stats.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(node_stats.public_key, 1, 20), alias) as alias, - CAST(COALESCE(node_stats.channels, 0) as INT) as channels, - CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, + SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, + CAST(COALESCE(nodes.channels, 0) as INT) as channels, + CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt, geo_names_city.names as city, geo_names_country.names as country - FROM node_stats - RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key + FROM nodes LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' - WHERE added = FROM_UNIXTIME(${latestDate}) ORDER BY channels DESC LIMIT 100 `; @@ -260,8 +248,8 @@ class NodesApi { public async $searchNodeByPublicKeyOrAlias(search: string) { try { const publicKeySearch = search.replace('%', '') + '%'; - const aliasSearch = search.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z ]/g, '').split(' ').map((search) => '+' + search + '*').join(' '); - const query = `SELECT nodes.public_key, nodes.alias, node_stats.capacity FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key WHERE nodes.public_key LIKE ? OR MATCH nodes.alias_search AGAINST (? IN BOOLEAN MODE) GROUP BY nodes.public_key ORDER BY node_stats.capacity DESC LIMIT 10`; + const aliasSearch = search.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '').split(' ').map((search) => '+' + search + '*').join(' '); + const query = `SELECT public_key, alias, capacity, channels, status FROM nodes WHERE public_key LIKE ? OR MATCH alias_search AGAINST (? IN BOOLEAN MODE) ORDER BY capacity DESC LIMIT 10`; const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]); return rows; } catch (e) { @@ -276,7 +264,7 @@ class NodesApi { // List all channels and the two linked ISP query = ` - SELECT short_id, capacity, + SELECT short_id, channels.capacity, channels.node1_public_key AS node1PublicKey, isp1.names AS isp1, isp1.id as isp1ID, channels.node2_public_key AS node2PublicKey, isp2.names AS isp2, isp2.id as isp2ID FROM channels @@ -391,17 +379,11 @@ class NodesApi { public async $getNodesPerCountry(countryId: string) { try { const query = ` - SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels, + SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels, nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at, geo_names_city.names as city, geo_names_country.names as country, geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision - FROM node_stats - JOIN ( - SELECT public_key, MAX(added) as last_added - FROM node_stats - GROUP BY public_key - ) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added - RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key + FROM nodes LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code' @@ -426,17 +408,11 @@ class NodesApi { public async $getNodesPerISP(ISPId: string) { try { const query = ` - SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels, + SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels, nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at, geo_names_city.names as city, geo_names_country.names as country, geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision - FROM node_stats - JOIN ( - SELECT public_key, MAX(added) as last_added - FROM node_stats - GROUP BY public_key - ) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added - RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key + FROM nodes LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code' @@ -464,7 +440,6 @@ class NodesApi { FROM nodes JOIN geo_names ON geo_names.id = nodes.country_id AND geo_names.type = 'country' JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code' - JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key GROUP BY country_id ORDER BY COUNT(DISTINCT nodes.public_key) DESC `; @@ -528,6 +503,18 @@ class NodesApi { } } + /** + * Update node sockets + */ + public async $updateNodeSockets(publicKey: string, sockets: {network: string; addr: string}[]): Promise { + const formattedSockets = (sockets.map(a => a.addr).join(',')) ?? ''; + try { + await DB.query(`UPDATE nodes SET sockets = ? WHERE public_key = ?`, [formattedSockets, publicKey]); + } catch (e) { + logger.err(`Cannot update node sockets for ${publicKey}. Reason: ${e instanceof Error ? e.message : e}`); + } + } + /** * Set all nodes not in `nodesPubkeys` as inactive (status = 0) */ @@ -555,7 +542,7 @@ class NodesApi { } private aliasToSearchText(str: string): string { - return str.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z ]/g, ''); + return str.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, ''); } } diff --git a/backend/src/api/explorer/statistics.api.ts b/backend/src/api/explorer/statistics.api.ts index 7bf3d9107..558ee86fd 100644 --- a/backend/src/api/explorer/statistics.api.ts +++ b/backend/src/api/explorer/statistics.api.ts @@ -27,7 +27,7 @@ class StatisticsApi { public async $getLatestStatistics(): Promise { try { const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1`); - const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1 OFFSET 7`); + const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats WHERE DATE(added) = DATE(NOW() - INTERVAL 7 DAY)`); return { latest: rows[0], previous: rows2[0], diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts index 656c3c6da..8b055832e 100644 --- a/backend/src/api/lightning/clightning/clightning-convert.ts +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -13,9 +13,13 @@ export function convertNode(clNode: any): ILightningApi.Node { features: [], // TODO parse and return clNode.feature pub_key: clNode.nodeid, addresses: clNode.addresses?.map((addr) => { + let address = addr.address; + if (addr.type === 'ipv6') { + address = `[${address}]`; + } return { network: addr.type, - addr: `${addr.address}:${addr.port}` + addr: `${address}:${addr.port}` }; }) ?? [], last_update: clNode?.last_timestamp ?? 0, diff --git a/backend/src/logger.ts b/backend/src/logger.ts index ea7e8cd3d..63774d513 100644 --- a/backend/src/logger.ts +++ b/backend/src/logger.ts @@ -74,7 +74,7 @@ class Logger { private getNetwork(): string { if (config.LIGHTNING.ENABLED) { - return 'lightning'; + return config.MEMPOOL.NETWORK === 'mainnet' ? 'lightning' : `${config.MEMPOOL.NETWORK}-lightning`; } if (config.BISQ.ENABLED) { return 'bisq'; diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index d704012f7..b9b7df92c 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -63,6 +63,9 @@ class NetworkSyncService { let deletedSockets = 0; const graphNodesPubkeys: string[] = []; for (const node of nodes) { + const latestUpdated = await channelsApi.$getLatestChannelUpdateForNode(node.pub_key); + node.last_update = Math.max(node.last_update, latestUpdated); + await nodesApi.$saveNode(node); graphNodesPubkeys.push(node.pub_key); ++progress; @@ -83,7 +86,7 @@ class NetworkSyncService { logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted`); // If a channel if not present in the graph, mark it as inactive - nodesApi.$setNodesInactive(graphNodesPubkeys); + await nodesApi.$setNodesInactive(graphNodesPubkeys); if (config.MAXMIND.ENABLED) { $lookupNodeLocation(); @@ -98,7 +101,7 @@ class NetworkSyncService { const [closedChannelsRaw]: any[] = await DB.query(`SELECT id FROM channels WHERE status = 2`); const closedChannels = {}; for (const closedChannel of closedChannelsRaw) { - closedChannels[Common.channelShortIdToIntegerId(closedChannel.id)] = true; + closedChannels[closedChannel.id] = true; } let progress = 0; @@ -121,7 +124,7 @@ class NetworkSyncService { logger.info(`${progress} channels updated`); // If a channel if not present in the graph, mark it as inactive - channelsApi.$setChannelsInactive(graphChannelsIds); + await channelsApi.$setChannelsInactive(graphChannelsIds); } catch (e) { logger.err(`Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`); } @@ -285,44 +288,66 @@ class NetworkSyncService { for (const channel of channels) { let reason = 0; // Only Esplora backend can retrieve spent transaction outputs - const outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id); - const lightningScriptReasons: number[] = []; - for (const outspend of outspends) { - if (outspend.spent && outspend.txid) { - const spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid); - const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]); - lightningScriptReasons.push(lightningScript); + try { + let outspends: IEsploraApi.Outspend[] | undefined; + try { + outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id); + } catch (e) { + logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`); + continue; } - } - if (lightningScriptReasons.length === outspends.length - && lightningScriptReasons.filter((r) => r === 1).length === outspends.length) { - reason = 1; - } else { - const filteredReasons = lightningScriptReasons.filter((r) => r !== 1); - if (filteredReasons.length) { - if (filteredReasons.some((r) => r === 2 || r === 4)) { - reason = 3; - } else { - reason = 2; + const lightningScriptReasons: number[] = []; + for (const outspend of outspends) { + if (outspend.spent && outspend.txid) { + let spendingTx: IEsploraApi.Transaction | undefined; + try { + spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid); + } catch (e) { + logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`); + continue; + } + const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]); + lightningScriptReasons.push(lightningScript); } + } + if (lightningScriptReasons.length === outspends.length + && lightningScriptReasons.filter((r) => r === 1).length === outspends.length) { + reason = 1; } else { - /* - We can detect a commitment transaction (force close) by reading Sequence and Locktime - https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction - */ - const closingTx = await bitcoinApi.$getRawTransaction(channel.closing_transaction_id); - const sequenceHex: string = closingTx.vin[0].sequence.toString(16); - const locktimeHex: string = closingTx.locktime.toString(16); - if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') { - reason = 2; // Here we can't be sure if it's a penalty or not + const filteredReasons = lightningScriptReasons.filter((r) => r !== 1); + if (filteredReasons.length) { + if (filteredReasons.some((r) => r === 2 || r === 4)) { + reason = 3; + } else { + reason = 2; + } } else { - reason = 1; + /* + We can detect a commitment transaction (force close) by reading Sequence and Locktime + https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction + */ + let closingTx: IEsploraApi.Transaction | undefined; + try { + closingTx = await bitcoinApi.$getRawTransaction(channel.closing_transaction_id); + } catch (e) { + logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id}. Reason ${e instanceof Error ? e.message : e}`); + continue; + } + const sequenceHex: string = closingTx.vin[0].sequence.toString(16); + const locktimeHex: string = closingTx.locktime.toString(16); + if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') { + reason = 2; // Here we can't be sure if it's a penalty or not + } else { + reason = 1; + } } } - } - if (reason) { - logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.'); - await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]); + if (reason) { + logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.'); + await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]); + } + } catch (e) { + logger.err(`$runClosedChannelsForensics() failed for channel ${channel.short_id}. Reason: ${e instanceof Error ? e.message : e}`); } ++progress; diff --git a/backend/src/tasks/lightning/sync-tasks/node-locations.ts b/backend/src/tasks/lightning/sync-tasks/node-locations.ts index 30a6bfc2a..9069e0fff 100644 --- a/backend/src/tasks/lightning/sync-tasks/node-locations.ts +++ b/backend/src/tasks/lightning/sync-tasks/node-locations.ts @@ -4,6 +4,7 @@ import nodesApi from '../../../api/explorer/nodes.api'; import config from '../../../config'; import DB from '../../../database'; import logger from '../../../logger'; +import * as IPCheck from '../../../utils/ipcheck.js'; export async function $lookupNodeLocation(): Promise { let loggerTimer = new Date().getTime() / 1000; @@ -27,6 +28,26 @@ export async function $lookupNodeLocation(): Promise { const asn = lookupAsn.get(ip); const isp = lookupIsp.get(ip); + let asOverwrite: any | undefined; + if (asn && (IPCheck.match(ip, '170.75.160.0/20') || IPCheck.match(ip, '172.81.176.0/21'))) { + asOverwrite = { + asn: 394745, + name: 'Lunanode', + }; + } + else if (asn && (IPCheck.match(ip, '50.7.0.0/16') || IPCheck.match(ip, '66.90.64.0/18'))) { + asOverwrite = { + asn: 30058, + name: 'FDCservers.net', + }; + } + else if (asn && asn.autonomous_system_number === 174) { + asOverwrite = { + asn: 174, + name: 'Cogent Communications', + }; + } + if (city && (asn || isp)) { const query = ` UPDATE nodes SET @@ -41,7 +62,7 @@ export async function $lookupNodeLocation(): Promise { `; const params = [ - isp?.autonomous_system_number ?? asn?.autonomous_system_number, + asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number, city.city?.geoname_id, city.country?.geoname_id, city.subdivisions ? city.subdivisions[0].geoname_id : null, @@ -91,7 +112,10 @@ export async function $lookupNodeLocation(): Promise { if (isp?.autonomous_system_organization ?? asn?.autonomous_system_organization) { await DB.query( `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'as_organization', ?)`, - [isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]); + [ + asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number, + JSON.stringify(asOverwrite?.name ?? isp?.isp ?? asn?.autonomous_system_organization) + ]); } } diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index e05ba4ab3..e3dfe6652 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -7,6 +7,8 @@ import { ILightningApi } from '../../../api/lightning/lightning-api.interface'; import { isIP } from 'net'; import { Common } from '../../../api/common'; import channelsApi from '../../../api/explorer/channels.api'; +import nodesApi from '../../../api/explorer/nodes.api'; +import { ResultSetHeader } from 'mysql2'; const fsPromises = promises; @@ -18,7 +20,12 @@ class LightningStatsImporter { logger.info('Caching funding txs for currently existing channels'); await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id)); + if (config.MEMPOOL.NETWORK !== 'mainnet' || config.DATABASE.ENABLED === false) { + return; + } + await this.$importHistoricalLightningStats(); + await this.$cleanupIncorrectSnapshot(); } /** @@ -32,7 +39,28 @@ class LightningStatsImporter { let clearnetTorNodes = 0; let unannouncedNodes = 0; + const [nodesInDbRaw]: any[] = await DB.query(`SELECT public_key FROM nodes`); + const nodesInDb = {}; + for (const node of nodesInDbRaw) { + nodesInDb[node.public_key] = node; + } + for (const node of networkGraph.nodes) { + // If we don't know about this node, insert it in db + if (isHistorical === true && !nodesInDb[node.pub_key]) { + await nodesApi.$saveNode({ + last_update: node.last_update, + pub_key: node.pub_key, + alias: node.alias, + addresses: node.addresses, + color: node.color, + features: node.features, + }); + nodesInDb[node.pub_key] = node; + } else { + await nodesApi.$updateNodeSockets(node.pub_key, node.addresses); + } + let hasOnion = false; let hasClearnet = false; let isUnnanounced = true; @@ -69,7 +97,7 @@ class LightningStatsImporter { const baseFees: number[] = []; const alreadyCountedChannels = {}; - const [channelsInDbRaw]: any[] = await DB.query(`SELECT short_id, created FROM channels`); + const [channelsInDbRaw]: any[] = await DB.query(`SELECT short_id FROM channels`); const channelsInDb = {}; for (const channel of channelsInDbRaw) { channelsInDb[channel.short_id] = channel; @@ -84,29 +112,19 @@ class LightningStatsImporter { continue; } - // Channel is already in db, check if we need to update 'created' field - if (isHistorical === true) { - //@ts-ignore - if (channelsInDb[short_id] && channel.timestamp < channel.created) { - await DB.query(` - UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.short_id = ?`, - //@ts-ignore - [channel.timestamp, short_id] - ); - } else if (!channelsInDb[short_id]) { - await channelsApi.$saveChannel({ - channel_id: short_id, - chan_point: `${tx.txid}:${short_id.split('x')[2]}`, - //@ts-ignore - last_update: channel.timestamp, - node1_pub: channel.node1_pub, - node2_pub: channel.node2_pub, - capacity: (tx.value * 100000000).toString(), - node1_policy: null, - node2_policy: null, - }, 0); - channelsInDb[channel.channel_id] = channel; - } + // If we don't know about this channel, insert it in db + if (isHistorical === true && !channelsInDb[short_id]) { + await channelsApi.$saveChannel({ + channel_id: short_id, + chan_point: `${tx.txid}:${short_id.split('x')[2]}`, + last_update: channel.last_update, + node1_pub: channel.node1_pub, + node2_pub: channel.node2_pub, + capacity: (tx.value * 100000000).toString(), + node1_policy: null, + node2_policy: null, + }, 0); + channelsInDb[channel.channel_id] = channel; } if (!nodeStats[channel.node1_pub]) { @@ -269,6 +287,17 @@ class LightningStatsImporter { nodeStats[public_key].capacity, nodeStats[public_key].channels, ]); + + if (!isHistorical) { + await DB.query( + `UPDATE nodes SET capacity = ?, channels = ? WHERE public_key = ?`, + [ + nodeStats[public_key].capacity, + nodeStats[public_key].channels, + public_key, + ] + ); + } } return { @@ -281,6 +310,7 @@ class LightningStatsImporter { * Import topology files LN historical data into the database */ async $importHistoricalLightningStats(): Promise { + logger.debug('Run the historical importer'); try { let fileList: string[] = []; try { @@ -294,7 +324,7 @@ class LightningStatsImporter { fileList.sort().reverse(); const [rows]: any[] = await DB.query(` - SELECT UNIX_TIMESTAMP(added) AS added, node_count + SELECT UNIX_TIMESTAMP(added) AS added FROM lightning_stats ORDER BY added DESC `); @@ -341,10 +371,16 @@ class LightningStatsImporter { graph = JSON.parse(fileContent); graph = await this.cleanupTopology(graph); } catch (e) { - logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`); + logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content. Reason: ${e instanceof Error ? e.message : e}`); continue; } + if (this.isIncorrectSnapshot(timestamp, graph)) { + logger.debug(`Ignoring ${this.topologiesFolder}/${filename}, because we defined it as an incorrect snapshot`); + ++totalProcessed; + continue; + } + if (!logStarted) { logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`); logStarted = true; @@ -375,7 +411,7 @@ class LightningStatsImporter { } } - async cleanupTopology(graph) { + cleanupTopology(graph): ILightningApi.NetworkGraph { const newGraph = { nodes: [], edges: [], @@ -385,18 +421,23 @@ class LightningStatsImporter { const addressesParts = (node.addresses ?? '').split(','); const addresses: any[] = []; for (const address of addressesParts) { + const formatted = Common.findSocketNetwork(address); addresses.push({ - network: '', - addr: address + network: formatted.network, + addr: formatted.url }); } + let rgb = node.rgb_color ?? '#000000'; + if (rgb.indexOf('#') === -1) { + rgb = `#${rgb}`; + } newGraph.nodes.push({ last_update: node.timestamp ?? 0, pub_key: node.id ?? null, - alias: node.alias ?? null, + alias: node.alias ?? node.id.slice(0, 20), addresses: addresses, - color: node.rgb_color ?? null, + color: rgb, features: {}, }); } @@ -430,6 +471,69 @@ class LightningStatsImporter { return newGraph; } + + private isIncorrectSnapshot(timestamp, graph): boolean { + if (timestamp >= 1549065600 /* 2019-02-02 */ && timestamp <= 1550620800 /* 2019-02-20 */ && graph.nodes.length < 2600) { + return true; + } + if (timestamp >= 1552953600 /* 2019-03-19 */ && timestamp <= 1556323200 /* 2019-05-27 */ && graph.nodes.length < 4000) { + return true; + } + if (timestamp >= 1557446400 /* 2019-05-10 */ && timestamp <= 1560470400 /* 2019-06-14 */ && graph.nodes.length < 4000) { + return true; + } + if (timestamp >= 1561680000 /* 2019-06-28 */ && timestamp <= 1563148800 /* 2019-07-15 */ && graph.nodes.length < 4000) { + return true; + } + if (timestamp >= 1571270400 /* 2019-11-17 */ && timestamp <= 1580601600 /* 2020-02-02 */ && graph.nodes.length < 4500) { + return true; + } + if (timestamp >= 1591142400 /* 2020-06-03 */ && timestamp <= 1592006400 /* 2020-06-13 */ && graph.nodes.length < 5500) { + return true; + } + if (timestamp >= 1632787200 /* 2021-09-28 */ && timestamp <= 1633564800 /* 2021-10-07 */ && graph.nodes.length < 13000) { + return true; + } + if (timestamp >= 1634256000 /* 2021-10-15 */ && timestamp <= 1645401600 /* 2022-02-21 */ && graph.nodes.length < 17000) { + return true; + } + if (timestamp >= 1654992000 /* 2022-06-12 */ && timestamp <= 1661472000 /* 2022-08-26 */ && graph.nodes.length < 14000) { + return true; + } + + return false; + } + + private async $cleanupIncorrectSnapshot(): Promise { + // We do not run this one automatically because those stats are not supposed to be inserted in the first + // place, but I write them here to remind us we manually run those queries + + // DELETE FROM lightning_stats + // WHERE ( + // UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 AND node_count < 2600 OR + // UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 AND node_count < 4000 OR + // UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 AND node_count < 4000 OR + // UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 AND node_count < 4000 OR + // UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 AND node_count < 4500 OR + // UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 AND node_count < 5500 OR + // UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 AND node_count < 13000 OR + // UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 AND node_count < 17000 OR + // UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000 AND node_count < 14000 + // ) + + // DELETE FROM node_stats + // WHERE ( + // UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 OR + // UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 OR + // UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 OR + // UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 OR + // UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 OR + // UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 OR + // UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 OR + // UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 OR + // UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000 + // ) + } } export default new LightningStatsImporter; diff --git a/backend/src/utils/ipcheck.js b/backend/src/utils/ipcheck.js new file mode 100644 index 000000000..06d4a6f15 --- /dev/null +++ b/backend/src/utils/ipcheck.js @@ -0,0 +1,119 @@ +var net = require('net'); + +var IPCheck = module.exports = function(input) { + var self = this; + + if (!(self instanceof IPCheck)) { + return new IPCheck(input); + } + + self.input = input; + self.parse(); +}; + +IPCheck.prototype.parse = function() { + var self = this; + + if (!self.input || typeof self.input !== 'string') return self.valid = false; + + var ip; + + var pos = self.input.lastIndexOf('/'); + if (pos !== -1) { + ip = self.input.substring(0, pos); + self.mask = +self.input.substring(pos + 1); + } else { + ip = self.input; + self.mask = null; + } + + self.ipv = net.isIP(ip); + self.valid = !!self.ipv && !isNaN(self.mask); + + if (!self.valid) return; + + // default mask = 32 for ipv4 and 128 for ipv6 + if (self.mask === null) self.mask = self.ipv === 4 ? 32 : 128; + + if (self.ipv === 4) { + // difference between ipv4 and ipv6 masks + self.mask += 96; + } + + if (self.mask < 0 || self.mask > 128) { + self.valid = false; + return; + } + + self.address = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]; + + if(self.ipv === 4){ + self.parseIPv4(ip); + }else{ + self.parseIPv6(ip); + } +}; + +IPCheck.prototype.parseIPv4 = function(ip) { + var self = this; + + // ipv4 addresses live under ::ffff:0:0 + self.address[10] = self.address[11] = 0xff; + + var octets = ip.split('.'); + for (var i = 0; i < 4; i++) { + self.address[i + 12] = parseInt(octets[i], 10); + } +}; + + +var V6_TRANSITIONAL = /:(\d+\.\d+\.\d+\.\d+)$/; + +IPCheck.prototype.parseIPv6 = function(ip) { + var self = this; + + var transitionalMatch = V6_TRANSITIONAL.exec(ip); + if(transitionalMatch){ + self.parseIPv4(transitionalMatch[1]); + return; + } + + var bits = ip.split(':'); + if (bits.length < 8) { + ip = ip.replace('::', Array(11 - bits.length).join(':')); + bits = ip.split(':'); + } + + var j = 0; + for (var i = 0; i < bits.length; i += 1) { + var x = bits[i] ? parseInt(bits[i], 16) : 0; + self.address[j++] = x >> 8; + self.address[j++] = x & 0xff; + } +}; + +IPCheck.prototype.match = function(cidr) { + var self = this; + + if (!(cidr instanceof IPCheck)) cidr = new IPCheck(cidr); + if (!self.valid || !cidr.valid) return false; + + var mask = cidr.mask; + var i = 0; + + while (mask >= 8) { + if (self.address[i] !== cidr.address[i]) return false; + + i++; + mask -= 8; + } + + var shift = 8 - mask; + return (self.address[i] >>> shift) === (cidr.address[i] >>> shift); +}; + + +IPCheck.match = function(ip, cidr) { + ip = ip instanceof IPCheck ? ip : new IPCheck(ip); + return ip.match(cidr); +}; diff --git a/backend/testSetup.ts b/backend/testSetup.ts new file mode 100644 index 000000000..ca51bbbe6 --- /dev/null +++ b/backend/testSetup.ts @@ -0,0 +1,5 @@ +jest.mock('./mempool-config.json', () => ({}), { virtual: true }); +jest.mock('./src/logger.ts', () => ({}), { virtual: true }); +jest.mock('./src/api/rbf-cache.ts', () => ({}), { virtual: true }); +jest.mock('./src/api/mempool.ts', () => ({}), { virtual: true }); +jest.mock('./src/api/memory-cache.ts', () => ({}), { virtual: true }); diff --git a/backend/tsconfig.json b/backend/tsconfig.json index 6a1970331..0670010e1 100644 --- a/backend/tsconfig.json +++ b/backend/tsconfig.json @@ -13,7 +13,8 @@ "node_modules/@types" ], "allowSyntheticDefaultImports": true, - "esModuleInterop": true + "esModuleInterop": true, + "allowJs": true, }, "include": [ "src/**/*.ts" diff --git a/frontend/src/app/app-routing.module.ts b/frontend/src/app/app-routing.module.ts index f4c6dbbc8..2d12bc2e7 100644 --- a/frontend/src/app/app-routing.module.ts +++ b/frontend/src/app/app-routing.module.ts @@ -1,5 +1,6 @@ import { NgModule } from '@angular/core'; -import { Routes, RouterModule, PreloadAllModules } from '@angular/router'; +import { Routes, RouterModule } from '@angular/router'; +import { AppPreloadingStrategy } from './app.preloading-strategy' import { StartComponent } from './components/start/start.component'; import { TransactionComponent } from './components/transaction/transaction.component'; import { TransactionPreviewComponent } from './components/transaction/transaction-preview.component'; @@ -25,6 +26,10 @@ import { AssetsComponent } from './components/assets/assets.component'; import { AssetComponent } from './components/asset/asset.component'; import { AssetsNavComponent } from './components/assets/assets-nav/assets-nav.component'; +const browserWindow = window || {}; +// @ts-ignore +const browserWindowEnv = browserWindow.__env || {}; + let routes: Routes = [ { path: 'testnet', @@ -32,7 +37,8 @@ let routes: Routes = [ { path: '', pathMatch: 'full', - loadChildren: () => import('./graphs/graphs.module').then(m => m.GraphsModule) + loadChildren: () => import('./graphs/graphs.module').then(m => m.GraphsModule), + data: { preload: true }, }, { path: '', @@ -109,7 +115,8 @@ let routes: Routes = [ }, { path: 'docs', - loadChildren: () => import('./docs/docs.module').then(m => m.DocsModule) + loadChildren: () => import('./docs/docs.module').then(m => m.DocsModule), + data: { preload: true }, }, { path: 'api', @@ -117,7 +124,8 @@ let routes: Routes = [ }, { path: 'lightning', - loadChildren: () => import('./lightning/lightning.module').then(m => m.LightningModule) + loadChildren: () => import('./lightning/lightning.module').then(m => m.LightningModule), + data: { preload: browserWindowEnv && browserWindowEnv.LIGHTNING === true }, }, ], }, @@ -410,10 +418,6 @@ let routes: Routes = [ }, ]; -const browserWindow = window || {}; -// @ts-ignore -const browserWindowEnv = browserWindow.__env || {}; - if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'bisq') { routes = [{ path: '', @@ -691,7 +695,7 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') { initialNavigation: 'enabled', scrollPositionRestoration: 'enabled', anchorScrolling: 'enabled', - preloadingStrategy: PreloadAllModules + preloadingStrategy: AppPreloadingStrategy })], }) export class AppRoutingModule { } diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index b6b8859f6..5ae0c6cb5 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -18,6 +18,7 @@ import { LanguageService } from './services/language.service'; import { FiatShortenerPipe } from './shared/pipes/fiat-shortener.pipe'; import { ShortenStringPipe } from './shared/pipes/shorten-string-pipe/shorten-string.pipe'; import { CapAddressPipe } from './shared/pipes/cap-address-pipe/cap-address-pipe'; +import { AppPreloadingStrategy } from './app.preloading-strategy'; @NgModule({ declarations: [ @@ -44,6 +45,7 @@ import { CapAddressPipe } from './shared/pipes/cap-address-pipe/cap-address-pipe ShortenStringPipe, FiatShortenerPipe, CapAddressPipe, + AppPreloadingStrategy, { provide: HTTP_INTERCEPTORS, useClass: HttpCacheInterceptor, multi: true } ], bootstrap: [AppComponent] diff --git a/frontend/src/app/app.preloading-strategy.ts b/frontend/src/app/app.preloading-strategy.ts new file mode 100644 index 000000000..f62d072da --- /dev/null +++ b/frontend/src/app/app.preloading-strategy.ts @@ -0,0 +1,10 @@ +import { PreloadingStrategy, Route } from '@angular/router'; +import { Observable, timer, mergeMap, of } from 'rxjs'; + +export class AppPreloadingStrategy implements PreloadingStrategy { + preload(route: Route, load: Function): Observable { + return route.data && route.data.preload + ? timer(1500).pipe(mergeMap(() => load())) + : of(null); + } +} diff --git a/frontend/src/app/components/address/address-preview.component.ts b/frontend/src/app/components/address/address-preview.component.ts index c0f6fff81..35e72435b 100644 --- a/frontend/src/app/components/address/address-preview.component.ts +++ b/frontend/src/app/components/address/address-preview.component.ts @@ -19,6 +19,7 @@ import { AddressInformation } from 'src/app/interfaces/node-api.interface'; export class AddressPreviewComponent implements OnInit, OnDestroy { network = ''; + rawAddress: string; address: Address; addressString: string; isLoadingAddress = true; @@ -55,7 +56,8 @@ export class AddressPreviewComponent implements OnInit, OnDestroy { this.mainSubscription = this.route.paramMap .pipe( switchMap((params: ParamMap) => { - this.openGraphService.waitFor('address-data'); + this.rawAddress = params.get('id') || ''; + this.openGraphService.waitFor('address-data-' + this.rawAddress); this.error = undefined; this.isLoadingAddress = true; this.loadedConfirmedTxCount = 0; @@ -73,7 +75,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy { this.isLoadingAddress = false; this.error = err; console.log(err); - this.openGraphService.fail('address-data'); + this.openGraphService.fail('address-data-' + this.rawAddress); return of(null); }) ); @@ -91,7 +93,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy { this.address = address; this.updateChainStats(); this.isLoadingAddress = false; - this.openGraphService.waitOver('address-data'); + this.openGraphService.waitOver('address-data-' + this.rawAddress); }) ) .subscribe(() => {}, @@ -99,7 +101,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy { console.log(error); this.error = error; this.isLoadingAddress = false; - this.openGraphService.fail('address-data'); + this.openGraphService.fail('address-data-' + this.rawAddress); } ); } diff --git a/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html b/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html index dfd0f0695..d07f9d60c 100644 --- a/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html +++ b/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html @@ -2,7 +2,36 @@