From e0952a4c1d7fd1fd41ccd2360c1ba34e6eb07a05 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sat, 16 Jul 2022 09:22:45 +0200 Subject: [PATCH 01/17] Wait for the price updater to complete before saving blocks prices --- backend/src/api/blocks.ts | 15 ++++++++++ backend/src/api/mining/mining.ts | 14 +++++----- backend/src/indexer.ts | 29 +++++++++++++++++--- backend/src/repositories/PricesRepository.ts | 5 ++++ backend/src/tasks/price-updater.ts | 9 +++--- 5 files changed, 57 insertions(+), 15 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 30f9fbf78..45ffd6079 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -22,6 +22,8 @@ import poolsParser from './pools-parser'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; import mining from './mining/mining'; import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; +import PricesRepository from '../repositories/PricesRepository'; +import priceUpdater from '../tasks/price-updater'; class Blocks { private blocks: BlockExtended[] = []; @@ -457,6 +459,19 @@ class Blocks { } await blocksRepository.$saveBlockInDatabase(blockExtended); + const lastestPriceId = await PricesRepository.$getLatestPriceId(); + if (priceUpdater.historyInserted === true && lastestPriceId !== null) { + await blocksRepository.$saveBlockPrices([{ + height: blockExtended.height, + priceId: lastestPriceId, + }]); + } else { + logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`) + setTimeout(() => { + indexer.runSingleTask('blocksPrices'); + }, 10000); + } + // Save blocks summary for visualization if it's enabled if (Common.blocksSummariesIndexingEnabled() === true) { await this.$getStrippedBlockTransactions(blockExtended.id, true); diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index 55e749596..55cd33bd3 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -473,7 +473,7 @@ class Mining { for (const block of blocksWithoutPrices) { // Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks - if (block.height < 68951) { + if (['mainnet', 'testnet'].includes(config.MEMPOOL.NETWORK) && block.height < 68951) { blocksPrices.push({ height: block.height, priceId: prices[0].id, @@ -492,11 +492,11 @@ class Mining { if (blocksPrices.length >= 100000) { totalInserted += blocksPrices.length; + let logStr = `Linking ${blocksPrices.length} blocks to their closest price`; if (blocksWithoutPrices.length > 200000) { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`); - } else { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`); + logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`; } + logger.debug(logStr); await BlocksRepository.$saveBlockPrices(blocksPrices); blocksPrices.length = 0; } @@ -504,11 +504,11 @@ class Mining { if (blocksPrices.length > 0) { totalInserted += blocksPrices.length; + let logStr = `Linking ${blocksPrices.length} blocks to their closest price`; if (blocksWithoutPrices.length > 200000) { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`); - } else { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`); + logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`; } + logger.debug(logStr); await BlocksRepository.$saveBlockPrices(blocksPrices); } } catch (e) { diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index e452a42f4..26a407291 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -6,13 +6,12 @@ import logger from './logger'; import HashratesRepository from './repositories/HashratesRepository'; import bitcoinClient from './api/bitcoin/bitcoin-client'; import priceUpdater from './tasks/price-updater'; +import PricesRepository from './repositories/PricesRepository'; class Indexer { runIndexer = true; indexerRunning = false; - - constructor() { - } + tasksRunning: string[] = []; public reindex() { if (Common.indexingEnabled()) { @@ -20,6 +19,28 @@ class Indexer { } } + public async runSingleTask(task: 'blocksPrices') { + if (!Common.indexingEnabled()) { + return; + } + + if (task === 'blocksPrices' && !this.tasksRunning.includes(task)) { + this.tasksRunning.push(task); + const lastestPriceId = await PricesRepository.$getLatestPriceId(); + if (priceUpdater.historyInserted === false || lastestPriceId === null) { + logger.debug(`Blocks prices indexer is waiting for the price updater to complete`) + setTimeout(() => { + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task) + this.runSingleTask('blocksPrices'); + }, 10000); + } else { + logger.debug(`Blocks prices indexer will run now`) + await mining.$indexBlockPrices(); + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task) + } + } + } + public async $run() { if (!Common.indexingEnabled() || this.runIndexer === false || this.indexerRunning === true || mempool.hasPriority() @@ -50,7 +71,7 @@ class Indexer { return; } - await mining.$indexBlockPrices(); + this.runSingleTask('blocksPrices'); await mining.$indexDifficultyAdjustments(); await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient await mining.$generateNetworkHashrateHistory(); diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index 92fb4860f..cc79ff2a6 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -27,6 +27,11 @@ class PricesRepository { return oldestRow[0] ? oldestRow[0].time : 0; } + public async $getLatestPriceId(): Promise { + const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`); + return oldestRow[0] ? oldestRow[0].id : null; + } + public async $getLatestPriceTime(): Promise { const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`); return oldestRow[0] ? oldestRow[0].time : 0; diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index a5901d7f7..81066efb2 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -1,4 +1,5 @@ import * as fs from 'fs'; +import { Common } from '../api/common'; import config from '../config'; import logger from '../logger'; import PricesRepository from '../repositories/PricesRepository'; @@ -34,10 +35,10 @@ export interface Prices { } class PriceUpdater { - historyInserted: boolean = false; - lastRun: number = 0; - lastHistoricalRun: number = 0; - running: boolean = false; + public historyInserted = false; + lastRun = 0; + lastHistoricalRun = 0; + running = false; feeds: PriceFeed[] = []; currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY']; latestPrices: Prices; From cacd4abd9d8f5153454ba2ef2ccde20712ecc30b Mon Sep 17 00:00:00 2001 From: Stephan Oeste Date: Fri, 5 Aug 2022 16:41:00 +0200 Subject: [PATCH 02/17] Add Core Lighting for FreeBSD in prod installer --- production/install | 52 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/production/install b/production/install index 729ff33e0..db48abaab 100755 --- a/production/install +++ b/production/install @@ -34,10 +34,11 @@ esac TOR_INSTALL=ON CERTBOT_INSTALL=ON -# install 3 network daemons +# install 4 network daemons BITCOIN_INSTALL=ON BISQ_INSTALL=ON ELEMENTS_INSTALL=ON +CLN_INSTALL=ON # configure 4 network instances BITCOIN_MAINNET_ENABLE=ON @@ -187,6 +188,7 @@ case $OS in NGINX_ETC_FOLDER=/usr/local/etc/nginx NGINX_CONFIGURATION=/usr/local/etc/nginx/nginx.conf CERTBOT_PKG=py39-certbot + CLN_PKG=c-lightning ;; Debian) @@ -272,6 +274,12 @@ ELECTRS_LIQUID_DATA=${ELECTRS_DATA_ROOT}/liquid ELECTRS_LIQUIDTESTNET_ZPOOL=${ZPOOL} ELECTRS_LIQUIDTESTNET_DATA=${ELECTRS_DATA_ROOT}/liquidtestnet +# Core Lightning user/group +CLN_USER=cln +CLN_GROUP=cln +# Core Lightning home folder +CLN_HOME=/cln + # bisq user/group BISQ_USER=bisq BISQ_GROUP=bisq @@ -572,6 +580,10 @@ zfsCreateFilesystems() done fi + if [ "${CLN_INSTALL}" = ON ];then + zfs create -o "mountpoint=${CLN_HOME}" "${ZPOOL}/cln" + fi + if [ "${BISQ_INSTALL}" = ON ];then zfs create -o "mountpoint=${BISQ_HOME}" "${ZPOOL}/bisq" fi @@ -651,6 +663,10 @@ ext4CreateDir() done fi + if [ "${CLN_INSTALL}" = ON ];then + mkdir -p "${CLN_HOME}" + fi + if [ "${BISQ_INSTALL}" = ON ];then mkdir -p "${BISQ_HOME}" fi @@ -711,6 +727,7 @@ Testnet:Enable Bitcoin Testnet:ON Signet:Enable Bitcoin Signet:ON Liquid:Enable Elements Liquid:ON Liquidtestnet:Enable Elements Liquidtestnet:ON +CoreLN:Enable Core Lightning:ON Bisq:Enable Bisq:ON EOF @@ -785,6 +802,12 @@ else ELEMENTS_INSTALL=OFF fi +if grep CoreLN $tempfile >/dev/null 2>&1;then + CLN_INSTALL=ON +else + CLN_INSTALL=OFF +fi + if grep Bisq $tempfile >/dev/null 2>&1;then BISQ_INSTALL=ON BISQ_MAINNET_ENABLE=ON @@ -1165,6 +1188,33 @@ if [ "${ELEMENTS_INSTALL}" = ON ;then osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_ELECTRS_HOME} && cargo run --release --features liquid --bin electrs -- --network liquid --version" || true fi +##################################### +# Core Lightning for Bitcoin Mainnet # +##################################### + +echo "[*] Installing Core Lightning" +case $OS in + FreeBSD) + echo "[*] Creating Core Lightning user" + osGroupCreate "${CLN_GROUP}" + osUserCreate "${CLN_USER}" "${CLN_HOME}" "${CLN_GROUP}" + osSudo "${ROOT_USER}" chsh -s `which zsh` "${CLN_USER}" + osSudo "${ROOT_USER}" chown -R "${CLN_USER}:${CLN_GROUP}" "${CLN_HOME}" + osSudo "${CLN_USER}" touch "${CLN_HOME}/.zshrc" + + echo "[*] Installing Core Lightning package" + osPackageInstall ${CLN_PKG} + + echo "[*] Installing Core Lightning mainnet Cronjob" + crontab_cln+='@reboot sleep 30 ; screen -dmS main lightningd --alias `hostname` --bitcoin-datadir /bitcoin\n' + crontab_cln+='@reboot sleep 60 ; screen -dmS sig lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network signet\n' + crontab_cln+='@reboot sleep 90 ; screen -dmS tes lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network testnet\n' + echo "${crontab_cln}" | crontab -u "${CLN_USER}" - + ;; + Debian) + ;; +esac + ##################### # Bisq installation # ##################### From 47363b477e5515b505ee66437104ea2599ecd92b Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 8 Aug 2022 09:00:11 +0200 Subject: [PATCH 03/17] Refactor the LN backend and add more logs --- backend/src/api/common.ts | 34 ++ backend/src/api/explorer/channels.api.ts | 136 ++++++ backend/src/api/explorer/nodes.api.ts | 62 +++ backend/src/config.ts | 6 +- backend/src/database.ts | 6 +- .../tasks/lightning/network-sync.service.ts | 406 +++++++----------- .../tasks/lightning/stats-updater.service.ts | 19 +- .../sync-tasks/funding-tx-fetcher.ts | 2 +- .../lightning/sync-tasks/node-locations.ts | 55 ++- 9 files changed, 434 insertions(+), 292 deletions(-) diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index fe6b858e0..410d34a01 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -1,5 +1,6 @@ import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces'; import config from '../config'; +import { convertChannelId } from './lightning/clightning/clightning-convert'; export class Common { static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ? '144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49' @@ -184,4 +185,37 @@ export class Common { config.MEMPOOL.BLOCKS_SUMMARIES_INDEXING === true ); } + + static setDateMidnight(date: Date): void { + date.setUTCHours(0); + date.setUTCMinutes(0); + date.setUTCSeconds(0); + date.setUTCMilliseconds(0); + } + + static channelShortIdToIntegerId(id: string): string { + if (config.LIGHTNING.BACKEND === 'lnd') { + return id; + } + return convertChannelId(id); + } + + /** Decodes a channel id returned by lnd as uint64 to a short channel id */ + static channelIntegerIdToShortId(id: string): string { + if (config.LIGHTNING.BACKEND === 'cln') { + return id; + } + + const n = BigInt(id); + return [ + n >> 40n, // nth block + (n >> 16n) & 0xffffffn, // nth tx of the block + n & 0xffffn // nth output of the tx + ].join('x'); + } + + static utcDateToMysql(date?: number): string { + const d = new Date((date || 0) * 1000); + return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0]; + } } diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index 9928cc85b..6023d4c94 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -1,6 +1,9 @@ import logger from '../../logger'; import DB from '../../database'; import nodesApi from './nodes.api'; +import { ResultSetHeader } from 'mysql2'; +import { ILightningApi } from '../lightning/lightning-api.interface'; +import { Common } from '../common'; class ChannelsApi { public async $getAllChannels(): Promise { @@ -302,6 +305,139 @@ class ChannelsApi { }, }; } + + /** + * Save or update a channel present in the graph + */ + public async $saveChannel(channel: ILightningApi.Channel): Promise { + const [ txid, vout ] = channel.chan_point.split(':'); + + const policy1: Partial = channel.node1_policy || {}; + const policy2: Partial = channel.node2_policy || {}; + + try { + const query = `INSERT INTO channels + ( + id, + short_id, + capacity, + transaction_id, + transaction_vout, + updated_at, + status, + node1_public_key, + node1_base_fee_mtokens, + node1_cltv_delta, + node1_fee_rate, + node1_is_disabled, + node1_max_htlc_mtokens, + node1_min_htlc_mtokens, + node1_updated_at, + node2_public_key, + node2_base_fee_mtokens, + node2_cltv_delta, + node2_fee_rate, + node2_is_disabled, + node2_max_htlc_mtokens, + node2_min_htlc_mtokens, + node2_updated_at + ) + VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + capacity = ?, + updated_at = ?, + status = 1, + node1_public_key = ?, + node1_base_fee_mtokens = ?, + node1_cltv_delta = ?, + node1_fee_rate = ?, + node1_is_disabled = ?, + node1_max_htlc_mtokens = ?, + node1_min_htlc_mtokens = ?, + node1_updated_at = ?, + node2_public_key = ?, + node2_base_fee_mtokens = ?, + node2_cltv_delta = ?, + node2_fee_rate = ?, + node2_is_disabled = ?, + node2_max_htlc_mtokens = ?, + node2_min_htlc_mtokens = ?, + node2_updated_at = ? + ;`; + + await DB.query(query, [ + Common.channelShortIdToIntegerId(channel.channel_id), + Common.channelIntegerIdToShortId(channel.channel_id), + channel.capacity, + txid, + vout, + Common.utcDateToMysql(channel.last_update), + channel.node1_pub, + policy1.fee_base_msat, + policy1.time_lock_delta, + policy1.fee_rate_milli_msat, + policy1.disabled, + policy1.max_htlc_msat, + policy1.min_htlc, + Common.utcDateToMysql(policy1.last_update), + channel.node2_pub, + policy2.fee_base_msat, + policy2.time_lock_delta, + policy2.fee_rate_milli_msat, + policy2.disabled, + policy2.max_htlc_msat, + policy2.min_htlc, + Common.utcDateToMysql(policy2.last_update), + channel.capacity, + Common.utcDateToMysql(channel.last_update), + channel.node1_pub, + policy1.fee_base_msat, + policy1.time_lock_delta, + policy1.fee_rate_milli_msat, + policy1.disabled, + policy1.max_htlc_msat, + policy1.min_htlc, + Common.utcDateToMysql(policy1.last_update), + channel.node2_pub, + policy2.fee_base_msat, + policy2.time_lock_delta, + policy2.fee_rate_milli_msat, + policy2.disabled, + policy2.max_htlc_msat, + policy2.min_htlc, + Common.utcDateToMysql(policy2.last_update) + ]); + } catch (e) { + logger.err('$saveChannel() error: ' + (e instanceof Error ? e.message : e)); + } + } + + /** + * Set all channels not in `graphChannelsIds` as inactive (status = 0) + */ + public async $setChannelsInactive(graphChannelsIds: string[]): Promise { + if (graphChannelsIds.length === 0) { + return; + } + + try { + const result = await DB.query(` + UPDATE channels + SET status = 0 + WHERE short_id NOT IN ( + ${graphChannelsIds.map(id => `"${id}"`).join(',')} + ) + AND status != 2 + `); + if (result[0].changedRows ?? 0 > 0) { + logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`); + } else { + logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`); + } + } catch (e) { + logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e)); + } + } } export default new ChannelsApi(); diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 93eef9a48..d4857a3a4 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -1,5 +1,7 @@ import logger from '../../logger'; import DB from '../../database'; +import { ResultSetHeader } from 'mysql2'; +import { ILightningApi } from '../lightning/lightning-api.interface'; class NodesApi { public async $getNode(public_key: string): Promise { @@ -321,6 +323,66 @@ class NodesApi { throw e; } } + + /** + * Save or update a node present in the graph + */ + public async $saveNode(node: ILightningApi.Node): Promise { + try { + const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? ''; + const query = `INSERT INTO nodes( + public_key, + first_seen, + updated_at, + alias, + color, + sockets, + status + ) + VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, 1) + ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, color = ?, sockets = ?, status = 1`; + + await DB.query(query, [ + node.pub_key, + node.last_update, + node.alias, + node.color, + sockets, + node.last_update, + node.alias, + node.color, + sockets, + ]); + } catch (e) { + logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e)); + } + } + + /** + * Set all nodes not in `nodesPubkeys` as inactive (status = 0) + */ + public async $setNodesInactive(graphNodesPubkeys: string[]): Promise { + if (graphNodesPubkeys.length === 0) { + return; + } + + try { + const result = await DB.query(` + UPDATE nodes + SET status = 0 + WHERE public_key NOT IN ( + ${graphNodesPubkeys.map(pubkey => `"${pubkey}"`).join(',')} + ) + `); + if (result[0].changedRows ?? 0 > 0) { + logger.info(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`); + } else { + logger.debug(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`); + } + } catch (e) { + logger.err('$setNodesInactive() error: ' + (e instanceof Error ? e.message : e)); + } + } } export default new NodesApi(); diff --git a/backend/src/config.ts b/backend/src/config.ts index d4dfc9edd..ddf1fd3d4 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -32,7 +32,8 @@ interface IConfig { ENABLED: boolean; BACKEND: 'lnd' | 'cln' | 'ldk'; TOPOLOGY_FOLDER: string; - NODE_STATS_REFRESH_INTERVAL: number; + STATS_REFRESH_INTERVAL: number; + GRAPH_REFRESH_INTERVAL: number; }; LND: { TLS_CERT_PATH: string; @@ -184,7 +185,8 @@ const defaults: IConfig = { 'ENABLED': false, 'BACKEND': 'lnd', 'TOPOLOGY_FOLDER': '', - 'NODE_STATS_REFRESH_INTERVAL': 600, + 'STATS_REFRESH_INTERVAL': 600, + 'GRAPH_REFRESH_INTERVAL': 600, }, 'LND': { 'TLS_CERT_PATH': '', diff --git a/backend/src/database.ts b/backend/src/database.ts index 66c876378..c2fb0980b 100644 --- a/backend/src/database.ts +++ b/backend/src/database.ts @@ -1,7 +1,7 @@ import config from './config'; import { createPool, Pool, PoolConnection } from 'mysql2/promise'; import logger from './logger'; -import { PoolOptions } from 'mysql2/typings/mysql'; +import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } from 'mysql2/typings/mysql'; class DB { constructor() { @@ -28,7 +28,9 @@ import { PoolOptions } from 'mysql2/typings/mysql'; } } - public async query(query, params?) { + public async query(query, params?): Promise<[T, FieldPacket[]]> + { this.checkDBFlag(); const pool = await this.getPool(); return pool.query(query, params); diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index c6bfdcbe3..8f2f77534 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -1,60 +1,43 @@ import DB from '../../database'; import logger from '../../logger'; import channelsApi from '../../api/explorer/channels.api'; -import bitcoinClient from '../../api/bitcoin/bitcoin-client'; import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory'; import config from '../../config'; import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface'; import { ILightningApi } from '../../api/lightning/lightning-api.interface'; import { $lookupNodeLocation } from './sync-tasks/node-locations'; import lightningApi from '../../api/lightning/lightning-api-factory'; -import { convertChannelId } from '../../api/lightning/clightning/clightning-convert'; -import { Common } from '../../api/common'; +import nodesApi from '../../api/explorer/nodes.api'; +import { ResultSetHeader } from 'mysql2'; +import fundingTxFetcher from './sync-tasks/funding-tx-fetcher'; class NetworkSyncService { + loggerTimer = 0; + constructor() {} - public async $startService() { - logger.info('Starting node sync service'); + public async $startService(): Promise { + logger.info('Starting lightning network sync service'); - await this.$runUpdater(); + this.loggerTimer = new Date().getTime() / 1000; - setInterval(async () => { - await this.$runUpdater(); - }, 1000 * 60 * 60); + await this.$runTasks(); } - private async $runUpdater(): Promise { + private async $runTasks(): Promise { try { - logger.info(`Updating nodes and channels...`); + logger.info(`Updating nodes and channels`); const networkGraph = await lightningApi.$getNetworkGraph(); if (networkGraph.nodes.length === 0 || networkGraph.edges.length === 0) { logger.info(`LN Network graph is empty, retrying in 10 seconds`); - await Common.sleep$(10000); - this.$runUpdater(); + setTimeout(() => { this.$runTasks(); }, 10000); return; } - for (const node of networkGraph.nodes) { - await this.$saveNode(node); - } - logger.info(`Nodes updated.`); - - if (config.MAXMIND.ENABLED) { - await $lookupNodeLocation(); - } - - const graphChannelsIds: string[] = []; - for (const channel of networkGraph.edges) { - await this.$saveChannel(channel); - graphChannelsIds.push(channel.channel_id); - } - await this.$setChannelsInactive(graphChannelsIds); - - logger.info(`Channels updated.`); - - await this.$findInactiveNodesAndChannels(); + await this.$updateNodesList(networkGraph.nodes); + await this.$updateChannelsList(networkGraph.edges); + await this.$deactivateChannelsWithoutActiveNodes(); await this.$lookUpCreationDateFromChain(); await this.$updateNodeFirstSeen(); await this.$scanForClosedChannels(); @@ -63,60 +46,148 @@ class NetworkSyncService { } } catch (e) { - logger.err('$runUpdater() error: ' + (e instanceof Error ? e.message : e)); + logger.err('$runTasks() error: ' + (e instanceof Error ? e.message : e)); } + + setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL); + } + + /** + * Update the `nodes` table to reflect the current network graph state + */ + private async $updateNodesList(nodes: ILightningApi.Node[]): Promise { + let progress = 0; + + const graphNodesPubkeys: string[] = []; + for (const node of nodes) { + await nodesApi.$saveNode(node); + graphNodesPubkeys.push(node.pub_key); + ++progress; + + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Updating node ${progress}/${nodes.length}`); + this.loggerTimer = new Date().getTime() / 1000; + } + } + logger.info(`${progress} nodes updated`); + + // If a channel if not present in the graph, mark it as inactive + // nodesApi.$setNodesInactive(graphNodesPubkeys); + + if (config.MAXMIND.ENABLED) { + $lookupNodeLocation(); + } + } + + /** + * Update the `channels` table to reflect the current network graph state + */ + private async $updateChannelsList(channels: ILightningApi.Channel[]): Promise { + let progress = 0; + + const graphChannelsIds: string[] = []; + for (const channel of channels) { + await channelsApi.$saveChannel(channel); + graphChannelsIds.push(channel.channel_id); + ++progress; + + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Updating channel ${progress}/${channels.length}`); + this.loggerTimer = new Date().getTime() / 1000; + } + } + + logger.info(`${progress} channels updated`); + + // If a channel if not present in the graph, mark it as inactive + channelsApi.$setChannelsInactive(graphChannelsIds); } // This method look up the creation date of the earliest channel of the node // and update the node to that date in order to get the earliest first seen date - private async $updateNodeFirstSeen() { + private async $updateNodeFirstSeen(): Promise { + let progress = 0; + let updated = 0; + try { - const [nodes]: any[] = await DB.query(`SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node1_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created1, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node2_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created2 FROM nodes`); + const [nodes]: any[] = await DB.query(` + SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen, + ( + SELECT MIN(UNIX_TIMESTAMP(created)) + FROM channels + WHERE channels.node1_public_key = nodes.public_key + ) AS created1, + ( + SELECT MIN(UNIX_TIMESTAMP(created)) + FROM channels + WHERE channels.node2_public_key = nodes.public_key + ) AS created2 + FROM nodes + `); + for (const node of nodes) { - let lowest = 0; - if (node.created1) { - if (node.created2 && node.created2 < node.created1) { - lowest = node.created2; - } else { - lowest = node.created1; - } - } else if (node.created2) { - lowest = node.created2; - } - if (lowest && lowest < node.first_seen) { + const lowest = Math.min( + node.created1 ?? Number.MAX_SAFE_INTEGER, + node.created2 ?? Number.MAX_SAFE_INTEGER, + node.first_seen ?? Number.MAX_SAFE_INTEGER + ); + if (lowest < node.first_seen) { const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`; const params = [lowest, node.public_key]; await DB.query(query, params); } + ++progress; + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Updating node first seen date ${progress}/${nodes.length}`); + this.loggerTimer = new Date().getTime() / 1000; + ++updated; + } } - logger.info(`Node first seen dates scan complete.`); + logger.info(`Updated ${updated} node first seen dates`); } catch (e) { logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e)); } } - private async $lookUpCreationDateFromChain() { - logger.info(`Running channel creation date lookup...`); + private async $lookUpCreationDateFromChain(): Promise { + let progress = 0; + + logger.info(`Running channel creation date lookup`); try { const channels = await channelsApi.$getChannelsWithoutCreatedDate(); for (const channel of channels) { - const transaction = await bitcoinClient.getRawTransaction(channel.transaction_id, 1); - await DB.query(`UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`, [transaction.blocktime, channel.id]); + const transaction = await fundingTxFetcher.$fetchChannelOpenTx(channel.short_id); + await DB.query(` + UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`, + [transaction.timestamp, channel.id] + ); + ++progress; + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Updating channel creation date ${progress}/${channels.length}`); + this.loggerTimer = new Date().getTime() / 1000; + } } - logger.info(`Channel creation dates scan complete.`); + logger.info(`Updated ${channels.length} channels' creation date`); } catch (e) { - logger.err('$setCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e)); + logger.err('$lookUpCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e)); } } - // Looking for channels whos nodes are inactive - private async $findInactiveNodesAndChannels(): Promise { - logger.info(`Running inactive channels scan...`); + /** + * If a channel does not have any active node linked to it, then also + * mark that channel as inactive + */ + private async $deactivateChannelsWithoutActiveNodes(): Promise { + logger.info(`Find channels which nodes are offline`); try { - const [channels]: [{ id: string }[]] = await DB.query(` - SELECT channels.id - FROM channels + const result = await DB.query(` + UPDATE channels + SET status = 0 WHERE channels.status = 1 AND ( ( @@ -131,16 +202,19 @@ class NetworkSyncService { ) = 0) `); - for (const channel of channels) { - await this.$updateChannelStatus(channel.id, 0); + if (result[0].changedRows ?? 0 > 0) { + logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`); + } else { + logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`); } - logger.info(`Inactive channels scan complete.`); } catch (e) { - logger.err('$findInactiveNodesAndChannels() error: ' + (e instanceof Error ? e.message : e)); + logger.err('$deactivateChannelsWithoutActiveNodes() error: ' + (e instanceof Error ? e.message : e)); } } private async $scanForClosedChannels(): Promise { + let progress = 0; + try { logger.info(`Starting closed channels scan...`); const channels = await channelsApi.$getChannelsByStatus(0); @@ -154,6 +228,13 @@ class NetworkSyncService { await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]); } } + + ++progress; + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Checking if channel has been closed ${progress}/${channels.length}`); + this.loggerTimer = new Date().getTime() / 1000; + } } logger.info(`Closed channels scan complete.`); } catch (e) { @@ -171,6 +252,9 @@ class NetworkSyncService { if (!config.ESPLORA.REST_API_URL) { return; } + + let progress = 0; + try { logger.info(`Started running closed channel forensics...`); const channels = await channelsApi.$getClosedChannelsWithoutReason(); @@ -216,6 +300,13 @@ class NetworkSyncService { logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.'); await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]); } + + ++progress; + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Updating channel closed channel forensics ${progress}/${channels.length}`); + this.loggerTimer = new Date().getTime() / 1000; + } } logger.info(`Closed channels forensics scan complete.`); } catch (e) { @@ -270,195 +361,6 @@ class NetworkSyncService { } return 1; } - - private async $saveChannel(channel: ILightningApi.Channel): Promise { - const [ txid, vout ] = channel.chan_point.split(':'); - - const policy1: Partial = channel.node1_policy || {}; - const policy2: Partial = channel.node2_policy || {}; - - try { - const query = `INSERT INTO channels - ( - id, - short_id, - capacity, - transaction_id, - transaction_vout, - updated_at, - status, - node1_public_key, - node1_base_fee_mtokens, - node1_cltv_delta, - node1_fee_rate, - node1_is_disabled, - node1_max_htlc_mtokens, - node1_min_htlc_mtokens, - node1_updated_at, - node2_public_key, - node2_base_fee_mtokens, - node2_cltv_delta, - node2_fee_rate, - node2_is_disabled, - node2_max_htlc_mtokens, - node2_min_htlc_mtokens, - node2_updated_at - ) - VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON DUPLICATE KEY UPDATE - capacity = ?, - updated_at = ?, - status = 1, - node1_public_key = ?, - node1_base_fee_mtokens = ?, - node1_cltv_delta = ?, - node1_fee_rate = ?, - node1_is_disabled = ?, - node1_max_htlc_mtokens = ?, - node1_min_htlc_mtokens = ?, - node1_updated_at = ?, - node2_public_key = ?, - node2_base_fee_mtokens = ?, - node2_cltv_delta = ?, - node2_fee_rate = ?, - node2_is_disabled = ?, - node2_max_htlc_mtokens = ?, - node2_min_htlc_mtokens = ?, - node2_updated_at = ? - ;`; - - await DB.query(query, [ - this.toIntegerId(channel.channel_id), - this.toShortId(channel.channel_id), - channel.capacity, - txid, - vout, - this.utcDateToMysql(channel.last_update), - channel.node1_pub, - policy1.fee_base_msat, - policy1.time_lock_delta, - policy1.fee_rate_milli_msat, - policy1.disabled, - policy1.max_htlc_msat, - policy1.min_htlc, - this.utcDateToMysql(policy1.last_update), - channel.node2_pub, - policy2.fee_base_msat, - policy2.time_lock_delta, - policy2.fee_rate_milli_msat, - policy2.disabled, - policy2.max_htlc_msat, - policy2.min_htlc, - this.utcDateToMysql(policy2.last_update), - channel.capacity, - this.utcDateToMysql(channel.last_update), - channel.node1_pub, - policy1.fee_base_msat, - policy1.time_lock_delta, - policy1.fee_rate_milli_msat, - policy1.disabled, - policy1.max_htlc_msat, - policy1.min_htlc, - this.utcDateToMysql(policy1.last_update), - channel.node2_pub, - policy2.fee_base_msat, - policy2.time_lock_delta, - policy2.fee_rate_milli_msat, - policy2.disabled, - policy2.max_htlc_msat, - policy2.min_htlc, - this.utcDateToMysql(policy2.last_update) - ]); - } catch (e) { - logger.err('$saveChannel() error: ' + (e instanceof Error ? e.message : e)); - } - } - - private async $updateChannelStatus(channelId: string, status: number): Promise { - try { - await DB.query(`UPDATE channels SET status = ? WHERE id = ?`, [status, channelId]); - } catch (e) { - logger.err('$updateChannelStatus() error: ' + (e instanceof Error ? e.message : e)); - } - } - - private async $setChannelsInactive(graphChannelsIds: string[]): Promise { - if (graphChannelsIds.length === 0) { - return; - } - - try { - await DB.query(` - UPDATE channels - SET status = 0 - WHERE short_id NOT IN ( - ${graphChannelsIds.map(id => `"${id}"`).join(',')} - ) - AND status != 2 - `); - } catch (e) { - logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e)); - } - } - - private async $saveNode(node: ILightningApi.Node): Promise { - try { - const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? ''; - const query = `INSERT INTO nodes( - public_key, - first_seen, - updated_at, - alias, - color, - sockets - ) - VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?) - ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, color = ?, sockets = ?`; - - await DB.query(query, [ - node.pub_key, - node.last_update, - node.alias, - node.color, - sockets, - node.last_update, - node.alias, - node.color, - sockets, - ]); - } catch (e) { - logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e)); - } - } - - private toIntegerId(id: string): string { - if (config.LIGHTNING.BACKEND === 'cln') { - return convertChannelId(id); - } - else if (config.LIGHTNING.BACKEND === 'lnd') { - return id; - } - return ''; - } - - /** Decodes a channel id returned by lnd as uint64 to a short channel id */ - private toShortId(id: string): string { - if (config.LIGHTNING.BACKEND === 'cln') { - return id; - } - - const n = BigInt(id); - return [ - n >> 40n, // nth block - (n >> 16n) & 0xffffffn, // nth tx of the block - n & 0xffffn // nth output of the tx - ].join('x'); - } - - private utcDateToMysql(date?: number): string { - const d = new Date((date || 0) * 1000); - return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0]; - } } export default new NetworkSyncService(); diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index d58ff0ae6..c0db48976 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -1,8 +1,8 @@ -import DB from '../../database'; import logger from '../../logger'; import lightningApi from '../../api/lightning/lightning-api-factory'; import LightningStatsImporter from './sync-tasks/stats-importer'; import config from '../../config'; +import { Common } from '../../api/common'; class LightningStatsUpdater { public async $startService(): Promise { @@ -12,29 +12,20 @@ class LightningStatsUpdater { LightningStatsImporter.$run(); } - private setDateMidnight(date: Date): void { - date.setUTCHours(0); - date.setUTCMinutes(0); - date.setUTCSeconds(0); - date.setUTCMilliseconds(0); - } - private async $runTasks(): Promise { await this.$logStatsDaily(); - setTimeout(() => { - this.$runTasks(); - }, 1000 * config.LIGHTNING.NODE_STATS_REFRESH_INTERVAL); + setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL); } /** - * Update the latest entry for each node every config.LIGHTNING.NODE_STATS_REFRESH_INTERVAL seconds + * Update the latest entry for each node every config.LIGHTNING.STATS_REFRESH_INTERVAL seconds */ private async $logStatsDaily(): Promise { const date = new Date(); - this.setDateMidnight(date); + Common.setDateMidnight(date); - logger.info(`Updating latest networks stats`); + logger.info(`Updating latest network stats`); const networkGraph = await lightningApi.$getNetworkGraph(); LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph); } diff --git a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts index 6ca72aef7..8ca05b929 100644 --- a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts +++ b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts @@ -69,7 +69,7 @@ class FundingTxFetcher { this.running = false; } - public async $fetchChannelOpenTx(channelId: string): Promise { + public async $fetchChannelOpenTx(channelId: string): Promise<{timestamp: number, txid: string, value: number}> { if (this.fundingTxCache[channelId]) { return this.fundingTxCache[channelId]; } diff --git a/backend/src/tasks/lightning/sync-tasks/node-locations.ts b/backend/src/tasks/lightning/sync-tasks/node-locations.ts index 483131b26..30a6bfc2a 100644 --- a/backend/src/tasks/lightning/sync-tasks/node-locations.ts +++ b/backend/src/tasks/lightning/sync-tasks/node-locations.ts @@ -6,7 +6,10 @@ import DB from '../../../database'; import logger from '../../../logger'; export async function $lookupNodeLocation(): Promise { - logger.info(`Running node location updater using Maxmind...`); + let loggerTimer = new Date().getTime() / 1000; + let progress = 0; + + logger.info(`Running node location updater using Maxmind`); try { const nodes = await nodesApi.$getAllNodes(); const lookupCity = await maxmind.open(config.MAXMIND.GEOLITE2_CITY); @@ -18,21 +21,24 @@ export async function $lookupNodeLocation(): Promise { for (const socket of sockets) { const ip = socket.substring(0, socket.lastIndexOf(':')).replace('[', '').replace(']', ''); const hasClearnet = [4, 6].includes(net.isIP(ip)); + if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') { const city = lookupCity.get(ip); const asn = lookupAsn.get(ip); const isp = lookupIsp.get(ip); if (city && (asn || isp)) { - const query = `UPDATE nodes SET - as_number = ?, - city_id = ?, - country_id = ?, - subdivision_id = ?, - longitude = ?, - latitude = ?, - accuracy_radius = ? - WHERE public_key = ?`; + const query = ` + UPDATE nodes SET + as_number = ?, + city_id = ?, + country_id = ?, + subdivision_id = ?, + longitude = ?, + latitude = ?, + accuracy_radius = ? + WHERE public_key = ? + `; const params = [ isp?.autonomous_system_number ?? asn?.autonomous_system_number, @@ -46,25 +52,25 @@ export async function $lookupNodeLocation(): Promise { ]; await DB.query(query, params); - // Store Continent - if (city.continent?.geoname_id) { - await DB.query( + // Store Continent + if (city.continent?.geoname_id) { + await DB.query( `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'continent', ?)`, [city.continent?.geoname_id, JSON.stringify(city.continent?.names)]); - } + } - // Store Country - if (city.country?.geoname_id) { - await DB.query( + // Store Country + if (city.country?.geoname_id) { + await DB.query( `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country', ?)`, [city.country?.geoname_id, JSON.stringify(city.country?.names)]); - } + } // Store Country ISO code if (city.country?.iso_code) { await DB.query( - `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`, - [city.country?.geoname_id, city.country?.iso_code]); + `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`, + [city.country?.geoname_id, city.country?.iso_code]); } // Store Division @@ -88,10 +94,17 @@ export async function $lookupNodeLocation(): Promise { [isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]); } } + + ++progress; + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Updating node location data ${progress}/${nodes.length}`); + loggerTimer = new Date().getTime() / 1000; + } } } } - logger.info(`Node location data updated.`); + logger.info(`${progress} nodes location data updated`); } catch (e) { logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e)); } From abb078f7ee36695f8065fa6f20dc611597839526 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 9 Aug 2022 09:21:31 +0200 Subject: [PATCH 04/17] Convert to short_id before fetching the funding tx --- backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts index 8ca05b929..6ee50b8e9 100644 --- a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts +++ b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts @@ -1,5 +1,6 @@ import { existsSync, promises } from 'fs'; import bitcoinClient from '../../../api/bitcoin/bitcoin-client'; +import { Common } from '../../../api/common'; import config from '../../../config'; import logger from '../../../logger'; @@ -70,6 +71,10 @@ class FundingTxFetcher { } public async $fetchChannelOpenTx(channelId: string): Promise<{timestamp: number, txid: string, value: number}> { + if (channelId.indexOf('x') === -1) { + channelId = Common.channelIntegerIdToShortId(channelId); + } + if (this.fundingTxCache[channelId]) { return this.fundingTxCache[channelId]; } From 6a52725b63080a621676c416898e50bd820b2c78 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 9 Aug 2022 10:28:40 +0200 Subject: [PATCH 05/17] Make sure we work with integer in the stats importer --- .../tasks/lightning/stats-updater.service.ts | 3 ++- .../lightning/sync-tasks/stats-importer.ts | 27 ++++++++++--------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index c0db48976..ab5b3cccb 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -25,9 +25,10 @@ class LightningStatsUpdater { const date = new Date(); Common.setDateMidnight(date); - logger.info(`Updating latest network stats`); const networkGraph = await lightningApi.$getNetworkGraph(); LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph); + + logger.info(`Updated latest network stats`); } } diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index ba4adc71c..8c823e2ef 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -128,32 +128,32 @@ class LightningStatsImporter { if (channel.node1_policy !== undefined) { // Coming from the node for (const policy of [channel.node1_policy, channel.node2_policy]) { if (policy && policy.fee_rate_milli_msat < 5000) { - avgFeeRate += policy.fee_rate_milli_msat; - feeRates.push(policy.fee_rate_milli_msat); + avgFeeRate += parseInt(policy.fee_rate_milli_msat, 10); + feeRates.push(parseInt(policy.fee_rate_milli_msat, 10)); } if (policy && policy.fee_base_msat < 5000) { - avgBaseFee += policy.fee_base_msat; - baseFees.push(policy.fee_base_msat); + avgBaseFee += parseInt(policy.fee_base_msat, 10); + baseFees.push(parseInt(policy.fee_base_msat, 10)); } } } else { // Coming from the historical import if (channel.fee_rate_milli_msat < 5000) { - avgFeeRate += channel.fee_rate_milli_msat; - feeRates.push(channel.fee_rate_milli_msat); + avgFeeRate += parseInt(channel.fee_rate_milli_msat, 10); + feeRates.push(parseInt(channel.fee_rate_milli_msat), 10); } if (channel.fee_base_msat < 5000) { - avgBaseFee += channel.fee_base_msat; - baseFees.push(channel.fee_base_msat); + avgBaseFee += parseInt(channel.fee_base_msat, 10); + baseFees.push(parseInt(channel.fee_base_msat), 10); } } } - - avgFeeRate /= networkGraph.edges.length; - avgBaseFee /= networkGraph.edges.length; + + avgFeeRate /= Math.max(networkGraph.edges.length, 1); + avgBaseFee /= Math.max(networkGraph.edges.length, 1); const medCapacity = capacities.sort((a, b) => b - a)[Math.round(capacities.length / 2 - 1)]; const medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)]; const medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)]; - const avgCapacity = Math.round(capacity / capacities.length); + const avgCapacity = Math.round(capacity / Math.max(capacities.length, 1)); let query = `INSERT INTO lightning_stats( added, @@ -251,6 +251,9 @@ class LightningStatsImporter { }; } + /** + * Import topology files LN historical data into the database + */ async $importHistoricalLightningStats(): Promise { let latestNodeCount = 1; From 2a6f48d8c817bad3954bdcd117f9e4f4fe51e3ed Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 9 Aug 2022 11:07:13 +0200 Subject: [PATCH 06/17] Handle core timeout during closed channel scan, using correct config variable --- backend/src/api/explorer/channels.api.ts | 184 +++++++++--------- .../tasks/lightning/network-sync.service.ts | 36 ++-- 2 files changed, 110 insertions(+), 110 deletions(-) diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index 6023d4c94..55043197d 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -315,101 +315,97 @@ class ChannelsApi { const policy1: Partial = channel.node1_policy || {}; const policy2: Partial = channel.node2_policy || {}; - try { - const query = `INSERT INTO channels - ( - id, - short_id, - capacity, - transaction_id, - transaction_vout, - updated_at, - status, - node1_public_key, - node1_base_fee_mtokens, - node1_cltv_delta, - node1_fee_rate, - node1_is_disabled, - node1_max_htlc_mtokens, - node1_min_htlc_mtokens, - node1_updated_at, - node2_public_key, - node2_base_fee_mtokens, - node2_cltv_delta, - node2_fee_rate, - node2_is_disabled, - node2_max_htlc_mtokens, - node2_min_htlc_mtokens, - node2_updated_at - ) - VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON DUPLICATE KEY UPDATE - capacity = ?, - updated_at = ?, - status = 1, - node1_public_key = ?, - node1_base_fee_mtokens = ?, - node1_cltv_delta = ?, - node1_fee_rate = ?, - node1_is_disabled = ?, - node1_max_htlc_mtokens = ?, - node1_min_htlc_mtokens = ?, - node1_updated_at = ?, - node2_public_key = ?, - node2_base_fee_mtokens = ?, - node2_cltv_delta = ?, - node2_fee_rate = ?, - node2_is_disabled = ?, - node2_max_htlc_mtokens = ?, - node2_min_htlc_mtokens = ?, - node2_updated_at = ? - ;`; + const query = `INSERT INTO channels + ( + id, + short_id, + capacity, + transaction_id, + transaction_vout, + updated_at, + status, + node1_public_key, + node1_base_fee_mtokens, + node1_cltv_delta, + node1_fee_rate, + node1_is_disabled, + node1_max_htlc_mtokens, + node1_min_htlc_mtokens, + node1_updated_at, + node2_public_key, + node2_base_fee_mtokens, + node2_cltv_delta, + node2_fee_rate, + node2_is_disabled, + node2_max_htlc_mtokens, + node2_min_htlc_mtokens, + node2_updated_at + ) + VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + capacity = ?, + updated_at = ?, + status = 1, + node1_public_key = ?, + node1_base_fee_mtokens = ?, + node1_cltv_delta = ?, + node1_fee_rate = ?, + node1_is_disabled = ?, + node1_max_htlc_mtokens = ?, + node1_min_htlc_mtokens = ?, + node1_updated_at = ?, + node2_public_key = ?, + node2_base_fee_mtokens = ?, + node2_cltv_delta = ?, + node2_fee_rate = ?, + node2_is_disabled = ?, + node2_max_htlc_mtokens = ?, + node2_min_htlc_mtokens = ?, + node2_updated_at = ? + ;`; - await DB.query(query, [ - Common.channelShortIdToIntegerId(channel.channel_id), - Common.channelIntegerIdToShortId(channel.channel_id), - channel.capacity, - txid, - vout, - Common.utcDateToMysql(channel.last_update), - channel.node1_pub, - policy1.fee_base_msat, - policy1.time_lock_delta, - policy1.fee_rate_milli_msat, - policy1.disabled, - policy1.max_htlc_msat, - policy1.min_htlc, - Common.utcDateToMysql(policy1.last_update), - channel.node2_pub, - policy2.fee_base_msat, - policy2.time_lock_delta, - policy2.fee_rate_milli_msat, - policy2.disabled, - policy2.max_htlc_msat, - policy2.min_htlc, - Common.utcDateToMysql(policy2.last_update), - channel.capacity, - Common.utcDateToMysql(channel.last_update), - channel.node1_pub, - policy1.fee_base_msat, - policy1.time_lock_delta, - policy1.fee_rate_milli_msat, - policy1.disabled, - policy1.max_htlc_msat, - policy1.min_htlc, - Common.utcDateToMysql(policy1.last_update), - channel.node2_pub, - policy2.fee_base_msat, - policy2.time_lock_delta, - policy2.fee_rate_milli_msat, - policy2.disabled, - policy2.max_htlc_msat, - policy2.min_htlc, - Common.utcDateToMysql(policy2.last_update) - ]); - } catch (e) { - logger.err('$saveChannel() error: ' + (e instanceof Error ? e.message : e)); - } + await DB.query(query, [ + Common.channelShortIdToIntegerId(channel.channel_id), + Common.channelIntegerIdToShortId(channel.channel_id), + channel.capacity, + txid, + vout, + Common.utcDateToMysql(channel.last_update), + channel.node1_pub, + policy1.fee_base_msat, + policy1.time_lock_delta, + policy1.fee_rate_milli_msat, + policy1.disabled, + policy1.max_htlc_msat, + policy1.min_htlc, + Common.utcDateToMysql(policy1.last_update), + channel.node2_pub, + policy2.fee_base_msat, + policy2.time_lock_delta, + policy2.fee_rate_milli_msat, + policy2.disabled, + policy2.max_htlc_msat, + policy2.min_htlc, + Common.utcDateToMysql(policy2.last_update), + channel.capacity, + Common.utcDateToMysql(channel.last_update), + channel.node1_pub, + policy1.fee_base_msat, + policy1.time_lock_delta, + policy1.fee_rate_milli_msat, + policy1.disabled, + policy1.max_htlc_msat, + policy1.min_htlc, + Common.utcDateToMysql(policy1.last_update), + channel.node2_pub, + policy2.fee_base_msat, + policy2.time_lock_delta, + policy2.fee_rate_milli_msat, + policy2.disabled, + policy2.max_htlc_msat, + policy2.min_htlc, + Common.utcDateToMysql(policy2.last_update) + ]); } /** diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index 8f2f77534..857ebceb2 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -49,7 +49,7 @@ class NetworkSyncService { logger.err('$runTasks() error: ' + (e instanceof Error ? e.message : e)); } - setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL); + setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL); } /** @@ -84,25 +84,29 @@ class NetworkSyncService { * Update the `channels` table to reflect the current network graph state */ private async $updateChannelsList(channels: ILightningApi.Channel[]): Promise { - let progress = 0; + try { + let progress = 0; - const graphChannelsIds: string[] = []; - for (const channel of channels) { - await channelsApi.$saveChannel(channel); - graphChannelsIds.push(channel.channel_id); - ++progress; + const graphChannelsIds: string[] = []; + for (const channel of channels) { + await channelsApi.$saveChannel(channel); + graphChannelsIds.push(channel.channel_id); + ++progress; - const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); - if (elapsedSeconds > 10) { - logger.info(`Updating channel ${progress}/${channels.length}`); - this.loggerTimer = new Date().getTime() / 1000; + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Updating channel ${progress}/${channels.length}`); + this.loggerTimer = new Date().getTime() / 1000; + } } + + logger.info(`${progress} channels updated`); + + // If a channel if not present in the graph, mark it as inactive + channelsApi.$setChannelsInactive(graphChannelsIds); + } catch (e) { + logger.err(`Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`); } - - logger.info(`${progress} channels updated`); - - // If a channel if not present in the graph, mark it as inactive - channelsApi.$setChannelsInactive(graphChannelsIds); } // This method look up the creation date of the earliest channel of the node From 61e512b8f708ef19b41bd125bcc1e05a20896ef8 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 8 Aug 2022 09:00:11 +0200 Subject: [PATCH 07/17] Refactor the LN backend and add more logs --- .../tasks/lightning/network-sync.service.ts | 57 ++++++++++++++++++- .../tasks/lightning/stats-updater.service.ts | 1 - .../sync-tasks/funding-tx-fetcher.ts | 2 +- 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index 857ebceb2..83c6f21cc 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -73,7 +73,7 @@ class NetworkSyncService { logger.info(`${progress} nodes updated`); // If a channel if not present in the graph, mark it as inactive - // nodesApi.$setNodesInactive(graphNodesPubkeys); + nodesApi.$setNodesInactive(graphNodesPubkeys); if (config.MAXMIND.ENABLED) { $lookupNodeLocation(); @@ -107,6 +107,61 @@ class NetworkSyncService { } catch (e) { logger.err(`Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`); } + + setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL); + } + + /** + * Update the `nodes` table to reflect the current network graph state + */ + private async $updateNodesList(nodes: ILightningApi.Node[]): Promise { + let progress = 0; + + const graphNodesPubkeys: string[] = []; + for (const node of nodes) { + await nodesApi.$saveNode(node); + graphNodesPubkeys.push(node.pub_key); + ++progress; + + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Updating node ${progress}/${nodes.length}`); + this.loggerTimer = new Date().getTime() / 1000; + } + } + logger.info(`${progress} nodes updated`); + + // If a channel if not present in the graph, mark it as inactive + // nodesApi.$setNodesInactive(graphNodesPubkeys); + + if (config.MAXMIND.ENABLED) { + $lookupNodeLocation(); + } + } + + /** + * Update the `channels` table to reflect the current network graph state + */ + private async $updateChannelsList(channels: ILightningApi.Channel[]): Promise { + let progress = 0; + + const graphChannelsIds: string[] = []; + for (const channel of channels) { + await channelsApi.$saveChannel(channel); + graphChannelsIds.push(channel.channel_id); + ++progress; + + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Updating channel ${progress}/${channels.length}`); + this.loggerTimer = new Date().getTime() / 1000; + } + } + + logger.info(`${progress} channels updated`); + + // If a channel if not present in the graph, mark it as inactive + channelsApi.$setChannelsInactive(graphChannelsIds); } // This method look up the creation date of the earliest channel of the node diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index ab5b3cccb..ecb056859 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -24,7 +24,6 @@ class LightningStatsUpdater { private async $logStatsDaily(): Promise { const date = new Date(); Common.setDateMidnight(date); - const networkGraph = await lightningApi.$getNetworkGraph(); LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph); diff --git a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts index 6ee50b8e9..9dbc21c72 100644 --- a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts +++ b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts @@ -115,4 +115,4 @@ class FundingTxFetcher { } } -export default new FundingTxFetcher; \ No newline at end of file +export default new FundingTxFetcher; From 9b974dfbd97bfda692412cbee8cc9f85a24c6276 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 9 Aug 2022 11:17:37 +0200 Subject: [PATCH 08/17] Add nodes.status db field to mark nodes as inactive if needed --- backend/src/api/database-migration.ts | 6 +- .../tasks/lightning/network-sync.service.ts | 55 +------------------ 2 files changed, 7 insertions(+), 54 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 19f523eb3..cfc0092d8 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 35; + private static currentVersion = 36; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -320,6 +320,10 @@ class DatabaseMigration { await this.$executeQuery('DELETE from `lightning_stats` WHERE added > "2021-09-19"'); await this.$executeQuery('ALTER TABLE `lightning_stats` ADD CONSTRAINT added_unique UNIQUE (added);'); } + + if (databaseSchemaVersion < 36 && isBitcoin == true) { + await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"'); + } } /** diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index 83c6f21cc..b87c63031 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -111,59 +111,6 @@ class NetworkSyncService { setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL); } - /** - * Update the `nodes` table to reflect the current network graph state - */ - private async $updateNodesList(nodes: ILightningApi.Node[]): Promise { - let progress = 0; - - const graphNodesPubkeys: string[] = []; - for (const node of nodes) { - await nodesApi.$saveNode(node); - graphNodesPubkeys.push(node.pub_key); - ++progress; - - const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); - if (elapsedSeconds > 10) { - logger.info(`Updating node ${progress}/${nodes.length}`); - this.loggerTimer = new Date().getTime() / 1000; - } - } - logger.info(`${progress} nodes updated`); - - // If a channel if not present in the graph, mark it as inactive - // nodesApi.$setNodesInactive(graphNodesPubkeys); - - if (config.MAXMIND.ENABLED) { - $lookupNodeLocation(); - } - } - - /** - * Update the `channels` table to reflect the current network graph state - */ - private async $updateChannelsList(channels: ILightningApi.Channel[]): Promise { - let progress = 0; - - const graphChannelsIds: string[] = []; - for (const channel of channels) { - await channelsApi.$saveChannel(channel); - graphChannelsIds.push(channel.channel_id); - ++progress; - - const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); - if (elapsedSeconds > 10) { - logger.info(`Updating channel ${progress}/${channels.length}`); - this.loggerTimer = new Date().getTime() / 1000; - } - } - - logger.info(`${progress} channels updated`); - - // If a channel if not present in the graph, mark it as inactive - channelsApi.$setChannelsInactive(graphChannelsIds); - } - // This method look up the creation date of the earliest channel of the node // and update the node to that date in order to get the earliest first seen date private async $updateNodeFirstSeen(): Promise { @@ -253,11 +200,13 @@ class NetworkSyncService { SELECT COUNT(*) FROM nodes WHERE nodes.public_key = channels.node1_public_key + AND nodes.status = 1 ) = 0 OR ( SELECT COUNT(*) FROM nodes WHERE nodes.public_key = channels.node2_public_key + AND nodes.status = 1 ) = 0) `); From a64cb4bbad104da0f2f7240702785ca6ba0d8ba2 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 9 Aug 2022 15:52:24 +0200 Subject: [PATCH 09/17] Make mining pools url configurable --- backend/src/config.ts | 4 ++++ backend/src/tasks/pools-updater.ts | 13 +++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/backend/src/config.ts b/backend/src/config.ts index d4dfc9edd..3d7a05242 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -24,6 +24,8 @@ interface IConfig { USER_AGENT: string; STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug'; AUTOMATIC_BLOCK_REINDEXING: boolean; + POOLS_JSON_URL: string, + POOLS_JSON_TREE_URL: string, }; ESPLORA: { REST_API_URL: string; @@ -135,6 +137,8 @@ const defaults: IConfig = { 'USER_AGENT': 'mempool', 'STDOUT_LOG_MIN_PRIORITY': 'debug', 'AUTOMATIC_BLOCK_REINDEXING': false, + 'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json', + 'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 04d9d5d07..11bb8060f 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -12,14 +12,11 @@ import * as https from 'https'; */ class PoolsUpdater { lastRun: number = 0; - currentSha: any = undefined; - poolsUrl: string = 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json'; - treeUrl: string = 'https://api.github.com/repos/mempool/mining-pools/git/trees/master'; + currentSha: string | undefined = undefined; + poolsUrl: string = config.MEMPOOL.POOLS_JSON_URL; + treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL; - constructor() { - } - - public async updatePoolsJson() { + public async updatePoolsJson(): Promise { if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { return; } @@ -77,7 +74,7 @@ class PoolsUpdater { /** * Fetch our latest pools.json sha from the db */ - private async updateDBSha(githubSha: string) { + private async updateDBSha(githubSha: string): Promise { this.currentSha = githubSha; if (config.DATABASE.ENABLED === true) { try { From aed37afb3e3bbc5cf1ddbb1578984b8bfc74c4b3 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 9 Aug 2022 17:26:14 +0200 Subject: [PATCH 10/17] Add missing config in docker script and sample --- backend/mempool-config.sample.json | 4 +++- docker/README.md | 6 +++++- docker/backend/start.sh | 4 ++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 312d9d18d..e636c9e2d 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -21,7 +21,9 @@ "EXTERNAL_RETRY_INTERVAL": 0, "USER_AGENT": "mempool", "STDOUT_LOG_MIN_PRIORITY": "debug", - "AUTOMATIC_BLOCK_REINDEXING": false + "AUTOMATIC_BLOCK_REINDEXING": false, + "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json", + "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master" }, "CORE_RPC": { "HOST": "127.0.0.1", diff --git a/docker/README.md b/docker/README.md index 14b80c19e..dd42462a7 100644 --- a/docker/README.md +++ b/docker/README.md @@ -102,7 +102,9 @@ Below we list all settings from `mempool-config.json` and the corresponding over "PRICE_FEED_UPDATE_INTERVAL": 600, "USE_SECOND_NODE_FOR_MINFEE": false, "EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"], - "STDOUT_LOG_MIN_PRIORITY": "info" + "STDOUT_LOG_MIN_PRIORITY": "info", + "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json", + "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master" }, ``` @@ -126,6 +128,8 @@ Corresponding `docker-compose.yml` overrides: MEMPOOL_USE_SECOND_NODE_FOR_MINFEE: "" MEMPOOL_EXTERNAL_ASSETS: "" MEMPOOL_STDOUT_LOG_MIN_PRIORITY: "" + MEMPOOL_POOLS_JSON_URL: "" + MEMPOOL_POOLS_JSON_TREE_URL: "" ... ``` diff --git a/docker/backend/start.sh b/docker/backend/start.sh index c31273bb6..3d754c979 100644 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -24,6 +24,8 @@ __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool} __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info} __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false} __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false} +__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=false} +__MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=false} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} @@ -114,6 +116,8 @@ sed -i "s!__MEMPOOL_USER_AGENT__!${__MEMPOOL_USER_AGENT__}!g" mempool-config.jso sed -i "s/__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__/${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}/g" mempool-config.json sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json +sed -i "s/__MEMPOOL_POOLS_JSON_URL__/${__MEMPOOL_POOLS_JSON_URL__}/g" mempool-config.json +sed -i "s/__MEMPOOL_POOLS_JSON_TREE_URL__/${__MEMPOOL_POOLS_JSON_TREE_URL__}/g" mempool-config.json sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json From d6a42cdf6b8d8b1b471b97f51215b51d9e631379 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 10 Aug 2022 11:28:54 +0200 Subject: [PATCH 11/17] Rewrite channels map component using native echart --- .../nodes-channels-map.component.ts | 211 +++++++++++------- 1 file changed, 125 insertions(+), 86 deletions(-) diff --git a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts index e0063858b..b88621bba 100644 --- a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts +++ b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts @@ -20,7 +20,11 @@ export class NodesChannelsMap implements OnInit, OnDestroy { @Input() publicKey: string | undefined; observable$: Observable; - center: number[] | undefined = undefined; + + center: number[] | undefined; + zoom: number | undefined; + channelWidth = 0.6; + channelOpacity = 0.1; chartInstance = undefined; chartOptions: EChartsOption = {}; @@ -42,7 +46,8 @@ export class NodesChannelsMap implements OnInit, OnDestroy { ngOnDestroy(): void {} ngOnInit(): void { - this.center = this.style === 'widget' ? [0, 0, -10] : undefined; + this.center = this.style === 'widget' ? [0, 40] : [0, 5]; + this.zoom = this.style === 'widget' ? 3.5 : 1.3; if (this.style === 'graph') { this.seoService.setTitle($localize`Lightning nodes channels world map`); @@ -69,29 +74,46 @@ export class NodesChannelsMap implements OnInit, OnDestroy { thisNodeGPS = [channel[6], channel[7]]; } - channelsLoc.push([[channel[2], channel[3]], [channel[6], channel[7]]]); + // We add a bit of noise so nodes at the same location are not all + // on top of each other + let random = Math.random() * 2 * Math.PI; + let random2 = Math.random() * 0.01; + if (!nodesPubkeys[channel[0]]) { - nodes.push({ - publicKey: channel[0], - name: channel[1], - value: [channel[2], channel[3]], - }); - nodesPubkeys[channel[0]] = true; + nodes.push([ + channel[2] + random2 * Math.cos(random), + channel[3] + random2 * Math.sin(random), + 1, + channel[0], + channel[1] + ]); + nodesPubkeys[channel[0]] = nodes[nodes.length - 1]; } + + random = Math.random() * 2 * Math.PI; + random2 = Math.random() * 0.01; + if (!nodesPubkeys[channel[4]]) { - nodes.push({ - publicKey: channel[4], - name: channel[5], - value: [channel[6], channel[7]], - }); - nodesPubkeys[channel[4]] = true; + nodes.push([ + channel[6] + random2 * Math.cos(random), + channel[7] + random2 * Math.sin(random), + 1, + channel[4], + channel[5] + ]); + nodesPubkeys[channel[4]] = nodes[nodes.length - 1]; } + + const channelLoc = []; + channelLoc.push(nodesPubkeys[channel[0]].slice(0, 2)); + channelLoc.push(nodesPubkeys[channel[4]].slice(0, 2)); + channelsLoc.push(channelLoc); } if (this.style === 'nodepage' && thisNodeGPS) { - // 1ML 0217890e3aad8d35bc054f43acc00084b25229ecff0ab68debd82883ad65ee8266 - // New York GPS [-74.0068, 40.7123] - // Map center [-20.55, 0, -9.85] - this.center = [thisNodeGPS[0] * -20.55 / -74.0068, 0, thisNodeGPS[1] * -9.85 / 40.7123]; + this.center = [thisNodeGPS[0], thisNodeGPS[1]]; + this.zoom = 10; + this.channelWidth = 1; + this.channelOpacity = 1; } this.prepareChartOptions(nodes, channelsLoc); @@ -115,87 +137,84 @@ export class NodesChannelsMap implements OnInit, OnDestroy { } this.chartOptions = { - silent: this.style === 'widget' ? true : false, + silent: this.style === 'widget', title: title ?? undefined, - geo3D: { - map: 'world', - shading: 'color', + tooltip: {}, + geo: { + animation: false, silent: true, - postEffect: { - enable: true, - bloom: { - intensity: 0.1, - } - }, - viewControl: { - center: this.center, - minDistance: 1, - maxDistance: 60, - distance: this.style === 'widget' ? 22 : this.style === 'nodepage' ? 22 : 60, - alpha: 90, - rotateSensitivity: 0, - panSensitivity: this.style === 'widget' ? 0 : 1, - zoomSensitivity: this.style === 'widget' ? 0 : 0.5, - panMouseButton: this.style === 'widget' ? null : 'left', - rotateMouseButton: undefined, + center: this.center, + zoom: this.zoom, + tooltip: { + show: true }, + map: 'world', + roam: this.style === 'widget' ? false : true, itemStyle: { - color: 'white', - opacity: 0.02, - borderWidth: 1, borderColor: 'black', + color: '#ffffff44' }, - regionHeight: 0.01, + scaleLimit: { + min: 1.3, + max: 100000, + } }, series: [ { - // @ts-ignore - type: 'lines3D', - coordinateSystem: 'geo3D', - blendMode: 'lighter', - lineStyle: { - width: 1, - opacity: ['widget', 'graph'].includes(this.style) ? 0.025 : 1, + large: true, + progressive: 200, + type: 'scatter', + data: nodes, + coordinateSystem: 'geo', + geoIndex: 0, + symbolSize: 4, + tooltip: { + backgroundColor: 'rgba(17, 19, 31, 1)', + borderRadius: 4, + shadowColor: 'rgba(0, 0, 0, 0.5)', + textStyle: { + color: '#b1b1b1', + align: 'left', + }, + borderColor: '#000', + formatter: (value) => { + const data = value.data; + const alias = data[4].length > 0 ? data[4] : data[3].slice(0, 20); + return `${alias}`; + } }, - data: channels + itemStyle: { + color: 'white', + borderColor: 'black', + borderWidth: 2, + opacity: 1, + }, + blendMode: 'lighter', + zlevel: 1, }, { - // @ts-ignore - type: 'scatter3D', - symbol: 'circle', - blendMode: 'lighter', - coordinateSystem: 'geo3D', - symbolSize: 3, - itemStyle: { - color: '#BBFFFF', - opacity: 1, - borderColor: '#FFFFFF00', + large: true, + progressive: 200, + silent: true, + type: 'lines', + coordinateSystem: 'geo', + data: channels, + lineStyle: { + opacity: this.channelOpacity, + width: this.channelWidth, + curveness: 0, + color: '#466d9d', }, - data: nodes, - emphasis: { - label: { - position: 'top', - color: 'white', - fontSize: 16, - formatter: function(value) { - return value.name; - }, - show: true, - } - } - }, + blendMode: 'lighter', + tooltip: { + show: false, + }, + zlevel: 2, + } ] }; } - @HostListener('window:wheel', ['$event']) - onWindowScroll(e): void { - // Not very smooth when using the mouse - if (this.style === 'widget' && e.target.tagName === 'CANVAS') { - window.scrollBy({left: 0, top: e.deltaY, behavior: 'auto'}); - } - } - onChartInit(ec) { if (this.chartInstance !== undefined) { return; @@ -211,14 +230,34 @@ export class NodesChannelsMap implements OnInit, OnDestroy { }); }); } - + this.chartInstance.on('click', (e) => { - if (e.data && e.data.publicKey) { + if (e.data) { this.zone.run(() => { - const url = new RelativeUrlPipe(this.stateService).transform(`/lightning/node/${e.data.publicKey}`); + const url = new RelativeUrlPipe(this.stateService).transform(`/lightning/node/${e.data[3]}`); this.router.navigate([url]); }); } }); + + this.chartInstance.on('georoam', (e) => { + if (!e.zoom || this.style === 'nodepage') { + return; + } + + const speed = 0.005; + const chartOptions = { + series: this.chartOptions.series + }; + + chartOptions.series[1].lineStyle.opacity += e.zoom > 1 ? speed : -speed; + chartOptions.series[1].lineStyle.width += e.zoom > 1 ? speed : -speed; + chartOptions.series[0].symbolSize += e.zoom > 1 ? speed * 10 : -speed * 10; + chartOptions.series[1].lineStyle.opacity = Math.max(0.05, Math.min(0.5, chartOptions.series[1].lineStyle.opacity)); + chartOptions.series[1].lineStyle.width = Math.max(0.5, Math.min(1, chartOptions.series[1].lineStyle.width)); + chartOptions.series[0].symbolSize = Math.max(4, Math.min(5.5, chartOptions.series[0].symbolSize)); + + this.chartInstance.setOption(chartOptions); + }); } } From 48a0a6c7e3202bb64f16630fb3518a50a5b2138d Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 10 Aug 2022 15:09:34 +0200 Subject: [PATCH 12/17] Convert short_id to integer id with clightning backend before returning the graph --- backend/src/api/common.ts | 4 ---- backend/src/api/explorer/channels.api.ts | 2 +- backend/src/api/lightning/clightning/clightning-convert.ts | 4 ++-- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 410d34a01..c97d02ba2 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -202,10 +202,6 @@ export class Common { /** Decodes a channel id returned by lnd as uint64 to a short channel id */ static channelIntegerIdToShortId(id: string): string { - if (config.LIGHTNING.BACKEND === 'cln') { - return id; - } - const n = BigInt(id); return [ n >> 40n, // nth block diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index 55043197d..072449d60 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -420,7 +420,7 @@ class ChannelsApi { const result = await DB.query(` UPDATE channels SET status = 0 - WHERE short_id NOT IN ( + WHERE id NOT IN ( ${graphChannelsIds.map(id => `"${id}"`).join(',')} ) AND status != 2 diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts index 5df51aadc..2bdde31d3 100644 --- a/backend/src/api/lightning/clightning/clightning-convert.ts +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -90,7 +90,7 @@ async function buildFullChannel(clChannelA: any, clChannelB: any): Promise Date: Wed, 10 Aug 2022 22:37:22 +0900 Subject: [PATCH 13/17] [ops] Fix minor issue for /cln/.zshrc in prod installer --- production/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/install b/production/install index db48abaab..f2ecc5dd1 100755 --- a/production/install +++ b/production/install @@ -1199,8 +1199,8 @@ case $OS in osGroupCreate "${CLN_GROUP}" osUserCreate "${CLN_USER}" "${CLN_HOME}" "${CLN_GROUP}" osSudo "${ROOT_USER}" chsh -s `which zsh` "${CLN_USER}" - osSudo "${ROOT_USER}" chown -R "${CLN_USER}:${CLN_GROUP}" "${CLN_HOME}" osSudo "${CLN_USER}" touch "${CLN_HOME}/.zshrc" + osSudo "${ROOT_USER}" chown -R "${CLN_USER}:${CLN_GROUP}" "${CLN_HOME}" echo "[*] Installing Core Lightning package" osPackageInstall ${CLN_PKG} From db41aed44b44c8d330f92352b5e1ad142f10d9bc Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 10 Aug 2022 16:00:12 +0200 Subject: [PATCH 14/17] Show channel on the map in channel page --- backend/src/api/explorer/channels.api.ts | 30 ++++++++++++- backend/src/api/explorer/channels.routes.ts | 3 ++ .../lightning/channel/channel.component.html | 4 +- .../lightning/channel/channel.component.ts | 19 +++++++- .../nodes-channels-map.component.ts | 43 +++++++++++++++++-- 5 files changed, 91 insertions(+), 8 deletions(-) diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index 55043197d..dec9af1e5 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -96,7 +96,31 @@ class ChannelsApi { public async $getChannel(id: string): Promise { try { - const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*, ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key WHERE (ns1.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node1_public_key) AND ns2.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node2_public_key)) AND channels.id = ?`; + const query = ` + SELECT n1.alias AS alias_left, n1.longitude as node1_longitude, n1.latitude as node1_latitude, + n2.alias AS alias_right, n2.longitude as node2_longitude, n2.latitude as node2_latitude, + channels.*, + ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right + FROM channels + LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key + LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key + LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key + LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key + WHERE ( + ns1.id = ( + SELECT MAX(id) + FROM node_stats + WHERE public_key = channels.node1_public_key + ) + AND ns2.id = ( + SELECT MAX(id) + FROM node_stats + WHERE public_key = channels.node2_public_key + ) + ) + AND channels.id = ? + `; + const [rows]: any = await DB.query(query, [id]); if (rows[0]) { return this.convertChannel(rows[0]); @@ -289,6 +313,8 @@ class ChannelsApi { 'max_htlc_mtokens': channel.node1_max_htlc_mtokens, 'min_htlc_mtokens': channel.node1_min_htlc_mtokens, 'updated_at': channel.node1_updated_at, + 'longitude': channel.node1_longitude, + 'latitude': channel.node1_latitude, }, 'node_right': { 'alias': channel.alias_right, @@ -302,6 +328,8 @@ class ChannelsApi { 'max_htlc_mtokens': channel.node2_max_htlc_mtokens, 'min_htlc_mtokens': channel.node2_min_htlc_mtokens, 'updated_at': channel.node2_updated_at, + 'longitude': channel.node2_longitude, + 'latitude': channel.node2_latitude, }, }; } diff --git a/backend/src/api/explorer/channels.routes.ts b/backend/src/api/explorer/channels.routes.ts index bbb075aa6..09c3da668 100644 --- a/backend/src/api/explorer/channels.routes.ts +++ b/backend/src/api/explorer/channels.routes.ts @@ -32,6 +32,9 @@ class ChannelsRoutes { res.status(404).send('Channel not found'); return; } + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); res.json(channel); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); diff --git a/frontend/src/app/lightning/channel/channel.component.html b/frontend/src/app/lightning/channel/channel.component.html index 41c2f3254..ec49c78a0 100644 --- a/frontend/src/app/lightning/channel/channel.component.html +++ b/frontend/src/app/lightning/channel/channel.component.html @@ -14,7 +14,9 @@
-
+ + +
diff --git a/frontend/src/app/lightning/channel/channel.component.ts b/frontend/src/app/lightning/channel/channel.component.ts index bc66f7180..9c3cdd57e 100644 --- a/frontend/src/app/lightning/channel/channel.component.ts +++ b/frontend/src/app/lightning/channel/channel.component.ts @@ -1,7 +1,7 @@ import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; import { ActivatedRoute, ParamMap } from '@angular/router'; import { Observable, of } from 'rxjs'; -import { catchError, switchMap } from 'rxjs/operators'; +import { catchError, switchMap, tap } from 'rxjs/operators'; import { SeoService } from 'src/app/services/seo.service'; import { LightningApiService } from '../lightning-api.service'; @@ -14,6 +14,7 @@ import { LightningApiService } from '../lightning-api.service'; export class ChannelComponent implements OnInit { channel$: Observable; error: any = null; + channelGeo: number[] = []; constructor( private lightningApiService: LightningApiService, @@ -29,9 +30,23 @@ export class ChannelComponent implements OnInit { this.seoService.setTitle(`Channel: ${params.get('short_id')}`); return this.lightningApiService.getChannel$(params.get('short_id')) .pipe( + tap((data) => { + if (!data.node_left.longitude || !data.node_left.latitude || + !data.node_right.longitude || !data.node_right.latitude) { + this.channelGeo = []; + } else { + this.channelGeo = [ + data.node_left.public_key, + data.node_left.alias, + data.node_left.longitude, data.node_left.latitude, + data.node_right.public_key, + data.node_right.alias, + data.node_right.longitude, data.node_right.latitude, + ]; + } + }), catchError((err) => { this.error = err; - console.log(this.error); return of(null); }) ); diff --git a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts index b88621bba..43da510f0 100644 --- a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts +++ b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts @@ -16,8 +16,9 @@ import 'echarts-gl'; changeDetection: ChangeDetectionStrategy.OnPush, }) export class NodesChannelsMap implements OnInit, OnDestroy { - @Input() style: 'graph' | 'nodepage' | 'widget' = 'graph'; + @Input() style: 'graph' | 'nodepage' | 'widget' | 'channelpage' = 'graph'; @Input() publicKey: string | undefined; + @Input() channel: any[] = []; observable$: Observable; @@ -25,6 +26,8 @@ export class NodesChannelsMap implements OnInit, OnDestroy { zoom: number | undefined; channelWidth = 0.6; channelOpacity = 0.1; + channelColor = '#466d9d'; + channelCurve = 0; chartInstance = undefined; chartOptions: EChartsOption = {}; @@ -67,13 +70,29 @@ export class NodesChannelsMap implements OnInit, OnDestroy { const nodes = []; const nodesPubkeys = {}; let thisNodeGPS: number[] | undefined = undefined; - for (const channel of data[1]) { + + let geoloc = data[1]; + if (this.style === 'channelpage') { + if (this.channel.length === 0) { + geoloc = []; + } else { + geoloc = [this.channel]; + } + } + for (const channel of geoloc) { if (!thisNodeGPS && data[2] === channel[0]) { thisNodeGPS = [channel[2], channel[3]]; } else if (!thisNodeGPS && data[2] === channel[4]) { thisNodeGPS = [channel[6], channel[7]]; } + // 0 - node1 pubkey + // 1 - node1 alias + // 2,3 - node1 GPS + // 4 - node2 pubkey + // 5 - node2 alias + // 6,7 - node2 GPS + // We add a bit of noise so nodes at the same location are not all // on top of each other let random = Math.random() * 2 * Math.PI; @@ -115,6 +134,22 @@ export class NodesChannelsMap implements OnInit, OnDestroy { this.channelWidth = 1; this.channelOpacity = 1; } + if (this.style === 'channelpage' && this.channel.length > 0) { + this.channelWidth = 2; + this.channelOpacity = 1; + this.channelColor = '#bafcff'; + this.channelCurve = 0.1; + this.center = [ + (this.channel[2] + this.channel[6]) / 2, + (this.channel[3] + this.channel[7]) / 2 + ]; + const distance = Math.sqrt( + Math.pow(this.channel[7] - this.channel[3], 2) + + Math.pow(this.channel[6] - this.channel[2], 2) + ); + + this.zoom = -0.05 * distance + 8; + } this.prepareChartOptions(nodes, channelsLoc); })); @@ -202,8 +237,8 @@ export class NodesChannelsMap implements OnInit, OnDestroy { lineStyle: { opacity: this.channelOpacity, width: this.channelWidth, - curveness: 0, - color: '#466d9d', + curveness: this.channelCurve, + color: this.channelColor, }, blendMode: 'lighter', tooltip: { From dc7231537f7c2342c818f80f867e5bc3d6bde03e Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 10 Aug 2022 16:58:29 +0200 Subject: [PATCH 15/17] Refactor channel id conversion utils --- backend/src/api/common.ts | 17 ++++++++++++----- .../lightning/clightning/clightning-convert.ts | 13 +++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index c97d02ba2..8d9de53c9 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -1,6 +1,5 @@ import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces'; import config from '../config'; -import { convertChannelId } from './lightning/clightning/clightning-convert'; export class Common { static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ? '144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49' @@ -193,15 +192,23 @@ export class Common { date.setUTCMilliseconds(0); } - static channelShortIdToIntegerId(id: string): string { - if (config.LIGHTNING.BACKEND === 'lnd') { - return id; + static channelShortIdToIntegerId(channelId: string): string { + if (channelId.indexOf('x') === -1) { // Already an integer id + return channelId; } - return convertChannelId(id); + if (channelId.indexOf('/') !== -1) { // Topology import + channelId = channelId.slice(0, -2); + } + const s = channelId.split('x').map(part => BigInt(part)); + return ((s[0] << 40n) | (s[1] << 16n) | s[2]).toString(); } /** Decodes a channel id returned by lnd as uint64 to a short channel id */ static channelIntegerIdToShortId(id: string): string { + if (id.indexOf('x') !== -1) { // Already a short id + return id; + } + const n = BigInt(id); return [ n >> 40n, // nth block diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts index 2bdde31d3..15d8d8766 100644 --- a/backend/src/api/lightning/clightning/clightning-convert.ts +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -1,6 +1,7 @@ import { ILightningApi } from '../lightning-api.interface'; import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher'; import logger from '../../../logger'; +import { Common } from '../../common'; /** * Convert a clightning "listnode" entry to a lnd node entry @@ -70,14 +71,6 @@ export async function convertAndmergeBidirectionalChannels(clChannels: any[]): P return consolidatedChannelList; } -export function convertChannelId(channelId): string { - if (channelId.indexOf('/') !== -1) { - channelId = channelId.slice(0, -2); - } - const s = channelId.split('x').map(part => BigInt(part)); - return ((s[0] << 40n) | (s[1] << 16n) | s[2]).toString(); -} - /** * Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format * In this case, clightning knows the channel policy for both nodes @@ -90,7 +83,7 @@ async function buildFullChannel(clChannelA: any, clChannelB: any): Promise Date: Wed, 10 Aug 2022 17:03:11 +0200 Subject: [PATCH 16/17] Fix channel rendering issue --- .../nodes-channels-map/nodes-channels-map.component.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts index 43da510f0..c055ad415 100644 --- a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts +++ b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts @@ -197,7 +197,6 @@ export class NodesChannelsMap implements OnInit, OnDestroy { series: [ { large: true, - progressive: 200, type: 'scatter', data: nodes, coordinateSystem: 'geo', @@ -228,7 +227,7 @@ export class NodesChannelsMap implements OnInit, OnDestroy { zlevel: 1, }, { - large: true, + large: false, progressive: 200, silent: true, type: 'lines', From 1d71e26a12c1c1875bf5015643139d555b4be380 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 11 Aug 2022 10:19:13 +0200 Subject: [PATCH 17/17] Add ISP chart in the dashboard - Fix mobile layout - Start polishing --- backend/src/api/explorer/nodes.api.ts | 2 +- backend/src/api/explorer/nodes.routes.ts | 2 +- .../pool-ranking/pool-ranking.component.html | 6 +- .../lightning-dashboard.component.html | 24 +++- .../lightning-dashboard.component.scss | 34 +++++- .../nodes-channels-map.component.scss | 13 ++- .../nodes-channels-map.component.ts | 12 +- .../nodes-list/nodes-list.component.html | 8 +- .../nodes-list/nodes-list.component.scss | 11 ++ .../nodes-list/nodes-list.component.ts | 1 + .../nodes-per-isp-chart.component.html | 64 +++++++++-- .../nodes-per-isp-chart.component.scss | 108 +++++++++++++++++- .../nodes-per-isp-chart.component.ts | 58 +++++++--- .../lightning-statistics-chart.component.scss | 3 +- 14 files changed, 292 insertions(+), 54 deletions(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index d4857a3a4..2d838524e 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -168,7 +168,7 @@ class NodesApi { } } - public async $getNodesISP(groupBy: string, showTor: boolean) { + public async $getNodesISPRanking(groupBy: string, showTor: boolean) { try { const orderBy = groupBy === 'capacity' ? `CAST(SUM(capacity) as INT)` : `COUNT(DISTINCT nodes.public_key)`; diff --git a/backend/src/api/explorer/nodes.routes.ts b/backend/src/api/explorer/nodes.routes.ts index a850b6a09..5e0f95acb 100644 --- a/backend/src/api/explorer/nodes.routes.ts +++ b/backend/src/api/explorer/nodes.routes.ts @@ -79,7 +79,7 @@ class NodesRoutes { return; } - const nodesPerAs = await nodesApi.$getNodesISP(groupBy, showTor); + const nodesPerAs = await nodesApi.$getNodesISPRanking(groupBy, showTor); res.header('Pragma', 'public'); res.header('Cache-control', 'public'); diff --git a/frontend/src/app/components/pool-ranking/pool-ranking.component.html b/frontend/src/app/components/pool-ranking/pool-ranking.component.html index ae1bb2eb2..1888b3eee 100644 --- a/frontend/src/app/components/pool-ranking/pool-ranking.component.html +++ b/frontend/src/app/components/pool-ranking/pool-ranking.component.html @@ -76,10 +76,8 @@
-
-
-
+
diff --git a/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html b/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html index 999183e09..3fbbaa4e2 100644 --- a/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html +++ b/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html @@ -4,6 +4,7 @@
+
Network Statistics  @@ -17,6 +18,7 @@
+
Channels Statistics  @@ -30,18 +32,28 @@
-
+ +
+
+
+ +
View more »
+
+
+
+ +
-
-
+
+
View more »
@@ -52,7 +64,7 @@
Top Capacity Nodes
- +
@@ -62,7 +74,7 @@
Most Connected Nodes
- +
diff --git a/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.scss b/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.scss index 4fdadd57b..303591974 100644 --- a/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.scss +++ b/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.scss @@ -14,6 +14,13 @@ background-color: #1d1f31; } +.graph-card { + height: 100%; + @media (min-width: 992px) { + height: 385px; + } +} + .card-title { font-size: 1rem; color: #4a68b9; @@ -22,9 +29,6 @@ color: #4a68b9; } -.card-body { - padding: 1.25rem 1rem 0.75rem 1rem; -} .card-body.pool-ranking { padding: 1.25rem 0.25rem 0.75rem 0.25rem; } @@ -32,6 +36,21 @@ font-size: 22px; } +#blockchain-container { + position: relative; + overflow-x: scroll; + overflow-y: hidden; + scrollbar-width: none; + -ms-overflow-style: none; +} + +#blockchain-container::-webkit-scrollbar { + display: none; +} + +.fade-border { + -webkit-mask-image: linear-gradient(to right, transparent 0%, black 10%, black 80%, transparent 100%) +} .main-title { position: relative; @@ -45,7 +64,7 @@ } .more-padding { - padding: 18px; + padding: 24px 20px !important; } .card-wrapper { @@ -78,3 +97,10 @@ .card-text { font-size: 22px; } + +.title-link, .title-link:hover, .title-link:focus, .title-link:active { + display: block; + margin-bottom: 10px; + text-decoration: none; + color: inherit; +} \ No newline at end of file diff --git a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss index 7e6b9f050..578bffc3a 100644 --- a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss +++ b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss @@ -30,21 +30,28 @@ } .widget { - width: 99vw; + width: 90vw; + margin-left: auto; + margin-right: auto; height: 250px; -webkit-mask: linear-gradient(0deg, #11131f00 5%, #11131fff 25%); + @media (max-width: 767.98px) { + width: 100vw; + } } .widget > .chart { - -webkit-mask: linear-gradient(180deg, #11131f00 0%, #11131fff 20%); min-height: 250px; + -webkit-mask: linear-gradient(180deg, #11131f00 0%, #11131fff 20%); + @media (max-width: 767.98px) { + padding-bottom: 0px; + } } .chart { min-height: 500px; width: 100%; height: 100%; - padding-right: 10px; @media (max-width: 992px) { padding-bottom: 25px; } diff --git a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts index 43da510f0..92c566156 100644 --- a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts +++ b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts @@ -8,6 +8,7 @@ import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url. import { StateService } from 'src/app/services/state.service'; import { EChartsOption, registerMap } from 'echarts'; import 'echarts-gl'; +import { isMobile } from 'src/app/shared/common.utils'; @Component({ selector: 'app-nodes-channels-map', @@ -50,8 +51,15 @@ export class NodesChannelsMap implements OnInit, OnDestroy { ngOnInit(): void { this.center = this.style === 'widget' ? [0, 40] : [0, 5]; - this.zoom = this.style === 'widget' ? 3.5 : 1.3; - + this.zoom = 1.3; + if (this.style === 'widget' && !isMobile()) { + this.zoom = 3.5; + } + if (this.style === 'widget' && isMobile()) { + this.zoom = 1.4; + this.center = [0, 10]; + } + if (this.style === 'graph') { this.seoService.setTitle($localize`Lightning nodes channels world map`); } diff --git a/frontend/src/app/lightning/nodes-list/nodes-list.component.html b/frontend/src/app/lightning/nodes-list/nodes-list.component.html index 65a7a558a..d21f0b30a 100644 --- a/frontend/src/app/lightning/nodes-list/nodes-list.component.html +++ b/frontend/src/app/lightning/nodes-list/nodes-list.component.html @@ -3,18 +3,18 @@ - - + + - - diff --git a/frontend/src/app/lightning/nodes-list/nodes-list.component.scss b/frontend/src/app/lightning/nodes-list/nodes-list.component.scss index e69de29bb..85a1339ea 100644 --- a/frontend/src/app/lightning/nodes-list/nodes-list.component.scss +++ b/frontend/src/app/lightning/nodes-list/nodes-list.component.scss @@ -0,0 +1,11 @@ +.capacity.mobile-channels { + @media (max-width: 767.98px) { + display: none; + } +} + +.channels.mobile-capacity { + @media (max-width: 767.98px) { + display: none; + } +} \ No newline at end of file diff --git a/frontend/src/app/lightning/nodes-list/nodes-list.component.ts b/frontend/src/app/lightning/nodes-list/nodes-list.component.ts index d6d05833e..9b9e2d594 100644 --- a/frontend/src/app/lightning/nodes-list/nodes-list.component.ts +++ b/frontend/src/app/lightning/nodes-list/nodes-list.component.ts @@ -9,6 +9,7 @@ import { Observable } from 'rxjs'; }) export class NodesListComponent implements OnInit { @Input() nodes$: Observable; + @Input() show: string; constructor() { } diff --git a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html index 23f54bbba..5e14ac67b 100644 --- a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html +++ b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html @@ -1,6 +1,29 @@ -
+
-
+
+
+
+
Tagged ISPs
+

+ {{ stats.taggedISP }} +

+
+
+
Tagged capacity
+

+ +

+
+
+
Tagged nodes
+

+ {{ stats.taggedNodeCount }} +

+
+
+
+ +
Lightning nodes per ISP
-
-
-
-
+
+
-
+
-
AliasCapacityChannelsCapacityChannels
{{ node.alias }} + + {{ node.channels | number }}
+
@@ -39,7 +60,7 @@ - +
Rank
{{ asEntry.rank }} {{ asEntry.name }} @@ -54,3 +75,26 @@ + + +
+
+
Tagged ISPs
+

+ +

+
+
+
Tagged capacity
+

+ +

+
+
+
Tagged nodes
+

+ +

+
+
+ diff --git a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss index 10ad39372..874d901b2 100644 --- a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss +++ b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss @@ -22,7 +22,40 @@ max-height: 400px; @media (max-width: 767.98px) { max-height: 230px; - margin-top: -35px; + margin-top: -40px; + } +} +.chart-widget { + width: 100%; + height: 100%; + height: 240px; + @media (max-width: 485px) { + max-height: 200px; + } +} + +.formRadioGroup { + margin-top: 6px; + display: flex; + flex-direction: column; + @media (min-width: 991px) { + position: relative; + top: -65px; + } + @media (min-width: 830px) and (max-width: 991px) { + position: relative; + top: 0px; + } + @media (min-width: 830px) { + flex-direction: row; + float: right; + margin-top: 0px; + } + .btn-sm { + font-size: 9px; + @media (min-width: 830px) { + font-size: 14px; + } } } @@ -35,6 +68,79 @@ }; } +@media (max-width: 767.98px) { + .pools-table th, + .pools-table td { + padding: .3em !important; + } +} + +.loadingGraphs { + position: absolute; + top: 50%; + left: calc(50% - 15px); + z-index: 100; +} + +.pool-distribution { + min-height: 56px; + display: block; + @media (min-width: 485px) { + display: flex; + flex-direction: row; + } + h5 { + margin-bottom: 5px; + } + .item { + max-width: 160px; + width: 50%; + display: inline-block; + margin: 0px auto 20px; + &:nth-child(2) { + order: 2; + @media (min-width: 485px) { + order: 3; + } + } + &:nth-child(3) { + width: 50%; + order: 3; + @media (min-width: 485px) { + order: 2; + display: block; + } + @media (min-width: 768px) { + display: none; + } + @media (min-width: 992px) { + display: block; + } + } + .card-title { + font-size: 1rem; + color: #4a68b9; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + .card-text { + font-size: 18px; + span { + color: #ffffff66; + font-size: 12px; + } + } + } +} + +.skeleton-loader { + width: 100%; + display: block; + max-width: 80px; + margin: 15px auto 3px; +} + .rank { width: 15%; @media (max-width: 576px) { diff --git a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.ts b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.ts index 6b9d41e74..cd8a72884 100644 --- a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.ts +++ b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.ts @@ -1,11 +1,12 @@ -import { ChangeDetectionStrategy, Component, OnInit, HostBinding, NgZone } from '@angular/core'; +import { ChangeDetectionStrategy, Component, OnInit, HostBinding, NgZone, Input } from '@angular/core'; import { Router } from '@angular/router'; import { EChartsOption, PieSeriesOption } from 'echarts'; -import { combineLatest, map, Observable, share, Subject, switchMap, tap } from 'rxjs'; +import { combineLatest, map, Observable, share, startWith, Subject, switchMap, tap } from 'rxjs'; import { chartColors } from 'src/app/app.constants'; import { ApiService } from 'src/app/services/api.service'; import { SeoService } from 'src/app/services/seo.service'; import { StateService } from 'src/app/services/state.service'; +import { isMobile } from 'src/app/shared/common.utils'; import { download } from 'src/app/shared/graphs.utils'; import { AmountShortenerPipe } from 'src/app/shared/pipes/amount-shortener.pipe'; import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.pipe'; @@ -17,6 +18,8 @@ import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url. changeDetection: ChangeDetectionStrategy.OnPush, }) export class NodesPerISPChartComponent implements OnInit { + @Input() widget: boolean = false; + isLoading = true; chartOptions: EChartsOption = {}; chartInitOptions = { @@ -46,7 +49,11 @@ export class NodesPerISPChartComponent implements OnInit { this.seoService.setTitle($localize`Lightning nodes per ISP`); this.showTorObservable$ = this.showTorSubject.asObservable(); - this.nodesPerAsObservable$ = combineLatest([this.groupBySubject, this.showTorSubject]) + + this.nodesPerAsObservable$ = combineLatest([ + this.groupBySubject.pipe(startWith(false)), + this.showTorSubject.pipe(startWith(false)), + ]) .pipe( switchMap((selectedFilters) => { return this.apiService.getNodesPerAs( @@ -62,23 +69,41 @@ export class NodesPerISPChartComponent implements OnInit { for (let i = 0; i < data.length; ++i) { data[i].rank = i + 1; } - return data.slice(0, 100); + return { + taggedISP: data.length, + taggedCapacity: data.reduce((partialSum, isp) => partialSum + isp.capacity, 0), + taggedNodeCount: data.reduce((partialSum, isp) => partialSum + isp.count, 0), + data: data.slice(0, 100), + }; }) ); }), share() ); + + if (this.widget) { + this.showTorSubject.next(false); + this.groupBySubject.next(false); + } } generateChartSerieData(as): PieSeriesOption[] { - const shareThreshold = this.isMobile() ? 2 : 0.5; + let shareThreshold = 0.5; + if (this.widget && isMobile() || isMobile()) { + shareThreshold = 1; + } else if (this.widget) { + shareThreshold = 0.75; + } + const data: object[] = []; let totalShareOther = 0; let totalNodeOther = 0; let edgeDistance: string | number = '10%'; - if (this.isMobile()) { + if (isMobile() && this.widget) { edgeDistance = 0; + } else if (isMobile() && !this.widget || this.widget) { + edgeDistance = 10; } as.forEach((as) => { @@ -92,15 +117,16 @@ export class NodesPerISPChartComponent implements OnInit { color: as.ispId === null ? '#7D4698' : undefined, }, value: as.share, - name: as.name + (this.isMobile() ? `` : ` (${as.share}%)`), + name: as.name + (isMobile() || this.widget ? `` : ` (${as.share}%)`), label: { overflow: 'truncate', + width: isMobile() ? 75 : this.widget ? 125 : 250, color: '#b1b1b1', alignTo: 'edge', edgeDistance: edgeDistance, }, tooltip: { - show: !this.isMobile(), + show: !isMobile(), backgroundColor: 'rgba(17, 19, 31, 1)', borderRadius: 4, shadowColor: 'rgba(0, 0, 0, 0.5)', @@ -125,7 +151,7 @@ export class NodesPerISPChartComponent implements OnInit { color: 'grey', }, value: totalShareOther, - name: 'Other' + (this.isMobile() ? `` : ` (${totalShareOther.toFixed(2)}%)`), + name: 'Other' + (isMobile() || this.widget ? `` : ` (${totalShareOther.toFixed(2)}%)`), label: { overflow: 'truncate', color: '#b1b1b1', @@ -153,7 +179,7 @@ export class NodesPerISPChartComponent implements OnInit { prepareChartOptions(as): void { let pieSize = ['20%', '80%']; // Desktop - if (this.isMobile()) { + if (isMobile() && !this.widget) { pieSize = ['15%', '60%']; } @@ -177,8 +203,8 @@ export class NodesPerISPChartComponent implements OnInit { lineStyle: { width: 2, }, - length: this.isMobile() ? 1 : 20, - length2: this.isMobile() ? 1 : undefined, + length: isMobile() ? 1 : 20, + length2: isMobile() ? 1 : undefined, }, label: { fontSize: 14, @@ -204,10 +230,6 @@ export class NodesPerISPChartComponent implements OnInit { }; } - isMobile(): boolean { - return (window.innerWidth <= 767.98); - } - onChartInit(ec): void { if (this.chartInstance !== undefined) { return; @@ -244,5 +266,9 @@ export class NodesPerISPChartComponent implements OnInit { onGroupToggleStatusChanged(e): void { this.groupBySubject.next(e); } + + isEllipsisActive(e) { + return (e.offsetWidth < e.scrollWidth); + } } diff --git a/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.scss b/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.scss index fa044a4d6..5f59539e3 100644 --- a/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.scss +++ b/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.scss @@ -51,8 +51,7 @@ } .chart-widget { width: 100%; - height: 100%; - max-height: 270px; + height: 320px; } .formRadioGroup {