Merge branch 'master' into nofollow-api-links

This commit is contained in:
wiz 2022-08-11 20:09:12 +09:00 committed by GitHub
commit 631de8d85f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
38 changed files with 1103 additions and 488 deletions

View File

@ -21,7 +21,9 @@
"EXTERNAL_RETRY_INTERVAL": 0, "EXTERNAL_RETRY_INTERVAL": 0,
"USER_AGENT": "mempool", "USER_AGENT": "mempool",
"STDOUT_LOG_MIN_PRIORITY": "debug", "STDOUT_LOG_MIN_PRIORITY": "debug",
"AUTOMATIC_BLOCK_REINDEXING": false "AUTOMATIC_BLOCK_REINDEXING": false,
"POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
"POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master"
}, },
"CORE_RPC": { "CORE_RPC": {
"HOST": "127.0.0.1", "HOST": "127.0.0.1",

View File

@ -22,6 +22,8 @@ import poolsParser from './pools-parser';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import mining from './mining/mining'; import mining from './mining/mining';
import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
import PricesRepository from '../repositories/PricesRepository';
import priceUpdater from '../tasks/price-updater';
class Blocks { class Blocks {
private blocks: BlockExtended[] = []; private blocks: BlockExtended[] = [];
@ -457,6 +459,19 @@ class Blocks {
} }
await blocksRepository.$saveBlockInDatabase(blockExtended); await blocksRepository.$saveBlockInDatabase(blockExtended);
const lastestPriceId = await PricesRepository.$getLatestPriceId();
if (priceUpdater.historyInserted === true && lastestPriceId !== null) {
await blocksRepository.$saveBlockPrices([{
height: blockExtended.height,
priceId: lastestPriceId,
}]);
} else {
logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`)
setTimeout(() => {
indexer.runSingleTask('blocksPrices');
}, 10000);
}
// Save blocks summary for visualization if it's enabled // Save blocks summary for visualization if it's enabled
if (Common.blocksSummariesIndexingEnabled() === true) { if (Common.blocksSummariesIndexingEnabled() === true) {
await this.$getStrippedBlockTransactions(blockExtended.id, true); await this.$getStrippedBlockTransactions(blockExtended.id, true);

View File

@ -184,4 +184,41 @@ export class Common {
config.MEMPOOL.BLOCKS_SUMMARIES_INDEXING === true config.MEMPOOL.BLOCKS_SUMMARIES_INDEXING === true
); );
} }
static setDateMidnight(date: Date): void {
date.setUTCHours(0);
date.setUTCMinutes(0);
date.setUTCSeconds(0);
date.setUTCMilliseconds(0);
}
static channelShortIdToIntegerId(channelId: string): string {
if (channelId.indexOf('x') === -1) { // Already an integer id
return channelId;
}
if (channelId.indexOf('/') !== -1) { // Topology import
channelId = channelId.slice(0, -2);
}
const s = channelId.split('x').map(part => BigInt(part));
return ((s[0] << 40n) | (s[1] << 16n) | s[2]).toString();
}
/** Decodes a channel id returned by lnd as uint64 to a short channel id */
static channelIntegerIdToShortId(id: string): string {
if (id.indexOf('x') !== -1) { // Already a short id
return id;
}
const n = BigInt(id);
return [
n >> 40n, // nth block
(n >> 16n) & 0xffffffn, // nth tx of the block
n & 0xffffn // nth output of the tx
].join('x');
}
static utcDateToMysql(date?: number): string {
const d = new Date((date || 0) * 1000);
return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
}
} }

View File

@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common'; import { Common } from './common';
class DatabaseMigration { class DatabaseMigration {
private static currentVersion = 35; private static currentVersion = 36;
private queryTimeout = 120000; private queryTimeout = 120000;
private statisticsAddedIndexed = false; private statisticsAddedIndexed = false;
private uniqueLogs: string[] = []; private uniqueLogs: string[] = [];
@ -320,6 +320,10 @@ class DatabaseMigration {
await this.$executeQuery('DELETE from `lightning_stats` WHERE added > "2021-09-19"'); await this.$executeQuery('DELETE from `lightning_stats` WHERE added > "2021-09-19"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD CONSTRAINT added_unique UNIQUE (added);'); await this.$executeQuery('ALTER TABLE `lightning_stats` ADD CONSTRAINT added_unique UNIQUE (added);');
} }
if (databaseSchemaVersion < 36 && isBitcoin == true) {
await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"');
}
} }
/** /**

View File

@ -1,6 +1,9 @@
import logger from '../../logger'; import logger from '../../logger';
import DB from '../../database'; import DB from '../../database';
import nodesApi from './nodes.api'; import nodesApi from './nodes.api';
import { ResultSetHeader } from 'mysql2';
import { ILightningApi } from '../lightning/lightning-api.interface';
import { Common } from '../common';
class ChannelsApi { class ChannelsApi {
public async $getAllChannels(): Promise<any[]> { public async $getAllChannels(): Promise<any[]> {
@ -93,7 +96,31 @@ class ChannelsApi {
public async $getChannel(id: string): Promise<any> { public async $getChannel(id: string): Promise<any> {
try { try {
const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*, ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key WHERE (ns1.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node1_public_key) AND ns2.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node2_public_key)) AND channels.id = ?`; const query = `
SELECT n1.alias AS alias_left, n1.longitude as node1_longitude, n1.latitude as node1_latitude,
n2.alias AS alias_right, n2.longitude as node2_longitude, n2.latitude as node2_latitude,
channels.*,
ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right
FROM channels
LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key
LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key
WHERE (
ns1.id = (
SELECT MAX(id)
FROM node_stats
WHERE public_key = channels.node1_public_key
)
AND ns2.id = (
SELECT MAX(id)
FROM node_stats
WHERE public_key = channels.node2_public_key
)
)
AND channels.id = ?
`;
const [rows]: any = await DB.query(query, [id]); const [rows]: any = await DB.query(query, [id]);
if (rows[0]) { if (rows[0]) {
return this.convertChannel(rows[0]); return this.convertChannel(rows[0]);
@ -286,6 +313,8 @@ class ChannelsApi {
'max_htlc_mtokens': channel.node1_max_htlc_mtokens, 'max_htlc_mtokens': channel.node1_max_htlc_mtokens,
'min_htlc_mtokens': channel.node1_min_htlc_mtokens, 'min_htlc_mtokens': channel.node1_min_htlc_mtokens,
'updated_at': channel.node1_updated_at, 'updated_at': channel.node1_updated_at,
'longitude': channel.node1_longitude,
'latitude': channel.node1_latitude,
}, },
'node_right': { 'node_right': {
'alias': channel.alias_right, 'alias': channel.alias_right,
@ -299,9 +328,140 @@ class ChannelsApi {
'max_htlc_mtokens': channel.node2_max_htlc_mtokens, 'max_htlc_mtokens': channel.node2_max_htlc_mtokens,
'min_htlc_mtokens': channel.node2_min_htlc_mtokens, 'min_htlc_mtokens': channel.node2_min_htlc_mtokens,
'updated_at': channel.node2_updated_at, 'updated_at': channel.node2_updated_at,
'longitude': channel.node2_longitude,
'latitude': channel.node2_latitude,
}, },
}; };
} }
/**
* Save or update a channel present in the graph
*/
public async $saveChannel(channel: ILightningApi.Channel): Promise<void> {
const [ txid, vout ] = channel.chan_point.split(':');
const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
const query = `INSERT INTO channels
(
id,
short_id,
capacity,
transaction_id,
transaction_vout,
updated_at,
status,
node1_public_key,
node1_base_fee_mtokens,
node1_cltv_delta,
node1_fee_rate,
node1_is_disabled,
node1_max_htlc_mtokens,
node1_min_htlc_mtokens,
node1_updated_at,
node2_public_key,
node2_base_fee_mtokens,
node2_cltv_delta,
node2_fee_rate,
node2_is_disabled,
node2_max_htlc_mtokens,
node2_min_htlc_mtokens,
node2_updated_at
)
VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
capacity = ?,
updated_at = ?,
status = 1,
node1_public_key = ?,
node1_base_fee_mtokens = ?,
node1_cltv_delta = ?,
node1_fee_rate = ?,
node1_is_disabled = ?,
node1_max_htlc_mtokens = ?,
node1_min_htlc_mtokens = ?,
node1_updated_at = ?,
node2_public_key = ?,
node2_base_fee_mtokens = ?,
node2_cltv_delta = ?,
node2_fee_rate = ?,
node2_is_disabled = ?,
node2_max_htlc_mtokens = ?,
node2_min_htlc_mtokens = ?,
node2_updated_at = ?
;`;
await DB.query(query, [
Common.channelShortIdToIntegerId(channel.channel_id),
Common.channelIntegerIdToShortId(channel.channel_id),
channel.capacity,
txid,
vout,
Common.utcDateToMysql(channel.last_update),
channel.node1_pub,
policy1.fee_base_msat,
policy1.time_lock_delta,
policy1.fee_rate_milli_msat,
policy1.disabled,
policy1.max_htlc_msat,
policy1.min_htlc,
Common.utcDateToMysql(policy1.last_update),
channel.node2_pub,
policy2.fee_base_msat,
policy2.time_lock_delta,
policy2.fee_rate_milli_msat,
policy2.disabled,
policy2.max_htlc_msat,
policy2.min_htlc,
Common.utcDateToMysql(policy2.last_update),
channel.capacity,
Common.utcDateToMysql(channel.last_update),
channel.node1_pub,
policy1.fee_base_msat,
policy1.time_lock_delta,
policy1.fee_rate_milli_msat,
policy1.disabled,
policy1.max_htlc_msat,
policy1.min_htlc,
Common.utcDateToMysql(policy1.last_update),
channel.node2_pub,
policy2.fee_base_msat,
policy2.time_lock_delta,
policy2.fee_rate_milli_msat,
policy2.disabled,
policy2.max_htlc_msat,
policy2.min_htlc,
Common.utcDateToMysql(policy2.last_update)
]);
}
/**
* Set all channels not in `graphChannelsIds` as inactive (status = 0)
*/
public async $setChannelsInactive(graphChannelsIds: string[]): Promise<void> {
if (graphChannelsIds.length === 0) {
return;
}
try {
const result = await DB.query<ResultSetHeader>(`
UPDATE channels
SET status = 0
WHERE id NOT IN (
${graphChannelsIds.map(id => `"${id}"`).join(',')}
)
AND status != 2
`);
if (result[0].changedRows ?? 0 > 0) {
logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`);
} else {
logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`);
}
} catch (e) {
logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
}
}
} }
export default new ChannelsApi(); export default new ChannelsApi();

View File

@ -32,6 +32,9 @@ class ChannelsRoutes {
res.status(404).send('Channel not found'); res.status(404).send('Channel not found');
return; return;
} }
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(channel); res.json(channel);
} catch (e) { } catch (e) {
res.status(500).send(e instanceof Error ? e.message : e); res.status(500).send(e instanceof Error ? e.message : e);

View File

@ -1,5 +1,7 @@
import logger from '../../logger'; import logger from '../../logger';
import DB from '../../database'; import DB from '../../database';
import { ResultSetHeader } from 'mysql2';
import { ILightningApi } from '../lightning/lightning-api.interface';
class NodesApi { class NodesApi {
public async $getNode(public_key: string): Promise<any> { public async $getNode(public_key: string): Promise<any> {
@ -166,7 +168,7 @@ class NodesApi {
} }
} }
public async $getNodesISP(groupBy: string, showTor: boolean) { public async $getNodesISPRanking(groupBy: string, showTor: boolean) {
try { try {
const orderBy = groupBy === 'capacity' ? `CAST(SUM(capacity) as INT)` : `COUNT(DISTINCT nodes.public_key)`; const orderBy = groupBy === 'capacity' ? `CAST(SUM(capacity) as INT)` : `COUNT(DISTINCT nodes.public_key)`;
@ -321,6 +323,66 @@ class NodesApi {
throw e; throw e;
} }
} }
/**
* Save or update a node present in the graph
*/
public async $saveNode(node: ILightningApi.Node): Promise<void> {
try {
const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
const query = `INSERT INTO nodes(
public_key,
first_seen,
updated_at,
alias,
color,
sockets,
status
)
VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, 1)
ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, color = ?, sockets = ?, status = 1`;
await DB.query(query, [
node.pub_key,
node.last_update,
node.alias,
node.color,
sockets,
node.last_update,
node.alias,
node.color,
sockets,
]);
} catch (e) {
logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));
}
}
/**
* Set all nodes not in `nodesPubkeys` as inactive (status = 0)
*/
public async $setNodesInactive(graphNodesPubkeys: string[]): Promise<void> {
if (graphNodesPubkeys.length === 0) {
return;
}
try {
const result = await DB.query<ResultSetHeader>(`
UPDATE nodes
SET status = 0
WHERE public_key NOT IN (
${graphNodesPubkeys.map(pubkey => `"${pubkey}"`).join(',')}
)
`);
if (result[0].changedRows ?? 0 > 0) {
logger.info(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`);
} else {
logger.debug(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`);
}
} catch (e) {
logger.err('$setNodesInactive() error: ' + (e instanceof Error ? e.message : e));
}
}
} }
export default new NodesApi(); export default new NodesApi();

View File

@ -79,7 +79,7 @@ class NodesRoutes {
return; return;
} }
const nodesPerAs = await nodesApi.$getNodesISP(groupBy, showTor); const nodesPerAs = await nodesApi.$getNodesISPRanking(groupBy, showTor);
res.header('Pragma', 'public'); res.header('Pragma', 'public');
res.header('Cache-control', 'public'); res.header('Cache-control', 'public');

View File

@ -1,6 +1,7 @@
import { ILightningApi } from '../lightning-api.interface'; import { ILightningApi } from '../lightning-api.interface';
import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher'; import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher';
import logger from '../../../logger'; import logger from '../../../logger';
import { Common } from '../../common';
/** /**
* Convert a clightning "listnode" entry to a lnd node entry * Convert a clightning "listnode" entry to a lnd node entry
@ -70,14 +71,6 @@ export async function convertAndmergeBidirectionalChannels(clChannels: any[]): P
return consolidatedChannelList; return consolidatedChannelList;
} }
export function convertChannelId(channelId): string {
if (channelId.indexOf('/') !== -1) {
channelId = channelId.slice(0, -2);
}
const s = channelId.split('x').map(part => BigInt(part));
return ((s[0] << 40n) | (s[1] << 16n) | s[2]).toString();
}
/** /**
* Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format * Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format
* In this case, clightning knows the channel policy for both nodes * In this case, clightning knows the channel policy for both nodes
@ -90,7 +83,7 @@ async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILigh
const outputIdx = parts[2]; const outputIdx = parts[2];
return { return {
channel_id: clChannelA.short_channel_id, channel_id: Common.channelShortIdToIntegerId(clChannelA.short_channel_id),
capacity: clChannelA.satoshis, capacity: clChannelA.satoshis,
last_update: lastUpdate, last_update: lastUpdate,
node1_policy: convertPolicy(clChannelA), node1_policy: convertPolicy(clChannelA),
@ -111,7 +104,7 @@ async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Cha
const outputIdx = parts[2]; const outputIdx = parts[2];
return { return {
channel_id: clChannel.short_channel_id, channel_id: Common.channelShortIdToIntegerId(clChannel.short_channel_id),
capacity: clChannel.satoshis, capacity: clChannel.satoshis,
last_update: clChannel.last_update ?? 0, last_update: clChannel.last_update ?? 0,
node1_policy: convertPolicy(clChannel), node1_policy: convertPolicy(clChannel),

View File

@ -473,7 +473,7 @@ class Mining {
for (const block of blocksWithoutPrices) { for (const block of blocksWithoutPrices) {
// Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks // Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks
if (block.height < 68951) { if (['mainnet', 'testnet'].includes(config.MEMPOOL.NETWORK) && block.height < 68951) {
blocksPrices.push({ blocksPrices.push({
height: block.height, height: block.height,
priceId: prices[0].id, priceId: prices[0].id,
@ -492,11 +492,11 @@ class Mining {
if (blocksPrices.length >= 100000) { if (blocksPrices.length >= 100000) {
totalInserted += blocksPrices.length; totalInserted += blocksPrices.length;
let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
if (blocksWithoutPrices.length > 200000) { if (blocksWithoutPrices.length > 200000) {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`); logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
} else {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
} }
logger.debug(logStr);
await BlocksRepository.$saveBlockPrices(blocksPrices); await BlocksRepository.$saveBlockPrices(blocksPrices);
blocksPrices.length = 0; blocksPrices.length = 0;
} }
@ -504,11 +504,11 @@ class Mining {
if (blocksPrices.length > 0) { if (blocksPrices.length > 0) {
totalInserted += blocksPrices.length; totalInserted += blocksPrices.length;
let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
if (blocksWithoutPrices.length > 200000) { if (blocksWithoutPrices.length > 200000) {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`); logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
} else {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
} }
logger.debug(logStr);
await BlocksRepository.$saveBlockPrices(blocksPrices); await BlocksRepository.$saveBlockPrices(blocksPrices);
} }
} catch (e) { } catch (e) {

View File

@ -24,6 +24,8 @@ interface IConfig {
USER_AGENT: string; USER_AGENT: string;
STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug'; STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
AUTOMATIC_BLOCK_REINDEXING: boolean; AUTOMATIC_BLOCK_REINDEXING: boolean;
POOLS_JSON_URL: string,
POOLS_JSON_TREE_URL: string,
}; };
ESPLORA: { ESPLORA: {
REST_API_URL: string; REST_API_URL: string;
@ -32,7 +34,8 @@ interface IConfig {
ENABLED: boolean; ENABLED: boolean;
BACKEND: 'lnd' | 'cln' | 'ldk'; BACKEND: 'lnd' | 'cln' | 'ldk';
TOPOLOGY_FOLDER: string; TOPOLOGY_FOLDER: string;
NODE_STATS_REFRESH_INTERVAL: number; STATS_REFRESH_INTERVAL: number;
GRAPH_REFRESH_INTERVAL: number;
}; };
LND: { LND: {
TLS_CERT_PATH: string; TLS_CERT_PATH: string;
@ -135,6 +138,8 @@ const defaults: IConfig = {
'USER_AGENT': 'mempool', 'USER_AGENT': 'mempool',
'STDOUT_LOG_MIN_PRIORITY': 'debug', 'STDOUT_LOG_MIN_PRIORITY': 'debug',
'AUTOMATIC_BLOCK_REINDEXING': false, 'AUTOMATIC_BLOCK_REINDEXING': false,
'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
}, },
'ESPLORA': { 'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000', 'REST_API_URL': 'http://127.0.0.1:3000',
@ -184,7 +189,8 @@ const defaults: IConfig = {
'ENABLED': false, 'ENABLED': false,
'BACKEND': 'lnd', 'BACKEND': 'lnd',
'TOPOLOGY_FOLDER': '', 'TOPOLOGY_FOLDER': '',
'NODE_STATS_REFRESH_INTERVAL': 600, 'STATS_REFRESH_INTERVAL': 600,
'GRAPH_REFRESH_INTERVAL': 600,
}, },
'LND': { 'LND': {
'TLS_CERT_PATH': '', 'TLS_CERT_PATH': '',

View File

@ -1,7 +1,7 @@
import config from './config'; import config from './config';
import { createPool, Pool, PoolConnection } from 'mysql2/promise'; import { createPool, Pool, PoolConnection } from 'mysql2/promise';
import logger from './logger'; import logger from './logger';
import { PoolOptions } from 'mysql2/typings/mysql'; import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } from 'mysql2/typings/mysql';
class DB { class DB {
constructor() { constructor() {
@ -28,7 +28,9 @@ import { PoolOptions } from 'mysql2/typings/mysql';
} }
} }
public async query(query, params?) { public async query<T extends RowDataPacket[][] | RowDataPacket[] | OkPacket |
OkPacket[] | ResultSetHeader>(query, params?): Promise<[T, FieldPacket[]]>
{
this.checkDBFlag(); this.checkDBFlag();
const pool = await this.getPool(); const pool = await this.getPool();
return pool.query(query, params); return pool.query(query, params);

View File

@ -6,13 +6,12 @@ import logger from './logger';
import HashratesRepository from './repositories/HashratesRepository'; import HashratesRepository from './repositories/HashratesRepository';
import bitcoinClient from './api/bitcoin/bitcoin-client'; import bitcoinClient from './api/bitcoin/bitcoin-client';
import priceUpdater from './tasks/price-updater'; import priceUpdater from './tasks/price-updater';
import PricesRepository from './repositories/PricesRepository';
class Indexer { class Indexer {
runIndexer = true; runIndexer = true;
indexerRunning = false; indexerRunning = false;
tasksRunning: string[] = [];
constructor() {
}
public reindex() { public reindex() {
if (Common.indexingEnabled()) { if (Common.indexingEnabled()) {
@ -20,6 +19,28 @@ class Indexer {
} }
} }
public async runSingleTask(task: 'blocksPrices') {
if (!Common.indexingEnabled()) {
return;
}
if (task === 'blocksPrices' && !this.tasksRunning.includes(task)) {
this.tasksRunning.push(task);
const lastestPriceId = await PricesRepository.$getLatestPriceId();
if (priceUpdater.historyInserted === false || lastestPriceId === null) {
logger.debug(`Blocks prices indexer is waiting for the price updater to complete`)
setTimeout(() => {
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
this.runSingleTask('blocksPrices');
}, 10000);
} else {
logger.debug(`Blocks prices indexer will run now`)
await mining.$indexBlockPrices();
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
}
}
}
public async $run() { public async $run() {
if (!Common.indexingEnabled() || this.runIndexer === false || if (!Common.indexingEnabled() || this.runIndexer === false ||
this.indexerRunning === true || mempool.hasPriority() this.indexerRunning === true || mempool.hasPriority()
@ -50,7 +71,7 @@ class Indexer {
return; return;
} }
await mining.$indexBlockPrices(); this.runSingleTask('blocksPrices');
await mining.$indexDifficultyAdjustments(); await mining.$indexDifficultyAdjustments();
await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient
await mining.$generateNetworkHashrateHistory(); await mining.$generateNetworkHashrateHistory();

View File

@ -27,6 +27,11 @@ class PricesRepository {
return oldestRow[0] ? oldestRow[0].time : 0; return oldestRow[0] ? oldestRow[0].time : 0;
} }
public async $getLatestPriceId(): Promise<number | null> {
const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
return oldestRow[0] ? oldestRow[0].id : null;
}
public async $getLatestPriceTime(): Promise<number> { public async $getLatestPriceTime(): Promise<number> {
const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`); const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
return oldestRow[0] ? oldestRow[0].time : 0; return oldestRow[0] ? oldestRow[0].time : 0;

View File

@ -1,60 +1,43 @@
import DB from '../../database'; import DB from '../../database';
import logger from '../../logger'; import logger from '../../logger';
import channelsApi from '../../api/explorer/channels.api'; import channelsApi from '../../api/explorer/channels.api';
import bitcoinClient from '../../api/bitcoin/bitcoin-client';
import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory'; import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
import config from '../../config'; import config from '../../config';
import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface'; import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
import { ILightningApi } from '../../api/lightning/lightning-api.interface'; import { ILightningApi } from '../../api/lightning/lightning-api.interface';
import { $lookupNodeLocation } from './sync-tasks/node-locations'; import { $lookupNodeLocation } from './sync-tasks/node-locations';
import lightningApi from '../../api/lightning/lightning-api-factory'; import lightningApi from '../../api/lightning/lightning-api-factory';
import { convertChannelId } from '../../api/lightning/clightning/clightning-convert'; import nodesApi from '../../api/explorer/nodes.api';
import { Common } from '../../api/common'; import { ResultSetHeader } from 'mysql2';
import fundingTxFetcher from './sync-tasks/funding-tx-fetcher';
class NetworkSyncService { class NetworkSyncService {
loggerTimer = 0;
constructor() {} constructor() {}
public async $startService() { public async $startService(): Promise<void> {
logger.info('Starting node sync service'); logger.info('Starting lightning network sync service');
await this.$runUpdater(); this.loggerTimer = new Date().getTime() / 1000;
setInterval(async () => { await this.$runTasks();
await this.$runUpdater();
}, 1000 * 60 * 60);
} }
private async $runUpdater(): Promise<void> { private async $runTasks(): Promise<void> {
try { try {
logger.info(`Updating nodes and channels...`); logger.info(`Updating nodes and channels`);
const networkGraph = await lightningApi.$getNetworkGraph(); const networkGraph = await lightningApi.$getNetworkGraph();
if (networkGraph.nodes.length === 0 || networkGraph.edges.length === 0) { if (networkGraph.nodes.length === 0 || networkGraph.edges.length === 0) {
logger.info(`LN Network graph is empty, retrying in 10 seconds`); logger.info(`LN Network graph is empty, retrying in 10 seconds`);
await Common.sleep$(10000); setTimeout(() => { this.$runTasks(); }, 10000);
this.$runUpdater();
return; return;
} }
for (const node of networkGraph.nodes) { await this.$updateNodesList(networkGraph.nodes);
await this.$saveNode(node); await this.$updateChannelsList(networkGraph.edges);
} await this.$deactivateChannelsWithoutActiveNodes();
logger.info(`Nodes updated.`);
if (config.MAXMIND.ENABLED) {
await $lookupNodeLocation();
}
const graphChannelsIds: string[] = [];
for (const channel of networkGraph.edges) {
await this.$saveChannel(channel);
graphChannelsIds.push(channel.channel_id);
}
await this.$setChannelsInactive(graphChannelsIds);
logger.info(`Channels updated.`);
await this.$findInactiveNodesAndChannels();
await this.$lookUpCreationDateFromChain(); await this.$lookUpCreationDateFromChain();
await this.$updateNodeFirstSeen(); await this.$updateNodeFirstSeen();
await this.$scanForClosedChannels(); await this.$scanForClosedChannels();
@ -63,84 +46,183 @@ class NetworkSyncService {
} }
} catch (e) { } catch (e) {
logger.err('$runUpdater() error: ' + (e instanceof Error ? e.message : e)); logger.err('$runTasks() error: ' + (e instanceof Error ? e.message : e));
} }
setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL);
}
/**
* Update the `nodes` table to reflect the current network graph state
*/
private async $updateNodesList(nodes: ILightningApi.Node[]): Promise<void> {
let progress = 0;
const graphNodesPubkeys: string[] = [];
for (const node of nodes) {
await nodesApi.$saveNode(node);
graphNodesPubkeys.push(node.pub_key);
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating node ${progress}/${nodes.length}`);
this.loggerTimer = new Date().getTime() / 1000;
}
}
logger.info(`${progress} nodes updated`);
// If a channel if not present in the graph, mark it as inactive
nodesApi.$setNodesInactive(graphNodesPubkeys);
if (config.MAXMIND.ENABLED) {
$lookupNodeLocation();
}
}
/**
* Update the `channels` table to reflect the current network graph state
*/
private async $updateChannelsList(channels: ILightningApi.Channel[]): Promise<void> {
try {
let progress = 0;
const graphChannelsIds: string[] = [];
for (const channel of channels) {
await channelsApi.$saveChannel(channel);
graphChannelsIds.push(channel.channel_id);
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating channel ${progress}/${channels.length}`);
this.loggerTimer = new Date().getTime() / 1000;
}
}
logger.info(`${progress} channels updated`);
// If a channel if not present in the graph, mark it as inactive
channelsApi.$setChannelsInactive(graphChannelsIds);
} catch (e) {
logger.err(`Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`);
}
setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL);
} }
// This method look up the creation date of the earliest channel of the node // This method look up the creation date of the earliest channel of the node
// and update the node to that date in order to get the earliest first seen date // and update the node to that date in order to get the earliest first seen date
private async $updateNodeFirstSeen() { private async $updateNodeFirstSeen(): Promise<void> {
let progress = 0;
let updated = 0;
try { try {
const [nodes]: any[] = await DB.query(`SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node1_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created1, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node2_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created2 FROM nodes`); const [nodes]: any[] = await DB.query(`
SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen,
(
SELECT MIN(UNIX_TIMESTAMP(created))
FROM channels
WHERE channels.node1_public_key = nodes.public_key
) AS created1,
(
SELECT MIN(UNIX_TIMESTAMP(created))
FROM channels
WHERE channels.node2_public_key = nodes.public_key
) AS created2
FROM nodes
`);
for (const node of nodes) { for (const node of nodes) {
let lowest = 0; const lowest = Math.min(
if (node.created1) { node.created1 ?? Number.MAX_SAFE_INTEGER,
if (node.created2 && node.created2 < node.created1) { node.created2 ?? Number.MAX_SAFE_INTEGER,
lowest = node.created2; node.first_seen ?? Number.MAX_SAFE_INTEGER
} else { );
lowest = node.created1; if (lowest < node.first_seen) {
}
} else if (node.created2) {
lowest = node.created2;
}
if (lowest && lowest < node.first_seen) {
const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`; const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`;
const params = [lowest, node.public_key]; const params = [lowest, node.public_key];
await DB.query(query, params); await DB.query(query, params);
} }
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating node first seen date ${progress}/${nodes.length}`);
this.loggerTimer = new Date().getTime() / 1000;
++updated;
}
} }
logger.info(`Node first seen dates scan complete.`); logger.info(`Updated ${updated} node first seen dates`);
} catch (e) { } catch (e) {
logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e)); logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e));
} }
} }
private async $lookUpCreationDateFromChain() { private async $lookUpCreationDateFromChain(): Promise<void> {
logger.info(`Running channel creation date lookup...`); let progress = 0;
logger.info(`Running channel creation date lookup`);
try { try {
const channels = await channelsApi.$getChannelsWithoutCreatedDate(); const channels = await channelsApi.$getChannelsWithoutCreatedDate();
for (const channel of channels) { for (const channel of channels) {
const transaction = await bitcoinClient.getRawTransaction(channel.transaction_id, 1); const transaction = await fundingTxFetcher.$fetchChannelOpenTx(channel.short_id);
await DB.query(`UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`, [transaction.blocktime, channel.id]); await DB.query(`
UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`,
[transaction.timestamp, channel.id]
);
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating channel creation date ${progress}/${channels.length}`);
this.loggerTimer = new Date().getTime() / 1000;
}
} }
logger.info(`Channel creation dates scan complete.`); logger.info(`Updated ${channels.length} channels' creation date`);
} catch (e) { } catch (e) {
logger.err('$setCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e)); logger.err('$lookUpCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e));
} }
} }
// Looking for channels whos nodes are inactive /**
private async $findInactiveNodesAndChannels(): Promise<void> { * If a channel does not have any active node linked to it, then also
logger.info(`Running inactive channels scan...`); * mark that channel as inactive
*/
private async $deactivateChannelsWithoutActiveNodes(): Promise<void> {
logger.info(`Find channels which nodes are offline`);
try { try {
const [channels]: [{ id: string }[]] = await <any>DB.query(` const result = await DB.query<ResultSetHeader>(`
SELECT channels.id UPDATE channels
FROM channels SET status = 0
WHERE channels.status = 1 WHERE channels.status = 1
AND ( AND (
( (
SELECT COUNT(*) SELECT COUNT(*)
FROM nodes FROM nodes
WHERE nodes.public_key = channels.node1_public_key WHERE nodes.public_key = channels.node1_public_key
AND nodes.status = 1
) = 0 ) = 0
OR ( OR (
SELECT COUNT(*) SELECT COUNT(*)
FROM nodes FROM nodes
WHERE nodes.public_key = channels.node2_public_key WHERE nodes.public_key = channels.node2_public_key
AND nodes.status = 1
) = 0) ) = 0)
`); `);
for (const channel of channels) { if (result[0].changedRows ?? 0 > 0) {
await this.$updateChannelStatus(channel.id, 0); logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`);
} else {
logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`);
} }
logger.info(`Inactive channels scan complete.`);
} catch (e) { } catch (e) {
logger.err('$findInactiveNodesAndChannels() error: ' + (e instanceof Error ? e.message : e)); logger.err('$deactivateChannelsWithoutActiveNodes() error: ' + (e instanceof Error ? e.message : e));
} }
} }
private async $scanForClosedChannels(): Promise<void> { private async $scanForClosedChannels(): Promise<void> {
let progress = 0;
try { try {
logger.info(`Starting closed channels scan...`); logger.info(`Starting closed channels scan...`);
const channels = await channelsApi.$getChannelsByStatus(0); const channels = await channelsApi.$getChannelsByStatus(0);
@ -154,6 +236,13 @@ class NetworkSyncService {
await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]); await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]);
} }
} }
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Checking if channel has been closed ${progress}/${channels.length}`);
this.loggerTimer = new Date().getTime() / 1000;
}
} }
logger.info(`Closed channels scan complete.`); logger.info(`Closed channels scan complete.`);
} catch (e) { } catch (e) {
@ -171,6 +260,9 @@ class NetworkSyncService {
if (!config.ESPLORA.REST_API_URL) { if (!config.ESPLORA.REST_API_URL) {
return; return;
} }
let progress = 0;
try { try {
logger.info(`Started running closed channel forensics...`); logger.info(`Started running closed channel forensics...`);
const channels = await channelsApi.$getClosedChannelsWithoutReason(); const channels = await channelsApi.$getClosedChannelsWithoutReason();
@ -216,6 +308,13 @@ class NetworkSyncService {
logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.'); logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]); await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
} }
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating channel closed channel forensics ${progress}/${channels.length}`);
this.loggerTimer = new Date().getTime() / 1000;
}
} }
logger.info(`Closed channels forensics scan complete.`); logger.info(`Closed channels forensics scan complete.`);
} catch (e) { } catch (e) {
@ -270,195 +369,6 @@ class NetworkSyncService {
} }
return 1; return 1;
} }
private async $saveChannel(channel: ILightningApi.Channel): Promise<void> {
const [ txid, vout ] = channel.chan_point.split(':');
const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
try {
const query = `INSERT INTO channels
(
id,
short_id,
capacity,
transaction_id,
transaction_vout,
updated_at,
status,
node1_public_key,
node1_base_fee_mtokens,
node1_cltv_delta,
node1_fee_rate,
node1_is_disabled,
node1_max_htlc_mtokens,
node1_min_htlc_mtokens,
node1_updated_at,
node2_public_key,
node2_base_fee_mtokens,
node2_cltv_delta,
node2_fee_rate,
node2_is_disabled,
node2_max_htlc_mtokens,
node2_min_htlc_mtokens,
node2_updated_at
)
VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
capacity = ?,
updated_at = ?,
status = 1,
node1_public_key = ?,
node1_base_fee_mtokens = ?,
node1_cltv_delta = ?,
node1_fee_rate = ?,
node1_is_disabled = ?,
node1_max_htlc_mtokens = ?,
node1_min_htlc_mtokens = ?,
node1_updated_at = ?,
node2_public_key = ?,
node2_base_fee_mtokens = ?,
node2_cltv_delta = ?,
node2_fee_rate = ?,
node2_is_disabled = ?,
node2_max_htlc_mtokens = ?,
node2_min_htlc_mtokens = ?,
node2_updated_at = ?
;`;
await DB.query(query, [
this.toIntegerId(channel.channel_id),
this.toShortId(channel.channel_id),
channel.capacity,
txid,
vout,
this.utcDateToMysql(channel.last_update),
channel.node1_pub,
policy1.fee_base_msat,
policy1.time_lock_delta,
policy1.fee_rate_milli_msat,
policy1.disabled,
policy1.max_htlc_msat,
policy1.min_htlc,
this.utcDateToMysql(policy1.last_update),
channel.node2_pub,
policy2.fee_base_msat,
policy2.time_lock_delta,
policy2.fee_rate_milli_msat,
policy2.disabled,
policy2.max_htlc_msat,
policy2.min_htlc,
this.utcDateToMysql(policy2.last_update),
channel.capacity,
this.utcDateToMysql(channel.last_update),
channel.node1_pub,
policy1.fee_base_msat,
policy1.time_lock_delta,
policy1.fee_rate_milli_msat,
policy1.disabled,
policy1.max_htlc_msat,
policy1.min_htlc,
this.utcDateToMysql(policy1.last_update),
channel.node2_pub,
policy2.fee_base_msat,
policy2.time_lock_delta,
policy2.fee_rate_milli_msat,
policy2.disabled,
policy2.max_htlc_msat,
policy2.min_htlc,
this.utcDateToMysql(policy2.last_update)
]);
} catch (e) {
logger.err('$saveChannel() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $updateChannelStatus(channelId: string, status: number): Promise<void> {
try {
await DB.query(`UPDATE channels SET status = ? WHERE id = ?`, [status, channelId]);
} catch (e) {
logger.err('$updateChannelStatus() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $setChannelsInactive(graphChannelsIds: string[]): Promise<void> {
if (graphChannelsIds.length === 0) {
return;
}
try {
await DB.query(`
UPDATE channels
SET status = 0
WHERE short_id NOT IN (
${graphChannelsIds.map(id => `"${id}"`).join(',')}
)
AND status != 2
`);
} catch (e) {
logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $saveNode(node: ILightningApi.Node): Promise<void> {
try {
const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
const query = `INSERT INTO nodes(
public_key,
first_seen,
updated_at,
alias,
color,
sockets
)
VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?)
ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, color = ?, sockets = ?`;
await DB.query(query, [
node.pub_key,
node.last_update,
node.alias,
node.color,
sockets,
node.last_update,
node.alias,
node.color,
sockets,
]);
} catch (e) {
logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));
}
}
private toIntegerId(id: string): string {
if (config.LIGHTNING.BACKEND === 'cln') {
return convertChannelId(id);
}
else if (config.LIGHTNING.BACKEND === 'lnd') {
return id;
}
return '';
}
/** Decodes a channel id returned by lnd as uint64 to a short channel id */
private toShortId(id: string): string {
if (config.LIGHTNING.BACKEND === 'cln') {
return id;
}
const n = BigInt(id);
return [
n >> 40n, // nth block
(n >> 16n) & 0xffffffn, // nth tx of the block
n & 0xffffn // nth output of the tx
].join('x');
}
private utcDateToMysql(date?: number): string {
const d = new Date((date || 0) * 1000);
return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
}
} }
export default new NetworkSyncService(); export default new NetworkSyncService();

View File

@ -1,8 +1,8 @@
import DB from '../../database';
import logger from '../../logger'; import logger from '../../logger';
import lightningApi from '../../api/lightning/lightning-api-factory'; import lightningApi from '../../api/lightning/lightning-api-factory';
import LightningStatsImporter from './sync-tasks/stats-importer'; import LightningStatsImporter from './sync-tasks/stats-importer';
import config from '../../config'; import config from '../../config';
import { Common } from '../../api/common';
class LightningStatsUpdater { class LightningStatsUpdater {
public async $startService(): Promise<void> { public async $startService(): Promise<void> {
@ -12,31 +12,22 @@ class LightningStatsUpdater {
LightningStatsImporter.$run(); LightningStatsImporter.$run();
} }
private setDateMidnight(date: Date): void {
date.setUTCHours(0);
date.setUTCMinutes(0);
date.setUTCSeconds(0);
date.setUTCMilliseconds(0);
}
private async $runTasks(): Promise<void> { private async $runTasks(): Promise<void> {
await this.$logStatsDaily(); await this.$logStatsDaily();
setTimeout(() => { setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL);
this.$runTasks();
}, 1000 * config.LIGHTNING.NODE_STATS_REFRESH_INTERVAL);
} }
/** /**
* Update the latest entry for each node every config.LIGHTNING.NODE_STATS_REFRESH_INTERVAL seconds * Update the latest entry for each node every config.LIGHTNING.STATS_REFRESH_INTERVAL seconds
*/ */
private async $logStatsDaily(): Promise<void> { private async $logStatsDaily(): Promise<void> {
const date = new Date(); const date = new Date();
this.setDateMidnight(date); Common.setDateMidnight(date);
logger.info(`Updating latest networks stats`);
const networkGraph = await lightningApi.$getNetworkGraph(); const networkGraph = await lightningApi.$getNetworkGraph();
LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph); LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph);
logger.info(`Updated latest network stats`);
} }
} }

View File

@ -1,5 +1,6 @@
import { existsSync, promises } from 'fs'; import { existsSync, promises } from 'fs';
import bitcoinClient from '../../../api/bitcoin/bitcoin-client'; import bitcoinClient from '../../../api/bitcoin/bitcoin-client';
import { Common } from '../../../api/common';
import config from '../../../config'; import config from '../../../config';
import logger from '../../../logger'; import logger from '../../../logger';
@ -69,7 +70,11 @@ class FundingTxFetcher {
this.running = false; this.running = false;
} }
public async $fetchChannelOpenTx(channelId: string): Promise<any> { public async $fetchChannelOpenTx(channelId: string): Promise<{timestamp: number, txid: string, value: number}> {
if (channelId.indexOf('x') === -1) {
channelId = Common.channelIntegerIdToShortId(channelId);
}
if (this.fundingTxCache[channelId]) { if (this.fundingTxCache[channelId]) {
return this.fundingTxCache[channelId]; return this.fundingTxCache[channelId];
} }
@ -110,4 +115,4 @@ class FundingTxFetcher {
} }
} }
export default new FundingTxFetcher; export default new FundingTxFetcher;

View File

@ -6,7 +6,10 @@ import DB from '../../../database';
import logger from '../../../logger'; import logger from '../../../logger';
export async function $lookupNodeLocation(): Promise<void> { export async function $lookupNodeLocation(): Promise<void> {
logger.info(`Running node location updater using Maxmind...`); let loggerTimer = new Date().getTime() / 1000;
let progress = 0;
logger.info(`Running node location updater using Maxmind`);
try { try {
const nodes = await nodesApi.$getAllNodes(); const nodes = await nodesApi.$getAllNodes();
const lookupCity = await maxmind.open<CityResponse>(config.MAXMIND.GEOLITE2_CITY); const lookupCity = await maxmind.open<CityResponse>(config.MAXMIND.GEOLITE2_CITY);
@ -18,21 +21,24 @@ export async function $lookupNodeLocation(): Promise<void> {
for (const socket of sockets) { for (const socket of sockets) {
const ip = socket.substring(0, socket.lastIndexOf(':')).replace('[', '').replace(']', ''); const ip = socket.substring(0, socket.lastIndexOf(':')).replace('[', '').replace(']', '');
const hasClearnet = [4, 6].includes(net.isIP(ip)); const hasClearnet = [4, 6].includes(net.isIP(ip));
if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') { if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') {
const city = lookupCity.get(ip); const city = lookupCity.get(ip);
const asn = lookupAsn.get(ip); const asn = lookupAsn.get(ip);
const isp = lookupIsp.get(ip); const isp = lookupIsp.get(ip);
if (city && (asn || isp)) { if (city && (asn || isp)) {
const query = `UPDATE nodes SET const query = `
as_number = ?, UPDATE nodes SET
city_id = ?, as_number = ?,
country_id = ?, city_id = ?,
subdivision_id = ?, country_id = ?,
longitude = ?, subdivision_id = ?,
latitude = ?, longitude = ?,
accuracy_radius = ? latitude = ?,
WHERE public_key = ?`; accuracy_radius = ?
WHERE public_key = ?
`;
const params = [ const params = [
isp?.autonomous_system_number ?? asn?.autonomous_system_number, isp?.autonomous_system_number ?? asn?.autonomous_system_number,
@ -46,25 +52,25 @@ export async function $lookupNodeLocation(): Promise<void> {
]; ];
await DB.query(query, params); await DB.query(query, params);
// Store Continent // Store Continent
if (city.continent?.geoname_id) { if (city.continent?.geoname_id) {
await DB.query( await DB.query(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'continent', ?)`, `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'continent', ?)`,
[city.continent?.geoname_id, JSON.stringify(city.continent?.names)]); [city.continent?.geoname_id, JSON.stringify(city.continent?.names)]);
} }
// Store Country // Store Country
if (city.country?.geoname_id) { if (city.country?.geoname_id) {
await DB.query( await DB.query(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country', ?)`, `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country', ?)`,
[city.country?.geoname_id, JSON.stringify(city.country?.names)]); [city.country?.geoname_id, JSON.stringify(city.country?.names)]);
} }
// Store Country ISO code // Store Country ISO code
if (city.country?.iso_code) { if (city.country?.iso_code) {
await DB.query( await DB.query(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`, `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`,
[city.country?.geoname_id, city.country?.iso_code]); [city.country?.geoname_id, city.country?.iso_code]);
} }
// Store Division // Store Division
@ -88,10 +94,17 @@ export async function $lookupNodeLocation(): Promise<void> {
[isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]); [isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]);
} }
} }
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating node location data ${progress}/${nodes.length}`);
loggerTimer = new Date().getTime() / 1000;
}
} }
} }
} }
logger.info(`Node location data updated.`); logger.info(`${progress} nodes location data updated`);
} catch (e) { } catch (e) {
logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e)); logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e));
} }

View File

@ -128,32 +128,32 @@ class LightningStatsImporter {
if (channel.node1_policy !== undefined) { // Coming from the node if (channel.node1_policy !== undefined) { // Coming from the node
for (const policy of [channel.node1_policy, channel.node2_policy]) { for (const policy of [channel.node1_policy, channel.node2_policy]) {
if (policy && policy.fee_rate_milli_msat < 5000) { if (policy && policy.fee_rate_milli_msat < 5000) {
avgFeeRate += policy.fee_rate_milli_msat; avgFeeRate += parseInt(policy.fee_rate_milli_msat, 10);
feeRates.push(policy.fee_rate_milli_msat); feeRates.push(parseInt(policy.fee_rate_milli_msat, 10));
} }
if (policy && policy.fee_base_msat < 5000) { if (policy && policy.fee_base_msat < 5000) {
avgBaseFee += policy.fee_base_msat; avgBaseFee += parseInt(policy.fee_base_msat, 10);
baseFees.push(policy.fee_base_msat); baseFees.push(parseInt(policy.fee_base_msat, 10));
} }
} }
} else { // Coming from the historical import } else { // Coming from the historical import
if (channel.fee_rate_milli_msat < 5000) { if (channel.fee_rate_milli_msat < 5000) {
avgFeeRate += channel.fee_rate_milli_msat; avgFeeRate += parseInt(channel.fee_rate_milli_msat, 10);
feeRates.push(channel.fee_rate_milli_msat); feeRates.push(parseInt(channel.fee_rate_milli_msat), 10);
} }
if (channel.fee_base_msat < 5000) { if (channel.fee_base_msat < 5000) {
avgBaseFee += channel.fee_base_msat; avgBaseFee += parseInt(channel.fee_base_msat, 10);
baseFees.push(channel.fee_base_msat); baseFees.push(parseInt(channel.fee_base_msat), 10);
} }
} }
} }
avgFeeRate /= networkGraph.edges.length; avgFeeRate /= Math.max(networkGraph.edges.length, 1);
avgBaseFee /= networkGraph.edges.length; avgBaseFee /= Math.max(networkGraph.edges.length, 1);
const medCapacity = capacities.sort((a, b) => b - a)[Math.round(capacities.length / 2 - 1)]; const medCapacity = capacities.sort((a, b) => b - a)[Math.round(capacities.length / 2 - 1)];
const medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)]; const medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)];
const medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)]; const medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)];
const avgCapacity = Math.round(capacity / capacities.length); const avgCapacity = Math.round(capacity / Math.max(capacities.length, 1));
let query = `INSERT INTO lightning_stats( let query = `INSERT INTO lightning_stats(
added, added,
@ -251,6 +251,9 @@ class LightningStatsImporter {
}; };
} }
/**
* Import topology files LN historical data into the database
*/
async $importHistoricalLightningStats(): Promise<void> { async $importHistoricalLightningStats(): Promise<void> {
let latestNodeCount = 1; let latestNodeCount = 1;

View File

@ -12,14 +12,11 @@ import * as https from 'https';
*/ */
class PoolsUpdater { class PoolsUpdater {
lastRun: number = 0; lastRun: number = 0;
currentSha: any = undefined; currentSha: string | undefined = undefined;
poolsUrl: string = 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json'; poolsUrl: string = config.MEMPOOL.POOLS_JSON_URL;
treeUrl: string = 'https://api.github.com/repos/mempool/mining-pools/git/trees/master'; treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL;
constructor() { public async updatePoolsJson(): Promise<void> {
}
public async updatePoolsJson() {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
return; return;
} }
@ -77,7 +74,7 @@ class PoolsUpdater {
/** /**
* Fetch our latest pools.json sha from the db * Fetch our latest pools.json sha from the db
*/ */
private async updateDBSha(githubSha: string) { private async updateDBSha(githubSha: string): Promise<void> {
this.currentSha = githubSha; this.currentSha = githubSha;
if (config.DATABASE.ENABLED === true) { if (config.DATABASE.ENABLED === true) {
try { try {

View File

@ -1,4 +1,5 @@
import * as fs from 'fs'; import * as fs from 'fs';
import { Common } from '../api/common';
import config from '../config'; import config from '../config';
import logger from '../logger'; import logger from '../logger';
import PricesRepository from '../repositories/PricesRepository'; import PricesRepository from '../repositories/PricesRepository';
@ -34,10 +35,10 @@ export interface Prices {
} }
class PriceUpdater { class PriceUpdater {
historyInserted: boolean = false; public historyInserted = false;
lastRun: number = 0; lastRun = 0;
lastHistoricalRun: number = 0; lastHistoricalRun = 0;
running: boolean = false; running = false;
feeds: PriceFeed[] = []; feeds: PriceFeed[] = [];
currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY']; currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
latestPrices: Prices; latestPrices: Prices;

View File

@ -102,7 +102,9 @@ Below we list all settings from `mempool-config.json` and the corresponding over
"PRICE_FEED_UPDATE_INTERVAL": 600, "PRICE_FEED_UPDATE_INTERVAL": 600,
"USE_SECOND_NODE_FOR_MINFEE": false, "USE_SECOND_NODE_FOR_MINFEE": false,
"EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"], "EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"],
"STDOUT_LOG_MIN_PRIORITY": "info" "STDOUT_LOG_MIN_PRIORITY": "info",
"POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
"POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master"
}, },
``` ```
@ -126,6 +128,8 @@ Corresponding `docker-compose.yml` overrides:
MEMPOOL_USE_SECOND_NODE_FOR_MINFEE: "" MEMPOOL_USE_SECOND_NODE_FOR_MINFEE: ""
MEMPOOL_EXTERNAL_ASSETS: "" MEMPOOL_EXTERNAL_ASSETS: ""
MEMPOOL_STDOUT_LOG_MIN_PRIORITY: "" MEMPOOL_STDOUT_LOG_MIN_PRIORITY: ""
MEMPOOL_POOLS_JSON_URL: ""
MEMPOOL_POOLS_JSON_TREE_URL: ""
... ...
``` ```

View File

@ -24,6 +24,8 @@ __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool}
__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info} __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
__MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false} __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false}
__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false} __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false}
__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=false}
__MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=false}
# CORE_RPC # CORE_RPC
__CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@ -114,6 +116,8 @@ sed -i "s!__MEMPOOL_USER_AGENT__!${__MEMPOOL_USER_AGENT__}!g" mempool-config.jso
sed -i "s/__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__/${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}/g" mempool-config.json sed -i "s/__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__/${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}/g" mempool-config.json
sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json
sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json
sed -i "s/__MEMPOOL_POOLS_JSON_URL__/${__MEMPOOL_POOLS_JSON_URL__}/g" mempool-config.json
sed -i "s/__MEMPOOL_POOLS_JSON_TREE_URL__/${__MEMPOOL_POOLS_JSON_TREE_URL__}/g" mempool-config.json
sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json

View File

@ -76,10 +76,8 @@
</div> </div>
<div [class]="!widget ? 'bottom-padding' : 'pb-0'" class="container pb-lg-0"> <div [class]="!widget ? 'bottom-padding' : 'pb-0'" class="container pb-lg-0">
<div> <div [class]="widget ? 'chart-widget' : 'chart'" echarts [initOpts]="chartInitOptions" [options]="chartOptions"
<div [class]="widget ? 'chart-widget' : 'chart'" echarts [initOpts]="chartInitOptions" [options]="chartOptions" (chartInit)="onChartInit($event)">
(chartInit)="onChartInit($event)">
</div>
</div> </div>
<div class="text-center loadingGraphs" *ngIf="isLoading"> <div class="text-center loadingGraphs" *ngIf="isLoading">

View File

@ -14,7 +14,9 @@
<div class="clearfix"></div> <div class="clearfix"></div>
<div class="box"> <app-nodes-channels-map *ngIf="!error" [style]="'channelpage'" [channel]="channelGeo"></app-nodes-channels-map>
<div class="box">
<div class="row"> <div class="row">
<div class="col-md"> <div class="col-md">

View File

@ -1,7 +1,7 @@
import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core';
import { ActivatedRoute, ParamMap } from '@angular/router'; import { ActivatedRoute, ParamMap } from '@angular/router';
import { Observable, of } from 'rxjs'; import { Observable, of } from 'rxjs';
import { catchError, switchMap } from 'rxjs/operators'; import { catchError, switchMap, tap } from 'rxjs/operators';
import { SeoService } from 'src/app/services/seo.service'; import { SeoService } from 'src/app/services/seo.service';
import { LightningApiService } from '../lightning-api.service'; import { LightningApiService } from '../lightning-api.service';
@ -14,6 +14,7 @@ import { LightningApiService } from '../lightning-api.service';
export class ChannelComponent implements OnInit { export class ChannelComponent implements OnInit {
channel$: Observable<any>; channel$: Observable<any>;
error: any = null; error: any = null;
channelGeo: number[] = [];
constructor( constructor(
private lightningApiService: LightningApiService, private lightningApiService: LightningApiService,
@ -29,9 +30,23 @@ export class ChannelComponent implements OnInit {
this.seoService.setTitle(`Channel: ${params.get('short_id')}`); this.seoService.setTitle(`Channel: ${params.get('short_id')}`);
return this.lightningApiService.getChannel$(params.get('short_id')) return this.lightningApiService.getChannel$(params.get('short_id'))
.pipe( .pipe(
tap((data) => {
if (!data.node_left.longitude || !data.node_left.latitude ||
!data.node_right.longitude || !data.node_right.latitude) {
this.channelGeo = [];
} else {
this.channelGeo = [
data.node_left.public_key,
data.node_left.alias,
data.node_left.longitude, data.node_left.latitude,
data.node_right.public_key,
data.node_right.alias,
data.node_right.longitude, data.node_right.latitude,
];
}
}),
catchError((err) => { catchError((err) => {
this.error = err; this.error = err;
console.log(this.error);
return of(null); return of(null);
}) })
); );

View File

@ -4,6 +4,7 @@
<div class="row row-cols-1 row-cols-md-2"> <div class="row row-cols-1 row-cols-md-2">
<!-- Network capacity/channels/nodes -->
<div class="col"> <div class="col">
<div class="main-title"> <div class="main-title">
<span i18n="lightning.statistics-title">Network Statistics</span>&nbsp; <span i18n="lightning.statistics-title">Network Statistics</span>&nbsp;
@ -17,6 +18,7 @@
</div> </div>
</div> </div>
<!-- Channels stats -->
<div class="col"> <div class="col">
<div class="main-title"> <div class="main-title">
<span i18n="lightning.statistics-title">Channels Statistics</span>&nbsp; <span i18n="lightning.statistics-title">Channels Statistics</span>&nbsp;
@ -30,18 +32,28 @@
</div> </div>
</div> </div>
<div class="col"> <!-- ISP pie chart -->
<div class="col" style="margin-bottom: 1.47rem">
<div class="card graph-card">
<div class="card-body pl-2 pr-2">
<app-nodes-per-isp-chart [widget]="true"></app-nodes-per-isp-chart>
<div class="mt-1"><a [attr.data-cy]="'pool-distribution-view-more'" [routerLink]="['/graphs/lightning/nodes-per-isp' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div>
</div>
</div>
</div>
<!-- <div class="col">
<div class="card"> <div class="card">
<div class="card-body"> <div class="card-body">
<app-nodes-networks-chart [widget]=true></app-nodes-networks-chart> <app-nodes-networks-chart [widget]=true></app-nodes-networks-chart>
<div class="mt-1"><a [routerLink]="['/graphs/lightning/nodes-networks' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div> <div class="mt-1"><a [routerLink]="['/graphs/lightning/nodes-networks' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div>
</div> </div>
</div> </div>
</div> </div> -->
<div class="col"> <div class="col">
<div class="card"> <div class="card graph-card">
<div class="card-body"> <div class="card-body pl-2 pr-2">
<app-lightning-statistics-chart [widget]=true></app-lightning-statistics-chart> <app-lightning-statistics-chart [widget]=true></app-lightning-statistics-chart>
<div class="mt-1"><a [routerLink]="['/graphs/lightning/capacity' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div> <div class="mt-1"><a [routerLink]="['/graphs/lightning/capacity' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div>
</div> </div>
@ -52,7 +64,7 @@
<div class="card"> <div class="card">
<div class="card-body"> <div class="card-body">
<h5 class="card-title">Top Capacity Nodes</h5> <h5 class="card-title">Top Capacity Nodes</h5>
<app-nodes-list [nodes$]="nodesByCapacity$"></app-nodes-list> <app-nodes-list [nodes$]="nodesByCapacity$" [show]="'mobile-capacity'"></app-nodes-list>
<!-- <div><a [routerLink]="['/lightning/nodes' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div> --> <!-- <div><a [routerLink]="['/lightning/nodes' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div> -->
</div> </div>
</div> </div>
@ -62,7 +74,7 @@
<div class="card"> <div class="card">
<div class="card-body"> <div class="card-body">
<h5 class="card-title">Most Connected Nodes</h5> <h5 class="card-title">Most Connected Nodes</h5>
<app-nodes-list [nodes$]="nodesByChannels$"></app-nodes-list> <app-nodes-list [nodes$]="nodesByChannels$" [show]="'mobile-channels'"></app-nodes-list>
<!-- <div><a [routerLink]="['/lightning/nodes' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div> --> <!-- <div><a [routerLink]="['/lightning/nodes' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div> -->
</div> </div>
</div> </div>

View File

@ -14,6 +14,13 @@
background-color: #1d1f31; background-color: #1d1f31;
} }
.graph-card {
height: 100%;
@media (min-width: 992px) {
height: 385px;
}
}
.card-title { .card-title {
font-size: 1rem; font-size: 1rem;
color: #4a68b9; color: #4a68b9;
@ -22,9 +29,6 @@
color: #4a68b9; color: #4a68b9;
} }
.card-body {
padding: 1.25rem 1rem 0.75rem 1rem;
}
.card-body.pool-ranking { .card-body.pool-ranking {
padding: 1.25rem 0.25rem 0.75rem 0.25rem; padding: 1.25rem 0.25rem 0.75rem 0.25rem;
} }
@ -32,6 +36,21 @@
font-size: 22px; font-size: 22px;
} }
#blockchain-container {
position: relative;
overflow-x: scroll;
overflow-y: hidden;
scrollbar-width: none;
-ms-overflow-style: none;
}
#blockchain-container::-webkit-scrollbar {
display: none;
}
.fade-border {
-webkit-mask-image: linear-gradient(to right, transparent 0%, black 10%, black 80%, transparent 100%)
}
.main-title { .main-title {
position: relative; position: relative;
@ -45,7 +64,7 @@
} }
.more-padding { .more-padding {
padding: 18px; padding: 24px 20px !important;
} }
.card-wrapper { .card-wrapper {
@ -78,3 +97,10 @@
.card-text { .card-text {
font-size: 22px; font-size: 22px;
} }
.title-link, .title-link:hover, .title-link:focus, .title-link:active {
display: block;
margin-bottom: 10px;
text-decoration: none;
color: inherit;
}

View File

@ -30,21 +30,28 @@
} }
.widget { .widget {
width: 99vw; width: 90vw;
margin-left: auto;
margin-right: auto;
height: 250px; height: 250px;
-webkit-mask: linear-gradient(0deg, #11131f00 5%, #11131fff 25%); -webkit-mask: linear-gradient(0deg, #11131f00 5%, #11131fff 25%);
@media (max-width: 767.98px) {
width: 100vw;
}
} }
.widget > .chart { .widget > .chart {
-webkit-mask: linear-gradient(180deg, #11131f00 0%, #11131fff 20%);
min-height: 250px; min-height: 250px;
-webkit-mask: linear-gradient(180deg, #11131f00 0%, #11131fff 20%);
@media (max-width: 767.98px) {
padding-bottom: 0px;
}
} }
.chart { .chart {
min-height: 500px; min-height: 500px;
width: 100%; width: 100%;
height: 100%; height: 100%;
padding-right: 10px;
@media (max-width: 992px) { @media (max-width: 992px) {
padding-bottom: 25px; padding-bottom: 25px;
} }

View File

@ -8,6 +8,7 @@ import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.
import { StateService } from 'src/app/services/state.service'; import { StateService } from 'src/app/services/state.service';
import { EChartsOption, registerMap } from 'echarts'; import { EChartsOption, registerMap } from 'echarts';
import 'echarts-gl'; import 'echarts-gl';
import { isMobile } from 'src/app/shared/common.utils';
@Component({ @Component({
selector: 'app-nodes-channels-map', selector: 'app-nodes-channels-map',
@ -16,11 +17,18 @@ import 'echarts-gl';
changeDetection: ChangeDetectionStrategy.OnPush, changeDetection: ChangeDetectionStrategy.OnPush,
}) })
export class NodesChannelsMap implements OnInit, OnDestroy { export class NodesChannelsMap implements OnInit, OnDestroy {
@Input() style: 'graph' | 'nodepage' | 'widget' = 'graph'; @Input() style: 'graph' | 'nodepage' | 'widget' | 'channelpage' = 'graph';
@Input() publicKey: string | undefined; @Input() publicKey: string | undefined;
@Input() channel: any[] = [];
observable$: Observable<any>; observable$: Observable<any>;
center: number[] | undefined = undefined;
center: number[] | undefined;
zoom: number | undefined;
channelWidth = 0.6;
channelOpacity = 0.1;
channelColor = '#466d9d';
channelCurve = 0;
chartInstance = undefined; chartInstance = undefined;
chartOptions: EChartsOption = {}; chartOptions: EChartsOption = {};
@ -42,8 +50,16 @@ export class NodesChannelsMap implements OnInit, OnDestroy {
ngOnDestroy(): void {} ngOnDestroy(): void {}
ngOnInit(): void { ngOnInit(): void {
this.center = this.style === 'widget' ? [0, 0, -10] : undefined; this.center = this.style === 'widget' ? [0, 40] : [0, 5];
this.zoom = 1.3;
if (this.style === 'widget' && !isMobile()) {
this.zoom = 3.5;
}
if (this.style === 'widget' && isMobile()) {
this.zoom = 1.4;
this.center = [0, 10];
}
if (this.style === 'graph') { if (this.style === 'graph') {
this.seoService.setTitle($localize`Lightning nodes channels world map`); this.seoService.setTitle($localize`Lightning nodes channels world map`);
} }
@ -62,36 +78,85 @@ export class NodesChannelsMap implements OnInit, OnDestroy {
const nodes = []; const nodes = [];
const nodesPubkeys = {}; const nodesPubkeys = {};
let thisNodeGPS: number[] | undefined = undefined; let thisNodeGPS: number[] | undefined = undefined;
for (const channel of data[1]) {
let geoloc = data[1];
if (this.style === 'channelpage') {
if (this.channel.length === 0) {
geoloc = [];
} else {
geoloc = [this.channel];
}
}
for (const channel of geoloc) {
if (!thisNodeGPS && data[2] === channel[0]) { if (!thisNodeGPS && data[2] === channel[0]) {
thisNodeGPS = [channel[2], channel[3]]; thisNodeGPS = [channel[2], channel[3]];
} else if (!thisNodeGPS && data[2] === channel[4]) { } else if (!thisNodeGPS && data[2] === channel[4]) {
thisNodeGPS = [channel[6], channel[7]]; thisNodeGPS = [channel[6], channel[7]];
} }
channelsLoc.push([[channel[2], channel[3]], [channel[6], channel[7]]]); // 0 - node1 pubkey
// 1 - node1 alias
// 2,3 - node1 GPS
// 4 - node2 pubkey
// 5 - node2 alias
// 6,7 - node2 GPS
// We add a bit of noise so nodes at the same location are not all
// on top of each other
let random = Math.random() * 2 * Math.PI;
let random2 = Math.random() * 0.01;
if (!nodesPubkeys[channel[0]]) { if (!nodesPubkeys[channel[0]]) {
nodes.push({ nodes.push([
publicKey: channel[0], channel[2] + random2 * Math.cos(random),
name: channel[1], channel[3] + random2 * Math.sin(random),
value: [channel[2], channel[3]], 1,
}); channel[0],
nodesPubkeys[channel[0]] = true; channel[1]
]);
nodesPubkeys[channel[0]] = nodes[nodes.length - 1];
} }
random = Math.random() * 2 * Math.PI;
random2 = Math.random() * 0.01;
if (!nodesPubkeys[channel[4]]) { if (!nodesPubkeys[channel[4]]) {
nodes.push({ nodes.push([
publicKey: channel[4], channel[6] + random2 * Math.cos(random),
name: channel[5], channel[7] + random2 * Math.sin(random),
value: [channel[6], channel[7]], 1,
}); channel[4],
nodesPubkeys[channel[4]] = true; channel[5]
]);
nodesPubkeys[channel[4]] = nodes[nodes.length - 1];
} }
const channelLoc = [];
channelLoc.push(nodesPubkeys[channel[0]].slice(0, 2));
channelLoc.push(nodesPubkeys[channel[4]].slice(0, 2));
channelsLoc.push(channelLoc);
} }
if (this.style === 'nodepage' && thisNodeGPS) { if (this.style === 'nodepage' && thisNodeGPS) {
// 1ML 0217890e3aad8d35bc054f43acc00084b25229ecff0ab68debd82883ad65ee8266 this.center = [thisNodeGPS[0], thisNodeGPS[1]];
// New York GPS [-74.0068, 40.7123] this.zoom = 10;
// Map center [-20.55, 0, -9.85] this.channelWidth = 1;
this.center = [thisNodeGPS[0] * -20.55 / -74.0068, 0, thisNodeGPS[1] * -9.85 / 40.7123]; this.channelOpacity = 1;
}
if (this.style === 'channelpage' && this.channel.length > 0) {
this.channelWidth = 2;
this.channelOpacity = 1;
this.channelColor = '#bafcff';
this.channelCurve = 0.1;
this.center = [
(this.channel[2] + this.channel[6]) / 2,
(this.channel[3] + this.channel[7]) / 2
];
const distance = Math.sqrt(
Math.pow(this.channel[7] - this.channel[3], 2) +
Math.pow(this.channel[6] - this.channel[2], 2)
);
this.zoom = -0.05 * distance + 8;
} }
this.prepareChartOptions(nodes, channelsLoc); this.prepareChartOptions(nodes, channelsLoc);
@ -115,87 +180,83 @@ export class NodesChannelsMap implements OnInit, OnDestroy {
} }
this.chartOptions = { this.chartOptions = {
silent: this.style === 'widget' ? true : false, silent: this.style === 'widget',
title: title ?? undefined, title: title ?? undefined,
geo3D: { tooltip: {},
map: 'world', geo: {
shading: 'color', animation: false,
silent: true, silent: true,
postEffect: { center: this.center,
enable: true, zoom: this.zoom,
bloom: { tooltip: {
intensity: 0.1, show: true
}
},
viewControl: {
center: this.center,
minDistance: 1,
maxDistance: 60,
distance: this.style === 'widget' ? 22 : this.style === 'nodepage' ? 22 : 60,
alpha: 90,
rotateSensitivity: 0,
panSensitivity: this.style === 'widget' ? 0 : 1,
zoomSensitivity: this.style === 'widget' ? 0 : 0.5,
panMouseButton: this.style === 'widget' ? null : 'left',
rotateMouseButton: undefined,
}, },
map: 'world',
roam: this.style === 'widget' ? false : true,
itemStyle: { itemStyle: {
color: 'white',
opacity: 0.02,
borderWidth: 1,
borderColor: 'black', borderColor: 'black',
color: '#ffffff44'
}, },
regionHeight: 0.01, scaleLimit: {
min: 1.3,
max: 100000,
}
}, },
series: [ series: [
{ {
// @ts-ignore large: true,
type: 'lines3D', type: 'scatter',
coordinateSystem: 'geo3D', data: nodes,
blendMode: 'lighter', coordinateSystem: 'geo',
lineStyle: { geoIndex: 0,
width: 1, symbolSize: 4,
opacity: ['widget', 'graph'].includes(this.style) ? 0.025 : 1, tooltip: {
backgroundColor: 'rgba(17, 19, 31, 1)',
borderRadius: 4,
shadowColor: 'rgba(0, 0, 0, 0.5)',
textStyle: {
color: '#b1b1b1',
align: 'left',
},
borderColor: '#000',
formatter: (value) => {
const data = value.data;
const alias = data[4].length > 0 ? data[4] : data[3].slice(0, 20);
return `<b style="color: white">${alias}</b>`;
}
}, },
data: channels itemStyle: {
color: 'white',
borderColor: 'black',
borderWidth: 2,
opacity: 1,
},
blendMode: 'lighter',
zlevel: 1,
}, },
{ {
// @ts-ignore large: false,
type: 'scatter3D', progressive: 200,
symbol: 'circle', silent: true,
blendMode: 'lighter', type: 'lines',
coordinateSystem: 'geo3D', coordinateSystem: 'geo',
symbolSize: 3, data: channels,
itemStyle: { lineStyle: {
color: '#BBFFFF', opacity: this.channelOpacity,
opacity: 1, width: this.channelWidth,
borderColor: '#FFFFFF00', curveness: this.channelCurve,
color: this.channelColor,
}, },
data: nodes, blendMode: 'lighter',
emphasis: { tooltip: {
label: { show: false,
position: 'top', },
color: 'white', zlevel: 2,
fontSize: 16, }
formatter: function(value) {
return value.name;
},
show: true,
}
}
},
] ]
}; };
} }
@HostListener('window:wheel', ['$event'])
onWindowScroll(e): void {
// Not very smooth when using the mouse
if (this.style === 'widget' && e.target.tagName === 'CANVAS') {
window.scrollBy({left: 0, top: e.deltaY, behavior: 'auto'});
}
}
onChartInit(ec) { onChartInit(ec) {
if (this.chartInstance !== undefined) { if (this.chartInstance !== undefined) {
return; return;
@ -211,14 +272,34 @@ export class NodesChannelsMap implements OnInit, OnDestroy {
}); });
}); });
} }
this.chartInstance.on('click', (e) => { this.chartInstance.on('click', (e) => {
if (e.data && e.data.publicKey) { if (e.data) {
this.zone.run(() => { this.zone.run(() => {
const url = new RelativeUrlPipe(this.stateService).transform(`/lightning/node/${e.data.publicKey}`); const url = new RelativeUrlPipe(this.stateService).transform(`/lightning/node/${e.data[3]}`);
this.router.navigate([url]); this.router.navigate([url]);
}); });
} }
}); });
this.chartInstance.on('georoam', (e) => {
if (!e.zoom || this.style === 'nodepage') {
return;
}
const speed = 0.005;
const chartOptions = {
series: this.chartOptions.series
};
chartOptions.series[1].lineStyle.opacity += e.zoom > 1 ? speed : -speed;
chartOptions.series[1].lineStyle.width += e.zoom > 1 ? speed : -speed;
chartOptions.series[0].symbolSize += e.zoom > 1 ? speed * 10 : -speed * 10;
chartOptions.series[1].lineStyle.opacity = Math.max(0.05, Math.min(0.5, chartOptions.series[1].lineStyle.opacity));
chartOptions.series[1].lineStyle.width = Math.max(0.5, Math.min(1, chartOptions.series[1].lineStyle.width));
chartOptions.series[0].symbolSize = Math.max(4, Math.min(5.5, chartOptions.series[0].symbolSize));
this.chartInstance.setOption(chartOptions);
});
} }
} }

View File

@ -3,18 +3,18 @@
<table class="table table-borderless"> <table class="table table-borderless">
<thead> <thead>
<th class="alias text-left" i18n="nodes.alias">Alias</th> <th class="alias text-left" i18n="nodes.alias">Alias</th>
<th class="capacity text-right" i18n="node.capacity">Capacity</th> <th class="capacity text-right" [class]="show" i18n="node.capacity">Capacity</th>
<th class="channels text-right" i18n="node.channels">Channels</th> <th class="channels text-right" [class]="show" i18n="node.channels">Channels</th>
</thead> </thead>
<tbody *ngIf="nodes$ | async as nodes; else skeleton"> <tbody *ngIf="nodes$ | async as nodes; else skeleton">
<tr *ngFor="let node of nodes; let i = index;"> <tr *ngFor="let node of nodes; let i = index;">
<td class="alias text-left"> <td class="alias text-left">
<a [routerLink]="['/lightning/node' | relativeUrl, node.public_key]">{{ node.alias }}</a> <a [routerLink]="['/lightning/node' | relativeUrl, node.public_key]">{{ node.alias }}</a>
</td> </td>
<td class="capacity text-right"> <td class="capacity text-right" [class]="show">
<app-amount [satoshis]="node.capacity" digitsInfo="1.2-2"></app-amount> <app-amount [satoshis]="node.capacity" digitsInfo="1.2-2"></app-amount>
</td> </td>
<td class="channels text-right"> <td class="channels text-right" [class]="show">
{{ node.channels | number }} {{ node.channels | number }}
</td> </td>
</tr> </tr>

View File

@ -0,0 +1,11 @@
.capacity.mobile-channels {
@media (max-width: 767.98px) {
display: none;
}
}
.channels.mobile-capacity {
@media (max-width: 767.98px) {
display: none;
}
}

View File

@ -9,6 +9,7 @@ import { Observable } from 'rxjs';
}) })
export class NodesListComponent implements OnInit { export class NodesListComponent implements OnInit {
@Input() nodes$: Observable<any>; @Input() nodes$: Observable<any>;
@Input() show: string;
constructor() { } constructor() { }

View File

@ -1,6 +1,29 @@
<div class="full-container h-100"> <div [class]="widget === false ? 'full-container' : ''">
<div class="card-header"> <div *ngIf="widget">
<div class="pool-distribution" *ngIf="(nodesPerAsObservable$ | async) as stats; else loadingReward">
<div class="item">
<h5 class="card-title d-inline-block" i18n="lightning.tagged-isp">Tagged ISPs</h5>
<p class="card-text">
{{ stats.taggedISP }}
</p>
</div>
<div class="item">
<h5 class="card-title d-inline-block" i18n="lightning.tagged-capacity">Tagged capacity</h5>
<p class="card-text" i18n-ngbTooltip="mining.blocks-count-desc">
<app-amount [satoshis]="stats.taggedCapacity" [digitsInfo]="'1.2-2'" [noFiat]="true"></app-amount>
</p>
</div>
<div class="item">
<h5 class="card-title d-inline-block" i18n="lightning.tagged-nodes">Tagged nodes</h5>
<p class="card-text" i18n-ngbTooltip="mining.pools-count-desc">
{{ stats.taggedNodeCount }}
</p>
</div>
</div>
</div>
<div class="card-header" *ngIf="!widget">
<div class="d-flex d-md-block align-items-baseline" style="margin-bottom: -5px"> <div class="d-flex d-md-block align-items-baseline" style="margin-bottom: -5px">
<span i18n="lightning.nodes-per-isp">Lightning nodes per ISP</span> <span i18n="lightning.nodes-per-isp">Lightning nodes per ISP</span>
<button class="btn p-0 pl-2" style="margin: 0 0 4px 0px" (click)="onSaveChart()"> <button class="btn p-0 pl-2" style="margin: 0 0 4px 0px" (click)="onSaveChart()">
@ -12,23 +35,21 @@
</small> </small>
</div> </div>
<div class="container pb-lg-0 bottom-padding"> <div [class]="!widget ? 'bottom-padding' : 'pb-0'" class="container pb-lg-0">
<div class="pb-lg-5" *ngIf="nodesPerAsObservable$ | async"> <div [class]="widget ? 'chart-widget' : 'chart'" echarts [initOpts]="chartInitOptions" [options]="chartOptions"
<div class="chart w-100" echarts [initOpts]="chartInitOptions" [options]="chartOptions" (chartInit)="onChartInit($event)">
(chartInit)="onChartInit($event)">
</div>
</div> </div>
<div class="text-center loadingGraphs" *ngIf="isLoading"> <div class="text-center loadingGraphs" *ngIf="isLoading">
<div class="spinner-border text-light"></div> <div class="spinner-border text-light"></div>
</div> </div>
<div class="d-flex toggle"> <div class="d-flex toggle" *ngIf="!widget">
<app-toggle [textLeft]="'Show Tor'" [textRight]="" (toggleStatusChanged)="onTorToggleStatusChanged($event)"></app-toggle> <app-toggle [textLeft]="'Show Tor'" [textRight]="" (toggleStatusChanged)="onTorToggleStatusChanged($event)"></app-toggle>
<app-toggle [textLeft]="'Nodes'" [textRight]="'Capacity'" (toggleStatusChanged)="onGroupToggleStatusChanged($event)"></app-toggle> <app-toggle [textLeft]="'Nodes'" [textRight]="'Capacity'" (toggleStatusChanged)="onGroupToggleStatusChanged($event)"></app-toggle>
</div> </div>
<table class="table table-borderless text-center m-auto" style="max-width: 900px"> <table class="table table-borderless text-center m-auto" style="max-width: 900px" *ngIf="!widget">
<thead> <thead>
<tr> <tr>
<th class="rank text-left pl-0" i18n="mining.rank">Rank</th> <th class="rank text-left pl-0" i18n="mining.rank">Rank</th>
@ -39,7 +60,7 @@
</tr> </tr>
</thead> </thead>
<tbody [attr.data-cy]="'pools-table'" *ngIf="(nodesPerAsObservable$ | async) as asList"> <tbody [attr.data-cy]="'pools-table'" *ngIf="(nodesPerAsObservable$ | async) as asList">
<tr *ngFor="let asEntry of asList"> <tr *ngFor="let asEntry of asList.data">
<td class="rank text-left pl-0">{{ asEntry.rank }}</td> <td class="rank text-left pl-0">{{ asEntry.rank }}</td>
<td class="name text-left text-truncate"> <td class="name text-left text-truncate">
<a *ngIf="asEntry.ispId" [routerLink]="[('/lightning/nodes/isp/' + asEntry.ispId) | relativeUrl]">{{ asEntry.name }}</a> <a *ngIf="asEntry.ispId" [routerLink]="[('/lightning/nodes/isp/' + asEntry.ispId) | relativeUrl]">{{ asEntry.name }}</a>
@ -54,3 +75,26 @@
</div> </div>
</div> </div>
<ng-template #loadingReward>
<div class="pool-distribution">
<div class="item">
<h5 class="card-title" i18n="lightning.tagged-isp">Tagged ISPs</h5>
<p class="card-text">
<span class="skeleton-loader skeleton-loader-big"></span>
</p>
</div>
<div class="item">
<h5 class="card-title" i18n="lightning.tagged-capacity">Tagged capacity</h5>
<p class="card-text">
<span class="skeleton-loader skeleton-loader-big"></span>
</p>
</div>
<div class="item">
<h5 class="card-title" i18n="lightning.tagged-nodes">Tagged nodes</h5>
<p class="card-text">
<span class="skeleton-loader skeleton-loader-big"></span>
</p>
</div>
</div>
</ng-template>

View File

@ -22,7 +22,40 @@
max-height: 400px; max-height: 400px;
@media (max-width: 767.98px) { @media (max-width: 767.98px) {
max-height: 230px; max-height: 230px;
margin-top: -35px; margin-top: -40px;
}
}
.chart-widget {
width: 100%;
height: 100%;
height: 240px;
@media (max-width: 485px) {
max-height: 200px;
}
}
.formRadioGroup {
margin-top: 6px;
display: flex;
flex-direction: column;
@media (min-width: 991px) {
position: relative;
top: -65px;
}
@media (min-width: 830px) and (max-width: 991px) {
position: relative;
top: 0px;
}
@media (min-width: 830px) {
flex-direction: row;
float: right;
margin-top: 0px;
}
.btn-sm {
font-size: 9px;
@media (min-width: 830px) {
font-size: 14px;
}
} }
} }
@ -35,6 +68,79 @@
}; };
} }
@media (max-width: 767.98px) {
.pools-table th,
.pools-table td {
padding: .3em !important;
}
}
.loadingGraphs {
position: absolute;
top: 50%;
left: calc(50% - 15px);
z-index: 100;
}
.pool-distribution {
min-height: 56px;
display: block;
@media (min-width: 485px) {
display: flex;
flex-direction: row;
}
h5 {
margin-bottom: 5px;
}
.item {
max-width: 160px;
width: 50%;
display: inline-block;
margin: 0px auto 20px;
&:nth-child(2) {
order: 2;
@media (min-width: 485px) {
order: 3;
}
}
&:nth-child(3) {
width: 50%;
order: 3;
@media (min-width: 485px) {
order: 2;
display: block;
}
@media (min-width: 768px) {
display: none;
}
@media (min-width: 992px) {
display: block;
}
}
.card-title {
font-size: 1rem;
color: #4a68b9;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.card-text {
font-size: 18px;
span {
color: #ffffff66;
font-size: 12px;
}
}
}
}
.skeleton-loader {
width: 100%;
display: block;
max-width: 80px;
margin: 15px auto 3px;
}
.rank { .rank {
width: 15%; width: 15%;
@media (max-width: 576px) { @media (max-width: 576px) {

View File

@ -1,11 +1,12 @@
import { ChangeDetectionStrategy, Component, OnInit, HostBinding, NgZone } from '@angular/core'; import { ChangeDetectionStrategy, Component, OnInit, HostBinding, NgZone, Input } from '@angular/core';
import { Router } from '@angular/router'; import { Router } from '@angular/router';
import { EChartsOption, PieSeriesOption } from 'echarts'; import { EChartsOption, PieSeriesOption } from 'echarts';
import { combineLatest, map, Observable, share, Subject, switchMap, tap } from 'rxjs'; import { combineLatest, map, Observable, share, startWith, Subject, switchMap, tap } from 'rxjs';
import { chartColors } from 'src/app/app.constants'; import { chartColors } from 'src/app/app.constants';
import { ApiService } from 'src/app/services/api.service'; import { ApiService } from 'src/app/services/api.service';
import { SeoService } from 'src/app/services/seo.service'; import { SeoService } from 'src/app/services/seo.service';
import { StateService } from 'src/app/services/state.service'; import { StateService } from 'src/app/services/state.service';
import { isMobile } from 'src/app/shared/common.utils';
import { download } from 'src/app/shared/graphs.utils'; import { download } from 'src/app/shared/graphs.utils';
import { AmountShortenerPipe } from 'src/app/shared/pipes/amount-shortener.pipe'; import { AmountShortenerPipe } from 'src/app/shared/pipes/amount-shortener.pipe';
import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.pipe'; import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.pipe';
@ -17,6 +18,8 @@ import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.
changeDetection: ChangeDetectionStrategy.OnPush, changeDetection: ChangeDetectionStrategy.OnPush,
}) })
export class NodesPerISPChartComponent implements OnInit { export class NodesPerISPChartComponent implements OnInit {
@Input() widget: boolean = false;
isLoading = true; isLoading = true;
chartOptions: EChartsOption = {}; chartOptions: EChartsOption = {};
chartInitOptions = { chartInitOptions = {
@ -46,7 +49,11 @@ export class NodesPerISPChartComponent implements OnInit {
this.seoService.setTitle($localize`Lightning nodes per ISP`); this.seoService.setTitle($localize`Lightning nodes per ISP`);
this.showTorObservable$ = this.showTorSubject.asObservable(); this.showTorObservable$ = this.showTorSubject.asObservable();
this.nodesPerAsObservable$ = combineLatest([this.groupBySubject, this.showTorSubject])
this.nodesPerAsObservable$ = combineLatest([
this.groupBySubject.pipe(startWith(false)),
this.showTorSubject.pipe(startWith(false)),
])
.pipe( .pipe(
switchMap((selectedFilters) => { switchMap((selectedFilters) => {
return this.apiService.getNodesPerAs( return this.apiService.getNodesPerAs(
@ -62,23 +69,41 @@ export class NodesPerISPChartComponent implements OnInit {
for (let i = 0; i < data.length; ++i) { for (let i = 0; i < data.length; ++i) {
data[i].rank = i + 1; data[i].rank = i + 1;
} }
return data.slice(0, 100); return {
taggedISP: data.length,
taggedCapacity: data.reduce((partialSum, isp) => partialSum + isp.capacity, 0),
taggedNodeCount: data.reduce((partialSum, isp) => partialSum + isp.count, 0),
data: data.slice(0, 100),
};
}) })
); );
}), }),
share() share()
); );
if (this.widget) {
this.showTorSubject.next(false);
this.groupBySubject.next(false);
}
} }
generateChartSerieData(as): PieSeriesOption[] { generateChartSerieData(as): PieSeriesOption[] {
const shareThreshold = this.isMobile() ? 2 : 0.5; let shareThreshold = 0.5;
if (this.widget && isMobile() || isMobile()) {
shareThreshold = 1;
} else if (this.widget) {
shareThreshold = 0.75;
}
const data: object[] = []; const data: object[] = [];
let totalShareOther = 0; let totalShareOther = 0;
let totalNodeOther = 0; let totalNodeOther = 0;
let edgeDistance: string | number = '10%'; let edgeDistance: string | number = '10%';
if (this.isMobile()) { if (isMobile() && this.widget) {
edgeDistance = 0; edgeDistance = 0;
} else if (isMobile() && !this.widget || this.widget) {
edgeDistance = 10;
} }
as.forEach((as) => { as.forEach((as) => {
@ -92,15 +117,16 @@ export class NodesPerISPChartComponent implements OnInit {
color: as.ispId === null ? '#7D4698' : undefined, color: as.ispId === null ? '#7D4698' : undefined,
}, },
value: as.share, value: as.share,
name: as.name + (this.isMobile() ? `` : ` (${as.share}%)`), name: as.name + (isMobile() || this.widget ? `` : ` (${as.share}%)`),
label: { label: {
overflow: 'truncate', overflow: 'truncate',
width: isMobile() ? 75 : this.widget ? 125 : 250,
color: '#b1b1b1', color: '#b1b1b1',
alignTo: 'edge', alignTo: 'edge',
edgeDistance: edgeDistance, edgeDistance: edgeDistance,
}, },
tooltip: { tooltip: {
show: !this.isMobile(), show: !isMobile(),
backgroundColor: 'rgba(17, 19, 31, 1)', backgroundColor: 'rgba(17, 19, 31, 1)',
borderRadius: 4, borderRadius: 4,
shadowColor: 'rgba(0, 0, 0, 0.5)', shadowColor: 'rgba(0, 0, 0, 0.5)',
@ -125,7 +151,7 @@ export class NodesPerISPChartComponent implements OnInit {
color: 'grey', color: 'grey',
}, },
value: totalShareOther, value: totalShareOther,
name: 'Other' + (this.isMobile() ? `` : ` (${totalShareOther.toFixed(2)}%)`), name: 'Other' + (isMobile() || this.widget ? `` : ` (${totalShareOther.toFixed(2)}%)`),
label: { label: {
overflow: 'truncate', overflow: 'truncate',
color: '#b1b1b1', color: '#b1b1b1',
@ -153,7 +179,7 @@ export class NodesPerISPChartComponent implements OnInit {
prepareChartOptions(as): void { prepareChartOptions(as): void {
let pieSize = ['20%', '80%']; // Desktop let pieSize = ['20%', '80%']; // Desktop
if (this.isMobile()) { if (isMobile() && !this.widget) {
pieSize = ['15%', '60%']; pieSize = ['15%', '60%'];
} }
@ -177,8 +203,8 @@ export class NodesPerISPChartComponent implements OnInit {
lineStyle: { lineStyle: {
width: 2, width: 2,
}, },
length: this.isMobile() ? 1 : 20, length: isMobile() ? 1 : 20,
length2: this.isMobile() ? 1 : undefined, length2: isMobile() ? 1 : undefined,
}, },
label: { label: {
fontSize: 14, fontSize: 14,
@ -204,10 +230,6 @@ export class NodesPerISPChartComponent implements OnInit {
}; };
} }
isMobile(): boolean {
return (window.innerWidth <= 767.98);
}
onChartInit(ec): void { onChartInit(ec): void {
if (this.chartInstance !== undefined) { if (this.chartInstance !== undefined) {
return; return;
@ -244,5 +266,9 @@ export class NodesPerISPChartComponent implements OnInit {
onGroupToggleStatusChanged(e): void { onGroupToggleStatusChanged(e): void {
this.groupBySubject.next(e); this.groupBySubject.next(e);
} }
isEllipsisActive(e) {
return (e.offsetWidth < e.scrollWidth);
}
} }

View File

@ -51,8 +51,7 @@
} }
.chart-widget { .chart-widget {
width: 100%; width: 100%;
height: 100%; height: 320px;
max-height: 270px;
} }
.formRadioGroup { .formRadioGroup {

View File

@ -34,10 +34,11 @@ esac
TOR_INSTALL=ON TOR_INSTALL=ON
CERTBOT_INSTALL=ON CERTBOT_INSTALL=ON
# install 3 network daemons # install 4 network daemons
BITCOIN_INSTALL=ON BITCOIN_INSTALL=ON
BISQ_INSTALL=ON BISQ_INSTALL=ON
ELEMENTS_INSTALL=ON ELEMENTS_INSTALL=ON
CLN_INSTALL=ON
# install UNFURL # install UNFURL
UNFURL_INSTALL=ON UNFURL_INSTALL=ON
@ -191,6 +192,7 @@ case $OS in
NGINX_ETC_FOLDER=/usr/local/etc/nginx NGINX_ETC_FOLDER=/usr/local/etc/nginx
NGINX_CONFIGURATION=/usr/local/etc/nginx/nginx.conf NGINX_CONFIGURATION=/usr/local/etc/nginx/nginx.conf
CERTBOT_PKG=py39-certbot CERTBOT_PKG=py39-certbot
CLN_PKG=c-lightning
;; ;;
Debian) Debian)
@ -275,6 +277,12 @@ ELECTRS_LIQUID_DATA=${ELECTRS_DATA_ROOT}/liquid
ELECTRS_LIQUIDTESTNET_ZPOOL=${ZPOOL} ELECTRS_LIQUIDTESTNET_ZPOOL=${ZPOOL}
ELECTRS_LIQUIDTESTNET_DATA=${ELECTRS_DATA_ROOT}/liquidtestnet ELECTRS_LIQUIDTESTNET_DATA=${ELECTRS_DATA_ROOT}/liquidtestnet
# Core Lightning user/group
CLN_USER=cln
CLN_GROUP=cln
# Core Lightning home folder
CLN_HOME=/cln
# bisq user/group # bisq user/group
BISQ_USER=bisq BISQ_USER=bisq
BISQ_GROUP=bisq BISQ_GROUP=bisq
@ -596,6 +604,10 @@ zfsCreateFilesystems()
done done
fi fi
if [ "${CLN_INSTALL}" = ON ];then
zfs create -o "mountpoint=${CLN_HOME}" "${ZPOOL}/cln"
fi
if [ "${BISQ_INSTALL}" = ON ];then if [ "${BISQ_INSTALL}" = ON ];then
zfs create -o "mountpoint=${BISQ_HOME}" "${ZPOOL}/bisq" zfs create -o "mountpoint=${BISQ_HOME}" "${ZPOOL}/bisq"
fi fi
@ -675,6 +687,10 @@ ext4CreateDir()
done done
fi fi
if [ "${CLN_INSTALL}" = ON ];then
mkdir -p "${CLN_HOME}"
fi
if [ "${BISQ_INSTALL}" = ON ];then if [ "${BISQ_INSTALL}" = ON ];then
mkdir -p "${BISQ_HOME}" mkdir -p "${BISQ_HOME}"
fi fi
@ -735,6 +751,7 @@ Testnet:Enable Bitcoin Testnet:ON
Signet:Enable Bitcoin Signet:ON Signet:Enable Bitcoin Signet:ON
Liquid:Enable Elements Liquid:ON Liquid:Enable Elements Liquid:ON
Liquidtestnet:Enable Elements Liquidtestnet:ON Liquidtestnet:Enable Elements Liquidtestnet:ON
CoreLN:Enable Core Lightning:ON
Bisq:Enable Bisq:ON Bisq:Enable Bisq:ON
Unfurl:Enable Unfurl:ON Unfurl:Enable Unfurl:ON
EOF EOF
@ -810,6 +827,11 @@ else
ELEMENTS_INSTALL=OFF ELEMENTS_INSTALL=OFF
fi fi
if grep CoreLN $tempfile >/dev/null 2>&1;then
CLN_INSTALL=ON
else
CLN_INSTALL=OFF
if [ "${BITCOIN_MAINNET_ENABLE}" = ON -o "${BITCOIN_TESTNET_ENABLE}" = ON -o "${BITCOIN_SIGNET_ENABLE}" = ON ];then if [ "${BITCOIN_MAINNET_ENABLE}" = ON -o "${BITCOIN_TESTNET_ENABLE}" = ON -o "${BITCOIN_SIGNET_ENABLE}" = ON ];then
BITCOIN_ELECTRS_INSTALL=ON BITCOIN_ELECTRS_INSTALL=ON
else else
@ -1234,6 +1256,33 @@ if [ "${ELEMENTS_ELECTRS_INSTALL}" = ON ];then
osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_ELECTRS_HOME} && cargo run --release --features liquid --bin electrs -- --network liquid --version" || true osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_ELECTRS_HOME} && cargo run --release --features liquid --bin electrs -- --network liquid --version" || true
fi fi
#####################################
# Core Lightning for Bitcoin Mainnet #
#####################################
echo "[*] Installing Core Lightning"
case $OS in
FreeBSD)
echo "[*] Creating Core Lightning user"
osGroupCreate "${CLN_GROUP}"
osUserCreate "${CLN_USER}" "${CLN_HOME}" "${CLN_GROUP}"
osSudo "${ROOT_USER}" chsh -s `which zsh` "${CLN_USER}"
osSudo "${CLN_USER}" touch "${CLN_HOME}/.zshrc"
osSudo "${ROOT_USER}" chown -R "${CLN_USER}:${CLN_GROUP}" "${CLN_HOME}"
echo "[*] Installing Core Lightning package"
osPackageInstall ${CLN_PKG}
echo "[*] Installing Core Lightning mainnet Cronjob"
crontab_cln+='@reboot sleep 30 ; screen -dmS main lightningd --alias `hostname` --bitcoin-datadir /bitcoin\n'
crontab_cln+='@reboot sleep 60 ; screen -dmS sig lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network signet\n'
crontab_cln+='@reboot sleep 90 ; screen -dmS tes lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network testnet\n'
echo "${crontab_cln}" | crontab -u "${CLN_USER}" -
;;
Debian)
;;
esac
##################### #####################
# Bisq installation # # Bisq installation #
##################### #####################