machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into knorrium/backend_unit_tests

Browse Source
This commit is contained in:
Felipe Knorr Kuhn
2022-08-12 22:46:35 -07:00
parent 4a9456eef7 ca47cd69f5
commit f6aee4262e
47 changed files with 1338 additions and 569 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/mempool-config.sample.json
View File

@@ -21,7 +21,9 @@
"EXTERNAL_RETRY_INTERVAL": 0,
"USER_AGENT": "mempool",
"STDOUT_LOG_MIN_PRIORITY": "debug",
"AUTOMATIC_BLOCK_REINDEXING": false
"AUTOMATIC_BLOCK_REINDEXING": false,
"POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
"POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master"
},
"CORE_RPC": {
"HOST": "127.0.0.1",

15
backend/src/api/blocks.ts
View File

@@ -22,6 +22,8 @@ import poolsParser from './pools-parser';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import mining from './mining/mining';
import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
import PricesRepository from '../repositories/PricesRepository';
import priceUpdater from '../tasks/price-updater';
class Blocks {
private blocks: BlockExtended[] = [];
@@ -457,6 +459,19 @@ class Blocks {
}
await blocksRepository.$saveBlockInDatabase(blockExtended);
const lastestPriceId = await PricesRepository.$getLatestPriceId();
if (priceUpdater.historyInserted === true && lastestPriceId !== null) {
await blocksRepository.$saveBlockPrices([{
height: blockExtended.height,
priceId: lastestPriceId,
}]);
} else {
logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`)
setTimeout(() => {
indexer.runSingleTask('blocksPrices');
}, 10000);
}
// Save blocks summary for visualization if it's enabled
if (Common.blocksSummariesIndexingEnabled() === true) {
await this.$getStrippedBlockTransactions(blockExtended.id, true);

37
backend/src/api/common.ts
View File

@@ -184,4 +184,41 @@ export class Common {
config.MEMPOOL.BLOCKS_SUMMARIES_INDEXING === true
);
}
static setDateMidnight(date: Date): void {
date.setUTCHours(0);
date.setUTCMinutes(0);
date.setUTCSeconds(0);
date.setUTCMilliseconds(0);
}
static channelShortIdToIntegerId(channelId: string): string {
if (channelId.indexOf('x') === -1) { // Already an integer id
return channelId;
}
if (channelId.indexOf('/') !== -1) { // Topology import
channelId = channelId.slice(0, -2);
}
const s = channelId.split('x').map(part => BigInt(part));
return ((s[0] << 40n) | (s[1] << 16n) | s[2]).toString();
}
/** Decodes a channel id returned by lnd as uint64 to a short channel id */
static channelIntegerIdToShortId(id: string): string {
if (id.indexOf('x') !== -1) { // Already a short id
return id;
}
const n = BigInt(id);
return [
n >> 40n, // nth block
(n >> 16n) & 0xffffffn, // nth tx of the block
n & 0xffffn // nth output of the tx
].join('x');
}
static utcDateToMysql(date?: number): string {
const d = new Date((date || 0) * 1000);
return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
}
}

6
backend/src/api/database-migration.ts
View File

@@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 35;
private static currentVersion = 36;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -320,6 +320,10 @@ class DatabaseMigration {
await this.$executeQuery('DELETE from `lightning_stats` WHERE added > "2021-09-19"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD CONSTRAINT added_unique UNIQUE (added);');
}
if (databaseSchemaVersion < 36 && isBitcoin == true) {
await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"');
}
}
/**

162
backend/src/api/explorer/channels.api.ts
View File

@@ -1,6 +1,9 @@
import logger from '../../logger';
import DB from '../../database';
import nodesApi from './nodes.api';
import { ResultSetHeader } from 'mysql2';
import { ILightningApi } from '../lightning/lightning-api.interface';
import { Common } from '../common';
class ChannelsApi {
public async $getAllChannels(): Promise<any[]> {
@@ -93,7 +96,31 @@ class ChannelsApi {
public async $getChannel(id: string): Promise<any> {
try {
const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*, ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key WHERE (ns1.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node1_public_key) AND ns2.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node2_public_key)) AND channels.id = ?`;
const query = `
SELECT n1.alias AS alias_left, n1.longitude as node1_longitude, n1.latitude as node1_latitude,
n2.alias AS alias_right, n2.longitude as node2_longitude, n2.latitude as node2_latitude,
channels.*,
ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right
FROM channels
LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key
LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key
WHERE (
ns1.id = (
SELECT MAX(id)
FROM node_stats
WHERE public_key = channels.node1_public_key
)
AND ns2.id = (
SELECT MAX(id)
FROM node_stats
WHERE public_key = channels.node2_public_key
)
)
AND channels.id = ?
`;
const [rows]: any = await DB.query(query, [id]);
if (rows[0]) {
return this.convertChannel(rows[0]);
@@ -286,6 +313,8 @@ class ChannelsApi {
'max_htlc_mtokens': channel.node1_max_htlc_mtokens,
'min_htlc_mtokens': channel.node1_min_htlc_mtokens,
'updated_at': channel.node1_updated_at,
'longitude': channel.node1_longitude,
'latitude': channel.node1_latitude,
},
'node_right': {
'alias': channel.alias_right,
@@ -299,9 +328,140 @@ class ChannelsApi {
'max_htlc_mtokens': channel.node2_max_htlc_mtokens,
'min_htlc_mtokens': channel.node2_min_htlc_mtokens,
'updated_at': channel.node2_updated_at,
'longitude': channel.node2_longitude,
'latitude': channel.node2_latitude,
},
};
}
/**
* Save or update a channel present in the graph
*/
public async $saveChannel(channel: ILightningApi.Channel): Promise<void> {
const [ txid, vout ] = channel.chan_point.split(':');
const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
const query = `INSERT INTO channels
(
id,
short_id,
capacity,
transaction_id,
transaction_vout,
updated_at,
status,
node1_public_key,
node1_base_fee_mtokens,
node1_cltv_delta,
node1_fee_rate,
node1_is_disabled,
node1_max_htlc_mtokens,
node1_min_htlc_mtokens,
node1_updated_at,
node2_public_key,
node2_base_fee_mtokens,
node2_cltv_delta,
node2_fee_rate,
node2_is_disabled,
node2_max_htlc_mtokens,
node2_min_htlc_mtokens,
node2_updated_at
)
VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
capacity = ?,
updated_at = ?,
status = 1,
node1_public_key = ?,
node1_base_fee_mtokens = ?,
node1_cltv_delta = ?,
node1_fee_rate = ?,
node1_is_disabled = ?,
node1_max_htlc_mtokens = ?,
node1_min_htlc_mtokens = ?,
node1_updated_at = ?,
node2_public_key = ?,
node2_base_fee_mtokens = ?,
node2_cltv_delta = ?,
node2_fee_rate = ?,
node2_is_disabled = ?,
node2_max_htlc_mtokens = ?,
node2_min_htlc_mtokens = ?,
node2_updated_at = ?
;`;
await DB.query(query, [
Common.channelShortIdToIntegerId(channel.channel_id),
Common.channelIntegerIdToShortId(channel.channel_id),
channel.capacity,
txid,
vout,
Common.utcDateToMysql(channel.last_update),
channel.node1_pub,
policy1.fee_base_msat,
policy1.time_lock_delta,
policy1.fee_rate_milli_msat,
policy1.disabled,
policy1.max_htlc_msat,
policy1.min_htlc,
Common.utcDateToMysql(policy1.last_update),
channel.node2_pub,
policy2.fee_base_msat,
policy2.time_lock_delta,
policy2.fee_rate_milli_msat,
policy2.disabled,
policy2.max_htlc_msat,
policy2.min_htlc,
Common.utcDateToMysql(policy2.last_update),
channel.capacity,
Common.utcDateToMysql(channel.last_update),
channel.node1_pub,
policy1.fee_base_msat,
policy1.time_lock_delta,
policy1.fee_rate_milli_msat,
policy1.disabled,
policy1.max_htlc_msat,
policy1.min_htlc,
Common.utcDateToMysql(policy1.last_update),
channel.node2_pub,
policy2.fee_base_msat,
policy2.time_lock_delta,
policy2.fee_rate_milli_msat,
policy2.disabled,
policy2.max_htlc_msat,
policy2.min_htlc,
Common.utcDateToMysql(policy2.last_update)
]);
}
/**
* Set all channels not in `graphChannelsIds` as inactive (status = 0)
*/
public async $setChannelsInactive(graphChannelsIds: string[]): Promise<void> {
if (graphChannelsIds.length === 0) {
return;
}
try {
const result = await DB.query<ResultSetHeader>(`
UPDATE channels
SET status = 0
WHERE id NOT IN (
${graphChannelsIds.map(id => `"${id}"`).join(',')}
)
AND status != 2
`);
if (result[0].changedRows ?? 0 > 0) {
logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`);
} else {
logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`);
}
} catch (e) {
logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
}
}
}
export default new ChannelsApi();

3
backend/src/api/explorer/channels.routes.ts
View File

@@ -32,6 +32,9 @@ class ChannelsRoutes {
res.status(404).send('Channel not found');
return;
}
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(channel);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);

64
backend/src/api/explorer/nodes.api.ts
View File

@@ -1,5 +1,7 @@
import logger from '../../logger';
import DB from '../../database';
import { ResultSetHeader } from 'mysql2';
import { ILightningApi } from '../lightning/lightning-api.interface';
class NodesApi {
public async $getNode(public_key: string): Promise<any> {
@@ -166,7 +168,7 @@ class NodesApi {
}
}
public async $getNodesISP(groupBy: string, showTor: boolean) {
public async $getNodesISPRanking(groupBy: string, showTor: boolean) {
try {
const orderBy = groupBy === 'capacity' ? `CAST(SUM(capacity) as INT)` : `COUNT(DISTINCT nodes.public_key)`;
@@ -321,6 +323,66 @@ class NodesApi {
throw e;
}
}
/**
* Save or update a node present in the graph
*/
public async $saveNode(node: ILightningApi.Node): Promise<void> {
try {
const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
const query = `INSERT INTO nodes(
public_key,
first_seen,
updated_at,
alias,
color,
sockets,
status
)
VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, 1)
ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, color = ?, sockets = ?, status = 1`;
await DB.query(query, [
node.pub_key,
node.last_update,
node.alias,
node.color,
sockets,
node.last_update,
node.alias,
node.color,
sockets,
]);
} catch (e) {
logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));
}
}
/**
* Set all nodes not in `nodesPubkeys` as inactive (status = 0)
*/
public async $setNodesInactive(graphNodesPubkeys: string[]): Promise<void> {
if (graphNodesPubkeys.length === 0) {
return;
}
try {
const result = await DB.query<ResultSetHeader>(`
UPDATE nodes
SET status = 0
WHERE public_key NOT IN (
${graphNodesPubkeys.map(pubkey => `"${pubkey}"`).join(',')}
)
`);
if (result[0].changedRows ?? 0 > 0) {
logger.info(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`);
} else {
logger.debug(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`);
}
} catch (e) {
logger.err('$setNodesInactive() error: ' + (e instanceof Error ? e.message : e));
}
}
}
export default new NodesApi();

2
backend/src/api/explorer/nodes.routes.ts
View File

@@ -79,7 +79,7 @@ class NodesRoutes {
return;
}
const nodesPerAs = await nodesApi.$getNodesISP(groupBy, showTor);
const nodesPerAs = await nodesApi.$getNodesISPRanking(groupBy, showTor);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');

13
backend/src/api/lightning/clightning/clightning-convert.ts
View File

@@ -1,6 +1,7 @@
import { ILightningApi } from '../lightning-api.interface';
import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher';
import logger from '../../../logger';
import { Common } from '../../common';
/**
* Convert a clightning "listnode" entry to a lnd node entry
@@ -70,14 +71,6 @@ export async function convertAndmergeBidirectionalChannels(clChannels: any[]): P
return consolidatedChannelList;
}
export function convertChannelId(channelId): string {
if (channelId.indexOf('/') !== -1) {
channelId = channelId.slice(0, -2);
}
const s = channelId.split('x').map(part => BigInt(part));
return ((s[0] << 40n) | (s[1] << 16n) | s[2]).toString();
}
/**
* Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format
* In this case, clightning knows the channel policy for both nodes
@@ -90,7 +83,7 @@ async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILigh
const outputIdx = parts[2];
return {
channel_id: clChannelA.short_channel_id,
channel_id: Common.channelShortIdToIntegerId(clChannelA.short_channel_id),
capacity: clChannelA.satoshis,
last_update: lastUpdate,
node1_policy: convertPolicy(clChannelA),
@@ -111,7 +104,7 @@ async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Cha
const outputIdx = parts[2];
return {
channel_id: clChannel.short_channel_id,
channel_id: Common.channelShortIdToIntegerId(clChannel.short_channel_id),
capacity: clChannel.satoshis,
last_update: clChannel.last_update ?? 0,
node1_policy: convertPolicy(clChannel),

14
backend/src/api/mining/mining.ts
View File

@@ -473,7 +473,7 @@ class Mining {
for (const block of blocksWithoutPrices) {
// Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks
if (block.height < 68951) {
if (['mainnet', 'testnet'].includes(config.MEMPOOL.NETWORK) && block.height < 68951) {
blocksPrices.push({
height: block.height,
priceId: prices[0].id,
@@ -492,11 +492,11 @@ class Mining {
if (blocksPrices.length >= 100000) {
totalInserted += blocksPrices.length;
let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
if (blocksWithoutPrices.length > 200000) {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`);
} else {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
}
logger.debug(logStr);
await BlocksRepository.$saveBlockPrices(blocksPrices);
blocksPrices.length = 0;
}
@@ -504,11 +504,11 @@ class Mining {
if (blocksPrices.length > 0) {
totalInserted += blocksPrices.length;
let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
if (blocksWithoutPrices.length > 200000) {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`);
} else {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
}
logger.debug(logStr);
await BlocksRepository.$saveBlockPrices(blocksPrices);
}
} catch (e) {

10
backend/src/config.ts
View File

@@ -24,6 +24,8 @@ interface IConfig {
USER_AGENT: string;
STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
AUTOMATIC_BLOCK_REINDEXING: boolean;
POOLS_JSON_URL: string,
POOLS_JSON_TREE_URL: string,
};
ESPLORA: {
REST_API_URL: string;
@@ -32,7 +34,8 @@ interface IConfig {
ENABLED: boolean;
BACKEND: 'lnd' | 'cln' | 'ldk';
TOPOLOGY_FOLDER: string;
NODE_STATS_REFRESH_INTERVAL: number;
STATS_REFRESH_INTERVAL: number;
GRAPH_REFRESH_INTERVAL: number;
};
LND: {
TLS_CERT_PATH: string;
@@ -135,6 +138,8 @@ const defaults: IConfig = {
'USER_AGENT': 'mempool',
'STDOUT_LOG_MIN_PRIORITY': 'debug',
'AUTOMATIC_BLOCK_REINDEXING': false,
'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
},
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',
@@ -184,7 +189,8 @@ const defaults: IConfig = {
'ENABLED': false,
'BACKEND': 'lnd',
'TOPOLOGY_FOLDER': '',
'NODE_STATS_REFRESH_INTERVAL': 600,
'STATS_REFRESH_INTERVAL': 600,
'GRAPH_REFRESH_INTERVAL': 600,
},
'LND': {
'TLS_CERT_PATH': '',

6
backend/src/database.ts
View File

@@ -1,7 +1,7 @@
import config from './config';
import { createPool, Pool, PoolConnection } from 'mysql2/promise';
import logger from './logger';
import { PoolOptions } from 'mysql2/typings/mysql';
import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } from 'mysql2/typings/mysql';
class DB {
constructor() {
@@ -28,7 +28,9 @@ import { PoolOptions } from 'mysql2/typings/mysql';
}
}
public async query(query, params?) {
public async query<T extends RowDataPacket[][] | RowDataPacket[] | OkPacket |
OkPacket[] | ResultSetHeader>(query, params?): Promise<[T, FieldPacket[]]>
{
this.checkDBFlag();
const pool = await this.getPool();
return pool.query(query, params);

29
backend/src/indexer.ts
View File

@@ -6,13 +6,12 @@ import logger from './logger';
import HashratesRepository from './repositories/HashratesRepository';
import bitcoinClient from './api/bitcoin/bitcoin-client';
import priceUpdater from './tasks/price-updater';
import PricesRepository from './repositories/PricesRepository';
class Indexer {
runIndexer = true;
indexerRunning = false;
constructor() {
}
tasksRunning: string[] = [];
public reindex() {
if (Common.indexingEnabled()) {
@@ -20,6 +19,28 @@ class Indexer {
}
}
public async runSingleTask(task: 'blocksPrices') {
if (!Common.indexingEnabled()) {
return;
}
if (task === 'blocksPrices' && !this.tasksRunning.includes(task)) {
this.tasksRunning.push(task);
const lastestPriceId = await PricesRepository.$getLatestPriceId();
if (priceUpdater.historyInserted === false || lastestPriceId === null) {
logger.debug(`Blocks prices indexer is waiting for the price updater to complete`)
setTimeout(() => {
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
this.runSingleTask('blocksPrices');
}, 10000);
} else {
logger.debug(`Blocks prices indexer will run now`)
await mining.$indexBlockPrices();
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
}
}
}
public async $run() {
if (!Common.indexingEnabled() || this.runIndexer === false ||
this.indexerRunning === true || mempool.hasPriority()
@@ -50,7 +71,7 @@ class Indexer {
return;
}
await mining.$indexBlockPrices();
this.runSingleTask('blocksPrices');
await mining.$indexDifficultyAdjustments();
await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient
await mining.$generateNetworkHashrateHistory();

5
backend/src/repositories/PricesRepository.ts
View File

@@ -27,6 +27,11 @@ class PricesRepository {
return oldestRow[0] ? oldestRow[0].time : 0;
}
public async $getLatestPriceId(): Promise<number | null> {
const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
return oldestRow[0] ? oldestRow[0].id : null;
}
public async $getLatestPriceTime(): Promise<number> {
const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
return oldestRow[0] ? oldestRow[0].time : 0;

412
backend/src/tasks/lightning/network-sync.service.ts
View File

@@ -1,60 +1,43 @@
import DB from '../../database';
import logger from '../../logger';
import channelsApi from '../../api/explorer/channels.api';
import bitcoinClient from '../../api/bitcoin/bitcoin-client';
import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
import config from '../../config';
import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
import { ILightningApi } from '../../api/lightning/lightning-api.interface';
import { $lookupNodeLocation } from './sync-tasks/node-locations';
import lightningApi from '../../api/lightning/lightning-api-factory';
import { convertChannelId } from '../../api/lightning/clightning/clightning-convert';
import { Common } from '../../api/common';
import nodesApi from '../../api/explorer/nodes.api';
import { ResultSetHeader } from 'mysql2';
import fundingTxFetcher from './sync-tasks/funding-tx-fetcher';
class NetworkSyncService {
loggerTimer = 0;
constructor() {}
public async $startService() {
logger.info('Starting node sync service');
public async $startService(): Promise<void> {
logger.info('Starting lightning network sync service');
await this.$runUpdater();
this.loggerTimer = new Date().getTime() / 1000;
setInterval(async () => {
await this.$runUpdater();
}, 1000 * 60 * 60);
await this.$runTasks();
}
private async $runUpdater(): Promise<void> {
private async $runTasks(): Promise<void> {
try {
logger.info(`Updating nodes and channels...`);
logger.info(`Updating nodes and channels`);
const networkGraph = await lightningApi.$getNetworkGraph();
if (networkGraph.nodes.length === 0 || networkGraph.edges.length === 0) {
logger.info(`LN Network graph is empty, retrying in 10 seconds`);
await Common.sleep$(10000);
this.$runUpdater();
setTimeout(() => { this.$runTasks(); }, 10000);
return;
}
for (const node of networkGraph.nodes) {
await this.$saveNode(node);
}
logger.info(`Nodes updated.`);
if (config.MAXMIND.ENABLED) {
await $lookupNodeLocation();
}
const graphChannelsIds: string[] = [];
for (const channel of networkGraph.edges) {
await this.$saveChannel(channel);
graphChannelsIds.push(channel.channel_id);
}
await this.$setChannelsInactive(graphChannelsIds);
logger.info(`Channels updated.`);
await this.$findInactiveNodesAndChannels();
await this.$updateNodesList(networkGraph.nodes);
await this.$updateChannelsList(networkGraph.edges);
await this.$deactivateChannelsWithoutActiveNodes();
await this.$lookUpCreationDateFromChain();
await this.$updateNodeFirstSeen();
await this.$scanForClosedChannels();
@@ -63,84 +46,181 @@ class NetworkSyncService {
}
} catch (e) {
logger.err('$runUpdater() error: ' + (e instanceof Error ? e.message : e));
logger.err('$runTasks() error: ' + (e instanceof Error ? e.message : e));
}
setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL);
}
/**
* Update the `nodes` table to reflect the current network graph state
*/
private async $updateNodesList(nodes: ILightningApi.Node[]): Promise<void> {
let progress = 0;
const graphNodesPubkeys: string[] = [];
for (const node of nodes) {
await nodesApi.$saveNode(node);
graphNodesPubkeys.push(node.pub_key);
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating node ${progress}/${nodes.length}`);
this.loggerTimer = new Date().getTime() / 1000;
}
}
logger.info(`${progress} nodes updated`);
// If a channel if not present in the graph, mark it as inactive
nodesApi.$setNodesInactive(graphNodesPubkeys);
if (config.MAXMIND.ENABLED) {
$lookupNodeLocation();
}
}
/**
* Update the `channels` table to reflect the current network graph state
*/
private async $updateChannelsList(channels: ILightningApi.Channel[]): Promise<void> {
try {
let progress = 0;
const graphChannelsIds: string[] = [];
for (const channel of channels) {
await channelsApi.$saveChannel(channel);
graphChannelsIds.push(channel.channel_id);
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating channel ${progress}/${channels.length}`);
this.loggerTimer = new Date().getTime() / 1000;
}
}
logger.info(`${progress} channels updated`);
// If a channel if not present in the graph, mark it as inactive
channelsApi.$setChannelsInactive(graphChannelsIds);
} catch (e) {
logger.err(`Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`);
}
}
// This method look up the creation date of the earliest channel of the node
// and update the node to that date in order to get the earliest first seen date
private async $updateNodeFirstSeen() {
private async $updateNodeFirstSeen(): Promise<void> {
let progress = 0;
let updated = 0;
try {
const [nodes]: any[] = await DB.query(`SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node1_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created1, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node2_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created2 FROM nodes`);
const [nodes]: any[] = await DB.query(`
SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen,
(
SELECT MIN(UNIX_TIMESTAMP(created))
FROM channels
WHERE channels.node1_public_key = nodes.public_key
) AS created1,
(
SELECT MIN(UNIX_TIMESTAMP(created))
FROM channels
WHERE channels.node2_public_key = nodes.public_key
) AS created2
FROM nodes
`);
for (const node of nodes) {
let lowest = 0;
if (node.created1) {
if (node.created2 && node.created2 < node.created1) {
lowest = node.created2;
} else {
lowest = node.created1;
}
} else if (node.created2) {
lowest = node.created2;
}
if (lowest && lowest < node.first_seen) {
const lowest = Math.min(
node.created1 ?? Number.MAX_SAFE_INTEGER,
node.created2 ?? Number.MAX_SAFE_INTEGER,
node.first_seen ?? Number.MAX_SAFE_INTEGER
);
if (lowest < node.first_seen) {
const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`;
const params = [lowest, node.public_key];
await DB.query(query, params);
}
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating node first seen date ${progress}/${nodes.length}`);
this.loggerTimer = new Date().getTime() / 1000;
++updated;
}
}
logger.info(`Node first seen dates scan complete.`);
logger.info(`Updated ${updated} node first seen dates`);
} catch (e) {
logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $lookUpCreationDateFromChain() {
logger.info(`Running channel creation date lookup...`);
private async $lookUpCreationDateFromChain(): Promise<void> {
let progress = 0;
logger.info(`Running channel creation date lookup`);
try {
const channels = await channelsApi.$getChannelsWithoutCreatedDate();
for (const channel of channels) {
const transaction = await bitcoinClient.getRawTransaction(channel.transaction_id, 1);
await DB.query(`UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`, [transaction.blocktime, channel.id]);
const transaction = await fundingTxFetcher.$fetchChannelOpenTx(channel.short_id);
await DB.query(`
UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`,
[transaction.timestamp, channel.id]
);
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating channel creation date ${progress}/${channels.length}`);
this.loggerTimer = new Date().getTime() / 1000;
}
}
logger.info(`Channel creation dates scan complete.`);
logger.info(`Updated ${channels.length} channels' creation date`);
} catch (e) {
logger.err('$setCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e));
logger.err('$lookUpCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e));
}
}
// Looking for channels whos nodes are inactive
private async $findInactiveNodesAndChannels(): Promise<void> {
logger.info(`Running inactive channels scan...`);
/**
* If a channel does not have any active node linked to it, then also
* mark that channel as inactive
*/
private async $deactivateChannelsWithoutActiveNodes(): Promise<void> {
logger.info(`Find channels which nodes are offline`);
try {
const [channels]: [{ id: string }[]] = await <any>DB.query(`
SELECT channels.id
FROM channels
const result = await DB.query<ResultSetHeader>(`
UPDATE channels
SET status = 0
WHERE channels.status = 1
AND (
(
SELECT COUNT(*)
FROM nodes
WHERE nodes.public_key = channels.node1_public_key
AND nodes.status = 1
) = 0
OR (
SELECT COUNT(*)
FROM nodes
WHERE nodes.public_key = channels.node2_public_key
AND nodes.status = 1
) = 0)
`);
for (const channel of channels) {
await this.$updateChannelStatus(channel.id, 0);
if (result[0].changedRows ?? 0 > 0) {
logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`);
} else {
logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`);
}
logger.info(`Inactive channels scan complete.`);
} catch (e) {
logger.err('$findInactiveNodesAndChannels() error: ' + (e instanceof Error ? e.message : e));
logger.err('$deactivateChannelsWithoutActiveNodes() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $scanForClosedChannels(): Promise<void> {
let progress = 0;
try {
logger.info(`Starting closed channels scan...`);
const channels = await channelsApi.$getChannelsByStatus(0);
@@ -154,6 +234,13 @@ class NetworkSyncService {
await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]);
}
}
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Checking if channel has been closed ${progress}/${channels.length}`);
this.loggerTimer = new Date().getTime() / 1000;
}
}
logger.info(`Closed channels scan complete.`);
} catch (e) {
@@ -171,6 +258,9 @@ class NetworkSyncService {
if (!config.ESPLORA.REST_API_URL) {
return;
}
let progress = 0;
try {
logger.info(`Started running closed channel forensics...`);
const channels = await channelsApi.$getClosedChannelsWithoutReason();
@@ -216,6 +306,13 @@ class NetworkSyncService {
logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
}
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating channel closed channel forensics ${progress}/${channels.length}`);
this.loggerTimer = new Date().getTime() / 1000;
}
}
logger.info(`Closed channels forensics scan complete.`);
} catch (e) {
@@ -270,195 +367,6 @@ class NetworkSyncService {
}
return 1;
}
private async $saveChannel(channel: ILightningApi.Channel): Promise<void> {
const [ txid, vout ] = channel.chan_point.split(':');
const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
try {
const query = `INSERT INTO channels
(
id,
short_id,
capacity,
transaction_id,
transaction_vout,
updated_at,
status,
node1_public_key,
node1_base_fee_mtokens,
node1_cltv_delta,
node1_fee_rate,
node1_is_disabled,
node1_max_htlc_mtokens,
node1_min_htlc_mtokens,
node1_updated_at,
node2_public_key,
node2_base_fee_mtokens,
node2_cltv_delta,
node2_fee_rate,
node2_is_disabled,
node2_max_htlc_mtokens,
node2_min_htlc_mtokens,
node2_updated_at
)
VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
capacity = ?,
updated_at = ?,
status = 1,
node1_public_key = ?,
node1_base_fee_mtokens = ?,
node1_cltv_delta = ?,
node1_fee_rate = ?,
node1_is_disabled = ?,
node1_max_htlc_mtokens = ?,
node1_min_htlc_mtokens = ?,
node1_updated_at = ?,
node2_public_key = ?,
node2_base_fee_mtokens = ?,
node2_cltv_delta = ?,
node2_fee_rate = ?,
node2_is_disabled = ?,
node2_max_htlc_mtokens = ?,
node2_min_htlc_mtokens = ?,
node2_updated_at = ?
;`;
await DB.query(query, [
this.toIntegerId(channel.channel_id),
this.toShortId(channel.channel_id),
channel.capacity,
txid,
vout,
this.utcDateToMysql(channel.last_update),
channel.node1_pub,
policy1.fee_base_msat,
policy1.time_lock_delta,
policy1.fee_rate_milli_msat,
policy1.disabled,
policy1.max_htlc_msat,
policy1.min_htlc,
this.utcDateToMysql(policy1.last_update),
channel.node2_pub,
policy2.fee_base_msat,
policy2.time_lock_delta,
policy2.fee_rate_milli_msat,
policy2.disabled,
policy2.max_htlc_msat,
policy2.min_htlc,
this.utcDateToMysql(policy2.last_update),
channel.capacity,
this.utcDateToMysql(channel.last_update),
channel.node1_pub,
policy1.fee_base_msat,
policy1.time_lock_delta,
policy1.fee_rate_milli_msat,
policy1.disabled,
policy1.max_htlc_msat,
policy1.min_htlc,
this.utcDateToMysql(policy1.last_update),
channel.node2_pub,
policy2.fee_base_msat,
policy2.time_lock_delta,
policy2.fee_rate_milli_msat,
policy2.disabled,
policy2.max_htlc_msat,
policy2.min_htlc,
this.utcDateToMysql(policy2.last_update)
]);
} catch (e) {
logger.err('$saveChannel() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $updateChannelStatus(channelId: string, status: number): Promise<void> {
try {
await DB.query(`UPDATE channels SET status = ? WHERE id = ?`, [status, channelId]);
} catch (e) {
logger.err('$updateChannelStatus() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $setChannelsInactive(graphChannelsIds: string[]): Promise<void> {
if (graphChannelsIds.length === 0) {
return;
}
try {
await DB.query(`
UPDATE channels
SET status = 0
WHERE short_id NOT IN (
${graphChannelsIds.map(id => `"${id}"`).join(',')}
)
AND status != 2
`);
} catch (e) {
logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $saveNode(node: ILightningApi.Node): Promise<void> {
try {
const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
const query = `INSERT INTO nodes(
public_key,
first_seen,
updated_at,
alias,
color,
sockets
)
VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?)
ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, color = ?, sockets = ?`;
await DB.query(query, [
node.pub_key,
node.last_update,
node.alias,
node.color,
sockets,
node.last_update,
node.alias,
node.color,
sockets,
]);
} catch (e) {
logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));
}
}
private toIntegerId(id: string): string {
if (config.LIGHTNING.BACKEND === 'cln') {
return convertChannelId(id);
}
else if (config.LIGHTNING.BACKEND === 'lnd') {
return id;
}
return '';
}
/** Decodes a channel id returned by lnd as uint64 to a short channel id */
private toShortId(id: string): string {
if (config.LIGHTNING.BACKEND === 'cln') {
return id;
}
const n = BigInt(id);
return [
n >> 40n, // nth block
(n >> 16n) & 0xffffffn, // nth tx of the block
n & 0xffffn // nth output of the tx
].join('x');
}
private utcDateToMysql(date?: number): string {
const d = new Date((date || 0) * 1000);
return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
}
}
export default new NetworkSyncService();

23
backend/src/tasks/lightning/stats-updater.service.ts
View File

@@ -1,8 +1,8 @@
import DB from '../../database';
import logger from '../../logger';
import lightningApi from '../../api/lightning/lightning-api-factory';
import LightningStatsImporter from './sync-tasks/stats-importer';
import config from '../../config';
import { Common } from '../../api/common';
class LightningStatsUpdater {
public async $startService(): Promise<void> {
@@ -12,31 +12,22 @@ class LightningStatsUpdater {
LightningStatsImporter.$run();
}
private setDateMidnight(date: Date): void {
date.setUTCHours(0);
date.setUTCMinutes(0);
date.setUTCSeconds(0);
date.setUTCMilliseconds(0);
}
private async $runTasks(): Promise<void> {
await this.$logStatsDaily();
setTimeout(() => {
this.$runTasks();
}, 1000 * config.LIGHTNING.NODE_STATS_REFRESH_INTERVAL);
setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL);
}
/**
* Update the latest entry for each node every config.LIGHTNING.NODE_STATS_REFRESH_INTERVAL seconds
* Update the latest entry for each node every config.LIGHTNING.STATS_REFRESH_INTERVAL seconds
*/
private async $logStatsDaily(): Promise<void> {
const date = new Date();
this.setDateMidnight(date);
logger.info(`Updating latest networks stats`);
Common.setDateMidnight(date);
const networkGraph = await lightningApi.$getNetworkGraph();
LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph);
await LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph);
logger.info(`Updated latest network stats`);
}
}

9
backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts
View File

@@ -1,5 +1,6 @@
import { existsSync, promises } from 'fs';
import bitcoinClient from '../../../api/bitcoin/bitcoin-client';
import { Common } from '../../../api/common';
import config from '../../../config';
import logger from '../../../logger';
@@ -69,7 +70,11 @@ class FundingTxFetcher {
this.running = false;
}
public async $fetchChannelOpenTx(channelId: string): Promise<any> {
public async $fetchChannelOpenTx(channelId: string): Promise<{timestamp: number, txid: string, value: number}> {
if (channelId.indexOf('x') === -1) {
channelId = Common.channelIntegerIdToShortId(channelId);
}
if (this.fundingTxCache[channelId]) {
return this.fundingTxCache[channelId];
}
@@ -110,4 +115,4 @@ class FundingTxFetcher {
}
}
export default new FundingTxFetcher;
export default new FundingTxFetcher;

55
backend/src/tasks/lightning/sync-tasks/node-locations.ts
View File

@@ -6,7 +6,10 @@ import DB from '../../../database';
import logger from '../../../logger';
export async function $lookupNodeLocation(): Promise<void> {
logger.info(`Running node location updater using Maxmind...`);
let loggerTimer = new Date().getTime() / 1000;
let progress = 0;
logger.info(`Running node location updater using Maxmind`);
try {
const nodes = await nodesApi.$getAllNodes();
const lookupCity = await maxmind.open<CityResponse>(config.MAXMIND.GEOLITE2_CITY);
@@ -18,21 +21,24 @@ export async function $lookupNodeLocation(): Promise<void> {
for (const socket of sockets) {
const ip = socket.substring(0, socket.lastIndexOf(':')).replace('[', '').replace(']', '');
const hasClearnet = [4, 6].includes(net.isIP(ip));
if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') {
const city = lookupCity.get(ip);
const asn = lookupAsn.get(ip);
const isp = lookupIsp.get(ip);
if (city && (asn || isp)) {
const query = `UPDATE nodes SET
as_number = ?,
city_id = ?,
country_id = ?,
subdivision_id = ?,
longitude = ?,
latitude = ?,
accuracy_radius = ?
WHERE public_key = ?`;
const query = `
UPDATE nodes SET
as_number = ?,
city_id = ?,
country_id = ?,
subdivision_id = ?,
longitude = ?,
latitude = ?,
accuracy_radius = ?
WHERE public_key = ?
`;
const params = [
isp?.autonomous_system_number ?? asn?.autonomous_system_number,
@@ -46,25 +52,25 @@ export async function $lookupNodeLocation(): Promise<void> {
];
await DB.query(query, params);
// Store Continent
if (city.continent?.geoname_id) {
await DB.query(
// Store Continent
if (city.continent?.geoname_id) {
await DB.query(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'continent', ?)`,
[city.continent?.geoname_id, JSON.stringify(city.continent?.names)]);
}
}
// Store Country
if (city.country?.geoname_id) {
await DB.query(
// Store Country
if (city.country?.geoname_id) {
await DB.query(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country', ?)`,
[city.country?.geoname_id, JSON.stringify(city.country?.names)]);
}
}
// Store Country ISO code
if (city.country?.iso_code) {
await DB.query(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`,
[city.country?.geoname_id, city.country?.iso_code]);
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`,
[city.country?.geoname_id, city.country?.iso_code]);
}
// Store Division
@@ -88,10 +94,17 @@ export async function $lookupNodeLocation(): Promise<void> {
[isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]);
}
}
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating node location data ${progress}/${nodes.length}`);
loggerTimer = new Date().getTime() / 1000;
}
}
}
}
logger.info(`Node location data updated.`);
logger.info(`${progress} nodes location data updated`);
} catch (e) {
logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e));
}

47
backend/src/tasks/lightning/sync-tasks/stats-importer.ts
View File

@@ -4,6 +4,8 @@ import { XMLParser } from 'fast-xml-parser';
import logger from '../../../logger';
import fundingTxFetcher from './funding-tx-fetcher';
import config from '../../../config';
import { ILightningApi } from '../../../api/lightning/lightning-api.interface';
import { isIP } from 'net';
const fsPromises = promises;
@@ -48,7 +50,7 @@ class LightningStatsImporter {
/**
* Generate LN network stats for one day
*/
public async computeNetworkStats(timestamp: number, networkGraph): Promise<unknown> {
public async computeNetworkStats(timestamp: number, networkGraph: ILightningApi.NetworkGraph): Promise<unknown> {
// Node counts and network shares
let clearnetNodes = 0;
let torNodes = 0;
@@ -61,8 +63,8 @@ class LightningStatsImporter {
let isUnnanounced = true;
for (const socket of (node.addresses ?? [])) {
hasOnion = hasOnion || ['torv2', 'torv3'].includes(socket.network);
hasClearnet = hasClearnet || ['ipv4', 'ipv6'].includes(socket.network);
hasOnion = hasOnion || ['torv2', 'torv3'].includes(socket.network) || socket.addr.indexOf('onion') !== -1;
hasClearnet = hasClearnet || ['ipv4', 'ipv6'].includes(socket.network) || [4, 6].includes(isIP(socket.addr.split(':')[0]));
}
if (hasOnion && hasClearnet) {
clearnetTorNodes++;
@@ -127,33 +129,39 @@ class LightningStatsImporter {
if (channel.node1_policy !== undefined) { // Coming from the node
for (const policy of [channel.node1_policy, channel.node2_policy]) {
if (policy && policy.fee_rate_milli_msat < 5000) {
avgFeeRate += policy.fee_rate_milli_msat;
feeRates.push(policy.fee_rate_milli_msat);
if (policy && parseInt(policy.fee_rate_milli_msat, 10) < 5000) {
avgFeeRate += parseInt(policy.fee_rate_milli_msat, 10);
feeRates.push(parseInt(policy.fee_rate_milli_msat, 10));
}
if (policy && policy.fee_base_msat < 5000) {
avgBaseFee += policy.fee_base_msat;
baseFees.push(policy.fee_base_msat);
if (policy && parseInt(policy.fee_base_msat, 10) < 5000) {
avgBaseFee += parseInt(policy.fee_base_msat, 10);
baseFees.push(parseInt(policy.fee_base_msat, 10));
}
}
} else { // Coming from the historical import
// @ts-ignore
if (channel.fee_rate_milli_msat < 5000) {
avgFeeRate += channel.fee_rate_milli_msat;
feeRates.push(channel.fee_rate_milli_msat);
}
// @ts-ignore
avgFeeRate += parseInt(channel.fee_rate_milli_msat, 10);
// @ts-ignore
feeRates.push(parseInt(channel.fee_rate_milli_msat), 10);
}
// @ts-ignore
if (channel.fee_base_msat < 5000) {
avgBaseFee += channel.fee_base_msat;
baseFees.push(channel.fee_base_msat);
// @ts-ignore
avgBaseFee += parseInt(channel.fee_base_msat, 10);
// @ts-ignore
baseFees.push(parseInt(channel.fee_base_msat), 10);
}
}
}
avgFeeRate /= networkGraph.edges.length;
avgBaseFee /= networkGraph.edges.length;
avgFeeRate /= Math.max(networkGraph.edges.length, 1);
avgBaseFee /= Math.max(networkGraph.edges.length, 1);
const medCapacity = capacities.sort((a, b) => b - a)[Math.round(capacities.length / 2 - 1)];
const medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)];
const medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)];
const avgCapacity = Math.round(capacity / capacities.length);
const avgCapacity = Math.round(capacity / Math.max(capacities.length, 1));
let query = `INSERT INTO lightning_stats(
added,
@@ -251,6 +259,9 @@ class LightningStatsImporter {
};
}
/**
* Import topology files LN historical data into the database
*/
async $importHistoricalLightningStats(): Promise<void> {
let latestNodeCount = 1;

13
backend/src/tasks/pools-updater.ts
View File

@@ -12,14 +12,11 @@ import * as https from 'https';
*/
class PoolsUpdater {
lastRun: number = 0;
currentSha: any = undefined;
poolsUrl: string = 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json';
treeUrl: string = 'https://api.github.com/repos/mempool/mining-pools/git/trees/master';
currentSha: string | undefined = undefined;
poolsUrl: string = config.MEMPOOL.POOLS_JSON_URL;
treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL;
constructor() {
}
public async updatePoolsJson() {
public async updatePoolsJson(): Promise<void> {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
return;
}
@@ -77,7 +74,7 @@ class PoolsUpdater {
/**
* Fetch our latest pools.json sha from the db
*/
private async updateDBSha(githubSha: string) {
private async updateDBSha(githubSha: string): Promise<void> {
this.currentSha = githubSha;
if (config.DATABASE.ENABLED === true) {
try {

9
backend/src/tasks/price-updater.ts
View File

@@ -1,4 +1,5 @@
import * as fs from 'fs';
import { Common } from '../api/common';
import config from '../config';
import logger from '../logger';
import PricesRepository from '../repositories/PricesRepository';
@@ -34,10 +35,10 @@ export interface Prices {
}
class PriceUpdater {
historyInserted: boolean = false;
lastRun: number = 0;
lastHistoricalRun: number = 0;
running: boolean = false;
public historyInserted = false;
lastRun = 0;
lastHistoricalRun = 0;
running = false;
feeds: PriceFeed[] = [];
currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
latestPrices: Prices;