machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge Lightning backend into Mempool backend

Browse Source
This commit is contained in:
nymkappa
2022-07-06 11:58:06 +02:00
committed by softsimon
parent faafa6db3b
commit a238420d7f
47 changed files with 298 additions and 5424 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts
View File

@@ -13,6 +13,7 @@ export interface AbstractBitcoinApi {
$getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
$getAddressPrefix(prefix: string): string[];
$sendRawTransaction(rawTransaction: string): Promise<string>;
$getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend>;
$getOutspends(txId: string): Promise<IEsploraApi.Outspend[]>;
$getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]>;
}

14
backend/src/api/bitcoin/bitcoin-api.ts
View File

@@ -130,6 +130,16 @@ class BitcoinApi implements AbstractBitcoinApi {
return this.bitcoindClient.sendRawTransaction(rawTransaction);
}
async $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
const txOut = await this.bitcoindClient.getTxOut(txId, vout, false);
return {
spent: txOut === null,
status: {
confirmed: true,
}
};
}
async $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
const outSpends: IEsploraApi.Outspend[] = [];
const tx = await this.$getRawTransaction(txId, true, false);
@@ -195,7 +205,9 @@ class BitcoinApi implements AbstractBitcoinApi {
sequence: vin.sequence,
txid: vin.txid || '',
vout: vin.vout || 0,
witness: vin.txinwitness,
witness: vin.txinwitness || [],
inner_redeemscript_asm: '',
inner_witnessscript_asm: '',
};
});

6
backend/src/api/bitcoin/esplora-api.interface.ts
View File

@@ -25,10 +25,10 @@ export namespace IEsploraApi {
is_coinbase: boolean;
scriptsig: string;
scriptsig_asm: string;
inner_redeemscript_asm?: string;
inner_witnessscript_asm?: string;
inner_redeemscript_asm: string;
inner_witnessscript_asm: string;
sequence: any;
witness?: string[];
witness: string[];
prevout: Vout | null;
// Elements
is_pegin?: boolean;

5
backend/src/api/bitcoin/esplora-api.ts
View File

@@ -66,6 +66,11 @@ class ElectrsApi implements AbstractBitcoinApi {
throw new Error('Method not implemented.');
}
$getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
return axios.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
.then((response) => response.data);
}
$getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
return axios.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
.then((response) => response.data);

87
backend/src/api/database-migration.ts
View File

@@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 24;
private static currentVersion = 25;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -248,6 +248,15 @@ class DatabaseMigration {
await this.$executeQuery('DROP TABLE IF EXISTS `blocks_audits`');
await this.$executeQuery(this.getCreateBlocksAuditsTableQuery(), await this.$checkIfTableExists('blocks_audits'));
}
if (databaseSchemaVersion < 25 && isBitcoin === true) {
await this.$executeQuery(`INSERT INTO state VALUES('last_node_stats', 0, '1970-01-01');`);
await this.$executeQuery(this.getCreateLightningStatisticsQuery(), await this.$checkIfTableExists('lightning_stats'));
await this.$executeQuery(this.getCreateNodesQuery(), await this.$checkIfTableExists('nodes'));
await this.$executeQuery(this.getCreateChannelsQuery(), await this.$checkIfTableExists('channels'));
await this.$executeQuery(this.getCreateNodesStatsQuery(), await this.$checkIfTableExists('node_stats'));
}
} catch (e) {
throw e;
}
@@ -569,6 +578,82 @@ class DatabaseMigration {
adjustment float NOT NULL,
PRIMARY KEY (height),
INDEX (time)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateLightningStatisticsQuery(): string {
return `CREATE TABLE IF NOT EXISTS lightning_stats (
id int(11) NOT NULL AUTO_INCREMENT,
added datetime NOT NULL,
channel_count int(11) NOT NULL,
node_count int(11) NOT NULL,
total_capacity double unsigned NOT NULL,
PRIMARY KEY (id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateNodesQuery(): string {
return `CREATE TABLE IF NOT EXISTS nodes (
public_key varchar(66) NOT NULL,
first_seen datetime NOT NULL,
updated_at datetime NOT NULL,
alias varchar(200) CHARACTER SET utf8mb4 NOT NULL,
color varchar(200) NOT NULL,
sockets text DEFAULT NULL,
PRIMARY KEY (public_key),
KEY alias (alias(10))
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateChannelsQuery(): string {
return `CREATE TABLE IF NOT EXISTS channels (
id bigint(11) unsigned NOT NULL,
short_id varchar(15) NOT NULL DEFAULT '',
capacity bigint(20) unsigned NOT NULL,
transaction_id varchar(64) NOT NULL,
transaction_vout int(11) NOT NULL,
updated_at datetime DEFAULT NULL,
created datetime DEFAULT NULL,
status int(11) NOT NULL DEFAULT 0,
closing_transaction_id varchar(64) DEFAULT NULL,
closing_date datetime DEFAULT NULL,
closing_reason int(11) DEFAULT NULL,
node1_public_key varchar(66) NOT NULL,
node1_base_fee_mtokens bigint(20) unsigned DEFAULT NULL,
node1_cltv_delta int(11) DEFAULT NULL,
node1_fee_rate bigint(11) DEFAULT NULL,
node1_is_disabled tinyint(1) DEFAULT NULL,
node1_max_htlc_mtokens bigint(20) unsigned DEFAULT NULL,
node1_min_htlc_mtokens bigint(20) DEFAULT NULL,
node1_updated_at datetime DEFAULT NULL,
node2_public_key varchar(66) NOT NULL,
node2_base_fee_mtokens bigint(20) unsigned DEFAULT NULL,
node2_cltv_delta int(11) DEFAULT NULL,
node2_fee_rate bigint(11) DEFAULT NULL,
node2_is_disabled tinyint(1) DEFAULT NULL,
node2_max_htlc_mtokens bigint(20) unsigned DEFAULT NULL,
node2_min_htlc_mtokens bigint(20) unsigned DEFAULT NULL,
node2_updated_at datetime DEFAULT NULL,
PRIMARY KEY (id),
KEY node1_public_key (node1_public_key),
KEY node2_public_key (node2_public_key),
KEY status (status),
KEY short_id (short_id),
KEY transaction_id (transaction_id),
KEY closing_transaction_id (closing_transaction_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateNodesStatsQuery(): string {
return `CREATE TABLE IF NOT EXISTS node_stats (
id int(11) unsigned NOT NULL AUTO_INCREMENT,
public_key varchar(66) NOT NULL DEFAULT '',
added date NOT NULL,
capacity bigint(20) unsigned NOT NULL DEFAULT 0,
channels int(11) unsigned NOT NULL DEFAULT 0,
PRIMARY KEY (id),
UNIQUE KEY added (added,public_key),
KEY public_key (public_key)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}

164
backend/src/api/explorer/channels.api.ts Normal file
View File

@@ -0,0 +1,164 @@
import logger from '../../logger';
import DB from '../../database';
class ChannelsApi {
public async $getAllChannels(): Promise<any[]> {
try {
const query = `SELECT * FROM channels`;
const [rows]: any = await DB.query(query);
return rows;
} catch (e) {
logger.err('$getAllChannels error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $searchChannelsById(search: string): Promise<any[]> {
try {
const searchStripped = search.replace('%', '') + '%';
const query = `SELECT id, short_id, capacity FROM channels WHERE id LIKE ? OR short_id LIKE ? LIMIT 10`;
const [rows]: any = await DB.query(query, [searchStripped, searchStripped]);
return rows;
} catch (e) {
logger.err('$searchChannelsById error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getChannelsByStatus(status: number): Promise<any[]> {
try {
const query = `SELECT * FROM channels WHERE status = ?`;
const [rows]: any = await DB.query(query, [status]);
return rows;
} catch (e) {
logger.err('$getChannelsByStatus error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getClosedChannelsWithoutReason(): Promise<any[]> {
try {
const query = `SELECT * FROM channels WHERE status = 2 AND closing_reason IS NULL AND closing_transaction_id != ''`;
const [rows]: any = await DB.query(query);
return rows;
} catch (e) {
logger.err('$getClosedChannelsWithoutReason error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getChannelsWithoutCreatedDate(): Promise<any[]> {
try {
const query = `SELECT * FROM channels WHERE created IS NULL`;
const [rows]: any = await DB.query(query);
return rows;
} catch (e) {
logger.err('$getChannelsWithoutCreatedDate error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getChannel(id: string): Promise<any> {
try {
const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*, ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key WHERE (ns1.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node1_public_key) AND ns2.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node2_public_key)) AND channels.id = ?`;
const [rows]: any = await DB.query(query, [id]);
if (rows[0]) {
return this.convertChannel(rows[0]);
}
} catch (e) {
logger.err('$getChannel error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getChannelsByTransactionId(transactionIds: string[]): Promise<any[]> {
try {
transactionIds = transactionIds.map((id) => '\'' + id + '\'');
const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.* FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key WHERE channels.transaction_id IN (${transactionIds.join(', ')}) OR channels.closing_transaction_id IN (${transactionIds.join(', ')})`;
const [rows]: any = await DB.query(query);
const channels = rows.map((row) => this.convertChannel(row));
return channels;
} catch (e) {
logger.err('$getChannelByTransactionId error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getChannelsForNode(public_key: string, index: number, length: number, status: string): Promise<any[]> {
try {
// Default active and inactive channels
let statusQuery = '< 2';
// Closed channels only
if (status === 'closed') {
statusQuery = '= 2';
}
const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*, ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key WHERE (ns1.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node1_public_key) AND ns2.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node2_public_key)) AND (node1_public_key = ? OR node2_public_key = ?) AND status ${statusQuery} ORDER BY channels.capacity DESC LIMIT ?, ?`;
const [rows]: any = await DB.query(query, [public_key, public_key, index, length]);
const channels = rows.map((row) => this.convertChannel(row));
return channels;
} catch (e) {
logger.err('$getChannelsForNode error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getChannelsCountForNode(public_key: string, status: string): Promise<any> {
try {
// Default active and inactive channels
let statusQuery = '< 2';
// Closed channels only
if (status === 'closed') {
statusQuery = '= 2';
}
const query = `SELECT COUNT(*) AS count FROM channels WHERE (node1_public_key = ? OR node2_public_key = ?) AND status ${statusQuery}`;
const [rows]: any = await DB.query(query, [public_key, public_key]);
return rows[0]['count'];
} catch (e) {
logger.err('$getChannelsForNode error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
private convertChannel(channel: any): any {
return {
'id': channel.id,
'short_id': channel.short_id,
'capacity': channel.capacity,
'transaction_id': channel.transaction_id,
'transaction_vout': channel.transaction_vout,
'closing_transaction_id': channel.closing_transaction_id,
'closing_reason': channel.closing_reason,
'updated_at': channel.updated_at,
'created': channel.created,
'status': channel.status,
'node_left': {
'alias': channel.alias_left,
'public_key': channel.node1_public_key,
'channels': channel.channels_left,
'capacity': channel.capacity_left,
'base_fee_mtokens': channel.node1_base_fee_mtokens,
'cltv_delta': channel.node1_cltv_delta,
'fee_rate': channel.node1_fee_rate,
'is_disabled': channel.node1_is_disabled,
'max_htlc_mtokens': channel.node1_max_htlc_mtokens,
'min_htlc_mtokens': channel.node1_min_htlc_mtokens,
'updated_at': channel.node1_updated_at,
},
'node_right': {
'alias': channel.alias_right,
'public_key': channel.node2_public_key,
'channels': channel.channels_right,
'capacity': channel.capacity_right,
'base_fee_mtokens': channel.node2_base_fee_mtokens,
'cltv_delta': channel.node2_cltv_delta,
'fee_rate': channel.node2_fee_rate,
'is_disabled': channel.node2_is_disabled,
'max_htlc_mtokens': channel.node2_max_htlc_mtokens,
'min_htlc_mtokens': channel.node2_min_htlc_mtokens,
'updated_at': channel.node2_updated_at,
},
};
}
}
export default new ChannelsApi();

98
backend/src/api/explorer/channels.routes.ts Normal file
View File

@@ -0,0 +1,98 @@
import config from '../../config';
import { Application, Request, Response } from 'express';
import channelsApi from './channels.api';
class ChannelsRoutes {
constructor() { }
public initRoutes(app: Application) {
app
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/txids', this.$getChannelsByTransactionIds)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/search/:search', this.$searchChannelsById)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/:short_id', this.$getChannel)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels', this.$getChannelsForNode)
;
}
private async $searchChannelsById(req: Request, res: Response) {
try {
const channels = await channelsApi.$searchChannelsById(req.params.search);
res.json(channels);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getChannel(req: Request, res: Response) {
try {
const channel = await channelsApi.$getChannel(req.params.short_id);
if (!channel) {
res.status(404).send('Channel not found');
return;
}
res.json(channel);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getChannelsForNode(req: Request, res: Response) {
try {
if (typeof req.query.public_key !== 'string') {
res.status(501).send('Missing parameter: public_key');
return;
}
const index = parseInt(typeof req.query.index === 'string' ? req.query.index : '0', 10) || 0;
const status: string = typeof req.query.status === 'string' ? req.query.status : '';
const length = 25;
const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, length, status);
const channelsCount = await channelsApi.$getChannelsCountForNode(req.query.public_key, status);
res.header('X-Total-Count', channelsCount.toString());
res.json(channels);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getChannelsByTransactionIds(req: Request, res: Response) {
try {
if (!Array.isArray(req.query.txId)) {
res.status(500).send('Not an array');
return;
}
const txIds: string[] = [];
for (const _txId in req.query.txId) {
if (typeof req.query.txId[_txId] === 'string') {
txIds.push(req.query.txId[_txId].toString());
}
}
const channels = await channelsApi.$getChannelsByTransactionId(txIds);
const inputs: any[] = [];
const outputs: any[] = [];
for (const txid of txIds) {
const foundChannelInputs = channels.find((channel) => channel.closing_transaction_id === txid);
if (foundChannelInputs) {
inputs.push(foundChannelInputs);
} else {
inputs.push(null);
}
const foundChannelOutputs = channels.find((channel) => channel.transaction_id === txid);
if (foundChannelOutputs) {
outputs.push(foundChannelOutputs);
} else {
outputs.push(null);
}
}
res.json({
inputs: inputs,
outputs: outputs,
});
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
}
export default new ChannelsRoutes();

53
backend/src/api/explorer/general.routes.ts Normal file
View File

@@ -0,0 +1,53 @@
import config from '../../config';
import { Application, Request, Response } from 'express';
import nodesApi from './nodes.api';
import channelsApi from './channels.api';
import statisticsApi from './statistics.api';
class GeneralLightningRoutes {
constructor() { }
public initRoutes(app: Application) {
app
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/search', this.$searchNodesAndChannels)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/statistics/latest', this.$getGeneralStats)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/statistics', this.$getStatistics)
;
}
private async $searchNodesAndChannels(req: Request, res: Response) {
if (typeof req.query.searchText !== 'string') {
res.status(501).send('Missing parameter: searchText');
return;
}
try {
const nodes = await nodesApi.$searchNodeByPublicKeyOrAlias(req.query.searchText);
const channels = await channelsApi.$searchChannelsById(req.query.searchText);
res.json({
nodes: nodes,
channels: channels,
});
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getStatistics(req: Request, res: Response) {
try {
const statistics = await statisticsApi.$getStatistics();
res.json(statistics);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getGeneralStats(req: Request, res: Response) {
try {
const statistics = await statisticsApi.$getLatestStatistics();
res.json(statistics);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
}
export default new GeneralLightningRoutes();

62
backend/src/api/explorer/nodes.api.ts Normal file
View File

@@ -0,0 +1,62 @@
import logger from '../../logger';
import DB from '../../database';
class NodesApi {
public async $getNode(public_key: string): Promise<any> {
try {
const query = `SELECT nodes.*, (SELECT COUNT(*) FROM channels WHERE channels.status < 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)) AS channel_count, (SELECT SUM(capacity) FROM channels WHERE channels.status < 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)) AS capacity, (SELECT AVG(capacity) FROM channels WHERE status < 2 AND (node1_public_key = ? OR node2_public_key = ?)) AS channels_capacity_avg FROM nodes WHERE public_key = ?`;
const [rows]: any = await DB.query(query, [public_key, public_key, public_key, public_key, public_key, public_key, public_key]);
return rows[0];
} catch (e) {
logger.err('$getNode error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getNodeStats(public_key: string): Promise<any> {
try {
const query = `SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels FROM node_stats WHERE public_key = ? ORDER BY added DESC`;
const [rows]: any = await DB.query(query, [public_key]);
return rows;
} catch (e) {
logger.err('$getNodeStats error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getTopCapacityNodes(): Promise<any> {
try {
const query = `SELECT nodes.*, node_stats.capacity, node_stats.channels FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key ORDER BY node_stats.added DESC, node_stats.capacity DESC LIMIT 10`;
const [rows]: any = await DB.query(query);
return rows;
} catch (e) {
logger.err('$getTopCapacityNodes error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getTopChannelsNodes(): Promise<any> {
try {
const query = `SELECT nodes.*, node_stats.capacity, node_stats.channels FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key ORDER BY node_stats.added DESC, node_stats.channels DESC LIMIT 10`;
const [rows]: any = await DB.query(query);
return rows;
} catch (e) {
logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $searchNodeByPublicKeyOrAlias(search: string) {
try {
const searchStripped = search.replace('%', '') + '%';
const query = `SELECT nodes.public_key, nodes.alias, node_stats.capacity FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key WHERE nodes.public_key LIKE ? OR nodes.alias LIKE ? GROUP BY nodes.public_key ORDER BY node_stats.capacity DESC LIMIT 10`;
const [rows]: any = await DB.query(query, [searchStripped, searchStripped]);
return rows;
} catch (e) {
logger.err('$searchNodeByPublicKeyOrAlias error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
}
export default new NodesApi();

61
backend/src/api/explorer/nodes.routes.ts Normal file
View File

@@ -0,0 +1,61 @@
import config from '../../config';
import { Application, Request, Response } from 'express';
import nodesApi from './nodes.api';
class NodesRoutes {
constructor() { }
public initRoutes(app: Application) {
app
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/search/:search', this.$searchNode)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/top', this.$getTopNodes)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/statistics', this.$getHistoricalNodeStats)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key', this.$getNode)
;
}
private async $searchNode(req: Request, res: Response) {
try {
const nodes = await nodesApi.$searchNodeByPublicKeyOrAlias(req.params.search);
res.json(nodes);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getNode(req: Request, res: Response) {
try {
const node = await nodesApi.$getNode(req.params.public_key);
if (!node) {
res.status(404).send('Node not found');
return;
}
res.json(node);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getHistoricalNodeStats(req: Request, res: Response) {
try {
const statistics = await nodesApi.$getNodeStats(req.params.public_key);
res.json(statistics);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getTopNodes(req: Request, res: Response) {
try {
const topCapacityNodes = await nodesApi.$getTopCapacityNodes();
const topChannelsNodes = await nodesApi.$getTopChannelsNodes();
res.json({
topByCapacity: topCapacityNodes,
topByChannels: topChannelsNodes,
});
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
}
export default new NodesRoutes();

32
backend/src/api/explorer/statistics.api.ts Normal file
View File

@@ -0,0 +1,32 @@
import logger from '../../logger';
import DB from '../../database';
class StatisticsApi {
public async $getStatistics(): Promise<any> {
try {
const query = `SELECT UNIX_TIMESTAMP(added) AS added, channel_count, node_count, total_capacity FROM lightning_stats ORDER BY id DESC`;
const [rows]: any = await DB.query(query);
return rows;
} catch (e) {
logger.err('$getStatistics error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getLatestStatistics(): Promise<any> {
try {
const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY id DESC LIMIT 1`);
const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY id DESC LIMIT 1 OFFSET 72`);
return {
latest: rows[0],
previous: rows2[0],
};
} catch (e) {
logger.err('$getLatestStatistics error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
}
export default new StatisticsApi();

7
backend/src/api/lightning/lightning-api-abstract-factory.ts Normal file
View File

@@ -0,0 +1,7 @@
import { ILightningApi } from './lightning-api.interface';
export interface AbstractLightningApi {
$getNetworkInfo(): Promise<ILightningApi.NetworkInfo>;
$getNetworkGraph(): Promise<ILightningApi.NetworkGraph>;
$getChanInfo(id: string): Promise<ILightningApi.Channel>;
}

13
backend/src/api/lightning/lightning-api-factory.ts Normal file
View File

@@ -0,0 +1,13 @@
import config from '../../config';
import { AbstractLightningApi } from './lightning-api-abstract-factory';
import LndApi from './lnd/lnd-api';
function lightningApiFactory(): AbstractLightningApi {
switch (config.LIGHTNING.BACKEND) {
case 'lnd':
default:
return new LndApi();
}
}
export default lightningApiFactory();

53
backend/src/api/lightning/lightning-api.interface.ts Normal file
View File

@@ -0,0 +1,53 @@
export namespace ILightningApi {
export interface NetworkInfo {
average_channel_size: number;
channel_count: number;
max_channel_size: number;
median_channel_size: number;
min_channel_size: number;
node_count: number;
not_recently_updated_policy_count: number;
total_capacity: number;
}
export interface NetworkGraph {
channels: Channel[];
nodes: Node[];
}
export interface Channel {
id: string;
capacity: number;
policies: Policy[];
transaction_id: string;
transaction_vout: number;
updated_at: string;
}
interface Policy {
public_key: string;
base_fee_mtokens?: number;
cltv_delta?: number;
fee_rate?: number;
is_disabled?: boolean;
max_htlc_mtokens?: number;
min_htlc_mtokens?: number;
updated_at?: string;
}
export interface Node {
alias: string;
color: string;
features: Feature[];
public_key: string;
sockets: string[];
updated_at: string;
}
interface Feature {
bit: number;
is_known: boolean;
is_required: boolean;
type: string;
}
}

44
backend/src/api/lightning/lnd/lnd-api.ts Normal file
View File

@@ -0,0 +1,44 @@
import { AbstractLightningApi } from '../lightning-api-abstract-factory';
import { ILightningApi } from '../lightning-api.interface';
import * as fs from 'fs';
import * as lnService from 'ln-service';
import config from '../../../config';
import logger from '../../../logger';
class LndApi implements AbstractLightningApi {
private lnd: any;
constructor() {
if (!config.LIGHTNING.ENABLED) {
return;
}
try {
const tls = fs.readFileSync(config.LND_NODE_AUTH.TLS_CERT_PATH).toString('base64');
const macaroon = fs.readFileSync(config.LND_NODE_AUTH.MACAROON_PATH).toString('base64');
const { lnd } = lnService.authenticatedLndGrpc({
cert: tls,
macaroon: macaroon,
socket: config.LND_NODE_AUTH.SOCKET,
});
this.lnd = lnd;
} catch (e) {
logger.err('Could not initiate the LND service handler: ' + (e instanceof Error ? e.message : e));
process.exit(1);
}
}
async $getNetworkInfo(): Promise<ILightningApi.NetworkInfo> {
return await lnService.getNetworkInfo({ lnd: this.lnd });
}
async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
return await lnService.getNetworkGraph({ lnd: this.lnd });
}
async $getChanInfo(id: string): Promise<ILightningApi.Channel> {
return await lnService.getChannel({ lnd: this.lnd, id });
}
}
export default LndApi;

26
backend/src/config.ts
View File

@@ -27,6 +27,15 @@ interface IConfig {
ESPLORA: {
REST_API_URL: string;
};
LIGHTNING: {
ENABLED: boolean;
BACKEND: 'lnd' | 'cln' | 'ldk';
};
LND_NODE_AUTH: {
TLS_CERT_PATH: string;
MACAROON_PATH: string;
SOCKET: string;
};
ELECTRUM: {
HOST: string;
PORT: number;
@@ -158,6 +167,15 @@ const defaults: IConfig = {
'ENABLED': false,
'DATA_PATH': '/bisq/statsnode-data/btc_mainnet/db'
},
'LIGHTNING': {
'ENABLED': false,
'BACKEND': 'lnd'
},
'LND_NODE_AUTH': {
'TLS_CERT_PATH': '',
'MACAROON_PATH': '',
'SOCKET': 'localhost:10009',
},
'SOCKS5PROXY': {
'ENABLED': false,
'USE_ONION': true,
@@ -166,11 +184,11 @@ const defaults: IConfig = {
'USERNAME': '',
'PASSWORD': ''
},
"PRICE_DATA_SERVER": {
'PRICE_DATA_SERVER': {
'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices'
},
"EXTERNAL_DATA_SERVER": {
'EXTERNAL_DATA_SERVER': {
'MEMPOOL_API': 'https://mempool.space/api/v1',
'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
'LIQUID_API': 'https://liquid.network/api/v1',
@@ -190,6 +208,8 @@ class Config implements IConfig {
SYSLOG: IConfig['SYSLOG'];
STATISTICS: IConfig['STATISTICS'];
BISQ: IConfig['BISQ'];
LIGHTNING: IConfig['LIGHTNING'];
LND_NODE_AUTH: IConfig['LND_NODE_AUTH'];
SOCKS5PROXY: IConfig['SOCKS5PROXY'];
PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
@@ -205,6 +225,8 @@ class Config implements IConfig {
this.SYSLOG = configs.SYSLOG;
this.STATISTICS = configs.STATISTICS;
this.BISQ = configs.BISQ;
this.LIGHTNING = configs.LIGHTNING;
this.LND_NODE_AUTH = configs.LND_NODE_AUTH;
this.SOCKS5PROXY = configs.SOCKS5PROXY;
this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;

18
backend/src/index.ts
View File

@@ -29,6 +29,11 @@ import poolsUpdater from './tasks/pools-updater';
import indexer from './indexer';
import priceUpdater from './tasks/price-updater';
import BlocksAuditsRepository from './repositories/BlocksAuditsRepository';
import nodeSyncService from './tasks/lightning/node-sync.service';
import lightningStatsUpdater from './tasks/lightning/stats-updater.service';
import nodesRoutes from './api/explorer/nodes.routes';
import channelsRoutes from './api/explorer/channels.routes';
import generalLightningRoutes from './api/explorer/general.routes';
class Server {
private wss: WebSocket.Server | undefined;
@@ -130,6 +135,13 @@ class Server {
bisqMarkets.startBisqService();
}
if (config.LIGHTNING.ENABLED) {
nodeSyncService.$startService()
.then(() => {
lightningStatsUpdater.$startService();
});
}
this.server.listen(config.MEMPOOL.HTTP_PORT, () => {
if (worker) {
logger.info(`Mempool Server worker #${process.pid} started`);
@@ -362,6 +374,12 @@ class Server {
.get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/month', routes.$getElementsPegsByMonth)
;
}
if (config.LIGHTNING.ENABLED) {
generalLightningRoutes.initRoutes(this.app);
nodesRoutes.initRoutes(this.app);
channelsRoutes.initRoutes(this.app);
}
}
}

397
backend/src/tasks/lightning/node-sync.service.ts Normal file
View File

@@ -0,0 +1,397 @@
import { chanNumber } from 'bolt07';
import DB from '../../database';
import logger from '../../logger';
import channelsApi from '../../api/explorer/channels.api';
import bitcoinClient from '../../api/bitcoin/bitcoin-client';
import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
import config from '../../config';
import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
import lightningApi from '../../api/lightning/lightning-api-factory';
import { ILightningApi } from '../../api/lightning/lightning-api.interface';
class NodeSyncService {
constructor() {}
public async $startService() {
logger.info('Starting node sync service');
await this.$runUpdater();
setInterval(async () => {
await this.$runUpdater();
}, 1000 * 60 * 60);
}
private async $runUpdater() {
try {
logger.info(`Updating nodes and channels...`);
const networkGraph = await lightningApi.$getNetworkGraph();
for (const node of networkGraph.nodes) {
await this.$saveNode(node);
}
logger.info(`Nodes updated.`);
await this.$setChannelsInactive();
for (const channel of networkGraph.channels) {
await this.$saveChannel(channel);
}
logger.info(`Channels updated.`);
await this.$findInactiveNodesAndChannels();
await this.$lookUpCreationDateFromChain();
await this.$updateNodeFirstSeen();
await this.$scanForClosedChannels();
await this.$runClosedChannelsForensics();
} catch (e) {
logger.err('$updateNodes() error: ' + (e instanceof Error ? e.message : e));
}
}
// This method look up the creation date of the earliest channel of the node
// and update the node to that date in order to get the earliest first seen date
private async $updateNodeFirstSeen() {
try {
const [nodes]: any[] = await DB.query(`SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node1_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created1, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node2_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created2 FROM nodes`);
for (const node of nodes) {
let lowest = 0;
if (node.created1) {
if (node.created2 && node.created2 < node.created1) {
lowest = node.created2;
} else {
lowest = node.created1;
}
} else if (node.created2) {
lowest = node.created2;
}
if (lowest && lowest < node.first_seen) {
const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`;
const params = [lowest, node.public_key];
await DB.query(query, params);
}
}
logger.info(`Node first seen dates scan complete.`);
} catch (e) {
logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $lookUpCreationDateFromChain() {
logger.info(`Running channel creation date lookup...`);
try {
const channels = await channelsApi.$getChannelsWithoutCreatedDate();
for (const channel of channels) {
const transaction = await bitcoinClient.getRawTransaction(channel.transaction_id, 1);
await DB.query(`UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`, [transaction.blocktime, channel.id]);
}
logger.info(`Channel creation dates scan complete.`);
} catch (e) {
logger.err('$setCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e));
}
}
// Looking for channels whos nodes are inactive
private async $findInactiveNodesAndChannels(): Promise<void> {
logger.info(`Running inactive channels scan...`);
try {
// @ts-ignore
const [channels]: [ILightningApi.Channel[]] = await DB.query(`SELECT channels.id FROM channels WHERE channels.status = 1 AND ((SELECT COUNT(*) FROM nodes WHERE nodes.public_key = channels.node1_public_key) = 0 OR (SELECT COUNT(*) FROM nodes WHERE nodes.public_key = channels.node2_public_key) = 0)`);
for (const channel of channels) {
await this.$updateChannelStatus(channel.id, 0);
}
logger.info(`Inactive channels scan complete.`);
} catch (e) {
logger.err('$findInactiveNodesAndChannels() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $scanForClosedChannels(): Promise<void> {
try {
logger.info(`Starting closed channels scan...`);
const channels = await channelsApi.$getChannelsByStatus(0);
for (const channel of channels) {
const spendingTx = await bitcoinApi.$getOutspend(channel.transaction_id, channel.transaction_vout);
if (spendingTx.spent === true && spendingTx.status?.confirmed === true) {
logger.debug('Marking channel: ' + channel.id + ' as closed.');
await DB.query(`UPDATE channels SET status = 2, closing_date = FROM_UNIXTIME(?) WHERE id = ?`,
[spendingTx.status.block_time, channel.id]);
if (spendingTx.txid && !channel.closing_transaction_id) {
await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]);
}
}
}
logger.info(`Closed channels scan complete.`);
} catch (e) {
logger.err('$scanForClosedChannels() error: ' + (e instanceof Error ? e.message : e));
}
}
/*
1. Mutually closed
2. Forced closed
3. Forced closed with penalty
*/
private async $runClosedChannelsForensics(): Promise<void> {
if (!config.ESPLORA.REST_API_URL) {
return;
}
try {
logger.info(`Started running closed channel forensics...`);
const channels = await channelsApi.$getClosedChannelsWithoutReason();
for (const channel of channels) {
let reason = 0;
// Only Esplora backend can retrieve spent transaction outputs
const outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id);
const lightningScriptReasons: number[] = [];
for (const outspend of outspends) {
if (outspend.spent && outspend.txid) {
const spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid);
const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
lightningScriptReasons.push(lightningScript);
}
}
if (lightningScriptReasons.length === outspends.length
&& lightningScriptReasons.filter((r) => r === 1).length === outspends.length) {
reason = 1;
} else {
const filteredReasons = lightningScriptReasons.filter((r) => r !== 1);
if (filteredReasons.length) {
if (filteredReasons.some((r) => r === 2 || r === 4)) {
reason = 3;
} else {
reason = 2;
}
} else {
/*
We can detect a commitment transaction (force close) by reading Sequence and Locktime
https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction
*/
const closingTx = await bitcoinApi.$getRawTransaction(channel.closing_transaction_id);
const sequenceHex: string = closingTx.vin[0].sequence.toString(16);
const locktimeHex: string = closingTx.locktime.toString(16);
if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') {
reason = 2; // Here we can't be sure if it's a penalty or not
} else {
reason = 1;
}
}
}
if (reason) {
logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
}
}
logger.info(`Closed channels forensics scan complete.`);
} catch (e) {
logger.err('$runClosedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
}
}
private findLightningScript(vin: IEsploraApi.Vin): number {
const topElement = vin.witness[vin.witness.length - 2];
if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) {
// https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs
if (topElement === '01') {
// top element is '01' to get in the revocation path
// 'Revoked Lightning Force Close';
// Penalty force closed
return 2;
} else {
// top element is '', this is a delayed to_local output
// 'Lightning Force Close';
return 3;
}
} else if (
/^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_NOTIF OP_DROP OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) ||
/^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_IF OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_DROP OP_PUSHBYTES_3 \w{6} OP_CLTV OP_DROP OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm)
) {
// https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs
// https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs
if (topElement.length === 66) {
// top element is a public key
// 'Revoked Lightning HTLC'; Penalty force closed
return 4;
} else if (topElement) {
// top element is a preimage
// 'Lightning HTLC';
return 5;
} else {
// top element is '' to get in the expiry of the script
// 'Expired Lightning HTLC';
return 6;
}
} else if (/^OP_PUSHBYTES_33 \w{66} OP_CHECKSIG OP_IFDUP OP_NOTIF OP_PUSHNUM_16 OP_CSV OP_ENDIF$/.test(vin.inner_witnessscript_asm)) {
// https://github.com/lightning/bolts/blob/master/03-transactions.md#to_local_anchor-and-to_remote_anchor-output-option_anchors
if (topElement) {
// top element is a signature
// 'Lightning Anchor';
return 7;
} else {
// top element is '', it has been swept after 16 blocks
// 'Swept Lightning Anchor';
return 8;
}
}
return 1;
}
private async $saveChannel(channel: ILightningApi.Channel): Promise<void> {
const fromChannel = chanNumber({ channel: channel.id }).number;
try {
const d = new Date(Date.parse(channel.updated_at));
const query = `INSERT INTO channels
(
id,
short_id,
capacity,
transaction_id,
transaction_vout,
updated_at,
status,
node1_public_key,
node1_base_fee_mtokens,
node1_cltv_delta,
node1_fee_rate,
node1_is_disabled,
node1_max_htlc_mtokens,
node1_min_htlc_mtokens,
node1_updated_at,
node2_public_key,
node2_base_fee_mtokens,
node2_cltv_delta,
node2_fee_rate,
node2_is_disabled,
node2_max_htlc_mtokens,
node2_min_htlc_mtokens,
node2_updated_at
)
VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
capacity = ?,
updated_at = ?,
status = 1,
node1_public_key = ?,
node1_base_fee_mtokens = ?,
node1_cltv_delta = ?,
node1_fee_rate = ?,
node1_is_disabled = ?,
node1_max_htlc_mtokens = ?,
node1_min_htlc_mtokens = ?,
node1_updated_at = ?,
node2_public_key = ?,
node2_base_fee_mtokens = ?,
node2_cltv_delta = ?,
node2_fee_rate = ?,
node2_is_disabled = ?,
node2_max_htlc_mtokens = ?,
node2_min_htlc_mtokens = ?,
node2_updated_at = ?
;`;
await DB.query(query, [
fromChannel,
channel.id,
channel.capacity,
channel.transaction_id,
channel.transaction_vout,
channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0,
channel.policies[0].public_key,
channel.policies[0].base_fee_mtokens,
channel.policies[0].cltv_delta,
channel.policies[0].fee_rate,
channel.policies[0].is_disabled,
channel.policies[0].max_htlc_mtokens,
channel.policies[0].min_htlc_mtokens,
channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0,
channel.policies[1].public_key,
channel.policies[1].base_fee_mtokens,
channel.policies[1].cltv_delta,
channel.policies[1].fee_rate,
channel.policies[1].is_disabled,
channel.policies[1].max_htlc_mtokens,
channel.policies[1].min_htlc_mtokens,
channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0,
channel.capacity,
channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0,
channel.policies[0].public_key,
channel.policies[0].base_fee_mtokens,
channel.policies[0].cltv_delta,
channel.policies[0].fee_rate,
channel.policies[0].is_disabled,
channel.policies[0].max_htlc_mtokens,
channel.policies[0].min_htlc_mtokens,
channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0,
channel.policies[1].public_key,
channel.policies[1].base_fee_mtokens,
channel.policies[1].cltv_delta,
channel.policies[1].fee_rate,
channel.policies[1].is_disabled,
channel.policies[1].max_htlc_mtokens,
channel.policies[1].min_htlc_mtokens,
channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0,
]);
} catch (e) {
logger.err('$saveChannel() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $updateChannelStatus(channelShortId: string, status: number): Promise<void> {
try {
await DB.query(`UPDATE channels SET status = ? WHERE id = ?`, [status, channelShortId]);
} catch (e) {
logger.err('$updateChannelStatus() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $setChannelsInactive(): Promise<void> {
try {
await DB.query(`UPDATE channels SET status = 0 WHERE status = 1`);
} catch (e) {
logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $saveNode(node: ILightningApi.Node): Promise<void> {
try {
const updatedAt = this.utcDateToMysql(node.updated_at);
const sockets = node.sockets.join(',');
const query = `INSERT INTO nodes(
public_key,
first_seen,
updated_at,
alias,
color,
sockets
)
VALUES (?, NOW(), ?, ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?, sockets = ?;`;
await DB.query(query, [
node.public_key,
updatedAt,
node.alias,
node.color,
sockets,
updatedAt,
node.alias,
node.color,
sockets,
]);
} catch (e) {
logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));
}
}
private utcDateToMysql(dateString: string): string {
const d = new Date(Date.parse(dateString));
return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
}
}
export default new NodeSyncService();

179
backend/src/tasks/lightning/stats-updater.service.ts Normal file
View File

@@ -0,0 +1,179 @@
import logger from "../../logger";
import DB from "../../database";
import lightningApi from "../../api/lightning/lightning-api-factory";
class LightningStatsUpdater {
constructor() {}
public async $startService() {
logger.info('Starting Stats service');
const now = new Date();
const nextHourInterval = new Date(now.getFullYear(), now.getMonth(), now.getDate(), Math.floor(now.getHours() / 1) + 1, 0, 0, 0);
const difference = nextHourInterval.getTime() - now.getTime();
// setTimeout(() => {
setInterval(async () => {
await this.$runTasks();
}, 1000 * 60 * 60);
//}, difference);
await this.$runTasks();
}
private async $runTasks() {
await this.$populateHistoricalData();
await this.$logLightningStatsDaily();
await this.$logNodeStatsDaily();
}
private async $logNodeStatsDaily() {
logger.info(`Running daily node stats update...`);
const currentDate = new Date().toISOString().split('T')[0];
try {
const [state]: any = await DB.query(`SELECT string FROM state WHERE name = 'last_node_stats'`);
// Only store once per day
if (state[0].string === currentDate) {
return;
}
const query = `SELECT nodes.public_key, c1.channels_count_left, c2.channels_count_right, c1.channels_capacity_left, c2.channels_capacity_right FROM nodes LEFT JOIN (SELECT node1_public_key, COUNT(id) AS channels_count_left, SUM(capacity) AS channels_capacity_left FROM channels WHERE channels.status < 2 GROUP BY node1_public_key) c1 ON c1.node1_public_key = nodes.public_key LEFT JOIN (SELECT node2_public_key, COUNT(id) AS channels_count_right, SUM(capacity) AS channels_capacity_right FROM channels WHERE channels.status < 2 GROUP BY node2_public_key) c2 ON c2.node2_public_key = nodes.public_key`;
const [nodes]: any = await DB.query(query);
// First run we won't have any nodes yet
if (nodes.length < 10) {
return;
}
for (const node of nodes) {
await DB.query(
`INSERT INTO node_stats(public_key, added, capacity, channels) VALUES (?, NOW(), ?, ?)`,
[node.public_key, (parseInt(node.channels_capacity_left || 0, 10)) + (parseInt(node.channels_capacity_right || 0, 10)),
node.channels_count_left + node.channels_count_right]);
}
await DB.query(`UPDATE state SET string = ? WHERE name = 'last_node_stats'`, [currentDate]);
logger.info('Daily node stats has updated.');
} catch (e) {
logger.err('$logNodeStatsDaily() error: ' + (e instanceof Error ? e.message : e));
}
}
// We only run this on first launch
private async $populateHistoricalData() {
logger.info(`Running historical stats population...`);
const startTime = '2018-01-13';
try {
const [rows]: any = await DB.query(`SELECT COUNT(*) FROM lightning_stats`);
// Only store once per day
if (rows[0]['COUNT(*)'] > 0) {
return;
}
const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels ORDER BY created ASC`);
let date: Date = new Date(startTime);
const currentDate = new Date();
while (date < currentDate) {
let totalCapacity = 0;
let channelsCount = 0;
for (const channel of channels) {
if (new Date(channel.created) > date) {
break;
}
if (channel.closing_date !== null && new Date(channel.closing_date) < date) {
continue;
}
totalCapacity += channel.capacity;
channelsCount++;
}
const query = `INSERT INTO lightning_stats(
added,
channel_count,
node_count,
total_capacity
)
VALUES (FROM_UNIXTIME(?), ?, ?, ?)`;
await DB.query(query, [
date.getTime() / 1000,
channelsCount,
0,
totalCapacity,
]);
// Add one day and continue
date.setDate(date.getDate() + 1);
}
const [nodes]: any = await DB.query(`SELECT first_seen FROM nodes ORDER BY first_seen ASC`);
date = new Date(startTime);
while (date < currentDate) {
let nodeCount = 0;
for (const node of nodes) {
if (new Date(node.first_seen) > date) {
break;
}
nodeCount++;
}
const query = `UPDATE lightning_stats SET node_count = ? WHERE added = FROM_UNIXTIME(?)`;
await DB.query(query, [
nodeCount,
date.getTime() / 1000,
]);
// Add one day and continue
date.setDate(date.getDate() + 1);
}
logger.info('Historical stats populated.');
} catch (e) {
logger.err('$populateHistoricalData() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $logLightningStatsDaily() {
logger.info(`Running lightning daily stats log...`);
const currentDate = new Date().toISOString().split('T')[0];
try {
const [state]: any = await DB.query(`SELECT string FROM state WHERE name = 'last_node_stats'`);
// Only store once per day
if (state[0].string === currentDate) {
return;
}
const networkGraph = await lightningApi.$getNetworkGraph();
let total_capacity = 0;
for (const channel of networkGraph.channels) {
if (channel.capacity) {
total_capacity += channel.capacity;
}
}
const query = `INSERT INTO lightning_stats(
added,
channel_count,
node_count,
total_capacity
)
VALUES (NOW(), ?, ?, ?)`;
await DB.query(query, [
networkGraph.channels.length,
networkGraph.nodes.length,
total_capacity,
]);
logger.info(`Lightning daily stats done.`);
} catch (e) {
logger.err('$logLightningStatsDaily() error: ' + (e instanceof Error ? e.message : e));
}
}
}
export default new LightningStatsUpdater();