machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Node stats updates

Browse Source
This commit is contained in:
softsimon
2022-05-05 23:19:24 +04:00
parent 7d6f0b5ebe
commit d9d7f8cc66
11 changed files with 127 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
lightning-backend/src/api/explorer/channels.api.ts
View File

@@ -24,6 +24,17 @@ class ChannelsApi {
}
}
public async $getChannelsWithoutCreatedDate(): Promise<any[]> {
try {
const query = `SELECT * FROM channels WHERE created IS NULL`;
const [rows]: any = await DB.query(query);
return rows;
} catch (e) {
logger.err('$getChannelsWithoutCreatedDate error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getChannel(shortId: string): Promise<any> {
try {
const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.* FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key WHERE channels.id = ?`;

19
lightning-backend/src/api/explorer/nodes.api.ts
View File

@@ -4,8 +4,8 @@ import DB from '../../database';
class NodesApi {
public async $getNode(public_key: string): Promise<any> {
try {
const query = `SELECT * FROM nodes WHERE public_key = ?`;
const [rows]: any = await DB.query(query, [public_key]);
const query = `SELECT nodes.*, (SELECT COUNT(*) FROM channels WHERE channels.status < 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)) AS channel_count, (SELECT SUM(capacity) FROM channels WHERE channels.status < 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)) AS capacity FROM nodes WHERE public_key = ?`;
const [rows]: any = await DB.query(query, [public_key, public_key, public_key, public_key, public_key]);
return rows[0];
} catch (e) {
logger.err('$getNode error: ' + (e instanceof Error ? e.message : e));
@@ -13,9 +13,20 @@ class NodesApi {
}
}
public async $getNodeStats(public_key: string): Promise<any> {
try {
const query = `SELECT * FROM nodes_stats WHERE public_key = ? ORDER BY added DESC`;
const [rows]: any = await DB.query(query, [public_key]);
return rows;
} catch (e) {
logger.err('$getNodeStats error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getTopCapacityNodes(): Promise<any> {
try {
const query = `SELECT nodes.*, nodes_stats.capacity_left, nodes_stats.capacity_right, nodes_stats.channels_left, nodes_stats.channels_right FROM nodes LEFT JOIN nodes_stats ON nodes_stats.public_key = nodes.public_key ORDER BY nodes_stats.added DESC, nodes_stats.capacity_left + nodes_stats.capacity_right DESC LIMIT 10`;
const query = `SELECT nodes.*, nodes_stats.capacity, nodes_stats.channels FROM nodes LEFT JOIN nodes_stats ON nodes_stats.public_key = nodes.public_key ORDER BY nodes_stats.added DESC, nodes_stats.capacity DESC LIMIT 10`;
const [rows]: any = await DB.query(query);
return rows;
} catch (e) {
@@ -26,7 +37,7 @@ class NodesApi {
public async $getTopChannelsNodes(): Promise<any> {
try {
const query = `SELECT nodes.*, nodes_stats.capacity_left, nodes_stats.capacity_right, nodes_stats.channels_left, nodes_stats.channels_right FROM nodes LEFT JOIN nodes_stats ON nodes_stats.public_key = nodes.public_key ORDER BY nodes_stats.added DESC, nodes_stats.channels_left + nodes_stats.channels_right DESC LIMIT 10`;
const query = `SELECT nodes.*, nodes_stats.capacity, nodes_stats.channels FROM nodes LEFT JOIN nodes_stats ON nodes_stats.public_key = nodes.public_key ORDER BY nodes_stats.added DESC, nodes_stats.channels DESC LIMIT 10`;
const [rows]: any = await DB.query(query);
return rows;
} catch (e) {

10
lightning-backend/src/api/explorer/nodes.routes.ts
View File

@@ -8,6 +8,7 @@ class NodesRoutes {
app
.get(config.MEMPOOL.API_URL_PREFIX + 'statistics/latest', this.$getGeneralStats)
.get(config.MEMPOOL.API_URL_PREFIX + 'nodes/top', this.$getTopNodes)
.get(config.MEMPOOL.API_URL_PREFIX + 'nodes/:public_key/statistics', this.$getHistoricalNodeStats)
.get(config.MEMPOOL.API_URL_PREFIX + 'nodes/:public_key', this.$getNode)
;
}
@@ -25,6 +26,15 @@ class NodesRoutes {
}
}
private async $getHistoricalNodeStats(req: Request, res: Response) {
try {
const statistics = await nodesApi.$getNodeStats(req.params.public_key);
res.json(statistics);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getGeneralStats(req: Request, res: Response) {
try {
const statistics = await nodesApi.$getLatestStatistics();

6
lightning-backend/src/database-migration.ts
View File

@@ -238,10 +238,8 @@ class DatabaseMigration {
id int(11) unsigned NOT NULL AUTO_INCREMENT,
public_key varchar(66) NOT NULL DEFAULT '',
added date NOT NULL,
capacity_left bigint(11) unsigned DEFAULT NULL,
capacity_right bigint(11) unsigned DEFAULT NULL,
channels_left int(11) unsigned DEFAULT NULL,
channels_right int(11) unsigned DEFAULT NULL,
capacity bigint(11) unsigned DEFAULT NULL,
channels int(11) unsigned DEFAULT NULL,
PRIMARY KEY (id),
KEY public_key (public_key)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;

58
lightning-backend/src/tasks/node-sync.service.ts
View File

@@ -26,21 +26,71 @@ class NodeSyncService {
for (const node of networkGraph.nodes) {
await this.$saveNode(node);
}
logger.debug(`Nodes updated`);
await this.$setChannelsInactive();
for (const channel of networkGraph.channels) {
await this.$saveChannel(channel);
}
logger.debug(`Channels updated`);
await this.$findInactiveNodesAndChannels();
logger.debug(`Inactive channels scan complete`);
await this.$scanForClosedChannels();
logger.debug(`Closed channels scan complete`);
await this.$lookUpCreationDateFromChain();
logger.debug(`Channel creation dates scan complete`);
await this.$updateNodeFirstSeen();
logger.debug(`Node first seen dates scan complete`);
} catch (e) {
logger.err('$updateNodes() error: ' + (e instanceof Error ? e.message : e));
}
}
// This method look up the creation date of the earliest channel of the node
// and update the node to that date in order to get the earliest first seen date
private async $updateNodeFirstSeen() {
try {
const [nodes]: any[] = await DB.query(`SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node1_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created1, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node2_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created2 FROM nodes`);
for (const node of nodes) {
let lowest = 0;
if (node.created1) {
if (node.created2 && node.created2 < node.created1) {
lowest = node.created2;
} else {
lowest = node.created1;
}
} else if (node.created2) {
lowest = node.created2;
}
if (lowest && lowest < node.first_seen) {
const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`;
const params = [lowest, node.public_key];
await DB.query(query, params);
}
}
} catch (e) {
logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $lookUpCreationDateFromChain() {
try {
const channels = await channelsApi.$getChannelsWithoutCreatedDate();
for (const channel of channels) {
const transaction = await bitcoinClient.getRawTransaction(channel.transaction_id, 1);
await DB.query(`UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`, [transaction.blocktime, channel.id]);
}
} catch (e) {
logger.err('$setCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e));
}
}
// Looking for channels whos nodes are inactive
private async $findInactiveNodesAndChannels(): Promise<void> {
try {
@@ -190,23 +240,27 @@ class NodeSyncService {
private async $saveNode(node: ILightningApi.Node): Promise<void> {
try {
const updatedAt = this.utcDateToMysql(node.updated_at);
const sockets = node.sockets.join(', ');
const query = `INSERT INTO nodes(
public_key,
first_seen,
updated_at,
alias,
color
color,
sockets
)
VALUES (?, NOW(), ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?;`;
VALUES (?, NOW(), ?, ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?, sockets = ?;`;
await DB.query(query, [
node.public_key,
updatedAt,
node.alias,
node.color,
sockets,
updatedAt,
node.alias,
node.color,
sockets,
]);
} catch (e) {
logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));

6
lightning-backend/src/tasks/stats-updater.service.ts
View File

@@ -38,9 +38,9 @@ class LightningStatsUpdater {
for (const node of nodes) {
await DB.query(
`INSERT INTO nodes_stats(public_key, added, capacity_left, capacity_right, channels_left, channels_right) VALUES (?, NOW(), ?, ?, ?, ?)`,
[node.public_key, node.channels_capacity_left, node.channels_capacity_right,
node.channels_count_left, node.channels_count_right]);
`INSERT INTO nodes_stats(public_key, added, capacity, channels) VALUES (?, NOW(), ?, ?)`,
[node.public_key, (parseInt(node.channels_capacity_left || 0, 10)) + (parseInt(node.channels_capacity_right || 0, 10)),
node.channels_count_left + node.channels_count_right]);
}
await DB.query(`UPDATE state SET string = ? WHERE name = 'last_node_stats'`, [currentDate]);
logger.debug('Daily node stats has updated.');