machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into nymkappa/bugfix/re-enable-ln-map-click

Browse Source
This commit is contained in:
wiz 2022-08-01 16:19:01 +00:00 committed by GitHub
commit fc463a9561
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 202 additions and 1003 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/mempool-config.sample.json
View File

@ -79,8 +79,8 @@
}, },
"LND": { "LND": {
"TLS_CERT_PATH": "tls.cert", "TLS_CERT_PATH": "tls.cert",
"MACAROON_PATH": "admin.macaroon", "MACAROON_PATH": "readonly.macaroon",
"SOCKET": "localhost:10009" "REST_API_URL": "https://localhost:8080"
}, },
"SOCKS5PROXY": { "SOCKS5PROXY": {
"ENABLED": false, "ENABLED": false,

892
backend/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

5
backend/package.json
View File

@ -16,7 +16,8 @@
"mempool", "mempool",
"blockchain", "blockchain",
"explorer", "explorer",
"liquid" "liquid",
"lightning"
], ],
"main": "index.ts", "main": "index.ts",
"scripts": { "scripts": {
@ -34,10 +35,8 @@
"@types/node": "^16.11.41", "@types/node": "^16.11.41",
"axios": "~0.27.2", "axios": "~0.27.2",
"bitcoinjs-lib": "6.0.1", "bitcoinjs-lib": "6.0.1",
"bolt07": "^1.8.1",
"crypto-js": "^4.0.0", "crypto-js": "^4.0.0",
"express": "^4.18.0", "express": "^4.18.0",
"lightning": "^5.16.3",
"maxmind": "^4.3.6", "maxmind": "^4.3.6",
"mysql2": "2.3.3", "mysql2": "2.3.3",
"node-worker-threads-pool": "^1.5.1", "node-worker-threads-pool": "^1.5.1",

10
backend/src/api/explorer/nodes.api.ts
View File

@ -163,8 +163,8 @@ class NodesApi {
public async $getNodesPerCountry(countryId: string) { public async $getNodesPerCountry(countryId: string) {
try { try {
const query = ` const query = `
SELECT node_stats.public_key, node_stats.capacity, node_stats.channels, nodes.alias, SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at, nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
geo_names_city.names as city geo_names_city.names as city
FROM node_stats FROM node_stats
JOIN ( JOIN (
@ -172,7 +172,7 @@ class NodesApi {
FROM node_stats FROM node_stats
GROUP BY public_key GROUP BY public_key
) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added ) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added
JOIN nodes ON nodes.public_key = node_stats.public_key RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
WHERE geo_names_country.id = ? WHERE geo_names_country.id = ?
@ -193,8 +193,8 @@ class NodesApi {
public async $getNodesPerISP(ISPId: string) { public async $getNodesPerISP(ISPId: string) {
try { try {
const query = ` const query = `
SELECT node_stats.public_key, node_stats.capacity, node_stats.channels, nodes.alias, SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at, nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
geo_names_city.names as city, geo_names_country.names as country geo_names_city.names as city, geo_names_country.names as country
FROM node_stats FROM node_stats
JOIN ( JOIN (

104
backend/src/api/lightning/lightning-api.interface.ts
View File

@ -1,71 +1,85 @@
export namespace ILightningApi { export namespace ILightningApi {
export interface NetworkInfo { export interface NetworkInfo {
average_channel_size: number; graph_diameter: number;
channel_count: number; avg_out_degree: number;
max_channel_size: number; max_out_degree: number;
median_channel_size: number; num_nodes: number;
min_channel_size: number; num_channels: number;
node_count: number; total_network_capacity: string;
not_recently_updated_policy_count: number; avg_channel_size: number;
total_capacity: number; min_channel_size: string;
max_channel_size: string;
median_channel_size_sat: string;
num_zombie_chans: string;
} }
export interface NetworkGraph { export interface NetworkGraph {
channels: Channel[];
nodes: Node[]; nodes: Node[];
edges: Channel[];
} }
export interface Channel { export interface Channel {
id: string; channel_id: string;
capacity: number; chan_point: string;
policies: Policy[]; last_update: number;
transaction_id: string; node1_pub: string;
transaction_vout: number; node2_pub: string;
updated_at?: string; capacity: string;
node1_policy: RoutingPolicy | null;
node2_policy: RoutingPolicy | null;
} }
interface Policy { export interface RoutingPolicy {
public_key: string; time_lock_delta: number;
base_fee_mtokens?: string; min_htlc: string;
cltv_delta?: number; fee_base_msat: string;
fee_rate?: number; fee_rate_milli_msat: string;
is_disabled?: boolean; disabled: boolean;
max_htlc_mtokens?: string; max_htlc_msat: string;
min_htlc_mtokens?: string; last_update: number;
updated_at?: string;
} }
export interface Node { export interface Node {
last_update: number;
pub_key: string;
alias: string; alias: string;
addresses: {
network: string;
addr: string;
}[];
color: string; color: string;
features: Feature[]; features: { [key: number]: Feature };
public_key: string;
sockets: string[];
updated_at?: string;
} }
export interface Info { export interface Info {
chains: string[]; identity_pubkey: string;
color: string;
active_channels_count: number;
alias: string; alias: string;
current_block_hash: string; num_pending_channels: number;
current_block_height: number; num_active_channels: number;
features: Feature[]; num_peers: number;
is_synced_to_chain: boolean; block_height: number;
is_synced_to_graph: boolean; block_hash: string;
latest_block_at: string; synced_to_chain: boolean;
peers_count: number; testnet: boolean;
pending_channels_count: number; uris: string[];
public_key: string; best_header_timestamp: string;
uris: any[];
version: string; version: string;
num_inactive_channels: number;
chains: {
chain: string;
network: string;
}[];
color: string;
synced_to_graph: boolean;
features: { [key: number]: Feature };
commit_hash: string;
/** Available on LND since v0.15.0-beta */
require_htlc_interceptor?: boolean;
} }
export interface Feature { export interface Feature {
bit: number; name: string;
is_known: boolean;
is_required: boolean; is_required: boolean;
type?: string; is_known: boolean;
} }
} }

46
backend/src/api/lightning/lnd/lnd-api.ts
View File

@ -1,44 +1,40 @@
import axios, { AxiosRequestConfig } from 'axios';
import { Agent } from 'https';
import * as fs from 'fs';
import { AbstractLightningApi } from '../lightning-api-abstract-factory'; import { AbstractLightningApi } from '../lightning-api-abstract-factory';
import { ILightningApi } from '../lightning-api.interface'; import { ILightningApi } from '../lightning-api.interface';
import * as fs from 'fs';
import { authenticatedLndGrpc, getWalletInfo, getNetworkGraph, getNetworkInfo } from 'lightning';
import config from '../../../config'; import config from '../../../config';
import logger from '../../../logger';
class LndApi implements AbstractLightningApi { class LndApi implements AbstractLightningApi {
private lnd: any; axiosConfig: AxiosRequestConfig = {};
constructor() { constructor() {
if (!config.LIGHTNING.ENABLED) { if (config.LIGHTNING.ENABLED) {
return; this.axiosConfig = {
} headers: {
try { 'Grpc-Metadata-macaroon': fs.readFileSync(config.LND.MACAROON_PATH).toString('hex')
const tls = fs.readFileSync(config.LND.TLS_CERT_PATH).toString('base64'); },
const macaroon = fs.readFileSync(config.LND.MACAROON_PATH).toString('base64'); httpsAgent: new Agent({
ca: fs.readFileSync(config.LND.TLS_CERT_PATH)
const { lnd } = authenticatedLndGrpc({ }),
cert: tls, timeout: 10000
macaroon: macaroon, };
socket: config.LND.SOCKET,
});
this.lnd = lnd;
} catch (e) {
logger.err('Could not initiate the LND service handler: ' + (e instanceof Error ? e.message : e));
process.exit(1);
} }
} }
async $getNetworkInfo(): Promise<ILightningApi.NetworkInfo> { async $getNetworkInfo(): Promise<ILightningApi.NetworkInfo> {
return await getNetworkInfo({ lnd: this.lnd }); return axios.get<ILightningApi.NetworkInfo>(config.LND.REST_API_URL + '/v1/graph/info', this.axiosConfig)
.then((response) => response.data);
} }
async $getInfo(): Promise<ILightningApi.Info> { async $getInfo(): Promise<ILightningApi.Info> {
// @ts-ignore return axios.get<ILightningApi.Info>(config.LND.REST_API_URL + '/v1/getinfo', this.axiosConfig)
return await getWalletInfo({ lnd: this.lnd }); .then((response) => response.data);
} }
async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> { async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
return await getNetworkGraph({ lnd: this.lnd }); return axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
.then((response) => response.data);
} }
} }

4
backend/src/config.ts
View File

@ -35,7 +35,7 @@ interface IConfig {
LND: { LND: {
TLS_CERT_PATH: string; TLS_CERT_PATH: string;
MACAROON_PATH: string; MACAROON_PATH: string;
SOCKET: string; REST_API_URL: string;
}; };
ELECTRUM: { ELECTRUM: {
HOST: string; HOST: string;
@ -182,7 +182,7 @@ const defaults: IConfig = {
'LND': { 'LND': {
'TLS_CERT_PATH': '', 'TLS_CERT_PATH': '',
'MACAROON_PATH': '', 'MACAROON_PATH': '',
'SOCKET': 'localhost:10009', 'REST_API_URL': 'https://localhost:8080',
}, },
'SOCKS5PROXY': { 'SOCKS5PROXY': {
'ENABLED': false, 'ENABLED': false,

115
backend/src/tasks/lightning/node-sync.service.ts
View File

@ -1,4 +1,3 @@
import { chanNumber } from 'bolt07';
import DB from '../../database'; import DB from '../../database';
import logger from '../../logger'; import logger from '../../logger';
import channelsApi from '../../api/explorer/channels.api'; import channelsApi from '../../api/explorer/channels.api';
@ -39,9 +38,9 @@ class NodeSyncService {
} }
const graphChannelsIds: string[] = []; const graphChannelsIds: string[] = [];
for (const channel of networkGraph.channels) { for (const channel of networkGraph.edges) {
await this.$saveChannel(channel); await this.$saveChannel(channel);
graphChannelsIds.push(channel.id); graphChannelsIds.push(channel.channel_id);
} }
await this.$setChannelsInactive(graphChannelsIds); await this.$setChannelsInactive(graphChannelsIds);
@ -56,7 +55,7 @@ class NodeSyncService {
} }
} catch (e) { } catch (e) {
logger.err('$updateNodes() error: ' + (e instanceof Error ? e.message : e)); logger.err('$runUpdater() error: ' + (e instanceof Error ? e.message : e));
} }
} }
@ -107,8 +106,7 @@ class NodeSyncService {
logger.info(`Running inactive channels scan...`); logger.info(`Running inactive channels scan...`);
try { try {
// @ts-ignore const [channels]: [{ id: string }[]] = await <any>DB.query(`
const [channels]: [ILightningApi.Channel[]] = await DB.query(`
SELECT channels.id SELECT channels.id
FROM channels FROM channels
WHERE channels.status = 1 WHERE channels.status = 1
@ -266,7 +264,10 @@ class NodeSyncService {
} }
private async $saveChannel(channel: ILightningApi.Channel): Promise<void> { private async $saveChannel(channel: ILightningApi.Channel): Promise<void> {
const fromChannel = chanNumber({ channel: channel.id }).number; const [ txid, vout ] = channel.chan_point.split(':');
const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
try { try {
const query = `INSERT INTO channels const query = `INSERT INTO channels
@ -319,55 +320,55 @@ class NodeSyncService {
;`; ;`;
await DB.query(query, [ await DB.query(query, [
fromChannel, channel.channel_id,
channel.id, this.toShortId(channel.channel_id),
channel.capacity, channel.capacity,
channel.transaction_id, txid,
channel.transaction_vout, vout,
channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0, this.utcDateToMysql(channel.last_update),
channel.policies[0].public_key, channel.node1_pub,
channel.policies[0].base_fee_mtokens, policy1.fee_base_msat,
channel.policies[0].cltv_delta, policy1.time_lock_delta,
channel.policies[0].fee_rate, policy1.fee_rate_milli_msat,
channel.policies[0].is_disabled, policy1.disabled,
channel.policies[0].max_htlc_mtokens, policy1.max_htlc_msat,
channel.policies[0].min_htlc_mtokens, policy1.min_htlc,
channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0, this.utcDateToMysql(policy1.last_update),
channel.policies[1].public_key, channel.node2_pub,
channel.policies[1].base_fee_mtokens, policy2.fee_base_msat,
channel.policies[1].cltv_delta, policy2.time_lock_delta,
channel.policies[1].fee_rate, policy2.fee_rate_milli_msat,
channel.policies[1].is_disabled, policy2.disabled,
channel.policies[1].max_htlc_mtokens, policy2.max_htlc_msat,
channel.policies[1].min_htlc_mtokens, policy2.min_htlc,
channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0, this.utcDateToMysql(policy2.last_update),
channel.capacity, channel.capacity,
channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0, this.utcDateToMysql(channel.last_update),
channel.policies[0].public_key, channel.node1_pub,
channel.policies[0].base_fee_mtokens, policy1.fee_base_msat,
channel.policies[0].cltv_delta, policy1.time_lock_delta,
channel.policies[0].fee_rate, policy1.fee_rate_milli_msat,
channel.policies[0].is_disabled, policy1.disabled,
channel.policies[0].max_htlc_mtokens, policy1.max_htlc_msat,
channel.policies[0].min_htlc_mtokens, policy1.min_htlc,
channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0, this.utcDateToMysql(policy1.last_update),
channel.policies[1].public_key, channel.node2_pub,
channel.policies[1].base_fee_mtokens, policy2.fee_base_msat,
channel.policies[1].cltv_delta, policy2.time_lock_delta,
channel.policies[1].fee_rate, policy2.fee_rate_milli_msat,
channel.policies[1].is_disabled, policy2.disabled,
channel.policies[1].max_htlc_mtokens, policy2.max_htlc_msat,
channel.policies[1].min_htlc_mtokens, policy2.min_htlc,
channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0, this.utcDateToMysql(policy2.last_update)
]); ]);
} catch (e) { } catch (e) {
logger.err('$saveChannel() error: ' + (e instanceof Error ? e.message : e)); logger.err('$saveChannel() error: ' + (e instanceof Error ? e.message : e));
} }
} }
private async $updateChannelStatus(channelShortId: string, status: number): Promise<void> { private async $updateChannelStatus(channelId: string, status: number): Promise<void> {
try { try {
await DB.query(`UPDATE channels SET status = ? WHERE id = ?`, [status, channelShortId]); await DB.query(`UPDATE channels SET status = ? WHERE id = ?`, [status, channelId]);
} catch (e) { } catch (e) {
logger.err('$updateChannelStatus() error: ' + (e instanceof Error ? e.message : e)); logger.err('$updateChannelStatus() error: ' + (e instanceof Error ? e.message : e));
} }
@ -390,8 +391,8 @@ class NodeSyncService {
private async $saveNode(node: ILightningApi.Node): Promise<void> { private async $saveNode(node: ILightningApi.Node): Promise<void> {
try { try {
const updatedAt = node.updated_at ? this.utcDateToMysql(node.updated_at) : '0000-00-00 00:00:00'; const updatedAt = this.utcDateToMysql(node.last_update);
const sockets = node.sockets.join(','); const sockets = node.addresses.map(a => a.addr).join(',');
const query = `INSERT INTO nodes( const query = `INSERT INTO nodes(
public_key, public_key,
first_seen, first_seen,
@ -403,7 +404,7 @@ class NodeSyncService {
VALUES (?, NOW(), ?, ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?, sockets = ?;`; VALUES (?, NOW(), ?, ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?, sockets = ?;`;
await DB.query(query, [ await DB.query(query, [
node.public_key, node.pub_key,
updatedAt, updatedAt,
node.alias, node.alias,
node.color, node.color,
@ -418,8 +419,18 @@ class NodeSyncService {
} }
} }
private utcDateToMysql(dateString: string): string { /** Decodes a channel id returned by lnd as uint64 to a short channel id */
const d = new Date(Date.parse(dateString)); private toShortId(id: string): string {
const n = BigInt(id);
return [
n >> 40n, // nth block
(n >> 16n) & 0xffffffn, // nth tx of the block
n & 0xffffn // nth output of the tx
].join('x');
}
private utcDateToMysql(date?: number): string {
const d = new Date((date || 0) * 1000);
return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0]; return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
} }
} }

25
backend/src/tasks/lightning/stats-updater.service.ts
View File

@ -52,7 +52,7 @@ class LightningStatsUpdater {
private async $lightningIsSynced(): Promise<boolean> { private async $lightningIsSynced(): Promise<boolean> {
const nodeInfo = await lightningApi.$getInfo(); const nodeInfo = await lightningApi.$getInfo();
return nodeInfo.is_synced_to_chain && nodeInfo.is_synced_to_graph; return nodeInfo.synced_to_chain && nodeInfo.synced_to_graph;
} }
private async $runTasks(): Promise<void> { private async $runTasks(): Promise<void> {
@ -66,13 +66,13 @@ class LightningStatsUpdater {
private async $logLightningStatsDaily() { private async $logLightningStatsDaily() {
try { try {
logger.info(`Running lightning daily stats log...`); logger.info(`Running lightning daily stats log...`);
const networkGraph = await lightningApi.$getNetworkGraph(); const networkGraph = await lightningApi.$getNetworkGraph();
let total_capacity = 0; let total_capacity = 0;
for (const channel of networkGraph.channels) { for (const channel of networkGraph.edges) {
if (channel.capacity) { if (channel.capacity) {
total_capacity += channel.capacity; total_capacity += parseInt(channel.capacity);
} }
} }
@ -80,20 +80,17 @@ class LightningStatsUpdater {
let torNodes = 0; let torNodes = 0;
let unannouncedNodes = 0; let unannouncedNodes = 0;
for (const node of networkGraph.nodes) { for (const node of networkGraph.nodes) {
let isUnnanounced = true; for (const socket of node.addresses) {
for (const socket of node.sockets) { const hasOnion = socket.addr.indexOf('.onion') !== -1;
const hasOnion = socket.indexOf('.onion') !== -1;
if (hasOnion) { if (hasOnion) {
torNodes++; torNodes++;
isUnnanounced = false;
} }
const hasClearnet = [4, 6].includes(net.isIP(socket.split(':')[0])); const hasClearnet = [4, 6].includes(net.isIP(socket.addr.split(':')[0]));
if (hasClearnet) { if (hasClearnet) {
clearnetNodes++; clearnetNodes++;
isUnnanounced = false;
} }
} }
if (isUnnanounced) { if (node.addresses.length === 0) {
unannouncedNodes++; unannouncedNodes++;
} }
} }
@ -118,7 +115,7 @@ class LightningStatsUpdater {
VALUES (NOW() - INTERVAL 1 DAY, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; VALUES (NOW() - INTERVAL 1 DAY, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
await DB.query(query, [ await DB.query(query, [
networkGraph.channels.length, networkGraph.edges.length,
networkGraph.nodes.length, networkGraph.nodes.length,
total_capacity, total_capacity,
torNodes, torNodes,
@ -292,7 +289,7 @@ class LightningStatsUpdater {
for (const node of nodes) { for (const node of nodes) {
const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels WHERE node1_public_key = ? OR node2_public_key = ? ORDER BY created ASC`, [node.public_key, node.public_key]); const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels WHERE node1_public_key = ? OR node2_public_key = ? ORDER BY created ASC`, [node.public_key, node.public_key]);
const date: Date = new Date(this.hardCodedStartTime); const date: Date = new Date(this.hardCodedStartTime);
const currentDate = new Date(); const currentDate = new Date();
this.setDateMidnight(currentDate); this.setDateMidnight(currentDate);
@ -322,7 +319,7 @@ class LightningStatsUpdater {
lastTotalCapacity = totalCapacity; lastTotalCapacity = totalCapacity;
lastChannelsCount = channelsCount; lastChannelsCount = channelsCount;
const query = `INSERT INTO node_stats( const query = `INSERT INTO node_stats(
public_key, public_key,
added, added,