machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into simon/updated-mempool-debug-output

Browse Source
This commit is contained in:
wiz
2022-11-21 17:12:07 +09:00
committed by GitHub
parent 9c09c00fab 5e5daca600
commit 8cc252642b
379 changed files with 89656 additions and 18884 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
backend/src/api/audit.ts Normal file
View File

@@ -0,0 +1,118 @@
import logger from '../logger';
import { BlockExtended, TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
class Audit {
auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended })
: { censored: string[], added: string[], score: number } {
if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
return { censored: [], added: [], score: 0 };
}
const matches: string[] = []; // present in both mined block and template
const added: string[] = []; // present in mined block, not in template
const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN
const isCensored = {}; // missing, without excuse
const isDisplaced = {};
let displacedWeight = 0;
const inBlock = {};
const inTemplate = {};
const now = Math.round((Date.now() / 1000));
for (const tx of transactions) {
inBlock[tx.txid] = tx;
}
// coinbase is always expected
if (transactions[0]) {
inTemplate[transactions[0].txid] = true;
}
// look for transactions that were expected in the template, but missing from the mined block
for (const txid of projectedBlocks[0].transactionIds) {
if (!inBlock[txid]) {
// tx is recent, may have reached the miner too late for inclusion
if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
fresh.push(txid);
} else {
isCensored[txid] = true;
}
displacedWeight += mempool[txid].weight;
}
inTemplate[txid] = true;
}
displacedWeight += (4000 - transactions[0].weight);
logger.warn(`${fresh.length} fresh, ${Object.keys(isCensored).length} possibly censored, ${displacedWeight} displaced weight`);
// we can expect an honest miner to include 'displaced' transactions in place of recent arrivals and censored txs
// these displaced transactions should occupy the first N weight units of the next projected block
let displacedWeightRemaining = displacedWeight;
let index = 0;
let lastFeeRate = Infinity;
let failures = 0;
while (projectedBlocks[1] && index < projectedBlocks[1].transactionIds.length && failures < 500) {
const txid = projectedBlocks[1].transactionIds[index];
const fits = (mempool[txid].weight - displacedWeightRemaining) < 4000;
const feeMatches = mempool[txid].effectiveFeePerVsize >= lastFeeRate;
if (fits || feeMatches) {
isDisplaced[txid] = true;
if (fits) {
lastFeeRate = Math.min(lastFeeRate, mempool[txid].effectiveFeePerVsize);
}
if (mempool[txid].firstSeen == null || (now - (mempool[txid]?.firstSeen || 0)) > PROPAGATION_MARGIN) {
displacedWeightRemaining -= mempool[txid].weight;
}
failures = 0;
} else {
failures++;
}
index++;
}
// mark unexpected transactions in the mined block as 'added'
let overflowWeight = 0;
for (const tx of transactions) {
if (inTemplate[tx.txid]) {
matches.push(tx.txid);
} else {
if (!isDisplaced[tx.txid]) {
added.push(tx.txid);
}
overflowWeight += tx.weight;
}
}
// transactions missing from near the end of our template are probably not being censored
let overflowWeightRemaining = overflowWeight;
let lastOverflowRate = 1.00;
index = projectedBlocks[0].transactionIds.length - 1;
while (index >= 0) {
const txid = projectedBlocks[0].transactionIds[index];
if (overflowWeightRemaining > 0) {
if (isCensored[txid]) {
delete isCensored[txid];
}
lastOverflowRate = mempool[txid].effectiveFeePerVsize;
} else if (Math.floor(mempool[txid].effectiveFeePerVsize * 100) <= Math.ceil(lastOverflowRate * 100)) { // tolerance of 0.01 sat/vb
if (isCensored[txid]) {
delete isCensored[txid];
}
}
overflowWeightRemaining -= (mempool[txid]?.weight || 0);
index--;
}
const numCensored = Object.keys(isCensored).length;
const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0;
return {
censored: Object.keys(isCensored),
added,
score
};
}
}
export default new Audit();

73
backend/src/api/backend-info.ts
View File

@@ -1,60 +1,37 @@
import * as fs from 'fs';
import * as os from 'os';
import logger from '../logger';
import fs from 'fs';
import path from 'path';
import os from 'os';
import { IBackendInfo } from '../mempool.interfaces';
const { spawnSync } = require('child_process');
class BackendInfo {
private gitCommitHash = '';
private hostname = '';
private version = '';
private backendInfo: IBackendInfo;
constructor() {
this.setLatestCommitHash();
this.setVersion();
this.hostname = os.hostname();
}
public getBackendInfo(): IBackendInfo {
return {
hostname: this.hostname,
gitCommit: this.gitCommitHash,
version: this.version,
// This file is created by ./fetch-version.ts during building
const versionFile = path.join(__dirname, 'version.json')
var versionInfo;
if (fs.existsSync(versionFile)) {
versionInfo = JSON.parse(fs.readFileSync(versionFile).toString());
} else {
// Use dummy values if `versionFile` doesn't exist (e.g., during testing)
versionInfo = {
version: '?',
gitCommit: '?'
};
}
this.backendInfo = {
hostname: os.hostname(),
version: versionInfo.version,
gitCommit: versionInfo.gitCommit
};
}
public getBackendInfo(): IBackendInfo {
return this.backendInfo;
}
public getShortCommitHash() {
return this.gitCommitHash.slice(0, 7);
}
private setLatestCommitHash(): void {
//TODO: share this logic with `generate-config.js`
if (process.env.DOCKER_COMMIT_HASH) {
this.gitCommitHash = process.env.DOCKER_COMMIT_HASH;
} else {
try {
const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
if (!gitRevParse.error) {
const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
this.gitCommitHash = output ? output : '?';
} else if (gitRevParse.error.code === 'ENOENT') {
console.log('git not found, cannot parse git hash');
this.gitCommitHash = '?';
}
} catch (e: any) {
console.log('Could not load git commit info: ' + e.message);
this.gitCommitHash = '?';
}
}
}
private setVersion(): void {
try {
const packageJson = fs.readFileSync('package.json').toString();
this.version = JSON.parse(packageJson).version;
} catch (e) {
throw new Error(e instanceof Error ? e.message : 'Error');
}
return this.backendInfo.gitCommit.slice(0, 7);
}
}

7
backend/src/api/bitcoin/bitcoin.routes.ts
View File

@@ -510,7 +510,12 @@ class BitcoinRoutes {
private getDifficultyChange(req: Request, res: Response) {
try {
res.json(difficultyAdjustment.getDifficultyAdjustment());
const da = difficultyAdjustment.getDifficultyAdjustment();
if (da) {
res.json(da);
} else {
res.status(503).send(`Service Temporarily Unavailable`);
}
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}

17
backend/src/api/blocks.ts
View File

@@ -20,6 +20,7 @@ import indexer from '../indexer';
import fiatConversion from './fiat-conversion';
import poolsParser from './pools-parser';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
import mining from './mining/mining';
import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
import PricesRepository from '../repositories/PricesRepository';
@@ -186,14 +187,18 @@ class Blocks {
if (!pool) { // We should never have this situation in practise
logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
`Check your "pools" table entries`);
return blockExtended;
} else {
blockExtended.extras.pool = {
id: pool.id,
name: pool.name,
slug: pool.slug,
};
}
blockExtended.extras.pool = {
id: pool.id,
name: pool.name,
slug: pool.slug,
};
const auditSummary = await BlocksAuditsRepository.$getShortBlockAudit(block.id);
if (auditSummary) {
blockExtended.extras.matchRate = auditSummary.matchRate;
}
}
return blockExtended;

81
backend/src/api/common.ts
View File

@@ -228,34 +228,75 @@ export class Common {
return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
}
static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket {
static findSocketNetwork(addr: string): {network: string | null, url: string} {
let network: string | null = null;
let url = addr.split('://')[1];
if (config.LIGHTNING.BACKEND === 'cln') {
network = socket.network;
} else if (config.LIGHTNING.BACKEND === 'lnd') {
if (socket.addr.indexOf('onion') !== -1) {
if (socket.addr.split('.')[0].length >= 56) {
network = 'torv3';
} else {
network = 'torv2';
}
} else if (socket.addr.indexOf('i2p') !== -1) {
network = 'i2p';
if (!url) {
return {
network: null,
url: addr,
};
}
if (addr.indexOf('onion') !== -1) {
if (url.split('.')[0].length >= 56) {
network = 'torv3';
} else {
const ipv = isIP(socket.addr.split(':')[0]);
if (ipv === 4) {
network = 'ipv4';
} else if (ipv === 6) {
network = 'ipv6';
}
network = 'torv2';
}
} else if (addr.indexOf('i2p') !== -1) {
network = 'i2p';
} else if (addr.indexOf('ipv4') !== -1) {
const ipv = isIP(url.split(':')[0]);
if (ipv === 4) {
network = 'ipv4';
} else {
return {
network: null,
url: addr,
};
}
} else if (addr.indexOf('ipv6') !== -1) {
url = url.split('[')[1].split(']')[0];
const ipv = isIP(url);
if (ipv === 6) {
const parts = addr.split(':');
network = 'ipv6';
url = `[${url}]:${parts[parts.length - 1]}`;
} else {
return {
network: null,
url: addr,
};
}
} else {
return {
network: null,
url: addr,
};
}
return {
publicKey: publicKey,
network: network,
addr: socket.addr,
url: url,
};
}
static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket {
if (config.LIGHTNING.BACKEND === 'cln') {
return {
publicKey: publicKey,
network: socket.network,
addr: socket.addr,
};
} else /* if (config.LIGHTNING.BACKEND === 'lnd') */ {
const formatted = this.findSocketNetwork(socket.addr);
return {
publicKey: publicKey,
network: formatted.network,
addr: formatted.url,
};
}
}
}

6
backend/src/api/database-migration.ts
View File

@@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 40;
private static currentVersion = 41;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -348,6 +348,10 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `nodes` ADD channels int(11) unsigned DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD INDEX `capacity` (`capacity`);');
}
if (databaseSchemaVersion < 41 && isBitcoin === true) {
await this.$executeQuery('UPDATE channels SET closing_reason = NULL WHERE closing_reason = 1');
}
}
/**

7
backend/src/api/difficulty-adjustment.ts
View File

@@ -81,14 +81,15 @@ export function calcDifficultyAdjustment(
}
class DifficultyAdjustmentApi {
constructor() { }
public getDifficultyAdjustment(): IDifficultyAdjustment {
public getDifficultyAdjustment(): IDifficultyAdjustment | null {
const DATime = blocks.getLastDifficultyAdjustmentTime();
const previousRetarget = blocks.getPreviousDifficultyRetarget();
const blockHeight = blocks.getCurrentBlockHeight();
const blocksCache = blocks.getBlocks();
const latestBlock = blocksCache[blocksCache.length - 1];
if (!latestBlock) {
return null;
}
const nowSeconds = Math.floor(new Date().getTime() / 1000);
return calcDifficultyAdjustment(

51
backend/src/api/explorer/channels.api.ts
View File

@@ -39,7 +39,8 @@ class ChannelsApi {
FROM channels
JOIN nodes AS nodes_1 on nodes_1.public_key = channels.node1_public_key
JOIN nodes AS nodes_2 on nodes_2.public_key = channels.node2_public_key
WHERE nodes_1.latitude IS NOT NULL AND nodes_1.longitude IS NOT NULL
WHERE channels.status = 1
AND nodes_1.latitude IS NOT NULL AND nodes_1.longitude IS NOT NULL
AND nodes_2.latitude IS NOT NULL AND nodes_2.longitude IS NOT NULL
`;
@@ -80,7 +81,7 @@ class ChannelsApi {
public async $searchChannelsById(search: string): Promise<any[]> {
try {
const searchStripped = search.replace('%', '') + '%';
const query = `SELECT id, short_id, capacity FROM channels WHERE id LIKE ? OR short_id LIKE ? LIMIT 10`;
const query = `SELECT id, short_id, capacity, status FROM channels WHERE id LIKE ? OR short_id LIKE ? LIMIT 10`;
const [rows]: any = await DB.query(query, [searchStripped, searchStripped]);
return rows;
} catch (e) {
@@ -229,9 +230,14 @@ class ChannelsApi {
public async $getChannelsByTransactionId(transactionIds: string[]): Promise<any[]> {
try {
transactionIds = transactionIds.map((id) => '\'' + id + '\'');
const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.* FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key WHERE channels.transaction_id IN (${transactionIds.join(', ')}) OR channels.closing_transaction_id IN (${transactionIds.join(', ')})`;
const [rows]: any = await DB.query(query);
const query = `
SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*
FROM channels
LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
WHERE channels.transaction_id IN ? OR channels.closing_transaction_id IN ?
`;
const [rows]: any = await DB.query(query, [[transactionIds], [transactionIds]]);
const channels = rows.map((row) => this.convertChannel(row));
return channels;
} catch (e) {
@@ -257,7 +263,8 @@ class ChannelsApi {
let query = `
SELECT COALESCE(node2.alias, SUBSTRING(node2_public_key, 0, 20)) AS alias, COALESCE(node2.public_key, node2_public_key) AS public_key,
channels.status, channels.node1_fee_rate,
channels.capacity, channels.short_id, channels.id, channels.closing_reason
channels.capacity, channels.short_id, channels.id, channels.closing_reason,
UNIX_TIMESTAMP(closing_date) as closing_date, UNIX_TIMESTAMP(channels.updated_at) as updated_at
FROM channels
LEFT JOIN nodes AS node2 ON node2.public_key = channels.node2_public_key
WHERE node1_public_key = ? AND channels.status ${channelStatusFilter}
@@ -268,7 +275,8 @@ class ChannelsApi {
query = `
SELECT COALESCE(node1.alias, SUBSTRING(node1_public_key, 0, 20)) AS alias, COALESCE(node1.public_key, node1_public_key) AS public_key,
channels.status, channels.node2_fee_rate,
channels.capacity, channels.short_id, channels.id, channels.closing_reason
channels.capacity, channels.short_id, channels.id, channels.closing_reason,
UNIX_TIMESTAMP(closing_date) as closing_date, UNIX_TIMESTAMP(channels.updated_at) as updated_at
FROM channels
LEFT JOIN nodes AS node1 ON node1.public_key = channels.node1_public_key
WHERE node2_public_key = ? AND channels.status ${channelStatusFilter}
@@ -277,7 +285,15 @@ class ChannelsApi {
let allChannels = channelsFromNode.concat(channelsToNode);
allChannels.sort((a, b) => {
return b.capacity - a.capacity;
if (status === 'closed') {
if (!b.closing_date && !a.closing_date) {
return (b.updated_at ?? 0) - (a.updated_at ?? 0);
} else {
return (b.closing_date ?? 0) - (a.closing_date ?? 0);
}
} else {
return b.capacity - a.capacity;
}
});
if (index >= 0) {
@@ -294,6 +310,7 @@ class ChannelsApi {
channel = {
status: row.status,
closing_reason: row.closing_reason,
closing_date: row.closing_date,
capacity: row.capacity ?? 0,
short_id: row.short_id,
id: row.id,
@@ -358,6 +375,7 @@ class ChannelsApi {
'transaction_vout': channel.transaction_vout,
'closing_transaction_id': channel.closing_transaction_id,
'closing_reason': channel.closing_reason,
'closing_date': channel.closing_date,
'updated_at': channel.updated_at,
'created': channel.created,
'status': channel.status,
@@ -522,6 +540,23 @@ class ChannelsApi {
logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
}
}
public async $getLatestChannelUpdateForNode(publicKey: string): Promise<number> {
try {
const query = `
SELECT MAX(UNIX_TIMESTAMP(updated_at)) as updated_at
FROM channels
WHERE node1_public_key = ?
`;
const [rows]: any[] = await DB.query(query, [publicKey]);
if (rows.length > 0) {
return rows[0].updated_at;
}
} catch (e) {
logger.err(`Can't getLatestChannelUpdateForNode for ${publicKey}. Reason ${e instanceof Error ? e.message : e}`);
}
return 0;
}
}
export default new ChannelsApi();

130
backend/src/api/explorer/nodes.api.ts
View File

@@ -5,6 +5,49 @@ import { ILightningApi } from '../lightning/lightning-api.interface';
import { ITopNodesPerCapacity, ITopNodesPerChannels } from '../../mempool.interfaces';
class NodesApi {
public async $getWorldNodes(): Promise<any> {
try {
let query = `
SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
CAST(COALESCE(nodes.channels, 0) as INT) as channels,
nodes.longitude, nodes.latitude,
geo_names_country.names as country, geo_names_iso.names as isoCode
FROM nodes
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
WHERE status = 1 AND nodes.as_number IS NOT NULL
ORDER BY capacity
`;
const [nodes]: any[] = await DB.query(query);
for (let i = 0; i < nodes.length; ++i) {
nodes[i].country = JSON.parse(nodes[i].country);
}
query = `
SELECT MAX(nodes.capacity) as maxLiquidity, MAX(nodes.channels) as maxChannels
FROM nodes
WHERE status = 1 AND nodes.as_number IS NOT NULL
`;
const [maximums]: any[] = await DB.query(query);
return {
maxLiquidity: maximums[0].maxLiquidity,
maxChannels: maximums[0].maxChannels,
nodes: nodes.map(node => [
node.longitude, node.latitude,
node.publicKey, node.alias, node.capacity, node.channels,
node.country, node.isoCode
])
};
} catch (e) {
logger.err(`Can't get world nodes list. Reason: ${e instanceof Error ? e.message : e}`);
}
}
public async $getNode(public_key: string): Promise<any> {
try {
// General info
@@ -86,6 +129,56 @@ class NodesApi {
}
}
public async $getFeeHistogram(node_public_key: string): Promise<unknown> {
try {
const inQuery = `
SELECT CASE WHEN fee_rate <= 10.0 THEN CEIL(fee_rate)
WHEN (fee_rate > 10.0 and fee_rate <= 100.0) THEN CEIL(fee_rate / 10.0) * 10.0
WHEN (fee_rate > 100.0 and fee_rate <= 1000.0) THEN CEIL(fee_rate / 100.0) * 100.0
WHEN fee_rate > 1000.0 THEN CEIL(fee_rate / 1000.0) * 1000.0
END as bucket,
count(short_id) as count,
sum(capacity) as capacity
FROM (
SELECT CASE WHEN node1_public_key = ? THEN node2_fee_rate WHEN node2_public_key = ? THEN node1_fee_rate END as fee_rate,
short_id as short_id,
capacity as capacity
FROM channels
WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
) as fee_rate_table
GROUP BY bucket;
`;
const [inRows]: any[] = await DB.query(inQuery, [node_public_key, node_public_key, node_public_key, node_public_key]);
const outQuery = `
SELECT CASE WHEN fee_rate <= 10.0 THEN CEIL(fee_rate)
WHEN (fee_rate > 10.0 and fee_rate <= 100.0) THEN CEIL(fee_rate / 10.0) * 10.0
WHEN (fee_rate > 100.0 and fee_rate <= 1000.0) THEN CEIL(fee_rate / 100.0) * 100.0
WHEN fee_rate > 1000.0 THEN CEIL(fee_rate / 1000.0) * 1000.0
END as bucket,
count(short_id) as count,
sum(capacity) as capacity
FROM (
SELECT CASE WHEN node1_public_key = ? THEN node1_fee_rate WHEN node2_public_key = ? THEN node2_fee_rate END as fee_rate,
short_id as short_id,
capacity as capacity
FROM channels
WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
) as fee_rate_table
GROUP BY bucket;
`;
const [outRows]: any[] = await DB.query(outQuery, [node_public_key, node_public_key, node_public_key, node_public_key]);
return {
incoming: inRows.length > 0 ? inRows : [],
outgoing: outRows.length > 0 ? outRows : [],
};
} catch (e) {
logger.err(`Cannot get node fee distribution for ${node_public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);
throw e;
}
}
public async $getAllNodes(): Promise<any> {
try {
const query = `SELECT * FROM nodes`;
@@ -133,10 +226,13 @@ class NodesApi {
CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
CAST(COALESCE(nodes.channels, 0) as INT) as channels,
UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
geo_names_city.names as city, geo_names_country.names as country
geo_names_city.names as city, geo_names_country.names as country,
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
FROM nodes
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
ORDER BY capacity DESC
LIMIT 100
`;
@@ -175,10 +271,13 @@ class NodesApi {
CAST(COALESCE(nodes.channels, 0) as INT) as channels,
CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
geo_names_city.names as city, geo_names_country.names as country
geo_names_city.names as city, geo_names_country.names as country,
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
FROM nodes
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
ORDER BY channels DESC
LIMIT 100
`;
@@ -221,11 +320,14 @@ class NodesApi {
CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity,
UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
geo_names_city.names as city, geo_names_country.names as country
geo_names_city.names as city, geo_names_country.names as country,
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
FROM node_stats
RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
WHERE added = FROM_UNIXTIME(${latestDate})
ORDER BY first_seen
LIMIT 100
@@ -249,7 +351,7 @@ class NodesApi {
try {
const publicKeySearch = search.replace('%', '') + '%';
const aliasSearch = search.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '').split(' ').map((search) => '+' + search + '*').join(' ');
const query = `SELECT public_key, alias, capacity, channels FROM nodes WHERE public_key LIKE ? OR MATCH alias_search AGAINST (? IN BOOLEAN MODE) ORDER BY capacity DESC LIMIT 10`;
const query = `SELECT public_key, alias, capacity, channels, status FROM nodes WHERE public_key LIKE ? OR MATCH alias_search AGAINST (? IN BOOLEAN MODE) ORDER BY capacity DESC LIMIT 10`;
const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]);
return rows;
} catch (e) {
@@ -382,12 +484,14 @@ class NodesApi {
SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
geo_names_city.names as city, geo_names_country.names as country,
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision,
nodes.longitude, nodes.latitude, nodes.as_number, geo_names_isp.names as isp
FROM nodes
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
LEFT JOIN geo_names geo_names_isp on geo_names_isp.id = nodes.as_number AND geo_names_isp.type = 'as_organization'
WHERE geo_names_country.id = ?
ORDER BY capacity DESC
`;
@@ -397,6 +501,7 @@ class NodesApi {
rows[i].country = JSON.parse(rows[i].country);
rows[i].city = JSON.parse(rows[i].city);
rows[i].subdivision = JSON.parse(rows[i].subdivision);
rows[i].isp = JSON.parse(rows[i].isp);
}
return rows;
} catch (e) {
@@ -411,7 +516,8 @@ class NodesApi {
SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
geo_names_city.names as city, geo_names_country.names as country,
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision,
nodes.longitude, nodes.latitude
FROM nodes
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
@@ -503,6 +609,18 @@ class NodesApi {
}
}
/**
* Update node sockets
*/
public async $updateNodeSockets(publicKey: string, sockets: {network: string; addr: string}[]): Promise<void> {
const formattedSockets = (sockets.map(a => a.addr).join(',')) ?? '';
try {
await DB.query(`UPDATE nodes SET sockets = ? WHERE public_key = ?`, [formattedSockets, publicKey]);
} catch (e) {
logger.err(`Cannot update node sockets for ${publicKey}. Reason: ${e instanceof Error ? e.message : e}`);
}
}
/**
* Set all nodes not in `nodesPubkeys` as inactive (status = 0)
*/

65
backend/src/api/explorer/nodes.routes.ts
View File

@@ -9,6 +9,7 @@ class NodesRoutes {
public initRoutes(app: Application) {
app
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/world', this.$getWorldNodes)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/country/:country', this.$getNodesPerCountry)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/search/:search', this.$searchNode)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp-ranking', this.$getISPRanking)
@@ -19,7 +20,9 @@ class NodesRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/connectivity', this.$getTopNodesByChannels)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/age', this.$getOldestNodes)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/statistics', this.$getHistoricalNodeStats)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/fees/histogram', this.$getFeeHistogram)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key', this.$getNode)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/group/:name', this.$getNodeGroup)
;
}
@@ -32,6 +35,39 @@ class NodesRoutes {
}
}
private async $getNodeGroup(req: Request, res: Response) {
try {
let nodesList;
let nodes: any[] = [];
switch (config.MEMPOOL.NETWORK) {
case 'testnet':
nodesList = ['032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b', '025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7', '0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55', '032ab2028c0b614c6d87824e2373529652fd7e4221b4c70cc4da7c7005c49afcf0', '029001b22fe70b48bee12d014df91982eb85ff1bd404ec772d5c83c4ee3e88d2c3', '0212e2848d79f928411da5f2ff0a8c95ec6ccb5a09d2031b6f71e91309dcde63af', '03e871a2229523d34f76e6311ff197cfe7f26c2fbec13554b93a46f4e710c47dab', '032202ec98d976b0e928bd1d91924e8bd3eab07231fc39feb3737b010071073df8', '02fa7c5a948d03d563a9f36940c2205a814e594d17c0042ced242c71a857d72605', '039c14fdec2d958e3d14cebf657451bbd9e039196615785e82c917f274e3fb2205', '033589bbcb233ffc416cefd5437c7f37e9d7cb7942d405e39e72c4c846d9b37f18', '029293110441c6e2eacb57e1255bf6ef05c41a6a676fe474922d33c19f98a7d584'];
break;
case 'signet':
nodesList = ['03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956', '033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de', '02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781', '025196512905b8a3f1597428b867bec63ec9a95e5089eb7dc7e63e2d2691669029', '027c625aa1fbe3768db68ebcb05b53b6dc0ce68b7b54b8900d326d167363e684fe', '03f1629af3101fcc56b7aac2667016be84e3defbf3d0c8719f836c9b41c9a57a43', '02dfb81e2f7a3c4c9e8a51b70ef82b4a24549cc2fab1f5b2fd636501774a918991', '02d01ccf832944c68f10d39006093769c5b8bda886d561b128534e313d729fdb34', '02499ed23027d4698a6904ff4ec1b6085a61f10b9a6937f90438f9947e38e8ea86', '038310e3a786340f2bd7770704c7ccfe560fd163d9a1c99d67894597419d12cbf7', '03e5e9d879b72c7d67ecd483bae023bd33e695bb32b981a4021260f7b9d62bc761', '028d16e1a0ace4c0c0a421536d8d32ce484dfe6e2f726b7b0e7c30f12a195f8cc7'];
break;
default:
nodesList = ['03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61', '03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437', '03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144', '0238bd27f02d67d6c51e269692bc8c9a32357a00e7777cba7f4f1f18a2a700b108', '03f983dcabed6baa1eab5b56c8b2e8fdc846ab3fd931155377897335e85a9fa57c', '03e399589533581e48796e29a825839a010036a61b20744fda929d6709fcbffcc5', '021f5288b5f72c42cd0d8801086af7ce09a816d8ee9a4c47a4b436399b26cb601a', '032b01b7585f781420cd4148841a82831ba37fa952342052cec16750852d4f2dd9', '02848036488d4b8fb1f1c4064261ec36151f43b085f0b51bd239ade3ddfc940c34', '02b6b1640fe029e304c216951af9fbefdb23b0bdc9baaf327540d31b6107841fdf', '03694289827203a5b3156d753071ddd5bf92e371f5a462943f9555eef6d2d6606c', '0283d850db7c3e8ea7cc9c4abc7afaab12bbdf72b677dcba1d608350d2537d7d43'];
}
for (let pubKey of nodesList) {
try {
const node = await nodesApi.$getNode(pubKey);
if (node) {
nodes.push(node);
}
} catch (e) {}
}
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(nodes);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getNode(req: Request, res: Response) {
try {
const node = await nodesApi.$getNode(req.params.public_key);
@@ -60,6 +96,22 @@ class NodesRoutes {
}
}
private async $getFeeHistogram(req: Request, res: Response) {
try {
const node = await nodesApi.$getFeeHistogram(req.params.public_key);
if (!node) {
res.status(404).send('Node not found');
return;
}
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(node);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getNodesRanking(req: Request, res: Response): Promise<void> {
try {
const topCapacityNodes = await nodesApi.$getTopCapacityNodes(false);
@@ -115,7 +167,6 @@ class NodesRoutes {
private async $getISPRanking(req: Request, res: Response): Promise<void> {
try {
const nodesPerAs = await nodesApi.$getNodesISPRanking();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
@@ -125,6 +176,18 @@ class NodesRoutes {
}
}
private async $getWorldNodes(req: Request, res: Response) {
try {
const worldNodes = await nodesApi.$getWorldNodes();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
res.json(worldNodes);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getNodesPerCountry(req: Request, res: Response) {
try {
const [country]: any[] = await DB.query(

5
backend/src/api/explorer/statistics.api.ts
View File

@@ -6,7 +6,8 @@ class StatisticsApi {
public async $getStatistics(interval: string | null = null): Promise<any> {
interval = Common.getSqlInterval(interval);
let query = `SELECT UNIX_TIMESTAMP(added) AS added, channel_count, total_capacity, tor_nodes, clearnet_nodes, unannounced_nodes
let query = `SELECT UNIX_TIMESTAMP(added) AS added, channel_count, total_capacity,
tor_nodes, clearnet_nodes, unannounced_nodes, clearnet_tor_nodes
FROM lightning_stats`;
if (interval) {
@@ -27,7 +28,7 @@ class StatisticsApi {
public async $getLatestStatistics(): Promise<any> {
try {
const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1`);
const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1 OFFSET 7`);
const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats WHERE DATE(added) = DATE(NOW() - INTERVAL 7 DAY)`);
return {
latest: rows[0],
previous: rows2[0],

37
backend/src/api/fetch-version.ts Normal file
View File

@@ -0,0 +1,37 @@
import fs from 'fs';
import path from "path";
const { spawnSync } = require('child_process');
function getVersion(): string {
const packageJson = fs.readFileSync('package.json').toString();
return JSON.parse(packageJson).version;
}
function getGitCommit(): string {
if (process.env.MEMPOOL_COMMIT_HASH) {
return process.env.MEMPOOL_COMMIT_HASH;
} else {
const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
if (!gitRevParse.error) {
const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
if (output) {
return output;
} else {
console.log('Could not fetch git commit: No repo available');
}
} else if (gitRevParse.error.code === 'ENOENT') {
console.log('Could not fetch git commit: Command `git` is unavailable');
}
}
return '?';
}
const versionInfo = {
version: getVersion(),
gitCommit: getGitCommit()
}
fs.writeFileSync(
path.join(__dirname, 'version.json'),
JSON.stringify(versionInfo, null, 2) + "\n"
);

10
backend/src/api/lightning/clightning/clightning-convert.ts
View File

@@ -13,9 +13,13 @@ export function convertNode(clNode: any): ILightningApi.Node {
features: [], // TODO parse and return clNode.feature
pub_key: clNode.nodeid,
addresses: clNode.addresses?.map((addr) => {
let address = addr.address;
if (addr.type === 'ipv6') {
address = `[${address}]`;
}
return {
network: addr.type,
addr: `${addr.address}:${addr.port}`
addr: `${address}:${addr.port}`
};
}) ?? [],
last_update: clNode?.last_timestamp ?? 0,
@@ -66,6 +70,8 @@ export async function convertAndmergeBidirectionalChannels(clChannels: any[]): P
logger.info(`Building partial channels from clightning output. Channels processed: ${channelProcessed + 1} of ${keys.length}`);
loggerTimer = new Date().getTime() / 1000;
}
channelProcessed++;
}
return consolidatedChannelList;
@@ -120,7 +126,7 @@ async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Cha
*/
function convertPolicy(clChannel: any): ILightningApi.RoutingPolicy {
return {
time_lock_delta: 0, // TODO
time_lock_delta: clChannel.delay,
min_htlc: clChannel.htlc_minimum_msat.slice(0, -4),
max_htlc_msat: clChannel.htlc_maximum_msat.slice(0, -4),
fee_base_msat: clChannel.base_fee_millisatoshi,

279
backend/src/api/mempool-blocks.ts
View File

@@ -1,7 +1,8 @@
import logger from '../logger';
import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta } from '../mempool.interfaces';
import { MempoolBlock, TransactionExtended, AuditTransaction, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor } from '../mempool.interfaces';
import { Common } from './common';
import config from '../config';
import { PairingHeap } from '../utils/pairing-heap';
class MempoolBlocks {
private mempoolBlocks: MempoolBlockWithTransactions[] = [];
@@ -72,6 +73,7 @@ class MempoolBlocks {
logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
const { blocks, deltas } = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
this.mempoolBlocks = blocks;
this.mempoolBlockDeltas = deltas;
}
@@ -99,6 +101,7 @@ class MempoolBlocks {
if (transactions.length) {
mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
}
// Calculate change from previous block states
for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
let added: TransactionStripped[] = [];
@@ -132,12 +135,286 @@ class MempoolBlocks {
removed
});
}
return {
blocks: mempoolBlocks,
deltas: mempoolBlockDeltas
};
}
/*
* Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core
* (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp)
*
* blockLimit: number of blocks to build in total.
* weightLimit: maximum weight of transactions to consider using the selection algorithm.
* if weightLimit is significantly lower than the mempool size, results may start to diverge from getBlockTemplate
* condenseRest: whether to ignore excess transactions or append them to the final block.
*/
public makeBlockTemplates(mempool: { [txid: string]: TransactionExtended }, blockLimit: number, weightLimit: number | null = null, condenseRest = false): MempoolBlockWithTransactions[] {
const start = Date.now();
const auditPool: { [txid: string]: AuditTransaction } = {};
const mempoolArray: AuditTransaction[] = [];
const restOfArray: TransactionExtended[] = [];
let weight = 0;
const maxWeight = weightLimit ? Math.max(4_000_000 * blockLimit, weightLimit) : Infinity;
// grab the top feerate txs up to maxWeight
Object.values(mempool).sort((a, b) => b.feePerVsize - a.feePerVsize).forEach(tx => {
weight += tx.weight;
if (weight >= maxWeight) {
restOfArray.push(tx);
return;
}
// initializing everything up front helps V8 optimize property access later
auditPool[tx.txid] = {
txid: tx.txid,
fee: tx.fee,
size: tx.size,
weight: tx.weight,
feePerVsize: tx.feePerVsize,
vin: tx.vin,
relativesSet: false,
ancestorMap: new Map<string, AuditTransaction>(),
children: new Set<AuditTransaction>(),
ancestorFee: 0,
ancestorWeight: 0,
score: 0,
used: false,
modified: false,
modifiedNode: null,
}
mempoolArray.push(auditPool[tx.txid]);
})
// Build relatives graph & calculate ancestor scores
for (const tx of mempoolArray) {
if (!tx.relativesSet) {
this.setRelatives(tx, auditPool);
}
}
// Sort by descending ancestor score
mempoolArray.sort((a, b) => (b.score || 0) - (a.score || 0));
// Build blocks by greedily choosing the highest feerate package
// (i.e. the package rooted in the transaction with the best ancestor score)
const blocks: MempoolBlockWithTransactions[] = [];
let blockWeight = 4000;
let blockSize = 0;
let transactions: AuditTransaction[] = [];
const modified: PairingHeap<AuditTransaction> = new PairingHeap((a, b): boolean => (a.score || 0) > (b.score || 0));
let overflow: AuditTransaction[] = [];
let failures = 0;
let top = 0;
while ((top < mempoolArray.length || !modified.isEmpty()) && (condenseRest || blocks.length < blockLimit)) {
// skip invalid transactions
while (top < mempoolArray.length && (mempoolArray[top].used || mempoolArray[top].modified)) {
top++;
}
// Select best next package
let nextTx;
const nextPoolTx = mempoolArray[top];
const nextModifiedTx = modified.peek();
if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) {
nextTx = nextPoolTx;
top++;
} else {
modified.pop();
if (nextModifiedTx) {
nextTx = nextModifiedTx;
nextTx.modifiedNode = undefined;
}
}
if (nextTx && !nextTx?.used) {
// Check if the package fits into this block
if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
blockWeight += nextTx.ancestorWeight;
const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values());
// sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4);
sortedTxSet.forEach((ancestor, i, arr) => {
const mempoolTx = mempool[ancestor.txid];
if (ancestor && !ancestor?.used) {
ancestor.used = true;
// update original copy of this tx with effective fee rate & relatives data
mempoolTx.effectiveFeePerVsize = effectiveFeeRate;
mempoolTx.ancestors = (Array.from(ancestor.ancestorMap?.values()) as AuditTransaction[]).map((a) => {
return {
txid: a.txid,
fee: a.fee,
weight: a.weight,
}
})
if (i < arr.length - 1) {
mempoolTx.bestDescendant = {
txid: arr[arr.length - 1].txid,
fee: arr[arr.length - 1].fee,
weight: arr[arr.length - 1].weight,
};
}
transactions.push(ancestor);
blockSize += ancestor.size;
}
});
// remove these as valid package ancestors for any descendants remaining in the mempool
if (sortedTxSet.length) {
sortedTxSet.forEach(tx => {
this.updateDescendants(tx, auditPool, modified);
});
}
failures = 0;
} else {
// hold this package in an overflow list while we check for smaller options
overflow.push(nextTx);
failures++;
}
}
// this block is full
const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000);
if (exceededPackageTries && (!condenseRest || blocks.length < blockLimit - 1)) {
// construct this block
if (transactions.length) {
blocks.push(this.dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length));
}
// reset for the next block
transactions = [];
blockSize = 0;
blockWeight = 4000;
// 'overflow' packages didn't fit in this block, but are valid candidates for the next
for (const overflowTx of overflow.reverse()) {
if (overflowTx.modified) {
overflowTx.modifiedNode = modified.add(overflowTx);
} else {
top--;
mempoolArray[top] = overflowTx;
}
}
overflow = [];
}
}
if (condenseRest) {
// pack any leftover transactions into the last block
for (const tx of overflow) {
if (!tx || tx?.used) {
continue;
}
blockWeight += tx.weight;
blockSize += tx.size;
transactions.push(tx);
tx.used = true;
}
const blockTransactions = transactions.map(t => mempool[t.txid])
restOfArray.forEach(tx => {
blockWeight += tx.weight;
blockSize += tx.size;
blockTransactions.push(tx);
});
if (blockTransactions.length) {
blocks.push(this.dataToMempoolBlocks(blockTransactions, blockSize, blockWeight, blocks.length));
}
transactions = [];
} else if (transactions.length) {
blocks.push(this.dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length));
}
const end = Date.now();
const time = end - start;
logger.debug('Mempool templates calculated in ' + time / 1000 + ' seconds');
return blocks;
}
// traverse in-mempool ancestors
// recursion unavoidable, but should be limited to depth < 25 by mempool policy
public setRelatives(
tx: AuditTransaction,
mempool: { [txid: string]: AuditTransaction },
): void {
for (const parent of tx.vin) {
const parentTx = mempool[parent.txid];
if (parentTx && !tx.ancestorMap!.has(parent.txid)) {
tx.ancestorMap.set(parent.txid, parentTx);
parentTx.children.add(tx);
// visit each node only once
if (!parentTx.relativesSet) {
this.setRelatives(parentTx, mempool);
}
parentTx.ancestorMap.forEach((ancestor) => {
tx.ancestorMap.set(ancestor.txid, ancestor);
});
}
};
tx.ancestorFee = tx.fee || 0;
tx.ancestorWeight = tx.weight || 0;
tx.ancestorMap.forEach((ancestor) => {
tx.ancestorFee += ancestor.fee;
tx.ancestorWeight += ancestor.weight;
});
tx.score = tx.ancestorFee / (tx.ancestorWeight || 1);
tx.relativesSet = true;
}
// iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score
// avoids recursion to limit call stack depth
private updateDescendants(
rootTx: AuditTransaction,
mempool: { [txid: string]: AuditTransaction },
modified: PairingHeap<AuditTransaction>,
): void {
const descendantSet: Set<AuditTransaction> = new Set();
// stack of nodes left to visit
const descendants: AuditTransaction[] = [];
let descendantTx;
let ancestorIndex;
let tmpScore;
rootTx.children.forEach(childTx => {
if (!descendantSet.has(childTx)) {
descendants.push(childTx);
descendantSet.add(childTx);
}
});
while (descendants.length) {
descendantTx = descendants.pop();
if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) {
// remove tx as ancestor
descendantTx.ancestorMap.delete(rootTx.txid);
descendantTx.ancestorFee -= rootTx.fee;
descendantTx.ancestorWeight -= rootTx.weight;
tmpScore = descendantTx.score;
descendantTx.score = descendantTx.ancestorFee / descendantTx.ancestorWeight;
if (!descendantTx.modifiedNode) {
descendantTx.modified = true;
descendantTx.modifiedNode = modified.add(descendantTx);
} else {
// rebalance modified heap if score has changed
if (descendantTx.score < tmpScore) {
modified.decreasePriority(descendantTx.modifiedNode);
} else if (descendantTx.score > tmpScore) {
modified.increasePriority(descendantTx.modifiedNode);
}
}
// add this node's children to the stack
descendantTx.children.forEach(childTx => {
// visit each node only once
if (!descendantSet.has(childTx)) {
descendants.push(childTx);
descendantSet.add(childTx);
}
});
}
}
}
private dataToMempoolBlocks(transactions: TransactionExtended[],
blockSize: number, blockWeight: number, blocksIndex: number): MempoolBlockWithTransactions {
let rangeLength = 4;

6
backend/src/api/mining/mining-routes.ts
View File

@@ -238,6 +238,12 @@ class MiningRoutes {
public async $getBlockAudit(req: Request, res: Response) {
try {
const audit = await BlocksAuditsRepository.$getBlockAudit(req.params.hash);
if (!audit) {
res.status(404).send(`This block has not been audited.`);
return;
}
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());

103
backend/src/api/websocket-handler.ts
View File

@@ -18,6 +18,7 @@ import difficultyAdjustment from './difficulty-adjustment';
import feeApi from './fee-api';
import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import Audit from './audit';
class WebsocketHandler {
private wss: WebSocket.Server | undefined;
@@ -405,75 +406,63 @@ class WebsocketHandler {
});
}
handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) {
handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]): void {
if (!this.wss) {
throw new Error('WebSocket.Server is not set');
}
let mBlocks: undefined | MempoolBlock[];
let mBlockDeltas: undefined | MempoolBlockDelta[];
let matchRate = 0;
let matchRate;
const _memPool = memPool.getMempool();
const _mempoolBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
if (_mempoolBlocks[0]) {
const matches: string[] = [];
const added: string[] = [];
const missing: string[] = [];
if (Common.indexingEnabled()) {
const mempoolCopy = cloneMempool(_memPool);
const projectedBlocks = mempoolBlocks.makeBlockTemplates(mempoolCopy, 2);
for (const txId of txIds) {
if (_mempoolBlocks[0].transactionIds.indexOf(txId) > -1) {
matches.push(txId);
} else {
added.push(txId);
const { censored, added, score } = Audit.auditBlock(transactions, projectedBlocks, mempoolCopy);
matchRate = Math.round(score * 100 * 100) / 100;
const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
return {
txid: tx.txid,
vsize: tx.vsize,
fee: tx.fee ? Math.round(tx.fee) : 0,
value: tx.value,
};
}) : [];
BlocksSummariesRepository.$saveSummary({
height: block.height,
template: {
id: block.id,
transactions: stripped
}
delete _memPool[txId];
}
});
for (const txId of _mempoolBlocks[0].transactionIds) {
if (matches.includes(txId) || added.includes(txId)) {
continue;
}
missing.push(txId);
}
BlocksAuditsRepository.$saveAudit({
time: block.timestamp,
height: block.height,
hash: block.id,
addedTxs: added,
missingTxs: censored,
matchRate: matchRate,
});
matchRate = Math.round((Math.max(0, matches.length - missing.length - added.length) / txIds.length * 100) * 100) / 100;
mempoolBlocks.updateMempoolBlocks(_memPool);
mBlocks = mempoolBlocks.getMempoolBlocks();
mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
if (Common.indexingEnabled()) {
const stripped = _mempoolBlocks[0].transactions.map((tx) => {
return {
txid: tx.txid,
vsize: tx.vsize,
fee: tx.fee ? Math.round(tx.fee) : 0,
value: tx.value,
};
});
BlocksSummariesRepository.$saveSummary({
height: block.height,
template: {
id: block.id,
transactions: stripped
}
});
BlocksAuditsRepository.$saveAudit({
time: block.timestamp,
height: block.height,
hash: block.id,
addedTxs: added,
missingTxs: missing,
matchRate: matchRate,
});
if (block.extras) {
block.extras.matchRate = matchRate;
}
}
if (block.extras) {
block.extras.matchRate = matchRate;
// Update mempool to remove transactions included in the new block
for (const txId of txIds) {
delete _memPool[txId];
}
mempoolBlocks.updateMempoolBlocks(_memPool);
mBlocks = mempoolBlocks.getMempoolBlocks();
mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
const da = difficultyAdjustment.getDifficultyAdjustment();
const fees = feeApi.getRecommendedFee();
@@ -580,4 +569,14 @@ class WebsocketHandler {
}
}
function cloneMempool(mempool: { [txid: string]: TransactionExtended }): { [txid: string]: TransactionExtended } {
const cloned = {};
Object.keys(mempool).forEach(id => {
cloned[id] = {
...mempool[id]
};
});
return cloned;
}
export default new WebsocketHandler();

6
backend/src/config.ts
View File

@@ -1,4 +1,6 @@
const configFile = require('../mempool-config.json');
const configFromFile = require(
process.env.MEMPOOL_CONFIG_FILE ? process.env.MEMPOOL_CONFIG_FILE : '../mempool-config.json'
);
interface IConfig {
MEMPOOL: {
@@ -249,7 +251,7 @@ class Config implements IConfig {
MAXMIND: IConfig['MAXMIND'];
constructor() {
const configs = this.merge(configFile, defaults);
const configs = this.merge(configFromFile, defaults);
this.MEMPOOL = configs.MEMPOOL;
this.ESPLORA = configs.ESPLORA;
this.ELECTRUM = configs.ELECTRUM;

2
backend/src/logger.ts
View File

@@ -74,7 +74,7 @@ class Logger {
private getNetwork(): string {
if (config.LIGHTNING.ENABLED) {
return 'lightning';
return config.MEMPOOL.NETWORK === 'mainnet' ? 'lightning' : `${config.MEMPOOL.NETWORK}-lightning`;
}
if (config.BISQ.ENABLED) {
return 'bisq';

31
backend/src/mempool.interfaces.ts
View File

@@ -1,4 +1,5 @@
import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
import { HeapNode } from "./utils/pairing-heap";
export interface PoolTag {
id: number; // mysql row id
@@ -70,12 +71,40 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
deleteAfter?: number;
}
interface Ancestor {
export interface AuditTransaction {
txid: string;
fee: number;
size: number;
weight: number;
feePerVsize: number;
vin: IEsploraApi.Vin[];
relativesSet: boolean;
ancestorMap: Map<string, AuditTransaction>;
children: Set<AuditTransaction>;
ancestorFee: number;
ancestorWeight: number;
score: number;
used: boolean;
modified: boolean;
modifiedNode: HeapNode<AuditTransaction>;
}
export interface Ancestor {
txid: string;
weight: number;
fee: number;
}
export interface TransactionSet {
fee: number;
weight: number;
score: number;
children?: Set<string>;
available?: boolean;
modified?: boolean;
modifiedNode?: HeapNode<string>;
}
interface BestDescendant {
txid: string;
weight: number;

24
backend/src/repositories/BlocksAuditsRepository.ts
View File

@@ -58,10 +58,12 @@ class BlocksAuditRepositories {
WHERE blocks_audits.hash = "${hash}"
`);
rows[0].missingTxs = JSON.parse(rows[0].missingTxs);
rows[0].addedTxs = JSON.parse(rows[0].addedTxs);
rows[0].transactions = JSON.parse(rows[0].transactions);
rows[0].template = JSON.parse(rows[0].template);
if (rows.length) {
rows[0].missingTxs = JSON.parse(rows[0].missingTxs);
rows[0].addedTxs = JSON.parse(rows[0].addedTxs);
rows[0].transactions = JSON.parse(rows[0].transactions);
rows[0].template = JSON.parse(rows[0].template);
}
return rows[0];
} catch (e: any) {
@@ -69,6 +71,20 @@ class BlocksAuditRepositories {
throw e;
}
}
public async $getShortBlockAudit(hash: string): Promise<any> {
try {
const [rows]: any[] = await DB.query(
`SELECT hash as id, match_rate as matchRate
FROM blocks_audits
WHERE blocks_audits.hash = "${hash}"
`);
return rows[0];
} catch (e: any) {
logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
}
export default new BlocksAuditRepositories();

94
backend/src/tasks/lightning/network-sync.service.ts
View File

@@ -12,9 +12,11 @@ import { ResultSetHeader } from 'mysql2';
import fundingTxFetcher from './sync-tasks/funding-tx-fetcher';
import NodesSocketsRepository from '../../repositories/NodesSocketsRepository';
import { Common } from '../../api/common';
import blocks from '../../api/blocks';
class NetworkSyncService {
loggerTimer = 0;
closedChannelsScanBlock = 0;
constructor() {}
@@ -63,6 +65,9 @@ class NetworkSyncService {
let deletedSockets = 0;
const graphNodesPubkeys: string[] = [];
for (const node of nodes) {
const latestUpdated = await channelsApi.$getLatestChannelUpdateForNode(node.pub_key);
node.last_update = Math.max(node.last_update, latestUpdated);
await nodesApi.$saveNode(node);
graphNodesPubkeys.push(node.pub_key);
++progress;
@@ -237,10 +242,22 @@ class NetworkSyncService {
}
private async $scanForClosedChannels(): Promise<void> {
if (this.closedChannelsScanBlock === blocks.getCurrentBlockHeight()) {
logger.debug(`We've already scan closed channels for this block, skipping.`);
return;
}
let progress = 0;
try {
logger.info(`Starting closed channels scan`);
let log = `Starting closed channels scan`;
if (this.closedChannelsScanBlock > 0) {
log += `. Last scan was at block ${this.closedChannelsScanBlock}`;
} else {
log += ` for the first time`;
}
logger.info(log);
const channels = await channelsApi.$getChannelsByStatus([0, 1]);
for (const channel of channels) {
const spendingTx = await bitcoinApi.$getOutspend(channel.transaction_id, channel.transaction_vout);
@@ -260,7 +277,9 @@ class NetworkSyncService {
this.loggerTimer = new Date().getTime() / 1000;
}
}
logger.info(`Closed channels scan complete.`);
this.closedChannelsScanBlock = blocks.getCurrentBlockHeight();
logger.info(`Closed channels scan completed at block ${this.closedChannelsScanBlock}`);
} catch (e) {
logger.err('$scanForClosedChannels() error: ' + (e instanceof Error ? e.message : e));
}
@@ -270,6 +289,24 @@ class NetworkSyncService {
1. Mutually closed
2. Forced closed
3. Forced closed with penalty
┌────────────────────────────────────┐ ┌────────────────────────────┐
│ outputs contain revocation script? ├──yes──► force close w/ penalty = 3 │
└──────────────┬─────────────────────┘ └────────────────────────────┘
no
┌──────────────▼──────────────────────────┐
│ outputs contain other lightning script? ├──┐
└──────────────┬──────────────────────────┘ │
no yes
┌──────────────▼─────────────┐ │
│ sequence starts with 0x80 │ ┌────────▼────────┐
│ and ├──────► force close = 2 │
│ locktime starts with 0x20? │ └─────────────────┘
└──────────────┬─────────────┘
no
┌─────────▼────────┐
│ mutual close = 1 │
└──────────────────┘
*/
private async $runClosedChannelsForensics(): Promise<void> {
@@ -307,36 +344,31 @@ class NetworkSyncService {
lightningScriptReasons.push(lightningScript);
}
}
if (lightningScriptReasons.length === outspends.length
&& lightningScriptReasons.filter((r) => r === 1).length === outspends.length) {
reason = 1;
} else {
const filteredReasons = lightningScriptReasons.filter((r) => r !== 1);
if (filteredReasons.length) {
if (filteredReasons.some((r) => r === 2 || r === 4)) {
reason = 3;
} else {
reason = 2;
}
const filteredReasons = lightningScriptReasons.filter((r) => r !== 1);
if (filteredReasons.length) {
if (filteredReasons.some((r) => r === 2 || r === 4)) {
reason = 3;
} else {
/*
We can detect a commitment transaction (force close) by reading Sequence and Locktime
https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction
*/
let closingTx: IEsploraApi.Transaction | undefined;
try {
closingTx = await bitcoinApi.$getRawTransaction(channel.closing_transaction_id);
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id}. Reason ${e instanceof Error ? e.message : e}`);
continue;
}
const sequenceHex: string = closingTx.vin[0].sequence.toString(16);
const locktimeHex: string = closingTx.locktime.toString(16);
if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') {
reason = 2; // Here we can't be sure if it's a penalty or not
} else {
reason = 1;
}
reason = 2;
}
} else {
/*
We can detect a commitment transaction (force close) by reading Sequence and Locktime
https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction
*/
let closingTx: IEsploraApi.Transaction | undefined;
try {
closingTx = await bitcoinApi.$getRawTransaction(channel.closing_transaction_id);
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id}. Reason ${e instanceof Error ? e.message : e}`);
continue;
}
const sequenceHex: string = closingTx.vin[0].sequence.toString(16);
const locktimeHex: string = closingTx.locktime.toString(16);
if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') {
reason = 2; // Here we can't be sure if it's a penalty or not
} else {
reason = 1;
}
}
if (reason) {

73
backend/src/tasks/lightning/sync-tasks/node-locations.ts
View File

@@ -4,10 +4,14 @@ import nodesApi from '../../../api/explorer/nodes.api';
import config from '../../../config';
import DB from '../../../database';
import logger from '../../../logger';
import { ResultSetHeader } from 'mysql2';
import * as IPCheck from '../../../utils/ipcheck.js';
export async function $lookupNodeLocation(): Promise<void> {
let loggerTimer = new Date().getTime() / 1000;
let progress = 0;
let nodesUpdated = 0;
let geoNamesInserted = 0;
logger.info(`Running node location updater using Maxmind`);
try {
@@ -27,6 +31,26 @@ export async function $lookupNodeLocation(): Promise<void> {
const asn = lookupAsn.get(ip);
const isp = lookupIsp.get(ip);
let asOverwrite: any | undefined;
if (asn && (IPCheck.match(ip, '170.75.160.0/20') || IPCheck.match(ip, '172.81.176.0/21'))) {
asOverwrite = {
asn: 394745,
name: 'Lunanode',
};
}
else if (asn && (IPCheck.match(ip, '50.7.0.0/16') || IPCheck.match(ip, '66.90.64.0/18'))) {
asOverwrite = {
asn: 30058,
name: 'FDCservers.net',
};
}
else if (asn && asn.autonomous_system_number === 174) {
asOverwrite = {
asn: 174,
name: 'Cogent Communications',
};
}
if (city && (asn || isp)) {
const query = `
UPDATE nodes SET
@@ -41,7 +65,7 @@ export async function $lookupNodeLocation(): Promise<void> {
`;
const params = [
isp?.autonomous_system_number ?? asn?.autonomous_system_number,
asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number,
city.city?.geoname_id,
city.country?.geoname_id,
city.subdivisions ? city.subdivisions[0].geoname_id : null,
@@ -50,48 +74,72 @@ export async function $lookupNodeLocation(): Promise<void> {
city.location?.accuracy_radius,
node.public_key
];
await DB.query(query, params);
let result = await DB.query<ResultSetHeader>(query, params);
if (result[0].changedRows ?? 0 > 0) {
++nodesUpdated;
}
// Store Continent
if (city.continent?.geoname_id) {
await DB.query(
result = await DB.query<ResultSetHeader>(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'continent', ?)`,
[city.continent?.geoname_id, JSON.stringify(city.continent?.names)]);
if (result[0].changedRows ?? 0 > 0) {
++geoNamesInserted;
}
}
// Store Country
if (city.country?.geoname_id) {
await DB.query(
result = await DB.query<ResultSetHeader>(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country', ?)`,
[city.country?.geoname_id, JSON.stringify(city.country?.names)]);
if (result[0].changedRows ?? 0 > 0) {
++geoNamesInserted;
}
}
// Store Country ISO code
if (city.country?.iso_code) {
await DB.query(
result = await DB.query<ResultSetHeader>(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`,
[city.country?.geoname_id, city.country?.iso_code]);
if (result[0].changedRows ?? 0 > 0) {
++geoNamesInserted;
}
}
// Store Division
if (city.subdivisions && city.subdivisions[0]) {
await DB.query(
result = await DB.query<ResultSetHeader>(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'division', ?)`,
[city.subdivisions[0].geoname_id, JSON.stringify(city.subdivisions[0]?.names)]);
if (result[0].changedRows ?? 0 > 0) {
++geoNamesInserted;
}
}
// Store City
if (city.city?.geoname_id) {
await DB.query(
result = await DB.query<ResultSetHeader>(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'city', ?)`,
[city.city?.geoname_id, JSON.stringify(city.city?.names)]);
if (result[0].changedRows ?? 0 > 0) {
++geoNamesInserted;
}
}
// Store AS name
if (isp?.autonomous_system_organization ?? asn?.autonomous_system_organization) {
await DB.query(
result = await DB.query<ResultSetHeader>(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'as_organization', ?)`,
[isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]);
[
asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number,
JSON.stringify(asOverwrite?.name ?? isp?.isp ?? asn?.autonomous_system_organization)
]);
if (result[0].changedRows ?? 0 > 0) {
++geoNamesInserted;
}
}
}
@@ -104,7 +152,12 @@ export async function $lookupNodeLocation(): Promise<void> {
}
}
}
logger.info(`${progress} nodes location data updated`);
if (nodesUpdated > 0) {
logger.info(`${nodesUpdated} nodes maxmind data updated, ${geoNamesInserted} geo names inserted`);
} else {
logger.debug(`${nodesUpdated} nodes maxmind data updated, ${geoNamesInserted} geo names inserted`);
}
} catch (e) {
logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e));
}

16
backend/src/tasks/lightning/sync-tasks/stats-importer.ts
View File

@@ -20,6 +20,10 @@ class LightningStatsImporter {
logger.info('Caching funding txs for currently existing channels');
await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
if (config.MEMPOOL.NETWORK !== 'mainnet' || config.DATABASE.ENABLED === false) {
return;
}
await this.$importHistoricalLightningStats();
await this.$cleanupIncorrectSnapshot();
}
@@ -53,6 +57,8 @@ class LightningStatsImporter {
features: node.features,
});
nodesInDb[node.pub_key] = node;
} else {
await nodesApi.$updateNodeSockets(node.pub_key, node.addresses);
}
let hasOnion = false;
@@ -354,9 +360,11 @@ class LightningStatsImporter {
fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8');
} catch (e: any) {
if (e.errno == -1) { // EISDIR - Ignore directorie
totalProcessed++;
continue;
}
logger.err(`Unable to open ${this.topologiesFolder}/${filename}`);
totalProcessed++;
continue;
}
@@ -365,7 +373,8 @@ class LightningStatsImporter {
graph = JSON.parse(fileContent);
graph = await this.cleanupTopology(graph);
} catch (e) {
logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`);
logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content. Reason: ${e instanceof Error ? e.message : e}`);
totalProcessed++;
continue;
}
@@ -415,9 +424,10 @@ class LightningStatsImporter {
const addressesParts = (node.addresses ?? '').split(',');
const addresses: any[] = [];
for (const address of addressesParts) {
const formatted = Common.findSocketNetwork(address);
addresses.push({
network: '',
addr: address
network: formatted.network,
addr: formatted.url
});
}

3
backend/src/tasks/price-updater.ts
View File

@@ -1,4 +1,5 @@
import * as fs from 'fs';
import path from "path";
import { Common } from '../api/common';
import config from '../config';
import logger from '../logger';
@@ -159,7 +160,7 @@ class PriceUpdater {
const existingPriceTimes = await PricesRepository.$getPricesTimes();
// Insert MtGox weekly prices
const pricesJson: any[] = JSON.parse(fs.readFileSync('./src/tasks/price-feeds/mtgox-weekly.json').toString());
const pricesJson: any[] = JSON.parse(fs.readFileSync(path.join(__dirname, 'mtgox-weekly.json')).toString());
const prices = this.getEmptyPricesObj();
let insertedCount: number = 0;
for (const price of pricesJson) {

119
backend/src/utils/ipcheck.js Normal file
View File

@@ -0,0 +1,119 @@
var net = require('net');
var IPCheck = module.exports = function(input) {
var self = this;
if (!(self instanceof IPCheck)) {
return new IPCheck(input);
}
self.input = input;
self.parse();
};
IPCheck.prototype.parse = function() {
var self = this;
if (!self.input || typeof self.input !== 'string') return self.valid = false;
var ip;
var pos = self.input.lastIndexOf('/');
if (pos !== -1) {
ip = self.input.substring(0, pos);
self.mask = +self.input.substring(pos + 1);
} else {
ip = self.input;
self.mask = null;
}
self.ipv = net.isIP(ip);
self.valid = !!self.ipv && !isNaN(self.mask);
if (!self.valid) return;
// default mask = 32 for ipv4 and 128 for ipv6
if (self.mask === null) self.mask = self.ipv === 4 ? 32 : 128;
if (self.ipv === 4) {
// difference between ipv4 and ipv6 masks
self.mask += 96;
}
if (self.mask < 0 || self.mask > 128) {
self.valid = false;
return;
}
self.address = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ];
if(self.ipv === 4){
self.parseIPv4(ip);
}else{
self.parseIPv6(ip);
}
};
IPCheck.prototype.parseIPv4 = function(ip) {
var self = this;
// ipv4 addresses live under ::ffff:0:0
self.address[10] = self.address[11] = 0xff;
var octets = ip.split('.');
for (var i = 0; i < 4; i++) {
self.address[i + 12] = parseInt(octets[i], 10);
}
};
var V6_TRANSITIONAL = /:(\d+\.\d+\.\d+\.\d+)$/;
IPCheck.prototype.parseIPv6 = function(ip) {
var self = this;
var transitionalMatch = V6_TRANSITIONAL.exec(ip);
if(transitionalMatch){
self.parseIPv4(transitionalMatch[1]);
return;
}
var bits = ip.split(':');
if (bits.length < 8) {
ip = ip.replace('::', Array(11 - bits.length).join(':'));
bits = ip.split(':');
}
var j = 0;
for (var i = 0; i < bits.length; i += 1) {
var x = bits[i] ? parseInt(bits[i], 16) : 0;
self.address[j++] = x >> 8;
self.address[j++] = x & 0xff;
}
};
IPCheck.prototype.match = function(cidr) {
var self = this;
if (!(cidr instanceof IPCheck)) cidr = new IPCheck(cidr);
if (!self.valid || !cidr.valid) return false;
var mask = cidr.mask;
var i = 0;
while (mask >= 8) {
if (self.address[i] !== cidr.address[i]) return false;
i++;
mask -= 8;
}
var shift = 8 - mask;
return (self.address[i] >>> shift) === (cidr.address[i] >>> shift);
};
IPCheck.match = function(ip, cidr) {
ip = ip instanceof IPCheck ? ip : new IPCheck(ip);
return ip.match(cidr);
};

174
backend/src/utils/pairing-heap.ts Normal file
View File

@@ -0,0 +1,174 @@
export type HeapNode<T> = {
element: T
child?: HeapNode<T>
next?: HeapNode<T>
prev?: HeapNode<T>
} | null | undefined;
// minimal pairing heap priority queue implementation
export class PairingHeap<T> {
private root: HeapNode<T> = null;
private comparator: (a: T, b: T) => boolean;
// comparator function should return 'true' if a is higher priority than b
constructor(comparator: (a: T, b: T) => boolean) {
this.comparator = comparator;
}
isEmpty(): boolean {
return !this.root;
}
add(element: T): HeapNode<T> {
const node: HeapNode<T> = {
element
};
this.root = this.meld(this.root, node);
return node;
}
// returns the top priority element without modifying the queue
peek(): T | void {
return this.root?.element;
}
// removes and returns the top priority element
pop(): T | void {
let element;
if (this.root) {
const node = this.root;
element = node.element;
this.root = this.mergePairs(node.child);
}
return element;
}
deleteNode(node: HeapNode<T>): void {
if (!node) {
return;
}
if (node === this.root) {
this.root = this.mergePairs(node.child);
}
else {
if (node.prev) {
if (node.prev.child === node) {
node.prev.child = node.next;
}
else {
node.prev.next = node.next;
}
}
if (node.next) {
node.next.prev = node.prev;
}
this.root = this.meld(this.root, this.mergePairs(node.child));
}
node.child = null;
node.prev = null;
node.next = null;
}
// fix the heap after increasing the priority of a given node
increasePriority(node: HeapNode<T>): void {
// already the top priority element
if (!node || node === this.root) {
return;
}
// extract from siblings
if (node.prev) {
if (node.prev?.child === node) {
if (this.comparator(node.prev.element, node.element)) {
// already in a valid position
return;
}
node.prev.child = node.next;
}
else {
node.prev.next = node.next;
}
}
if (node.next) {
node.next.prev = node.prev;
}
this.root = this.meld(this.root, node);
}
decreasePriority(node: HeapNode<T>): void {
this.deleteNode(node);
this.root = this.meld(this.root, node);
}
meld(a: HeapNode<T>, b: HeapNode<T>): HeapNode<T> {
if (!a) {
return b;
}
if (!b || a === b) {
return a;
}
let parent: HeapNode<T> = b;
let child: HeapNode<T> = a;
if (this.comparator(a.element, b.element)) {
parent = a;
child = b;
}
child.next = parent.child;
if (parent.child) {
parent.child.prev = child;
}
child.prev = parent;
parent.child = child;
parent.next = null;
parent.prev = null;
return parent;
}
mergePairs(node: HeapNode<T>): HeapNode<T> {
if (!node) {
return null;
}
let current: HeapNode<T> = node;
let next: HeapNode<T>;
let nextCurrent: HeapNode<T>;
let pairs: HeapNode<T>;
let melded: HeapNode<T>;
while (current) {
next = current.next;
if (next) {
nextCurrent = next.next;
melded = this.meld(current, next);
if (melded) {
melded.prev = pairs;
}
pairs = melded;
}
else {
nextCurrent = null;
current.prev = pairs;
pairs = current;
break;
}
current = nextCurrent;
}
melded = null;
let prev: HeapNode<T>;
while (pairs) {
prev = pairs.prev;
melded = this.meld(melded, pairs);
pairs = prev;
}
return melded;
}
}