machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into simon/eslint-triple-equals

Browse Source
This commit is contained in:
Felipe Knorr Kuhn
2022-09-01 22:51:07 -07:00
committed by GitHub
parent 2104570889 9047cb5998
commit 1aad4f2926
156 changed files with 2964 additions and 1049 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
backend/jest.config.ts
View File

@@ -7,11 +7,14 @@ const config: Config.InitialOptions = {
automock: false,
collectCoverage: true,
collectCoverageFrom: ["./src/**/**.ts"],
coverageProvider: "v8",
coverageProvider: "babel",
coverageThreshold: {
global: {
lines: 1
}
}
},
setupFiles: [
"./testSetup.ts",
],
}
export default config;

62
backend/src/__tests__/api/difficulty-adjustment.test.ts Normal file
View File

@@ -0,0 +1,62 @@
import { calcDifficultyAdjustment, DifficultyAdjustment } from '../../api/difficulty-adjustment';
describe('Mempool Difficulty Adjustment', () => {
test('should calculate Difficulty Adjustments properly', () => {
const dt = (dtString) => {
return Math.floor(new Date(dtString).getTime() / 1000);
};
const vectors = [
[ // Vector 1
[ // Inputs
dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds)
dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds)
750134, // Current block height
0.6280047707459726, // Previous retarget % (Passed through)
'mainnet', // Network (if testnet, next value is non-zero)
0, // If not testnet, not used
],
{ // Expected Result
progressPercent: 9.027777777777777,
difficultyChange: 12.562233927411782,
estimatedRetargetDate: 1661895424692,
remainingBlocks: 1834,
remainingTime: 977591692,
previousRetarget: 0.6280047707459726,
nextRetargetHeight: 751968,
timeAvg: 533038,
timeOffset: 0,
},
],
[ // Vector 2 (testnet)
[ // Inputs
dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds)
dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds)
750134, // Current block height
0.6280047707459726, // Previous retarget % (Passed through)
'testnet', // Network
dt('2022-08-19T13:52:46.000Z'), // Latest block timestamp in seconds
],
{ // Expected Result is same other than timeOffset
progressPercent: 9.027777777777777,
difficultyChange: 12.562233927411782,
estimatedRetargetDate: 1661895424692,
remainingBlocks: 1834,
remainingTime: 977591692,
previousRetarget: 0.6280047707459726,
nextRetargetHeight: 751968,
timeAvg: 533038,
timeOffset: -667000, // 11 min 7 seconds since last block (testnet only)
// If we add time avg to abs(timeOffset) it makes exactly 1200000 ms, or 20 minutes
},
],
] as [[number, number, number, number, string, number], DifficultyAdjustment][];
for (const vector of vectors) {
const result = calcDifficultyAdjustment(...vector[0]);
// previousRetarget is passed through untouched
expect(result.previousRetarget).toStrictEqual(vector[0][3]);
expect(result).toStrictEqual(vector[1]);
}
});
});

1
backend/src/__tests__/config.test.ts
View File

@@ -136,5 +136,4 @@ describe('Mempool Backend Config', () => {
expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER);
});
});
});

7
backend/src/api/bitcoin/bitcoin.routes.ts
View File

@@ -510,7 +510,12 @@ class BitcoinRoutes {
private getDifficultyChange(req: Request, res: Response) {
try {
res.json(difficultyAdjustment.getDifficultyAdjustment());
const da = difficultyAdjustment.getDifficultyAdjustment();
if (da) {
res.json(da);
} else {
res.status(503).send(`Service Temporarily Unavailable`);
}
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}

81
backend/src/api/common.ts
View File

@@ -228,34 +228,75 @@ export class Common {
return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
}
static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket {
static findSocketNetwork(addr: string): {network: string | null, url: string} {
let network: string | null = null;
let url = addr.split('://')[1];
if (config.LIGHTNING.BACKEND === 'cln') {
network = socket.network;
} else if (config.LIGHTNING.BACKEND === 'lnd') {
if (socket.addr.indexOf('onion') !== -1) {
if (socket.addr.split('.')[0].length >= 56) {
network = 'torv3';
} else {
network = 'torv2';
}
} else if (socket.addr.indexOf('i2p') !== -1) {
network = 'i2p';
if (!url) {
return {
network: null,
url: addr,
};
}
if (addr.indexOf('onion') !== -1) {
if (url.split('.')[0].length >= 56) {
network = 'torv3';
} else {
const ipv = isIP(socket.addr.split(':')[0]);
if (ipv === 4) {
network = 'ipv4';
} else if (ipv === 6) {
network = 'ipv6';
}
network = 'torv2';
}
} else if (addr.indexOf('i2p') !== -1) {
network = 'i2p';
} else if (addr.indexOf('ipv4') !== -1) {
const ipv = isIP(url.split(':')[0]);
if (ipv === 4) {
network = 'ipv4';
} else {
return {
network: null,
url: addr,
};
}
} else if (addr.indexOf('ipv6') !== -1) {
url = url.split('[')[1].split(']')[0];
const ipv = isIP(url);
if (ipv === 6) {
const parts = addr.split(':');
network = 'ipv6';
url = `[${url}]:${parts[parts.length - 1]}`;
} else {
return {
network: null,
url: addr,
};
}
} else {
return {
network: null,
url: addr,
};
}
return {
publicKey: publicKey,
network: network,
addr: socket.addr,
url: url,
};
}
static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket {
if (config.LIGHTNING.BACKEND === 'cln') {
return {
publicKey: publicKey,
network: socket.network,
addr: socket.addr,
};
} else /* if (config.LIGHTNING.BACKEND === 'lnd') */ {
const formatted = this.findSocketNetwork(socket.addr);
return {
publicKey: publicKey,
network: formatted.network,
addr: formatted.url,
};
}
}
}

8
backend/src/api/database-migration.ts
View File

@@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 39;
private static currentVersion = 40;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -342,6 +342,12 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `nodes` ADD alias_search TEXT NULL DEFAULT NULL AFTER `alias`');
await this.$executeQuery('ALTER TABLE nodes ADD FULLTEXT(alias_search)');
}
if (databaseSchemaVersion < 40 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `nodes` ADD capacity bigint(20) unsigned DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD channels int(11) unsigned DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD INDEX `capacity` (`capacity`);');
}
}
/**

137
backend/src/api/difficulty-adjustment.ts
View File

@@ -2,65 +2,100 @@ import config from '../config';
import { IDifficultyAdjustment } from '../mempool.interfaces';
import blocks from './blocks';
class DifficultyAdjustmentApi {
constructor() { }
export interface DifficultyAdjustment {
progressPercent: number; // Percent: 0 to 100
difficultyChange: number; // Percent: -75 to 300
estimatedRetargetDate: number; // Unix time in ms
remainingBlocks: number; // Block count
remainingTime: number; // Duration of time in ms
previousRetarget: number; // Percent: -75 to 300
nextRetargetHeight: number; // Block Height
timeAvg: number; // Duration of time in ms
timeOffset: number; // (Testnet) Time since last block (cap @ 20min) in ms
}
public getDifficultyAdjustment(): IDifficultyAdjustment {
export function calcDifficultyAdjustment(
DATime: number,
nowSeconds: number,
blockHeight: number,
previousRetarget: number,
network: string,
latestBlockTimestamp: number,
): DifficultyAdjustment {
const ESTIMATE_LAG_BLOCKS = 146; // For first 7.2% of epoch, don't estimate.
const EPOCH_BLOCK_LENGTH = 2016; // Bitcoin mainnet
const BLOCK_SECONDS_TARGET = 600; // Bitcoin mainnet
const TESTNET_MAX_BLOCK_SECONDS = 1200; // Bitcoin testnet
const diffSeconds = nowSeconds - DATime;
const blocksInEpoch = (blockHeight >= 0) ? blockHeight % EPOCH_BLOCK_LENGTH : 0;
const progressPercent = (blockHeight >= 0) ? blocksInEpoch / EPOCH_BLOCK_LENGTH * 100 : 100;
const remainingBlocks = EPOCH_BLOCK_LENGTH - blocksInEpoch;
const nextRetargetHeight = (blockHeight >= 0) ? blockHeight + remainingBlocks : 0;
let difficultyChange = 0;
let timeAvgSecs = BLOCK_SECONDS_TARGET;
// Only calculate the estimate once we have 7.2% of blocks in current epoch
if (blocksInEpoch >= ESTIMATE_LAG_BLOCKS) {
timeAvgSecs = diffSeconds / blocksInEpoch;
difficultyChange = (BLOCK_SECONDS_TARGET / timeAvgSecs - 1) * 100;
// Max increase is x4 (+300%)
if (difficultyChange > 300) {
difficultyChange = 300;
}
// Max decrease is /4 (-75%)
if (difficultyChange < -75) {
difficultyChange = -75;
}
}
// Testnet difficulty is set to 1 after 20 minutes of no blocks,
// therefore the time between blocks will always be below 20 minutes (1200s).
let timeOffset = 0;
if (network === 'testnet') {
if (timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) {
timeAvgSecs = TESTNET_MAX_BLOCK_SECONDS;
}
const secondsSinceLastBlock = nowSeconds - latestBlockTimestamp;
if (secondsSinceLastBlock + timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) {
timeOffset = -Math.min(secondsSinceLastBlock, TESTNET_MAX_BLOCK_SECONDS) * 1000;
}
}
const timeAvg = Math.floor(timeAvgSecs * 1000);
const remainingTime = remainingBlocks * timeAvg;
const estimatedRetargetDate = remainingTime + nowSeconds * 1000;
return {
progressPercent,
difficultyChange,
estimatedRetargetDate,
remainingBlocks,
remainingTime,
previousRetarget,
nextRetargetHeight,
timeAvg,
timeOffset,
};
}
class DifficultyAdjustmentApi {
public getDifficultyAdjustment(): IDifficultyAdjustment | null {
const DATime = blocks.getLastDifficultyAdjustmentTime();
const previousRetarget = blocks.getPreviousDifficultyRetarget();
const blockHeight = blocks.getCurrentBlockHeight();
const blocksCache = blocks.getBlocks();
const latestBlock = blocksCache[blocksCache.length - 1];
const now = new Date().getTime() / 1000;
const diff = now - DATime;
const blocksInEpoch = blockHeight % 2016;
const progressPercent = (blocksInEpoch >= 0) ? blocksInEpoch / 2016 * 100 : 100;
const remainingBlocks = 2016 - blocksInEpoch;
const nextRetargetHeight = blockHeight + remainingBlocks;
let difficultyChange = 0;
if (remainingBlocks < 1870) {
if (blocksInEpoch > 0) {
difficultyChange = (600 / (diff / blocksInEpoch) - 1) * 100;
}
if (difficultyChange > 300) {
difficultyChange = 300;
}
if (difficultyChange < -75) {
difficultyChange = -75;
}
if (!latestBlock) {
return null;
}
const nowSeconds = Math.floor(new Date().getTime() / 1000);
let timeAvgMins = blocksInEpoch && blocksInEpoch > 146 ? diff / blocksInEpoch / 60 : 10;
// Testnet difficulty is set to 1 after 20 minutes of no blocks,
// therefore the time between blocks will always be below 20 minutes (1200s).
let timeOffset = 0;
if (config.MEMPOOL.NETWORK === 'testnet') {
if (timeAvgMins > 20) {
timeAvgMins = 20;
}
if (now - latestBlock.timestamp + timeAvgMins * 60 > 1200) {
timeOffset = -Math.min(now - latestBlock.timestamp, 1200) * 1000;
}
}
const timeAvg = timeAvgMins * 60 * 1000 ;
const remainingTime = (remainingBlocks * timeAvg) + (now * 1000);
const estimatedRetargetDate = remainingTime + now;
return {
progressPercent,
difficultyChange,
estimatedRetargetDate,
remainingBlocks,
remainingTime,
previousRetarget,
nextRetargetHeight,
timeAvg,
timeOffset,
};
return calcDifficultyAdjustment(
DATime, nowSeconds, blockHeight, previousRetarget,
config.MEMPOOL.NETWORK, latestBlock.timestamp
);
}
}

103
backend/src/api/explorer/channels.api.ts
View File

@@ -80,7 +80,7 @@ class ChannelsApi {
public async $searchChannelsById(search: string): Promise<any[]> {
try {
const searchStripped = search.replace('%', '') + '%';
const query = `SELECT id, short_id, capacity FROM channels WHERE id LIKE ? OR short_id LIKE ? LIMIT 10`;
const query = `SELECT id, short_id, capacity, status FROM channels WHERE id LIKE ? OR short_id LIKE ? LIMIT 10`;
const [rows]: any = await DB.query(query, [searchStripped, searchStripped]);
return rows;
} catch (e) {
@@ -229,9 +229,14 @@ class ChannelsApi {
public async $getChannelsByTransactionId(transactionIds: string[]): Promise<any[]> {
try {
transactionIds = transactionIds.map((id) => '\'' + id + '\'');
const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.* FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key WHERE channels.transaction_id IN (${transactionIds.join(', ')}) OR channels.closing_transaction_id IN (${transactionIds.join(', ')})`;
const [rows]: any = await DB.query(query);
const query = `
SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*
FROM channels
LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
WHERE channels.transaction_id IN ? OR channels.closing_transaction_id IN ?
`;
const [rows]: any = await DB.query(query, [[transactionIds], [transactionIds]]);
const channels = rows.map((row) => this.convertChannel(row));
return channels;
} catch (e) {
@@ -245,15 +250,20 @@ class ChannelsApi {
let channelStatusFilter;
if (status === 'open') {
channelStatusFilter = '< 2';
} else if (status === 'active') {
channelStatusFilter = '= 1';
} else if (status === 'closed') {
channelStatusFilter = '= 2';
} else {
throw new Error('getChannelsForNode: Invalid status requested');
}
// Channels originating from node
let query = `
SELECT COALESCE(node2.alias, SUBSTRING(node2_public_key, 0, 20)) AS alias, COALESCE(node2.public_key, node2_public_key) AS public_key,
channels.status, channels.node1_fee_rate,
channels.capacity, channels.short_id, channels.id, channels.closing_reason
channels.capacity, channels.short_id, channels.id, channels.closing_reason,
UNIX_TIMESTAMP(closing_date) as closing_date, UNIX_TIMESTAMP(channels.updated_at) as updated_at
FROM channels
LEFT JOIN nodes AS node2 ON node2.public_key = channels.node2_public_key
WHERE node1_public_key = ? AND channels.status ${channelStatusFilter}
@@ -264,7 +274,8 @@ class ChannelsApi {
query = `
SELECT COALESCE(node1.alias, SUBSTRING(node1_public_key, 0, 20)) AS alias, COALESCE(node1.public_key, node1_public_key) AS public_key,
channels.status, channels.node2_fee_rate,
channels.capacity, channels.short_id, channels.id, channels.closing_reason
channels.capacity, channels.short_id, channels.id, channels.closing_reason,
UNIX_TIMESTAMP(closing_date) as closing_date, UNIX_TIMESTAMP(channels.updated_at) as updated_at
FROM channels
LEFT JOIN nodes AS node1 ON node1.public_key = channels.node1_public_key
WHERE node2_public_key = ? AND channels.status ${channelStatusFilter}
@@ -273,27 +284,56 @@ class ChannelsApi {
let allChannels = channelsFromNode.concat(channelsToNode);
allChannels.sort((a, b) => {
return b.capacity - a.capacity;
if (status === 'closed') {
if (!b.closing_date && !a.closing_date) {
return (b.updated_at ?? 0) - (a.updated_at ?? 0);
} else {
return (b.closing_date ?? 0) - (a.closing_date ?? 0);
}
} else {
return b.capacity - a.capacity;
}
});
allChannels = allChannels.slice(index, index + length);
if (index >= 0) {
allChannels = allChannels.slice(index, index + length);
} else if (index === -1) { // Node channels tree chart
allChannels = allChannels.slice(0, 1000);
}
const channels: any[] = []
for (const row of allChannels) {
const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key);
channels.push({
status: row.status,
closing_reason: row.closing_reason,
capacity: row.capacity ?? 0,
short_id: row.short_id,
id: row.id,
fee_rate: row.node1_fee_rate ?? row.node2_fee_rate ?? 0,
node: {
alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20),
public_key: row.public_key,
channels: activeChannelsStats.active_channel_count ?? 0,
capacity: activeChannelsStats.capacity ?? 0,
}
});
let channel;
if (index >= 0) {
const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key);
channel = {
status: row.status,
closing_reason: row.closing_reason,
closing_date: row.closing_date,
capacity: row.capacity ?? 0,
short_id: row.short_id,
id: row.id,
fee_rate: row.node1_fee_rate ?? row.node2_fee_rate ?? 0,
node: {
alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20),
public_key: row.public_key,
channels: activeChannelsStats.active_channel_count ?? 0,
capacity: activeChannelsStats.capacity ?? 0,
}
};
} else if (index === -1) {
channel = {
capacity: row.capacity ?? 0,
short_id: row.short_id,
id: row.id,
node: {
alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20),
public_key: row.public_key,
}
};
}
channels.push(channel);
}
return channels;
@@ -498,6 +538,23 @@ class ChannelsApi {
logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
}
}
public async $getLatestChannelUpdateForNode(publicKey: string): Promise<number> {
try {
const query = `
SELECT MAX(UNIX_TIMESTAMP(updated_at)) as updated_at
FROM channels
WHERE node1_public_key = ?
`;
const [rows]: any[] = await DB.query(query, [publicKey]);
if (rows.length > 0) {
return rows[0].updated_at;
}
} catch (e) {
logger.err(`Can't getLatestChannelUpdateForNode for ${publicKey}. Reason ${e instanceof Error ? e.message : e}`);
}
return 0;
}
}
export default new ChannelsApi();

42
backend/src/api/explorer/channels.routes.ts
View File

@@ -47,8 +47,17 @@ class ChannelsRoutes {
res.status(400).send('Missing parameter: public_key');
return;
}
const index = parseInt(typeof req.query.index === 'string' ? req.query.index : '0', 10) || 0;
const status: string = typeof req.query.status === 'string' ? req.query.status : '';
if (index < -1) {
res.status(400).send('Invalid index');
}
if (['open', 'active', 'closed'].includes(status) === false) {
res.status(400).send('Invalid status');
}
const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, 10, status);
const channelsCount = await channelsApi.$getChannelsCountForNode(req.query.public_key, status);
res.header('Pragma', 'public');
@@ -61,7 +70,7 @@ class ChannelsRoutes {
}
}
private async $getChannelsByTransactionIds(req: Request, res: Response) {
private async $getChannelsByTransactionIds(req: Request, res: Response): Promise<void> {
try {
if (!Array.isArray(req.query.txId)) {
res.status(400).send('Not an array');
@@ -74,27 +83,26 @@ class ChannelsRoutes {
}
}
const channels = await channelsApi.$getChannelsByTransactionId(txIds);
const inputs: any[] = [];
const outputs: any[] = [];
const result: any[] = [];
for (const txid of txIds) {
const foundChannelInputs = channels.find((channel) => channel.closing_transaction_id === txid);
if (foundChannelInputs) {
inputs.push(foundChannelInputs);
} else {
inputs.push(null);
const inputs: any = {};
const outputs: any = {};
// Assuming that we only have one lightning close input in each transaction. This may not be true in the future
const foundChannelsFromInput = channels.find((channel) => channel.closing_transaction_id === txid);
if (foundChannelsFromInput) {
inputs[0] = foundChannelsFromInput;
}
const foundChannelOutputs = channels.find((channel) => channel.transaction_id === txid);
if (foundChannelOutputs) {
outputs.push(foundChannelOutputs);
} else {
outputs.push(null);
const foundChannelsFromOutputs = channels.filter((channel) => channel.transaction_id === txid);
for (const output of foundChannelsFromOutputs) {
outputs[output.transaction_vout] = output;
}
result.push({
inputs,
outputs,
});
}
res.json({
inputs: inputs,
outputs: outputs,
});
res.json(result);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}

81
backend/src/api/explorer/nodes.api.ts
View File

@@ -115,17 +115,13 @@ class NodesApi {
public async $getTopCapacityNodes(full: boolean): Promise<ITopNodesPerCapacity[]> {
try {
let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats');
const latestDate = rows[0].maxAdded;
let rows: any;
let query: string;
if (full === false) {
query = `
SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
node_stats.capacity
FROM node_stats
JOIN nodes ON nodes.public_key = node_stats.public_key
WHERE added = FROM_UNIXTIME(${latestDate})
nodes.capacity
FROM nodes
ORDER BY capacity DESC
LIMIT 100
`;
@@ -133,16 +129,14 @@ class NodesApi {
[rows] = await DB.query(query);
} else {
query = `
SELECT node_stats.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(node_stats.public_key, 1, 20), alias) as alias,
CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity,
CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
CAST(COALESCE(nodes.channels, 0) as INT) as channels,
UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
geo_names_city.names as city, geo_names_country.names as country
FROM node_stats
RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
FROM nodes
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
WHERE added = FROM_UNIXTIME(${latestDate})
ORDER BY capacity DESC
LIMIT 100
`;
@@ -163,17 +157,13 @@ class NodesApi {
public async $getTopChannelsNodes(full: boolean): Promise<ITopNodesPerChannels[]> {
try {
let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats');
const latestDate = rows[0].maxAdded;
let rows: any;
let query: string;
if (full === false) {
query = `
SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
node_stats.channels
FROM node_stats
JOIN nodes ON nodes.public_key = node_stats.public_key
WHERE added = FROM_UNIXTIME(${latestDate})
nodes.channels
FROM nodes
ORDER BY channels DESC
LIMIT 100;
`;
@@ -181,16 +171,14 @@ class NodesApi {
[rows] = await DB.query(query);
} else {
query = `
SELECT node_stats.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(node_stats.public_key, 1, 20), alias) as alias,
CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity,
SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
CAST(COALESCE(nodes.channels, 0) as INT) as channels,
CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
geo_names_city.names as city, geo_names_country.names as country
FROM node_stats
RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
FROM nodes
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
WHERE added = FROM_UNIXTIME(${latestDate})
ORDER BY channels DESC
LIMIT 100
`;
@@ -260,8 +248,8 @@ class NodesApi {
public async $searchNodeByPublicKeyOrAlias(search: string) {
try {
const publicKeySearch = search.replace('%', '') + '%';
const aliasSearch = search.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z ]/g, '').split(' ').map((search) => '+' + search + '*').join(' ');
const query = `SELECT nodes.public_key, nodes.alias, node_stats.capacity FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key WHERE nodes.public_key LIKE ? OR MATCH nodes.alias_search AGAINST (? IN BOOLEAN MODE) GROUP BY nodes.public_key ORDER BY node_stats.capacity DESC LIMIT 10`;
const aliasSearch = search.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '').split(' ').map((search) => '+' + search + '*').join(' ');
const query = `SELECT public_key, alias, capacity, channels, status FROM nodes WHERE public_key LIKE ? OR MATCH alias_search AGAINST (? IN BOOLEAN MODE) ORDER BY capacity DESC LIMIT 10`;
const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]);
return rows;
} catch (e) {
@@ -276,7 +264,7 @@ class NodesApi {
// List all channels and the two linked ISP
query = `
SELECT short_id, capacity,
SELECT short_id, channels.capacity,
channels.node1_public_key AS node1PublicKey, isp1.names AS isp1, isp1.id as isp1ID,
channels.node2_public_key AS node2PublicKey, isp2.names AS isp2, isp2.id as isp2ID
FROM channels
@@ -391,17 +379,11 @@ class NodesApi {
public async $getNodesPerCountry(countryId: string) {
try {
const query = `
SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
geo_names_city.names as city, geo_names_country.names as country,
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
FROM node_stats
JOIN (
SELECT public_key, MAX(added) as last_added
FROM node_stats
GROUP BY public_key
) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added
RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
FROM nodes
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
@@ -426,17 +408,11 @@ class NodesApi {
public async $getNodesPerISP(ISPId: string) {
try {
const query = `
SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
geo_names_city.names as city, geo_names_country.names as country,
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
FROM node_stats
JOIN (
SELECT public_key, MAX(added) as last_added
FROM node_stats
GROUP BY public_key
) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added
RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
FROM nodes
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
@@ -464,7 +440,6 @@ class NodesApi {
FROM nodes
JOIN geo_names ON geo_names.id = nodes.country_id AND geo_names.type = 'country'
JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key
GROUP BY country_id
ORDER BY COUNT(DISTINCT nodes.public_key) DESC
`;
@@ -528,6 +503,18 @@ class NodesApi {
}
}
/**
* Update node sockets
*/
public async $updateNodeSockets(publicKey: string, sockets: {network: string; addr: string}[]): Promise<void> {
const formattedSockets = (sockets.map(a => a.addr).join(',')) ?? '';
try {
await DB.query(`UPDATE nodes SET sockets = ? WHERE public_key = ?`, [formattedSockets, publicKey]);
} catch (e) {
logger.err(`Cannot update node sockets for ${publicKey}. Reason: ${e instanceof Error ? e.message : e}`);
}
}
/**
* Set all nodes not in `nodesPubkeys` as inactive (status = 0)
*/
@@ -555,7 +542,7 @@ class NodesApi {
}
private aliasToSearchText(str: string): string {
return str.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z ]/g, '');
return str.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '');
}
}

2
backend/src/api/explorer/statistics.api.ts
View File

@@ -27,7 +27,7 @@ class StatisticsApi {
public async $getLatestStatistics(): Promise<any> {
try {
const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1`);
const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1 OFFSET 7`);
const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats WHERE DATE(added) = DATE(NOW() - INTERVAL 7 DAY)`);
return {
latest: rows[0],
previous: rows2[0],

6
backend/src/api/lightning/clightning/clightning-convert.ts
View File

@@ -13,9 +13,13 @@ export function convertNode(clNode: any): ILightningApi.Node {
features: [], // TODO parse and return clNode.feature
pub_key: clNode.nodeid,
addresses: clNode.addresses?.map((addr) => {
let address = addr.address;
if (addr.type === 'ipv6') {
address = `[${address}]`;
}
return {
network: addr.type,
addr: `${addr.address}:${addr.port}`
addr: `${address}:${addr.port}`
};
}) ?? [],
last_update: clNode?.last_timestamp ?? 0,

2
backend/src/logger.ts
View File

@@ -74,7 +74,7 @@ class Logger {
private getNetwork(): string {
if (config.LIGHTNING.ENABLED) {
return 'lightning';
return config.MEMPOOL.NETWORK === 'mainnet' ? 'lightning' : `${config.MEMPOOL.NETWORK}-lightning`;
}
if (config.BISQ.ENABLED) {
return 'bisq';

95
backend/src/tasks/lightning/network-sync.service.ts
View File

@@ -63,6 +63,9 @@ class NetworkSyncService {
let deletedSockets = 0;
const graphNodesPubkeys: string[] = [];
for (const node of nodes) {
const latestUpdated = await channelsApi.$getLatestChannelUpdateForNode(node.pub_key);
node.last_update = Math.max(node.last_update, latestUpdated);
await nodesApi.$saveNode(node);
graphNodesPubkeys.push(node.pub_key);
++progress;
@@ -83,7 +86,7 @@ class NetworkSyncService {
logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted`);
// If a channel if not present in the graph, mark it as inactive
nodesApi.$setNodesInactive(graphNodesPubkeys);
await nodesApi.$setNodesInactive(graphNodesPubkeys);
if (config.MAXMIND.ENABLED) {
$lookupNodeLocation();
@@ -98,7 +101,7 @@ class NetworkSyncService {
const [closedChannelsRaw]: any[] = await DB.query(`SELECT id FROM channels WHERE status = 2`);
const closedChannels = {};
for (const closedChannel of closedChannelsRaw) {
closedChannels[Common.channelShortIdToIntegerId(closedChannel.id)] = true;
closedChannels[closedChannel.id] = true;
}
let progress = 0;
@@ -121,7 +124,7 @@ class NetworkSyncService {
logger.info(`${progress} channels updated`);
// If a channel if not present in the graph, mark it as inactive
channelsApi.$setChannelsInactive(graphChannelsIds);
await channelsApi.$setChannelsInactive(graphChannelsIds);
} catch (e) {
logger.err(`Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`);
}
@@ -285,44 +288,66 @@ class NetworkSyncService {
for (const channel of channels) {
let reason = 0;
// Only Esplora backend can retrieve spent transaction outputs
const outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id);
const lightningScriptReasons: number[] = [];
for (const outspend of outspends) {
if (outspend.spent && outspend.txid) {
const spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid);
const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
lightningScriptReasons.push(lightningScript);
try {
let outspends: IEsploraApi.Outspend[] | undefined;
try {
outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id);
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
continue;
}
}
if (lightningScriptReasons.length === outspends.length
&& lightningScriptReasons.filter((r) => r === 1).length === outspends.length) {
reason = 1;
} else {
const filteredReasons = lightningScriptReasons.filter((r) => r !== 1);
if (filteredReasons.length) {
if (filteredReasons.some((r) => r === 2 || r === 4)) {
reason = 3;
} else {
reason = 2;
const lightningScriptReasons: number[] = [];
for (const outspend of outspends) {
if (outspend.spent && outspend.txid) {
let spendingTx: IEsploraApi.Transaction | undefined;
try {
spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid);
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`);
continue;
}
const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
lightningScriptReasons.push(lightningScript);
}
}
if (lightningScriptReasons.length === outspends.length
&& lightningScriptReasons.filter((r) => r === 1).length === outspends.length) {
reason = 1;
} else {
/*
We can detect a commitment transaction (force close) by reading Sequence and Locktime
https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction
*/
const closingTx = await bitcoinApi.$getRawTransaction(channel.closing_transaction_id);
const sequenceHex: string = closingTx.vin[0].sequence.toString(16);
const locktimeHex: string = closingTx.locktime.toString(16);
if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') {
reason = 2; // Here we can't be sure if it's a penalty or not
const filteredReasons = lightningScriptReasons.filter((r) => r !== 1);
if (filteredReasons.length) {
if (filteredReasons.some((r) => r === 2 || r === 4)) {
reason = 3;
} else {
reason = 2;
}
} else {
reason = 1;
/*
We can detect a commitment transaction (force close) by reading Sequence and Locktime
https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction
*/
let closingTx: IEsploraApi.Transaction | undefined;
try {
closingTx = await bitcoinApi.$getRawTransaction(channel.closing_transaction_id);
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id}. Reason ${e instanceof Error ? e.message : e}`);
continue;
}
const sequenceHex: string = closingTx.vin[0].sequence.toString(16);
const locktimeHex: string = closingTx.locktime.toString(16);
if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') {
reason = 2; // Here we can't be sure if it's a penalty or not
} else {
reason = 1;
}
}
}
}
if (reason) {
logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
if (reason) {
logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
}
} catch (e) {
logger.err(`$runClosedChannelsForensics() failed for channel ${channel.short_id}. Reason: ${e instanceof Error ? e.message : e}`);
}
++progress;

28
backend/src/tasks/lightning/sync-tasks/node-locations.ts
View File

@@ -4,6 +4,7 @@ import nodesApi from '../../../api/explorer/nodes.api';
import config from '../../../config';
import DB from '../../../database';
import logger from '../../../logger';
import * as IPCheck from '../../../utils/ipcheck.js';
export async function $lookupNodeLocation(): Promise<void> {
let loggerTimer = new Date().getTime() / 1000;
@@ -27,6 +28,26 @@ export async function $lookupNodeLocation(): Promise<void> {
const asn = lookupAsn.get(ip);
const isp = lookupIsp.get(ip);
let asOverwrite: any | undefined;
if (asn && (IPCheck.match(ip, '170.75.160.0/20') || IPCheck.match(ip, '172.81.176.0/21'))) {
asOverwrite = {
asn: 394745,
name: 'Lunanode',
};
}
else if (asn && (IPCheck.match(ip, '50.7.0.0/16') || IPCheck.match(ip, '66.90.64.0/18'))) {
asOverwrite = {
asn: 30058,
name: 'FDCservers.net',
};
}
else if (asn && asn.autonomous_system_number === 174) {
asOverwrite = {
asn: 174,
name: 'Cogent Communications',
};
}
if (city && (asn || isp)) {
const query = `
UPDATE nodes SET
@@ -41,7 +62,7 @@ export async function $lookupNodeLocation(): Promise<void> {
`;
const params = [
isp?.autonomous_system_number ?? asn?.autonomous_system_number,
asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number,
city.city?.geoname_id,
city.country?.geoname_id,
city.subdivisions ? city.subdivisions[0].geoname_id : null,
@@ -91,7 +112,10 @@ export async function $lookupNodeLocation(): Promise<void> {
if (isp?.autonomous_system_organization ?? asn?.autonomous_system_organization) {
await DB.query(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'as_organization', ?)`,
[isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]);
[
asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number,
JSON.stringify(asOverwrite?.name ?? isp?.isp ?? asn?.autonomous_system_organization)
]);
}
}

166
backend/src/tasks/lightning/sync-tasks/stats-importer.ts
View File

@@ -7,6 +7,8 @@ import { ILightningApi } from '../../../api/lightning/lightning-api.interface';
import { isIP } from 'net';
import { Common } from '../../../api/common';
import channelsApi from '../../../api/explorer/channels.api';
import nodesApi from '../../../api/explorer/nodes.api';
import { ResultSetHeader } from 'mysql2';
const fsPromises = promises;
@@ -18,7 +20,12 @@ class LightningStatsImporter {
logger.info('Caching funding txs for currently existing channels');
await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
if (config.MEMPOOL.NETWORK !== 'mainnet' || config.DATABASE.ENABLED === false) {
return;
}
await this.$importHistoricalLightningStats();
await this.$cleanupIncorrectSnapshot();
}
/**
@@ -32,7 +39,28 @@ class LightningStatsImporter {
let clearnetTorNodes = 0;
let unannouncedNodes = 0;
const [nodesInDbRaw]: any[] = await DB.query(`SELECT public_key FROM nodes`);
const nodesInDb = {};
for (const node of nodesInDbRaw) {
nodesInDb[node.public_key] = node;
}
for (const node of networkGraph.nodes) {
// If we don't know about this node, insert it in db
if (isHistorical === true && !nodesInDb[node.pub_key]) {
await nodesApi.$saveNode({
last_update: node.last_update,
pub_key: node.pub_key,
alias: node.alias,
addresses: node.addresses,
color: node.color,
features: node.features,
});
nodesInDb[node.pub_key] = node;
} else {
await nodesApi.$updateNodeSockets(node.pub_key, node.addresses);
}
let hasOnion = false;
let hasClearnet = false;
let isUnnanounced = true;
@@ -69,7 +97,7 @@ class LightningStatsImporter {
const baseFees: number[] = [];
const alreadyCountedChannels = {};
const [channelsInDbRaw]: any[] = await DB.query(`SELECT short_id, created FROM channels`);
const [channelsInDbRaw]: any[] = await DB.query(`SELECT short_id FROM channels`);
const channelsInDb = {};
for (const channel of channelsInDbRaw) {
channelsInDb[channel.short_id] = channel;
@@ -84,29 +112,19 @@ class LightningStatsImporter {
continue;
}
// Channel is already in db, check if we need to update 'created' field
if (isHistorical === true) {
//@ts-ignore
if (channelsInDb[short_id] && channel.timestamp < channel.created) {
await DB.query(`
UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.short_id = ?`,
//@ts-ignore
[channel.timestamp, short_id]
);
} else if (!channelsInDb[short_id]) {
await channelsApi.$saveChannel({
channel_id: short_id,
chan_point: `${tx.txid}:${short_id.split('x')[2]}`,
//@ts-ignore
last_update: channel.timestamp,
node1_pub: channel.node1_pub,
node2_pub: channel.node2_pub,
capacity: (tx.value * 100000000).toString(),
node1_policy: null,
node2_policy: null,
}, 0);
channelsInDb[channel.channel_id] = channel;
}
// If we don't know about this channel, insert it in db
if (isHistorical === true && !channelsInDb[short_id]) {
await channelsApi.$saveChannel({
channel_id: short_id,
chan_point: `${tx.txid}:${short_id.split('x')[2]}`,
last_update: channel.last_update,
node1_pub: channel.node1_pub,
node2_pub: channel.node2_pub,
capacity: (tx.value * 100000000).toString(),
node1_policy: null,
node2_policy: null,
}, 0);
channelsInDb[channel.channel_id] = channel;
}
if (!nodeStats[channel.node1_pub]) {
@@ -269,6 +287,17 @@ class LightningStatsImporter {
nodeStats[public_key].capacity,
nodeStats[public_key].channels,
]);
if (!isHistorical) {
await DB.query(
`UPDATE nodes SET capacity = ?, channels = ? WHERE public_key = ?`,
[
nodeStats[public_key].capacity,
nodeStats[public_key].channels,
public_key,
]
);
}
}
return {
@@ -281,6 +310,7 @@ class LightningStatsImporter {
* Import topology files LN historical data into the database
*/
async $importHistoricalLightningStats(): Promise<void> {
logger.debug('Run the historical importer');
try {
let fileList: string[] = [];
try {
@@ -294,7 +324,7 @@ class LightningStatsImporter {
fileList.sort().reverse();
const [rows]: any[] = await DB.query(`
SELECT UNIX_TIMESTAMP(added) AS added, node_count
SELECT UNIX_TIMESTAMP(added) AS added
FROM lightning_stats
ORDER BY added DESC
`);
@@ -341,10 +371,16 @@ class LightningStatsImporter {
graph = JSON.parse(fileContent);
graph = await this.cleanupTopology(graph);
} catch (e) {
logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`);
logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content. Reason: ${e instanceof Error ? e.message : e}`);
continue;
}
if (this.isIncorrectSnapshot(timestamp, graph)) {
logger.debug(`Ignoring ${this.topologiesFolder}/${filename}, because we defined it as an incorrect snapshot`);
++totalProcessed;
continue;
}
if (!logStarted) {
logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`);
logStarted = true;
@@ -375,7 +411,7 @@ class LightningStatsImporter {
}
}
async cleanupTopology(graph) {
cleanupTopology(graph): ILightningApi.NetworkGraph {
const newGraph = {
nodes: <ILightningApi.Node[]>[],
edges: <ILightningApi.Channel[]>[],
@@ -385,18 +421,23 @@ class LightningStatsImporter {
const addressesParts = (node.addresses ?? '').split(',');
const addresses: any[] = [];
for (const address of addressesParts) {
const formatted = Common.findSocketNetwork(address);
addresses.push({
network: '',
addr: address
network: formatted.network,
addr: formatted.url
});
}
let rgb = node.rgb_color ?? '#000000';
if (rgb.indexOf('#') === -1) {
rgb = `#${rgb}`;
}
newGraph.nodes.push({
last_update: node.timestamp ?? 0,
pub_key: node.id ?? null,
alias: node.alias ?? null,
alias: node.alias ?? node.id.slice(0, 20),
addresses: addresses,
color: node.rgb_color ?? null,
color: rgb,
features: {},
});
}
@@ -430,6 +471,69 @@ class LightningStatsImporter {
return newGraph;
}
private isIncorrectSnapshot(timestamp, graph): boolean {
if (timestamp >= 1549065600 /* 2019-02-02 */ && timestamp <= 1550620800 /* 2019-02-20 */ && graph.nodes.length < 2600) {
return true;
}
if (timestamp >= 1552953600 /* 2019-03-19 */ && timestamp <= 1556323200 /* 2019-05-27 */ && graph.nodes.length < 4000) {
return true;
}
if (timestamp >= 1557446400 /* 2019-05-10 */ && timestamp <= 1560470400 /* 2019-06-14 */ && graph.nodes.length < 4000) {
return true;
}
if (timestamp >= 1561680000 /* 2019-06-28 */ && timestamp <= 1563148800 /* 2019-07-15 */ && graph.nodes.length < 4000) {
return true;
}
if (timestamp >= 1571270400 /* 2019-11-17 */ && timestamp <= 1580601600 /* 2020-02-02 */ && graph.nodes.length < 4500) {
return true;
}
if (timestamp >= 1591142400 /* 2020-06-03 */ && timestamp <= 1592006400 /* 2020-06-13 */ && graph.nodes.length < 5500) {
return true;
}
if (timestamp >= 1632787200 /* 2021-09-28 */ && timestamp <= 1633564800 /* 2021-10-07 */ && graph.nodes.length < 13000) {
return true;
}
if (timestamp >= 1634256000 /* 2021-10-15 */ && timestamp <= 1645401600 /* 2022-02-21 */ && graph.nodes.length < 17000) {
return true;
}
if (timestamp >= 1654992000 /* 2022-06-12 */ && timestamp <= 1661472000 /* 2022-08-26 */ && graph.nodes.length < 14000) {
return true;
}
return false;
}
private async $cleanupIncorrectSnapshot(): Promise<void> {
// We do not run this one automatically because those stats are not supposed to be inserted in the first
// place, but I write them here to remind us we manually run those queries
// DELETE FROM lightning_stats
// WHERE (
// UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 AND node_count < 2600 OR
// UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 AND node_count < 4000 OR
// UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 AND node_count < 4000 OR
// UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 AND node_count < 4000 OR
// UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 AND node_count < 4500 OR
// UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 AND node_count < 5500 OR
// UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 AND node_count < 13000 OR
// UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 AND node_count < 17000 OR
// UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000 AND node_count < 14000
// )
// DELETE FROM node_stats
// WHERE (
// UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 OR
// UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 OR
// UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 OR
// UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 OR
// UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 OR
// UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 OR
// UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 OR
// UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 OR
// UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000
// )
}
}
export default new LightningStatsImporter;

119
backend/src/utils/ipcheck.js Normal file
View File

@@ -0,0 +1,119 @@
var net = require('net');
var IPCheck = module.exports = function(input) {
var self = this;
if (!(self instanceof IPCheck)) {
return new IPCheck(input);
}
self.input = input;
self.parse();
};
IPCheck.prototype.parse = function() {
var self = this;
if (!self.input || typeof self.input !== 'string') return self.valid = false;
var ip;
var pos = self.input.lastIndexOf('/');
if (pos !== -1) {
ip = self.input.substring(0, pos);
self.mask = +self.input.substring(pos + 1);
} else {
ip = self.input;
self.mask = null;
}
self.ipv = net.isIP(ip);
self.valid = !!self.ipv && !isNaN(self.mask);
if (!self.valid) return;
// default mask = 32 for ipv4 and 128 for ipv6
if (self.mask === null) self.mask = self.ipv === 4 ? 32 : 128;
if (self.ipv === 4) {
// difference between ipv4 and ipv6 masks
self.mask += 96;
}
if (self.mask < 0 || self.mask > 128) {
self.valid = false;
return;
}
self.address = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ];
if(self.ipv === 4){
self.parseIPv4(ip);
}else{
self.parseIPv6(ip);
}
};
IPCheck.prototype.parseIPv4 = function(ip) {
var self = this;
// ipv4 addresses live under ::ffff:0:0
self.address[10] = self.address[11] = 0xff;
var octets = ip.split('.');
for (var i = 0; i < 4; i++) {
self.address[i + 12] = parseInt(octets[i], 10);
}
};
var V6_TRANSITIONAL = /:(\d+\.\d+\.\d+\.\d+)$/;
IPCheck.prototype.parseIPv6 = function(ip) {
var self = this;
var transitionalMatch = V6_TRANSITIONAL.exec(ip);
if(transitionalMatch){
self.parseIPv4(transitionalMatch[1]);
return;
}
var bits = ip.split(':');
if (bits.length < 8) {
ip = ip.replace('::', Array(11 - bits.length).join(':'));
bits = ip.split(':');
}
var j = 0;
for (var i = 0; i < bits.length; i += 1) {
var x = bits[i] ? parseInt(bits[i], 16) : 0;
self.address[j++] = x >> 8;
self.address[j++] = x & 0xff;
}
};
IPCheck.prototype.match = function(cidr) {
var self = this;
if (!(cidr instanceof IPCheck)) cidr = new IPCheck(cidr);
if (!self.valid || !cidr.valid) return false;
var mask = cidr.mask;
var i = 0;
while (mask >= 8) {
if (self.address[i] !== cidr.address[i]) return false;
i++;
mask -= 8;
}
var shift = 8 - mask;
return (self.address[i] >>> shift) === (cidr.address[i] >>> shift);
};
IPCheck.match = function(ip, cidr) {
ip = ip instanceof IPCheck ? ip : new IPCheck(ip);
return ip.match(cidr);
};

5
backend/testSetup.ts Normal file
View File

@@ -0,0 +1,5 @@
jest.mock('./mempool-config.json', () => ({}), { virtual: true });
jest.mock('./src/logger.ts', () => ({}), { virtual: true });
jest.mock('./src/api/rbf-cache.ts', () => ({}), { virtual: true });
jest.mock('./src/api/mempool.ts', () => ({}), { virtual: true });
jest.mock('./src/api/memory-cache.ts', () => ({}), { virtual: true });

3
backend/tsconfig.json
View File

@@ -13,7 +13,8 @@
"node_modules/@types"
],
"allowSyntheticDefaultImports": true,
"esModuleInterop": true
"esModuleInterop": true,
"allowJs": true,
},
"include": [
"src/**/*.ts"