machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into nymkappa/feature/cltv

Browse Source
This commit is contained in:
wiz
2022-09-09 18:06:44 +02:00
committed by GitHub
parent dac3a43c1b ffbb4e0b9e
commit b30483572d
73 changed files with 3449 additions and 468 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
backend/src/api/backend-info.ts
View File

@@ -1,60 +1,37 @@
import * as fs from 'fs';
import * as os from 'os';
import logger from '../logger';
import fs from 'fs';
import path from 'path';
import os from 'os';
import { IBackendInfo } from '../mempool.interfaces';
const { spawnSync } = require('child_process');
class BackendInfo {
private gitCommitHash = '';
private hostname = '';
private version = '';
private backendInfo: IBackendInfo;
constructor() {
this.setLatestCommitHash();
this.setVersion();
this.hostname = os.hostname();
}
public getBackendInfo(): IBackendInfo {
return {
hostname: this.hostname,
gitCommit: this.gitCommitHash,
version: this.version,
// This file is created by ./fetch-version.ts during building
const versionFile = path.join(__dirname, 'version.json')
var versionInfo;
if (fs.existsSync(versionFile)) {
versionInfo = JSON.parse(fs.readFileSync(versionFile).toString());
} else {
// Use dummy values if `versionFile` doesn't exist (e.g., during testing)
versionInfo = {
version: '?',
gitCommit: '?'
};
}
this.backendInfo = {
hostname: os.hostname(),
version: versionInfo.version,
gitCommit: versionInfo.gitCommit
};
}
public getBackendInfo(): IBackendInfo {
return this.backendInfo;
}
public getShortCommitHash() {
return this.gitCommitHash.slice(0, 7);
}
private setLatestCommitHash(): void {
//TODO: share this logic with `generate-config.js`
if (process.env.DOCKER_COMMIT_HASH) {
this.gitCommitHash = process.env.DOCKER_COMMIT_HASH;
} else {
try {
const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
if (!gitRevParse.error) {
const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
this.gitCommitHash = output ? output : '?';
} else if (gitRevParse.error.code === 'ENOENT') {
console.log('git not found, cannot parse git hash');
this.gitCommitHash = '?';
}
} catch (e: any) {
console.log('Could not load git commit info: ' + e.message);
this.gitCommitHash = '?';
}
}
}
private setVersion(): void {
try {
const packageJson = fs.readFileSync('package.json').toString();
this.version = JSON.parse(packageJson).version;
} catch (e) {
throw new Error(e instanceof Error ? e.message : 'Error');
}
return this.backendInfo.gitCommit.slice(0, 7);
}
}

4
backend/src/api/explorer/channels.api.ts
View File

@@ -39,7 +39,8 @@ class ChannelsApi {
FROM channels
JOIN nodes AS nodes_1 on nodes_1.public_key = channels.node1_public_key
JOIN nodes AS nodes_2 on nodes_2.public_key = channels.node2_public_key
WHERE nodes_1.latitude IS NOT NULL AND nodes_1.longitude IS NOT NULL
WHERE channels.status = 1
AND nodes_1.latitude IS NOT NULL AND nodes_1.longitude IS NOT NULL
AND nodes_2.latitude IS NOT NULL AND nodes_2.longitude IS NOT NULL
`;
@@ -374,6 +375,7 @@ class ChannelsApi {
'transaction_vout': channel.transaction_vout,
'closing_transaction_id': channel.closing_transaction_id,
'closing_reason': channel.closing_reason,
'closing_date': channel.closing_date,
'updated_at': channel.updated_at,
'created': channel.created,
'status': channel.status,

66
backend/src/api/explorer/nodes.api.ts
View File

@@ -5,6 +5,49 @@ import { ILightningApi } from '../lightning/lightning-api.interface';
import { ITopNodesPerCapacity, ITopNodesPerChannels } from '../../mempool.interfaces';
class NodesApi {
public async $getWorldNodes(): Promise<any> {
try {
let query = `
SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
CAST(COALESCE(nodes.channels, 0) as INT) as channels,
nodes.longitude, nodes.latitude,
geo_names_country.names as country, geo_names_iso.names as isoCode
FROM nodes
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
WHERE status = 1 AND nodes.as_number IS NOT NULL
ORDER BY capacity
`;
const [nodes]: any[] = await DB.query(query);
for (let i = 0; i < nodes.length; ++i) {
nodes[i].country = JSON.parse(nodes[i].country);
}
query = `
SELECT MAX(nodes.capacity) as maxLiquidity, MAX(nodes.channels) as maxChannels
FROM nodes
WHERE status = 1 AND nodes.as_number IS NOT NULL
`;
const [maximums]: any[] = await DB.query(query);
return {
maxLiquidity: maximums[0].maxLiquidity,
maxChannels: maximums[0].maxChannels,
nodes: nodes.map(node => [
node.longitude, node.latitude,
node.publicKey, node.alias, node.capacity, node.channels,
node.country, node.isoCode
])
};
} catch (e) {
logger.err(`Can't get world nodes list. Reason: ${e instanceof Error ? e.message : e}`);
}
}
public async $getNode(public_key: string): Promise<any> {
try {
// General info
@@ -133,10 +176,13 @@ class NodesApi {
CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
CAST(COALESCE(nodes.channels, 0) as INT) as channels,
UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
geo_names_city.names as city, geo_names_country.names as country
geo_names_city.names as city, geo_names_country.names as country,
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
FROM nodes
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
ORDER BY capacity DESC
LIMIT 100
`;
@@ -175,10 +221,13 @@ class NodesApi {
CAST(COALESCE(nodes.channels, 0) as INT) as channels,
CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
geo_names_city.names as city, geo_names_country.names as country
geo_names_city.names as city, geo_names_country.names as country,
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
FROM nodes
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
ORDER BY channels DESC
LIMIT 100
`;
@@ -221,11 +270,14 @@ class NodesApi {
CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity,
UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
geo_names_city.names as city, geo_names_country.names as country
geo_names_city.names as city, geo_names_country.names as country,
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
FROM node_stats
RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
WHERE added = FROM_UNIXTIME(${latestDate})
ORDER BY first_seen
LIMIT 100
@@ -382,12 +434,14 @@ class NodesApi {
SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
geo_names_city.names as city, geo_names_country.names as country,
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision,
nodes.longitude, nodes.latitude, nodes.as_number, geo_names_isp.names as isp
FROM nodes
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
LEFT JOIN geo_names geo_names_isp on geo_names_isp.id = nodes.as_number AND geo_names_isp.type = 'as_organization'
WHERE geo_names_country.id = ?
ORDER BY capacity DESC
`;
@@ -397,6 +451,7 @@ class NodesApi {
rows[i].country = JSON.parse(rows[i].country);
rows[i].city = JSON.parse(rows[i].city);
rows[i].subdivision = JSON.parse(rows[i].subdivision);
rows[i].isp = JSON.parse(rows[i].isp);
}
return rows;
} catch (e) {
@@ -411,7 +466,8 @@ class NodesApi {
SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
geo_names_city.names as city, geo_names_country.names as country,
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision,
nodes.longitude, nodes.latitude
FROM nodes
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'

14
backend/src/api/explorer/nodes.routes.ts
View File

@@ -9,6 +9,7 @@ class NodesRoutes {
public initRoutes(app: Application) {
app
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/world', this.$getWorldNodes)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/country/:country', this.$getNodesPerCountry)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/search/:search', this.$searchNode)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp-ranking', this.$getISPRanking)
@@ -115,7 +116,6 @@ class NodesRoutes {
private async $getISPRanking(req: Request, res: Response): Promise<void> {
try {
const nodesPerAs = await nodesApi.$getNodesISPRanking();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
@@ -125,6 +125,18 @@ class NodesRoutes {
}
}
private async $getWorldNodes(req: Request, res: Response) {
try {
const worldNodes = await nodesApi.$getWorldNodes();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
res.json(worldNodes);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getNodesPerCountry(req: Request, res: Response) {
try {
const [country]: any[] = await DB.query(

37
backend/src/api/fetch-version.ts Normal file
View File

@@ -0,0 +1,37 @@
import fs from 'fs';
import path from "path";
const { spawnSync } = require('child_process');
function getVersion(): string {
const packageJson = fs.readFileSync('package.json').toString();
return JSON.parse(packageJson).version;
}
function getGitCommit(): string {
if (process.env.MEMPOOL_COMMIT_HASH) {
return process.env.MEMPOOL_COMMIT_HASH;
} else {
const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
if (!gitRevParse.error) {
const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
if (output) {
return output;
} else {
console.log('Could not fetch git commit: No repo available');
}
} else if (gitRevParse.error.code === 'ENOENT') {
console.log('Could not fetch git commit: Command `git` is unavailable');
}
}
return '?';
}
const versionInfo = {
version: getVersion(),
gitCommit: getGitCommit()
}
fs.writeFileSync(
path.join(__dirname, 'version.json'),
JSON.stringify(versionInfo, null, 2) + "\n"
);

6
backend/src/config.ts
View File

@@ -1,4 +1,6 @@
const configFile = require('../mempool-config.json');
const configFromFile = require(
process.env.MEMPOOL_CONFIG_FILE ? process.env.MEMPOOL_CONFIG_FILE : '../mempool-config.json'
);
interface IConfig {
MEMPOOL: {
@@ -249,7 +251,7 @@ class Config implements IConfig {
MAXMIND: IConfig['MAXMIND'];
constructor() {
const configs = this.merge(configFile, defaults);
const configs = this.merge(configFromFile, defaults);
this.MEMPOOL = configs.MEMPOOL;
this.ESPLORA = configs.ESPLORA;
this.ELECTRUM = configs.ELECTRUM;

20
backend/src/tasks/lightning/network-sync.service.ts
View File

@@ -12,9 +12,11 @@ import { ResultSetHeader } from 'mysql2';
import fundingTxFetcher from './sync-tasks/funding-tx-fetcher';
import NodesSocketsRepository from '../../repositories/NodesSocketsRepository';
import { Common } from '../../api/common';
import blocks from '../../api/blocks';
class NetworkSyncService {
loggerTimer = 0;
closedChannelsScanBlock = 0;
constructor() {}
@@ -240,10 +242,22 @@ class NetworkSyncService {
}
private async $scanForClosedChannels(): Promise<void> {
if (this.closedChannelsScanBlock === blocks.getCurrentBlockHeight()) {
logger.debug(`We've already scan closed channels for this block, skipping.`);
return;
}
let progress = 0;
try {
logger.info(`Starting closed channels scan`);
let log = `Starting closed channels scan`;
if (this.closedChannelsScanBlock > 0) {
log += `. Last scan was at block ${this.closedChannelsScanBlock}`;
} else {
log += ` for the first time`;
}
logger.info(log);
const channels = await channelsApi.$getChannelsByStatus([0, 1]);
for (const channel of channels) {
const spendingTx = await bitcoinApi.$getOutspend(channel.transaction_id, channel.transaction_vout);
@@ -263,7 +277,9 @@ class NetworkSyncService {
this.loggerTimer = new Date().getTime() / 1000;
}
}
logger.info(`Closed channels scan complete.`);
this.closedChannelsScanBlock = blocks.getCurrentBlockHeight();
logger.info(`Closed channels scan completed at block ${this.closedChannelsScanBlock}`);
} catch (e) {
logger.err('$scanForClosedChannels() error: ' + (e instanceof Error ? e.message : e));
}

3
backend/src/tasks/price-updater.ts
View File

@@ -1,4 +1,5 @@
import * as fs from 'fs';
import path from "path";
import { Common } from '../api/common';
import config from '../config';
import logger from '../logger';
@@ -159,7 +160,7 @@ class PriceUpdater {
const existingPriceTimes = await PricesRepository.$getPricesTimes();
// Insert MtGox weekly prices
const pricesJson: any[] = JSON.parse(fs.readFileSync('./src/tasks/price-feeds/mtgox-weekly.json').toString());
const pricesJson: any[] = JSON.parse(fs.readFileSync(path.join(__dirname, 'mtgox-weekly.json')).toString());
const prices = this.getEmptyPricesObj();
let insertedCount: number = 0;
for (const price of pricesJson) {