machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into nymkappa/bugfix/block-audit-code-refactor

Browse Source
This commit is contained in:
Felipe Knorr Kuhn
2022-07-20 20:59:11 -07:00
committed by GitHub
parent 2872d2e299 c7909a1ca8
commit 76b6d2a21b
49 changed files with 1663 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
backend/src/api/database-migration.ts
View File

@@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 32;
private static currentVersion = 33;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -302,6 +302,10 @@ class DatabaseMigration {
if (databaseSchemaVersion < 32 && isBitcoin == true) {
await this.$executeQuery('ALTER TABLE `blocks_summaries` ADD `template` JSON DEFAULT "[]"');
}
if (databaseSchemaVersion < 33 && isBitcoin == true) {
await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization", "country_iso_code") NOT NULL');
}
}
/**

126
backend/src/api/explorer/nodes.api.ts
View File

@@ -93,6 +93,132 @@ class NodesApi {
throw e;
}
}
public async $getNodesISP() {
try {
let query = `SELECT nodes.as_number as ispId, geo_names.names as names, COUNT(DISTINCT nodes.public_key) as nodesCount, SUM(capacity) as capacity
FROM nodes
JOIN geo_names ON geo_names.id = nodes.as_number
JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key
GROUP BY as_number
ORDER BY COUNT(DISTINCT nodes.public_key) DESC
`;
const [nodesCountPerAS]: any = await DB.query(query);
query = `SELECT COUNT(*) as total FROM nodes WHERE as_number IS NOT NULL`;
const [nodesWithAS]: any = await DB.query(query);
const nodesPerAs: any[] = [];
for (const as of nodesCountPerAS) {
nodesPerAs.push({
ispId: as.ispId,
name: JSON.parse(as.names),
count: as.nodesCount,
share: Math.floor(as.nodesCount / nodesWithAS[0].total * 10000) / 100,
capacity: as.capacity,
})
}
return nodesPerAs;
} catch (e) {
logger.err(`Cannot get nodes grouped by AS. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
public async $getNodesPerCountry(countryId: string) {
try {
const query = `
SELECT node_stats.public_key, node_stats.capacity, node_stats.channels, nodes.alias,
UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
geo_names_city.names as city
FROM node_stats
JOIN (
SELECT public_key, MAX(added) as last_added
FROM node_stats
GROUP BY public_key
) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added
JOIN nodes ON nodes.public_key = node_stats.public_key
JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
WHERE geo_names_country.id = ?
ORDER BY capacity DESC
`;
const [rows]: any = await DB.query(query, [countryId]);
for (let i = 0; i < rows.length; ++i) {
rows[i].city = JSON.parse(rows[i].city);
}
return rows;
} catch (e) {
logger.err(`Cannot get nodes for country id ${countryId}. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
public async $getNodesPerISP(ISPId: string) {
try {
const query = `
SELECT node_stats.public_key, node_stats.capacity, node_stats.channels, nodes.alias,
UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
geo_names_city.names as city, geo_names_country.names as country
FROM node_stats
JOIN (
SELECT public_key, MAX(added) as last_added
FROM node_stats
GROUP BY public_key
) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added
JOIN nodes ON nodes.public_key = node_stats.public_key
JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
WHERE nodes.as_number = ?
ORDER BY capacity DESC
`;
const [rows]: any = await DB.query(query, [ISPId]);
for (let i = 0; i < rows.length; ++i) {
rows[i].country = JSON.parse(rows[i].country);
rows[i].city = JSON.parse(rows[i].city);
}
return rows;
} catch (e) {
logger.err(`Cannot get nodes for ISP id ${ISPId}. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
public async $getNodesCountries() {
try {
let query = `SELECT geo_names.names as names, geo_names_iso.names as iso_code, COUNT(DISTINCT nodes.public_key) as nodesCount, SUM(capacity) as capacity
FROM nodes
JOIN geo_names ON geo_names.id = nodes.country_id AND geo_names.type = 'country'
JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key
GROUP BY country_id
ORDER BY COUNT(DISTINCT nodes.public_key) DESC
`;
const [nodesCountPerCountry]: any = await DB.query(query);
query = `SELECT COUNT(*) as total FROM nodes WHERE country_id IS NOT NULL`;
const [nodesWithAS]: any = await DB.query(query);
const nodesPerCountry: any[] = [];
for (const country of nodesCountPerCountry) {
nodesPerCountry.push({
name: JSON.parse(country.names),
iso: country.iso_code,
count: country.nodesCount,
share: Math.floor(country.nodesCount / nodesWithAS[0].total * 10000) / 100,
capacity: country.capacity,
})
}
return nodesPerCountry;
} catch (e) {
logger.err(`Cannot get nodes grouped by AS. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
}
export default new NodesApi();

85
backend/src/api/explorer/nodes.routes.ts
View File

@@ -1,13 +1,19 @@
import config from '../../config';
import { Application, Request, Response } from 'express';
import nodesApi from './nodes.api';
import DB from '../../database';
class NodesRoutes {
constructor() { }
public initRoutes(app: Application) {
app
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/country/:country', this.$getNodesPerCountry)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp/:isp', this.$getNodesPerISP)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/search/:search', this.$searchNode)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/top', this.$getTopNodes)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp', this.$getNodesISP)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/countries', this.$getNodesCountries)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/statistics', this.$getHistoricalNodeStats)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key', this.$getNode)
;
@@ -56,6 +62,85 @@ class NodesRoutes {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getNodesISP(req: Request, res: Response) {
try {
const nodesPerAs = await nodesApi.$getNodesISP();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
res.json(nodesPerAs);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getNodesPerCountry(req: Request, res: Response) {
try {
const [country]: any[] = await DB.query(
`SELECT geo_names.id, geo_names_country.names as country_names
FROM geo_names
JOIN geo_names geo_names_country on geo_names.id = geo_names_country.id AND geo_names_country.type = 'country'
WHERE geo_names.type = 'country_iso_code' AND geo_names.names = ?`,
[req.params.country]
);
if (country.length === 0) {
res.status(404).send(`This country does not exist or does not host any lightning nodes on clearnet`);
return;
}
const nodes = await nodesApi.$getNodesPerCountry(country[0].id);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json({
country: JSON.parse(country[0].country_names),
nodes: nodes,
});
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getNodesPerISP(req: Request, res: Response) {
try {
const [isp]: any[] = await DB.query(
`SELECT geo_names.names as isp_name
FROM geo_names
WHERE geo_names.type = 'as_organization' AND geo_names.id = ?`,
[req.params.isp]
);
if (isp.length === 0) {
res.status(404).send(`This ISP does not exist or does not host any lightning nodes on clearnet`);
return;
}
const nodes = await nodesApi.$getNodesPerISP(req.params.isp);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json({
isp: JSON.parse(isp[0].isp_name),
nodes: nodes,
});
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getNodesCountries(req: Request, res: Response) {
try {
const nodesPerAs = await nodesApi.$getNodesCountries();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
res.json(nodesPerAs);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
}
export default new NodesRoutes();

7
backend/src/tasks/lightning/sync-tasks/node-locations.ts
View File

@@ -39,6 +39,13 @@ export async function $lookupNodeLocation(): Promise<void> {
[city.country?.geoname_id, JSON.stringify(city.country?.names)]);
}
// Store Country ISO code
if (city.country?.iso_code) {
await DB.query(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`,
[city.country?.geoname_id, city.country?.iso_code]);
}
// Store Division
if (city.subdivisions && city.subdivisions[0]) {
await DB.query(

6
backend/src/tasks/price-feeds/kraken-api.ts
View File

@@ -62,7 +62,7 @@ class KrakenApi implements PriceFeed {
// CHF weekly price history goes back to timestamp 1575504000 (December 5, 2019)
// AUD weekly price history goes back to timestamp 1591833600 (June 11, 2020)
const priceHistory: any = {}; // map: timestamp -> Prices
let priceHistory: any = {}; // map: timestamp -> Prices
for (const currency of this.currencies) {
const response = await query(this.urlHist.replace('{GRANULARITY}', '10080') + currency);
@@ -83,6 +83,10 @@ class KrakenApi implements PriceFeed {
}
for (const time in priceHistory) {
if (priceHistory[time].USD === -1) {
delete priceHistory[time];
continue;
}
await PricesRepository.$savePrices(parseInt(time, 10), priceHistory[time]);
}