machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Integrate GeoIP2 ISP database

Browse Source
This commit is contained in:
nymkappa
2022-07-23 23:33:13 +02:00
parent ecbdee0087
commit 6795f69343
7 changed files with 42 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
backend/src/api/explorer/nodes.api.ts
View File

@@ -96,11 +96,11 @@ class NodesApi {
public async $getNodesISP() {
try {
let query = `SELECT nodes.as_number as ispId, geo_names.names as names, COUNT(DISTINCT nodes.public_key) as nodesCount, SUM(capacity) as capacity
let query = `SELECT GROUP_CONCAT(DISTINCT(nodes.as_number)) as ispId, geo_names.names as names, COUNT(DISTINCT nodes.public_key) as nodesCount, SUM(capacity) as capacity
FROM nodes
JOIN geo_names ON geo_names.id = nodes.as_number
JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key
GROUP BY as_number
GROUP BY geo_names.names
ORDER BY COUNT(DISTINCT nodes.public_key) DESC
`;
const [nodesCountPerAS]: any = await DB.query(query);
@@ -168,14 +168,14 @@ class NodesApi {
FROM node_stats
GROUP BY public_key
) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added
JOIN nodes ON nodes.public_key = node_stats.public_key
RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
WHERE nodes.as_number = ?
WHERE nodes.as_number IN (?)
ORDER BY capacity DESC
`;
const [rows]: any = await DB.query(query, [ISPId]);
const [rows]: any = await DB.query(query, [ISPId.split(',')]);
for (let i = 0; i < rows.length; ++i) {
rows[i].country = JSON.parse(rows[i].country);
rows[i].city = JSON.parse(rows[i].city);

4
backend/src/config.ts
View File

@@ -102,6 +102,7 @@ interface IConfig {
ENABLED: boolean;
GEOLITE2_CITY: string;
GEOLITE2_ASN: string;
GEOLITE2_ISP: string;
},
}
@@ -206,7 +207,8 @@ const defaults: IConfig = {
"MAXMIND": {
'ENABLED': false,
"GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
"GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb"
"GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
"GEOLITE2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
},
};

33
backend/src/tasks/lightning/sync-tasks/node-locations.ts
View File

@@ -1,5 +1,5 @@
import * as net from 'net';
import maxmind, { CityResponse, AsnResponse } from 'maxmind';
import maxmind, { CityResponse, AsnResponse, IspResponse } from 'maxmind';
import nodesApi from '../../../api/explorer/nodes.api';
import config from '../../../config';
import DB from '../../../database';
@@ -11,6 +11,7 @@ export async function $lookupNodeLocation(): Promise<void> {
const nodes = await nodesApi.$getAllNodes();
const lookupCity = await maxmind.open<CityResponse>(config.MAXMIND.GEOLITE2_CITY);
const lookupAsn = await maxmind.open<AsnResponse>(config.MAXMIND.GEOLITE2_ASN);
const lookupIsp = await maxmind.open<IspResponse>(config.MAXMIND.GEOLITE2_ISP);
for (const node of nodes) {
const sockets: string[] = node.sockets.split(',');
@@ -20,9 +21,29 @@ export async function $lookupNodeLocation(): Promise<void> {
if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') {
const city = lookupCity.get(ip);
const asn = lookupAsn.get(ip);
if (city && asn) {
const query = `UPDATE nodes SET as_number = ?, city_id = ?, country_id = ?, subdivision_id = ?, longitude = ?, latitude = ?, accuracy_radius = ? WHERE public_key = ?`;
const params = [asn.autonomous_system_number, city.city?.geoname_id, city.country?.geoname_id, city.subdivisions ? city.subdivisions[0].geoname_id : null, city.location?.longitude, city.location?.latitude, city.location?.accuracy_radius, node.public_key];
const isp = lookupIsp.get(ip);
if (city && (asn || isp)) {
const query = `UPDATE nodes SET
as_number = ?,
city_id = ?,
country_id = ?,
subdivision_id = ?,
longitude = ?,
latitude = ?,
accuracy_radius = ?
WHERE public_key = ?`;
const params = [
isp?.autonomous_system_number ?? asn?.autonomous_system_number,
city.city?.geoname_id,
city.country?.geoname_id,
city.subdivisions ? city.subdivisions[0].geoname_id : null,
city.location?.longitude,
city.location?.latitude,
city.location?.accuracy_radius,
node.public_key
];
await DB.query(query, params);
// Store Continent
@@ -61,10 +82,10 @@ export async function $lookupNodeLocation(): Promise<void> {
}
// Store AS name
if (asn.autonomous_system_organization) {
if (isp?.autonomous_system_organization ?? asn?.autonomous_system_organization) {
await DB.query(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'as_organization', ?)`,
[asn.autonomous_system_number, JSON.stringify(asn.autonomous_system_organization)]);
[isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]);
}
}
}