machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #2166 from mempool/nymkappa/feature/isp-maxmind

Browse Source 
Integrate GeoIP2 ISP database
This commit is contained in:
wiz 2022-07-24 00:14:21 +02:00 committed by GitHub
commit 0f91778970
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 43 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backend/mempool-config.sample.json
View File

@ -66,7 +66,8 @@
"MAXMIND": { "MAXMIND": {
"ENABLED": false, "ENABLED": false,
"GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb", "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
"GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb" "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
"GEOIP2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
}, },
"BISQ": { "BISQ": {
"ENABLED": false, "ENABLED": false,

10
backend/src/api/explorer/nodes.api.ts
View File

@ -96,11 +96,11 @@ class NodesApi {
public async $getNodesISP() { public async $getNodesISP() {
try { try {
let query = `SELECT nodes.as_number as ispId, geo_names.names as names, COUNT(DISTINCT nodes.public_key) as nodesCount, SUM(capacity) as capacity let query = `SELECT GROUP_CONCAT(DISTINCT(nodes.as_number)) as ispId, geo_names.names as names, COUNT(DISTINCT nodes.public_key) as nodesCount, SUM(capacity) as capacity
FROM nodes FROM nodes
JOIN geo_names ON geo_names.id = nodes.as_number JOIN geo_names ON geo_names.id = nodes.as_number
JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key
GROUP BY as_number GROUP BY geo_names.names
ORDER BY COUNT(DISTINCT nodes.public_key) DESC ORDER BY COUNT(DISTINCT nodes.public_key) DESC
`; `;
const [nodesCountPerAS]: any = await DB.query(query); const [nodesCountPerAS]: any = await DB.query(query);
@ -168,14 +168,14 @@ class NodesApi {
FROM node_stats FROM node_stats
GROUP BY public_key GROUP BY public_key
) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added ) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added
JOIN nodes ON nodes.public_key = node_stats.public_key RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
WHERE nodes.as_number = ? WHERE nodes.as_number IN (?)
ORDER BY capacity DESC ORDER BY capacity DESC
`; `;
const [rows]: any = await DB.query(query, [ISPId]); const [rows]: any = await DB.query(query, [ISPId.split(',')]);
for (let i = 0; i < rows.length; ++i) { for (let i = 0; i < rows.length; ++i) {
rows[i].country = JSON.parse(rows[i].country); rows[i].country = JSON.parse(rows[i].country);
rows[i].city = JSON.parse(rows[i].city); rows[i].city = JSON.parse(rows[i].city);

4
backend/src/config.ts
View File

@ -102,6 +102,7 @@ interface IConfig {
ENABLED: boolean; ENABLED: boolean;
GEOLITE2_CITY: string; GEOLITE2_CITY: string;
GEOLITE2_ASN: string; GEOLITE2_ASN: string;
GEOIP2_ISP: string;
}, },
} }
@ -206,7 +207,8 @@ const defaults: IConfig = {
"MAXMIND": { "MAXMIND": {
'ENABLED': false, 'ENABLED': false,
"GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb", "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
"GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb" "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
"GEOIP2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
}, },
}; };

33
backend/src/tasks/lightning/sync-tasks/node-locations.ts
View File

@ -1,5 +1,5 @@
import * as net from 'net'; import * as net from 'net';
import maxmind, { CityResponse, AsnResponse } from 'maxmind'; import maxmind, { CityResponse, AsnResponse, IspResponse } from 'maxmind';
import nodesApi from '../../../api/explorer/nodes.api'; import nodesApi from '../../../api/explorer/nodes.api';
import config from '../../../config'; import config from '../../../config';
import DB from '../../../database'; import DB from '../../../database';
@ -11,6 +11,7 @@ export async function $lookupNodeLocation(): Promise<void> {
const nodes = await nodesApi.$getAllNodes(); const nodes = await nodesApi.$getAllNodes();
const lookupCity = await maxmind.open<CityResponse>(config.MAXMIND.GEOLITE2_CITY); const lookupCity = await maxmind.open<CityResponse>(config.MAXMIND.GEOLITE2_CITY);
const lookupAsn = await maxmind.open<AsnResponse>(config.MAXMIND.GEOLITE2_ASN); const lookupAsn = await maxmind.open<AsnResponse>(config.MAXMIND.GEOLITE2_ASN);
const lookupIsp = await maxmind.open<IspResponse>(config.MAXMIND.GEOIP2_ISP);
for (const node of nodes) { for (const node of nodes) {
const sockets: string[] = node.sockets.split(','); const sockets: string[] = node.sockets.split(',');
@ -20,9 +21,29 @@ export async function $lookupNodeLocation(): Promise<void> {
if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') { if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') {
const city = lookupCity.get(ip); const city = lookupCity.get(ip);
const asn = lookupAsn.get(ip); const asn = lookupAsn.get(ip);
if (city && asn) { const isp = lookupIsp.get(ip);
const query = `UPDATE nodes SET as_number = ?, city_id = ?, country_id = ?, subdivision_id = ?, longitude = ?, latitude = ?, accuracy_radius = ? WHERE public_key = ?`;
const params = [asn.autonomous_system_number, city.city?.geoname_id, city.country?.geoname_id, city.subdivisions ? city.subdivisions[0].geoname_id : null, city.location?.longitude, city.location?.latitude, city.location?.accuracy_radius, node.public_key]; if (city && (asn || isp)) {
const query = `UPDATE nodes SET
as_number = ?,
city_id = ?,
country_id = ?,
subdivision_id = ?,
longitude = ?,
latitude = ?,
accuracy_radius = ?
WHERE public_key = ?`;
const params = [
isp?.autonomous_system_number ?? asn?.autonomous_system_number,
city.city?.geoname_id,
city.country?.geoname_id,
city.subdivisions ? city.subdivisions[0].geoname_id : null,
city.location?.longitude,
city.location?.latitude,
city.location?.accuracy_radius,
node.public_key
];
await DB.query(query, params); await DB.query(query, params);
// Store Continent // Store Continent
@ -61,10 +82,10 @@ export async function $lookupNodeLocation(): Promise<void> {
} }
// Store AS name // Store AS name
if (asn.autonomous_system_organization) { if (isp?.autonomous_system_organization ?? asn?.autonomous_system_organization) {
await DB.query( await DB.query(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'as_organization', ?)`, `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'as_organization', ?)`,
[asn.autonomous_system_number, JSON.stringify(asn.autonomous_system_organization)]); [isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]);
} }
} }
} }

2
frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html
View File

@ -34,7 +34,7 @@
<tbody [attr.data-cy]="'pools-table'" *ngIf="(nodesPerAsObservable$ | async) as asList"> <tbody [attr.data-cy]="'pools-table'" *ngIf="(nodesPerAsObservable$ | async) as asList">
<tr *ngFor="let asEntry of asList"> <tr *ngFor="let asEntry of asList">
<td class="rank text-left pl-0">{{ asEntry.rank }}</td> <td class="rank text-left pl-0">{{ asEntry.rank }}</td>
<td class="name text-left text-truncate" style="max-width: 100px"> <td class="name text-left text-truncate" style="max-width: 250px">
<a [routerLink]="[('/lightning/nodes/isp/' + asEntry.ispId) | relativeUrl]">{{ asEntry.name }}</a> <a [routerLink]="[('/lightning/nodes/isp/' + asEntry.ispId) | relativeUrl]">{{ asEntry.name }}</a>
</td> </td>
<td class="share text-right">{{ asEntry.share }}%</td> <td class="share text-right">{{ asEntry.share }}%</td>

4
frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss
View File

@ -36,14 +36,14 @@
} }
.rank { .rank {
width: 20%; width: 15%;
@media (max-width: 576px) { @media (max-width: 576px) {
display: none display: none
} }
} }
.name { .name {
width: 20%; width: 25%;
@media (max-width: 576px) { @media (max-width: 576px) {
width: 80%; width: 80%;
max-width: 150px; max-width: 150px;

4
frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.ts
View File

@ -61,7 +61,7 @@ export class NodesPerISPChartComponent implements OnInit {
} }
generateChartSerieData(as) { generateChartSerieData(as) {
const shareThreshold = this.isMobile() ? 2 : 1; const shareThreshold = this.isMobile() ? 2 : 0.5;
const data: object[] = []; const data: object[] = [];
let totalShareOther = 0; let totalShareOther = 0;
let totalNodeOther = 0; let totalNodeOther = 0;
@ -155,7 +155,7 @@ export class NodesPerISPChartComponent implements OnInit {
series: [ series: [
{ {
zlevel: 0, zlevel: 0,
minShowLabelAngle: 3.6, minShowLabelAngle: 1.8,
name: 'Lightning nodes', name: 'Lightning nodes',
type: 'pie', type: 'pie',
radius: pieSize, radius: pieSize,