Merge pull request #2074 from mempool/simon/maxmind

Use maxmind to store node locations

backend/mempool-config.sample.json

@@ -63,6 +63,11 @@
     "ENABLED": true,
     "TX_PER_SECOND_SAMPLE_PERIOD": 150
   },
+  "MAXMIND": {
+    "ENABLED": false,
+    "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
+    "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb"
+  },
   "BISQ": {
     "ENABLED": false,
     "DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db"

backend/package-lock.json

@@ -17,6 +17,7 @@
         "crypto-js": "^4.0.0",
         "express": "^4.18.0",
         "lightning": "^5.16.3",
+        "maxmind": "^4.3.6",
         "mysql2": "2.3.3",
         "node-worker-threads-pool": "^1.5.1",
         "socks-proxy-agent": "~7.0.0",
@@ -2222,6 +2223,19 @@
         "node": ">=10"
       }
     },
+    "node_modules/maxmind": {
+      "version": "4.3.6",
+      "resolved": "https://registry.npmjs.org/maxmind/-/maxmind-4.3.6.tgz",
+      "integrity": "sha512-CwnEZqJX0T6b2rWrc0/V3n9hL/hWAMEn7fY09077YJUHiHx7cn/esA2ZIz8BpYLSJUf7cGVel0oUJa9jMwyQpg==",
+      "dependencies": {
+        "mmdb-lib": "2.0.2",
+        "tiny-lru": "8.0.2"
+      },
+      "engines": {
+        "node": ">=10",
+        "npm": ">=6"
+      }
+    },
     "node_modules/md5.js": {
       "version": "1.3.5",
       "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz",
@@ -2317,6 +2331,15 @@
         "node": "*"
       }
     },
+    "node_modules/mmdb-lib": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/mmdb-lib/-/mmdb-lib-2.0.2.tgz",
+      "integrity": "sha512-shi1I+fCPQonhTi7qyb6hr7hi87R7YS69FlfJiMFuJ12+grx0JyL56gLNzGTYXPU7EhAPkMLliGeyHer0K+AVA==",
+      "engines": {
+        "node": ">=10",
+        "npm": ">=6"
+      }
+    },
     "node_modules/ms": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@@ -3027,6 +3050,14 @@
       "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==",
       "dev": true
     },
+    "node_modules/tiny-lru": {
+      "version": "8.0.2",
+      "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz",
+      "integrity": "sha512-ApGvZ6vVvTNdsmt676grvCkUCGwzG9IqXma5Z07xJgiC5L7akUMof5U8G2JTI9Rz/ovtVhJBlY6mNhEvtjzOIg==",
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/tiny-secp256k1": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/tiny-secp256k1/-/tiny-secp256k1-2.2.1.tgz",
@@ -4971,6 +5002,15 @@
         "yallist": "^4.0.0"
       }
     },
+    "maxmind": {
+      "version": "4.3.6",
+      "resolved": "https://registry.npmjs.org/maxmind/-/maxmind-4.3.6.tgz",
+      "integrity": "sha512-CwnEZqJX0T6b2rWrc0/V3n9hL/hWAMEn7fY09077YJUHiHx7cn/esA2ZIz8BpYLSJUf7cGVel0oUJa9jMwyQpg==",
+      "requires": {
+        "mmdb-lib": "2.0.2",
+        "tiny-lru": "8.0.2"
+      }
+    },
     "md5.js": {
       "version": "1.3.5",
       "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz",
@@ -5039,6 +5079,11 @@
         "brace-expansion": "^1.1.7"
       }
     },
+    "mmdb-lib": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/mmdb-lib/-/mmdb-lib-2.0.2.tgz",
+      "integrity": "sha512-shi1I+fCPQonhTi7qyb6hr7hi87R7YS69FlfJiMFuJ12+grx0JyL56gLNzGTYXPU7EhAPkMLliGeyHer0K+AVA=="
+    },
     "ms": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@@ -5549,6 +5594,11 @@
       "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==",
       "dev": true
     },
+    "tiny-lru": {
+      "version": "8.0.2",
+      "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz",
+      "integrity": "sha512-ApGvZ6vVvTNdsmt676grvCkUCGwzG9IqXma5Z07xJgiC5L7akUMof5U8G2JTI9Rz/ovtVhJBlY6mNhEvtjzOIg=="
+    },
     "tiny-secp256k1": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/tiny-secp256k1/-/tiny-secp256k1-2.2.1.tgz",

backend/package.json

@@ -38,6 +38,7 @@
     "crypto-js": "^4.0.0",
     "express": "^4.18.0",
     "lightning": "^5.16.3",
+    "maxmind": "^4.3.6",
     "mysql2": "2.3.3",
     "node-worker-threads-pool": "^1.5.1",
     "socks-proxy-agent": "~7.0.0",

backend/src/api/database-migration.ts

@@ -4,7 +4,7 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 28;
+  private static currentVersion = 29;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -280,6 +280,17 @@ class DatabaseMigration {
         await this.$executeQuery(`ALTER TABLE lightning_stats MODIFY added DATE`);
       }
 
+      if (databaseSchemaVersion < 29 && isBitcoin === true) {
+        await this.$executeQuery(this.getCreateGeoNamesTableQuery(), await this.$checkIfTableExists('geo_names'));
+        await this.$executeQuery('ALTER TABLE `nodes` ADD as_number int(11) unsigned NULL DEFAULT NULL');
+        await this.$executeQuery('ALTER TABLE `nodes` ADD city_id int(11) unsigned NULL DEFAULT NULL');
+        await this.$executeQuery('ALTER TABLE `nodes` ADD country_id int(11) unsigned NULL DEFAULT NULL');
+        await this.$executeQuery('ALTER TABLE `nodes` ADD accuracy_radius int(11) unsigned NULL DEFAULT NULL');
+        await this.$executeQuery('ALTER TABLE `nodes` ADD subdivision_id int(11) unsigned NULL DEFAULT NULL');
+        await this.$executeQuery('ALTER TABLE `nodes` ADD longitude double NULL DEFAULT NULL');
+        await this.$executeQuery('ALTER TABLE `nodes` ADD latitude double NULL DEFAULT NULL');
+      }
+
     } catch (e) {
       throw e;
     }
@@ -693,6 +704,16 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateGeoNamesTableQuery(): string {
+    return `CREATE TABLE geo_names (
+      id int(11) unsigned NOT NULL,
+      type enum('city','country','division','continent') NOT NULL,
+      names text DEFAULT NULL,
+      UNIQUE KEY id (id,type),
+      KEY id_2 (id)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`
+  }
+
   public async $truncateIndexedData(tables: string[]) {
     const allowedTables = ['blocks', 'hashrates', 'prices'];
 

backend/src/api/explorer/nodes.api.ts

@@ -13,6 +13,17 @@ class NodesApi {
     }
   }
 
+  public async $getAllNodes(): Promise<any> {
+    try {
+      const query = `SELECT * FROM nodes`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getAllNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   public async $getNodeStats(public_key: string): Promise<any> {
     try {
       const query = `SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels FROM node_stats WHERE public_key = ? ORDER BY added DESC`;

backend/src/config.ts

@@ -98,6 +98,11 @@ interface IConfig {
     BISQ_URL: string;
     BISQ_ONION: string;
   };
+  MAXMIND: {
+    ENABLED: boolean;
+    GEOLITE2_CITY: string;
+    GEOLITE2_ASN: string;
+  },
 }
 
 const defaults: IConfig = {
@@ -197,7 +202,12 @@ const defaults: IConfig = {
     'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1',
     'BISQ_URL': 'https://bisq.markets/api',
     'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
-  }
+  },
+  "MAXMIND": {
+    'ENABLED': false,
+    "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
+    "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb"
+  },
 };
 
 class Config implements IConfig {
@@ -215,6 +225,7 @@ class Config implements IConfig {
   SOCKS5PROXY: IConfig['SOCKS5PROXY'];
   PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
   EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
+  MAXMIND: IConfig['MAXMIND'];
 
   constructor() {
     const configs = this.merge(configFile, defaults);
@@ -232,6 +243,7 @@ class Config implements IConfig {
     this.SOCKS5PROXY = configs.SOCKS5PROXY;
     this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
     this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
+    this.MAXMIND = configs.MAXMIND;
   }
 
   merge = (...objects: object[]): IConfig => {

backend/src/tasks/lightning/node-sync.service.ts

@@ -8,6 +8,7 @@ import config from '../../config';
 import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
 import lightningApi from '../../api/lightning/lightning-api-factory';
 import { ILightningApi } from '../../api/lightning/lightning-api.interface';
+import { $lookupNodeLocation } from './sync-tasks/node-locations';
 
 class NodeSyncService {
   constructor() {}
@@ -33,6 +34,10 @@ class NodeSyncService {
       }
       logger.info(`Nodes updated.`);
 
+      if (config.MAXMIND.ENABLED) {
+        await $lookupNodeLocation();
+      }
+
       await this.$setChannelsInactive();
 
       for (const channel of networkGraph.channels) {

backend/src/tasks/lightning/stats-updater.service.ts

@@ -206,7 +206,7 @@ class LightningStatsUpdater {
               torNodes++;
               isUnnanounced = false;
             }
-            const hasClearnet = [4, 6].includes(net.isIP(socket.split(':')[0]));
+            const hasClearnet = [4, 6].includes(net.isIP(socket.substring(0, socket.lastIndexOf(':'))));
             if (hasClearnet) {
               clearnetNodes++;
               isUnnanounced = false;

backend/src/tasks/lightning/sync-tasks/node-locations.ts

@@ -0,0 +1,63 @@
+import * as net from 'net';
+import maxmind, { CityResponse, AsnResponse } from 'maxmind';
+import nodesApi from '../../../api/explorer/nodes.api';
+import config from '../../../config';
+import DB from '../../../database';
+import logger from '../../../logger';
+
+export async function $lookupNodeLocation(): Promise<void> {
+  logger.info(`Running node location updater using Maxmind...`);
+  try {
+    const nodes = await nodesApi.$getAllNodes();
+    const lookupCity = await maxmind.open<CityResponse>(config.MAXMIND.GEOLITE2_CITY);
+    const lookupAsn = await maxmind.open<AsnResponse>(config.MAXMIND.GEOLITE2_ASN);
+
+    for (const node of nodes) {
+      const sockets: string[] = node.sockets.split(',');
+      for (const socket of sockets) {
+        const ip = socket.substring(0, socket.lastIndexOf(':')).replace('[', '').replace(']', '');
+        const hasClearnet = [4, 6].includes(net.isIP(ip));
+        if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') {
+          const city = lookupCity.get(ip);
+          const asn = lookupAsn.get(ip);
+          if (city && asn) {
+            const query = `UPDATE nodes SET as_number = ?, city_id = ?, country_id = ?, subdivision_id = ?, longitude = ?, latitude = ?, accuracy_radius = ? WHERE public_key = ?`;
+            const params = [asn.autonomous_system_number, city.city?.geoname_id, city.country?.geoname_id, city.subdivisions ? city.subdivisions[0].geoname_id : null, city.location?.longitude, city.location?.latitude, city.location?.accuracy_radius, node.public_key];
+            await DB.query(query, params);
+
+             // Store Continent
+             if (city.continent?.geoname_id) {
+               await DB.query(
+                `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'continent', ?)`,
+                [city.continent?.geoname_id, JSON.stringify(city.continent?.names)]);
+             }
+
+             // Store Country
+             if (city.country?.geoname_id) {
+               await DB.query(
+                `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country', ?)`,
+                [city.country?.geoname_id, JSON.stringify(city.country?.names)]);
+             }
+
+            // Store Division
+            if (city.subdivisions && city.subdivisions[0]) {
+              await DB.query(
+                `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'division', ?)`,
+                [city.subdivisions[0].geoname_id, JSON.stringify(city.subdivisions[0]?.names)]);
+            }
+
+            // Store City
+            if (city.city?.geoname_id) {
+              await DB.query(
+                `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'city', ?)`,
+                [city.city?.geoname_id, JSON.stringify(city.city?.names)]);
+            }
+          }
+        }
+      }
+    }
+    logger.info(`Node location data updated.`);
+  } catch (e) {
+    logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e));
+  }
+}