machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Log correct maxmind mysql updates - fix stats import processed files counter

Browse Source
This commit is contained in:
nymkappa 2022-08-24 15:39:52 +02:00
parent 12c188266a
commit faec398cf0
No known key found for this signature in database
GPG Key ID: E155910B16E8BD04
2 changed files with 40 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
backend/src/tasks/lightning/sync-tasks/node-locations.ts
View File

@ -4,11 +4,14 @@ import nodesApi from '../../../api/explorer/nodes.api';
import config from '../../../config';
import DB from '../../../database';
import logger from '../../../logger';
import { ResultSetHeader } from 'mysql2';
import * as IPCheck from '../../../utils/ipcheck.js';
export async function $lookupNodeLocation(): Promise<void> {
let loggerTimer = new Date().getTime() / 1000;
let progress = 0;
let nodesUpdated = 0;
let geoNamesInserted = 0;
logger.info(`Running node location updater using Maxmind`);
try {
@ -71,51 +74,72 @@ export async function $lookupNodeLocation(): Promise<void> {
city.location?.accuracy_radius,
node.public_key
];
await DB.query(query, params);
let result = await DB.query<ResultSetHeader>(query, params);
if (result[0].changedRows ?? 0 > 0) {
++nodesUpdated;
}
// Store Continent
if (city.continent?.geoname_id) {
await DB.query(
result = await DB.query<ResultSetHeader>(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'continent', ?)`,
[city.continent?.geoname_id, JSON.stringify(city.continent?.names)]);
if (result[0].changedRows ?? 0 > 0) {
++geoNamesInserted;
}
}
// Store Country
if (city.country?.geoname_id) {
await DB.query(
result = await DB.query<ResultSetHeader>(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country', ?)`,
[city.country?.geoname_id, JSON.stringify(city.country?.names)]);
if (result[0].changedRows ?? 0 > 0) {
++geoNamesInserted;
}
}
// Store Country ISO code
if (city.country?.iso_code) {
await DB.query(
result = await DB.query<ResultSetHeader>(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`,
[city.country?.geoname_id, city.country?.iso_code]);
if (result[0].changedRows ?? 0 > 0) {
++geoNamesInserted;
}
}
// Store Division
if (city.subdivisions && city.subdivisions[0]) {
await DB.query(
result = await DB.query<ResultSetHeader>(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'division', ?)`,
[city.subdivisions[0].geoname_id, JSON.stringify(city.subdivisions[0]?.names)]);
if (result[0].changedRows ?? 0 > 0) {
++geoNamesInserted;
}
}
// Store City
if (city.city?.geoname_id) {
await DB.query(
result = await DB.query<ResultSetHeader>(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'city', ?)`,
[city.city?.geoname_id, JSON.stringify(city.city?.names)]);
if (result[0].changedRows ?? 0 > 0) {
++geoNamesInserted;
}
}
// Store AS name
if (isp?.autonomous_system_organization ?? asn?.autonomous_system_organization) {
await DB.query(
result = await DB.query<ResultSetHeader>(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'as_organization', ?)`,
[
asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number,
JSON.stringify(asOverwrite?.name ?? isp?.isp ?? asn?.autonomous_system_organization)
]);
if (result[0].changedRows ?? 0 > 0) {
++geoNamesInserted;
}
}
}
@ -128,7 +152,12 @@ export async function $lookupNodeLocation(): Promise<void> {
}
}
}
logger.info(`${progress} nodes location data updated`);
if (nodesUpdated > 0) {
logger.info(`${nodesUpdated} nodes maxmind data updated, ${geoNamesInserted} geo names inserted`);
} else {
logger.debug(`${nodesUpdated} nodes maxmind data updated, ${geoNamesInserted} geo names inserted`);
}
} catch (e) {
logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e));
}

3
backend/src/tasks/lightning/sync-tasks/stats-importer.ts
View File

@ -360,9 +360,11 @@ class LightningStatsImporter {
fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8');
} catch (e: any) {
if (e.errno == -1) { // EISDIR - Ignore directorie
totalProcessed++;
continue;
}
logger.err(`Unable to open ${this.topologiesFolder}/${filename}`);
totalProcessed++;
continue;
}
@ -372,6 +374,7 @@ class LightningStatsImporter {
graph = await this.cleanupTopology(graph);
} catch (e) {
logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content. Reason: ${e instanceof Error ? e.message : e}`);
totalProcessed++;
continue;
}