machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into nymkappa/feature/align-dashboards

Browse Source
This commit is contained in:
softsimon
2023-03-01 06:31:25 +04:00
committed by GitHub
parent 5981e52534 ec7f0d1143
commit 0929d53c56
43 changed files with 5527 additions and 2485 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
backend/README.md
View File

@@ -160,7 +160,7 @@ npm install -g ts-node nodemon
Then, run the watcher:
```
nodemon src/index.ts --ignore cache/ --ignore pools.json
nodemon src/index.ts --ignore cache/
```
`nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`.
@@ -219,6 +219,16 @@ Generate block at regular interval (every 10 seconds in this example):
watch -n 10 "./src/bitcoin-cli -regtest -rpcport=8332 generatetoaddress 1 $address"
```
### Mining pools update
By default, mining pools will be not automatically updated regularly (`config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` is set to `false`).
To manually update your mining pools, you can use the `--update-pools` command line flag when you run the nodejs backend. For example `npm run start --update-pools`. This will trigger the mining pools update and automatically re-index appropriate blocks.
You can enabled the automatic mining pools update by settings `config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` to `true` in your `mempool-config.json`.
When a `coinbase tag` or `coinbase address` change is detected, all blocks tagged to the `unknown` mining pools (starting from height 130635) will be deleted from the `blocks` table. Additionaly, all blocks which were tagged to the pool which has been updated will also be deleted from the `blocks` table. Of course, those blocks will be automatically reindexed.
### Re-index tables
You can manually force the nodejs backend to drop all data from a specified set of tables for future re-index. This is mostly useful for the mining dashboard and the lightning explorer.
@@ -235,4 +245,4 @@ Feb 13 14:55:27 [63246] WARN: <lightning> Indexed data for "hashrates" tables wi
Feb 13 14:55:32 [63246] NOTICE: <lightning> Table hashrates has been truncated
```
Reference: https://github.com/mempool/mempool/pull/1269
Reference: https://github.com/mempool/mempool/pull/1269

2
backend/mempool-config.sample.json
View File

@@ -22,7 +22,7 @@
"USER_AGENT": "mempool",
"STDOUT_LOG_MIN_PRIORITY": "debug",
"AUTOMATIC_BLOCK_REINDEXING": false,
"POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
"POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
"POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
"AUDIT": false,
"ADVANCED_GBT_AUDIT": false,

2
backend/src/__fixtures__/mempool-config.template.json
View File

@@ -7,7 +7,7 @@
"HTTP_PORT": 1,
"SPAWN_CLUSTER_PROCS": 2,
"API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
"AUTOMATIC_BLOCK_REINDEXING": true,
"AUTOMATIC_BLOCK_REINDEXING": false,
"POLL_RATE_MS": 3,
"CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
"CLEAR_PROTECTION_MINUTES": 4,

3
backend/src/api/audit.ts
View File

@@ -119,7 +119,8 @@ class Audit {
}
const numCensored = Object.keys(isCensored).length;
const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0;
const numMatches = matches.length - 1; // adjust for coinbase tx
const score = numMatches > 0 ? (numMatches / (numMatches + numCensored)) : 0;
return {
censored: Object.keys(isCensored),

15
backend/src/api/common.ts
View File

@@ -237,14 +237,21 @@ export class Common {
].join('x');
}
static utcDateToMysql(date?: number): string {
static utcDateToMysql(date?: number | null): string | null {
if (date === null) {
return null;
}
const d = new Date((date || 0) * 1000);
return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
}
static findSocketNetwork(addr: string): {network: string | null, url: string} {
let network: string | null = null;
let url = addr.split('://')[1];
let url: string = addr;
if (config.LIGHTNING.BACKEND === 'cln') {
url = addr.split('://')[1];
}
if (!url) {
return {
@@ -261,7 +268,7 @@ export class Common {
}
} else if (addr.indexOf('i2p') !== -1) {
network = 'i2p';
} else if (addr.indexOf('ipv4') !== -1) {
} else if (addr.indexOf('ipv4') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && isIP(url.split(':')[0]) === 4)) {
const ipv = isIP(url.split(':')[0]);
if (ipv === 4) {
network = 'ipv4';
@@ -271,7 +278,7 @@ export class Common {
url: addr,
};
}
} else if (addr.indexOf('ipv6') !== -1) {
} else if (addr.indexOf('ipv6') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && url.indexOf(']:'))) {
url = url.split('[')[1].split(']')[0];
const ipv = isIP(url);
if (ipv === 6) {

35
backend/src/api/database-migration.ts
View File

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 56;
private static currentVersion = 57;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -500,6 +500,11 @@ class DatabaseMigration {
this.uniqueLog(logger.notice, '`pools` table has been truncated`');
await this.updateToSchemaVersion(56);
}
if (databaseSchemaVersion < 57) {
await this.$executeQuery(`ALTER TABLE nodes MODIFY updated_at datetime NULL`);
await this.updateToSchemaVersion(57);
}
}
/**
@@ -1012,26 +1017,16 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
public async $truncateIndexedData(tables: string[]) {
const allowedTables = ['blocks', 'hashrates', 'prices'];
public async $blocksReindexingTruncate(): Promise<void> {
logger.warn(`Truncating pools, blocks and hashrates for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
await Common.sleep$(5000);
try {
for (const table of tables) {
if (!allowedTables.includes(table)) {
logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`);
continue;
}
await this.$executeQuery(`TRUNCATE ${table}`, true);
if (table === 'hashrates') {
await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
}
logger.notice(`Table ${table} has been truncated`);
}
} catch (e) {
logger.warn(`Unable to erase indexed data`);
}
}
await this.$executeQuery(`TRUNCATE blocks`);
await this.$executeQuery(`TRUNCATE hashrates`);
await this.$executeQuery('DELETE FROM `pools`');
await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
}
private async $convertCompactCpfpTables(): Promise<void> {
try {

11
backend/src/api/explorer/channels.api.ts
View File

@@ -559,6 +559,17 @@ class ChannelsApi {
const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
// https://github.com/mempool/mempool/issues/3006
if ((channel.last_update ?? 0) < 1514736061) { // January 1st 2018
channel.last_update = null;
}
if ((policy1.last_update ?? 0) < 1514736061) { // January 1st 2018
policy1.last_update = null;
}
if ((policy2.last_update ?? 0) < 1514736061) { // January 1st 2018
policy2.last_update = null;
}
const query = `INSERT INTO channels
(
id,

5
backend/src/api/explorer/nodes.api.ts
View File

@@ -642,6 +642,11 @@ class NodesApi {
*/
public async $saveNode(node: ILightningApi.Node): Promise<void> {
try {
// https://github.com/mempool/mempool/issues/3006
if ((node.last_update ?? 0) < 1514736061) { // January 1st 2018
node.last_update = null;
}
const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
const query = `INSERT INTO nodes(
public_key,

6
backend/src/api/lightning/lightning-api.interface.ts
View File

@@ -21,7 +21,7 @@ export namespace ILightningApi {
export interface Channel {
channel_id: string;
chan_point: string;
last_update: number;
last_update: number | null;
node1_pub: string;
node2_pub: string;
capacity: string;
@@ -36,11 +36,11 @@ export namespace ILightningApi {
fee_rate_milli_msat: string;
disabled: boolean;
max_htlc_msat: string;
last_update: number;
last_update: number | null;
}
export interface Node {
last_update: number;
last_update: number | null;
pub_key: string;
alias: string;
addresses: {

4
backend/src/api/pools-parser.ts
View File

@@ -73,7 +73,7 @@ class PoolsParser {
}
}
logger.info('Mining pools.json import completed');
logger.info('Mining pools-v2.json import completed');
}
/**
@@ -115,7 +115,7 @@ class PoolsParser {
return;
}
// Get oldest blocks mined by the pool and assume pools.json updates only concern most recent years
// Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years
// Ignore early days of Bitcoin as there were no mining pool yet
const [oldestPoolBlock]: any[] = await DB.query(`
SELECT height

8
backend/src/index.ts
View File

@@ -36,7 +36,6 @@ import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
import forensicsService from './tasks/lightning/forensics.service';
import priceUpdater from './tasks/price-updater';
import mining from './api/mining/mining';
import chainTips from './api/chain-tips';
import { AxiosError } from 'axios';
@@ -84,11 +83,8 @@ class Server {
if (config.DATABASE.ENABLED) {
await DB.checkDbConnection();
try {
if (process.env.npm_config_reindex !== undefined) { // Re-index requests
const tables = process.env.npm_config_reindex.split(',');
logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds (using '--reindex')`);
await Common.sleep$(5000);
await databaseMigration.$truncateIndexedData(tables);
if (process.env.npm_config_reindex_blocks === 'true') { // Re-index requests
await databaseMigration.$blocksReindexingTruncate();
}
await databaseMigration.$initializeOrMigrateDatabase();
if (Common.indexingEnabled()) {

7
backend/src/repositories/BlocksRepository.ts
View File

@@ -16,6 +16,9 @@ class BlocksRepository {
* Save indexed block data in the database
*/
public async $saveBlockInDatabase(block: BlockExtended) {
const truncatedCoinbaseSignature = block?.extras?.coinbaseSignature?.substring(0, 500);
const truncatedCoinbaseSignatureAscii = block?.extras?.coinbaseSignatureAscii?.substring(0, 500);
try {
const query = `INSERT INTO blocks(
height, hash, blockTimestamp, size,
@@ -65,7 +68,7 @@ class BlocksRepository {
block.extras.medianTimestamp,
block.extras.header,
block.extras.coinbaseAddress,
block.extras.coinbaseSignature,
truncatedCoinbaseSignature,
block.extras.utxoSetSize,
block.extras.utxoSetChange,
block.extras.avgTxSize,
@@ -78,7 +81,7 @@ class BlocksRepository {
block.extras.segwitTotalSize,
block.extras.segwitTotalWeight,
block.extras.medianFeeAmt,
block.extras.coinbaseSignatureAscii,
truncatedCoinbaseSignatureAscii,
];
await DB.query(query, params);

2
backend/src/repositories/PoolsRepository.ts
View File

@@ -99,7 +99,7 @@ class PoolsRepository {
rows[0].regexes = JSON.parse(rows[0].regexes);
}
if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
rows[0].addresses = []; // pools.json only contains mainnet addresses
rows[0].addresses = []; // pools-v2.json only contains mainnet addresses
} else if (parse) {
rows[0].addresses = JSON.parse(rows[0].addresses);
}

2
backend/src/tasks/lightning/network-sync.service.ts
View File

@@ -72,7 +72,7 @@ class NetworkSyncService {
const graphNodesPubkeys: string[] = [];
for (const node of nodes) {
const latestUpdated = await channelsApi.$getLatestChannelUpdateForNode(node.pub_key);
node.last_update = Math.max(node.last_update, latestUpdated);
node.last_update = Math.max(node.last_update ?? 0, latestUpdated);
await nodesApi.$saveNode(node);
graphNodesPubkeys.push(node.pub_key);

26
backend/src/tasks/pools-updater.ts
View File

@@ -17,11 +17,6 @@ class PoolsUpdater {
treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL;
public async updatePoolsJson(): Promise<void> {
if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) {
logger.info(`Not updating mining pools to avoid inconsistency because AUTOMATIC_BLOCK_REINDEXING is set to false`)
return;
}
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
return;
}
@@ -36,12 +31,6 @@ class PoolsUpdater {
this.lastRun = now;
if (config.SOCKS5PROXY.ENABLED) {
logger.info(`Updating latest mining pools from ${this.poolsUrl} over the Tor network`, logger.tags.mining);
} else {
logger.info(`Updating latest mining pools from ${this.poolsUrl} over clearnet`, logger.tags.mining);
}
try {
const githubSha = await this.fetchPoolsSha(); // Fetch pools-v2.json sha from github
if (githubSha === undefined) {
@@ -57,10 +46,21 @@ class PoolsUpdater {
return;
}
// See backend README for more details about the mining pools update process
if (this.currentSha !== undefined && // If we don't have any mining pool, download it at least once
config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING !== true && // Automatic pools update is disabled
!process.env.npm_config_update_pools // We're not manually updating mining pool
) {
logger.warn(`Updated mining pools data is available (${githubSha}) but AUTOMATIC_BLOCK_REINDEXING is disabled`);
logger.info(`You can update your mining pools using the --update-pools command flag. You may want to clear your nginx cache as well if applicable`);
return;
}
const network = config.SOCKS5PROXY.ENABLED ? 'tor' : 'clearnet';
if (this.currentSha === undefined) {
logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl}`, logger.tags.mining);
logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining);
} else {
logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl}`, logger.tags.mining);
logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
}
const poolsJson = await this.query(this.poolsUrl);
if (poolsJson === undefined) {