machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into nymkappa/bugfix/ignore-too-low-lightning-timestamps

Browse Source
This commit is contained in:
nymkappa
2023-03-01 11:08:43 +09:00
committed by GitHub
parent 73a3b35296 09fd5c0a48
commit c2dff72387
48 changed files with 5848 additions and 2670 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/__fixtures__/mempool-config.template.json
View File

@@ -7,7 +7,7 @@
"HTTP_PORT": 1,
"SPAWN_CLUSTER_PROCS": 2,
"API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
"AUTOMATIC_BLOCK_REINDEXING": true,
"AUTOMATIC_BLOCK_REINDEXING": false,
"POLL_RATE_MS": 3,
"CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
"CLEAR_PROTECTION_MINUTES": 4,

3
backend/src/api/audit.ts
View File

@@ -119,7 +119,8 @@ class Audit {
}
const numCensored = Object.keys(isCensored).length;
const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0;
const numMatches = matches.length - 1; // adjust for coinbase tx
const score = numMatches > 0 ? (numMatches / (numMatches + numCensored)) : 0;
return {
censored: Object.keys(isCensored),

28
backend/src/api/database-migration.ts
View File

@@ -1017,26 +1017,16 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
public async $truncateIndexedData(tables: string[]) {
const allowedTables = ['blocks', 'hashrates', 'prices'];
public async $blocksReindexingTruncate(): Promise<void> {
logger.warn(`Truncating pools, blocks and hashrates for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
await Common.sleep$(5000);
try {
for (const table of tables) {
if (!allowedTables.includes(table)) {
logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`);
continue;
}
await this.$executeQuery(`TRUNCATE ${table}`, true);
if (table === 'hashrates') {
await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
}
logger.notice(`Table ${table} has been truncated`);
}
} catch (e) {
logger.warn(`Unable to erase indexed data`);
}
}
await this.$executeQuery(`TRUNCATE blocks`);
await this.$executeQuery(`TRUNCATE hashrates`);
await this.$executeQuery('DELETE FROM `pools`');
await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
}
private async $convertCompactCpfpTables(): Promise<void> {
try {

4
backend/src/api/pools-parser.ts
View File

@@ -73,7 +73,7 @@ class PoolsParser {
}
}
logger.info('Mining pools.json import completed');
logger.info('Mining pools-v2.json import completed');
}
/**
@@ -115,7 +115,7 @@ class PoolsParser {
return;
}
// Get oldest blocks mined by the pool and assume pools.json updates only concern most recent years
// Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years
// Ignore early days of Bitcoin as there were no mining pool yet
const [oldestPoolBlock]: any[] = await DB.query(`
SELECT height

8
backend/src/index.ts
View File

@@ -36,7 +36,6 @@ import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
import forensicsService from './tasks/lightning/forensics.service';
import priceUpdater from './tasks/price-updater';
import mining from './api/mining/mining';
import chainTips from './api/chain-tips';
import { AxiosError } from 'axios';
@@ -84,11 +83,8 @@ class Server {
if (config.DATABASE.ENABLED) {
await DB.checkDbConnection();
try {
if (process.env.npm_config_reindex !== undefined) { // Re-index requests
const tables = process.env.npm_config_reindex.split(',');
logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds (using '--reindex')`);
await Common.sleep$(5000);
await databaseMigration.$truncateIndexedData(tables);
if (process.env.npm_config_reindex_blocks === 'true') { // Re-index requests
await databaseMigration.$blocksReindexingTruncate();
}
await databaseMigration.$initializeOrMigrateDatabase();
if (Common.indexingEnabled()) {

7
backend/src/repositories/BlocksRepository.ts
View File

@@ -16,6 +16,9 @@ class BlocksRepository {
* Save indexed block data in the database
*/
public async $saveBlockInDatabase(block: BlockExtended) {
const truncatedCoinbaseSignature = block?.extras?.coinbaseSignature?.substring(0, 500);
const truncatedCoinbaseSignatureAscii = block?.extras?.coinbaseSignatureAscii?.substring(0, 500);
try {
const query = `INSERT INTO blocks(
height, hash, blockTimestamp, size,
@@ -65,7 +68,7 @@ class BlocksRepository {
block.extras.medianTimestamp,
block.extras.header,
block.extras.coinbaseAddress,
block.extras.coinbaseSignature,
truncatedCoinbaseSignature,
block.extras.utxoSetSize,
block.extras.utxoSetChange,
block.extras.avgTxSize,
@@ -78,7 +81,7 @@ class BlocksRepository {
block.extras.segwitTotalSize,
block.extras.segwitTotalWeight,
block.extras.medianFeeAmt,
block.extras.coinbaseSignatureAscii,
truncatedCoinbaseSignatureAscii,
];
await DB.query(query, params);

2
backend/src/repositories/PoolsRepository.ts
View File

@@ -99,7 +99,7 @@ class PoolsRepository {
rows[0].regexes = JSON.parse(rows[0].regexes);
}
if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
rows[0].addresses = []; // pools.json only contains mainnet addresses
rows[0].addresses = []; // pools-v2.json only contains mainnet addresses
} else if (parse) {
rows[0].addresses = JSON.parse(rows[0].addresses);
}

26
backend/src/tasks/pools-updater.ts
View File

@@ -17,11 +17,6 @@ class PoolsUpdater {
treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL;
public async updatePoolsJson(): Promise<void> {
if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) {
logger.info(`Not updating mining pools to avoid inconsistency because AUTOMATIC_BLOCK_REINDEXING is set to false`)
return;
}
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
return;
}
@@ -36,12 +31,6 @@ class PoolsUpdater {
this.lastRun = now;
if (config.SOCKS5PROXY.ENABLED) {
logger.info(`Updating latest mining pools from ${this.poolsUrl} over the Tor network`, logger.tags.mining);
} else {
logger.info(`Updating latest mining pools from ${this.poolsUrl} over clearnet`, logger.tags.mining);
}
try {
const githubSha = await this.fetchPoolsSha(); // Fetch pools-v2.json sha from github
if (githubSha === undefined) {
@@ -57,10 +46,21 @@ class PoolsUpdater {
return;
}
// See backend README for more details about the mining pools update process
if (this.currentSha !== undefined && // If we don't have any mining pool, download it at least once
config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING !== true && // Automatic pools update is disabled
!process.env.npm_config_update_pools // We're not manually updating mining pool
) {
logger.warn(`Updated mining pools data is available (${githubSha}) but AUTOMATIC_BLOCK_REINDEXING is disabled`);
logger.info(`You can update your mining pools using the --update-pools command flag. You may want to clear your nginx cache as well if applicable`);
return;
}
const network = config.SOCKS5PROXY.ENABLED ? 'tor' : 'clearnet';
if (this.currentSha === undefined) {
logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl}`, logger.tags.mining);
logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining);
} else {
logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl}`, logger.tags.mining);
logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
}
const poolsJson = await this.query(this.poolsUrl);
if (poolsJson === undefined) {