machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into nymkappa/bugfix/node-map

Browse Source
This commit is contained in:
wiz
2022-12-01 16:55:22 +09:00
committed by GitHub
parent 1461cb1b17 f6813f1d1c
commit 13b52c427c
110 changed files with 8382 additions and 9459 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/mempool-config.sample.json
View File

@@ -25,7 +25,9 @@
"AUTOMATIC_BLOCK_REINDEXING": false,
"POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
"POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
"ADVANCED_TRANSACTION_SELECTION": false
"ADVANCED_GBT_AUDIT": false,
"ADVANCED_GBT_MEMPOOL": false,
"TRANSACTION_INDEXING": false
},
"CORE_RPC": {
"HOST": "127.0.0.1",

4
backend/src/__fixtures__/mempool-config.template.json
View File

@@ -26,7 +26,9 @@
"INDEXING_BLOCKS_AMOUNT": 14,
"POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
"POOLS_JSON_URL": "__POOLS_JSON_URL__",
"ADVANCED_TRANSACTION_SELECTION": "__ADVANCED_TRANSACTION_SELECTION__"
"ADVANCED_GBT_AUDIT": "__ADVANCED_GBT_AUDIT__",
"ADVANCED_GBT_MEMPOOL": "__ADVANCED_GBT_MEMPOOL__",
"TRANSACTION_INDEXING": "__TRANSACTION_INDEXING__"
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",

4
backend/src/__tests__/config.test.ts
View File

@@ -38,7 +38,9 @@ describe('Mempool Backend Config', () => {
STDOUT_LOG_MIN_PRIORITY: 'debug',
POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
ADVANCED_TRANSACTION_SELECTION: false,
ADVANCED_GBT_AUDIT: false,
ADVANCED_GBT_MEMPOOL: false,
TRANSACTION_INDEXING: false,
});
expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });

54
backend/src/api/audit.ts
View File

@@ -10,9 +10,9 @@ const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first
class Audit {
auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended })
: { censored: string[], added: string[], score: number } {
: { censored: string[], added: string[], fresh: string[], score: number } {
if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
return { censored: [], added: [], score: 0 };
return { censored: [], added: [], fresh: [], score: 0 };
}
const matches: string[] = []; // present in both mined block and template
@@ -83,7 +83,17 @@ class Audit {
} else {
if (!isDisplaced[tx.txid]) {
added.push(tx.txid);
} else {
}
let blockIndex = -1;
let index = -1;
projectedBlocks.forEach((block, bi) => {
const i = block.transactionIds.indexOf(tx.txid);
if (i >= 0) {
blockIndex = bi;
index = i;
}
});
overflowWeight += tx.weight;
}
totalWeight += tx.weight;
@@ -119,48 +129,10 @@ class Audit {
return {
censored: Object.keys(isCensored),
added,
fresh,
score
};
}
public async $getBlockAuditScores(fromHeight?: number, limit: number = 15): Promise<AuditScore[]> {
let currentHeight = fromHeight !== undefined ? fromHeight : await blocksRepository.$mostRecentBlockHeight();
const returnScores: AuditScore[] = [];
if (currentHeight < 0) {
return returnScores;
}
for (let i = 0; i < limit && currentHeight >= 0; i++) {
const block = blocks.getBlocks().find((b) => b.height === currentHeight);
if (block?.extras?.matchRate != null) {
returnScores.push({
hash: block.id,
matchRate: block.extras.matchRate
});
} else {
let currentHash;
if (!currentHash && Common.indexingEnabled()) {
const dbBlock = await blocksRepository.$getBlockByHeight(currentHeight);
if (dbBlock && dbBlock['id']) {
currentHash = dbBlock['id'];
}
}
if (!currentHash) {
currentHash = await bitcoinApi.$getBlockHash(currentHeight);
}
if (currentHash) {
const auditScore = await blocksAuditsRepository.$getBlockAuditScore(currentHash);
returnScores.push({
hash: currentHash,
matchRate: auditScore?.matchRate
});
}
}
currentHeight--;
}
return returnScores;
}
}
export default new Audit();

2
backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts
View File

@@ -10,7 +10,7 @@ export interface AbstractBitcoinApi {
$getBlockHash(height: number): Promise<string>;
$getBlockHeader(hash: string): Promise<string>;
$getBlock(hash: string): Promise<IEsploraApi.Block>;
$getRawBlock(hash: string): Promise<string>;
$getRawBlock(hash: string): Promise<Buffer>;
$getAddress(address: string): Promise<IEsploraApi.Address>;
$getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
$getAddressPrefix(prefix: string): string[];

2
backend/src/api/bitcoin/bitcoin-api.ts
View File

@@ -81,7 +81,7 @@ class BitcoinApi implements AbstractBitcoinApi {
.then((rpcBlock: IBitcoinApi.Block) => rpcBlock.tx);
}
$getRawBlock(hash: string): Promise<string> {
$getRawBlock(hash: string): Promise<Buffer> {
return this.bitcoindClient.getBlock(hash, 0)
.then((raw: string) => Buffer.from(raw, "hex"));
}

55
backend/src/api/bitcoin/bitcoin.routes.ts
View File

@@ -17,13 +17,14 @@ import logger from '../../logger';
import blocks from '../blocks';
import bitcoinClient from './bitcoin-client';
import difficultyAdjustment from '../difficulty-adjustment';
import transactionRepository from '../../repositories/TransactionRepository';
class BitcoinRoutes {
public initRoutes(app: Application) {
app
.get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', this.getTransactionTimes)
.get(config.MEMPOOL.API_URL_PREFIX + 'outspends', this.$getBatchedOutspends)
.get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.getCpfpInfo)
.get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.$getCpfpInfo)
.get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', this.getDifficultyChange)
.get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', this.getRecommendedFees)
.get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', this.getMempoolBlocks)
@@ -89,6 +90,7 @@ class BitcoinRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', this.getBlocks.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', this.getBlock)
.get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
.post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
;
@@ -187,29 +189,36 @@ class BitcoinRoutes {
}
}
private getCpfpInfo(req: Request, res: Response) {
private async $getCpfpInfo(req: Request, res: Response) {
if (!/^[a-fA-F0-9]{64}$/.test(req.params.txId)) {
res.status(501).send(`Invalid transaction ID.`);
return;
}
const tx = mempool.getMempool()[req.params.txId];
if (!tx) {
res.status(404).send(`Transaction doesn't exist in the mempool.`);
if (tx) {
if (tx?.cpfpChecked) {
res.json({
ancestors: tx.ancestors,
bestDescendant: tx.bestDescendant || null,
descendants: tx.descendants || null,
effectiveFeePerVsize: tx.effectiveFeePerVsize || null,
});
return;
}
const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool());
res.json(cpfpInfo);
return;
} else {
const cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
if (cpfpInfo) {
res.json(cpfpInfo);
return;
}
}
if (tx.cpfpChecked) {
res.json({
ancestors: tx.ancestors,
bestDescendant: tx.bestDescendant || null,
});
return;
}
const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool());
res.json(cpfpInfo);
res.status(404).send(`Transaction has no CPFP info available.`);
}
private getBackendInfo(req: Request, res: Response) {
@@ -324,6 +333,16 @@ class BitcoinRoutes {
}
}
private async getStrippedBlockTransactions(req: Request, res: Response) {
try {
const transactions = await blocks.$getStrippedBlockTransactions(req.params.hash);
res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
res.json(transactions);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async getBlock(req: Request, res: Response) {
try {
const block = await blocks.$getBlock(req.params.hash);
@@ -356,9 +375,9 @@ class BitcoinRoutes {
}
}
private async getStrippedBlockTransactions(req: Request, res: Response) {
private async getBlockAuditSummary(req: Request, res: Response) {
try {
const transactions = await blocks.$getStrippedBlockTransactions(req.params.hash);
const transactions = await blocks.$getBlockAuditSummary(req.params.hash);
res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
res.json(transactions);
} catch (e) {

6
backend/src/api/bitcoin/esplora-api.ts
View File

@@ -55,9 +55,9 @@ class ElectrsApi implements AbstractBitcoinApi {
.then((response) => response.data);
}
$getRawBlock(hash: string): Promise<string> {
return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", this.axiosConfig)
.then((response) => response.data);
$getRawBlock(hash: string): Promise<Buffer> {
return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
.then((response) => { return Buffer.from(response.data); });
}
$getAddress(address: string): Promise<IEsploraApi.Address> {

141
backend/src/api/blocks.ts
View File

@@ -21,10 +21,13 @@ import fiatConversion from './fiat-conversion';
import poolsParser from './pools-parser';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
import cpfpRepository from '../repositories/CpfpRepository';
import transactionRepository from '../repositories/TransactionRepository';
import mining from './mining/mining';
import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
import PricesRepository from '../repositories/PricesRepository';
import priceUpdater from '../tasks/price-updater';
import { Block } from 'bitcoinjs-lib';
class Blocks {
private blocks: BlockExtended[] = [];
@@ -260,7 +263,7 @@ class Blocks {
/**
* [INDEXING] Index all blocks summaries for the block txs visualization
*/
public async $generateBlocksSummariesDatabase() {
public async $generateBlocksSummariesDatabase(): Promise<void> {
if (Common.blocksSummariesIndexingEnabled() === false) {
return;
}
@@ -316,6 +319,57 @@ class Blocks {
}
}
/**
* [INDEXING] Index transaction CPFP data for all blocks
*/
public async $generateCPFPDatabase(): Promise<void> {
if (Common.cpfpIndexingEnabled() === false) {
return;
}
try {
// Get all indexed block hash
const unindexedBlocks = await blocksRepository.$getCPFPUnindexedBlocks();
if (!unindexedBlocks?.length) {
return;
}
// Logging
let count = 0;
let countThisRun = 0;
let timer = new Date().getTime() / 1000;
const startedAt = new Date().getTime() / 1000;
for (const block of unindexedBlocks) {
// Logging
const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
if (elapsedSeconds > 5) {
const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
const blockPerSeconds = Math.max(1, countThisRun / elapsedSeconds);
const progress = Math.round(count / unindexedBlocks.length * 10000) / 100;
logger.debug(`Indexing cpfp clusters for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`);
timer = new Date().getTime() / 1000;
countThisRun = 0;
}
await this.$indexCPFP(block.hash, block.height); // Calculate and save CPFP data for transactions in this block
// Logging
count++;
countThisRun++;
}
if (count > 0) {
logger.notice(`CPFP indexing completed: indexed ${count} blocks`);
} else {
logger.debug(`CPFP indexing completed: indexed ${count} blocks`);
}
} catch (e) {
logger.err(`CPFP indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
throw e;
}
}
/**
* [INDEXING] Index all blocks metadata for the mining dashboard
*/
@@ -359,7 +413,7 @@ class Blocks {
}
++indexedThisRun;
++totalIndexed;
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) {
const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
@@ -461,9 +515,13 @@ class Blocks {
await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
await HashratesRepository.$deleteLastEntries();
await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
await cpfpRepository.$deleteClustersFrom(lastBlock['height'] - 10);
for (let i = 10; i >= 0; --i) {
const newBlock = await this.$indexBlock(lastBlock['height'] - i);
await this.$getStrippedBlockTransactions(newBlock.id, true, true);
if (config.MEMPOOL.TRANSACTION_INDEXING) {
await this.$indexCPFP(newBlock.id, lastBlock['height'] - i);
}
}
await mining.$indexDifficultyAdjustments();
await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
@@ -489,6 +547,9 @@ class Blocks {
if (Common.blocksSummariesIndexingEnabled() === true) {
await this.$getStrippedBlockTransactions(blockExtended.id, true);
}
if (config.MEMPOOL.TRANSACTION_INDEXING) {
this.$indexCPFP(blockExtended.id, this.currentBlockHeight);
}
}
}
@@ -590,7 +651,7 @@ class Blocks {
if (skipMemoryCache === false) {
// Check the memory cache
const cachedSummary = this.getBlockSummaries().find((b) => b.id === hash);
if (cachedSummary) {
if (cachedSummary?.transactions?.length) {
return cachedSummary.transactions;
}
}
@@ -598,7 +659,7 @@ class Blocks {
// Check if it's indexed in db
if (skipDBLookup === false && Common.blocksSummariesIndexingEnabled() === true) {
const indexedSummary = await BlocksSummariesRepository.$getByBlockId(hash);
if (indexedSummary !== undefined) {
if (indexedSummary !== undefined && indexedSummary?.transactions?.length) {
return indexedSummary.transactions;
}
}
@@ -651,6 +712,22 @@ class Blocks {
return returnBlocks;
}
public async $getBlockAuditSummary(hash: string): Promise<any> {
let summary;
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
summary = await BlocksAuditsRepository.$getBlockAudit(hash);
}
// fallback to non-audited transaction summary
if (!summary?.transactions?.length) {
const strippedTransactions = await this.$getStrippedBlockTransactions(hash);
summary = {
transactions: strippedTransactions
};
}
return summary;
}
public getLastDifficultyAdjustmentTime(): number {
return this.lastDifficultyAdjustmentTime;
}
@@ -662,6 +739,62 @@ class Blocks {
public getCurrentBlockHeight(): number {
return this.currentBlockHeight;
}
public async $indexCPFP(hash: string, height: number): Promise<void> {
let transactions;
if (false/*Common.blocksSummariesIndexingEnabled()*/) {
transactions = await this.$getStrippedBlockTransactions(hash);
const rawBlock = await bitcoinApi.$getRawBlock(hash);
const block = Block.fromBuffer(rawBlock);
const txMap = {};
for (const tx of block.transactions || []) {
txMap[tx.getId()] = tx;
}
for (const tx of transactions) {
if (txMap[tx.txid]?.ins) {
tx.vin = txMap[tx.txid].ins.map(vin => {
return {
txid: vin.hash
};
});
}
}
} else {
const block = await bitcoinClient.getBlock(hash, 2);
transactions = block.tx.map(tx => {
tx.vsize = tx.weight / 4;
return tx;
});
}
let cluster: TransactionStripped[] = [];
let ancestors: { [txid: string]: boolean } = {};
for (let i = transactions.length - 1; i >= 0; i--) {
const tx = transactions[i];
if (!ancestors[tx.txid]) {
let totalFee = 0;
let totalVSize = 0;
cluster.forEach(tx => {
totalFee += tx?.fee || 0;
totalVSize += tx.vsize;
});
const effectiveFeePerVsize = (totalFee * 100_000_000) / totalVSize;
if (cluster.length > 1) {
await cpfpRepository.$saveCluster(height, cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: (tx.fee || 0) * 100_000_000 }; }), effectiveFeePerVsize);
for (const tx of cluster) {
await transactionRepository.$setCluster(tx.txid, cluster[0].txid);
}
}
cluster = [];
ancestors = {};
}
cluster.push(tx);
tx.vin.forEach(vin => {
ancestors[vin.txid] = true;
});
}
await blocksRepository.$setCPFPIndexed(hash);
}
}
export default new Blocks();

7
backend/src/api/common.ts
View File

@@ -187,6 +187,13 @@ export class Common {
);
}
static cpfpIndexingEnabled(): boolean {
return (
Common.indexingEnabled() &&
config.MEMPOOL.TRANSACTION_INDEXING === true
);
}
static setDateMidnight(date: Date): void {
date.setUTCHours(0);
date.setUTCMinutes(0);

106
backend/src/api/database-migration.ts
View File

@@ -4,8 +4,8 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 44;
private queryTimeout = 900_000;
private static currentVersion = 49;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -107,18 +107,22 @@ class DatabaseMigration {
await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) {
await this.$executeQuery(`CREATE INDEX added ON statistics (added);`);
await this.updateToSchemaVersion(2);
}
if (databaseSchemaVersion < 3) {
await this.$executeQuery(this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
await this.updateToSchemaVersion(3);
}
if (databaseSchemaVersion < 4) {
await this.$executeQuery('DROP table IF EXISTS blocks;');
await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
await this.updateToSchemaVersion(4);
}
if (databaseSchemaVersion < 5 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
await this.updateToSchemaVersion(5);
}
if (databaseSchemaVersion < 6 && isBitcoin === true) {
@@ -141,11 +145,13 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
await this.$executeQuery('ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
await this.updateToSchemaVersion(6);
}
if (databaseSchemaVersion < 7 && isBitcoin === true) {
await this.$executeQuery('DROP table IF EXISTS hashrates;');
await this.$executeQuery(this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates'));
await this.updateToSchemaVersion(7);
}
if (databaseSchemaVersion < 8 && isBitcoin === true) {
@@ -155,6 +161,7 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
await this.$executeQuery('ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
await this.updateToSchemaVersion(8);
}
if (databaseSchemaVersion < 9 && isBitcoin === true) {
@@ -162,10 +169,12 @@ class DatabaseMigration {
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
await this.updateToSchemaVersion(9);
}
if (databaseSchemaVersion < 10 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
await this.updateToSchemaVersion(10);
}
if (databaseSchemaVersion < 11 && isBitcoin === true) {
@@ -178,11 +187,13 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
await this.updateToSchemaVersion(11);
}
if (databaseSchemaVersion < 12 && isBitcoin === true) {
// No need to re-index because the new data type can contain larger values
await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.updateToSchemaVersion(12);
}
if (databaseSchemaVersion < 13 && isBitcoin === true) {
@@ -190,6 +201,7 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.updateToSchemaVersion(13);
}
if (databaseSchemaVersion < 14 && isBitcoin === true) {
@@ -197,37 +209,45 @@ class DatabaseMigration {
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
await this.updateToSchemaVersion(14);
}
if (databaseSchemaVersion < 16 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps
await this.updateToSchemaVersion(16);
}
if (databaseSchemaVersion < 17 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `pools` ADD `slug` CHAR(50) NULL');
await this.updateToSchemaVersion(17);
}
if (databaseSchemaVersion < 18 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `hash` (`hash`);');
await this.updateToSchemaVersion(18);
}
if (databaseSchemaVersion < 19) {
await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates'));
await this.updateToSchemaVersion(19);
}
if (databaseSchemaVersion < 20 && isBitcoin === true) {
await this.$executeQuery(this.getCreateBlocksSummariesTableQuery(), await this.$checkIfTableExists('blocks_summaries'));
await this.updateToSchemaVersion(20);
}
if (databaseSchemaVersion < 21) {
await this.$executeQuery('DROP TABLE IF EXISTS `rates`');
await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices'));
await this.updateToSchemaVersion(21);
}
if (databaseSchemaVersion < 22 && isBitcoin === true) {
await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`');
await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments'));
await this.updateToSchemaVersion(22);
}
if (databaseSchemaVersion < 23) {
@@ -240,11 +260,13 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `prices` ADD `CHF` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `AUD` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `JPY` float DEFAULT "0"');
await this.updateToSchemaVersion(23);
}
if (databaseSchemaVersion < 24 && isBitcoin == true) {
await this.$executeQuery('DROP TABLE IF EXISTS `blocks_audits`');
await this.$executeQuery(this.getCreateBlocksAuditsTableQuery(), await this.$checkIfTableExists('blocks_audits'));
await this.updateToSchemaVersion(24);
}
if (databaseSchemaVersion < 25 && isBitcoin === true) {
@@ -252,6 +274,7 @@ class DatabaseMigration {
await this.$executeQuery(this.getCreateNodesQuery(), await this.$checkIfTableExists('nodes'));
await this.$executeQuery(this.getCreateChannelsQuery(), await this.$checkIfTableExists('channels'));
await this.$executeQuery(this.getCreateNodesStatsQuery(), await this.$checkIfTableExists('node_stats'));
await this.updateToSchemaVersion(25);
}
if (databaseSchemaVersion < 26 && isBitcoin === true) {
@@ -262,6 +285,7 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD tor_nodes int(11) NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_nodes int(11) NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD unannounced_nodes int(11) NOT NULL DEFAULT "0"');
await this.updateToSchemaVersion(26);
}
if (databaseSchemaVersion < 27 && isBitcoin === true) {
@@ -271,6 +295,7 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_capacity bigint(20) unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_fee_rate int(11) unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"');
await this.updateToSchemaVersion(27);
}
if (databaseSchemaVersion < 28 && isBitcoin === true) {
@@ -280,6 +305,7 @@ class DatabaseMigration {
await this.$executeQuery(`TRUNCATE lightning_stats`);
await this.$executeQuery(`TRUNCATE node_stats`);
await this.$executeQuery(`ALTER TABLE lightning_stats MODIFY added DATE`);
await this.updateToSchemaVersion(28);
}
if (databaseSchemaVersion < 29 && isBitcoin === true) {
@@ -291,41 +317,50 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `nodes` ADD subdivision_id int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD longitude double NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD latitude double NULL DEFAULT NULL');
await this.updateToSchemaVersion(29);
}
if (databaseSchemaVersion < 30 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization") NOT NULL');
await this.updateToSchemaVersion(30);
}
if (databaseSchemaVersion < 31 && isBitcoin == true) { // Link blocks to prices
await this.$executeQuery('ALTER TABLE `prices` ADD `id` int NULL AUTO_INCREMENT UNIQUE');
await this.$executeQuery('DROP TABLE IF EXISTS `blocks_prices`');
await this.$executeQuery(this.getCreateBlocksPricesTableQuery(), await this.$checkIfTableExists('blocks_prices'));
await this.updateToSchemaVersion(31);
}
if (databaseSchemaVersion < 32 && isBitcoin == true) {
await this.$executeQuery('ALTER TABLE `blocks_summaries` ADD `template` JSON DEFAULT "[]"');
await this.updateToSchemaVersion(32);
}
if (databaseSchemaVersion < 33 && isBitcoin == true) {
await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization", "country_iso_code") NOT NULL');
await this.updateToSchemaVersion(33);
}
if (databaseSchemaVersion < 34 && isBitcoin == true) {
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_tor_nodes int(11) NOT NULL DEFAULT "0"');
await this.updateToSchemaVersion(34);
}
if (databaseSchemaVersion < 35 && isBitcoin == true) {
await this.$executeQuery('DELETE from `lightning_stats` WHERE added > "2021-09-19"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD CONSTRAINT added_unique UNIQUE (added);');
await this.updateToSchemaVersion(35);
}
if (databaseSchemaVersion < 36 && isBitcoin == true) {
await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"');
await this.updateToSchemaVersion(36);
}
if (databaseSchemaVersion < 37 && isBitcoin == true) {
await this.$executeQuery(this.getCreateLNNodesSocketsTableQuery(), await this.$checkIfTableExists('nodes_sockets'));
await this.updateToSchemaVersion(37);
}
if (databaseSchemaVersion < 38 && isBitcoin == true) {
@@ -336,34 +371,76 @@ class DatabaseMigration {
await this.$executeQuery(`TRUNCATE node_stats`);
await this.$executeQuery('ALTER TABLE `lightning_stats` CHANGE `added` `added` timestamp NULL');
await this.$executeQuery('ALTER TABLE `node_stats` CHANGE `added` `added` timestamp NULL');
await this.updateToSchemaVersion(38);
}
if (databaseSchemaVersion < 39 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `nodes` ADD alias_search TEXT NULL DEFAULT NULL AFTER `alias`');
await this.$executeQuery('ALTER TABLE nodes ADD FULLTEXT(alias_search)');
await this.updateToSchemaVersion(39);
}
if (databaseSchemaVersion < 40 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `nodes` ADD capacity bigint(20) unsigned DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD channels int(11) unsigned DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD INDEX `capacity` (`capacity`);');
await this.updateToSchemaVersion(40);
}
if (databaseSchemaVersion < 41 && isBitcoin === true) {
await this.$executeQuery('UPDATE channels SET closing_reason = NULL WHERE closing_reason = 1');
await this.updateToSchemaVersion(41);
}
if (databaseSchemaVersion < 42 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `channels` ADD closing_resolved tinyint(1) DEFAULT 0');
await this.updateToSchemaVersion(42);
}
if (databaseSchemaVersion < 43 && isBitcoin === true) {
await this.$executeQuery(this.getCreateLNNodeRecordsTableQuery(), await this.$checkIfTableExists('nodes_records'));
await this.updateToSchemaVersion(43);
}
if (databaseSchemaVersion < 44 && isBitcoin === true) {
await this.$executeQuery('TRUNCATE TABLE `blocks_audits`');
await this.$executeQuery('UPDATE blocks_summaries SET template = NULL');
await this.updateToSchemaVersion(44);
}
if (databaseSchemaVersion < 45 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fresh_txs JSON DEFAULT "[]"');
await this.updateToSchemaVersion(45);
}
if (databaseSchemaVersion < 46) {
await this.$executeQuery(`ALTER TABLE blocks MODIFY blockTimestamp timestamp NOT NULL DEFAULT 0`);
await this.updateToSchemaVersion(46);
}
if (databaseSchemaVersion < 47) {
await this.$executeQuery('ALTER TABLE `blocks` ADD cpfp_indexed tinyint(1) DEFAULT 0');
await this.$executeQuery(this.getCreateCPFPTableQuery(), await this.$checkIfTableExists('cpfp_clusters'));
await this.$executeQuery(this.getCreateTransactionsTableQuery(), await this.$checkIfTableExists('transactions'));
await this.updateToSchemaVersion(47);
}
if (databaseSchemaVersion < 48 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `channels` ADD source_checked tinyint(1) DEFAULT 0');
await this.$executeQuery('ALTER TABLE `channels` ADD closing_fee bigint(20) unsigned DEFAULT 0');
await this.$executeQuery('ALTER TABLE `channels` ADD node1_funding_balance bigint(20) unsigned DEFAULT 0');
await this.$executeQuery('ALTER TABLE `channels` ADD node2_funding_balance bigint(20) unsigned DEFAULT 0');
await this.$executeQuery('ALTER TABLE `channels` ADD node1_closing_balance bigint(20) unsigned DEFAULT 0');
await this.$executeQuery('ALTER TABLE `channels` ADD node2_closing_balance bigint(20) unsigned DEFAULT 0');
await this.$executeQuery('ALTER TABLE `channels` ADD funding_ratio float unsigned DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `channels` ADD closed_by varchar(66) DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `channels` ADD single_funded tinyint(1) DEFAULT 0');
await this.$executeQuery('ALTER TABLE `channels` ADD outputs JSON DEFAULT "[]"');
await this.updateToSchemaVersion(48);
}
if (databaseSchemaVersion < 49 && isBitcoin === true) {
await this.$executeQuery('TRUNCATE TABLE `blocks_audits`');
await this.updateToSchemaVersion(49);
}
}
@@ -502,6 +579,10 @@ class DatabaseMigration {
return `UPDATE state SET number = ${DatabaseMigration.currentVersion} WHERE name = 'schema_version';`;
}
private async updateToSchemaVersion(version): Promise<void> {
await this.$executeQuery(`UPDATE state SET number = ${version} WHERE name = 'schema_version';`);
}
/**
* Print current database version
*/
@@ -813,6 +894,25 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateCPFPTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS cpfp_clusters (
root varchar(65) NOT NULL,
height int(10) NOT NULL,
txs JSON DEFAULT NULL,
fee_rate double unsigned NOT NULL,
PRIMARY KEY (root)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateTransactionsTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS transactions (
txid varchar(65) NOT NULL,
cluster varchar(65) DEFAULT NULL,
PRIMARY KEY (txid),
FOREIGN KEY (cluster) REFERENCES cpfp_clusters (root) ON DELETE SET NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
public async $truncateIndexedData(tables: string[]) {
const allowedTables = ['blocks', 'hashrates', 'prices'];

127
backend/src/api/explorer/channels.api.ts
View File

@@ -128,6 +128,21 @@ class ChannelsApi {
}
}
public async $getChannelsWithoutSourceChecked(): Promise<any[]> {
try {
const query = `
SELECT channels.*
FROM channels
WHERE channels.source_checked != 1
`;
const [rows]: any = await DB.query(query);
return rows;
} catch (e) {
logger.err('$getUnresolvedClosedChannels error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getChannelsWithoutCreatedDate(): Promise<any[]> {
try {
const query = `SELECT * FROM channels WHERE created IS NULL`;
@@ -257,6 +272,108 @@ class ChannelsApi {
}
}
public async $getChannelByClosingId(transactionId: string): Promise<any> {
try {
const query = `
SELECT
channels.*
FROM channels
WHERE channels.closing_transaction_id = ?
`;
const [rows]: any = await DB.query(query, [transactionId]);
if (rows.length > 0) {
rows[0].outputs = JSON.parse(rows[0].outputs);
return rows[0];
}
} catch (e) {
logger.err('$getChannelByClosingId error: ' + (e instanceof Error ? e.message : e));
// don't throw - this data isn't essential
}
}
public async $getChannelsByOpeningId(transactionId: string): Promise<any> {
try {
const query = `
SELECT
channels.*
FROM channels
WHERE channels.transaction_id = ?
`;
const [rows]: any = await DB.query(query, [transactionId]);
if (rows.length > 0) {
return rows.map(row => {
row.outputs = JSON.parse(row.outputs);
return row;
});
}
} catch (e) {
logger.err('$getChannelsByOpeningId error: ' + (e instanceof Error ? e.message : e));
// don't throw - this data isn't essential
}
}
public async $updateClosingInfo(channelInfo: { id: string, node1_closing_balance: number, node2_closing_balance: number, closed_by: string | null, closing_fee: number, outputs: ILightningApi.ForensicOutput[]}): Promise<void> {
try {
const query = `
UPDATE channels SET
node1_closing_balance = ?,
node2_closing_balance = ?,
closed_by = ?,
closing_fee = ?,
outputs = ?
WHERE channels.id = ?
`;
await DB.query<ResultSetHeader>(query, [
channelInfo.node1_closing_balance || 0,
channelInfo.node2_closing_balance || 0,
channelInfo.closed_by,
channelInfo.closing_fee || 0,
JSON.stringify(channelInfo.outputs),
channelInfo.id,
]);
} catch (e) {
logger.err('$updateClosingInfo error: ' + (e instanceof Error ? e.message : e));
// don't throw - this data isn't essential
}
}
public async $updateOpeningInfo(channelInfo: { id: string, node1_funding_balance: number, node2_funding_balance: number, funding_ratio: number, single_funded: boolean | void }): Promise<void> {
try {
const query = `
UPDATE channels SET
node1_funding_balance = ?,
node2_funding_balance = ?,
funding_ratio = ?,
single_funded = ?
WHERE channels.id = ?
`;
await DB.query<ResultSetHeader>(query, [
channelInfo.node1_funding_balance || 0,
channelInfo.node2_funding_balance || 0,
channelInfo.funding_ratio,
channelInfo.single_funded ? 1 : 0,
channelInfo.id,
]);
} catch (e) {
logger.err('$updateOpeningInfo error: ' + (e instanceof Error ? e.message : e));
// don't throw - this data isn't essential
}
}
public async $markChannelSourceChecked(id: string): Promise<void> {
try {
const query = `
UPDATE channels
SET source_checked = 1
WHERE id = ?
`;
await DB.query<ResultSetHeader>(query, [id]);
} catch (e) {
logger.err('$markChannelSourceChecked error: ' + (e instanceof Error ? e.message : e));
// don't throw - this data isn't essential
}
}
public async $getChannelsForNode(public_key: string, index: number, length: number, status: string): Promise<any[]> {
try {
let channelStatusFilter;
@@ -385,11 +502,15 @@ class ChannelsApi {
'transaction_id': channel.transaction_id,
'transaction_vout': channel.transaction_vout,
'closing_transaction_id': channel.closing_transaction_id,
'closing_fee': channel.closing_fee,
'closing_reason': channel.closing_reason,
'closing_date': channel.closing_date,
'updated_at': channel.updated_at,
'created': channel.created,
'status': channel.status,
'funding_ratio': channel.funding_ratio,
'closed_by': channel.closed_by,
'single_funded': !!channel.single_funded,
'node_left': {
'alias': channel.alias_left,
'public_key': channel.node1_public_key,
@@ -404,6 +525,9 @@ class ChannelsApi {
'updated_at': channel.node1_updated_at,
'longitude': channel.node1_longitude,
'latitude': channel.node1_latitude,
'funding_balance': channel.node1_funding_balance,
'closing_balance': channel.node1_closing_balance,
'initiated_close': channel.closed_by === channel.node1_public_key ? true : undefined,
},
'node_right': {
'alias': channel.alias_right,
@@ -419,6 +543,9 @@ class ChannelsApi {
'updated_at': channel.node2_updated_at,
'longitude': channel.node2_longitude,
'latitude': channel.node2_latitude,
'funding_balance': channel.node2_funding_balance,
'closing_balance': channel.node2_closing_balance,
'initiated_close': channel.closed_by === channel.node2_public_key ? true : undefined,
},
};
}

6
backend/src/api/lightning/lightning-api.interface.ts
View File

@@ -83,4 +83,10 @@ export namespace ILightningApi {
is_required: boolean;
is_known: boolean;
}
export interface ForensicOutput {
node?: 1 | 2;
type: number;
value: number;
}
}

1
backend/src/api/mempool-blocks.ts
View File

@@ -155,6 +155,7 @@ class MempoolBlocks {
if (newMempool[txid] && mempool[txid]) {
newMempool[txid].effectiveFeePerVsize = mempool[txid].effectiveFeePerVsize;
newMempool[txid].ancestors = mempool[txid].ancestors;
newMempool[txid].descendants = mempool[txid].descendants;
newMempool[txid].bestDescendant = mempool[txid].bestDescendant;
newMempool[txid].cpfpChecked = mempool[txid].cpfpChecked;
}

7
backend/src/api/mining/mining-routes.ts
View File

@@ -283,9 +283,12 @@ class MiningRoutes {
private async $getBlockAuditScores(req: Request, res: Response) {
try {
const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
let height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
if (height == null) {
height = await BlocksRepository.$mostRecentBlockHeight();
}
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(await audits.$getBlockAuditScores(height, 15));
res.json(await BlocksAuditsRepository.$getBlockAuditScores(height, height - 15));
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}

26
backend/src/api/tx-selection-worker.ts
View File

@@ -108,36 +108,38 @@ function makeBlockTemplates({ mempool, blockLimit, weightLimit, condenseRest }:
if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
blockWeight += nextTx.ancestorWeight;
const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values());
const descendants: AuditTransaction[] = [];
// sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4);
sortedTxSet.forEach((ancestor, i, arr) => {
while (sortedTxSet.length) {
const ancestor = sortedTxSet.pop();
const mempoolTx = mempool[ancestor.txid];
if (ancestor && !ancestor?.used) {
ancestor.used = true;
// update original copy of this tx with effective fee rate & relatives data
mempoolTx.effectiveFeePerVsize = effectiveFeeRate;
mempoolTx.ancestors = (Array.from(ancestor.ancestorMap?.values()) as AuditTransaction[]).map((a) => {
mempoolTx.ancestors = sortedTxSet.map((a) => {
return {
txid: a.txid,
fee: a.fee,
weight: a.weight,
};
}).reverse();
mempoolTx.descendants = descendants.map((a) => {
return {
txid: a.txid,
fee: a.fee,
weight: a.weight,
};
});
descendants.push(ancestor);
mempoolTx.cpfpChecked = true;
if (i < arr.length - 1) {
mempoolTx.bestDescendant = {
txid: arr[arr.length - 1].txid,
fee: arr[arr.length - 1].fee,
weight: arr[arr.length - 1].weight,
};
} else {
mempoolTx.bestDescendant = null;
}
transactions.push(ancestor);
blockSize += ancestor.size;
}
});
}
// remove these as valid package ancestors for any descendants remaining in the mempool
if (sortedTxSet.length) {

16
backend/src/api/websocket-handler.ts
View File

@@ -250,12 +250,12 @@ class WebsocketHandler {
throw new Error('WebSocket.Server is not set');
}
if (config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) {
if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
await mempoolBlocks.makeBlockTemplates(newMempool, 8, null, true);
}
else {
} else {
mempoolBlocks.updateMempoolBlocks(newMempool);
}
const mBlocks = mempoolBlocks.getMempoolBlocks();
const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
const mempoolInfo = memPool.getMempoolInfo();
@@ -417,9 +417,8 @@ class WebsocketHandler {
}
const _memPool = memPool.getMempool();
let matchRate;
if (config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) {
if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
await mempoolBlocks.makeBlockTemplates(_memPool, 2);
} else {
mempoolBlocks.updateMempoolBlocks(_memPool);
@@ -428,8 +427,8 @@ class WebsocketHandler {
if (Common.indexingEnabled() && memPool.isInSync()) {
const projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
const { censored, added, score } = Audit.auditBlock(transactions, projectedBlocks, _memPool);
matchRate = Math.round(score * 100 * 100) / 100;
const { censored, added, fresh, score } = Audit.auditBlock(transactions, projectedBlocks, _memPool);
const matchRate = Math.round(score * 100 * 100) / 100;
const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
return {
@@ -454,6 +453,7 @@ class WebsocketHandler {
hash: block.id,
addedTxs: added,
missingTxs: censored,
freshTxs: fresh,
matchRate: matchRate,
});
@@ -467,7 +467,7 @@ class WebsocketHandler {
delete _memPool[txId];
}
if (config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) {
if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
await mempoolBlocks.makeBlockTemplates(_memPool, 2);
} else {
mempoolBlocks.updateMempoolBlocks(_memPool);

8
backend/src/config.ts
View File

@@ -29,7 +29,9 @@ interface IConfig {
AUTOMATIC_BLOCK_REINDEXING: boolean;
POOLS_JSON_URL: string,
POOLS_JSON_TREE_URL: string,
ADVANCED_TRANSACTION_SELECTION: boolean;
ADVANCED_GBT_AUDIT: boolean;
ADVANCED_GBT_MEMPOOL: boolean;
TRANSACTION_INDEXING: boolean;
};
ESPLORA: {
REST_API_URL: string;
@@ -147,7 +149,9 @@ const defaults: IConfig = {
'AUTOMATIC_BLOCK_REINDEXING': false,
'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
'ADVANCED_TRANSACTION_SELECTION': false,
'ADVANCED_GBT_AUDIT': false,
'ADVANCED_GBT_MEMPOOL': false,
'TRANSACTION_INDEXING': false,
},
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',

1
backend/src/indexer.ts
View File

@@ -77,6 +77,7 @@ class Indexer {
await mining.$generateNetworkHashrateHistory();
await mining.$generatePoolHashrateHistory();
await blocks.$generateBlocksSummariesDatabase();
await blocks.$generateCPFPDatabase();
} catch (e) {
this.indexerRunning = false;
logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));

6
backend/src/mempool.interfaces.ts
View File

@@ -28,6 +28,7 @@ export interface BlockAudit {
height: number,
hash: string,
missingTxs: string[],
freshTxs: string[],
addedTxs: string[],
matchRate: number,
}
@@ -71,6 +72,7 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
firstSeen?: number;
effectiveFeePerVsize: number;
ancestors?: Ancestor[];
descendants?: Ancestor[];
bestDescendant?: BestDescendant | null;
cpfpChecked?: boolean;
deleteAfter?: number;
@@ -118,7 +120,9 @@ interface BestDescendant {
export interface CpfpInfo {
ancestors: Ancestor[];
bestDescendant: BestDescendant | null;
bestDescendant?: BestDescendant | null;
descendants?: Ancestor[];
effectiveFeePerVsize?: number;
}
export interface TransactionStripped {

31
backend/src/repositories/BlocksAuditsRepository.ts
View File

@@ -1,3 +1,4 @@
import blocks from '../api/blocks';
import DB from '../database';
import logger from '../logger';
import { BlockAudit, AuditScore } from '../mempool.interfaces';
@@ -5,9 +6,9 @@ import { BlockAudit, AuditScore } from '../mempool.interfaces';
class BlocksAuditRepositories {
public async $saveAudit(audit: BlockAudit): Promise<void> {
try {
await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, match_rate)
VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
JSON.stringify(audit.addedTxs), audit.matchRate]);
await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, match_rate)
VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), audit.matchRate]);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`);
@@ -51,7 +52,7 @@ class BlocksAuditRepositories {
const [rows]: any[] = await DB.query(
`SELECT blocks.height, blocks.hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.size,
blocks.weight, blocks.tx_count,
transactions, template, missing_txs as missingTxs, added_txs as addedTxs, match_rate as matchRate
transactions, template, missing_txs as missingTxs, added_txs as addedTxs, fresh_txs as freshTxs, match_rate as matchRate
FROM blocks_audits
JOIN blocks ON blocks.hash = blocks_audits.hash
JOIN blocks_summaries ON blocks_summaries.id = blocks_audits.hash
@@ -61,11 +62,15 @@ class BlocksAuditRepositories {
if (rows.length) {
rows[0].missingTxs = JSON.parse(rows[0].missingTxs);
rows[0].addedTxs = JSON.parse(rows[0].addedTxs);
rows[0].freshTxs = JSON.parse(rows[0].freshTxs);
rows[0].transactions = JSON.parse(rows[0].transactions);
rows[0].template = JSON.parse(rows[0].template);
if (rows[0].transactions.length) {
return rows[0];
}
}
return rows[0];
return null;
} catch (e: any) {
logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
@@ -85,6 +90,20 @@ class BlocksAuditRepositories {
throw e;
}
}
public async $getBlockAuditScores(maxHeight: number, minHeight: number): Promise<AuditScore[]> {
try {
const [rows]: any[] = await DB.query(
`SELECT hash, match_rate as matchRate
FROM blocks_audits
WHERE blocks_audits.height BETWEEN ? AND ?
`, [minHeight, maxHeight]);
return rows;
} catch (e: any) {
logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
}
export default new BlocksAuditRepositories();

17
backend/src/repositories/BlocksRepository.ts
View File

@@ -662,6 +662,23 @@ class BlocksRepository {
}
}
/**
* Get a list of blocks that have not had CPFP data indexed
*/
public async $getCPFPUnindexedBlocks(): Promise<any[]> {
try {
const [rows]: any = await DB.query(`SELECT height, hash FROM blocks WHERE cpfp_indexed = 0 ORDER BY height DESC`);
return rows;
} catch (e) {
logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $setCPFPIndexed(hash: string): Promise<void> {
await DB.query(`UPDATE blocks SET cpfp_indexed = 1 WHERE hash = ?`, [hash]);
}
/**
* Return the oldest block from a consecutive chain of block from the most recent one
*/

43
backend/src/repositories/CpfpRepository.ts Normal file
View File

@@ -0,0 +1,43 @@
import DB from '../database';
import logger from '../logger';
import { Ancestor } from '../mempool.interfaces';
class CpfpRepository {
public async $saveCluster(height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise<void> {
try {
const txsJson = JSON.stringify(txs);
await DB.query(
`
INSERT INTO cpfp_clusters(root, height, txs, fee_rate)
VALUE (?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
height = ?,
txs = ?,
fee_rate = ?
`,
[txs[0].txid, height, txsJson, effectiveFeePerVsize, height, txsJson, effectiveFeePerVsize, height]
);
} catch (e: any) {
logger.err(`Cannot save cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $deleteClustersFrom(height: number): Promise<void> {
logger.info(`Delete newer cpfp clusters from height ${height} from the database`);
try {
await DB.query(
`
DELETE from cpfp_clusters
WHERE height >= ?
`,
[height]
);
} catch (e: any) {
logger.err(`Cannot delete cpfp clusters from db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
}
export default new CpfpRepository();

77
backend/src/repositories/TransactionRepository.ts Normal file
View File

@@ -0,0 +1,77 @@
import DB from '../database';
import logger from '../logger';
import { Ancestor, CpfpInfo } from '../mempool.interfaces';
interface CpfpSummary {
txid: string;
cluster: string;
root: string;
txs: Ancestor[];
height: number;
fee_rate: number;
}
class TransactionRepository {
public async $setCluster(txid: string, cluster: string): Promise<void> {
try {
await DB.query(
`
INSERT INTO transactions
(
txid,
cluster
)
VALUE (?, ?)
ON DUPLICATE KEY UPDATE
cluster = ?
;`,
[txid, cluster, cluster]
);
} catch (e: any) {
logger.err(`Cannot save transaction cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getCpfpInfo(txid: string): Promise<CpfpInfo | void> {
try {
let query = `
SELECT *
FROM transactions
LEFT JOIN cpfp_clusters AS cluster ON cluster.root = transactions.cluster
WHERE transactions.txid = ?
`;
const [rows]: any = await DB.query(query, [txid]);
if (rows.length) {
rows[0].txs = JSON.parse(rows[0].txs) as Ancestor[];
return this.convertCpfp(rows[0]);
}
} catch (e) {
logger.err('Cannot get transaction cpfp info from db. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
private convertCpfp(cpfp: CpfpSummary): CpfpInfo {
const descendants: Ancestor[] = [];
const ancestors: Ancestor[] = [];
let matched = false;
for (const tx of cpfp.txs) {
if (tx.txid === cpfp.txid) {
matched = true;
} else if (!matched) {
descendants.push(tx);
} else {
ancestors.push(tx);
}
}
return {
descendants,
ancestors,
effectiveFeePerVsize: cpfp.fee_rate
};
}
}
export default new TransactionRepository();

273
backend/src/tasks/lightning/forensics.service.ts
View File

@@ -5,13 +5,16 @@ import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
import config from '../../config';
import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
import { Common } from '../../api/common';
import { ILightningApi } from '../../api/lightning/lightning-api.interface';
const throttleDelay = 20; //ms
const tempCacheSize = 10000;
class ForensicsService {
loggerTimer = 0;
closedChannelsScanBlock = 0;
txCache: { [txid: string]: IEsploraApi.Transaction } = {};
tempCached: string[] = [];
constructor() {}
@@ -29,6 +32,7 @@ class ForensicsService {
if (config.MEMPOOL.BACKEND === 'esplora') {
await this.$runClosedChannelsForensics(false);
await this.$runOpenedChannelsForensics();
}
} catch (e) {
@@ -95,16 +99,9 @@ class ForensicsService {
const lightningScriptReasons: number[] = [];
for (const outspend of outspends) {
if (outspend.spent && outspend.txid) {
let spendingTx: IEsploraApi.Transaction | undefined = this.txCache[outspend.txid];
let spendingTx = await this.fetchTransaction(outspend.txid);
if (!spendingTx) {
try {
spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid);
await Common.sleep$(throttleDelay);
this.txCache[outspend.txid] = spendingTx;
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`);
continue;
}
continue;
}
cached.push(spendingTx.txid);
const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
@@ -124,16 +121,9 @@ class ForensicsService {
We can detect a commitment transaction (force close) by reading Sequence and Locktime
https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction
*/
let closingTx: IEsploraApi.Transaction | undefined = this.txCache[channel.closing_transaction_id];
let closingTx = await this.fetchTransaction(channel.closing_transaction_id, true);
if (!closingTx) {
try {
closingTx = await bitcoinApi.$getRawTransaction(channel.closing_transaction_id);
await Common.sleep$(throttleDelay);
this.txCache[channel.closing_transaction_id] = closingTx;
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id}. Reason ${e instanceof Error ? e.message : e}`);
continue;
}
continue;
}
cached.push(closingTx.txid);
const sequenceHex: string = closingTx.vin[0].sequence.toString(16);
@@ -174,7 +164,7 @@ class ForensicsService {
}
private findLightningScript(vin: IEsploraApi.Vin): number {
const topElement = vin.witness[vin.witness.length - 2];
const topElement = vin.witness?.length > 2 ? vin.witness[vin.witness.length - 2] : null;
if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) {
// https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs
if (topElement === '01') {
@@ -193,7 +183,7 @@ class ForensicsService {
) {
// https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs
// https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs
if (topElement.length === 66) {
if (topElement?.length === 66) {
// top element is a public key
// 'Revoked Lightning HTLC'; Penalty force closed
return 4;
@@ -220,6 +210,249 @@ class ForensicsService {
}
return 1;
}
// If a channel open tx spends funds from a another channel transaction,
// we can attribute that output to a specific counterparty
private async $runOpenedChannelsForensics(): Promise<void> {
const runTimer = Date.now();
let progress = 0;
try {
logger.info(`Started running open channel forensics...`);
const channels = await channelsApi.$getChannelsWithoutSourceChecked();
for (const openChannel of channels) {
let openTx = await this.fetchTransaction(openChannel.transaction_id, true);
if (!openTx) {
continue;
}
for (const input of openTx.vin) {
const closeChannel = await channelsApi.$getChannelByClosingId(input.txid);
if (closeChannel) {
// this input directly spends a channel close output
await this.$attributeChannelBalances(closeChannel, openChannel, input);
} else {
const prevOpenChannels = await channelsApi.$getChannelsByOpeningId(input.txid);
if (prevOpenChannels?.length) {
// this input spends a channel open change output
for (const prevOpenChannel of prevOpenChannels) {
await this.$attributeChannelBalances(prevOpenChannel, openChannel, input, null, null, true);
}
} else {
// check if this input spends any swept channel close outputs
await this.$attributeSweptChannelCloses(openChannel, input);
}
}
}
// calculate how much of the total input value is attributable to the channel open output
openChannel.funding_ratio = openTx.vout[openChannel.transaction_vout].value / ((openTx.vout.reduce((sum, v) => sum + v.value, 0) || 1) + openTx.fee);
// save changes to the opening channel, and mark it as checked
if (openTx?.vin?.length === 1) {
openChannel.single_funded = true;
}
if (openChannel.node1_funding_balance || openChannel.node2_funding_balance || openChannel.node1_closing_balance || openChannel.node2_closing_balance || openChannel.closed_by) {
await channelsApi.$updateOpeningInfo(openChannel);
}
await channelsApi.$markChannelSourceChecked(openChannel.id);
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating opened channel forensics ${progress}/${channels?.length}`);
this.loggerTimer = new Date().getTime() / 1000;
this.truncateTempCache();
}
if (Date.now() - runTimer > (config.LIGHTNING.FORENSICS_INTERVAL * 1000)) {
break;
}
}
logger.info(`Open channels forensics scan complete.`);
} catch (e) {
logger.err('$runOpenedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
} finally {
this.clearTempCache();
}
}
// Check if a channel open tx input spends the result of a swept channel close output
private async $attributeSweptChannelCloses(openChannel: ILightningApi.Channel, input: IEsploraApi.Vin): Promise<void> {
let sweepTx = await this.fetchTransaction(input.txid, true);
if (!sweepTx) {
logger.err(`couldn't find input transaction for channel forensics ${openChannel.channel_id} ${input.txid}`);
return;
}
const openContribution = sweepTx.vout[input.vout].value;
for (const sweepInput of sweepTx.vin) {
const lnScriptType = this.findLightningScript(sweepInput);
if (lnScriptType > 1) {
const closeChannel = await channelsApi.$getChannelByClosingId(sweepInput.txid);
if (closeChannel) {
const initiator = (lnScriptType === 2 || lnScriptType === 4) ? 'remote' : (lnScriptType === 3 ? 'local' : null);
await this.$attributeChannelBalances(closeChannel, openChannel, sweepInput, openContribution, initiator);
}
}
}
}
private async $attributeChannelBalances(
prevChannel, openChannel, input: IEsploraApi.Vin, openContribution: number | null = null,
initiator: 'remote' | 'local' | null = null, linkedOpenings: boolean = false
): Promise<void> {
// figure out which node controls the input/output
let openSide;
let prevLocal;
let prevRemote;
let matched = false;
let ambiguous = false; // if counterparties are the same in both channels, we can't tell them apart
if (openChannel.node1_public_key === prevChannel.node1_public_key) {
openSide = 1;
prevLocal = 1;
prevRemote = 2;
matched = true;
} else if (openChannel.node1_public_key === prevChannel.node2_public_key) {
openSide = 1;
prevLocal = 2;
prevRemote = 1;
matched = true;
}
if (openChannel.node2_public_key === prevChannel.node1_public_key) {
openSide = 2;
prevLocal = 1;
prevRemote = 2;
if (matched) {
ambiguous = true;
}
matched = true;
} else if (openChannel.node2_public_key === prevChannel.node2_public_key) {
openSide = 2;
prevLocal = 2;
prevRemote = 1;
if (matched) {
ambiguous = true;
}
matched = true;
}
if (matched && !ambiguous) {
// fetch closing channel transaction and perform forensics on the outputs
let prevChannelTx = await this.fetchTransaction(input.txid, true);
let outspends: IEsploraApi.Outspend[] | undefined;
try {
outspends = await bitcoinApi.$getOutspends(input.txid);
await Common.sleep$(throttleDelay);
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + input.txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
}
if (!outspends || !prevChannelTx) {
return;
}
if (!linkedOpenings) {
if (!prevChannel.outputs || !prevChannel.outputs.length) {
prevChannel.outputs = prevChannelTx.vout.map(vout => {
return {
type: 0,
value: vout.value,
};
});
}
for (let i = 0; i < outspends?.length; i++) {
const outspend = outspends[i];
const output = prevChannel.outputs[i];
if (outspend.spent && outspend.txid) {
try {
const spendingTx = await this.fetchTransaction(outspend.txid, true);
if (spendingTx) {
output.type = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
}
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`);
}
} else {
output.type = 0;
}
}
// attribute outputs to each counterparty, and sum up total known balances
prevChannel.outputs[input.vout].node = prevLocal;
const isPenalty = prevChannel.outputs.filter((out) => out.type === 2 || out.type === 4)?.length > 0;
const normalOutput = [1,3].includes(prevChannel.outputs[input.vout].type);
const mutualClose = ((prevChannel.status === 2 || prevChannel.status === 'closed') && prevChannel.closing_reason === 1);
let localClosingBalance = 0;
let remoteClosingBalance = 0;
for (const output of prevChannel.outputs) {
if (isPenalty) {
// penalty close, so local node takes everything
localClosingBalance += output.value;
} else if (output.node) {
// this output determinstically linked to one of the counterparties
if (output.node === prevLocal) {
localClosingBalance += output.value;
} else {
remoteClosingBalance += output.value;
}
} else if (normalOutput && (output.type === 1 || output.type === 3 || (mutualClose && prevChannel.outputs.length === 2))) {
// local node had one main output, therefore remote node takes the other
remoteClosingBalance += output.value;
}
}
prevChannel[`node${prevLocal}_closing_balance`] = localClosingBalance;
prevChannel[`node${prevRemote}_closing_balance`] = remoteClosingBalance;
prevChannel.closing_fee = prevChannelTx.fee;
if (initiator && !linkedOpenings) {
const initiatorSide = initiator === 'remote' ? prevRemote : prevLocal;
prevChannel.closed_by = prevChannel[`node${initiatorSide}_public_key`];
}
// save changes to the closing channel
await channelsApi.$updateClosingInfo(prevChannel);
} else {
if (prevChannelTx.vin.length <= 1) {
prevChannel[`node${prevLocal}_funding_balance`] = prevChannel.capacity;
prevChannel.single_funded = true;
prevChannel.funding_ratio = 1;
// save changes to the closing channel
await channelsApi.$updateOpeningInfo(prevChannel);
}
}
openChannel[`node${openSide}_funding_balance`] = openChannel[`node${openSide}_funding_balance`] + (openContribution || prevChannelTx?.vout[input.vout]?.value || 0);
}
}
async fetchTransaction(txid: string, temp: boolean = false): Promise<IEsploraApi.Transaction | null> {
let tx = this.txCache[txid];
if (!tx) {
try {
tx = await bitcoinApi.$getRawTransaction(txid);
this.txCache[txid] = tx;
if (temp) {
this.tempCached.push(txid);
}
await Common.sleep$(throttleDelay);
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
return null;
}
}
return tx;
}
clearTempCache(): void {
for (const txid of this.tempCached) {
delete this.txCache[txid];
}
this.tempCached = [];
}
truncateTempCache(): void {
if (this.tempCached.length > tempCacheSize) {
const removed = this.tempCached.splice(0, this.tempCached.length - tempCacheSize);
for (const txid of removed) {
delete this.txCache[txid];
}
}
}
}
export default new ForensicsService();

3
backend/src/tasks/lightning/network-sync.service.ts
View File

@@ -31,6 +31,7 @@ class NetworkSyncService {
}
private async $runTasks(): Promise<void> {
const taskStartTime = Date.now();
try {
logger.info(`Updating nodes and channels`);
@@ -57,7 +58,7 @@ class NetworkSyncService {
logger.err('$runTasks() error: ' + (e instanceof Error ? e.message : e));
}
setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL);
setTimeout(() => { this.$runTasks(); }, Math.max(1, (1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL) - (Date.now() - taskStartTime)));
}
/**