machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into dependabot/npm_and_yarn/backend/mysql2-3.9.1

Browse Source
This commit is contained in:
softsimon
2024-02-07 19:47:41 +08:00
committed by GitHub
parent 5f3f483546 0acb797494
commit 582eca1fdd
90 changed files with 3793 additions and 295 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
backend/.gitignore vendored
View File

@@ -7,6 +7,12 @@ mempool-config.json
pools.json
icons.json
# docker
Dockerfile
GeoIP
start.sh
wait-for-it.sh
# compiled output
/dist
/tmp

2
backend/src/api/bisq/markets-api.ts
View File

@@ -646,7 +646,7 @@ class BisqMarketsApi {
case 'year':
return strtotime('midnight first day of january', ts);
default:
throw new Error('Unsupported interval: ' + interval);
throw new Error('Unsupported interval');
}
}

1
backend/src/api/bitcoin/bitcoin-api.interface.ts
View File

@@ -106,6 +106,7 @@ export namespace IBitcoinApi {
address?: string; // (string) bitcoin address
addresses?: string[]; // (string) bitcoin addresses
pegout_chain?: string; // (string) Elements peg-out chain
pegout_address?: string; // (string) Elements peg-out address
pegout_addresses?: string[]; // (string) Elements peg-out addresses
};
}

37
backend/src/api/blocks.ts
View File

@@ -2,7 +2,7 @@ import config from '../config';
import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
import logger from '../logger';
import memPool from './mempool';
import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended, TransactionClassified } from '../mempool.interfaces';
import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended, TransactionClassified, BlockAudit } from '../mempool.interfaces';
import { Common } from './common';
import diskCache from './disk-cache';
import transactionUtils from './transaction-utils';
@@ -451,7 +451,9 @@ class Blocks {
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendTransaction(tx));
const cpfpSummary = await this.$indexCPFP(block.hash, block.height, txs);
await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary
if (cpfpSummary) {
await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary
}
} else {
await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary
}
@@ -995,11 +997,11 @@ class Blocks {
return state;
}
private updateTimerProgress(state, msg) {
private updateTimerProgress(state, msg): void {
state.progress = msg;
}
private clearTimer(state) {
private clearTimer(state): void {
if (state.timer) {
clearTimeout(state.timer);
}
@@ -1088,13 +1090,19 @@ class Blocks {
summary = {
id: hash,
transactions: cpfpSummary.transactions.map(tx => {
let flags: number = 0;
try {
flags = tx.flags || Common.getTransactionFlags(tx);
} catch (e) {
logger.warn('Failed to classify transaction: ' + (e instanceof Error ? e.message : e));
}
return {
txid: tx.txid,
fee: tx.fee || 0,
vsize: tx.vsize,
value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0)),
rate: tx.effectiveFeePerVsize,
flags: tx.flags || Common.getTransactionFlags(tx),
flags: flags,
};
}),
};
@@ -1284,7 +1292,7 @@ class Blocks {
return blocks;
}
public async $getBlockAuditSummary(hash: string): Promise<any> {
public async $getBlockAuditSummary(hash: string): Promise<BlockAudit | null> {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
return BlocksAuditsRepository.$getBlockAudit(hash);
} else {
@@ -1304,7 +1312,7 @@ class Blocks {
return this.currentBlockHeight;
}
public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise<CpfpSummary> {
public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise<CpfpSummary | null> {
let transactions = txs;
if (!transactions) {
if (config.MEMPOOL.BACKEND === 'esplora') {
@@ -1319,14 +1327,19 @@ class Blocks {
}
}
const summary = Common.calculateCpfp(height, transactions as TransactionExtended[]);
if (transactions?.length != null) {
const summary = Common.calculateCpfp(height, transactions as TransactionExtended[]);
await this.$saveCpfp(hash, height, summary);
await this.$saveCpfp(hash, height, summary);
const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions);
await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats);
const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions);
await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats);
return summary;
return summary;
} else {
logger.err(`Cannot index CPFP for block ${height} - missing transaction data`);
return null;
}
}
public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise<void> {

22
backend/src/api/common.ts
View File

@@ -6,6 +6,7 @@ import { NodeSocket } from '../repositories/NodesSocketsRepository';
import { isIP } from 'net';
import transactionUtils from './transaction-utils';
import { isPoint } from '../utils/secp256k1';
import logger from '../logger';
export class Common {
static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ?
'144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49'
@@ -245,7 +246,8 @@ export class Common {
} else if (tx.version === 2) {
flags |= TransactionFlags.v2;
}
const reusedAddresses: { [address: string ]: number } = {};
const reusedInputAddresses: { [address: string ]: number } = {};
const reusedOutputAddresses: { [address: string ]: number } = {};
const inValues = {};
const outValues = {};
let rbf = false;
@@ -261,6 +263,9 @@ export class Common {
case 'v0_p2wpkh': flags |= TransactionFlags.p2wpkh; break;
case 'v0_p2wsh': flags |= TransactionFlags.p2wsh; break;
case 'v1_p2tr': {
if (!vin.witness?.length) {
throw new Error('Taproot input missing witness data');
}
flags |= TransactionFlags.p2tr;
// in taproot, if the last witness item begins with 0x50, it's an annex
const hasAnnex = vin.witness?.[vin.witness.length - 1].startsWith('50');
@@ -286,7 +291,7 @@ export class Common {
}
if (vin.prevout?.scriptpubkey_address) {
reusedAddresses[vin.prevout?.scriptpubkey_address] = (reusedAddresses[vin.prevout?.scriptpubkey_address] || 0) + 1;
reusedInputAddresses[vin.prevout?.scriptpubkey_address] = (reusedInputAddresses[vin.prevout?.scriptpubkey_address] || 0) + 1;
}
inValues[vin.prevout?.value || Math.random()] = (inValues[vin.prevout?.value || Math.random()] || 0) + 1;
}
@@ -301,7 +306,7 @@ export class Common {
case 'p2pk': {
flags |= TransactionFlags.p2pk;
// detect fake pubkey (i.e. not a valid DER point on the secp256k1 curve)
hasFakePubkey = hasFakePubkey || !isPoint(vout.scriptpubkey.slice(2, -2));
hasFakePubkey = hasFakePubkey || !isPoint(vout.scriptpubkey?.slice(2, -2));
} break;
case 'multisig': {
flags |= TransactionFlags.p2ms;
@@ -321,7 +326,7 @@ export class Common {
case 'op_return': flags |= TransactionFlags.op_return; break;
}
if (vout.scriptpubkey_address) {
reusedAddresses[vout.scriptpubkey_address] = (reusedAddresses[vout.scriptpubkey_address] || 0) + 1;
reusedOutputAddresses[vout.scriptpubkey_address] = (reusedOutputAddresses[vout.scriptpubkey_address] || 0) + 1;
}
outValues[vout.value || Math.random()] = (outValues[vout.value || Math.random()] || 0) + 1;
}
@@ -331,7 +336,7 @@ export class Common {
// fast but bad heuristic to detect possible coinjoins
// (at least 5 inputs and 5 outputs, less than half of which are unique amounts, with no address reuse)
const addressReuse = Object.values(reusedAddresses).reduce((acc, count) => Math.max(acc, count), 0) > 1;
const addressReuse = Object.keys(reusedOutputAddresses).reduce((acc, key) => Math.max(acc, (reusedInputAddresses[key] || 0) + (reusedOutputAddresses[key] || 0)), 0) > 1;
if (!addressReuse && tx.vin.length >= 5 && tx.vout.length >= 5 && (Object.keys(inValues).length + Object.keys(outValues).length) <= (tx.vin.length + tx.vout.length) / 2 ) {
flags |= TransactionFlags.coinjoin;
}
@@ -348,7 +353,12 @@ export class Common {
}
static classifyTransaction(tx: TransactionExtended): TransactionClassified {
const flags = Common.getTransactionFlags(tx);
let flags = 0;
try {
flags = Common.getTransactionFlags(tx);
} catch (e) {
logger.warn('Failed to add classification flags to transaction: ' + (e instanceof Error ? e.message : e));
}
tx.flags = flags;
return {
...Common.stripTransaction(tx),

42
backend/src/api/database-migration.ts
View File

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 67;
private static currentVersion = 68;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -566,6 +566,20 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `blocks_templates` ADD INDEX `version` (`version`)');
await this.updateToSchemaVersion(67);
}
if (databaseSchemaVersion < 68 && config.MEMPOOL.NETWORK === "liquid") {
await this.$executeQuery('TRUNCATE TABLE elements_pegs');
await this.$executeQuery('ALTER TABLE elements_pegs ADD PRIMARY KEY (txid, txindex);');
await this.$executeQuery(`UPDATE state SET number = 0 WHERE name = 'last_elements_block';`);
// Create the federation_addresses table and add the two Liquid Federation change addresses in
await this.$executeQuery(this.getCreateFederationAddressesTableQuery(), await this.$checkIfTableExists('federation_addresses'));
await this.$executeQuery(`INSERT INTO federation_addresses (bitcoinaddress) VALUES ('bc1qxvay4an52gcghxq5lavact7r6qe9l4laedsazz8fj2ee2cy47tlqff4aj4')`); // Federation change address
await this.$executeQuery(`INSERT INTO federation_addresses (bitcoinaddress) VALUES ('3EiAcrzq1cELXScc98KeCswGWZaPGceT1d')`); // Federation change address
// Create the federation_txos table that uses the federation_addresses table as a foreign key
await this.$executeQuery(this.getCreateFederationTxosTableQuery(), await this.$checkIfTableExists('federation_txos'));
await this.$executeQuery(`INSERT INTO state VALUES('last_bitcoin_block_audit', 0, NULL);`);
await this.updateToSchemaVersion(68);
}
}
/**
@@ -813,6 +827,32 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateFederationAddressesTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS federation_addresses (
bitcoinaddress varchar(100) NOT NULL,
PRIMARY KEY (bitcoinaddress)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateFederationTxosTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS federation_txos (
txid varchar(65) NOT NULL,
txindex int(11) NOT NULL,
bitcoinaddress varchar(100) NOT NULL,
amount bigint(20) unsigned NOT NULL,
blocknumber int(11) unsigned NOT NULL,
blocktime int(11) unsigned NOT NULL,
unspent tinyint(1) NOT NULL,
lastblockupdate int(11) unsigned NOT NULL,
lasttimeupdate int(11) unsigned NOT NULL,
pegtxid varchar(65) NOT NULL,
pegindex int(11) NOT NULL,
pegblocktime int(11) unsigned NOT NULL,
PRIMARY KEY (txid, txindex),
FOREIGN KEY (bitcoinaddress) REFERENCES federation_addresses (bitcoinaddress)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreatePoolsTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS pools (
id int(11) NOT NULL AUTO_INCREMENT,

359
backend/src/api/liquid/elements-parser.ts
View File

@@ -5,8 +5,12 @@ import { Common } from '../common';
import DB from '../../database';
import logger from '../../logger';
const federationChangeAddresses = ['bc1qxvay4an52gcghxq5lavact7r6qe9l4laedsazz8fj2ee2cy47tlqff4aj4', '3EiAcrzq1cELXScc98KeCswGWZaPGceT1d'];
const auditBlockOffsetWithTip = 1; // Wait for 1 block confirmation before processing the block in the audit process to reduce the risk of reorgs
class ElementsParser {
private isRunning = false;
private isUtxosUpdatingRunning = false;
constructor() { }
@@ -32,12 +36,6 @@ class ElementsParser {
}
}
public async $getPegDataByMonth(): Promise<any> {
const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`;
const [rows] = await DB.query(query);
return rows;
}
protected async $parseBlock(block: IBitcoinApi.Block) {
for (const tx of block.tx) {
await this.$parseInputs(tx, block);
@@ -55,29 +53,30 @@ class ElementsParser {
protected async $parsePegIn(input: IBitcoinApi.Vin, vindex: number, txid: string, block: IBitcoinApi.Block) {
const bitcoinTx: IBitcoinApi.Transaction = await bitcoinSecondClient.getRawTransaction(input.txid, true);
const bitcoinBlock: IBitcoinApi.Block = await bitcoinSecondClient.getBlock(bitcoinTx.blockhash);
const prevout = bitcoinTx.vout[input.vout || 0];
const outputAddress = prevout.scriptPubKey.address || (prevout.scriptPubKey.addresses && prevout.scriptPubKey.addresses[0]) || '';
await this.$savePegToDatabase(block.height, block.time, prevout.value * 100000000, txid, vindex,
outputAddress, bitcoinTx.txid, prevout.n, 1);
outputAddress, bitcoinTx.txid, prevout.n, bitcoinBlock.height, bitcoinBlock.time, 1);
}
protected async $parseOutputs(tx: IBitcoinApi.Transaction, block: IBitcoinApi.Block) {
for (const output of tx.vout) {
if (output.scriptPubKey.pegout_chain) {
await this.$savePegToDatabase(block.height, block.time, 0 - output.value * 100000000, tx.txid, output.n,
(output.scriptPubKey.pegout_addresses && output.scriptPubKey.pegout_addresses[0] || ''), '', 0, 0);
(output.scriptPubKey.pegout_address || ''), '', 0, 0, 0, 0);
}
if (!output.scriptPubKey.pegout_chain && output.scriptPubKey.type === 'nulldata'
&& output.value && output.value > 0 && output.asset && output.asset === Common.nativeAssetId) {
await this.$savePegToDatabase(block.height, block.time, 0 - output.value * 100000000, tx.txid, output.n,
(output.scriptPubKey.pegout_addresses && output.scriptPubKey.pegout_addresses[0] || ''), '', 0, 1);
(output.scriptPubKey.pegout_address || ''), '', 0, 0, 0, 1);
}
}
}
protected async $savePegToDatabase(height: number, blockTime: number, amount: number, txid: string,
txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, final_tx: number): Promise<void> {
const query = `INSERT INTO elements_pegs(
txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, bitcoinblock: number, bitcoinBlockTime: number, final_tx: number): Promise<void> {
const query = `INSERT IGNORE INTO elements_pegs(
block, datetime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`;
@@ -85,7 +84,22 @@ class ElementsParser {
height, blockTime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
];
await DB.query(query, params);
logger.debug(`Saved L-BTC peg from block height #${height} with TXID ${txid}.`);
logger.debug(`Saved L-BTC peg from Liquid block height #${height} with TXID ${txid}.`);
if (amount > 0) { // Peg-in
// Add the address to the federation addresses table
await DB.query(`INSERT IGNORE INTO federation_addresses (bitcoinaddress) VALUES (?)`, [bitcoinaddress]);
// Add the UTXO to the federation txos table
const query_utxos = `INSERT IGNORE INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate, pegtxid, pegindex, pegblocktime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
const params_utxos: (string | number)[] = [bitcointxid, bitcoinindex, bitcoinaddress, amount, bitcoinblock, bitcoinBlockTime, 1, bitcoinblock - 1, 0, txid, txindex, blockTime];
await DB.query(query_utxos, params_utxos);
const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`)
await this.$saveLastBlockAuditToDatabase(minBlockUpdate[0]['lastblockupdate']);
logger.debug(`Saved new Federation UTXO ${bitcointxid}:${bitcoinindex} belonging to ${bitcoinaddress} to federation txos`);
}
}
protected async $getLatestBlockHeightFromDatabase(): Promise<number> {
@@ -98,6 +112,327 @@ class ElementsParser {
const query = `UPDATE state SET number = ? WHERE name = 'last_elements_block'`;
await DB.query(query, [blockHeight]);
}
///////////// FEDERATION AUDIT //////////////
public async $updateFederationUtxos() {
if (this.isUtxosUpdatingRunning) {
return;
}
this.isUtxosUpdatingRunning = true;
try {
let auditProgress = await this.$getAuditProgress();
// If no peg in transaction was found in the database, return
if (!auditProgress.lastBlockAudit) {
logger.debug(`No Federation UTXOs found in the database. Waiting for some to be confirmed before starting the Federation UTXOs audit`);
this.isUtxosUpdatingRunning = false;
return;
}
const bitcoinBlocksToSync = await this.$getBitcoinBlockchainState();
// If the bitcoin blockchain is not synced yet, return
if (bitcoinBlocksToSync.bitcoinHeaders > bitcoinBlocksToSync.bitcoinBlocks + 1) {
logger.debug(`Bitcoin client is not synced yet. ${bitcoinBlocksToSync.bitcoinHeaders - bitcoinBlocksToSync.bitcoinBlocks} blocks remaining to sync before the Federation audit process can start`);
this.isUtxosUpdatingRunning = false;
return;
}
auditProgress.lastBlockAudit++;
// Logging
let indexedThisRun = 0;
let timer = Date.now() / 1000;
const startedAt = Date.now() / 1000;
const indexingSpeeds: number[] = [];
while (auditProgress.lastBlockAudit <= auditProgress.confirmedTip) {
// First, get the current UTXOs that need to be scanned in the block
const utxos = await this.$getFederationUtxosToScan(auditProgress.lastBlockAudit);
// Get the peg-out addresses that need to be scanned
const redeemAddresses = await this.$getRedeemAddressesToScan();
// The fast way: check if these UTXOs are still unspent as of the current block with gettxout
let spentAsTip: any[];
let unspentAsTip: any[];
if (auditProgress.confirmedTip - auditProgress.lastBlockAudit <= 150) { // If the audit status is not too far in the past, we can use gettxout (fast way)
const utxosToParse = await this.$getFederationUtxosToParse(utxos);
spentAsTip = utxosToParse.spentAsTip;
unspentAsTip = utxosToParse.unspentAsTip;
logger.debug(`Found ${utxos.length} Federation UTXOs and ${redeemAddresses.length} Peg-Out Addresses to scan in Bitcoin block height #${auditProgress.lastBlockAudit} / #${auditProgress.confirmedTip}`);
logger.debug(`${unspentAsTip.length} / ${utxos.length} Federation UTXOs are unspent as of tip`);
} else { // If the audit status is too far in the past, it is useless and wasteful to look for still unspent txos since they will all be spent as of the tip
spentAsTip = utxos;
unspentAsTip = [];
// Logging
const elapsedSeconds = (Date.now() / 1000) - timer;
if (elapsedSeconds > 5) {
const runningFor = (Date.now() / 1000) - startedAt;
const blockPerSeconds = indexedThisRun / elapsedSeconds;
indexingSpeeds.push(blockPerSeconds);
if (indexingSpeeds.length > 100) indexingSpeeds.shift(); // Keep the length of the up to 100 last indexing speeds
const meanIndexingSpeed = indexingSpeeds.reduce((a, b) => a + b, 0) / indexingSpeeds.length;
const eta = (auditProgress.confirmedTip - auditProgress.lastBlockAudit) / meanIndexingSpeed;
logger.debug(`Scanning ${utxos.length} Federation UTXOs and ${redeemAddresses.length} Peg-Out Addresses at Bitcoin block height #${auditProgress.lastBlockAudit} / #${auditProgress.confirmedTip} | ~${meanIndexingSpeed.toFixed(2)} blocks/sec | elapsed: ${(runningFor / 60).toFixed(0)} minutes | ETA: ${(eta / 60).toFixed(0)} minutes`);
timer = Date.now() / 1000;
indexedThisRun = 0;
}
}
// The slow way: parse the block to look for the spending tx
const blockHash: IBitcoinApi.ChainTips = await bitcoinSecondClient.getBlockHash(auditProgress.lastBlockAudit);
const block: IBitcoinApi.Block = await bitcoinSecondClient.getBlock(blockHash, 2);
await this.$parseBitcoinBlock(block, spentAsTip, unspentAsTip, auditProgress.confirmedTip, redeemAddresses);
// Finally, update the lastblockupdate of the remaining UTXOs and save to the database
const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`)
await this.$saveLastBlockAuditToDatabase(minBlockUpdate[0]['lastblockupdate']);
auditProgress = await this.$getAuditProgress();
auditProgress.lastBlockAudit++;
indexedThisRun++;
}
this.isUtxosUpdatingRunning = false;
} catch (e) {
this.isUtxosUpdatingRunning = false;
throw new Error(e instanceof Error ? e.message : 'Error');
}
}
// Get the UTXOs that need to be scanned in block height (UTXOs that were last updated in the block height - 1)
protected async $getFederationUtxosToScan(height: number) {
const query = `SELECT txid, txindex, bitcoinaddress, amount FROM federation_txos WHERE lastblockupdate = ? AND unspent = 1`;
const [rows] = await DB.query(query, [height - 1]);
return rows as any[];
}
// Returns the UTXOs that are spent as of tip and need to be scanned
protected async $getFederationUtxosToParse(utxos: any[]): Promise<any> {
const spentAsTip: any[] = [];
const unspentAsTip: any[] = [];
for (const utxo of utxos) {
const result = await bitcoinSecondClient.getTxOut(utxo.txid, utxo.txindex, false);
result ? unspentAsTip.push(utxo) : spentAsTip.push(utxo);
}
return {spentAsTip, unspentAsTip};
}
protected async $parseBitcoinBlock(block: IBitcoinApi.Block, spentAsTip: any[], unspentAsTip: any[], confirmedTip: number, redeemAddressesData: any[] = []) {
const redeemAddresses: string[] = redeemAddressesData.map(redeemAddress => redeemAddress.bitcoinaddress);
for (const tx of block.tx) {
let mightRedeemInThisTx = false; // If a Federation UTXO is spent in this block, we might find a peg-out address in the outputs...
// Check if the Federation UTXOs that was spent as of tip are spent in this block
for (const input of tx.vin) {
const txo = spentAsTip.find(txo => txo.txid === input.txid && txo.txindex === input.vout);
if (txo) {
mightRedeemInThisTx = true;
await DB.query(`UPDATE federation_txos SET unspent = 0, lastblockupdate = ?, lasttimeupdate = ? WHERE txid = ? AND txindex = ?`, [block.height, block.time, txo.txid, txo.txindex]);
// Remove the TXO from the utxo array
spentAsTip.splice(spentAsTip.indexOf(txo), 1);
logger.debug(`Federation UTXO ${txo.txid}:${txo.txindex} (${txo.amount} sats) was spent in block ${block.height}`);
}
}
// Check if an output is sent to a change address of the federation
for (const output of tx.vout) {
if (output.scriptPubKey.address && federationChangeAddresses.includes(output.scriptPubKey.address)) {
// Check that the UTXO was not already added in the DB by previous scans
const [rows_check] = await DB.query(`SELECT txid FROM federation_txos WHERE txid = ? AND txindex = ?`, [tx.txid, output.n]) as any[];
if (rows_check.length === 0) {
const query_utxos = `INSERT INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate, pegtxid, pegindex, pegblocktime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
const params_utxos: (string | number)[] = [tx.txid, output.n, output.scriptPubKey.address, output.value * 100000000, block.height, block.time, 1, block.height, 0, '', 0, 0];
await DB.query(query_utxos, params_utxos);
// Add the UTXO to the utxo array
spentAsTip.push({
txid: tx.txid,
txindex: output.n,
bitcoinaddress: output.scriptPubKey.address,
amount: output.value * 100000000
});
logger.debug(`Added new Federation UTXO ${tx.txid}:${output.n} (${output.value * 100000000} sats), change address: ${output.scriptPubKey.address}`);
}
}
if (mightRedeemInThisTx && output.scriptPubKey.address && redeemAddresses.includes(output.scriptPubKey.address)) {
// Find the number of times output.scriptPubKey.address appears in redeemAddresses. There can be address reuse for peg-outs...
const matchingAddress: any[] = redeemAddressesData.filter(redeemAddress => redeemAddress.bitcoinaddress === output.scriptPubKey.address && -redeemAddress.amount === Math.round(output.value * 100000000));
if (matchingAddress.length > 0) {
if (matchingAddress.length > 1) {
// If there are more than one peg out address with the same amount, we can't know which one redeemed the UTXO: we take the oldest one
matchingAddress.sort((a, b) => a.datetime - b.datetime);
logger.debug(`Found redeem txid ${tx.txid}:${output.n} to peg-out address ${matchingAddress[0].bitcoinaddress}, amount ${matchingAddress[0].amount}, datetime ${matchingAddress[0].datetime}`);
} else {
logger.debug(`Found redeem txid ${tx.txid}:${output.n} to peg-out address ${matchingAddress[0].bitcoinaddress}, amount ${matchingAddress[0].amount}`);
}
const query_add_redeem = `UPDATE elements_pegs SET bitcointxid = ?, bitcoinindex = ? WHERE bitcoinaddress = ? AND amount = ? AND datetime = ?`;
const params_add_redeem: (string | number)[] = [tx.txid, output.n, matchingAddress[0].bitcoinaddress, matchingAddress[0].amount, matchingAddress[0].datetime];
await DB.query(query_add_redeem, params_add_redeem);
const index = redeemAddressesData.indexOf(matchingAddress[0]);
redeemAddressesData.splice(index, 1);
redeemAddresses.splice(index, 1);
} else { // The output amount does not match the peg-out amount... log it
logger.debug(`Found redeem txid ${tx.txid}:${output.n} to peg-out address ${output.scriptPubKey.address} but output amount ${Math.round(output.value * 100000000)} does not match the peg-out amount!`);
}
}
}
}
for (const utxo of spentAsTip) {
await DB.query(`UPDATE federation_txos SET lastblockupdate = ? WHERE txid = ? AND txindex = ?`, [block.height, utxo.txid, utxo.txindex]);
}
for (const utxo of unspentAsTip) {
await DB.query(`UPDATE federation_txos SET lastblockupdate = ? WHERE txid = ? AND txindex = ?`, [confirmedTip, utxo.txid, utxo.txindex]);
}
}
protected async $saveLastBlockAuditToDatabase(blockHeight: number) {
const query = `UPDATE state SET number = ? WHERE name = 'last_bitcoin_block_audit'`;
await DB.query(query, [blockHeight]);
}
// Get the bitcoin block where the audit process was last updated
protected async $getAuditProgress(): Promise<any> {
const lastblockaudit = await this.$getLastBlockAudit();
const bitcoinBlocksToSync = await this.$getBitcoinBlockchainState();
return {
lastBlockAudit: lastblockaudit,
confirmedTip: bitcoinBlocksToSync.bitcoinBlocks - auditBlockOffsetWithTip,
};
}
// Get the bitcoin blocks remaining to be synced
protected async $getBitcoinBlockchainState(): Promise<any> {
const result = await bitcoinSecondClient.getBlockchainInfo();
return {
bitcoinBlocks: result.blocks,
bitcoinHeaders: result.headers,
}
}
protected async $getLastBlockAudit(): Promise<number> {
const query = `SELECT number FROM state WHERE name = 'last_bitcoin_block_audit'`;
const [rows] = await DB.query(query);
return rows[0]['number'];
}
protected async $getRedeemAddressesToScan(): Promise<any[]> {
const query = `SELECT datetime, amount, bitcoinaddress FROM elements_pegs where amount < 0 AND bitcoinaddress != '' AND bitcointxid = '';`;
const [rows]: any[] = await DB.query(query);
return rows;
}
///////////// DATA QUERY //////////////
public async $getAuditStatus(): Promise<any> {
const lastBlockAudit = await this.$getLastBlockAudit();
const bitcoinBlocksToSync = await this.$getBitcoinBlockchainState();
return {
bitcoinBlocks: bitcoinBlocksToSync.bitcoinBlocks,
bitcoinHeaders: bitcoinBlocksToSync.bitcoinHeaders,
lastBlockAudit: lastBlockAudit,
isAuditSynced: bitcoinBlocksToSync.bitcoinHeaders - bitcoinBlocksToSync.bitcoinBlocks <= 2 && bitcoinBlocksToSync.bitcoinBlocks - lastBlockAudit <= 3,
};
}
public async $getPegDataByMonth(): Promise<any> {
const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`;
const [rows] = await DB.query(query);
return rows;
}
public async $getFederationReservesByMonth(): Promise<any> {
const query = `
SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(blocktime), '%Y-%m-01') AS date FROM federation_txos
WHERE
(blocktime > UNIX_TIMESTAMP(LAST_DAY(FROM_UNIXTIME(blocktime) - INTERVAL 1 MONTH) + INTERVAL 1 DAY))
AND
((unspent = 1) OR (unspent = 0 AND lasttimeupdate > UNIX_TIMESTAMP(LAST_DAY(FROM_UNIXTIME(blocktime)) + INTERVAL 1 DAY)))
GROUP BY
date;`;
const [rows] = await DB.query(query);
return rows;
}
// Get the current L-BTC pegs and the last Liquid block it was updated
public async $getCurrentLbtcSupply(): Promise<any> {
const [rows] = await DB.query(`SELECT SUM(amount) AS LBTC_supply FROM elements_pegs;`);
const lastblockupdate = await this.$getLatestBlockHeightFromDatabase();
const hash = await bitcoinClient.getBlockHash(lastblockupdate);
return {
amount: rows[0]['LBTC_supply'],
lastBlockUpdate: lastblockupdate,
hash: hash
};
}
// Get the current reserves of the federation and the last Bitcoin block it was updated
public async $getCurrentFederationReserves(): Promise<any> {
const [rows] = await DB.query(`SELECT SUM(amount) AS total_balance FROM federation_txos WHERE unspent = 1;`);
const lastblockaudit = await this.$getLastBlockAudit();
const hash = await bitcoinSecondClient.getBlockHash(lastblockaudit);
return {
amount: rows[0]['total_balance'],
lastBlockUpdate: lastblockaudit,
hash: hash
};
}
// Get all of the federation addresses, most balances first
public async $getFederationAddresses(): Promise<any> {
const query = `SELECT bitcoinaddress, SUM(amount) AS balance FROM federation_txos WHERE unspent = 1 GROUP BY bitcoinaddress ORDER BY balance DESC;`;
const [rows] = await DB.query(query);
return rows;
}
// Get all of the UTXOs held by the federation, most recent first
public async $getFederationUtxos(): Promise<any> {
const query = `SELECT txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, pegtxid, pegindex, pegblocktime FROM federation_txos WHERE unspent = 1 ORDER BY blocktime DESC;`;
const [rows] = await DB.query(query);
return rows;
}
// Get all of the federation addresses one month ago, most balances first
public async $getFederationAddressesOneMonthAgo(): Promise<any> {
const query = `
SELECT COUNT(*) AS addresses_count_one_month FROM (
SELECT bitcoinaddress, SUM(amount) AS balance
FROM federation_txos
WHERE
(blocktime < UNIX_TIMESTAMP(TIMESTAMPADD(DAY, -30, CURRENT_TIMESTAMP())))
AND
((unspent = 1) OR (unspent = 0 AND lasttimeupdate > UNIX_TIMESTAMP(TIMESTAMPADD(DAY, -30, CURRENT_TIMESTAMP()))))
GROUP BY bitcoinaddress
) AS result;`;
const [rows] = await DB.query(query);
return rows[0];
}
// Get all of the UTXOs held by the federation one month ago, most recent first
public async $getFederationUtxosOneMonthAgo(): Promise<any> {
const query = `
SELECT COUNT(*) AS utxos_count_one_month FROM federation_txos
WHERE
(blocktime < UNIX_TIMESTAMP(TIMESTAMPADD(DAY, -30, CURRENT_TIMESTAMP())))
AND
((unspent = 1) OR (unspent = 0 AND lasttimeupdate > UNIX_TIMESTAMP(TIMESTAMPADD(DAY, -30, CURRENT_TIMESTAMP()))))
ORDER BY blocktime DESC;`;
const [rows] = await DB.query(query);
return rows[0];
}
// Get recent pegouts from the federation (3 months old)
public async $getRecentPegouts(): Promise<any> {
const query = `SELECT txid, txindex, amount, bitcoinaddress, bitcointxid, bitcoinindex, datetime AS blocktime FROM elements_pegs WHERE amount < 0 AND datetime > UNIX_TIMESTAMP(TIMESTAMPADD(DAY, -90, CURRENT_TIMESTAMP())) ORDER BY blocktime;`;
const [rows] = await DB.query(query);
return rows;
}
}
export default new ElementsParser();

121
backend/src/api/liquid/liquid.routes.ts
View File

@@ -15,7 +15,16 @@ class LiquidRoutes {
if (config.DATABASE.ENABLED) {
app
.get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs', this.$getElementsPegs)
.get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/month', this.$getElementsPegsByMonth)
.get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves', this.$getFederationReserves)
.get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/month', this.$getFederationReservesByMonth)
.get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegouts', this.$getPegOuts)
.get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/addresses', this.$getFederationAddresses)
.get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/addresses/previous-month', this.$getFederationAddressesOneMonthAgo)
.get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/utxos', this.$getFederationUtxos)
.get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/utxos/previous-month', this.$getFederationUtxosOneMonthAgo)
.get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/status', this.$getFederationAuditStatus)
;
}
}
@@ -63,11 +72,123 @@ class LiquidRoutes {
private async $getElementsPegsByMonth(req: Request, res: Response) {
try {
const pegs = await elementsParser.$getPegDataByMonth();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60 * 60).toUTCString());
res.json(pegs);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getFederationReservesByMonth(req: Request, res: Response) {
try {
const reserves = await elementsParser.$getFederationReservesByMonth();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60 * 60).toUTCString());
res.json(reserves);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getElementsPegs(req: Request, res: Response) {
try {
const currentSupply = await elementsParser.$getCurrentLbtcSupply();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(currentSupply);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getFederationReserves(req: Request, res: Response) {
try {
const currentReserves = await elementsParser.$getCurrentFederationReserves();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(currentReserves);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getFederationAuditStatus(req: Request, res: Response) {
try {
const auditStatus = await elementsParser.$getAuditStatus();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(auditStatus);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getFederationAddresses(req: Request, res: Response) {
try {
const federationAddresses = await elementsParser.$getFederationAddresses();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(federationAddresses);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getFederationAddressesOneMonthAgo(req: Request, res: Response) {
try {
const federationAddresses = await elementsParser.$getFederationAddressesOneMonthAgo();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60 * 60 * 24).toUTCString());
res.json(federationAddresses);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getFederationUtxos(req: Request, res: Response) {
try {
const federationUtxos = await elementsParser.$getFederationUtxos();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(federationUtxos);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getFederationUtxosOneMonthAgo(req: Request, res: Response) {
try {
const federationUtxos = await elementsParser.$getFederationUtxosOneMonthAgo();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60 * 60 * 24).toUTCString());
res.json(federationUtxos);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getPegOuts(req: Request, res: Response) {
try {
const recentPegOuts = await elementsParser.$getRecentPegouts();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(recentPegOuts);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
}
export default new LiquidRoutes();

2
backend/src/api/mining/mining.ts
View File

@@ -142,7 +142,7 @@ class Mining {
public async $getPoolStat(slug: string): Promise<object> {
const pool = await PoolsRepository.$getPool(slug);
if (!pool) {
throw new Error('This mining pool does not exist ' + escape(slug));
throw new Error('This mining pool does not exist');
}
const blockCount: number = await BlocksRepository.$blockCount(pool.id);

1
backend/src/index.ts
View File

@@ -266,6 +266,7 @@ class Server {
blocks.setNewBlockCallback(async () => {
try {
await elementsParser.$parse();
await elementsParser.$updateFederationUtxos();
} catch (e) {
logger.warn('Elements parsing error: ' + (e instanceof Error ? e.message : e));
}

10
backend/src/repositories/BlocksAuditsRepository.ts
View File

@@ -59,7 +59,7 @@ class BlocksAuditRepositories {
}
}
public async $getBlockAudit(hash: string): Promise<any> {
public async $getBlockAudit(hash: string): Promise<BlockAudit | null> {
try {
const [rows]: any[] = await DB.query(
`SELECT blocks_audits.height, blocks_audits.hash as id, UNIX_TIMESTAMP(blocks_audits.time) as timestamp,
@@ -75,8 +75,8 @@ class BlocksAuditRepositories {
expected_weight as expectedWeight
FROM blocks_audits
JOIN blocks_templates ON blocks_templates.id = blocks_audits.hash
WHERE blocks_audits.hash = "${hash}"
`);
WHERE blocks_audits.hash = ?
`, [hash]);
if (rows.length) {
rows[0].missingTxs = JSON.parse(rows[0].missingTxs);
@@ -101,8 +101,8 @@ class BlocksAuditRepositories {
const [rows]: any[] = await DB.query(
`SELECT hash, match_rate as matchRate, expected_fees as expectedFees, expected_weight as expectedWeight
FROM blocks_audits
WHERE blocks_audits.hash = "${hash}"
`);
WHERE blocks_audits.hash = ?
`, [hash]);
return rows[0];
} catch (e: any) {
logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));

16
backend/src/repositories/BlocksRepository.ts
View File

@@ -5,7 +5,7 @@ import logger from '../logger';
import { Common } from '../api/common';
import PoolsRepository from './PoolsRepository';
import HashratesRepository from './HashratesRepository';
import { escape } from 'mysql2';
import { RowDataPacket, escape } from 'mysql2';
import BlocksSummariesRepository from './BlocksSummariesRepository';
import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
import bitcoinClient from '../api/bitcoin/bitcoin-client';
@@ -478,7 +478,7 @@ class BlocksRepository {
public async $getBlocksByPool(slug: string, startHeight?: number): Promise<BlockExtended[]> {
const pool = await PoolsRepository.$getPool(slug);
if (!pool) {
throw new Error('This mining pool does not exist ' + escape(slug));
throw new Error('This mining pool does not exist');
}
const params: any[] = [];
@@ -802,10 +802,10 @@ class BlocksRepository {
/**
* Get a list of blocks that have been indexed
*/
public async $getIndexedBlocks(): Promise<any[]> {
public async $getIndexedBlocks(): Promise<{ height: number, hash: string }[]> {
try {
const [rows]: any = await DB.query(`SELECT height, hash FROM blocks ORDER BY height DESC`);
return rows;
const [rows] = await DB.query(`SELECT height, hash FROM blocks ORDER BY height DESC`) as RowDataPacket[][];
return rows as { height: number, hash: string }[];
} catch (e) {
logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
@@ -815,7 +815,7 @@ class BlocksRepository {
/**
* Get a list of blocks that have not had CPFP data indexed
*/
public async $getCPFPUnindexedBlocks(): Promise<any[]> {
public async $getCPFPUnindexedBlocks(): Promise<number[]> {
try {
const blockchainInfo = await bitcoinClient.getBlockchainInfo();
const currentBlockHeight = blockchainInfo.blocks;
@@ -825,13 +825,13 @@ class BlocksRepository {
}
const minHeight = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
const [rows]: any[] = await DB.query(`
const [rows] = await DB.query(`
SELECT height
FROM compact_cpfp_clusters
WHERE height <= ? AND height >= ?
GROUP BY height
ORDER BY height DESC;
`, [currentBlockHeight, minHeight]);
`, [currentBlockHeight, minHeight]) as RowDataPacket[][];
const indexedHeights = {};
rows.forEach((row) => { indexedHeights[row.height] = true; });

3
backend/src/repositories/BlocksSummariesRepository.ts
View File

@@ -1,3 +1,4 @@
import { RowDataPacket } from 'mysql2';
import DB from '../database';
import logger from '../logger';
import { BlockSummary, TransactionClassified } from '../mempool.interfaces';
@@ -69,7 +70,7 @@ class BlocksSummariesRepository {
public async $getIndexedSummariesId(): Promise<string[]> {
try {
const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`);
const [rows] = await DB.query(`SELECT id from blocks_summaries`) as RowDataPacket[][];
return rows.map(row => row.id);
} catch (e) {
logger.err(`Cannot get block summaries id list. Reason: ` + (e instanceof Error ? e.message : e));

2
backend/src/repositories/HashratesRepository.ts
View File

@@ -139,7 +139,7 @@ class HashratesRepository {
public async $getPoolWeeklyHashrate(slug: string): Promise<any[]> {
const pool = await PoolsRepository.$getPool(slug);
if (!pool) {
throw new Error('This mining pool does not exist ' + escape(slug));
throw new Error('This mining pool does not exist');
}
// Find hashrate boundaries

3
backend/src/utils/secp256k1.ts
View File

@@ -31,6 +31,9 @@ const curveP = BigInt(`0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
* @returns {boolean} true if the point is on the SECP256K1 curve
*/
export function isPoint(pointHex: string): boolean {
if (!pointHex?.length) {
return false;
}
if (
!(
// is uncompressed