machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into simon/bisq-dashboard

Browse Source 
# Conflicts:
#	frontend/package-lock.json
#	frontend/package.json
This commit is contained in:
softsimon
2021-04-12 22:22:50 +04:00
parent 8e57aac934 0bf49c390e
commit 8b7850e6e1
40 changed files with 10786 additions and 7546 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
backend/package-lock.json generated
View File

@@ -7,7 +7,7 @@
"": {
"name": "mempool-backend",
"version": "2.0.0",
"license": "MIT",
"license": "GNU Affero General Public License v3.0",
"dependencies": {
"@mempool/bitcoin": "^3.0.2",
"@mempool/electrum-client": "^1.1.7",
@@ -16,17 +16,17 @@
"crypto-js": "^4.0.0",
"express": "^4.17.1",
"locutus": "^2.0.12",
"mysql2": "^2.2.5",
"node-worker-threads-pool": "^1.4.2",
"ws": "^7.3.1"
"mysql2": "2.2.5",
"node-worker-threads-pool": "^1.4.3",
"ws": "^7.4.4"
},
"devDependencies": {
"@types/compression": "^1.0.1",
"@types/express": "^4.17.2",
"@types/locutus": "^0.0.6",
"@types/ws": "^6.0.4",
"tslint": "~6.1.0",
"typescript": "~3.9.7"
"tslint": "^6.1.0",
"typescript": "^4.1.5"
}
},
"node_modules/@babel/code-frame": {
@@ -1480,9 +1480,9 @@
"integrity": "sha512-7uc1O8h1M1g0rArakJdf0uLRSSgFcYexrVoKo+bzJd32gd4gDy2L/Z+8/FjPnU9ydY3pEnVPtr9FyscYY60K1g=="
},
"node_modules/typescript": {
"version": "3.9.7",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.7.tgz",
"integrity": "sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==",
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.2.3.tgz",
"integrity": "sha512-qOcYwxaByStAWrBf4x0fibwZvMRG+r4cQoTjbPtUlrWjBHbmCAww1i448U0GJ+3cNNEtebDteo/cHOR3xJ4wEw==",
"dev": true,
"bin": {
"tsc": "bin/tsc",
@@ -1544,9 +1544,9 @@
"dev": true
},
"node_modules/ws": {
"version": "7.4.2",
"resolved": "https://registry.npmjs.org/ws/-/ws-7.4.2.tgz",
"integrity": "sha512-T4tewALS3+qsrpGI/8dqNMLIVdq/g/85U98HPMa6F0m6xTbvhXU6RCQLqPH3+SlomNV/LdY6RXEbBpMH6EOJnA==",
"version": "7.4.4",
"resolved": "https://registry.npmjs.org/ws/-/ws-7.4.4.tgz",
"integrity": "sha512-Qm8k8ojNQIMx7S+Zp8u/uHOx7Qazv3Yv4q68MiWWWOJhiwG5W3x7iqmRtJo8xxrciZUY4vRxUTJCKuRnF28ZZw==",
"engines": {
"node": ">=8.3.0"
},
@@ -2785,9 +2785,9 @@
"integrity": "sha512-7uc1O8h1M1g0rArakJdf0uLRSSgFcYexrVoKo+bzJd32gd4gDy2L/Z+8/FjPnU9ydY3pEnVPtr9FyscYY60K1g=="
},
"typescript": {
"version": "3.9.7",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.7.tgz",
"integrity": "sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==",
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.2.3.tgz",
"integrity": "sha512-qOcYwxaByStAWrBf4x0fibwZvMRG+r4cQoTjbPtUlrWjBHbmCAww1i448U0GJ+3cNNEtebDteo/cHOR3xJ4wEw==",
"dev": true
},
"unpipe": {
@@ -2833,9 +2833,9 @@
"dev": true
},
"ws": {
"version": "7.4.2",
"resolved": "https://registry.npmjs.org/ws/-/ws-7.4.2.tgz",
"integrity": "sha512-T4tewALS3+qsrpGI/8dqNMLIVdq/g/85U98HPMa6F0m6xTbvhXU6RCQLqPH3+SlomNV/LdY6RXEbBpMH6EOJnA==",
"version": "7.4.4",
"resolved": "https://registry.npmjs.org/ws/-/ws-7.4.4.tgz",
"integrity": "sha512-Qm8k8ojNQIMx7S+Zp8u/uHOx7Qazv3Yv4q68MiWWWOJhiwG5W3x7iqmRtJo8xxrciZUY4vRxUTJCKuRnF28ZZw==",
"requires": {}
},
"yallist": {

8
backend/package.json
View File

@@ -36,15 +36,15 @@
"express": "^4.17.1",
"locutus": "^2.0.12",
"mysql2": "2.2.5",
"node-worker-threads-pool": "^1.4.2",
"ws": "^7.3.1"
"node-worker-threads-pool": "^1.4.3",
"ws": "^7.4.4"
},
"devDependencies": {
"@types/compression": "^1.0.1",
"@types/express": "^4.17.2",
"@types/locutus": "^0.0.6",
"@types/ws": "^6.0.4",
"tslint": "~6.1.0",
"typescript": "~3.9.7"
"tslint": "^6.1.0",
"typescript": "^4.1.5"
}
}

19
backend/src/api/common.ts
View File

@@ -13,9 +13,13 @@ export class Common {
}
static percentile(numbers: number[], percentile: number) {
if (percentile === 50) return this.median(numbers);
if (percentile === 50) {
return this.median(numbers);
}
const index = Math.ceil(numbers.length * (100 - percentile) * 1e-2);
if (index < 0 || index > numbers.length - 1) return 0;
if (index < 0 || index > numbers.length - 1) {
return 0;
}
return numbers[index];
}
@@ -71,7 +75,7 @@ export class Common {
}, ms);
});
}
static shuffleArray(array: any[]) {
for (let i = array.length - 1; i > 0; i--) {
const j = Math.floor(Math.random() * (i + 1));
@@ -81,9 +85,10 @@ export class Common {
static setRelativesAndGetCpfpInfo(tx: TransactionExtended, memPool: { [txid: string]: TransactionExtended }): CpfpInfo {
const parents = this.findAllParents(tx, memPool);
const lowerFeeParents = parents.filter((parent) => parent.feePerVsize < tx.effectiveFeePerVsize);
let totalWeight = tx.weight + parents.reduce((prev, val) => prev + val.weight, 0);
let totalFees = tx.fee + parents.reduce((prev, val) => prev + val.fee, 0);
let totalWeight = tx.weight + lowerFeeParents.reduce((prev, val) => prev + val.weight, 0);
let totalFees = tx.fee + lowerFeeParents.reduce((prev, val) => prev + val.fee, 0);
tx.ancestors = parents
.map((t) => {
@@ -113,6 +118,10 @@ export class Common {
private static findAllParents(tx: TransactionExtended, memPool: { [txid: string]: TransactionExtended }): TransactionExtended[] {
let parents: TransactionExtended[] = [];
tx.vin.forEach((parent) => {
if (parents.find((p) => p.txid === parent.txid)) {
return;
}
const parentTx = memPool[parent.txid];
if (parentTx) {
if (tx.bestDescendant && tx.bestDescendant.fee / (tx.bestDescendant.weight / 4) > parentTx.feePerVsize) {

28
backend/src/api/disk-cache.ts
View File

@@ -12,6 +12,7 @@ class DiskCache {
private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';
private static CHUNK_FILES = 25;
private isWritingCache = false;
constructor() { }
@@ -19,8 +20,13 @@ class DiskCache {
if (!cluster.isMaster) {
return;
}
if (this.isWritingCache) {
logger.debug('Saving cache already in progress. Skipping.')
return;
}
try {
logger.debug('Writing mempool and blocks data to disk cache (async)...');
this.isWritingCache = true;
const mempool = memPool.getMempool();
const mempoolArray: TransactionExtended[] = [];
@@ -44,8 +50,10 @@ class DiskCache {
}), {flag: 'w'});
}
logger.debug('Mempool and blocks data saved to disk cache');
this.isWritingCache = false;
} catch (e) {
logger.warn('Error writing to cache file: ' + e.message || e);
this.isWritingCache = false;
}
}
@@ -68,22 +76,26 @@ class DiskCache {
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
const fileName = DiskCache.FILE_NAMES.replace('{number}', i.toString());
if (fs.existsSync(fileName)) {
const cacheData2 = JSON.parse(fs.readFileSync(fileName, 'utf8'));
if (cacheData2.mempoolArray) {
for (const tx of cacheData2.mempoolArray) {
data.mempool[tx.txid] = tx;
try {
if (fs.existsSync(fileName)) {
const cacheData2 = JSON.parse(fs.readFileSync(fileName, 'utf8'));
if (cacheData2.mempoolArray) {
for (const tx of cacheData2.mempoolArray) {
data.mempool[tx.txid] = tx;
}
} else {
Object.assign(data.mempool, cacheData2.mempool);
}
} else {
Object.assign(data.mempool, cacheData2.mempool);
}
} catch (e) {
logger.debug('Error parsing ' + fileName + '. Skipping.');
}
}
memPool.setMempool(data.mempool);
blocks.setBlocks(data.blocks);
} catch (e) {
logger.warn('Failed to parse mempoool and blocks cache. Skipping...');
logger.warn('Failed to parse mempoool and blocks cache. Skipping.');
}
}
}

2
backend/src/api/mempool-blocks.ts
View File

@@ -53,7 +53,7 @@ class MempoolBlocks {
// Pass down size + fee to all unconfirmed children
let sizes = 0;
memPoolArray.forEach((tx, i) => {
sizes += tx.weight
sizes += tx.weight;
if (sizes > 4000000 * 8) {
return;
}

29
backend/src/api/mempool.ts
View File

@@ -10,10 +10,11 @@ import loadingIndicators from './loading-indicators';
class Mempool {
private static WEBSOCKET_REFRESH_RATE_MS = 10000;
private static LAZY_DELETE_AFTER_SECONDS = 30;
private inSync: boolean = false;
private mempoolCache: { [txId: string]: TransactionExtended } = {};
private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0,
maxmempool: 0, mempoolminfee: 0, minrelaytxfee: 0 };
maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
deletedTransactions: TransactionExtended[]) => void) | undefined;
@@ -27,6 +28,7 @@ class Mempool {
constructor() {
setInterval(this.updateTxPerSecond.bind(this), 1000);
setInterval(this.deleteExpiredTransactions.bind(this), 20000);
}
public isInSync(): boolean {
@@ -145,7 +147,6 @@ class Mempool {
}, 1000 * 60 * config.MEMPOOL.CLEAR_PROTECTION_MINUTES);
}
let newMempool = {};
const deletedTransactions: TransactionExtended[] = [];
if (this.mempoolProtection !== 1) {
@@ -154,35 +155,31 @@ class Mempool {
const transactionsObject = {};
transactions.forEach((txId) => transactionsObject[txId] = true);
// Replace mempool to separate deleted transactions
// Flag transactions for lazy deletion
for (const tx in this.mempoolCache) {
if (transactionsObject[tx]) {
newMempool[tx] = this.mempoolCache[tx];
} else {
if (!transactionsObject[tx] && !this.mempoolCache[tx].deleteAfter) {
deletedTransactions.push(this.mempoolCache[tx]);
this.mempoolCache[tx].deleteAfter = new Date().getTime() + Mempool.LAZY_DELETE_AFTER_SECONDS * 1000;
}
}
} else {
newMempool = this.mempoolCache;
}
const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx));
this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6);
if (!this.inSync && transactions.length === Object.keys(newMempool).length) {
if (!this.inSync && transactions.length === Object.keys(this.mempoolCache).length) {
this.inSync = true;
logger.info('The mempool is now in sync!');
loadingIndicators.setProgress('mempool', 100);
}
if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
this.mempoolCache = newMempool;
this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
}
const end = new Date().getTime();
const time = end - start;
logger.debug(`New mempool size: ${Object.keys(newMempool).length} Change: ${diff}`);
logger.debug(`New mempool size: ${Object.keys(this.mempoolCache).length} Change: ${diff}`);
logger.debug('Mempool updated in ' + time / 1000 + ' seconds');
}
@@ -198,6 +195,16 @@ class Mempool {
);
}
}
private deleteExpiredTransactions() {
const now = new Date().getTime();
for (const tx in this.mempoolCache) {
const lazyDeleteAt = this.mempoolCache[tx].deleteAfter;
if (lazyDeleteAt && lazyDeleteAt < now) {
delete this.mempoolCache[tx];
}
}
}
}
export default new Mempool();

20
backend/src/api/websocket-handler.ts
View File

@@ -202,10 +202,15 @@ class WebsocketHandler {
mempoolBlocks.updateMempoolBlocks(newMempool);
const mBlocks = mempoolBlocks.getMempoolBlocks();
const mempool = memPool.getMempool();
const mempoolInfo = memPool.getMempoolInfo();
const vBytesPerSecond = memPool.getVBytesPerSecond();
const rbfTransactions = Common.findRbfTransactions(newTransactions, deletedTransactions);
for (const rbfTransaction in rbfTransactions) {
delete mempool[rbfTransaction];
}
this.wss.clients.forEach(async (client: WebSocket) => {
if (client.readyState !== WebSocket.OPEN) {
return;
@@ -337,28 +342,23 @@ class WebsocketHandler {
throw new Error('WebSocket.Server is not set');
}
// Check how many transactions in the new block matches the latest projected mempool block
// If it's more than 0, recalculate the mempool blocks and send to client in the same update
let mBlocks: undefined | MempoolBlock[];
let matchRate = 0;
const _memPool = memPool.getMempool();
const _mempoolBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
if (_mempoolBlocks[0]) {
const matches: string[] = [];
for (const txId of txIds) {
if (_mempoolBlocks[0].transactionIds.indexOf(txId) > -1) {
matches.push(txId);
}
delete _memPool[txId];
}
matchRate = Math.round((matches.length / (txIds.length - 1)) * 100);
if (matchRate > 0) {
const currentMemPool = memPool.getMempool();
for (const txId of matches) {
delete currentMemPool[txId];
}
mempoolBlocks.updateMempoolBlocks(currentMemPool);
mBlocks = mempoolBlocks.getMempoolBlocks();
}
mempoolBlocks.updateMempoolBlocks(_memPool);
mBlocks = mempoolBlocks.getMempoolBlocks();
}
block.matchRate = matchRate;

29
backend/src/index.ts
View File

@@ -111,7 +111,16 @@ class Server {
async runMainUpdateLoop() {
try {
await memPool.$updateMemPoolInfo();
try {
await memPool.$updateMemPoolInfo();
} catch (e) {
const msg = `updateMempoolInfo: ${(e.message || e)}`;
if (config.CORE_RPC_MINFEE.ENABLED) {
logger.warn(msg);
} else {
logger.debug(msg);
}
}
await blocks.$updateBlocks();
await memPool.$updateMempool();
setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
@@ -169,6 +178,24 @@ class Server {
res.status(500).end();
}
})
.get(config.MEMPOOL.API_URL_PREFIX + 'contributors', async (req, res) => {
try {
const response = await axios.get('https://mempool.space/api/v1/contributors', { responseType: 'stream', timeout: 10000 });
response.data.pipe(res);
} catch (e) {
res.status(500).end();
}
})
.get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => {
try {
const response = await axios.get('https://mempool.space/api/v1/contributors/images/' + req.params.id, {
responseType: 'stream', timeout: 10000
});
response.data.pipe(res);
} catch (e) {
res.status(500).end();
}
})
;
if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {

1
backend/src/mempool.interfaces.ts
View File

@@ -30,6 +30,7 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
ancestors?: Ancestor[];
bestDescendant?: BestDescendant | null;
cpfpChecked?: boolean;
deleteAfter?: number;
}
interface Ancestor {