machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into mononaut/confirmed-tx-status

Browse Source
This commit is contained in:
softsimon
2023-11-12 13:48:35 +09:00
committed by GitHub
parent 9e6072c371 1e69f3b331
commit 90634c4343
44 changed files with 998 additions and 728 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
backend/src/__fixtures__/mempool-config.template.json
View File

@@ -41,7 +41,9 @@
"PORT": 15,
"USERNAME": "__CORE_RPC_USERNAME__",
"PASSWORD": "__CORE_RPC_PASSWORD__",
"TIMEOUT": 1000
"TIMEOUT": 1000,
"COOKIE": false,
"COOKIE_PATH": "__CORE_RPC_COOKIE_PATH__"
},
"ELECTRUM": {
"HOST": "__ELECTRUM_HOST__",
@@ -61,7 +63,9 @@
"PORT": 17,
"USERNAME": "__SECOND_CORE_RPC_USERNAME__",
"PASSWORD": "__SECOND_CORE_RPC_PASSWORD__",
"TIMEOUT": 2000
"TIMEOUT": 2000,
"COOKIE": false,
"COOKIE_PATH": "__SECOND_CORE_RPC_COOKIE_PATH__"
},
"DATABASE": {
"ENABLED": false,

8
backend/src/__tests__/config.test.ts
View File

@@ -66,7 +66,9 @@ describe('Mempool Backend Config', () => {
PORT: 8332,
USERNAME: 'mempool',
PASSWORD: 'mempool',
TIMEOUT: 60000
TIMEOUT: 60000,
COOKIE: false,
COOKIE_PATH: '/bitcoin/.cookie'
});
expect(config.SECOND_CORE_RPC).toStrictEqual({
@@ -74,7 +76,9 @@ describe('Mempool Backend Config', () => {
PORT: 8332,
USERNAME: 'mempool',
PASSWORD: 'mempool',
TIMEOUT: 60000
TIMEOUT: 60000,
COOKIE: false,
COOKIE_PATH: '/bitcoin/.cookie'
});
expect(config.DATABASE).toStrictEqual({

1
backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts
View File

@@ -32,4 +32,5 @@ export interface BitcoinRpcCredentials {
user: string;
pass: string;
timeout: number;
cookie?: string;
}

1
backend/src/api/bitcoin/bitcoin-client.ts
View File

@@ -8,6 +8,7 @@ const nodeRpcCredentials: BitcoinRpcCredentials = {
user: config.CORE_RPC.USERNAME,
pass: config.CORE_RPC.PASSWORD,
timeout: config.CORE_RPC.TIMEOUT,
cookie: config.CORE_RPC.COOKIE ? config.CORE_RPC.COOKIE_PATH : undefined,
};
export default new bitcoin.Client(nodeRpcCredentials);

1
backend/src/api/bitcoin/bitcoin-second-client.ts
View File

@@ -8,6 +8,7 @@ const nodeRpcCredentials: BitcoinRpcCredentials = {
user: config.SECOND_CORE_RPC.USERNAME,
pass: config.SECOND_CORE_RPC.PASSWORD,
timeout: config.SECOND_CORE_RPC.TIMEOUT,
cookie: config.SECOND_CORE_RPC.COOKIE ? config.SECOND_CORE_RPC.COOKIE_PATH : undefined,
};
export default new bitcoin.Client(nodeRpcCredentials);

6
backend/src/api/disk-cache.ts
View File

@@ -252,7 +252,11 @@ class DiskCache {
}
if (rbfData?.rbf) {
rbfCache.load(rbfData.rbf);
rbfCache.load({
txs: rbfData.rbf.txs.map(([txid, entry]) => ({ value: entry })),
trees: rbfData.rbf.trees,
expiring: rbfData.rbf.expiring.map(([txid, value]) => ({ key: txid, value })),
});
}
} catch (e) {
logger.warn('Failed to parse rbf cache. Skipping. Reason: ' + (e instanceof Error ? e.message : e));

49
backend/src/api/rbf-cache.ts
View File

@@ -53,6 +53,9 @@ class RbfCache {
private expiring: Map<string, number> = new Map();
private cacheQueue: CacheEvent[] = [];
private evictionCount = 0;
private staleCount = 0;
constructor() {
setInterval(this.cleanup.bind(this), 1000 * 60 * 10);
}
@@ -245,6 +248,7 @@ class RbfCache {
// flag a transaction as removed from the mempool
public evict(txid: string, fast: boolean = false): void {
this.evictionCount++;
if (this.txs.has(txid) && (fast || !this.expiring.has(txid))) {
const expiryTime = fast ? Date.now() + (1000 * 60 * 10) : Date.now() + (1000 * 86400); // 24 hours
this.addExpiration(txid, expiryTime);
@@ -272,18 +276,23 @@ class RbfCache {
this.remove(txid);
}
}
logger.debug(`rbf cache contains ${this.txs.size} txs, ${this.rbfTrees.size} trees, ${this.expiring.size} due to expire`);
logger.debug(`rbf cache contains ${this.txs.size} txs, ${this.rbfTrees.size} trees, ${this.expiring.size} due to expire (${this.evictionCount} newly expired)`);
this.evictionCount = 0;
}
// remove a transaction & all previous versions from the cache
private remove(txid): void {
// don't remove a transaction if a newer version remains in the mempool
if (!this.replacedBy.has(txid)) {
const root = this.treeMap.get(txid);
const replaces = this.replaces.get(txid);
this.replaces.delete(txid);
this.treeMap.delete(txid);
this.removeTx(txid);
this.removeExpiration(txid);
if (root === txid) {
this.removeTree(txid);
}
for (const tx of (replaces || [])) {
// recursively remove prior versions from the cache
this.replacedBy.delete(tx);
@@ -359,18 +368,25 @@ class RbfCache {
}
public async load({ txs, trees, expiring }): Promise<void> {
txs.forEach(txEntry => {
this.txs.set(txEntry.key, txEntry.value);
});
for (const deflatedTree of trees) {
await this.importTree(deflatedTree.root, deflatedTree.root, deflatedTree, this.txs);
}
expiring.forEach(expiringEntry => {
if (this.txs.has(expiringEntry.key)) {
this.expiring.set(expiringEntry.key, new Date(expiringEntry.value).getTime());
try {
txs.forEach(txEntry => {
this.txs.set(txEntry.value.txid, txEntry.value);
});
this.staleCount = 0;
for (const deflatedTree of trees) {
await this.importTree(deflatedTree.root, deflatedTree.root, deflatedTree, this.txs);
}
});
this.cleanup();
expiring.forEach(expiringEntry => {
if (this.txs.has(expiringEntry.key)) {
this.expiring.set(expiringEntry.key, new Date(expiringEntry.value).getTime());
}
});
logger.debug(`loaded ${txs.length} txs, ${trees.length} trees into rbf cache, ${expiring.length} due to expire, ${this.staleCount} were stale`);
this.staleCount = 0;
this.cleanup();
} catch (e) {
logger.err('failed to restore RBF cache: ' + (e instanceof Error ? e.message : e));
}
}
exportTree(tree: RbfTree, deflated: any = null) {
@@ -398,6 +414,13 @@ class RbfCache {
const treeInfo = deflated[txid];
const replaces: RbfTree[] = [];
// if the root tx is unknown, remove this tree and return early
if (root === txid && !txs.has(txid)) {
this.staleCount++;
this.removeTree(deflated.key);
return;
}
// check if any transactions in this tree have already been confirmed
mined = mined || treeInfo.mined;
let exists = mined;
@@ -413,7 +436,7 @@ class RbfCache {
this.evict(txid, true);
}
} catch (e) {
// most transactions do not exist
// most transactions only exist in our cache
}
}

2
backend/src/api/redis-cache.ts
View File

@@ -219,7 +219,7 @@ class RedisCache {
await memPool.$setMempool(loadedMempool);
await rbfCache.load({
txs: rbfTxs,
trees: rbfTrees.map(loadedTree => loadedTree.value),
trees: rbfTrees.map(loadedTree => { loadedTree.value.key = loadedTree.key; return loadedTree.value; }),
expiring: rbfExpirations,
});
}

2
backend/src/api/websocket-handler.ts
View File

@@ -577,7 +577,7 @@ class WebsocketHandler {
response['utxoSpent'] = JSON.stringify(outspends);
}
const rbfReplacedBy = rbfCache.getReplacedBy(client['track-tx']);
const rbfReplacedBy = rbfChanges.map[client['track-tx']] ? rbfCache.getReplacedBy(client['track-tx']) : false;
if (rbfReplacedBy) {
response['rbfTransaction'] = JSON.stringify({
txid: rbfReplacedBy,

8
backend/src/config.ts
View File

@@ -78,6 +78,8 @@ interface IConfig {
USERNAME: string;
PASSWORD: string;
TIMEOUT: number;
COOKIE: boolean;
COOKIE_PATH: string;
};
SECOND_CORE_RPC: {
HOST: string;
@@ -85,6 +87,8 @@ interface IConfig {
USERNAME: string;
PASSWORD: string;
TIMEOUT: number;
COOKIE: boolean;
COOKIE_PATH: string;
};
DATABASE: {
ENABLED: boolean;
@@ -207,6 +211,8 @@ const defaults: IConfig = {
'USERNAME': 'mempool',
'PASSWORD': 'mempool',
'TIMEOUT': 60000,
'COOKIE': false,
'COOKIE_PATH': '/bitcoin/.cookie'
},
'SECOND_CORE_RPC': {
'HOST': '127.0.0.1',
@@ -214,6 +220,8 @@ const defaults: IConfig = {
'USERNAME': 'mempool',
'PASSWORD': 'mempool',
'TIMEOUT': 60000,
'COOKIE': false,
'COOKIE_PATH': '/bitcoin/.cookie'
},
'DATABASE': {
'ENABLED': true,

21
backend/src/rpc-api/jsonrpc.ts
View File

@@ -1,5 +1,6 @@
var http = require('http')
var https = require('https')
import { readFileSync } from 'fs';
var JsonRPC = function (opts) {
// @ts-ignore
@@ -55,7 +56,13 @@ JsonRPC.prototype.call = function (method, params) {
}
// use HTTP auth if user and password set
if (this.opts.user && this.opts.pass) {
if (this.opts.cookie) {
if (!this.cachedCookie) {
this.cachedCookie = readFileSync(this.opts.cookie).toString();
}
// @ts-ignore
requestOptions.auth = this.cachedCookie;
} else if (this.opts.user && this.opts.pass) {
// @ts-ignore
requestOptions.auth = this.opts.user + ':' + this.opts.pass
}
@@ -93,7 +100,7 @@ JsonRPC.prototype.call = function (method, params) {
reject(err)
})
request.on('response', function (response) {
request.on('response', (response) => {
clearTimeout(reqTimeout)
// We need to buffer the response chunks in a nonblocking way.
@@ -104,7 +111,7 @@ JsonRPC.prototype.call = function (method, params) {
// When all the responses are finished, we decode the JSON and
// depending on whether it's got a result or an error, we call
// emitSuccess or emitError on the promise.
response.on('end', function () {
response.on('end', () => {
var err
if (cbCalled) return
@@ -113,6 +120,14 @@ JsonRPC.prototype.call = function (method, params) {
try {
var decoded = JSON.parse(buffer)
} catch (e) {
// if we authenticated using a cookie and it failed, read the cookie file again
if (
response.statusCode === 401 /* Unauthorized */ &&
this.opts.cookie
) {
this.cachedCookie = undefined;
}
if (response.statusCode !== 200) {
err = new Error('Invalid params, response status code: ' + response.statusCode)
err.code = -32602