diff --git a/LICENSE b/LICENSE index ac267d120..9f8592854 100644 --- a/LICENSE +++ b/LICENSE @@ -13,7 +13,7 @@ the terms of (at your option) either: proxy statement published on . However, this copyright license does not include an implied right or license to -use our trademarks: The Mempool Open Source Project™, mempool.space™, the +use our trademarks: The Mempool Open Source Project®, mempool.space™, the mempool Logo™, the mempool.space Vertical Logo™, the mempool.space Horizontal Logo™, the mempool Square Logo™, and the mempool Blocks logo™ are registered trademarks or trademarks of Mempool Space K.K in Japan, the United States, diff --git a/README.md b/README.md index b7a455cd5..dd2e62478 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# The Mempool Open Source Project™ [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs) +# The Mempool Open Source Project® [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs) https://user-images.githubusercontent.com/93150691/226236121-375ea64f-b4a1-4cc0-8fad-a6fb33226840.mp4 diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 8f8b82475..83b3b1ad4 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -8,6 +8,7 @@ "API_URL_PREFIX": "/api/v1/", "POLL_RATE_MS": 2000, "CACHE_DIR": "./cache", + "CACHE_ENABLED": true, "CLEAR_PROTECTION_MINUTES": 20, "RECOMMENDED_FEE_PERCENTILE": 50, "BLOCK_WEIGHT_UNITS": 4000000, diff --git a/backend/package-lock.json b/backend/package-lock.json index 0a0b8b9d1..1a92552cb 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -19,6 +19,7 @@ "maxmind": "~4.3.11", "mysql2": "~3.5.2", "rust-gbt": "file:./rust-gbt", + "redis": "^4.6.6", "socks-proxy-agent": "~7.0.0", "typescript": "~4.9.3", "ws": "~8.13.0" @@ -1555,6 +1556,64 @@ "node": ">= 8" } }, + "node_modules/@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/client": { + "version": "1.5.7", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.7.tgz", + "integrity": "sha512-gaOBOuJPjK5fGtxSseaKgSvjiZXQCdLlGg9WYQst+/GRUjmXaiB5kVkeQMRtPc7Q2t93XZcJfBMSwzs/XS9UZw==", + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@redis/client/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "node_modules/@redis/graph": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.0.tgz", + "integrity": "sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/json": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.4.tgz", + "integrity": "sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/search": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.2.tgz", + "integrity": "sha512-/cMfstG/fOh/SsE+4/BQGeuH/JJloeWuH+qJzM8dbxuWvdWibWAOAHHCZTMPhV3xIlH4/cUEIA8OV5QnYpaVoA==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/time-series": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.4.tgz", + "integrity": "sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, "node_modules/@sinclair/typebox": { "version": "0.25.24", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.25.24.tgz", @@ -2718,6 +2777,14 @@ "node": ">=12" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -3678,6 +3745,14 @@ "is-property": "^1.0.2" } }, + "node_modules/generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "engines": { + "node": ">= 4" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -6577,6 +6652,19 @@ "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==", "dev": true }, + "node_modules/redis": { + "version": "4.6.6", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.6.6.tgz", + "integrity": "sha512-aLs2fuBFV/VJ28oLBqYykfnhGGkFxvx0HdCEBYdJ99FFbSEMZ7c1nVKwR6ZRv+7bb7JnC0mmCzaqu8frgOYhpA==", + "dependencies": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.5.7", + "@redis/graph": "1.1.0", + "@redis/json": "1.0.4", + "@redis/search": "1.1.2", + "@redis/time-series": "1.0.4" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -8704,6 +8792,53 @@ "fastq": "^1.6.0" } }, + "@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "requires": {} + }, + "@redis/client": { + "version": "1.5.7", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.7.tgz", + "integrity": "sha512-gaOBOuJPjK5fGtxSseaKgSvjiZXQCdLlGg9WYQst+/GRUjmXaiB5kVkeQMRtPc7Q2t93XZcJfBMSwzs/XS9UZw==", + "requires": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "dependencies": { + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + } + }, + "@redis/graph": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.0.tgz", + "integrity": "sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==", + "requires": {} + }, + "@redis/json": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.4.tgz", + "integrity": "sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw==", + "requires": {} + }, + "@redis/search": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.2.tgz", + "integrity": "sha512-/cMfstG/fOh/SsE+4/BQGeuH/JJloeWuH+qJzM8dbxuWvdWibWAOAHHCZTMPhV3xIlH4/cUEIA8OV5QnYpaVoA==", + "requires": {} + }, + "@redis/time-series": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.4.tgz", + "integrity": "sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==", + "requires": {} + }, "@sinclair/typebox": { "version": "0.25.24", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.25.24.tgz", @@ -9604,6 +9739,11 @@ "wrap-ansi": "^7.0.0" } }, + "cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==" + }, "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -10332,6 +10472,11 @@ "is-property": "^1.0.2" } }, + "generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==" + }, "gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -12454,6 +12599,19 @@ "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==", "dev": true }, + "redis": { + "version": "4.6.6", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.6.6.tgz", + "integrity": "sha512-aLs2fuBFV/VJ28oLBqYykfnhGGkFxvx0HdCEBYdJ99FFbSEMZ7c1nVKwR6ZRv+7bb7JnC0mmCzaqu8frgOYhpA==", + "requires": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.5.7", + "@redis/graph": "1.1.0", + "@redis/json": "1.0.4", + "@redis/search": "1.1.2", + "@redis/time-series": "1.0.4" + } + }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", diff --git a/backend/package.json b/backend/package.json index 7ebc2e970..24da55e17 100644 --- a/backend/package.json +++ b/backend/package.json @@ -47,13 +47,14 @@ "maxmind": "~4.3.11", "mysql2": "~3.5.2", "rust-gbt": "file:./rust-gbt", + "redis": "^4.6.6", "socks-proxy-agent": "~7.0.0", "typescript": "~4.9.3", "ws": "~8.13.0" }, "devDependencies": { - "@babel/core": "^7.21.3", "@babel/code-frame": "^7.18.6", + "@babel/core": "^7.21.3", "@types/compression": "^1.7.2", "@types/crypto-js": "^4.1.1", "@types/express": "^4.17.17", diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 21c8e51c0..dfd7f0030 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -10,6 +10,7 @@ "AUTOMATIC_BLOCK_REINDEXING": false, "POLL_RATE_MS": 3, "CACHE_DIR": "__MEMPOOL_CACHE_DIR__", + "CACHE_ENABLED": true, "CLEAR_PROTECTION_MINUTES": 4, "RECOMMENDED_FEE_PERCENTILE": 5, "BLOCK_WEIGHT_UNITS": 6, @@ -131,5 +132,9 @@ "MEMPOOL_SERVICES": { "API": "", "ACCELERATIONS": false + }, + "REDIS": { + "ENABLED": false, + "UNIX_SOCKET_PATH": "/tmp/redis.sock" } } diff --git a/backend/src/__tests__/api/difficulty-adjustment.test.ts b/backend/src/__tests__/api/difficulty-adjustment.test.ts index da1ac0d2c..c3e8e1a88 100644 --- a/backend/src/__tests__/api/difficulty-adjustment.test.ts +++ b/backend/src/__tests__/api/difficulty-adjustment.test.ts @@ -1,4 +1,8 @@ -import { calcDifficultyAdjustment, DifficultyAdjustment } from '../../api/difficulty-adjustment'; +import { + calcBitsDifference, + calcDifficultyAdjustment, + DifficultyAdjustment, +} from '../../api/difficulty-adjustment'; describe('Mempool Difficulty Adjustment', () => { test('should calculate Difficulty Adjustments properly', () => { @@ -86,4 +90,46 @@ describe('Mempool Difficulty Adjustment', () => { expect(result).toStrictEqual(vector[1]); } }); + + test('should calculate Difficulty change from bits fields of two blocks', () => { + // Check same exponent + check min max for output + expect(calcBitsDifference(0x1d000200, 0x1d000100)).toEqual(100); + expect(calcBitsDifference(0x1d000400, 0x1d000100)).toEqual(300); + expect(calcBitsDifference(0x1d000800, 0x1d000100)).toEqual(300); // Actually 700 + expect(calcBitsDifference(0x1d000100, 0x1d000200)).toEqual(-50); + expect(calcBitsDifference(0x1d000100, 0x1d000400)).toEqual(-75); + expect(calcBitsDifference(0x1d000100, 0x1d000800)).toEqual(-75); // Actually -87.5 + // Check new higher exponent + expect(calcBitsDifference(0x1c000200, 0x1d000001)).toEqual(100); + expect(calcBitsDifference(0x1c000400, 0x1d000001)).toEqual(300); + expect(calcBitsDifference(0x1c000800, 0x1d000001)).toEqual(300); + expect(calcBitsDifference(0x1c000100, 0x1d000002)).toEqual(-50); + expect(calcBitsDifference(0x1c000100, 0x1d000004)).toEqual(-75); + expect(calcBitsDifference(0x1c000100, 0x1d000008)).toEqual(-75); + // Check new lower exponent + expect(calcBitsDifference(0x1d000002, 0x1c000100)).toEqual(100); + expect(calcBitsDifference(0x1d000004, 0x1c000100)).toEqual(300); + expect(calcBitsDifference(0x1d000008, 0x1c000100)).toEqual(300); + expect(calcBitsDifference(0x1d000001, 0x1c000200)).toEqual(-50); + expect(calcBitsDifference(0x1d000001, 0x1c000400)).toEqual(-75); + expect(calcBitsDifference(0x1d000001, 0x1c000800)).toEqual(-75); + // Check error when exponents are too far apart + expect(() => calcBitsDifference(0x1d000001, 0x1a000800)).toThrow( + /Impossible exponent difference/ + ); + // Check invalid inputs + expect(() => calcBitsDifference(0x7f000001, 0x1a000800)).toThrow( + /Invalid bits/ + ); + expect(() => calcBitsDifference(0, 0x1a000800)).toThrow(/Invalid bits/); + expect(() => calcBitsDifference(100.2783, 0x1a000800)).toThrow( + /Invalid bits/ + ); + expect(() => calcBitsDifference(0x00800000, 0x1a000800)).toThrow( + /Invalid bits/ + ); + expect(() => calcBitsDifference(0x1c000000, 0x1a000800)).toThrow( + /Invalid bits/ + ); + }); }); diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 2c83d1af5..d575c95d3 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -23,6 +23,7 @@ describe('Mempool Backend Config', () => { AUTOMATIC_BLOCK_REINDEXING: false, POLL_RATE_MS: 2000, CACHE_DIR: './cache', + CACHE_ENABLED: true, CLEAR_PROTECTION_MINUTES: 20, RECOMMENDED_FEE_PERCENTILE: 50, BLOCK_WEIGHT_UNITS: 4000000, @@ -132,6 +133,11 @@ describe('Mempool Backend Config', () => { API: "", ACCELERATIONS: false, }); + + expect(config.REDIS).toStrictEqual({ + ENABLED: false, + UNIX_SOCKET_PATH: '' + }); }); }); @@ -167,6 +173,8 @@ describe('Mempool Backend Config', () => { expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER); expect(config.MEMPOOL_SERVICES).toStrictEqual(fixture.MEMPOOL_SERVICES); + + expect(config.REDIS).toStrictEqual(fixture.REDIS); }); }); @@ -180,12 +188,12 @@ describe('Mempool Backend Config', () => { // We have a few cases where we can't follow the pattern if (root === 'MEMPOOL' && key === 'HTTP_PORT') { console.log('skipping check for MEMPOOL_HTTP_PORT'); - return; + continue; } switch (typeof value) { case 'object': { if (Array.isArray(value)) { - return; + continue; } else { parseJson(value, key); } diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index a1cf767d9..132cda91a 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -5,6 +5,7 @@ import { IEsploraApi } from './esplora-api.interface'; import blocks from '../blocks'; import mempool from '../mempool'; import { TransactionExtended } from '../../mempool.interfaces'; +import transactionUtils from '../transaction-utils'; class BitcoinApi implements AbstractBitcoinApi { private rawMempoolCache: IBitcoinApi.RawMempool | null = null; @@ -63,9 +64,16 @@ class BitcoinApi implements AbstractBitcoinApi { return Promise.resolve([]); } - $getTransactionHex(txId: string): Promise { - return this.$getRawTransaction(txId, true) - .then((tx) => tx.hex || ''); + async $getTransactionHex(txId: string): Promise { + const txInMempool = mempool.getMempool()[txId]; + if (txInMempool && txInMempool.hex) { + return txInMempool.hex; + } + + return this.bitcoindClient.getRawTransaction(txId, true) + .then((transaction: IBitcoinApi.Transaction) => { + return transaction.hex; + }); } $getBlockHeightTip(): Promise { @@ -209,7 +217,7 @@ class BitcoinApi implements AbstractBitcoinApi { scriptpubkey: vout.scriptPubKey.hex, scriptpubkey_address: vout.scriptPubKey && vout.scriptPubKey.address ? vout.scriptPubKey.address : vout.scriptPubKey.addresses ? vout.scriptPubKey.addresses[0] : '', - scriptpubkey_asm: vout.scriptPubKey.asm ? this.convertScriptSigAsm(vout.scriptPubKey.hex) : '', + scriptpubkey_asm: vout.scriptPubKey.asm ? transactionUtils.convertScriptSigAsm(vout.scriptPubKey.hex) : '', scriptpubkey_type: this.translateScriptPubKeyType(vout.scriptPubKey.type), }; }); @@ -219,7 +227,7 @@ class BitcoinApi implements AbstractBitcoinApi { is_coinbase: !!vin.coinbase, prevout: null, scriptsig: vin.scriptSig && vin.scriptSig.hex || vin.coinbase || '', - scriptsig_asm: vin.scriptSig && this.convertScriptSigAsm(vin.scriptSig.hex) || '', + scriptsig_asm: vin.scriptSig && transactionUtils.convertScriptSigAsm(vin.scriptSig.hex) || '', sequence: vin.sequence, txid: vin.txid || '', vout: vin.vout || 0, @@ -291,7 +299,7 @@ class BitcoinApi implements AbstractBitcoinApi { } const innerTx = await this.$getRawTransaction(vin.txid, false, false); vin.prevout = innerTx.vout[vin.vout]; - this.addInnerScriptsToVin(vin); + transactionUtils.addInnerScriptsToVin(vin); } return transaction; } @@ -330,7 +338,7 @@ class BitcoinApi implements AbstractBitcoinApi { } const innerTx = await this.$getRawTransaction(transaction.vin[i].txid, false, false); transaction.vin[i].prevout = innerTx.vout[transaction.vin[i].vout]; - this.addInnerScriptsToVin(transaction.vin[i]); + transactionUtils.addInnerScriptsToVin(transaction.vin[i]); totalIn += innerTx.vout[transaction.vin[i].vout].value; } if (lazyPrevouts && transaction.vin.length > 12) { @@ -342,122 +350,6 @@ class BitcoinApi implements AbstractBitcoinApi { return transaction; } - private convertScriptSigAsm(hex: string): string { - const buf = Buffer.from(hex, 'hex'); - - const b: string[] = []; - - let i = 0; - while (i < buf.length) { - const op = buf[i]; - if (op >= 0x01 && op <= 0x4e) { - i++; - let push: number; - if (op === 0x4c) { - push = buf.readUInt8(i); - b.push('OP_PUSHDATA1'); - i += 1; - } else if (op === 0x4d) { - push = buf.readUInt16LE(i); - b.push('OP_PUSHDATA2'); - i += 2; - } else if (op === 0x4e) { - push = buf.readUInt32LE(i); - b.push('OP_PUSHDATA4'); - i += 4; - } else { - push = op; - b.push('OP_PUSHBYTES_' + push); - } - - const data = buf.slice(i, i + push); - if (data.length !== push) { - break; - } - - b.push(data.toString('hex')); - i += data.length; - } else { - if (op === 0x00) { - b.push('OP_0'); - } else if (op === 0x4f) { - b.push('OP_PUSHNUM_NEG1'); - } else if (op === 0xb1) { - b.push('OP_CLTV'); - } else if (op === 0xb2) { - b.push('OP_CSV'); - } else if (op === 0xba) { - b.push('OP_CHECKSIGADD'); - } else { - const opcode = bitcoinjs.script.toASM([ op ]); - if (opcode && op < 0xfd) { - if (/^OP_(\d+)$/.test(opcode)) { - b.push(opcode.replace(/^OP_(\d+)$/, 'OP_PUSHNUM_$1')); - } else { - b.push(opcode); - } - } else { - b.push('OP_RETURN_' + op); - } - } - i += 1; - } - } - - return b.join(' '); - } - - private addInnerScriptsToVin(vin: IEsploraApi.Vin): void { - if (!vin.prevout) { - return; - } - - if (vin.prevout.scriptpubkey_type === 'p2sh') { - const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0]; - vin.inner_redeemscript_asm = this.convertScriptSigAsm(redeemScript); - if (vin.witness && vin.witness.length > 2) { - const witnessScript = vin.witness[vin.witness.length - 1]; - vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript); - } - } - - if (vin.prevout.scriptpubkey_type === 'v0_p2wsh' && vin.witness) { - const witnessScript = vin.witness[vin.witness.length - 1]; - vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript); - } - - if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness) { - const witnessScript = this.witnessToP2TRScript(vin.witness); - if (witnessScript !== null) { - vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript); - } - } - } - - /** - * This function must only be called when we know the witness we are parsing - * is a taproot witness. - * @param witness An array of hex strings that represents the witness stack of - * the input. - * @returns null if the witness is not a script spend, and the hex string of - * the script item if it is a script spend. - */ - private witnessToP2TRScript(witness: string[]): string | null { - if (witness.length < 2) return null; - // Note: see BIP341 for parsing details of witness stack - - // If there are at least two witness elements, and the first byte of the - // last element is 0x50, this last element is called annex a and - // is removed from the witness stack. - const hasAnnex = witness[witness.length - 1].substring(0, 2) === '50'; - // If there are at least two witness elements left, script path spending is used. - // Call the second-to-last stack element s, the script. - // (Note: this phrasing from BIP341 assumes we've *removed* the annex from the stack) - if (hasAnnex && witness.length < 3) return null; - const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2; - return witness[positionOfScript]; - } - } export default BitcoinApi; diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index e2887b706..90a31ecae 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -6,7 +6,7 @@ import websocketHandler from '../websocket-handler'; import mempool from '../mempool'; import feeApi from '../fee-api'; import mempoolBlocks from '../mempool-blocks'; -import bitcoinApi, { bitcoinCoreApi } from './bitcoin-api-factory'; +import bitcoinApi from './bitcoin-api-factory'; import { Common } from '../common'; import backendInfo from '../backend-info'; import transactionUtils from '../transaction-utils'; @@ -484,7 +484,7 @@ class BitcoinRoutes { returnBlocks.push(localBlock); nextHash = localBlock.previousblockhash; } else { - const block = await bitcoinCoreApi.$getBlock(nextHash); + const block = await bitcoinApi.$getBlock(nextHash); returnBlocks.push(block); nextHash = block.previousblockhash; } @@ -577,7 +577,7 @@ class BitcoinRoutes { } try { - const addressData = await bitcoinApi.$getScriptHash(req.params.address); + const addressData = await bitcoinApi.$getScriptHash(req.params.scripthash); res.json(addressData); } catch (e) { if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) { @@ -598,7 +598,7 @@ class BitcoinRoutes { if (req.query.after_txid && typeof req.query.after_txid === 'string') { lastTxId = req.query.after_txid; } - const transactions = await bitcoinApi.$getScriptHashTransactions(req.params.address, lastTxId); + const transactions = await bitcoinApi.$getScriptHashTransactions(req.params.scripthash, lastTxId); res.json(transactions); } catch (e) { if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) { diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 4dbf4305e..33196b052 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -26,12 +26,15 @@ import PricesRepository from '../repositories/PricesRepository'; import priceUpdater from '../tasks/price-updater'; import chainTips from './chain-tips'; import websocketHandler from './websocket-handler'; +import redisCache from './redis-cache'; +import rbfCache from './rbf-cache'; +import { calcBitsDifference } from './difficulty-adjustment'; class Blocks { private blocks: BlockExtended[] = []; private blockSummaries: BlockSummary[] = []; private currentBlockHeight = 0; - private currentDifficulty = 0; + private currentBits = 0; private lastDifficultyAdjustmentTime = 0; private previousDifficultyRetarget = 0; private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = []; @@ -105,11 +108,16 @@ class Blocks { } } - // Skip expensive lookups while mempool has priority if (onlyCoinbase) { try { - const coinbase = await transactionUtils.$getTransactionExtended(txIds[0], false, false, false, addMempoolData); - return [coinbase]; + const coinbase = await transactionUtils.$getTransactionExtendedRetry(txIds[0], false, false, false, addMempoolData); + if (coinbase && coinbase.vin[0].is_coinbase) { + return [coinbase]; + } else { + const msg = `Expected a coinbase tx, but the backend API returned something else`; + logger.err(msg); + throw new Error(msg); + } } catch (e) { const msg = `Cannot fetch coinbase tx ${txIds[0]}. Reason: ` + (e instanceof Error ? e.message : e); logger.err(msg); @@ -134,17 +142,17 @@ class Blocks { // Fetch remaining txs individually for (const txid of txIds.filter(txid => !transactionMap[txid])) { - if (!transactionMap[txid]) { - if (!quiet && (totalFound % (Math.round((txIds.length) / 10)) === 0 || totalFound + 1 === txIds.length)) { // Avoid log spam - logger.debug(`Indexing tx ${totalFound + 1} of ${txIds.length} in block #${blockHeight}`); - } - try { - const tx = await transactionUtils.$getTransactionExtended(txid, false, false, false, addMempoolData); - transactionMap[txid] = tx; - totalFound++; - } catch (e) { - logger.err(`Cannot fetch tx ${txid}. Reason: ` + (e instanceof Error ? e.message : e)); - } + if (!quiet && (totalFound % (Math.round((txIds.length) / 10)) === 0 || totalFound + 1 === txIds.length)) { // Avoid log spam + logger.debug(`Indexing tx ${totalFound + 1} of ${txIds.length} in block #${blockHeight}`); + } + try { + const tx = await transactionUtils.$getTransactionExtendedRetry(txid, false, false, false, addMempoolData); + transactionMap[txid] = tx; + totalFound++; + } catch (e) { + const msg = `Cannot fetch tx ${txid}. Reason: ` + (e instanceof Error ? e.message : e); + logger.err(msg); + throw new Error(msg); } } @@ -152,8 +160,24 @@ class Blocks { logger.debug(`${foundInMempool} of ${txIds.length} found in mempool. ${totalFound - foundInMempool} fetched through backend service.`); } + // Require the first transaction to be a coinbase + const coinbase = transactionMap[txIds[0]]; + if (!coinbase || !coinbase.vin[0].is_coinbase) { + const msg = `Expected first tx in a block to be a coinbase, but found something else`; + logger.err(msg); + throw new Error(msg); + } + + // Require all transactions to be present + // (we should have thrown an error already if a tx request failed) + if (txIds.some(txid => !transactionMap[txid])) { + const msg = `Failed to fetch ${txIds.length - totalFound} transactions from block`; + logger.err(msg); + throw new Error(msg); + } + // Return list of transactions, preserving block order - return txIds.map(txid => transactionMap[txid]).filter(tx => tx != null); + return txIds.map(txid => transactionMap[txid]); } /** @@ -396,8 +420,8 @@ class Blocks { let newlyIndexed = 0; let totalIndexed = indexedBlockSummariesHashesArray.length; let indexedThisRun = 0; - let timer = new Date().getTime() / 1000; - const startedAt = new Date().getTime() / 1000; + let timer = Date.now() / 1000; + const startedAt = Date.now() / 1000; for (const block of indexedBlocks) { if (indexedBlockSummariesHashes[block.hash] === true) { @@ -405,17 +429,24 @@ class Blocks { } // Logging - const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); + const elapsedSeconds = (Date.now() / 1000) - timer; if (elapsedSeconds > 5) { - const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = indexedThisRun / elapsedSeconds; const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100; - logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining); - timer = new Date().getTime() / 1000; + logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`, logger.tags.mining); + timer = Date.now() / 1000; indexedThisRun = 0; } - await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary + + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendTransaction(tx)); + const cpfpSummary = await this.$indexCPFP(block.hash, block.height, txs); + await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary + } else { + await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary + } // Logging indexedThisRun++; @@ -454,18 +485,18 @@ class Blocks { // Logging let count = 0; let countThisRun = 0; - let timer = new Date().getTime() / 1000; - const startedAt = new Date().getTime() / 1000; + let timer = Date.now() / 1000; + const startedAt = Date.now() / 1000; for (const height of unindexedBlockHeights) { // Logging const hash = await bitcoinApi.$getBlockHash(height); - const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); + const elapsedSeconds = (Date.now() / 1000) - timer; if (elapsedSeconds > 5) { - const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = (countThisRun / elapsedSeconds); + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = countThisRun / elapsedSeconds; const progress = Math.round(count / unindexedBlockHeights.length * 10000) / 100; - logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor} seconds`); - timer = new Date().getTime() / 1000; + logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`); + timer = Date.now() / 1000; countThisRun = 0; } @@ -544,8 +575,8 @@ class Blocks { let totalIndexed = await blocksRepository.$blockCountBetweenHeight(currentBlockHeight, lastBlockToIndex); let indexedThisRun = 0; let newlyIndexed = 0; - const startedAt = new Date().getTime() / 1000; - let timer = new Date().getTime() / 1000; + const startedAt = Date.now() / 1000; + let timer = Date.now() / 1000; while (currentBlockHeight >= lastBlockToIndex) { const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1); @@ -565,18 +596,18 @@ class Blocks { } ++indexedThisRun; ++totalIndexed; - const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); + const elapsedSeconds = (Date.now() / 1000) - timer; if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) { - const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = indexedThisRun / elapsedSeconds; const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100; - logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining); - timer = new Date().getTime() / 1000; + logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress.toFixed(2)}%) | elapsed: ${runningFor.toFixed(2)} seconds`, logger.tags.mining); + timer = Date.now() / 1000; indexedThisRun = 0; loadingIndicators.setProgress('block-indexing', progress, false); } const blockHash = await bitcoinApi.$getBlockHash(blockHeight); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, null, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -633,17 +664,17 @@ class Blocks { const heightDiff = blockHeightTip % 2016; const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff); this.updateTimerProgress(timer, 'got block hash for initial difficulty adjustment'); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); this.updateTimerProgress(timer, 'got block for initial difficulty adjustment'); this.lastDifficultyAdjustmentTime = block.timestamp; - this.currentDifficulty = block.difficulty; + this.currentBits = block.bits; if (blockHeightTip >= 2016) { const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016); this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment'); - const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash); + const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash); this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment'); - this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100; + this.previousDifficultyRetarget = calcBitsDifference(previousPeriodBlock.bits, block.bits); logger.debug(`Initial difficulty adjustment data set.`); } } else { @@ -667,14 +698,14 @@ class Blocks { const block = BitcoinApi.convertBlock(verboseBlock); const txIds: string[] = verboseBlock.tx.map(tx => tx.txid); const transactions = await this.$getTransactionsExtended(blockHash, block.height, false, txIds, false, true) as MempoolTransactionExtended[]; - if (config.MEMPOOL.BACKEND !== 'esplora') { - // fill in missing transaction fee data from verboseBlock - for (let i = 0; i < transactions.length; i++) { - if (!transactions[i].fee && transactions[i].txid === verboseBlock.tx[i].txid) { - transactions[i].fee = verboseBlock.tx[i].fee * 100_000_000; - } + + // fill in missing transaction fee data from verboseBlock + for (let i = 0; i < transactions.length; i++) { + if (!transactions[i].fee && transactions[i].txid === verboseBlock.tx[i].txid) { + transactions[i].fee = (verboseBlock.tx[i].fee * 100_000_000) || 0; } } + const cpfpSummary: CpfpSummary = Common.calculateCpfp(block.height, transactions); const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions); const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, cpfpSummary.transactions); @@ -756,14 +787,18 @@ class Blocks { time: block.timestamp, height: block.height, difficulty: block.difficulty, - adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise + adjustment: Math.round( + // calcBitsDifference returns +- percentage, +100 returns to positive, /100 returns to ratio. + // Instead of actually doing /100, just reduce the multiplier. + (calcBitsDifference(this.currentBits, block.bits) + 100) * 10000 + ) / 1000000, // Remove float point noise }); this.updateTimerProgress(timer, `saved difficulty adjustment for ${this.currentBlockHeight}`); } - this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100; + this.previousDifficultyRetarget = calcBitsDifference(this.currentBits, block.bits); this.lastDifficultyAdjustmentTime = block.timestamp; - this.currentDifficulty = block.difficulty; + this.currentBits = block.bits; } // wait for pending async callbacks to finish @@ -783,10 +818,18 @@ class Blocks { if (this.newBlockCallbacks.length) { this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions)); } - if (!memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) { + if (config.MEMPOOL.CACHE_ENABLED && !memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) { diskCache.$saveCacheToDisk(); } + // Update Redis cache + if (config.REDIS.ENABLED) { + await redisCache.$updateBlocks(this.blocks); + await redisCache.$updateBlockSummaries(this.blockSummaries); + await redisCache.$removeTransactions(txIds); + await rbfCache.updateCache(); + } + handledBlocks++; } @@ -831,7 +874,7 @@ class Blocks { } const blockHash = await bitcoinApi.$getBlockHash(height); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -843,7 +886,7 @@ class Blocks { } public async $indexStaleBlock(hash: string): Promise { - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash); const transactions = await this.$getTransactionsExtended(hash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -868,7 +911,7 @@ class Blocks { } // Bitcoin network, add our custom data on top - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash); if (block.stale) { return await this.$indexStaleBlock(hash); } else { @@ -903,7 +946,7 @@ class Blocks { transactions: cpfpSummary.transactions.map(tx => { return { txid: tx.txid, - fee: tx.fee, + fee: tx.fee || 0, vsize: tx.vsize, value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0)), rate: tx.effectiveFeePerVsize @@ -911,10 +954,15 @@ class Blocks { }), }; } else { - // Call Core RPC - const block = await bitcoinClient.getBlock(hash, 2); - summary = this.summarizeBlock(block); - height = block.height; + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx)); + summary = this.summarizeBlockTransactions(hash, txs); + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(hash, 2); + summary = this.summarizeBlock(block); + height = block.height; + } } if (height == null) { const block = await bitcoinApi.$getBlock(hash); @@ -1037,8 +1085,17 @@ class Blocks { if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) { cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); if (cleanBlock.fee_amt_percentiles === null) { - const block = await bitcoinClient.getBlock(cleanBlock.hash, 2); - const summary = this.summarizeBlock(block); + + let summary; + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(cleanBlock.hash)).map(tx => transactionUtils.extendTransaction(tx)); + summary = this.summarizeBlockTransactions(cleanBlock.hash, txs); + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(cleanBlock.hash, 2); + summary = this.summarizeBlock(block); + } + await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions); cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); } @@ -1098,19 +1155,29 @@ class Blocks { return this.currentBlockHeight; } - public async $indexCPFP(hash: string, height: number): Promise { - const block = await bitcoinClient.getBlock(hash, 2); - const transactions = block.tx.map(tx => { - tx.fee *= 100_000_000; - return tx; - }); + public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise { + let transactions = txs; + if (!transactions) { + if (config.MEMPOOL.BACKEND === 'esplora') { + transactions = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx)); + } + if (!transactions) { + const block = await bitcoinClient.getBlock(hash, 2); + transactions = block.tx.map(tx => { + tx.fee *= 100_000_000; + return tx; + }); + } + } - const summary = Common.calculateCpfp(height, transactions); + const summary = Common.calculateCpfp(height, transactions as TransactionExtended[]); await this.$saveCpfp(hash, height, summary); const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions); await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats); + + return summary; } public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise { diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index b2f476da3..64bac2036 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -108,7 +108,7 @@ export class Common { static stripTransaction(tx: TransactionExtended): TransactionStripped { return { txid: tx.txid, - fee: tx.fee, + fee: tx.fee || 0, vsize: tx.weight / 4, value: tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0), acc: tx.acceleration || undefined, diff --git a/backend/src/api/difficulty-adjustment.ts b/backend/src/api/difficulty-adjustment.ts index 1f37d8be9..23d0c33de 100644 --- a/backend/src/api/difficulty-adjustment.ts +++ b/backend/src/api/difficulty-adjustment.ts @@ -16,6 +16,68 @@ export interface DifficultyAdjustment { expectedBlocks: number; // Block count } +/** + * Calculate the difficulty increase/decrease by using the `bits` integer contained in two + * block headers. + * + * Warning: Only compare `bits` from blocks in two adjacent difficulty periods. This code + * assumes the maximum difference is x4 or /4 (as per the protocol) and will throw an + * error if an exponent difference of 2 or more is seen. + * + * @param {number} oldBits The 32 bit `bits` integer from a block header. + * @param {number} newBits The 32 bit `bits` integer from a block header in the next difficulty period. + * @returns {number} A floating point decimal of the difficulty change from old to new. + * (ie. 21.3 means 21.3% increase in difficulty, -21.3 is a 21.3% decrease in difficulty) + */ +export function calcBitsDifference(oldBits: number, newBits: number): number { + // Must be + // - integer + // - highest exponent is 0x1f, so max value (as integer) is 0x1f0000ff + // - min value is 1 (exponent = 0) + // - highest bit of the number-part is +- sign, it must not be 1 + const verifyBits = (bits: number): void => { + if ( + Math.floor(bits) !== bits || + bits > 0x1f0000ff || + bits < 1 || + (bits & 0x00800000) !== 0 || + (bits & 0x007fffff) === 0 + ) { + throw new Error('Invalid bits'); + } + }; + verifyBits(oldBits); + verifyBits(newBits); + + // No need to mask exponents because we checked the bounds above + const oldExp = oldBits >> 24; + const newExp = newBits >> 24; + const oldNum = oldBits & 0x007fffff; + const newNum = newBits & 0x007fffff; + // The diff can only possibly be 1, 0, -1 + // (because maximum difficulty change is x4 or /4 (2 bits up or down)) + let result: number; + switch (newExp - oldExp) { + // New less than old, target lowered, difficulty increased + case -1: + result = ((oldNum << 8) * 100) / newNum - 100; + break; + // Same exponent, compare numbers as is. + case 0: + result = (oldNum * 100) / newNum - 100; + break; + // Old less than new, target raised, difficulty decreased + case 1: + result = (oldNum * 100) / (newNum << 8) - 100; + break; + default: + throw new Error('Impossible exponent difference'); + } + + // Min/Max values + return result > 300 ? 300 : result < -75 ? -75 : result; +} + export function calcDifficultyAdjustment( DATime: number, nowSeconds: number, diff --git a/backend/src/api/disk-cache.ts b/backend/src/api/disk-cache.ts index 1e428d8b6..6f603489a 100644 --- a/backend/src/api/disk-cache.ts +++ b/backend/src/api/disk-cache.ts @@ -29,7 +29,7 @@ class DiskCache { }; constructor() { - if (!cluster.isPrimary) { + if (!cluster.isPrimary || !config.MEMPOOL.CACHE_ENABLED) { return; } process.on('SIGINT', (e) => { @@ -39,7 +39,7 @@ class DiskCache { } async $saveCacheToDisk(sync: boolean = false): Promise { - if (!cluster.isPrimary) { + if (!cluster.isPrimary || !config.MEMPOOL.CACHE_ENABLED) { return; } if (this.isWritingCache) { @@ -175,10 +175,11 @@ class DiskCache { } async $loadMempoolCache(): Promise { - if (!fs.existsSync(DiskCache.FILE_NAME)) { + if (!config.MEMPOOL.CACHE_ENABLED || !fs.existsSync(DiskCache.FILE_NAME)) { return; } try { + const start = Date.now(); let data: any = {}; const cacheData = fs.readFileSync(DiskCache.FILE_NAME, 'utf8'); if (cacheData) { @@ -220,6 +221,8 @@ class DiskCache { } } + logger.info(`Loaded mempool from disk cache in ${Date.now() - start} ms`); + await memPool.$setMempool(data.mempool); if (!this.ignoreBlocksCache) { blocks.setBlocks(data.blocks); diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 703e24f09..23eefd4f9 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -10,6 +10,7 @@ import bitcoinClient from './bitcoin/bitcoin-client'; import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; import rbfCache from './rbf-cache'; import accelerationApi, { Acceleration } from './services/acceleration'; +import redisCache from './redis-cache'; class Mempool { private inSync: boolean = false; @@ -88,6 +89,10 @@ class Mempool { public async $setMempool(mempoolData: { [txId: string]: MempoolTransactionExtended }) { this.mempoolCache = mempoolData; let count = 0; + const redisTimer = Date.now(); + if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) { + logger.debug(`Migrating ${Object.keys(this.mempoolCache).length} transactions from disk cache to Redis cache`); + } for (const txid of Object.keys(this.mempoolCache)) { if (!this.mempoolCache[txid].sigops || this.mempoolCache[txid].effectiveFeePerVsize == null) { this.mempoolCache[txid] = transactionUtils.extendMempoolTransaction(this.mempoolCache[txid]); @@ -96,6 +101,13 @@ class Mempool { this.mempoolCache[txid].order = transactionUtils.txidToOrdering(txid); } count++; + if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) { + await redisCache.$addTransaction(this.mempoolCache[txid]); + } + } + if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) { + await redisCache.$flushTransactions(); + logger.debug(`Finished migrating cache transactions in ${((Date.now() - redisTimer) / 1000).toFixed(2)} seconds`); } if (this.mempoolChangedCallback) { this.mempoolChangedCallback(this.mempoolCache, [], [], []); @@ -140,8 +152,8 @@ class Mempool { logger.err('failed to fetch bulk mempool transactions from esplora'); } } - return newTransactions; logger.info(`Done inserting loaded mempool transactions into local cache`); + return newTransactions; } public async $updateMemPoolInfo() { @@ -173,7 +185,7 @@ class Mempool { return txTimes; } - public async $updateMempool(transactions: string[]): Promise { + public async $updateMempool(transactions: string[], pollRate: number): Promise { logger.debug(`Updating mempool...`); // warn if this run stalls the main loop for more than 2 minutes @@ -210,6 +222,11 @@ class Mempool { logger.info(`Missing ${transactions.length - currentMempoolSize} mempool transactions, attempting to reload in bulk from esplora`); try { newTransactions = await this.$reloadMempool(transactions.length); + if (config.REDIS.ENABLED) { + for (const tx of newTransactions) { + await redisCache.$addTransaction(tx); + } + } loaded = true; } catch (e) { logger.err('failed to load mempool in bulk from esplora, falling back to fetching individual transactions'); @@ -232,6 +249,10 @@ class Mempool { } hasChange = true; newTransactions.push(transaction); + + if (config.REDIS.ENABLED) { + await redisCache.$addTransaction(transaction); + } } catch (e: any) { if (config.MEMPOOL.BACKEND === 'esplora' && e.response?.status === 404) { this.missingTxCount++; @@ -240,7 +261,7 @@ class Mempool { } } - if (Date.now() - intervalTimer > 5_000) { + if (Date.now() - intervalTimer > Math.max(pollRate * 2, 5_000)) { if (this.inSync) { // Break and restart mempool loop if we spend too much time processing // new transactions that may lead to falling behind on block height @@ -252,7 +273,7 @@ class Mempool { if (Math.floor(progress) < 100) { loadingIndicators.setProgress('mempool', progress); } - intervalTimer = Date.now() + intervalTimer = Date.now(); } } } @@ -325,6 +346,13 @@ class Mempool { loadingIndicators.setProgress('mempool', 100); } + // Update Redis cache + if (config.REDIS.ENABLED) { + await redisCache.$flushTransactions(); + await redisCache.$removeTransactions(deletedTransactions.map(tx => tx.txid)); + await rbfCache.updateCache(); + } + const end = new Date().getTime(); const time = end - start; logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`); diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index e190492b8..7376e7cf4 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -11,7 +11,7 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust import config from '../../config'; import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository'; import PricesRepository from '../../repositories/PricesRepository'; -import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory'; +import bitcoinApi from '../bitcoin/bitcoin-api-factory'; import { IEsploraApi } from '../bitcoin/esplora-api.interface'; import database from '../../database'; @@ -201,7 +201,7 @@ class Mining { try { const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); const genesisTimestamp = genesisBlock.timestamp * 1000; const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps(); @@ -312,7 +312,7 @@ class Mining { const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; try { - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); const genesisTimestamp = genesisBlock.timestamp * 1000; const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); const lastMidnight = this.getDateMidnight(new Date()); @@ -421,8 +421,9 @@ class Mining { } const blocks: any = await BlocksRepository.$getBlocksDifficulty(); - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); let currentDifficulty = genesisBlock.difficulty; + let currentBits = genesisBlock.bits; let totalIndexed = 0; if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) { @@ -436,6 +437,7 @@ class Mining { const oldestConsecutiveBlock = await BlocksRepository.$getOldestConsecutiveBlock(); if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== -1) { + currentBits = oldestConsecutiveBlock.bits; currentDifficulty = oldestConsecutiveBlock.difficulty; } @@ -443,10 +445,11 @@ class Mining { let timer = new Date().getTime() / 1000; for (const block of blocks) { - if (block.difficulty !== currentDifficulty) { + if (block.bits !== currentBits) { if (indexedHeights[block.height] === true) { // Already indexed if (block.height >= oldestConsecutiveBlock.height) { currentDifficulty = block.difficulty; + currentBits = block.bits; } continue; } @@ -464,6 +467,7 @@ class Mining { totalIndexed++; if (block.height >= oldestConsecutiveBlock.height) { currentDifficulty = block.difficulty; + currentBits = block.bits; } } diff --git a/backend/src/api/rbf-cache.ts b/backend/src/api/rbf-cache.ts index f28dd0de3..b5592252c 100644 --- a/backend/src/api/rbf-cache.ts +++ b/backend/src/api/rbf-cache.ts @@ -1,15 +1,17 @@ +import config from "../config"; import logger from "../logger"; import { MempoolTransactionExtended, TransactionStripped } from "../mempool.interfaces"; import bitcoinApi from './bitcoin/bitcoin-api-factory'; import { Common } from "./common"; +import redisCache from "./redis-cache"; -interface RbfTransaction extends TransactionStripped { +export interface RbfTransaction extends TransactionStripped { rbf?: boolean; mined?: boolean; fullRbf?: boolean; } -interface RbfTree { +export interface RbfTree { tx: RbfTransaction; time: number; interval?: number; @@ -28,6 +30,19 @@ export interface ReplacementInfo { newVsize: number; } +enum CacheOp { + Remove = 0, + Add = 1, + Change = 2, +} + +interface CacheEvent { + op: CacheOp; + type: 'tx' | 'tree' | 'exp'; + txid: string, + value?: any, +} + class RbfCache { private replacedBy: Map = new Map(); private replaces: Map = new Map(); @@ -36,11 +51,43 @@ class RbfCache { private treeMap: Map = new Map(); // map of txids to sequence ids private txs: Map = new Map(); private expiring: Map = new Map(); + private cacheQueue: CacheEvent[] = []; constructor() { setInterval(this.cleanup.bind(this), 1000 * 60 * 10); } + private addTx(txid: string, tx: MempoolTransactionExtended): void { + this.txs.set(txid, tx); + this.cacheQueue.push({ op: CacheOp.Add, type: 'tx', txid }); + } + + private addTree(txid: string, tree: RbfTree): void { + this.rbfTrees.set(txid, tree); + this.dirtyTrees.add(txid); + this.cacheQueue.push({ op: CacheOp.Add, type: 'tree', txid }); + } + + private addExpiration(txid: string, expiry: number): void { + this.expiring.set(txid, expiry); + this.cacheQueue.push({ op: CacheOp.Add, type: 'exp', txid, value: expiry }); + } + + private removeTx(txid: string): void { + this.txs.delete(txid); + this.cacheQueue.push({ op: CacheOp.Remove, type: 'tx', txid }); + } + + private removeTree(txid: string): void { + this.rbfTrees.delete(txid); + this.cacheQueue.push({ op: CacheOp.Remove, type: 'tree', txid }); + } + + private removeExpiration(txid: string): void { + this.expiring.delete(txid); + this.cacheQueue.push({ op: CacheOp.Remove, type: 'exp', txid }); + } + public add(replaced: MempoolTransactionExtended[], newTxExtended: MempoolTransactionExtended): void { if (!newTxExtended || !replaced?.length || this.txs.has(newTxExtended.txid)) { return; @@ -49,7 +96,7 @@ class RbfCache { const newTx = Common.stripTransaction(newTxExtended) as RbfTransaction; const newTime = newTxExtended.firstSeen || (Date.now() / 1000); newTx.rbf = newTxExtended.vin.some((v) => v.sequence < 0xfffffffe); - this.txs.set(newTx.txid, newTxExtended); + this.addTx(newTx.txid, newTxExtended); // maintain rbf trees let txFullRbf = false; @@ -66,7 +113,7 @@ class RbfCache { const treeId = this.treeMap.get(replacedTx.txid); if (treeId) { const tree = this.rbfTrees.get(treeId); - this.rbfTrees.delete(treeId); + this.removeTree(treeId); if (tree) { tree.interval = newTime - tree?.time; replacedTrees.push(tree); @@ -83,7 +130,7 @@ class RbfCache { replaces: [], }); treeFullRbf = treeFullRbf || !replacedTx.rbf; - this.txs.set(replacedTx.txid, replacedTxExtended); + this.addTx(replacedTx.txid, replacedTxExtended); } } newTx.fullRbf = txFullRbf; @@ -94,10 +141,9 @@ class RbfCache { fullRbf: treeFullRbf, replaces: replacedTrees }; - this.rbfTrees.set(treeId, newTree); + this.addTree(treeId, newTree); this.updateTreeMap(treeId, newTree); this.replaces.set(newTx.txid, replacedTrees.map(tree => tree.tx.txid)); - this.dirtyTrees.add(treeId); } public has(txId: string): boolean { @@ -191,6 +237,7 @@ class RbfCache { this.setTreeMined(tree, txid); tree.mined = true; this.dirtyTrees.add(treeId); + this.cacheQueue.push({ op: CacheOp.Change, type: 'tree', txid: treeId }); } } this.evict(txid); @@ -199,7 +246,8 @@ class RbfCache { // flag a transaction as removed from the mempool public evict(txid: string, fast: boolean = false): void { if (this.txs.has(txid) && (fast || !this.expiring.has(txid))) { - this.expiring.set(txid, fast ? Date.now() + (1000 * 60 * 10) : Date.now() + (1000 * 86400)); // 24 hours + const expiryTime = fast ? Date.now() + (1000 * 60 * 10) : Date.now() + (1000 * 86400); // 24 hours + this.addExpiration(txid, expiryTime); } } @@ -220,11 +268,11 @@ class RbfCache { const now = Date.now(); for (const txid of this.expiring.keys()) { if ((this.expiring.get(txid) || 0) < now) { - this.expiring.delete(txid); + this.removeExpiration(txid); this.remove(txid); } } - logger.debug(`rbf cache contains ${this.txs.size} txs, ${this.expiring.size} due to expire`); + logger.debug(`rbf cache contains ${this.txs.size} txs, ${this.rbfTrees.size} trees, ${this.expiring.size} due to expire`); } // remove a transaction & all previous versions from the cache @@ -234,14 +282,14 @@ class RbfCache { const replaces = this.replaces.get(txid); this.replaces.delete(txid); this.treeMap.delete(txid); - this.txs.delete(txid); - this.expiring.delete(txid); + this.removeTx(txid); + this.removeExpiration(txid); for (const tx of (replaces || [])) { // recursively remove prior versions from the cache this.replacedBy.delete(tx); // if this is the id of a tree, remove that too if (this.treeMap.get(tx) === tx) { - this.rbfTrees.delete(tx); + this.removeTree(tx); } this.remove(tx); } @@ -273,6 +321,33 @@ class RbfCache { } } + public async updateCache(): Promise { + if (!config.REDIS.ENABLED) { + return; + } + // Update the Redis cache by replaying queued events + for (const e of this.cacheQueue) { + if (e.op === CacheOp.Add || e.op === CacheOp.Change) { + let value = e.value; + switch(e.type) { + case 'tx': { + value = this.txs.get(e.txid); + } break; + case 'tree': { + const tree = this.rbfTrees.get(e.txid); + value = tree ? this.exportTree(tree) : null; + } break; + } + if (value != null) { + await redisCache.$setRbfEntry(e.type, e.txid, value); + } + } else if (e.op === CacheOp.Remove) { + await redisCache.$removeRbfEntry(e.type, e.txid); + } + } + this.cacheQueue = []; + } + public dump(): any { const trees = Array.from(this.rbfTrees.values()).map((tree: RbfTree) => { return this.exportTree(tree); }); @@ -285,14 +360,14 @@ class RbfCache { public async load({ txs, trees, expiring }): Promise { txs.forEach(txEntry => { - this.txs.set(txEntry[0], txEntry[1]); + this.txs.set(txEntry.key, txEntry.value); }); for (const deflatedTree of trees) { await this.importTree(deflatedTree.root, deflatedTree.root, deflatedTree, this.txs); } expiring.forEach(expiringEntry => { - if (this.txs.has(expiringEntry[0])) { - this.expiring.set(expiringEntry[0], new Date(expiringEntry[1]).getTime()); + if (this.txs.has(expiringEntry.key)) { + this.expiring.set(expiringEntry.key, new Date(expiringEntry.value).getTime()); } }); this.cleanup(); @@ -378,8 +453,7 @@ class RbfCache { }; this.treeMap.set(txid, root); if (root === txid) { - this.rbfTrees.set(root, tree); - this.dirtyTrees.add(root); + this.addTree(root, tree); } return tree; } diff --git a/backend/src/api/redis-cache.ts b/backend/src/api/redis-cache.ts new file mode 100644 index 000000000..fcde8013a --- /dev/null +++ b/backend/src/api/redis-cache.ts @@ -0,0 +1,276 @@ +import { createClient } from 'redis'; +import memPool from './mempool'; +import blocks from './blocks'; +import logger from '../logger'; +import config from '../config'; +import { BlockExtended, BlockSummary, MempoolTransactionExtended } from '../mempool.interfaces'; +import rbfCache from './rbf-cache'; +import transactionUtils from './transaction-utils'; + +enum NetworkDB { + mainnet = 0, + testnet, + signet, + liquid, + liquidtestnet, +} + +class RedisCache { + private client; + private connected = false; + private schemaVersion = 1; + + private cacheQueue: MempoolTransactionExtended[] = []; + private txFlushLimit: number = 10000; + + constructor() { + if (config.REDIS.ENABLED) { + const redisConfig = { + socket: { + path: config.REDIS.UNIX_SOCKET_PATH + }, + database: NetworkDB[config.MEMPOOL.NETWORK], + }; + this.client = createClient(redisConfig); + this.client.on('error', (e) => { + logger.err(`Error in Redis client: ${e instanceof Error ? e.message : e}`); + }); + this.$ensureConnected(); + } + } + + private async $ensureConnected(): Promise { + if (!this.connected && config.REDIS.ENABLED) { + return this.client.connect().then(async () => { + this.connected = true; + logger.info(`Redis client connected`); + const version = await this.client.get('schema_version'); + if (version !== this.schemaVersion) { + // schema changed + // perform migrations or flush DB if necessary + logger.info(`Redis schema version changed from ${version} to ${this.schemaVersion}`); + await this.client.set('schema_version', this.schemaVersion); + } + }); + } + } + + async $updateBlocks(blocks: BlockExtended[]) { + try { + await this.$ensureConnected(); + await this.client.set('blocks', JSON.stringify(blocks)); + logger.debug(`Saved latest blocks to Redis cache`); + } catch (e) { + logger.warn(`Failed to update blocks in Redis cache: ${e instanceof Error ? e.message : e}`); + } + } + + async $updateBlockSummaries(summaries: BlockSummary[]) { + try { + await this.$ensureConnected(); + await this.client.set('block-summaries', JSON.stringify(summaries)); + logger.debug(`Saved latest block summaries to Redis cache`); + } catch (e) { + logger.warn(`Failed to update block summaries in Redis cache: ${e instanceof Error ? e.message : e}`); + } + } + + async $addTransaction(tx: MempoolTransactionExtended) { + this.cacheQueue.push(tx); + if (this.cacheQueue.length >= this.txFlushLimit) { + await this.$flushTransactions(); + } + } + + async $flushTransactions() { + const success = await this.$addTransactions(this.cacheQueue); + if (success) { + logger.debug(`Saved ${this.cacheQueue.length} transactions to Redis cache`); + this.cacheQueue = []; + } else { + logger.err(`Failed to save ${this.cacheQueue.length} transactions to Redis cache`); + } + } + + private async $addTransactions(newTransactions: MempoolTransactionExtended[]): Promise { + if (!newTransactions.length) { + return true; + } + try { + await this.$ensureConnected(); + const msetData = newTransactions.map(tx => { + const minified: any = { ...tx }; + delete minified.hex; + for (const vin of minified.vin) { + delete vin.inner_redeemscript_asm; + delete vin.inner_witnessscript_asm; + delete vin.scriptsig_asm; + } + for (const vout of minified.vout) { + delete vout.scriptpubkey_asm; + } + return [`mempool:tx:${tx.txid}`, JSON.stringify(minified)]; + }); + await this.client.MSET(msetData); + return true; + } catch (e) { + logger.warn(`Failed to add ${newTransactions.length} transactions to Redis cache: ${e instanceof Error ? e.message : e}`); + return false; + } + } + + async $removeTransactions(transactions: string[]) { + try { + await this.$ensureConnected(); + for (let i = 0; i < Math.ceil(transactions.length / 10000); i++) { + const slice = transactions.slice(i * 10000, (i + 1) * 10000); + await this.client.unlink(slice.map(txid => `mempool:tx:${txid}`)); + logger.debug(`Deleted ${slice.length} transactions from the Redis cache`); + } + } catch (e) { + logger.warn(`Failed to remove ${transactions.length} transactions from Redis cache: ${e instanceof Error ? e.message : e}`); + } + } + + async $setRbfEntry(type: string, txid: string, value: any): Promise { + try { + await this.$ensureConnected(); + await this.client.set(`rbf:${type}:${txid}`, JSON.stringify(value)); + } catch (e) { + logger.warn(`Failed to set RBF ${type} in Redis cache: ${e instanceof Error ? e.message : e}`); + } + } + + async $removeRbfEntry(type: string, txid: string): Promise { + try { + await this.$ensureConnected(); + await this.client.unlink(`rbf:${type}:${txid}`); + } catch (e) { + logger.warn(`Failed to remove RBF ${type} from Redis cache: ${e instanceof Error ? e.message : e}`); + } + } + + async $getBlocks(): Promise { + try { + await this.$ensureConnected(); + const json = await this.client.get('blocks'); + return JSON.parse(json); + } catch (e) { + logger.warn(`Failed to retrieve blocks from Redis cache: ${e instanceof Error ? e.message : e}`); + return []; + } + } + + async $getBlockSummaries(): Promise { + try { + await this.$ensureConnected(); + const json = await this.client.get('block-summaries'); + return JSON.parse(json); + } catch (e) { + logger.warn(`Failed to retrieve blocks from Redis cache: ${e instanceof Error ? e.message : e}`); + return []; + } + } + + async $getMempool(): Promise<{ [txid: string]: MempoolTransactionExtended }> { + const start = Date.now(); + const mempool = {}; + try { + await this.$ensureConnected(); + const mempoolList = await this.scanKeys('mempool:tx:*'); + for (const tx of mempoolList) { + mempool[tx.key] = tx.value; + } + logger.info(`Loaded mempool from Redis cache in ${Date.now() - start} ms`); + return mempool || {}; + } catch (e) { + logger.warn(`Failed to retrieve mempool from Redis cache: ${e instanceof Error ? e.message : e}`); + } + return {}; + } + + async $getRbfEntries(type: string): Promise { + try { + await this.$ensureConnected(); + const rbfEntries = await this.scanKeys(`rbf:${type}:*`); + return rbfEntries; + } catch (e) { + logger.warn(`Failed to retrieve Rbf ${type}s from Redis cache: ${e instanceof Error ? e.message : e}`); + return []; + } + } + + async $loadCache() { + logger.info('Restoring mempool and blocks data from Redis cache'); + // Load block data + const loadedBlocks = await this.$getBlocks(); + const loadedBlockSummaries = await this.$getBlockSummaries(); + // Load mempool + const loadedMempool = await this.$getMempool(); + this.inflateLoadedTxs(loadedMempool); + // Load rbf data + const rbfTxs = await this.$getRbfEntries('tx'); + const rbfTrees = await this.$getRbfEntries('tree'); + const rbfExpirations = await this.$getRbfEntries('exp'); + + // Set loaded data + blocks.setBlocks(loadedBlocks || []); + blocks.setBlockSummaries(loadedBlockSummaries || []); + await memPool.$setMempool(loadedMempool); + await rbfCache.load({ + txs: rbfTxs, + trees: rbfTrees.map(loadedTree => loadedTree.value), + expiring: rbfExpirations, + }); + } + + private inflateLoadedTxs(mempool: { [txid: string]: MempoolTransactionExtended }) { + for (const tx of Object.values(mempool)) { + for (const vin of tx.vin) { + if (vin.scriptsig) { + vin.scriptsig_asm = transactionUtils.convertScriptSigAsm(vin.scriptsig); + transactionUtils.addInnerScriptsToVin(vin); + } + } + for (const vout of tx.vout) { + if (vout.scriptpubkey) { + vout.scriptpubkey_asm = transactionUtils.convertScriptSigAsm(vout.scriptpubkey); + } + } + } + } + + private async scanKeys(pattern): Promise<{ key: string, value: T }[]> { + logger.info(`loading Redis entries for ${pattern}`); + let keys: string[] = []; + const result: { key: string, value: T }[] = []; + const patternLength = pattern.length - 1; + let count = 0; + const processValues = async (keys): Promise => { + const values = await this.client.MGET(keys); + for (let i = 0; i < values.length; i++) { + if (values[i]) { + result.push({ key: keys[i].slice(patternLength), value: JSON.parse(values[i]) }); + count++; + } + } + logger.info(`loaded ${count} entries from Redis cache`); + }; + for await (const key of this.client.scanIterator({ + MATCH: pattern, + COUNT: 100 + })) { + keys.push(key); + if (keys.length >= 10000) { + await processValues(keys); + keys = []; + } + } + if (keys.length) { + await processValues(keys); + } + return result; + } +} + +export default new RedisCache(); diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index 0b10afdfb..e141a6076 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -3,6 +3,7 @@ import { IEsploraApi } from './bitcoin/esplora-api.interface'; import { Common } from './common'; import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; import * as bitcoinjs from 'bitcoinjs-lib'; +import logger from '../logger'; class TransactionUtils { constructor() { } @@ -22,6 +23,23 @@ class TransactionUtils { }; } + // Wrapper for $getTransactionExtended with an automatic retry direct to Core if the first API request fails. + // Propagates any error from the retry request. + public async $getTransactionExtendedRetry(txid: string, addPrevouts = false, lazyPrevouts = false, forceCore = false, addMempoolData = false): Promise { + try { + const result = await this.$getTransactionExtended(txid, addPrevouts, lazyPrevouts, forceCore, addMempoolData); + if (result) { + return result; + } else { + logger.err(`Cannot fetch tx ${txid}. Reason: backend returned null data`); + } + } catch (e) { + logger.err(`Cannot fetch tx ${txid}. Reason: ` + (e instanceof Error ? e.message : e)); + } + // retry direct from Core if first request failed + return this.$getTransactionExtended(txid, addPrevouts, lazyPrevouts, true, addMempoolData); + } + /** * @param txId * @param addPrevouts @@ -31,7 +49,7 @@ class TransactionUtils { public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false, forceCore = false, addMempoolData = false): Promise { let transaction: IEsploraApi.Transaction; if (forceCore === true) { - transaction = await bitcoinCoreApi.$getRawTransaction(txId, true); + transaction = await bitcoinCoreApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts); } else { transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts); } @@ -170,6 +188,122 @@ class TransactionUtils { 16 ); } + + public addInnerScriptsToVin(vin: IEsploraApi.Vin): void { + if (!vin.prevout) { + return; + } + + if (vin.prevout.scriptpubkey_type === 'p2sh') { + const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0]; + vin.inner_redeemscript_asm = this.convertScriptSigAsm(redeemScript); + if (vin.witness && vin.witness.length > 2) { + const witnessScript = vin.witness[vin.witness.length - 1]; + vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript); + } + } + + if (vin.prevout.scriptpubkey_type === 'v0_p2wsh' && vin.witness) { + const witnessScript = vin.witness[vin.witness.length - 1]; + vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript); + } + + if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness) { + const witnessScript = this.witnessToP2TRScript(vin.witness); + if (witnessScript !== null) { + vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript); + } + } + } + + public convertScriptSigAsm(hex: string): string { + const buf = Buffer.from(hex, 'hex'); + + const b: string[] = []; + + let i = 0; + while (i < buf.length) { + const op = buf[i]; + if (op >= 0x01 && op <= 0x4e) { + i++; + let push: number; + if (op === 0x4c) { + push = buf.readUInt8(i); + b.push('OP_PUSHDATA1'); + i += 1; + } else if (op === 0x4d) { + push = buf.readUInt16LE(i); + b.push('OP_PUSHDATA2'); + i += 2; + } else if (op === 0x4e) { + push = buf.readUInt32LE(i); + b.push('OP_PUSHDATA4'); + i += 4; + } else { + push = op; + b.push('OP_PUSHBYTES_' + push); + } + + const data = buf.slice(i, i + push); + if (data.length !== push) { + break; + } + + b.push(data.toString('hex')); + i += data.length; + } else { + if (op === 0x00) { + b.push('OP_0'); + } else if (op === 0x4f) { + b.push('OP_PUSHNUM_NEG1'); + } else if (op === 0xb1) { + b.push('OP_CLTV'); + } else if (op === 0xb2) { + b.push('OP_CSV'); + } else if (op === 0xba) { + b.push('OP_CHECKSIGADD'); + } else { + const opcode = bitcoinjs.script.toASM([ op ]); + if (opcode && op < 0xfd) { + if (/^OP_(\d+)$/.test(opcode)) { + b.push(opcode.replace(/^OP_(\d+)$/, 'OP_PUSHNUM_$1')); + } else { + b.push(opcode); + } + } else { + b.push('OP_RETURN_' + op); + } + } + i += 1; + } + } + + return b.join(' '); + } + + /** + * This function must only be called when we know the witness we are parsing + * is a taproot witness. + * @param witness An array of hex strings that represents the witness stack of + * the input. + * @returns null if the witness is not a script spend, and the hex string of + * the script item if it is a script spend. + */ + public witnessToP2TRScript(witness: string[]): string | null { + if (witness.length < 2) return null; + // Note: see BIP341 for parsing details of witness stack + + // If there are at least two witness elements, and the first byte of the + // last element is 0x50, this last element is called annex a and + // is removed from the witness stack. + const hasAnnex = witness[witness.length - 1].substring(0, 2) === '50'; + // If there are at least two witness elements left, script path spending is used. + // Call the second-to-last stack element s, the script. + // (Note: this phrasing from BIP341 assumes we've *removed* the annex from the stack) + if (hasAnnex && witness.length < 3) return null; + const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2; + return witness[positionOfScript]; + } } export default new TransactionUtils(); diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index b77ce80f1..8ade49288 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -191,15 +191,18 @@ class WebsocketHandler { } if (parsedMessage && parsedMessage['track-address']) { - if (/^([a-km-zA-HJ-NP-Z1-9]{26,35}|[a-km-zA-HJ-NP-Z1-9]{80}|[a-z]{2,5}1[ac-hj-np-z02-9]{8,100}|[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|[0-9a-fA-F]{130})$/ + if (/^([a-km-zA-HJ-NP-Z1-9]{26,35}|[a-km-zA-HJ-NP-Z1-9]{80}|[a-z]{2,5}1[ac-hj-np-z02-9]{8,100}|[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64})$/ .test(parsedMessage['track-address'])) { let matchedAddress = parsedMessage['track-address']; if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}$/.test(parsedMessage['track-address'])) { matchedAddress = matchedAddress.toLowerCase(); } - if (/^[0-9a-fA-F]{130}$/.test(parsedMessage['track-address'])) { + if (/^04[a-fA-F0-9]{128}$/.test(parsedMessage['track-address'])) { client['track-address'] = null; client['track-scriptpubkey'] = '41' + matchedAddress + 'ac'; + } else if (/^|(02|03)[a-fA-F0-9]{64}$/.test(parsedMessage['track-address'])) { + client['track-address'] = null; + client['track-scriptpubkey'] = '21' + matchedAddress + 'ac'; } else { client['track-address'] = matchedAddress; client['track-scriptpubkey'] = null; diff --git a/backend/src/config.ts b/backend/src/config.ts index ceb569e06..269290125 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -12,6 +12,7 @@ interface IConfig { API_URL_PREFIX: string; POLL_RATE_MS: number; CACHE_DIR: string; + CACHE_ENABLED: boolean; CLEAR_PROTECTION_MINUTES: number; RECOMMENDED_FEE_PERCENTILE: number; BLOCK_WEIGHT_UNITS: number; @@ -142,6 +143,10 @@ interface IConfig { API: string; ACCELERATIONS: boolean; }, + REDIS: { + ENABLED: boolean; + UNIX_SOCKET_PATH: string; + }, } const defaults: IConfig = { @@ -154,6 +159,7 @@ const defaults: IConfig = { 'API_URL_PREFIX': '/api/v1/', 'POLL_RATE_MS': 2000, 'CACHE_DIR': './cache', + 'CACHE_ENABLED': true, 'CLEAR_PROTECTION_MINUTES': 20, 'RECOMMENDED_FEE_PERCENTILE': 50, 'BLOCK_WEIGHT_UNITS': 4000000, @@ -283,7 +289,11 @@ const defaults: IConfig = { 'MEMPOOL_SERVICES': { 'API': '', 'ACCELERATIONS': false, - } + }, + 'REDIS': { + 'ENABLED': false, + 'UNIX_SOCKET_PATH': '', + }, }; class Config implements IConfig { @@ -305,6 +315,7 @@ class Config implements IConfig { MAXMIND: IConfig['MAXMIND']; REPLICATION: IConfig['REPLICATION']; MEMPOOL_SERVICES: IConfig['MEMPOOL_SERVICES']; + REDIS: IConfig['REDIS']; constructor() { const configs = this.merge(configFromFile, defaults); @@ -326,6 +337,7 @@ class Config implements IConfig { this.MAXMIND = configs.MAXMIND; this.REPLICATION = configs.REPLICATION; this.MEMPOOL_SERVICES = configs.MEMPOOL_SERVICES; + this.REDIS = configs.REDIS; } merge = (...objects: object[]): IConfig => { diff --git a/backend/src/index.ts b/backend/src/index.ts index bbfaa9ff3..185a47067 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -41,6 +41,7 @@ import chainTips from './api/chain-tips'; import { AxiosError } from 'axios'; import v8 from 'v8'; import { formatBytes, getBytesUnit } from './utils/format'; +import redisCache from './api/redis-cache'; class Server { private wss: WebSocket.Server | undefined; @@ -122,7 +123,11 @@ class Server { await poolsUpdater.updatePoolsJson(); // Needs to be done before loading the disk cache because we sometimes wipe it await syncAssets.syncAssets$(); if (config.MEMPOOL.ENABLED) { - await diskCache.$loadMempoolCache(); + if (config.MEMPOOL.CACHE_ENABLED) { + await diskCache.$loadMempoolCache(); + } else if (config.REDIS.ENABLED) { + await redisCache.$loadCache(); + } } if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) { @@ -183,14 +188,15 @@ class Server { } const newMempool = await bitcoinApi.$getRawMempool(); const numHandledBlocks = await blocks.$updateBlocks(); + const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerRunning ? 10 : 1); if (numHandledBlocks === 0) { - await memPool.$updateMempool(newMempool); + await memPool.$updateMempool(newMempool, pollRate); } indexer.$run(); // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS const elapsed = Date.now() - start; - const remainingTime = Math.max(0, config.MEMPOOL.POLL_RATE_MS - elapsed) + const remainingTime = Math.max(0, pollRate - elapsed); setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 0 : remainingTime); this.backendRetryCount = 0; } catch (e: any) { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 078b85a03..0953f9b84 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -1,3 +1,4 @@ +import bitcoinApi from '../api/bitcoin/bitcoin-api-factory'; import { BlockExtended, BlockExtension, BlockPrice, EffectiveFeeStats } from '../mempool.interfaces'; import DB from '../database'; import logger from '../logger'; @@ -12,6 +13,7 @@ import config from '../config'; import chainTips from '../api/chain-tips'; import blocks from '../api/blocks'; import BlocksAuditsRepository from './BlocksAuditsRepository'; +import transactionUtils from '../api/transaction-utils'; interface DatabaseBlock { id: string; @@ -539,7 +541,7 @@ class BlocksRepository { */ public async $getBlocksDifficulty(): Promise { try { - const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty FROM blocks`); + const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty, bits FROM blocks`); return rows; } catch (e) { logger.err('Cannot get blocks difficulty list from the db. Reason: ' + (e instanceof Error ? e.message : e)); @@ -848,7 +850,7 @@ class BlocksRepository { */ public async $getOldestConsecutiveBlock(): Promise { try { - const [rows]: any = await DB.query(`SELECT height, UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty FROM blocks ORDER BY height DESC`); + const [rows]: any = await DB.query(`SELECT height, UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty, bits FROM blocks ORDER BY height DESC`); for (let i = 0; i < rows.length - 1; ++i) { if (rows[i].height - rows[i + 1].height > 1) { return rows[i]; @@ -1036,8 +1038,17 @@ class BlocksRepository { { extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); if (extras.feePercentiles === null) { - const block = await bitcoinClient.getBlock(dbBlk.id, 2); - const summary = blocks.summarizeBlock(block); + + let summary; + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(dbBlk.id)).map(tx => transactionUtils.extendTransaction(tx)); + summary = blocks.summarizeBlockTransactions(dbBlk.id, txs); + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(dbBlk.id, 2); + summary = blocks.summarizeBlock(block); + } + await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.id, summary.transactions); extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); } diff --git a/contributors/Czino.txt b/contributors/Czino.txt new file mode 100644 index 000000000..08affb095 --- /dev/null +++ b/contributors/Czino.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of July 29, 2023. + +Signed: Czino diff --git a/contributors/andrewtoth.txt b/contributors/andrewtoth.txt new file mode 100644 index 000000000..bd2ed3d06 --- /dev/null +++ b/contributors/andrewtoth.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of August 2, 2023. + +Signed: andrewtoth diff --git a/contributors/bguillaumat.txt b/contributors/bguillaumat.txt new file mode 100644 index 000000000..ac14a07c7 --- /dev/null +++ b/contributors/bguillaumat.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022. + +Signed: bguillaumat diff --git a/contributors/fiatjaf.txt b/contributors/fiatjaf.txt new file mode 100644 index 000000000..cdd716d3c --- /dev/null +++ b/contributors/fiatjaf.txt @@ -0,0 +1,5 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022. +I also regret having ever contributed to this repository since they keep asking me to sign this legalese timewaste things. +And finally I don't care about licenses and won't sue anyone over intellectual property, which is a fake statist construct invented by evil lobby lawyers. + +Signed: fiatjaf diff --git a/contributors/rishkwal.txt b/contributors/rishkwal.txt new file mode 100644 index 000000000..9a50bda6b --- /dev/null +++ b/contributors/rishkwal.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of July 29, 2023. + +Signed: rishkwal diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 71fe3bd65..92a6c9266 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -8,6 +8,7 @@ "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__", "POLL_RATE_MS": __MEMPOOL_POLL_RATE_MS__, "CACHE_DIR": "__MEMPOOL_CACHE_DIR__", + "CACHE_ENABLED": __MEMPOOL_CACHE_ENABLED__, "CLEAR_PROTECTION_MINUTES": __MEMPOOL_CLEAR_PROTECTION_MINUTES__, "RECOMMENDED_FEE_PERCENTILE": __MEMPOOL_RECOMMENDED_FEE_PERCENTILE__, "BLOCK_WEIGHT_UNITS": __MEMPOOL_BLOCK_WEIGHT_UNITS__, @@ -137,5 +138,9 @@ "MEMPOOL_SERVICES": { "API": "__MEMPOOL_SERVICES_API__", "ACCELERATIONS": __MEMPOOL_SERVICES_ACCELERATIONS__ + }, + "REDIS": { + "ENABLED": __REDIS_ENABLED__, + "UNIX_SOCKET_PATH": "__REDIS_UNIX_SOCKET_PATH__" } } diff --git a/docker/backend/start.sh b/docker/backend/start.sh index c9088c0a6..a7fedd8ff 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -9,6 +9,7 @@ __MEMPOOL_SPAWN_CLUSTER_PROCS__=${MEMPOOL_SPAWN_CLUSTER_PROCS:=0} __MEMPOOL_API_URL_PREFIX__=${MEMPOOL_API_URL_PREFIX:=/api/v1/} __MEMPOOL_POLL_RATE_MS__=${MEMPOOL_POLL_RATE_MS:=2000} __MEMPOOL_CACHE_DIR__=${MEMPOOL_CACHE_DIR:=./cache} +__MEMPOOL_CACHE_ENABLED__=${MEMPOOL_CACHE_ENABLED:=true} __MEMPOOL_CLEAR_PROTECTION_MINUTES__=${MEMPOOL_CLEAR_PROTECTION_MINUTES:=20} __MEMPOOL_RECOMMENDED_FEE_PERCENTILE__=${MEMPOOL_RECOMMENDED_FEE_PERCENTILE:=50} __MEMPOOL_BLOCK_WEIGHT_UNITS__=${MEMPOOL_BLOCK_WEIGHT_UNITS:=4000000} @@ -140,6 +141,9 @@ __REPLICATION_SERVERS__=${REPLICATION_SERVERS:=[]} __MEMPOOL_SERVICES_API__=${MEMPOOL_SERVICES_API:=""} __MEMPOOL_SERVICES_ACCELERATIONS__=${MEMPOOL_SERVICES_ACCELERATIONS:=false} +# REDIS +__REDIS_ENABLED__=${REDIS_ENABLED:=true} +__REDIS_UNIX_SOCKET_PATH__=${REDIS_UNIX_SOCKET_PATH:=true} mkdir -p "${__MEMPOOL_CACHE_DIR__}" @@ -151,6 +155,7 @@ sed -i "s!__MEMPOOL_SPAWN_CLUSTER_PROCS__!${__MEMPOOL_SPAWN_CLUSTER_PROCS__}!g" sed -i "s!__MEMPOOL_API_URL_PREFIX__!${__MEMPOOL_API_URL_PREFIX__}!g" mempool-config.json sed -i "s!__MEMPOOL_POLL_RATE_MS__!${__MEMPOOL_POLL_RATE_MS__}!g" mempool-config.json sed -i "s!__MEMPOOL_CACHE_DIR__!${__MEMPOOL_CACHE_DIR__}!g" mempool-config.json +sed -i "s!__MEMPOOL_CACHE_ENABLED__!${__MEMPOOL_CACHE_ENABLED__}!g" mempool-config.json sed -i "s!__MEMPOOL_CLEAR_PROTECTION_MINUTES__!${__MEMPOOL_CLEAR_PROTECTION_MINUTES__}!g" mempool-config.json sed -i "s!__MEMPOOL_RECOMMENDED_FEE_PERCENTILE__!${__MEMPOOL_RECOMMENDED_FEE_PERCENTILE__}!g" mempool-config.json sed -i "s!__MEMPOOL_BLOCK_WEIGHT_UNITS__!${__MEMPOOL_BLOCK_WEIGHT_UNITS__}!g" mempool-config.json @@ -169,7 +174,7 @@ sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-co sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json -sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_GBT__}!g" mempool-config.json +sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_RUST_GBT__}!g" mempool-config.json sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json @@ -270,5 +275,8 @@ sed -i "s!__REPLICATION_SERVERS__!${__REPLICATION_SERVERS__}!g" mempool-config.j sed -i "s!__MEMPOOL_SERVICES_API__!${__MEMPOOL_SERVICES_API__}!g" mempool-config.json sed -i "s!__MEMPOOL_SERVICES_ACCELERATIONS__!${__MEMPOOL_SERVICES_ACCELERATIONS__}!g" mempool-config.json +# REDIS +sed -i "s!__REDIS_ENABLED__!${__REDIS_ENABLED__}!g" mempool-config.json +sed -i "s!__REDIS_UNIX_SOCKET_PATH__!${__REDIS_UNIX_SOCKET_PATH__}!g" mempool-config.json node /backend/package/index.js diff --git a/docker/frontend/entrypoint.sh b/docker/frontend/entrypoint.sh index b6946578b..7d5ee313d 100644 --- a/docker/frontend/entrypoint.sh +++ b/docker/frontend/entrypoint.sh @@ -18,7 +18,7 @@ fi __TESTNET_ENABLED__=${TESTNET_ENABLED:=false} __SIGNET_ENABLED__=${SIGNET_ENABLED:=false} -__LIQUID_ENABLED__=${LIQUID_EANBLED:=false} +__LIQUID_ENABLED__=${LIQUID_ENABLED:=false} __LIQUID_TESTNET_ENABLED__=${LIQUID_TESTNET_ENABLED:=false} __BISQ_ENABLED__=${BISQ_ENABLED:=false} __BISQ_SEPARATE_BACKEND__=${BISQ_SEPARATE_BACKEND:=false} diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 381353948..23f50b544 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -411,7 +411,7 @@ Trademark Notice

- The Mempool Open Source Project™, mempool.space™, the mempool logo®, the mempool.space logos™, the mempool square logo®, and the mempool blocks logo™ are either registered trademarks or trademarks of Mempool Space K.K in Japan, the United States, and/or other countries. + The Mempool Open Source Project®, mempool.space™, the mempool logo®, the mempool.space logos™, the mempool square logo®, and the mempool blocks logo™ are either registered trademarks or trademarks of Mempool Space K.K in Japan, the United States, and/or other countries.

While our software is available under an open source software license, the copyright license does not include an implied right or license to use our trademarks. See our Trademark Policy and Guidelines for more details, published on <https://mempool.space/trademark-policy>. diff --git a/frontend/src/app/components/address/address-preview.component.ts b/frontend/src/app/components/address/address-preview.component.ts index 07ead8baa..844def9fd 100644 --- a/frontend/src/app/components/address/address-preview.component.ts +++ b/frontend/src/app/components/address/address-preview.component.ts @@ -64,12 +64,12 @@ export class AddressPreviewComponent implements OnInit, OnDestroy { this.address = null; this.addressInfo = null; this.addressString = params.get('id') || ''; - if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|[A-F0-9]{130}$/.test(this.addressString)) { + if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64}$/.test(this.addressString)) { this.addressString = this.addressString.toLowerCase(); } this.seoService.setTitle($localize`:@@address.component.browser-title:Address: ${this.addressString}:INTERPOLATION:`); - return (this.addressString.match(/[a-f0-9]{130}/) + return (this.addressString.match(/04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64}/) ? this.electrsApiService.getPubKeyAddress$(this.addressString) : this.electrsApiService.getAddress$(this.addressString) ).pipe( diff --git a/frontend/src/app/components/address/address.component.ts b/frontend/src/app/components/address/address.component.ts index 64d3a4143..e9bfaaa1b 100644 --- a/frontend/src/app/components/address/address.component.ts +++ b/frontend/src/app/components/address/address.component.ts @@ -72,7 +72,7 @@ export class AddressComponent implements OnInit, OnDestroy { this.addressInfo = null; document.body.scrollTo(0, 0); this.addressString = params.get('id') || ''; - if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|[A-F0-9]{130}$/.test(this.addressString)) { + if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64}$/.test(this.addressString)) { this.addressString = this.addressString.toLowerCase(); } this.seoService.setTitle($localize`:@@address.component.browser-title:Address: ${this.addressString}:INTERPOLATION:`); @@ -84,7 +84,7 @@ export class AddressComponent implements OnInit, OnDestroy { ) .pipe( switchMap(() => ( - this.addressString.match(/[a-f0-9]{130}/) + this.addressString.match(/04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64}/) ? this.electrsApiService.getPubKeyAddress$(this.addressString) : this.electrsApiService.getAddress$(this.addressString) ).pipe( @@ -118,7 +118,7 @@ export class AddressComponent implements OnInit, OnDestroy { this.isLoadingAddress = false; this.isLoadingTransactions = true; return address.is_pubkey - ? this.electrsApiService.getScriptHashTransactions$('41' + address.address + 'ac') + ? this.electrsApiService.getScriptHashTransactions$((address.address.length === 66 ? '21' : '41') + address.address + 'ac') : this.electrsApiService.getAddressTransactions$(address.address); }), switchMap((transactions) => { diff --git a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss index 47c87a45c..f4f4dcc77 100644 --- a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss +++ b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss @@ -25,7 +25,8 @@ flex-direction: column; padding: 0px 15px; width: 100%; - height: calc(100vh - 250px); + height: calc(100vh - 225px); + min-height: 400px; @media (min-width: 992px) { height: calc(100vh - 150px); } diff --git a/frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss b/frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss index fae81952b..b73d55685 100644 --- a/frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss +++ b/frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss @@ -25,7 +25,8 @@ flex-direction: column; padding: 0px 15px; width: 100%; - height: calc(100vh - 250px); + height: calc(100vh - 225px); + min-height: 400px; @media (min-width: 992px) { height: calc(100vh - 150px); } diff --git a/frontend/src/app/components/block-health-graph/block-health-graph.component.scss b/frontend/src/app/components/block-health-graph/block-health-graph.component.scss index f8403bad5..7b8154bae 100644 --- a/frontend/src/app/components/block-health-graph/block-health-graph.component.scss +++ b/frontend/src/app/components/block-health-graph/block-health-graph.component.scss @@ -25,7 +25,8 @@ flex-direction: column; padding: 0px 15px; width: 100%; - height: calc(100vh - 250px); + height: calc(100vh - 225px); + min-height: 400px; @media (min-width: 992px) { height: calc(100vh - 150px); } diff --git a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss index f8403bad5..7b8154bae 100644 --- a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss +++ b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss @@ -25,7 +25,8 @@ flex-direction: column; padding: 0px 15px; width: 100%; - height: calc(100vh - 250px); + height: calc(100vh - 225px); + min-height: 400px; @media (min-width: 992px) { height: calc(100vh - 150px); } diff --git a/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss b/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss index f8403bad5..7b8154bae 100644 --- a/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss +++ b/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss @@ -25,7 +25,8 @@ flex-direction: column; padding: 0px 15px; width: 100%; - height: calc(100vh - 250px); + height: calc(100vh - 225px); + min-height: 400px; @media (min-width: 992px) { height: calc(100vh - 150px); } diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.ts b/frontend/src/app/components/blocks-list/blocks-list.component.ts index 1af6572fc..cec925270 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.ts +++ b/frontend/src/app/components/blocks-list/blocks-list.component.ts @@ -68,7 +68,7 @@ export class BlocksList implements OnInit { for (const block of blocks) { // @ts-ignore: Need to add an extra field for the template block.extras.pool.logo = `/resources/mining-pools/` + - block.extras.pool.name.toLowerCase().replace(' ', '').replace('.', '') + '.svg'; + block.extras.pool.slug + '.svg'; } } if (this.widget) { @@ -102,7 +102,7 @@ export class BlocksList implements OnInit { if (this.stateService.env.MINING_DASHBOARD) { // @ts-ignore: Need to add an extra field for the template blocks[1][0].extras.pool.logo = `/resources/mining-pools/` + - blocks[1][0].extras.pool.name.toLowerCase().replace(' ', '').replace('.', '') + '.svg'; + blocks[1][0].extras.pool.slug + '.svg'; } acc.unshift(blocks[1][0]); acc = acc.slice(0, this.widget ? 6 : 15); diff --git a/frontend/src/app/components/difficulty/difficulty.component.ts b/frontend/src/app/components/difficulty/difficulty.component.ts index d3983c939..a2c03dc56 100644 --- a/frontend/src/app/components/difficulty/difficulty.component.ts +++ b/frontend/src/app/components/difficulty/difficulty.component.ts @@ -1,4 +1,4 @@ -import { ChangeDetectionStrategy, Component, HostListener, Inject, Input, LOCALE_ID, OnInit } from '@angular/core'; +import { ChangeDetectionStrategy, ChangeDetectorRef, Component, HostListener, Inject, Input, LOCALE_ID, OnInit } from '@angular/core'; import { combineLatest, Observable, timer } from 'rxjs'; import { map, switchMap } from 'rxjs/operators'; import { StateService } from '../..//services/state.service'; @@ -61,6 +61,7 @@ export class DifficultyComponent implements OnInit { constructor( public stateService: StateService, + private cd: ChangeDetectorRef, @Inject(LOCALE_ID) private locale: string, ) { } @@ -189,9 +190,15 @@ export class DifficultyComponent implements OnInit { return shapes; } + @HostListener('pointerdown', ['$event']) + onPointerDown(event) { + this.onPointerMove(event); + } + @HostListener('pointermove', ['$event']) onPointerMove(event) { this.tooltipPosition = { x: event.clientX, y: event.clientY }; + this.cd.markForCheck(); } onHover(event, rect): void { diff --git a/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts b/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts index 1d9c289d8..010466952 100644 --- a/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts +++ b/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts @@ -74,14 +74,14 @@ export class FeeDistributionGraphComponent implements OnInit, OnChanges, OnDestr this.labelInterval = this.numSamples / this.numLabels; while (nextSample <= maxBlockVSize) { if (txIndex >= txs.length) { - samples.push([(1 - (sampleIndex / this.numSamples)) * 100, 0]); + samples.push([(1 - (sampleIndex / this.numSamples)) * 100, 0.000001]); nextSample += sampleInterval; sampleIndex++; continue; } while (txs[txIndex] && nextSample < cumVSize + txs[txIndex].vsize) { - samples.push([(1 - (sampleIndex / this.numSamples)) * 100, txs[txIndex].rate]); + samples.push([(1 - (sampleIndex / this.numSamples)) * 100, txs[txIndex].rate || 0.000001]); nextSample += sampleInterval; sampleIndex++; } @@ -118,7 +118,9 @@ export class FeeDistributionGraphComponent implements OnInit, OnChanges, OnDestr }, }, yAxis: { - type: 'value', + type: 'log', + min: 1, + max: this.data.reduce((min, val) => Math.max(min, val[1]), 1), // name: 'Effective Fee Rate s/vb', // nameLocation: 'middle', splitLine: { @@ -129,12 +131,16 @@ export class FeeDistributionGraphComponent implements OnInit, OnChanges, OnDestr } }, axisLabel: { + show: true, formatter: (value: number): string => { const unitValue = this.weightMode ? value / 4 : value; const selectedPowerOfTen = selectPowerOfTen(unitValue); const newVal = Math.round(unitValue / selectedPowerOfTen.divider); return `${newVal}${selectedPowerOfTen.unit}`; }, + }, + axisTick: { + show: true, } }, series: [{ diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss index 0caa35f33..886608573 100644 --- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss @@ -25,7 +25,8 @@ flex-direction: column; padding: 0px 15px; width: 100%; - height: calc(100vh - 250px); + height: calc(100vh - 225px); + min-height: 400px; @media (min-width: 992px) { height: calc(100vh - 150px); } diff --git a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.scss b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.scss index 3b1083505..64a4dcb3d 100644 --- a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.scss +++ b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.scss @@ -25,7 +25,8 @@ flex-direction: column; padding: 0px 15px; width: 100%; - height: calc(100vh - 250px); + height: calc(100vh - 225px); + min-height: 400px; @media (min-width: 992px) { height: calc(100vh - 150px); } diff --git a/frontend/src/app/components/pool/pool-preview.component.ts b/frontend/src/app/components/pool/pool-preview.component.ts index 0431686d6..e03b73665 100644 --- a/frontend/src/app/components/pool/pool-preview.component.ts +++ b/frontend/src/app/components/pool/pool-preview.component.ts @@ -89,7 +89,7 @@ export class PoolPreviewComponent implements OnInit { this.openGraphService.waitOver('pool-stats-' + this.slug); - const logoSrc = `/resources/mining-pools/` + poolStats.pool.name.toLowerCase().replace(' ', '').replace('.', '') + '.svg'; + const logoSrc = `/resources/mining-pools/` + poolStats.pool.slug + '.svg'; if (logoSrc === this.lastImgSrc) { this.openGraphService.waitOver('pool-img-' + this.slug); } diff --git a/frontend/src/app/components/pool/pool.component.ts b/frontend/src/app/components/pool/pool.component.ts index f2fc79ff2..edd5801fe 100644 --- a/frontend/src/app/components/pool/pool.component.ts +++ b/frontend/src/app/components/pool/pool.component.ts @@ -79,7 +79,7 @@ export class PoolComponent implements OnInit { poolStats.pool.regexes = regexes.slice(0, -3); return Object.assign({ - logo: `/resources/mining-pools/` + poolStats.pool.name.toLowerCase().replace(' ', '').replace('.', '') + '.svg' + logo: `/resources/mining-pools/` + poolStats.pool.slug + '.svg' }, poolStats); }) ); diff --git a/frontend/src/app/components/privacy-policy/privacy-policy.component.html b/frontend/src/app/components/privacy-policy/privacy-policy.component.html index fc93b9aa9..e0979ce24 100644 --- a/frontend/src/app/components/privacy-policy/privacy-policy.component.html +++ b/frontend/src/app/components/privacy-policy/privacy-policy.component.html @@ -43,7 +43,7 @@

TRUST YOUR OWN SELF-HOSTED MEMPOOL EXPLORER

-

For maximum privacy, we recommend that you use your own self-hosted instance of The Mempool Open Source Project™ on your own hardware. You can easily install your own self-hosted instance of this website on a Raspberry Pi using a one-click installation method maintained by various Bitcoin fullnode distributions such as Umbrel, RaspiBlitz, MyNode, and RoninDojo. See our project's GitHub page for more details about self-hosting this website.

+

For maximum privacy, we recommend that you use your own self-hosted instance of The Mempool Open Source Project® on your own hardware. You can easily install your own self-hosted instance of this website on a Raspberry Pi using a one-click installation method maintained by various Bitcoin fullnode distributions such as Umbrel, RaspiBlitz, MyNode, and RoninDojo. See our project's GitHub page for more details about self-hosting this website.


diff --git a/frontend/src/app/components/search-form/search-form.component.ts b/frontend/src/app/components/search-form/search-form.component.ts index 61b3351b7..0a794d1f5 100644 --- a/frontend/src/app/components/search-form/search-form.component.ts +++ b/frontend/src/app/components/search-form/search-form.component.ts @@ -34,7 +34,7 @@ export class SearchFormComponent implements OnInit { } } - regexAddress = /^([a-km-zA-HJ-NP-Z1-9]{26,35}|[a-km-zA-HJ-NP-Z1-9]{80}|[A-z]{2,5}1[a-zA-HJ-NP-Z0-9]{39,59}|[0-9a-fA-F]{130})$/; + regexAddress = /^([a-km-zA-HJ-NP-Z1-9]{26,35}|[a-km-zA-HJ-NP-Z1-9]{80}|[A-z]{2,5}1[a-zA-HJ-NP-Z0-9]{39,59}|04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64})$/; regexBlockhash = /^[0]{8}[a-fA-F0-9]{56}$/; regexTransaction = /^([a-fA-F0-9]{64})(:\d+)?$/; regexBlockheight = /^[0-9]{1,9}$/; diff --git a/frontend/src/app/components/trademark-policy/trademark-policy.component.html b/frontend/src/app/components/trademark-policy/trademark-policy.component.html index 4f9419642..3a3da15dd 100644 --- a/frontend/src/app/components/trademark-policy/trademark-policy.component.html +++ b/frontend/src/app/components/trademark-policy/trademark-policy.component.html @@ -7,7 +7,7 @@

Trademark Policy and Guidelines

-
The Mempool Open Source Project ™
+
The Mempool Open Source Project ®
Updated: July 19, 2021

@@ -304,7 +304,7 @@

Also, if you are using our Marks in a way described in the sections "Uses for Which We Are Granting a License," you must include the following trademark attribution at the foot of the webpage where you have used the Mark (or, if in a book, on the credits page), on any packaging or labeling, and on advertising or marketing materials:

-

“The Mempool Space K.K.™, The Mempool Open Source Project™, mempool.space™, the mempool logo®, the mempool.space logos™, the mempool square logo®, and the mempool blocks logo™ are either registered trademarks or trademarks of Mempool Space K.K in Japan, the United States, and/or other countries, and are used with permission. Mempool Space K.K. has no affiliation with and does not sponsor or endorse the information provided herein.”

+

“The Mempool Space K.K.™, The Mempool Open Source Project®, mempool.space™, the mempool logo®, the mempool.space logos™, the mempool square logo®, and the mempool blocks logo™ are either registered trademarks or trademarks of Mempool Space K.K in Japan, the United States, and/or other countries, and are used with permission. Mempool Space K.K. has no affiliation with and does not sponsor or endorse the information provided herein.”

  • What to Do When You See Abuse
    • diff --git a/frontend/src/app/components/transactions-list/transactions-list.component.html b/frontend/src/app/components/transactions-list/transactions-list.component.html index d1d0673fe..22486d320 100644 --- a/frontend/src/app/components/transactions-list/transactions-list.component.html +++ b/frontend/src/app/components/transactions-list/transactions-list.component.html @@ -23,7 +23,7 @@ @@ -56,8 +56,8 @@ Peg-in - P2PK - + P2PK + @@ -184,7 +184,7 @@ @@ -192,8 +192,8 @@ - P2PK - + P2PK + diff --git a/frontend/src/app/dashboard/dashboard.component.ts b/frontend/src/app/dashboard/dashboard.component.ts index 05381453d..31fabd06f 100644 --- a/frontend/src/app/dashboard/dashboard.component.ts +++ b/frontend/src/app/dashboard/dashboard.component.ts @@ -1,6 +1,6 @@ import { AfterViewInit, ChangeDetectionStrategy, Component, OnDestroy, OnInit } from '@angular/core'; import { combineLatest, merge, Observable, of, Subscription } from 'rxjs'; -import { filter, map, scan, share, switchMap, tap } from 'rxjs/operators'; +import { catchError, filter, map, scan, share, switchMap, tap } from 'rxjs/operators'; import { BlockExtended, OptimizedMempoolStats } from '../interfaces/node-api.interface'; import { MempoolInfo, TransactionStripped, ReplacementInfo } from '../interfaces/websocket.interface'; import { ApiService } from '../services/api.service'; @@ -159,7 +159,7 @@ export class DashboardComponent implements OnInit, OnDestroy, AfterViewInit { for (const block of blocks) { // @ts-ignore: Need to add an extra field for the template block.extras.pool.logo = `/resources/mining-pools/` + - block.extras.pool.name.toLowerCase().replace(' ', '').replace('.', '') + '.svg'; + block.extras.pool.slug + '.svg'; } } return of(blocks.slice(0, 6)); @@ -171,7 +171,11 @@ export class DashboardComponent implements OnInit, OnDestroy, AfterViewInit { this.mempoolStats$ = this.stateService.connectionState$ .pipe( filter((state) => state === 2), - switchMap(() => this.apiService.list2HStatistics$()), + switchMap(() => this.apiService.list2HStatistics$().pipe( + catchError((e) => { + return of(null); + }) + )), switchMap((mempoolStats) => { return merge( this.stateService.live2Chart$ @@ -186,10 +190,14 @@ export class DashboardComponent implements OnInit, OnDestroy, AfterViewInit { ); }), map((mempoolStats) => { - return { - mempool: mempoolStats, - weightPerSecond: this.handleNewMempoolData(mempoolStats.concat([])), - }; + if (mempoolStats) { + return { + mempool: mempoolStats, + weightPerSecond: this.handleNewMempoolData(mempoolStats.concat([])), + }; + } else { + return null; + } }), share(), ); diff --git a/frontend/src/app/docs/api-docs/api-docs.component.html b/frontend/src/app/docs/api-docs/api-docs.component.html index 9945a3f05..40a7ae486 100644 --- a/frontend/src/app/docs/api-docs/api-docs.component.html +++ b/frontend/src/app/docs/api-docs/api-docs.component.html @@ -10,8 +10,8 @@
    -

    mempool.space merely provides data about the Bitcoin network. It cannot help you with retrieving funds, confirming your transaction quicker, etc.

    For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).

    -

    mempool.space merely provides data about the Bitcoin network. It cannot help you with retrieving funds, confirming your transaction quicker, etc.

    For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).

    +

    mempool.space merely provides data about the Bitcoin network. It cannot help you with retrieving funds, wallet issues, etc.

    For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).

    +

    mempool.space merely provides data about the Bitcoin network. It cannot help you with retrieving funds, wallet issues, etc.

    For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).

    diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index cb42eaed3..0fa0753f6 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -111,6 +111,7 @@ export interface PoolInfo { regexes: string; // JSON array addresses: string; // JSON array emptyBlocks: number; + slug: string; } export interface PoolStat { pool: PoolInfo; diff --git a/frontend/src/app/lightning/nodes-map/nodes-map.component.scss b/frontend/src/app/lightning/nodes-map/nodes-map.component.scss index d49b68957..a2f62e9c5 100644 --- a/frontend/src/app/lightning/nodes-map/nodes-map.component.scss +++ b/frontend/src/app/lightning/nodes-map/nodes-map.component.scss @@ -14,7 +14,8 @@ flex-direction: column; padding: 0px 15px; width: 100%; - height: calc(100vh - 250px); + height: calc(100vh - 225px); + min-height: 400px; @media (min-width: 992px) { height: calc(100vh - 150px); } diff --git a/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.scss b/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.scss index bb8f2cd87..0e6fb056d 100644 --- a/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.scss +++ b/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.scss @@ -25,7 +25,8 @@ flex-direction: column; padding: 0px 15px; width: 100%; - height: calc(100vh - 250px); + height: calc(100vh - 225px); + min-height: 400px; @media (min-width: 992px) { height: calc(100vh - 150px); } diff --git a/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.scss b/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.scss index 0d692a6c8..c885e4839 100644 --- a/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.scss +++ b/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.scss @@ -25,7 +25,8 @@ flex-direction: column; padding: 0px 15px; width: 100%; - height: calc(100vh - 250px); + height: calc(100vh - 225px); + min-height: 400px; @media (min-width: 992px) { height: calc(100vh - 150px); } diff --git a/frontend/src/app/services/electrs-api.service.ts b/frontend/src/app/services/electrs-api.service.ts index f866eb23d..d63d49f68 100644 --- a/frontend/src/app/services/electrs-api.service.ts +++ b/frontend/src/app/services/electrs-api.service.ts @@ -67,7 +67,8 @@ export class ElectrsApiService { } getPubKeyAddress$(pubkey: string): Observable
    { - return this.getScriptHash$('41' + pubkey + 'ac').pipe( + const scriptpubkey = (pubkey.length === 130 ? '41' : '21') + pubkey + 'ac'; + return this.getScriptHash$(scriptpubkey).pipe( switchMap((scripthash: ScriptHash) => { return of({ ...scripthash, diff --git a/frontend/src/app/services/mining.service.ts b/frontend/src/app/services/mining.service.ts index 63257fa74..96723921e 100644 --- a/frontend/src/app/services/mining.service.ts +++ b/frontend/src/app/services/mining.service.ts @@ -96,7 +96,7 @@ export class MiningService { share: parseFloat((poolStat.blockCount / stats.blockCount * 100).toFixed(2)), lastEstimatedHashrate: (poolStat.blockCount / stats.blockCount * stats.lastEstimatedHashrate / hashrateDivider).toFixed(2), emptyBlockRatio: (poolStat.emptyBlocks / poolStat.blockCount * 100).toFixed(2), - logo: `/resources/mining-pools/` + poolStat.name.toLowerCase().replace(' ', '').replace('.', '') + '.svg', + logo: `/resources/mining-pools/` + poolStat.slug + '.svg', ...poolStat }; }); diff --git a/frontend/src/app/services/websocket.service.ts b/frontend/src/app/services/websocket.service.ts index e70424cdc..4bd20e987 100644 --- a/frontend/src/app/services/websocket.service.ts +++ b/frontend/src/app/services/websocket.service.ts @@ -113,7 +113,7 @@ export class WebsocketService { this.stateService.connectionState$.next(2); } - if (this.stateService.connectionState$.value === 1) { + if (this.stateService.connectionState$.value !== 2) { this.stateService.connectionState$.next(2); } diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 8e5279a94..2b30714d1 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -2,7 +2,10 @@
    -
    The Mempool Open Source Project ™
    +

    Explore the full Bitcoin ecosystem ™

    @@ -17,17 +20,16 @@

    Sign In

    -

    Broadcast Transaction

    -

    Connect to our Nodes

    -

    About The Mempool Open Source Project™

    Explore

    Mining Dashboard

    -

    Lightning Dashboard

    +

    Lightning Dashboard

    Recent Blocks

    +

    Broadcast Transaction

    +

    Connect to our Nodes

    API Documentation

    @@ -38,25 +40,25 @@

    Why isn't my transaction confirming?

    More FAQs ›

    -
    -
    -

    More Networks

    -

    Mainnet Explorer

    -

    Testnet Explorer

    -

    Signet Explorer

    -

    Liquid Explorer

    -

    Bisq Explorer

    +
    +

    Networks

    +

    Mainnet Explorer

    +

    Testnet Explorer

    +

    Signet Explorer

    +

    Liquid Testnet Explorer

    +

    Liquid Explorer

    +

    Bisq Explorer

    + +
    +

    Tools

    +

    Clock (Mempool)

    +

    Clock (Mined)

    +

    BTC/Fiat Converter

    +
    +

    Legal

    Terms of Service

    diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.scss b/frontend/src/app/shared/components/global-footer/global-footer.component.scss index defbe937e..387a27ffc 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.scss +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.scss @@ -21,6 +21,10 @@ footer .row.main .branding { text-align: center; } +footer .row.main .branding > p { + margin-bottom: 45px; +} + footer .row.main .branding .btn { display: inline-block; color: #fff !important; @@ -89,6 +93,11 @@ footer .row.version p a { color: #09a3ba; } +.main-logo { + max-width: 220px; + margin: 0 auto 20px auto; +} + @media (max-width: 992px) { footer .row.main .links.outer { diff --git a/frontend/src/index.bisq.html b/frontend/src/index.bisq.html index 43c85b404..6d3bcb130 100644 --- a/frontend/src/index.bisq.html +++ b/frontend/src/index.bisq.html @@ -7,7 +7,7 @@ - + @@ -15,7 +15,7 @@ - + diff --git a/frontend/src/index.liquid.html b/frontend/src/index.liquid.html index 630f6a2ab..ce57e5873 100644 --- a/frontend/src/index.liquid.html +++ b/frontend/src/index.liquid.html @@ -7,7 +7,7 @@ - + @@ -15,7 +15,7 @@ - + diff --git a/frontend/src/index.mempool.html b/frontend/src/index.mempool.html index 60f1b4421..ec026b63b 100644 --- a/frontend/src/index.mempool.html +++ b/frontend/src/index.mempool.html @@ -7,7 +7,7 @@ - + @@ -15,7 +15,7 @@ - + diff --git a/frontend/src/resources/mempool-space-preview.png b/frontend/src/resources/mempool-space-preview.png index bfc59f600..60c8bcc6a 100644 Binary files a/frontend/src/resources/mempool-space-preview.png and b/frontend/src/resources/mempool-space-preview.png differ diff --git a/frontend/src/resources/previews/dashboard.png b/frontend/src/resources/previews/dashboard.png index e11588994..60c8bcc6a 100644 Binary files a/frontend/src/resources/previews/dashboard.png and b/frontend/src/resources/previews/dashboard.png differ diff --git a/production/electrs-start-liquid b/production/electrs-start-liquid deleted file mode 100755 index a28135836..000000000 --- a/production/electrs-start-liquid +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env zsh -cd "${HOME}/electrs" -#source "${HOME}/.cargo/env" -#export PATH="${HOME}/.cargo/bin:${PATH}" - -until false -do - cargo run \ - --release \ - --features liquid \ - --bin electrs \ - -- \ - -vvv \ - --asset-db-path "${HOME}/asset_registry_db" \ - --address-search \ - --cors '*' \ - --db-dir __ELECTRS_DATA_ROOT__ \ - --network liquid \ - --daemon-dir "${HOME}" \ - --http-socket-file '/elements/socket/esplora-liquid-mainnet' \ - --cookie '__ELEMENTS_RPC_USER__:__ELEMENTS_RPC_PASS__' \ - --precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" - sleep 1 -done diff --git a/production/electrs-start-liquidtestnet b/production/electrs-start-liquidtestnet deleted file mode 100755 index 828e96533..000000000 --- a/production/electrs-start-liquidtestnet +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/local/bin/zsh -cd "${HOME}/electrs" -#source "${HOME}/.cargo/env" -#export PATH="${HOME}/.cargo/bin:${PATH}" - -until false -do - cargo run \ - --release \ - --features liquid \ - --bin electrs \ - -- \ - -vv \ - --asset-db-path "${HOME}/asset_registry_testnet_db" \ - --address-search \ - --cors '*' \ - --db-dir __ELECTRS_DATA_ROOT__ \ - --network liquidtestnet \ - --daemon-dir "${HOME}" \ - --http-socket-file '/elements/socket/esplora-liquid-testnet' \ - --cookie '__ELEMENTS_RPC_USER__:__ELEMENTS_RPC_PASS__' \ - --precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" - sleep 1 -done diff --git a/production/electrs-start-mainnet b/production/electrs-start-mainnet deleted file mode 100755 index c6a8c4d54..000000000 --- a/production/electrs-start-mainnet +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env zsh -cd "${HOME}/electrs" -#source "${HOME}/.cargo/env" -#export PATH="${HOME}/.cargo/bin:${PATH}" - -until false -do - cargo run \ - --release \ - --bin electrs \ - -- \ - -vvvv \ - --address-search \ - --cors '*' \ - --db-dir __ELECTRS_DATA_ROOT__ \ - --daemon-dir "${HOME}" \ - --http-socket-file '/bitcoin/socket/esplora-bitcoin-mainnet' \ - --cookie '__BITCOIN_RPC_USER__:__BITCOIN_RPC_PASS__' \ - --precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" - - sleep 3 -done diff --git a/production/electrs-start-signet b/production/electrs-start-signet deleted file mode 100755 index 40e1d1115..000000000 --- a/production/electrs-start-signet +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env zsh -cd "${HOME}/electrs" -#source "${HOME}/.cargo/env" -#export PATH="${HOME}/.cargo/bin:${PATH}" - -until false -do - cargo run \ - --release \ - --bin electrs \ - -- \ - -vv \ - --network signet \ - --address-search \ - --cors '*' \ - --db-dir __ELECTRS_DATA_ROOT__ \ - --daemon-rpc-addr '127.0.0.1:38332' \ - --daemon-dir "${HOME}" \ - --http-socket-file '/bitcoin/socket/esplora-bitcoin-signet' \ - --cookie '__BITCOIN_RPC_USER__:__BITCOIN_RPC_PASS__' \ - --precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" - sleep 1 -done diff --git a/production/electrs-start-testnet b/production/electrs-start-testnet deleted file mode 100755 index ce05de2de..000000000 --- a/production/electrs-start-testnet +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env zsh -cd "${HOME}/electrs" -#source $HOME/.cargo/env -#export PATH=$HOME/.cargo/bin:$PATH - -until false -do - cargo run \ - --release \ - --bin electrs \ - -- \ - -vvvv \ - --network testnet \ - --address-search \ - --cors '*' \ - --db-dir __ELECTRS_DATA_ROOT__ \ - --daemon-dir "${HOME}" \ - --http-socket-file '/bitcoin/socket/esplora-bitcoin-testnet' \ - --cookie '__BITCOIN_RPC_USER__:__BITCOIN_RPC_PASS__' \ - --precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" - - sleep 3 -done diff --git a/production/install b/production/install index 1af975d64..addb4dc16 100755 --- a/production/install +++ b/production/install @@ -1603,15 +1603,8 @@ fi ######################################## # Electrs instance for Bitcoin Mainnet # ######################################## - if [ "${BITCOIN_MAINNET_ENABLE}" = ON ];then - echo "[*] Installing Bitcoin Mainnet electrs start script" - osSudo "${ROOT_USER}" install -c -o "${BITCOIN_USER}" -g "${BITCOIN_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-mainnet" "${BITCOIN_ELECTRS_HOME}" - - echo "[*] Configuring Bitcoin Mainnet RPC credentials in electrs start script" - osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_USER__/${BITCOIN_RPC_USER}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-mainnet" - osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PASS__/${BITCOIN_RPC_PASS}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-mainnet" - osSudo "${ROOT_USER}" sed -i.orig "s!__ELECTRS_DATA_ROOT__!${ELECTRS_DATA_ROOT}!" "${BITCOIN_ELECTRS_HOME}/electrs-start-mainnet" + echo "[*] FIXME: must only crontab enabled daemons" fi ######################################## @@ -1619,13 +1612,7 @@ fi ######################################## if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then - echo "[*] Installing Bitcoin Testnet electrs start script" - osSudo "${ROOT_USER}" install -c -o "${BITCOIN_USER}" -g "${BITCOIN_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-testnet" "${BITCOIN_ELECTRS_HOME}" - - echo "[*] Configuring Bitcoin Testnet RPC credentials in electrs start script" - osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_USER__/${BITCOIN_RPC_USER}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-testnet" - osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PASS__/${BITCOIN_RPC_PASS}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-testnet" - osSudo "${ROOT_USER}" sed -i.orig "s!__ELECTRS_DATA_ROOT__!${ELECTRS_DATA_ROOT}!" "${BITCOIN_ELECTRS_HOME}/electrs-start-testnet" + echo "[*] FIXME: must only crontab enabled daemons" fi ####################################### @@ -1633,13 +1620,7 @@ fi ####################################### if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then - echo "[*] Installing Bitcoin Signet electrs start script" - osSudo "${ROOT_USER}" install -c -o "${BITCOIN_USER}" -g "${BITCOIN_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-signet" "${BITCOIN_ELECTRS_HOME}" - - echo "[*] Configuring Bitcoin Signet RPC credentials in electrs start script" - osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_USER__/${BITCOIN_RPC_USER}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-signet" - osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PASS__/${BITCOIN_RPC_PASS}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-signet" - osSudo "${ROOT_USER}" sed -i.orig "s!__ELECTRS_DATA_ROOT__!${ELECTRS_DATA_ROOT}!" "${BITCOIN_ELECTRS_HOME}/electrs-start-signet" + echo "[*] FIXME: must only crontab enabled daemons" fi ######################################## @@ -1647,21 +1628,12 @@ fi ######################################## if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then - echo "[*] Installing Elements Liquid electrs start script" - osSudo "${ROOT_USER}" install -c -o "${ELEMENTS_USER}" -g "${ELEMENTS_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-liquid" "${ELEMENTS_ELECTRS_HOME}" - echo "[*] Installing Elements crontab" case $OS in FreeBSD) - echo "[*] FIXME: must only crontab enabled daemons" osSudo "${ROOT_USER}" crontab -u "${ELEMENTS_USER}" "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/elements.crontab" ;; esac - - echo "[*] Configuring Elements Liquid RPC credentials in electrs start script" - osSudo "${ROOT_USER}" sed -i.orig "s/__ELEMENTS_RPC_USER__/${ELEMENTS_RPC_USER}/" "${ELEMENTS_ELECTRS_HOME}/electrs-start-liquid" - osSudo "${ROOT_USER}" sed -i.orig "s/__ELEMENTS_RPC_PASS__/${ELEMENTS_RPC_PASS}/" "${ELEMENTS_ELECTRS_HOME}/electrs-start-liquid" - osSudo "${ROOT_USER}" sed -i.orig "s!__ELECTRS_DATA_ROOT__!${ELECTRS_DATA_ROOT}!" "${ELEMENTS_ELECTRS_HOME}/electrs-start-liquid" fi ################################################ @@ -1690,7 +1662,6 @@ fi echo "[*] Installing crontabs" case $OS in FreeBSD) - echo "[*] FIXME: must only crontab enabled daemons" osSudo "${ROOT_USER}" crontab -u "${BITCOIN_USER}" "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/bitcoin.crontab" osSudo "${ROOT_USER}" crontab -u "${MINFEE_USER}" "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/minfee.crontab" ;; diff --git a/production/unfurler-config.mainnet.json b/production/unfurler-config.mainnet.json index 77df23704..fd389476b 100644 --- a/production/unfurler-config.mainnet.json +++ b/production/unfurler-config.mainnet.json @@ -1,6 +1,6 @@ { "SERVER": { - "HOST": "https://mempool.fra.mempool.space", + "HOST": "https://mempool.tk7.mempool.space", "HTTP_PORT": 8001 }, "MEMPOOL": { diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index fedf32110..199d77c12 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -174,7 +174,7 @@ class Server { const { lang, path } = parseLanguageUrl(rawPath); const matchedRoute = matchRoute(this.network, path); let ogImageUrl = config.SERVER.HOST + (matchedRoute.staticImg || matchedRoute.fallbackImg); - let ogTitle = 'The Mempool Open Source Project™'; + let ogTitle = 'The Mempool Open Source Project®'; if (matchedRoute.render) { ogImageUrl = `${config.SERVER.HOST}/render/${lang || 'en'}/preview${path}`; @@ -187,7 +187,7 @@ class Server { ${ogTitle} - +