From 5dbf6789a7908246405dbb8a3d62e7412dbe846b Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 20 Dec 2020 22:36:36 +0700 Subject: [PATCH 01/31] Basic bitcoind/romanz-electrum support to sync the mempool and blocks. --- backend/mempool-config.sample.json | 11 +- backend/package.json | 2 + .../bitcoin/bitcoin-api-abstract-factory.ts | 16 +++ .../src/api/bitcoin/bitcoin-api-factory.ts | 19 +++ backend/src/api/bitcoin/bitcoind-api.ts | 83 ++++++++++++++ .../src/api/bitcoin/bitcoind-electrs-api.ts | 108 ++++++++++++++++++ backend/src/api/bitcoin/electrs-api.ts | 24 ++-- backend/src/api/blocks.ts | 39 ++++--- backend/src/api/common.ts | 4 +- backend/src/api/mempool.ts | 47 ++++++-- backend/src/config.ts | 22 +++- backend/src/interfaces.ts | 56 ++++++++- 12 files changed, 393 insertions(+), 38 deletions(-) create mode 100644 backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts create mode 100644 backend/src/api/bitcoin/bitcoin-api-factory.ts create mode 100644 backend/src/api/bitcoin/bitcoind-api.ts create mode 100644 backend/src/api/bitcoin/bitcoind-electrs-api.ts diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 2e7e09316..473622ef5 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -1,6 +1,7 @@ { "MEMPOOL": { "NETWORK": "mainnet", + "BACKEND": "electrs", "HTTP_PORT": 8999, "SPAWN_CLUSTER_PROCS": 0, "API_URL_PREFIX": "/api/v1/", @@ -8,7 +9,15 @@ }, "ELECTRS": { "REST_API_URL": "http://127.0.0.1:3000", - "POLL_RATE_MS": 2000 + "POLL_RATE_MS": 2000, + "HOST": "127.0.0.1", + "PORT": 50002 + }, + "BITCOIND": { + "HOST": "127.0.0.1", + "PORT": 3306, + "USERNAME": "mempool", + "PASSWORD": "mempool" }, "DATABASE": { "ENABLED": true, diff --git a/backend/package.json b/backend/package.json index 369308e71..fa2c79c11 100644 --- a/backend/package.json +++ b/backend/package.json @@ -27,6 +27,8 @@ "test": "echo \"Error: no test specified\" && exit 1" }, "dependencies": { + "@codewarriorr/electrum-client-js": "^0.1.1", + "@mempool/bitcoin": "^3.0.2", "axios": "^0.21.0", "express": "^4.17.1", "locutus": "^2.0.12", diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts new file mode 100644 index 000000000..01347884a --- /dev/null +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -0,0 +1,16 @@ +import { MempoolInfo, Transaction, Block, MempoolEntries, MempoolEntry } from '../../interfaces'; + +export interface AbstractBitcoinApi { + getMempoolInfo(): Promise; + getRawMempool(): Promise; + getRawTransaction(txId: string): Promise; + getBlockHeightTip(): Promise; + getTxIdsForBlock(hash: string): Promise; + getBlockHash(height: number): Promise; + getBlock(hash: string): Promise; + getMempoolEntry(txid: string): Promise; + + // Custom + getRawMempoolVerbose(): Promise; + getRawTransactionBitcond(txId: string): Promise; +} diff --git a/backend/src/api/bitcoin/bitcoin-api-factory.ts b/backend/src/api/bitcoin/bitcoin-api-factory.ts new file mode 100644 index 000000000..f625e99e1 --- /dev/null +++ b/backend/src/api/bitcoin/bitcoin-api-factory.ts @@ -0,0 +1,19 @@ +import config from '../../config'; +import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; +import BitcoindElectrsApi from './bitcoind-electrs-api'; +import BitcoindApi from './bitcoind-api'; +import ElectrsApi from './electrs-api'; + +function bitcoinApiFactory(): AbstractBitcoinApi { + switch (config.MEMPOOL.BACKEND) { + case 'electrs': + return new ElectrsApi(); + case 'bitcoind-electrs': + return new BitcoindElectrsApi(); + case 'bitcoind': + default: + return new BitcoindApi(); + } +} + +export default bitcoinApiFactory(); diff --git a/backend/src/api/bitcoin/bitcoind-api.ts b/backend/src/api/bitcoin/bitcoind-api.ts new file mode 100644 index 000000000..d9c7cb4a3 --- /dev/null +++ b/backend/src/api/bitcoin/bitcoind-api.ts @@ -0,0 +1,83 @@ +import config from '../../config'; +import { Transaction, Block, MempoolInfo, RpcBlock, MempoolEntries, MempoolEntry } from '../../interfaces'; +import * as bitcoin from '@mempool/bitcoin'; + +class BitcoindApi { + bitcoindClient: any; + + constructor() { + this.bitcoindClient = new bitcoin.Client({ + host: config.BITCOIND.HOST, + port: config.BITCOIND.PORT, + user: config.BITCOIND.USERNAME, + pass: config.BITCOIND.PASSWORD, + timeout: 60000, + }); + } + + getMempoolInfo(): Promise { + return this.bitcoindClient.getMempoolInfo(); + } + + getRawMempool(): Promise { + return this.bitcoindClient.getRawMemPool(); + } + + getRawMempoolVerbose(): Promise { + return this.bitcoindClient.getRawMemPool(true); + } + + getMempoolEntry(txid: string): Promise { + return this.bitcoindClient.getMempoolEntry(txid,); + } + + getRawTransaction(txId: string): Promise { + return this.bitcoindClient.getRawTransaction(txId, true) + .then((transaction: Transaction) => { + transaction.vout.forEach((vout) => vout.value = vout.value * 100000000); + return transaction; + }); + } + + getBlockHeightTip(): Promise { + return this.bitcoindClient.getChainTips() + .then((result) => result[0].height); + } + + getTxIdsForBlock(hash: string): Promise { + return this.bitcoindClient.getBlock(hash, 1) + .then((rpcBlock: RpcBlock) => { + return rpcBlock.tx; + }); + } + + getBlockHash(height: number): Promise { + return this.bitcoindClient.getBlockHash(height) + } + + getBlock(hash: string): Promise { + return this.bitcoindClient.getBlock(hash) + .then((rpcBlock: RpcBlock) => { + return { + id: rpcBlock.hash, + height: rpcBlock.height, + version: rpcBlock.version, + timestamp: rpcBlock.time, + bits: rpcBlock.bits, + nonce: rpcBlock.nonce, + difficulty: rpcBlock.difficulty, + merkle_root: rpcBlock.merkleroot, + tx_count: rpcBlock.nTx, + size: rpcBlock.size, + weight: rpcBlock.weight, + previousblockhash: rpcBlock.previousblockhash, + }; + }); + } + + getRawTransactionBitcond(txId: string): Promise { + throw new Error('Method not implemented.'); + } +} + +export default BitcoindApi; diff --git a/backend/src/api/bitcoin/bitcoind-electrs-api.ts b/backend/src/api/bitcoin/bitcoind-electrs-api.ts new file mode 100644 index 000000000..b99321a78 --- /dev/null +++ b/backend/src/api/bitcoin/bitcoind-electrs-api.ts @@ -0,0 +1,108 @@ +import config from '../../config'; +import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; +import { Transaction, Block, MempoolInfo, RpcBlock, MempoolEntries, MempoolEntry } from '../../interfaces'; +import * as bitcoin from '@mempool/bitcoin'; +import * as ElectrumClient from '@codewarriorr/electrum-client-js'; +import logger from '../../logger'; + +class BitcoindElectrsApi implements AbstractBitcoinApi { + bitcoindClient: any; + electrumClient: any; + + constructor() { + this.bitcoindClient = new bitcoin.Client({ + host: config.BITCOIND.HOST, + port: config.BITCOIND.PORT, + user: config.BITCOIND.USERNAME, + pass: config.BITCOIND.PASSWORD, + timeout: 60000, + }); + + this.electrumClient = new ElectrumClient( + config.ELECTRS.HOST, + config.ELECTRS.PORT, + 'ssl' + ); + + this.electrumClient.connect( + 'electrum-client-js', + '1.4' + ) + } + + getMempoolInfo(): Promise { + return this.bitcoindClient.getMempoolInfo(); + } + + getRawMempool(): Promise { + return this.bitcoindClient.getRawMemPool(); + } + + getRawMempoolVerbose(): Promise { + return this.bitcoindClient.getRawMemPool(true); + } + + getMempoolEntry(txid: string): Promise { + return this.bitcoindClient.getMempoolEntry(txid,); + } + + async getRawTransaction(txId: string): Promise { + try { + const transaction: Transaction = await this.electrumClient.blockchain_transaction_get(txId, true); + if (!transaction) { + throw new Error('not found'); + } + transaction.vout.forEach((vout) => vout.value = vout.value * 100000000); + return transaction; + } catch (e) { + logger.debug('getRawTransaction error: ' + (e.message || e)); + throw new Error(e); + } + } + + getRawTransactionBitcond(txId: string): Promise { + return this.bitcoindClient.getRawTransaction(txId, true) + .then((transaction: Transaction) => { + transaction.vout.forEach((vout) => vout.value = vout.value * 100000000); + return transaction; + }); + } + + getBlockHeightTip(): Promise { + return this.bitcoindClient.getChainTips() + .then((result) => result[0].height); + } + + getTxIdsForBlock(hash: string): Promise { + return this.bitcoindClient.getBlock(hash, 1) + .then((rpcBlock: RpcBlock) => { + return rpcBlock.tx; + }); + } + + getBlockHash(height: number): Promise { + return this.bitcoindClient.getBlockHash(height) + } + + getBlock(hash: string): Promise { + return this.bitcoindClient.getBlock(hash) + .then((rpcBlock: RpcBlock) => { + return { + id: rpcBlock.hash, + height: rpcBlock.height, + version: rpcBlock.version, + timestamp: rpcBlock.time, + bits: rpcBlock.bits, + nonce: rpcBlock.nonce, + difficulty: rpcBlock.difficulty, + merkle_root: rpcBlock.merkleroot, + tx_count: rpcBlock.nTx, + size: rpcBlock.size, + weight: rpcBlock.weight, + previousblockhash: rpcBlock.previousblockhash, + }; + }); + } +} + +export default BitcoindElectrsApi; diff --git a/backend/src/api/bitcoin/electrs-api.ts b/backend/src/api/bitcoin/electrs-api.ts index 0530f0cfa..a7028ab59 100644 --- a/backend/src/api/bitcoin/electrs-api.ts +++ b/backend/src/api/bitcoin/electrs-api.ts @@ -1,8 +1,9 @@ import config from '../../config'; -import { Transaction, Block, MempoolInfo } from '../../interfaces'; +import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; +import { Transaction, Block, MempoolInfo, MempoolEntry, MempoolEntries } from '../../interfaces'; import axios from 'axios'; -class ElectrsApi { +class ElectrsApi implements AbstractBitcoinApi { constructor() { } @@ -42,15 +43,22 @@ class ElectrsApi { .then((response) => response.data); } - getBlocksFromHeight(height: number): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/blocks/' + height) - .then((response) => response.data); - } - getBlock(hash: string): Promise { return axios.get(config.ELECTRS.REST_API_URL + '/block/' + hash) .then((response) => response.data); } + + getRawMempoolVerbose(): Promise { + throw new Error('Method not implemented.'); + } + + getMempoolEntry(): Promise { + throw new Error('Method not implemented.'); + } + + getRawTransactionBitcond(txId: string): Promise { + throw new Error('Method not implemented.'); + } } -export default new ElectrsApi(); +export default ElectrsApi; diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index de1d702aa..31e2525f6 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -1,7 +1,8 @@ -import bitcoinApi from './bitcoin/electrs-api'; +import config from '../config'; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; import logger from '../logger'; import memPool from './mempool'; -import { Block, TransactionExtended, TransactionMinerInfo } from '../interfaces'; +import { Block, Transaction, TransactionExtended, TransactionMinerInfo } from '../interfaces'; import { Common } from './common'; import diskCache from './disk-cache'; @@ -55,31 +56,38 @@ class Blocks { logger.debug(`New block found (#${this.currentBlockHeight})!`); } + let transactions: TransactionExtended[] = []; + const blockHash = await bitcoinApi.getBlockHash(this.currentBlockHeight); const block = await bitcoinApi.getBlock(blockHash); - const txIds = await bitcoinApi.getTxIdsForBlock(blockHash); + let txIds: string[] = await bitcoinApi.getTxIdsForBlock(blockHash); const mempool = memPool.getMempool(); let found = 0; - let notFound = 0; - - const transactions: TransactionExtended[] = []; for (let i = 0; i < txIds.length; i++) { if (mempool[txIds[i]]) { transactions.push(mempool[txIds[i]]); found++; } else { - logger.debug(`Fetching block tx ${i} of ${txIds.length}`); - const tx = await memPool.getTransactionExtended(txIds[i]); - if (tx) { - transactions.push(tx); + if (config.MEMPOOL.BACKEND === 'electrs') { + logger.debug(`Fetching block tx ${i} of ${txIds.length}`); + const tx = await memPool.getTransactionExtended(txIds[i]); + if (tx) { + transactions.push(tx); + } + } else { // When using bitcoind, just skip parsing past block tx's for now + if (i === 0) { + const tx = await memPool.getTransactionExtended(txIds[i], true); + if (tx) { + transactions.push(tx); + } + } } - notFound++; } } - logger.debug(`${found} of ${txIds.length} found in mempool. ${notFound} not found.`); + logger.debug(`${found} of ${txIds.length} found in mempool. ${txIds.length - found} not found.`); block.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); block.coinbaseTx = this.stripCoinbaseTransaction(transactions[0]); @@ -110,10 +118,13 @@ class Blocks { private stripCoinbaseTransaction(tx: TransactionExtended): TransactionMinerInfo { return { vin: [{ - scriptsig: tx.vin[0].scriptsig + scriptsig: tx.vin[0].scriptsig || tx.vin[0]['coinbase'] }], vout: tx.vout - .map((vout) => ({ scriptpubkey_address: vout.scriptpubkey_address, value: vout.value })) + .map((vout) => ({ + scriptpubkey_address: vout.scriptpubkey_address || (vout['scriptPubKey']['addresses'] && vout['scriptPubKey']['addresses'][0]) || null, + value: vout.value + })) .filter((vout) => vout.value) }; } diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 63256be9b..6e87c9896 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -1,4 +1,4 @@ -import { Transaction, TransactionExtended, TransactionStripped } from '../interfaces'; +import { TransactionExtended, TransactionStripped } from '../interfaces'; export class Common { static median(numbers: number[]) { @@ -53,7 +53,7 @@ export class Common { txid: tx.txid, fee: tx.fee, weight: tx.weight, - value: tx.vin.reduce((acc, vin) => acc + (vin.prevout ? vin.prevout.value : 0), 0), + value: tx.vout ? tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0) : 0, }; } } diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 2a509aa3f..6a477d5f7 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -1,6 +1,6 @@ import config from '../config'; -import bitcoinApi from './bitcoin/electrs-api'; -import { MempoolInfo, TransactionExtended, Transaction, VbytesPerSecond } from '../interfaces'; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; +import { MempoolInfo, TransactionExtended, Transaction, VbytesPerSecond, MempoolEntry, MempoolEntries } from '../interfaces'; import logger from '../logger'; import { Common } from './common'; @@ -18,6 +18,7 @@ class Mempool { private vBytesPerSecond: number = 0; private mempoolProtection = 0; private latestTransactions: any[] = []; + private mempoolEntriesCache: MempoolEntries | null = null; constructor() { setInterval(this.updateTxPerSecond.bind(this), 1000); @@ -75,20 +76,47 @@ class Mempool { return txTimes; } - public async getTransactionExtended(txId: string): Promise { + public async getTransactionExtended(txId: string, isCoinbase = false): Promise { try { - const transaction: Transaction = await bitcoinApi.getRawTransaction(txId); - return Object.assign({ - vsize: transaction.weight / 4, - feePerVsize: (transaction.fee || 0) / (transaction.weight / 4), - firstSeen: Math.round((new Date().getTime() / 1000)), - }, transaction); + let transaction: Transaction; + if (!isCoinbase && config.MEMPOOL.BACKEND === 'bitcoind-electrs') { + transaction = await bitcoinApi.getRawTransactionBitcond(txId); + } else { + transaction = await bitcoinApi.getRawTransaction(txId); + } + if (config.MEMPOOL.BACKEND !== 'electrs' && !isCoinbase) { + transaction = await this.$appendFeeData(transaction); + } + return this.extendTransaction(transaction); } catch (e) { logger.debug(txId + ' not found'); return false; } } + private async $appendFeeData(transaction: Transaction): Promise { + let mempoolEntry: MempoolEntry; + if (!this.inSync && !this.mempoolEntriesCache) { + this.mempoolEntriesCache = await bitcoinApi.getRawMempoolVerbose(); + } + if (this.mempoolEntriesCache && this.mempoolEntriesCache[transaction.txid]) { + mempoolEntry = this.mempoolEntriesCache[transaction.txid]; + } else { + mempoolEntry = await bitcoinApi.getMempoolEntry(transaction.txid); + } + transaction.fee = mempoolEntry.fees.base * 100000000; + return transaction; + } + + private extendTransaction(transaction: Transaction | MempoolEntry): TransactionExtended { + // @ts-ignore + return Object.assign({ + vsize: Math.round(transaction.weight / 4), + feePerVsize: Math.max(1, (transaction.fee || 0) / (transaction.weight / 4)), + firstSeen: Math.round((new Date().getTime() / 1000)), + }, transaction); + } + public async $updateMempool() { logger.debug('Updating mempool'); const start = new Date().getTime(); @@ -169,6 +197,7 @@ class Mempool { if (!this.inSync && transactions.length === Object.keys(newMempool).length) { this.inSync = true; + this.mempoolEntriesCache = null; logger.info('The mempool is now in sync!'); } diff --git a/backend/src/config.ts b/backend/src/config.ts index 7aa1d2a84..6e2e3dc9d 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -3,6 +3,7 @@ const configFile = require('../mempool-config.json'); interface IConfig { MEMPOOL: { NETWORK: 'mainnet' | 'testnet' | 'liquid'; + BACKEND: 'electrs' | 'bitcoind' | 'bitcoind-electrs'; HTTP_PORT: number; SPAWN_CLUSTER_PROCS: number; API_URL_PREFIX: string; @@ -11,7 +12,15 @@ interface IConfig { ELECTRS: { REST_API_URL: string; POLL_RATE_MS: number; + HOST: string; + PORT: number; }; + BITCOIND: { + HOST: string; + PORT: number; + USERNAME: string; + PASSWORD: string; + }, DATABASE: { ENABLED: boolean; HOST: string, @@ -44,6 +53,7 @@ interface IConfig { const defaults: IConfig = { 'MEMPOOL': { 'NETWORK': 'mainnet', + 'BACKEND': 'electrs', 'HTTP_PORT': 8999, 'SPAWN_CLUSTER_PROCS': 0, 'API_URL_PREFIX': '/api/v1/', @@ -51,7 +61,15 @@ const defaults: IConfig = { }, 'ELECTRS': { 'REST_API_URL': 'http://127.0.0.1:3000', - 'POLL_RATE_MS': 2000 + 'POLL_RATE_MS': 2000, + 'HOST': '127.0.0.1', + 'PORT': 3306 + }, + 'BITCOIND': { + 'HOST': "127.0.0.1", + 'PORT': 8332, + 'USERNAME': "mempoo", + 'PASSWORD': "mempool" }, 'DATABASE': { 'ENABLED': true, @@ -85,6 +103,7 @@ const defaults: IConfig = { class Config implements IConfig { MEMPOOL: IConfig['MEMPOOL']; ELECTRS: IConfig['ELECTRS']; + BITCOIND: IConfig['BITCOIND']; DATABASE: IConfig['DATABASE']; STATISTICS: IConfig['STATISTICS']; BISQ_BLOCKS: IConfig['BISQ_BLOCKS']; @@ -95,6 +114,7 @@ class Config implements IConfig { const configs = this.merge(configFile, defaults); this.MEMPOOL = configs.MEMPOOL; this.ELECTRS = configs.ELECTRS; + this.BITCOIND = configs.BITCOIND; this.DATABASE = configs.DATABASE; this.STATISTICS = configs.STATISTICS; this.BISQ_BLOCKS = configs.BISQ_BLOCKS; diff --git a/backend/src/interfaces.ts b/backend/src/interfaces.ts index 159ad868d..e4fab5181 100644 --- a/backend/src/interfaces.ts +++ b/backend/src/interfaces.ts @@ -119,7 +119,7 @@ export interface Block { version: number; timestamp: number; bits: number; - nounce: number; + nonce: number; difficulty: number; merkle_root: string; tx_count: number; @@ -132,8 +132,58 @@ export interface Block { feeRange?: number[]; reward?: number; coinbaseTx?: TransactionMinerInfo; - matchRate: number; - stage: number; + matchRate?: number; +} + +export interface RpcBlock { + hash: string; + confirmations: number; + size: number; + strippedsize: number; + weight: number; + height: number; + version: number, + versionHex: string; + merkleroot: string; + tx: Transaction[]; + time: number; + mediantime: number; + nonce: number; + bits: number; + difficulty: number; + chainwork: string; + nTx: number, + previousblockhash: string; + nextblockhash: string; +} + +export interface MempoolEntries { [txId: string]: MempoolEntry }; + +export interface MempoolEntry { + fees: Fees + vsize: number + weight: number + fee: number + modifiedfee: number + time: number + height: number + descendantcount: number + descendantsize: number + descendantfees: number + ancestorcount: number + ancestorsize: number + ancestorfees: number + wtxid: string + depends: any[] + spentby: any[] + 'bip125-replaceable': boolean +} + +export interface Fees { + base: number + modified: number + ancestor: number + descendant: number } export interface Address { From ecc0f316cc35580024545098887d18e84cb2fbf3 Mon Sep 17 00:00:00 2001 From: softsimon Date: Mon, 21 Dec 2020 23:08:34 +0700 Subject: [PATCH 02/31] Refactored transaction handling. --- .../bitcoin/bitcoin-api-abstract-factory.ts | 23 ++-- backend/src/api/bitcoin/bitcoind-api.ts | 30 +++-- .../src/api/bitcoin/bitcoind-electrs-api.ts | 50 ++++--- backend/src/api/bitcoin/electrs-api.ts | 26 ++-- backend/src/api/blocks.ts | 35 +++-- backend/src/api/common.ts | 2 +- backend/src/api/mempool.ts | 50 +------ backend/src/api/transaction-utils.ts | 124 ++++++++++++++++++ backend/src/index.ts | 15 +++ backend/src/interfaces.ts | 48 +++---- backend/src/routes.ts | 69 +++++++++- frontend/proxy.conf.json | 4 +- 12 files changed, 331 insertions(+), 145 deletions(-) create mode 100644 backend/src/api/transaction-utils.ts diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index 01347884a..94332b560 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -1,16 +1,17 @@ -import { MempoolInfo, Transaction, Block, MempoolEntries, MempoolEntry } from '../../interfaces'; +import { MempoolInfo, Transaction, Block, MempoolEntries, MempoolEntry, Address } from '../../interfaces'; export interface AbstractBitcoinApi { - getMempoolInfo(): Promise; - getRawMempool(): Promise; - getRawTransaction(txId: string): Promise; - getBlockHeightTip(): Promise; - getTxIdsForBlock(hash: string): Promise; - getBlockHash(height: number): Promise; - getBlock(hash: string): Promise; - getMempoolEntry(txid: string): Promise; + $getMempoolInfo(): Promise; + $getRawMempool(): Promise; + $getRawTransaction(txId: string): Promise; + $getBlockHeightTip(): Promise; + $getTxIdsForBlock(hash: string): Promise; + $getBlockHash(height: number): Promise; + $getBlock(hash: string): Promise; + $getMempoolEntry(txid: string): Promise; + $getAddress(address: string): Promise
; // Custom - getRawMempoolVerbose(): Promise; - getRawTransactionBitcond(txId: string): Promise; + $getRawMempoolVerbose(): Promise; + $getRawTransactionBitcond(txId: string): Promise; } diff --git a/backend/src/api/bitcoin/bitcoind-api.ts b/backend/src/api/bitcoin/bitcoind-api.ts index d9c7cb4a3..7f01f5f55 100644 --- a/backend/src/api/bitcoin/bitcoind-api.ts +++ b/backend/src/api/bitcoin/bitcoind-api.ts @@ -1,5 +1,5 @@ import config from '../../config'; -import { Transaction, Block, MempoolInfo, RpcBlock, MempoolEntries, MempoolEntry } from '../../interfaces'; +import { Transaction, Block, MempoolInfo, RpcBlock, MempoolEntries, MempoolEntry, Address } from '../../interfaces'; import * as bitcoin from '@mempool/bitcoin'; class BitcoindApi { @@ -15,23 +15,23 @@ class BitcoindApi { }); } - getMempoolInfo(): Promise { + $getMempoolInfo(): Promise { return this.bitcoindClient.getMempoolInfo(); } - getRawMempool(): Promise { + $getRawMempool(): Promise { return this.bitcoindClient.getRawMemPool(); } - getRawMempoolVerbose(): Promise { + $getRawMempoolVerbose(): Promise { return this.bitcoindClient.getRawMemPool(true); } - getMempoolEntry(txid: string): Promise { - return this.bitcoindClient.getMempoolEntry(txid,); + $getMempoolEntry(txid: string): Promise { + return this.bitcoindClient.getMempoolEntry(txid); } - getRawTransaction(txId: string): Promise { + $getRawTransaction(txId: string): Promise { return this.bitcoindClient.getRawTransaction(txId, true) .then((transaction: Transaction) => { transaction.vout.forEach((vout) => vout.value = vout.value * 100000000); @@ -39,23 +39,23 @@ class BitcoindApi { }); } - getBlockHeightTip(): Promise { + $getBlockHeightTip(): Promise { return this.bitcoindClient.getChainTips() .then((result) => result[0].height); } - getTxIdsForBlock(hash: string): Promise { + $getTxIdsForBlock(hash: string): Promise { return this.bitcoindClient.getBlock(hash, 1) .then((rpcBlock: RpcBlock) => { return rpcBlock.tx; }); } - getBlockHash(height: number): Promise { - return this.bitcoindClient.getBlockHash(height) + $getBlockHash(height: number): Promise { + return this.bitcoindClient.getBlockHash(height); } - getBlock(hash: string): Promise { + $getBlock(hash: string): Promise { return this.bitcoindClient.getBlock(hash) .then((rpcBlock: RpcBlock) => { return { @@ -75,7 +75,11 @@ class BitcoindApi { }); } - getRawTransactionBitcond(txId: string): Promise { + $getRawTransactionBitcond(txId: string): Promise { + throw new Error('Method not implemented.'); + } + + $getAddress(address: string): Promise
{ throw new Error('Method not implemented.'); } } diff --git a/backend/src/api/bitcoin/bitcoind-electrs-api.ts b/backend/src/api/bitcoin/bitcoind-electrs-api.ts index b99321a78..f80baa0c2 100644 --- a/backend/src/api/bitcoin/bitcoind-electrs-api.ts +++ b/backend/src/api/bitcoin/bitcoind-electrs-api.ts @@ -1,9 +1,10 @@ import config from '../../config'; import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; -import { Transaction, Block, MempoolInfo, RpcBlock, MempoolEntries, MempoolEntry } from '../../interfaces'; +import { Transaction, Block, MempoolInfo, RpcBlock, MempoolEntries, MempoolEntry, Address } from '../../interfaces'; import * as bitcoin from '@mempool/bitcoin'; import * as ElectrumClient from '@codewarriorr/electrum-client-js'; import logger from '../../logger'; +import transactionUtils from '../transaction-utils'; class BitcoindElectrsApi implements AbstractBitcoinApi { bitcoindClient: any; @@ -27,64 +28,64 @@ class BitcoindElectrsApi implements AbstractBitcoinApi { this.electrumClient.connect( 'electrum-client-js', '1.4' - ) + ); } - getMempoolInfo(): Promise { + $getMempoolInfo(): Promise { return this.bitcoindClient.getMempoolInfo(); } - getRawMempool(): Promise { + $getRawMempool(): Promise { return this.bitcoindClient.getRawMemPool(); } - getRawMempoolVerbose(): Promise { + $getRawMempoolVerbose(): Promise { return this.bitcoindClient.getRawMemPool(true); } - getMempoolEntry(txid: string): Promise { - return this.bitcoindClient.getMempoolEntry(txid,); + $getMempoolEntry(txid: string): Promise { + return this.bitcoindClient.getMempoolEntry(txid); } - async getRawTransaction(txId: string): Promise { + async $getRawTransaction(txId: string): Promise { try { const transaction: Transaction = await this.electrumClient.blockchain_transaction_get(txId, true); if (!transaction) { - throw new Error('not found'); + throw new Error(txId + ' not found!'); } - transaction.vout.forEach((vout) => vout.value = vout.value * 100000000); + transactionUtils.bitcoindToElectrsTransaction(transaction); return transaction; } catch (e) { - logger.debug('getRawTransaction error: ' + (e.message || e)); + logger.debug('getRawTransaction error: ' + (e.message || e)); throw new Error(e); } } - getRawTransactionBitcond(txId: string): Promise { + $getRawTransactionBitcond(txId: string): Promise { return this.bitcoindClient.getRawTransaction(txId, true) .then((transaction: Transaction) => { - transaction.vout.forEach((vout) => vout.value = vout.value * 100000000); + transactionUtils.bitcoindToElectrsTransaction(transaction); return transaction; }); } - getBlockHeightTip(): Promise { + $getBlockHeightTip(): Promise { return this.bitcoindClient.getChainTips() .then((result) => result[0].height); } - getTxIdsForBlock(hash: string): Promise { + $getTxIdsForBlock(hash: string): Promise { return this.bitcoindClient.getBlock(hash, 1) .then((rpcBlock: RpcBlock) => { return rpcBlock.tx; }); } - getBlockHash(height: number): Promise { - return this.bitcoindClient.getBlockHash(height) + $getBlockHash(height: number): Promise { + return this.bitcoindClient.getBlockHash(height); } - getBlock(hash: string): Promise { + $getBlock(hash: string): Promise { return this.bitcoindClient.getBlock(hash) .then((rpcBlock: RpcBlock) => { return { @@ -103,6 +104,19 @@ class BitcoindElectrsApi implements AbstractBitcoinApi { }; }); } + + async $getAddress(address: string): Promise
{ + try { + const addressInfo: Address = await this.electrumClient.blockchain_scripthash_getBalance(address); + if (!address) { + throw new Error('not found'); + } + return addressInfo; + } catch (e) { + logger.debug('getRawTransaction error: ' + (e.message || e)); + throw new Error(e); + } + } } export default BitcoindElectrsApi; diff --git a/backend/src/api/bitcoin/electrs-api.ts b/backend/src/api/bitcoin/electrs-api.ts index a7028ab59..a2dc86ad4 100644 --- a/backend/src/api/bitcoin/electrs-api.ts +++ b/backend/src/api/bitcoin/electrs-api.ts @@ -1,6 +1,6 @@ import config from '../../config'; import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; -import { Transaction, Block, MempoolInfo, MempoolEntry, MempoolEntries } from '../../interfaces'; +import { Transaction, Block, MempoolInfo, MempoolEntry, MempoolEntries, Address } from '../../interfaces'; import axios from 'axios'; class ElectrsApi implements AbstractBitcoinApi { @@ -8,7 +8,7 @@ class ElectrsApi implements AbstractBitcoinApi { constructor() { } - getMempoolInfo(): Promise { + $getMempoolInfo(): Promise { return axios.get(config.ELECTRS.REST_API_URL + '/mempool', { timeout: 10000 }) .then((response) => { return { @@ -18,45 +18,49 @@ class ElectrsApi implements AbstractBitcoinApi { }); } - getRawMempool(): Promise { + $getRawMempool(): Promise { return axios.get(config.ELECTRS.REST_API_URL + '/mempool/txids') .then((response) => response.data); } - getRawTransaction(txId: string): Promise { + $getRawTransaction(txId: string): Promise { return axios.get(config.ELECTRS.REST_API_URL + '/tx/' + txId) .then((response) => response.data); } - getBlockHeightTip(): Promise { + $getBlockHeightTip(): Promise { return axios.get(config.ELECTRS.REST_API_URL + '/blocks/tip/height') .then((response) => response.data); } - getTxIdsForBlock(hash: string): Promise { + $getTxIdsForBlock(hash: string): Promise { return axios.get(config.ELECTRS.REST_API_URL + '/block/' + hash + '/txids') .then((response) => response.data); } - getBlockHash(height: number): Promise { + $getBlockHash(height: number): Promise { return axios.get(config.ELECTRS.REST_API_URL + '/block-height/' + height) .then((response) => response.data); } - getBlock(hash: string): Promise { + $getBlock(hash: string): Promise { return axios.get(config.ELECTRS.REST_API_URL + '/block/' + hash) .then((response) => response.data); } - getRawMempoolVerbose(): Promise { + $getRawMempoolVerbose(): Promise { throw new Error('Method not implemented.'); } - getMempoolEntry(): Promise { + $getMempoolEntry(): Promise { throw new Error('Method not implemented.'); } - getRawTransactionBitcond(txId: string): Promise { + $getRawTransactionBitcond(txId: string): Promise { + throw new Error('Method not implemented.'); + } + + $getAddress(address: string): Promise
{ throw new Error('Method not implemented.'); } } diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 31e2525f6..a941f8fa2 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -2,9 +2,10 @@ import config from '../config'; import bitcoinApi from './bitcoin/bitcoin-api-factory'; import logger from '../logger'; import memPool from './mempool'; -import { Block, Transaction, TransactionExtended, TransactionMinerInfo } from '../interfaces'; +import { Block, TransactionExtended, TransactionMinerInfo } from '../interfaces'; import { Common } from './common'; import diskCache from './disk-cache'; +import transactionUtils from './transaction-utils'; class Blocks { private static KEEP_BLOCK_AMOUNT = 8; @@ -28,7 +29,7 @@ class Blocks { } public async $updateBlocks() { - const blockHeightTip = await bitcoinApi.getBlockHeightTip(); + const blockHeightTip = await bitcoinApi.$getBlockHeightTip(); if (this.blocks.length === 0) { this.currentBlockHeight = blockHeightTip - Blocks.KEEP_BLOCK_AMOUNT; @@ -43,8 +44,8 @@ class Blocks { if (!this.lastDifficultyAdjustmentTime) { const heightDiff = blockHeightTip % 2016; - const blockHash = await bitcoinApi.getBlockHash(blockHeightTip - heightDiff); - const block = await bitcoinApi.getBlock(blockHash); + const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff); + const block = await bitcoinApi.$getBlock(blockHash); this.lastDifficultyAdjustmentTime = block.timestamp; } @@ -56,11 +57,11 @@ class Blocks { logger.debug(`New block found (#${this.currentBlockHeight})!`); } - let transactions: TransactionExtended[] = []; + const transactions: TransactionExtended[] = []; - const blockHash = await bitcoinApi.getBlockHash(this.currentBlockHeight); - const block = await bitcoinApi.getBlock(blockHash); - let txIds: string[] = await bitcoinApi.getTxIdsForBlock(blockHash); + const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight); + const block = await bitcoinApi.$getBlock(blockHash); + const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); const mempool = memPool.getMempool(); let found = 0; @@ -70,19 +71,13 @@ class Blocks { transactions.push(mempool[txIds[i]]); found++; } else { - if (config.MEMPOOL.BACKEND === 'electrs') { + // When using bitcoind, just skip parsing past block tx's for now except for coinbase + if (config.MEMPOOL.BACKEND === 'electrs' || i === 0) { // logger.debug(`Fetching block tx ${i} of ${txIds.length}`); - const tx = await memPool.getTransactionExtended(txIds[i]); + const tx = await transactionUtils.getTransactionExtended(txIds[i]); if (tx) { transactions.push(tx); } - } else { // When using bitcoind, just skip parsing past block tx's for now - if (i === 0) { - const tx = await memPool.getTransactionExtended(txIds[i], true); - if (tx) { - transactions.push(tx); - } - } } } } @@ -115,6 +110,10 @@ class Blocks { return this.lastDifficultyAdjustmentTime; } + public getCurrentBlockHeight(): number { + return this.currentBlockHeight; + } + private stripCoinbaseTransaction(tx: TransactionExtended): TransactionMinerInfo { return { vin: [{ @@ -122,7 +121,7 @@ class Blocks { }], vout: tx.vout .map((vout) => ({ - scriptpubkey_address: vout.scriptpubkey_address || (vout['scriptPubKey']['addresses'] && vout['scriptPubKey']['addresses'][0]) || null, + scriptpubkey_address: vout.scriptpubkey_address, value: vout.value })) .filter((vout) => vout.value) diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 6e87c9896..fdd50a11a 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -53,7 +53,7 @@ export class Common { txid: tx.txid, fee: tx.fee, weight: tx.weight, - value: tx.vout ? tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0) : 0, + value: tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0), }; } } diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 6a477d5f7..cb13fb874 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -3,6 +3,7 @@ import bitcoinApi from './bitcoin/bitcoin-api-factory'; import { MempoolInfo, TransactionExtended, Transaction, VbytesPerSecond, MempoolEntry, MempoolEntries } from '../interfaces'; import logger from '../logger'; import { Common } from './common'; +import transactionUtils from './transaction-utils'; class Mempool { private inSync: boolean = false; @@ -18,7 +19,6 @@ class Mempool { private vBytesPerSecond: number = 0; private mempoolProtection = 0; private latestTransactions: any[] = []; - private mempoolEntriesCache: MempoolEntries | null = null; constructor() { setInterval(this.updateTxPerSecond.bind(this), 1000); @@ -49,7 +49,7 @@ class Mempool { } public async $updateMemPoolInfo() { - this.mempoolInfo = await bitcoinApi.getMempoolInfo(); + this.mempoolInfo = await bitcoinApi.$getMempoolInfo(); } public getMempoolInfo(): MempoolInfo | undefined { @@ -76,60 +76,19 @@ class Mempool { return txTimes; } - public async getTransactionExtended(txId: string, isCoinbase = false): Promise { - try { - let transaction: Transaction; - if (!isCoinbase && config.MEMPOOL.BACKEND === 'bitcoind-electrs') { - transaction = await bitcoinApi.getRawTransactionBitcond(txId); - } else { - transaction = await bitcoinApi.getRawTransaction(txId); - } - if (config.MEMPOOL.BACKEND !== 'electrs' && !isCoinbase) { - transaction = await this.$appendFeeData(transaction); - } - return this.extendTransaction(transaction); - } catch (e) { - logger.debug(txId + ' not found'); - return false; - } - } - - private async $appendFeeData(transaction: Transaction): Promise { - let mempoolEntry: MempoolEntry; - if (!this.inSync && !this.mempoolEntriesCache) { - this.mempoolEntriesCache = await bitcoinApi.getRawMempoolVerbose(); - } - if (this.mempoolEntriesCache && this.mempoolEntriesCache[transaction.txid]) { - mempoolEntry = this.mempoolEntriesCache[transaction.txid]; - } else { - mempoolEntry = await bitcoinApi.getMempoolEntry(transaction.txid); - } - transaction.fee = mempoolEntry.fees.base * 100000000; - return transaction; - } - - private extendTransaction(transaction: Transaction | MempoolEntry): TransactionExtended { - // @ts-ignore - return Object.assign({ - vsize: Math.round(transaction.weight / 4), - feePerVsize: Math.max(1, (transaction.fee || 0) / (transaction.weight / 4)), - firstSeen: Math.round((new Date().getTime() / 1000)), - }, transaction); - } - public async $updateMempool() { logger.debug('Updating mempool'); const start = new Date().getTime(); let hasChange: boolean = false; const currentMempoolSize = Object.keys(this.mempoolCache).length; let txCount = 0; - const transactions = await bitcoinApi.getRawMempool(); + const transactions = await bitcoinApi.$getRawMempool(); const diff = transactions.length - currentMempoolSize; const newTransactions: TransactionExtended[] = []; for (const txid of transactions) { if (!this.mempoolCache[txid]) { - const transaction = await this.getTransactionExtended(txid); + const transaction = await transactionUtils.getTransactionExtended(txid, false, true); if (transaction) { this.mempoolCache[txid] = transaction; txCount++; @@ -197,7 +156,6 @@ class Mempool { if (!this.inSync && transactions.length === Object.keys(newMempool).length) { this.inSync = true; - this.mempoolEntriesCache = null; logger.info('The mempool is now in sync!'); } diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts new file mode 100644 index 000000000..c94a903be --- /dev/null +++ b/backend/src/api/transaction-utils.ts @@ -0,0 +1,124 @@ +import bitcoinApi from './bitcoin/bitcoin-api-factory'; +import { MempoolEntries, MempoolEntry, Transaction, TransactionExtended, TransactionMinerInfo } from '../interfaces'; +import config from '../config'; +import logger from '../logger'; +import mempool from './mempool'; +import blocks from './blocks'; + +class TransactionUtils { + private mempoolEntriesCache: MempoolEntries | null = null; + + constructor() { } + + public async $addPrevoutsToTransaction(transaction: TransactionExtended): Promise { + for (const vin of transaction.vin) { + const innerTx = await bitcoinApi.$getRawTransaction(vin.txid); + vin.prevout = innerTx.vout[vin.vout]; + } + return transaction; + } + + public async $calculateFeeFromInputs(transaction: Transaction): Promise { + if (transaction.vin[0]['coinbase']) { + transaction.fee = 0; + // @ts-ignore + return transaction; + } + let totalIn = 0; + for (const vin of transaction.vin) { + const innerTx = await bitcoinApi.$getRawTransaction(vin.txid); + totalIn += innerTx.vout[vin.vout].value; + } + const totalOut = transaction.vout.reduce((prev, output) => prev + output.value, 0); + transaction.fee = parseFloat((totalIn - totalOut).toFixed(8)); + return this.extendTransaction(transaction); + } + + public extendTransaction(transaction: Transaction | MempoolEntry): TransactionExtended { + // @ts-ignore + return Object.assign({ + vsize: Math.round(transaction.weight / 4), + feePerVsize: Math.max(1, (transaction.fee || 0) / (transaction.weight / 4)), + firstSeen: Math.round((new Date().getTime() / 1000)), + }, transaction); + } + + public stripCoinbaseTransaction(tx: TransactionExtended): TransactionMinerInfo { + return { + vin: [{ + scriptsig: tx.vin[0].scriptsig || tx.vin[0]['coinbase'] + }], + vout: tx.vout + .map((vout) => ({ + scriptpubkey_address: vout.scriptpubkey_address, + value: vout.value + })) + .filter((vout) => vout.value) + }; + } + + public async getTransactionExtended(txId: string, isCoinbase = false, inMempool = false): Promise { + try { + let transaction: Transaction; + if (inMempool) { + transaction = await bitcoinApi.$getRawTransactionBitcond(txId); + } else { + transaction = await bitcoinApi.$getRawTransaction(txId); + } + if (config.MEMPOOL.BACKEND !== 'electrs' && !isCoinbase) { + if (inMempool) { + transaction = await this.$appendFeeData(transaction); + } else { + transaction = await this.$calculateFeeFromInputs(transaction); + } + } + return this.extendTransaction(transaction); + } catch (e) { + logger.debug('getTransactionExtended error: ' + (e.message || e)); + console.log(e); + return null; + } + } + + public bitcoindToElectrsTransaction(transaction: any): void { + try { + transaction.vout = transaction.vout.map((vout) => { + return { + value: vout.value * 100000000, + scriptpubkey: vout.scriptPubKey.hex, + scriptpubkey_address: vout.scriptPubKey && vout.scriptPubKey.addresses ? vout.scriptPubKey.addresses[0] : null, + scriptpubkey_asm: vout.scriptPubKey.asm, + scriptpubkey_type: vout.scriptPubKey.type, + }; + }); + if (transaction.confirmations) { + transaction['status'] = { + confirmed: true, + block_height: blocks.getCurrentBlockHeight() - transaction.confirmations, + block_hash: transaction.blockhash, + block_time: transaction.blocktime, + }; + } else { + transaction['status'] = { confirmed: false }; + } + } catch (e) { + console.log('augment failed: ' + (e.message || e)); + } + } + + private async $appendFeeData(transaction: Transaction): Promise { + let mempoolEntry: MempoolEntry; + if (!mempool.isInSync() && !this.mempoolEntriesCache) { + this.mempoolEntriesCache = await bitcoinApi.$getRawMempoolVerbose(); + } + if (this.mempoolEntriesCache && this.mempoolEntriesCache[transaction.txid]) { + mempoolEntry = this.mempoolEntriesCache[transaction.txid]; + } else { + mempoolEntry = await bitcoinApi.$getMempoolEntry(transaction.txid); + } + transaction.fee = mempoolEntry.fees.base * 100000000; + return transaction; + } +} + +export default new TransactionUtils(); diff --git a/backend/src/index.ts b/backend/src/index.ts index ab50afc36..11878460f 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -208,6 +208,21 @@ class Server { } }); } + + if (config.MEMPOOL.BACKEND === 'bitcoind' || config.MEMPOOL.BACKEND === 'bitcoind-electrs') { + this.app + .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId', routes.getTransaction) + .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', routes.getTransactionOutspends) + .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', routes.getBlock) + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', routes.getBlocks) + .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs/:index', routes.getBlockTransactions) + .get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', routes.getBlockHeight) + .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', routes.getAddress) + .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', routes.getAddressTransactions) + .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', routes.getAdressTxChain) + .get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', routes.getAddressPrefix) + ; + } } } diff --git a/backend/src/interfaces.ts b/backend/src/interfaces.ts index e4fab5181..302d43e12 100644 --- a/backend/src/interfaces.ts +++ b/backend/src/interfaces.ts @@ -142,7 +142,7 @@ export interface RpcBlock { strippedsize: number; weight: number; height: number; - version: number, + version: number; versionHex: string; merkleroot: string; tx: Transaction[]; @@ -152,38 +152,38 @@ export interface RpcBlock { bits: number; difficulty: number; chainwork: string; - nTx: number, + nTx: number; previousblockhash: string; nextblockhash: string; } -export interface MempoolEntries { [txId: string]: MempoolEntry }; +export interface MempoolEntries { [txId: string]: MempoolEntry; } export interface MempoolEntry { - fees: Fees - vsize: number - weight: number - fee: number - modifiedfee: number - time: number - height: number - descendantcount: number - descendantsize: number - descendantfees: number - ancestorcount: number - ancestorsize: number - ancestorfees: number - wtxid: string - depends: any[] - spentby: any[] - 'bip125-replaceable': boolean + fees: Fees; + vsize: number; + weight: number; + fee: number; + modifiedfee: number; + time: number; + height: number; + descendantcount: number; + descendantsize: number; + descendantfees: number; + ancestorcount: number; + ancestorsize: number; + ancestorfees: number; + wtxid: string; + depends: any[]; + spentby: any[]; + 'bip125-replaceable': boolean; } export interface Fees { - base: number - modified: number - ancestor: number - descendant: number + base: number; + modified: number; + ancestor: number; + descendant: number; } export interface Address { diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 3d60f7df2..bc24c7b6b 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -8,10 +8,12 @@ import mempool from './api/mempool'; import bisq from './api/bisq/bisq'; import websocketHandler from './api/websocket-handler'; import bisqMarket from './api/bisq/markets-api'; -import { OptimizedStatistic, RequiredSpec } from './interfaces'; +import { OptimizedStatistic, RequiredSpec, Transaction, TransactionExtended } from './interfaces'; import { MarketsApiError } from './api/bisq/interfaces'; import donations from './api/donations'; import logger from './logger'; +import bitcoinApi from './api/bitcoin/bitcoin-api-factory'; +import transactionUtils from './api/transaction-utils'; class Routes { private cache: { [date: string]: OptimizedStatistic[] } = { @@ -524,6 +526,71 @@ class Routes { }; } + public async getTransaction(req: Request, res: Response) { + try { + let transaction: TransactionExtended | null; + const txInMempool = mempool.getMempool()[req.params.txId]; + if (txInMempool) { + transaction = txInMempool; + } else { + transaction = await transactionUtils.getTransactionExtended(req.params.txId); + } + if (transaction) { + transaction = await transactionUtils.$addPrevoutsToTransaction(transaction); + res.json(transaction); + } else { + res.status(500).send('Error fetching transaction.'); + } + } catch (e) { + res.status(500).send(e.message); + } + } + + public async getBlock(req: Request, res: Response) { + try { + const result = await bitcoinApi.$getBlock(req.params.hash); + res.json(result); + } catch (e) { + res.status(500).send(e.message); + } + } + + public async getBlocks(req: Request, res: Response) { + res.status(404).send('Not implemented'); + } + + public async getBlockTransactions(req: Request, res: Response) { + res.status(404).send('Not implemented'); + } + + public async getBlockHeight(req: Request, res: Response) { + res.status(404).send('Not implemented'); + } + + public async getAddress(req: Request, res: Response) { + try { + const result = await bitcoinApi.$getAddress(req.params.hash); + res.json(result); + } catch (e) { + res.status(500).send(e.message); + } + } + + public async getAddressTransactions(req: Request, res: Response) { + res.status(404).send('Not implemented'); + } + + public async getAdressTxChain(req: Request, res: Response) { + res.status(404).send('Not implemented'); + } + + public async getAddressPrefix(req: Request, res: Response) { + res.json([]); + } + + public getTransactionOutspends(req: Request, res: Response) { + res.json([]); + } } export default new Routes(); diff --git a/frontend/proxy.conf.json b/frontend/proxy.conf.json index c02eb5e1e..5886ff104 100644 --- a/frontend/proxy.conf.json +++ b/frontend/proxy.conf.json @@ -9,10 +9,10 @@ "ws": true }, "/api/": { - "target": "http://localhost:50001/", + "target": "http://localhost:8999/", "secure": false, "pathRewrite": { - "^/api/": "" + "^/api/": "/api/v1/" } }, "/testnet/api/v1": { From f84b9e6582b6a0e80333d2f7ecf441ed48bccaa0 Mon Sep 17 00:00:00 2001 From: softsimon Date: Tue, 22 Dec 2020 06:04:31 +0700 Subject: [PATCH 03/31] Address page mostly working. --- backend/package.json | 1 + .../bitcoin/bitcoin-api-abstract-factory.ts | 6 +- backend/src/api/bitcoin/bitcoind-api.ts | 18 ++++- .../src/api/bitcoin/bitcoind-electrs-api.ts | 23 +++++- backend/src/api/bitcoin/electrs-api.ts | 16 +++- backend/src/api/blocks.ts | 21 +++-- backend/src/api/transaction-utils.ts | 34 ++++++-- backend/src/interfaces.ts | 24 ++++++ backend/src/routes.ts | 77 ++++++++++++++++++- 9 files changed, 195 insertions(+), 25 deletions(-) diff --git a/backend/package.json b/backend/package.json index fa2c79c11..3796c340e 100644 --- a/backend/package.json +++ b/backend/package.json @@ -30,6 +30,7 @@ "@codewarriorr/electrum-client-js": "^0.1.1", "@mempool/bitcoin": "^3.0.2", "axios": "^0.21.0", + "crypto-js": "^4.0.0", "express": "^4.17.1", "locutus": "^2.0.12", "mysql2": "^1.6.1", diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index 94332b560..cce6a352b 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -1,4 +1,5 @@ -import { MempoolInfo, Transaction, Block, MempoolEntries, MempoolEntry, Address } from '../../interfaces'; +import { MempoolInfo, Transaction, Block, MempoolEntries, MempoolEntry, Address, AddressInformation, + ScriptHashBalance, ScriptHashHistory } from '../../interfaces'; export interface AbstractBitcoinApi { $getMempoolInfo(): Promise; @@ -10,6 +11,9 @@ export interface AbstractBitcoinApi { $getBlock(hash: string): Promise; $getMempoolEntry(txid: string): Promise; $getAddress(address: string): Promise
; + $validateAddress(address: string): Promise; + $getScriptHashBalance(scriptHash: string): Promise; + $getScriptHashHistory(scriptHash: string): Promise; // Custom $getRawMempoolVerbose(): Promise; diff --git a/backend/src/api/bitcoin/bitcoind-api.ts b/backend/src/api/bitcoin/bitcoind-api.ts index 7f01f5f55..d008e95b2 100644 --- a/backend/src/api/bitcoin/bitcoind-api.ts +++ b/backend/src/api/bitcoin/bitcoind-api.ts @@ -1,8 +1,10 @@ import config from '../../config'; -import { Transaction, Block, MempoolInfo, RpcBlock, MempoolEntries, MempoolEntry, Address } from '../../interfaces'; +import { Transaction, Block, MempoolInfo, RpcBlock, MempoolEntries, MempoolEntry, Address, + AddressInformation, ScriptHashBalance, ScriptHashHistory } from '../../interfaces'; import * as bitcoin from '@mempool/bitcoin'; +import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; -class BitcoindApi { +class BitcoindApi implements AbstractBitcoinApi { bitcoindClient: any; constructor() { @@ -82,6 +84,18 @@ class BitcoindApi { $getAddress(address: string): Promise
{ throw new Error('Method not implemented.'); } + + $validateAddress(address: string): Promise { + return this.bitcoindClient.validateAddress(address); + } + + $getScriptHashBalance(scriptHash: string): Promise { + throw new Error('Method not implemented.'); + } + + $getScriptHashHistory(scriptHash: string): Promise { + throw new Error('Method not implemented.'); + } } export default BitcoindApi; diff --git a/backend/src/api/bitcoin/bitcoind-electrs-api.ts b/backend/src/api/bitcoin/bitcoind-electrs-api.ts index f80baa0c2..cbbdec8f2 100644 --- a/backend/src/api/bitcoin/bitcoind-electrs-api.ts +++ b/backend/src/api/bitcoin/bitcoind-electrs-api.ts @@ -1,11 +1,13 @@ import config from '../../config'; import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; -import { Transaction, Block, MempoolInfo, RpcBlock, MempoolEntries, MempoolEntry, Address } from '../../interfaces'; +import { Transaction, Block, MempoolInfo, RpcBlock, MempoolEntries, MempoolEntry, Address, + AddressInformation, ScriptHashBalance, ScriptHashHistory } from '../../interfaces'; import * as bitcoin from '@mempool/bitcoin'; import * as ElectrumClient from '@codewarriorr/electrum-client-js'; import logger from '../../logger'; import transactionUtils from '../transaction-utils'; - +import * as sha256 from 'crypto-js/sha256'; +import * as hexEnc from 'crypto-js/enc-hex'; class BitcoindElectrsApi implements AbstractBitcoinApi { bitcoindClient: any; electrumClient: any; @@ -117,6 +119,23 @@ class BitcoindElectrsApi implements AbstractBitcoinApi { throw new Error(e); } } + + $validateAddress(address: string): Promise { + return this.bitcoindClient.validateAddress(address); + } + + $getScriptHashBalance(scriptHash: string): Promise { + return this.electrumClient.blockchain_scripthash_getBalance(this.encodeScriptHash(scriptHash)); + } + + $getScriptHashHistory(scriptHash: string): Promise { + return this.electrumClient.blockchain_scripthash_getHistory(this.encodeScriptHash(scriptHash)); + } + + private encodeScriptHash(scriptPubKey: string): string { + const addrScripthash = hexEnc.stringify(sha256(hexEnc.parse(scriptPubKey))); + return addrScripthash.match(/.{2}/g).reverse().join(''); + } } export default BitcoindElectrsApi; diff --git a/backend/src/api/bitcoin/electrs-api.ts b/backend/src/api/bitcoin/electrs-api.ts index a2dc86ad4..521ff7ab5 100644 --- a/backend/src/api/bitcoin/electrs-api.ts +++ b/backend/src/api/bitcoin/electrs-api.ts @@ -1,6 +1,7 @@ import config from '../../config'; import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; -import { Transaction, Block, MempoolInfo, MempoolEntry, MempoolEntries, Address } from '../../interfaces'; +import { Transaction, Block, MempoolInfo, MempoolEntry, MempoolEntries, Address, + AddressInformation, ScriptHashBalance, ScriptHashHistory } from '../../interfaces'; import axios from 'axios'; class ElectrsApi implements AbstractBitcoinApi { @@ -63,6 +64,19 @@ class ElectrsApi implements AbstractBitcoinApi { $getAddress(address: string): Promise
{ throw new Error('Method not implemented.'); } + + $getScriptHashBalance(scriptHash: string): Promise { + throw new Error('Method not implemented.'); + } + + $getScriptHashHistory(scriptHash: string): Promise { + throw new Error('Method not implemented.'); + } + + $validateAddress(address: string): Promise { + throw new Error('Method not implemented.'); + } + } export default ElectrsApi; diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index a941f8fa2..76213f86b 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -67,17 +67,22 @@ class Blocks { let found = 0; for (let i = 0; i < txIds.length; i++) { + // When using bitcoind, just fetch the coinbase tx for now + if ((config.MEMPOOL.BACKEND === 'bitcoind' || + config.MEMPOOL.BACKEND === 'bitcoind-electrs') && i === 0) { + const tx = await transactionUtils.getTransactionExtended(txIds[i], true); + if (tx) { + transactions.push(tx); + } + } if (mempool[txIds[i]]) { transactions.push(mempool[txIds[i]]); found++; - } else { - // When using bitcoind, just skip parsing past block tx's for now except for coinbase - if (config.MEMPOOL.BACKEND === 'electrs' || i === 0) { // - logger.debug(`Fetching block tx ${i} of ${txIds.length}`); - const tx = await transactionUtils.getTransactionExtended(txIds[i]); - if (tx) { - transactions.push(tx); - } + } else if (config.MEMPOOL.BACKEND === 'electrs') { + logger.debug(`Fetching block tx ${i} of ${txIds.length}`); + const tx = await transactionUtils.getTransactionExtended(txIds[i]); + if (tx) { + transactions.push(tx); } } } diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index c94a903be..fc150d40b 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -34,13 +34,14 @@ class TransactionUtils { return this.extendTransaction(transaction); } - public extendTransaction(transaction: Transaction | MempoolEntry): TransactionExtended { + public extendTransaction(transaction: Transaction): TransactionExtended { + transaction['vsize'] = Math.round(transaction.weight / 4); + transaction['feePerVsize'] = Math.max(1, (transaction.fee || 0) / (transaction.weight / 4)); + if (!transaction.in_active_chain) { + transaction['firstSeen'] = Math.round((new Date().getTime() / 1000)); + } // @ts-ignore - return Object.assign({ - vsize: Math.round(transaction.weight / 4), - feePerVsize: Math.max(1, (transaction.fee || 0) / (transaction.weight / 4)), - firstSeen: Math.round((new Date().getTime() / 1000)), - }, transaction); + return transaction; } public stripCoinbaseTransaction(tx: TransactionExtended): TransactionMinerInfo { @@ -88,7 +89,7 @@ class TransactionUtils { scriptpubkey: vout.scriptPubKey.hex, scriptpubkey_address: vout.scriptPubKey && vout.scriptPubKey.addresses ? vout.scriptPubKey.addresses[0] : null, scriptpubkey_asm: vout.scriptPubKey.asm, - scriptpubkey_type: vout.scriptPubKey.type, + scriptpubkey_type: this.translateScriptPubKeyType(vout.scriptPubKey.type), }; }); if (transaction.confirmations) { @@ -106,6 +107,25 @@ class TransactionUtils { } } + private translateScriptPubKeyType(outputType: string): string { + const map = { + 'pubkey': 'p2pk', + 'pubkeyhash': 'p2pkh', + 'scripthash': 'p2sh', + 'witness_v0_keyhash': 'v0_p2wpkh', + 'witness_v0_scripthash': 'v0_p2wsh', + 'witness_v1_taproot': 'v1_p2tr', + 'nonstandard': 'nonstandard', + 'nulldata': 'nulldata' + }; + + if (map[outputType]) { + return map[outputType]; + } else { + return ''; + } + } + private async $appendFeeData(transaction: Transaction): Promise { let mempoolEntry: MempoolEntry; if (!mempool.isInSync() && !this.mempoolEntriesCache) { diff --git a/backend/src/interfaces.ts b/backend/src/interfaces.ts index 302d43e12..e7a77f670 100644 --- a/backend/src/interfaces.ts +++ b/backend/src/interfaces.ts @@ -30,6 +30,9 @@ export interface Transaction { vin: Vin[]; vout: Vout[]; status: Status; + + // bitcoind (temp?) + in_active_chain?: boolean; } export interface TransactionMinerInfo { @@ -294,3 +297,24 @@ interface RequiredParams { required: boolean; types: ('@string' | '@number' | '@boolean' | string)[]; } + +export interface AddressInformation { + isvalid: boolean; + address: string; + scriptPubKey: string; + isscript: boolean; + iswitness: boolean; + witness_version?: boolean; + witness_program: string; +} + +export interface ScriptHashBalance { + confirmed: number; + unconfirmed: number; +} + +export interface ScriptHashHistory { + height: number; + tx_hash: string; + fee?: number; +} diff --git a/backend/src/routes.ts b/backend/src/routes.ts index bc24c7b6b..0a2c39465 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -1,5 +1,5 @@ import config from './config'; -import { Request, Response } from 'express'; +import { json, Request, Response } from 'express'; import statistics from './api/statistics'; import feeApi from './api/fee-api'; import backendInfo from './api/backend-info'; @@ -568,16 +568,85 @@ class Routes { } public async getAddress(req: Request, res: Response) { + if (config.MEMPOOL.BACKEND === 'bitcoind') { + res.status(405).send('Address lookups cannot be used with bitcoind as backend.'); + return; + } try { - const result = await bitcoinApi.$getAddress(req.params.hash); - res.json(result); + const addressInfo = await bitcoinApi.$validateAddress(req.params.address); + if (!addressInfo || !addressInfo.isvalid) { + res.json({ + 'address': req.params.address, + 'chain_stats': { + 'funded_txo_count': 0, + 'funded_txo_sum': 0, + 'spent_txo_count': 0, + 'spent_txo_sum': 0, + 'tx_count': 0 + }, + 'mempool_stats': { + 'funded_txo_count': 0, + 'funded_txo_sum': 0, + 'spent_txo_count': 0, + 'spent_txo_sum': 0, + 'tx_count': 0 + } + }); + return; + } + + const balance = await bitcoinApi.$getScriptHashBalance(addressInfo.scriptPubKey); + const history = await bitcoinApi.$getScriptHashHistory(addressInfo.scriptPubKey); + + const unconfirmed = history.filter((h) => h.fee).length; + + res.json({ + 'address': addressInfo.address, + 'chain_stats': { + 'funded_txo_count': 0, + 'funded_txo_sum': balance.confirmed ? balance.confirmed : 0, + 'spent_txo_count': 0, + 'spent_txo_sum': balance.confirmed < 0 ? balance.confirmed : 0, + 'tx_count': history.length - unconfirmed, + }, + 'mempool_stats': { + 'funded_txo_count': 0, + 'funded_txo_sum': balance.unconfirmed > 0 ? balance.unconfirmed : 0, + 'spent_txo_count': 0, + 'spent_txo_sum': balance.unconfirmed < 0 ? -balance.unconfirmed : 0, + 'tx_count': unconfirmed, + } + }); + } catch (e) { res.status(500).send(e.message); } } public async getAddressTransactions(req: Request, res: Response) { - res.status(404).send('Not implemented'); + if (config.MEMPOOL.BACKEND === 'bitcoind') { + res.status(405).send('Address lookups cannot be used with bitcoind as backend.'); + return; + } + + try { + const addressInfo = await bitcoinApi.$validateAddress(req.params.address); + if (!addressInfo || !addressInfo.isvalid) { + res.json([]); + } + const history = await bitcoinApi.$getScriptHashHistory(addressInfo.scriptPubKey); + const transactions: TransactionExtended[] = []; + for (const h of history) { + let tx = await transactionUtils.getTransactionExtended(h.tx_hash); + if (tx) { + tx = await transactionUtils.$addPrevoutsToTransaction(tx); + transactions.push(tx); + } + } + res.json(transactions); + } catch (e) { + res.status(500).send(e.message); + } } public async getAdressTxChain(req: Request, res: Response) { From bb28a566225f1289ad03a5228249bcda73c89804 Mon Sep 17 00:00:00 2001 From: softsimon Date: Mon, 28 Dec 2020 04:47:22 +0700 Subject: [PATCH 04/31] Big refactor of multiple backends handling. --- backend/mempool-config.sample.json | 2 +- backend/src/api/bisq/bisq.ts | 4 +- .../bitcoin/bitcoin-api-abstract-factory.ts | 22 +- .../src/api/bitcoin/bitcoin-api-factory.ts | 18 +- .../src/api/bitcoin/bitcoin-api.interface.ts | 116 +++++++ backend/src/api/bitcoin/bitcoin-api.ts | 199 +++++++++++ backend/src/api/bitcoin/bitcoin-base.api.ts | 40 +++ backend/src/api/bitcoin/bitcoind-api.ts | 101 ------ .../src/api/bitcoin/bitcoind-electrs-api.ts | 141 -------- backend/src/api/bitcoin/electrs-api.ts | 82 ----- .../src/api/bitcoin/electrum-api.interface.ts | 12 + backend/src/api/bitcoin/electrum-api.ts | 119 +++++++ .../src/api/bitcoin/esplora-api.interface.ts | 168 +++++++++ backend/src/api/bitcoin/esplora-api.ts | 55 +++ backend/src/api/blocks.ts | 34 +- backend/src/api/common.ts | 2 +- backend/src/api/fee-api.ts | 2 +- backend/src/api/mempool-blocks.ts | 2 +- backend/src/api/mempool.ts | 21 +- backend/src/api/statistics.ts | 2 +- backend/src/api/transaction-utils.ts | 104 +----- backend/src/api/websocket-handler.ts | 6 +- backend/src/config.ts | 12 +- backend/src/index.ts | 4 +- backend/src/interfaces.ts | 320 ------------------ backend/src/mempool.interfaces.ts | 138 ++++++++ backend/src/routes.ts | 89 ++--- 27 files changed, 946 insertions(+), 869 deletions(-) create mode 100644 backend/src/api/bitcoin/bitcoin-api.interface.ts create mode 100644 backend/src/api/bitcoin/bitcoin-api.ts create mode 100644 backend/src/api/bitcoin/bitcoin-base.api.ts delete mode 100644 backend/src/api/bitcoin/bitcoind-api.ts delete mode 100644 backend/src/api/bitcoin/bitcoind-electrs-api.ts delete mode 100644 backend/src/api/bitcoin/electrs-api.ts create mode 100644 backend/src/api/bitcoin/electrum-api.interface.ts create mode 100644 backend/src/api/bitcoin/electrum-api.ts create mode 100644 backend/src/api/bitcoin/esplora-api.interface.ts create mode 100644 backend/src/api/bitcoin/esplora-api.ts delete mode 100644 backend/src/interfaces.ts create mode 100644 backend/src/mempool.interfaces.ts diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 473622ef5..786aaddc8 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -1,7 +1,7 @@ { "MEMPOOL": { "NETWORK": "mainnet", - "BACKEND": "electrs", + "BACKEND": "electrum", "HTTP_PORT": 8999, "SPAWN_CLUSTER_PROCS": 0, "API_URL_PREFIX": "/api/v1/", diff --git a/backend/src/api/bisq/bisq.ts b/backend/src/api/bisq/bisq.ts index 8c4d89ec6..7824d30ff 100644 --- a/backend/src/api/bisq/bisq.ts +++ b/backend/src/api/bisq/bisq.ts @@ -3,7 +3,7 @@ import * as fs from 'fs'; import axios from 'axios'; import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces'; import { Common } from '../common'; -import { Block } from '../../interfaces'; +import { BlockExtended } from '../../mempool.interfaces'; import { StaticPool } from 'node-worker-threads-pool'; import logger from '../../logger'; @@ -42,7 +42,7 @@ class Bisq { this.startSubDirectoryWatcher(); } - handleNewBitcoinBlock(block: Block): void { + handleNewBitcoinBlock(block: BlockExtended): void { if (block.height - 2 > this.latestBlockHeight && this.latestBlockHeight !== 0) { logger.warn(`Bitcoin block height (#${block.height}) has diverged from the latest Bisq block height (#${this.latestBlockHeight}). Restarting watchers...`); this.startTopDirectoryWatcher(); diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index cce6a352b..4dcd4b4d4 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -1,21 +1,13 @@ -import { MempoolInfo, Transaction, Block, MempoolEntries, MempoolEntry, Address, AddressInformation, - ScriptHashBalance, ScriptHashHistory } from '../../interfaces'; +import { IEsploraApi } from './esplora-api.interface'; export interface AbstractBitcoinApi { - $getMempoolInfo(): Promise; - $getRawMempool(): Promise; - $getRawTransaction(txId: string): Promise; + $getRawMempool(): Promise; + $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean): Promise; + $getRawTransactionBitcoind(txId: string, skipConversion?: boolean, addPrevout?: boolean): Promise; $getBlockHeightTip(): Promise; $getTxIdsForBlock(hash: string): Promise; $getBlockHash(height: number): Promise; - $getBlock(hash: string): Promise; - $getMempoolEntry(txid: string): Promise; - $getAddress(address: string): Promise
; - $validateAddress(address: string): Promise; - $getScriptHashBalance(scriptHash: string): Promise; - $getScriptHashHistory(scriptHash: string): Promise; - - // Custom - $getRawMempoolVerbose(): Promise; - $getRawTransactionBitcond(txId: string): Promise; + $getBlock(hash: string): Promise; + $getAddress(address: string): Promise; + $getAddressTransactions(address: string, lastSeenTxId: string): Promise; } diff --git a/backend/src/api/bitcoin/bitcoin-api-factory.ts b/backend/src/api/bitcoin/bitcoin-api-factory.ts index f625e99e1..a16521a0e 100644 --- a/backend/src/api/bitcoin/bitcoin-api-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-factory.ts @@ -1,18 +1,18 @@ import config from '../../config'; import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; -import BitcoindElectrsApi from './bitcoind-electrs-api'; -import BitcoindApi from './bitcoind-api'; -import ElectrsApi from './electrs-api'; +import EsploraApi from './esplora-api'; +import BitcoinApi from './bitcoin-api'; +import ElectrumApi from './electrum-api'; function bitcoinApiFactory(): AbstractBitcoinApi { switch (config.MEMPOOL.BACKEND) { - case 'electrs': - return new ElectrsApi(); - case 'bitcoind-electrs': - return new BitcoindElectrsApi(); - case 'bitcoind': + case 'esplora': + return new EsploraApi(); + case 'electrum': + return new ElectrumApi(); + case 'none': default: - return new BitcoindApi(); + return new BitcoinApi(); } } diff --git a/backend/src/api/bitcoin/bitcoin-api.interface.ts b/backend/src/api/bitcoin/bitcoin-api.interface.ts new file mode 100644 index 000000000..194beebee --- /dev/null +++ b/backend/src/api/bitcoin/bitcoin-api.interface.ts @@ -0,0 +1,116 @@ +export namespace IBitcoinApi { + export interface MempoolInfo { + loaded: boolean; // (boolean) True if the mempool is fully loaded + size: number; // (numeric) Current tx count + bytes: number; // (numeric) Sum of all virtual transaction sizes as defined in BIP 141. + usage: number; // (numeric) Total memory usage for the mempool + maxmempool: number; // (numeric) Maximum memory usage for the mempool + mempoolminfee: number; // (numeric) Minimum fee rate in BTC/kB for tx to be accepted. + minrelaytxfee: number; // (numeric) Current minimum relay fee for transactions + } + + export interface RawMempool { [txId: string]: MempoolEntry; } + + export interface MempoolEntry { + vsize: number; // (numeric) virtual transaction size as defined in BIP 141. + weight: number; // (numeric) transaction weight as defined in BIP 141. + time: number; // (numeric) local time transaction entered pool in seconds since 1 Jan 1970 GMT + height: number; // (numeric) block height when transaction entered pool + descendantcount: number; // (numeric) number of in-mempool descendant transactions (including this one) + descendantsize: number; // (numeric) virtual transaction size of in-mempool descendants (including this one) + ancestorcount: number; // (numeric) number of in-mempool ancestor transactions (including this one) + ancestorsize: number; // (numeric) virtual transaction size of in-mempool ancestors (including this one) + wtxid: string; // (string) hash of serialized transactionumber; including witness data + fees: { + base: number; // (numeric) transaction fee in BTC + modified: number; // (numeric) transaction fee with fee deltas used for mining priority in BTC + ancestor: number; // (numeric) modified fees (see above) of in-mempool ancestors (including this one) in BTC + descendant: number; // (numeric) modified fees (see above) of in-mempool descendants (including this one) in BTC + }; + depends: string[]; // (string) parent transaction id + spentby: string[]; // (array) unconfirmed transactions spending outputs from this transaction + 'bip125-replaceable': boolean; // (boolean) Whether this transaction could be replaced due to BIP125 (replace-by-fee) + } + + export interface Block { + hash: string; // (string) the block hash (same as provided) + confirmations: number; // (numeric) The number of confirmations, or -1 if the block is not on the main chain + size: number; // (numeric) The block size + strippedsize: number; // (numeric) The block size excluding witness data + weight: number; // (numeric) The block weight as defined in BIP 141 + height: number; // (numeric) The block height or index + version: number; // (numeric) The block version + versionHex: string; // (string) The block version formatted in hexadecimal + merkleroot: string; // (string) The merkle root + tx: Transaction[]; + time: number; // (numeric) The block time expressed in UNIX epoch time + mediantime: number; // (numeric) The median block time expressed in UNIX epoch time + nonce: number; // (numeric) The nonce + bits: string; // (string) The bits + difficulty: number; // (numeric) The difficulty + chainwork: string; // (string) Expected number of hashes required to produce the chain up to this block (in hex) + nTx: number; // (numeric) The number of transactions in the block + previousblockhash: string; // (string) The hash of the previous block + nextblockhash: string; // (string) The hash of the next block + } + + export interface Transaction { + in_active_chain: boolean; // (boolean) Whether specified block is in the active chain or not + hex: string; // (string) The serialized, hex-encoded data for 'txid' + txid: string; // (string) The transaction id (same as provided) + hash: string; // (string) The transaction hash (differs from txid for witness transactions) + size: number; // (numeric) The serialized transaction size + vsize: number; // (numeric) The virtual transaction size (differs from size for witness transactions) + weight: number; // (numeric) The transaction's weight (between vsize*4-3 and vsize*4) + version: number; // (numeric) The version + locktime: number; // (numeric) The lock time + vin: Vin[]; + vout: Vout[]; + blockhash: string; // (string) the block hash + confirmations: number; // (numeric) The confirmations + blocktime: number; // (numeric) The block time expressed in UNIX epoch time + time: number; // (numeric) Same as blocktime + } + + interface Vin { + txid?: string; // (string) The transaction id + vout?: number; // (string) + scriptSig?: { // (json object) The script + asm: string; // (string) asm + hex: string; // (string) hex + }; + sequence: number; // (numeric) The script sequence number + txinwitness?: string[]; // (string) hex-encoded witness data + coinbase?: string; + } + + interface Vout { + value: number; // (numeric) The value in BTC + n: number; // (numeric) index + scriptPubKey: { // (json object) + asm: string; // (string) the asm + hex: string; // (string) the hex + reqSigs: number; // (numeric) The required sigs + type: string; // (string) The type, eg 'pubkeyhash' + addresses: string[] // (string) bitcoin address + }; + } + + export interface AddressInformation { + isvalid: boolean; // (boolean) If the address is valid or not. If not, this is the only property returned. + address: string; // (string) The bitcoin address validated + scriptPubKey: string; // (string) The hex-encoded scriptPubKey generated by the address + isscript: boolean; // (boolean) If the key is a script + iswitness: boolean; // (boolean) If the address is a witness + witness_version?: boolean; // (numeric, optional) The version number of the witness program + witness_program: string; // (string, optional) The hex value of the witness program + } + + export interface ChainTips { + height: number; // (numeric) height of the chain tip + hash: string; // (string) block hash of the tip + branchlen: number; // (numeric) zero for main chain, otherwise length of branch connecting the tip to the main chain + status: 'invalid' | 'headers-only' | 'valid-headers' | 'valid-fork' | 'active'; + } + +} diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts new file mode 100644 index 000000000..c5452a533 --- /dev/null +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -0,0 +1,199 @@ +import config from '../../config'; +import * as bitcoin from '@mempool/bitcoin'; +import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; +import { IBitcoinApi } from './bitcoin-api.interface'; +import { IEsploraApi } from './esplora-api.interface'; +import blocks from '../blocks'; +import bitcoinBaseApi from './bitcoin-base.api'; +import mempool from '../mempool'; + +class BitcoinApi implements AbstractBitcoinApi { + private rawMempoolCache: IBitcoinApi.RawMempool | null = null; + private bitcoindClient: any; + + constructor() { + this.bitcoindClient = new bitcoin.Client({ + host: config.BITCOIND.HOST, + port: config.BITCOIND.PORT, + user: config.BITCOIND.USERNAME, + pass: config.BITCOIND.PASSWORD, + timeout: 60000, + }); + } + + $getRawTransactionBitcoind(txId: string, skipConversion = false, addPrevout = false): Promise { + return this.bitcoindClient.getRawTransaction(txId, true) + .then((transaction: IBitcoinApi.Transaction) => { + if (skipConversion) { + return transaction; + } + return this.$convertTransaction(transaction, addPrevout); + }); + } + + $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise { + return this.bitcoindClient.getRawTransaction(txId, true) + .then((transaction: IBitcoinApi.Transaction) => { + if (skipConversion) { + return transaction; + } + return this.$convertTransaction(transaction, addPrevout); + }); + } + + $getBlockHeightTip(): Promise { + return this.bitcoindClient.getChainTips() + .then((result: IBitcoinApi.ChainTips[]) => result[0].height); + } + + $getTxIdsForBlock(hash: string): Promise { + return this.bitcoindClient.getBlock(hash, 1) + .then((rpcBlock: IBitcoinApi.Block) => rpcBlock.tx); + } + + $getBlockHash(height: number): Promise { + return this.bitcoindClient.getBlockHash(height); + } + + $getBlock(hash: string): Promise { + return this.bitcoindClient.getBlock(hash) + .then((block: IBitcoinApi.Block) => this.convertBlock(block)); + } + + $getAddress(address: string): Promise { + throw new Error('Method getAddress not supported by the Bitcoin RPC API.'); + } + + $getAddressTransactions(address: string, lastSeenTxId: string): Promise { + throw new Error('Method getAddressTransactions not supported by the Bitcoin RPC API.'); + } + + $getRawMempool(): Promise { + return this.bitcoindClient.getRawMemPool(); + } + + protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean): Promise { + let esploraTransaction: IEsploraApi.Transaction = { + txid: transaction.txid, + version: transaction.version, + locktime: transaction.locktime, + size: transaction.size, + weight: transaction.weight, + fee: 0, + vin: [], + vout: [], + status: { confirmed: false }, + }; + + esploraTransaction.vout = transaction.vout.map((vout) => { + return { + value: vout.value * 100000000, + scriptpubkey: vout.scriptPubKey.hex, + scriptpubkey_address: vout.scriptPubKey && vout.scriptPubKey.addresses ? vout.scriptPubKey.addresses[0] : '', + scriptpubkey_asm: vout.scriptPubKey.asm, + scriptpubkey_type: this.translateScriptPubKeyType(vout.scriptPubKey.type), + }; + }); + + esploraTransaction.vin = transaction.vin.map((vin) => { + return { + is_coinbase: !!vin.coinbase, + prevout: null, + scriptsig: vin.scriptSig && vin.scriptSig.hex || vin.coinbase || '', + scriptsig_asm: vin.scriptSig && vin.scriptSig.asm || '', + sequence: vin.sequence, + txid: vin.txid || '', + vout: vin.vout || 0, + witness: vin.txinwitness, + }; + }); + + if (transaction.confirmations) { + esploraTransaction.status = { + confirmed: true, + block_height: blocks.getCurrentBlockHeight() - transaction.confirmations + 1, + block_hash: transaction.blockhash, + block_time: transaction.blocktime, + }; + } + + if (transaction.confirmations) { + esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, addPrevout); + } else { + esploraTransaction = await this.$appendMempoolFeeData(esploraTransaction); + } + + return esploraTransaction; + } + + private convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block { + return { + id: block.hash, + height: block.height, + version: block.version, + timestamp: block.time, + bits: parseInt(block.bits, 16), + nonce: block.nonce, + difficulty: block.difficulty, + merkle_root: block.merkleroot, + tx_count: block.nTx, + size: block.size, + weight: block.weight, + previousblockhash: block.previousblockhash, + }; + } + + private translateScriptPubKeyType(outputType: string): string { + const map = { + 'pubkey': 'p2pk', + 'pubkeyhash': 'p2pkh', + 'scripthash': 'p2sh', + 'witness_v0_keyhash': 'v0_p2wpkh', + 'witness_v0_scripthash': 'v0_p2wsh', + 'witness_v1_taproot': 'v1_p2tr', + 'nonstandard': 'nonstandard', + 'nulldata': 'op_return' + }; + + if (map[outputType]) { + return map[outputType]; + } else { + return ''; + } + } + + private async $appendMempoolFeeData(transaction: IEsploraApi.Transaction): Promise { + let mempoolEntry: IBitcoinApi.MempoolEntry; + if (!mempool.isInSync() && !this.rawMempoolCache) { + this.rawMempoolCache = await bitcoinBaseApi.$getRawMempoolVerbose(); + } + if (this.rawMempoolCache && this.rawMempoolCache[transaction.txid]) { + mempoolEntry = this.rawMempoolCache[transaction.txid]; + } else { + mempoolEntry = await bitcoinBaseApi.$getMempoolEntry(transaction.txid); + } + transaction.fee = mempoolEntry.fees.base * 100000000; + return transaction; + } + + private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean): Promise { + if (transaction.vin[0].is_coinbase) { + transaction.fee = 0; + return transaction; + } + let totalIn = 0; + for (const vin of transaction.vin) { + const innerTx = await this.$getRawTransaction(vin.txid, !addPrevout); + if (addPrevout) { + vin.prevout = innerTx.vout[vin.vout]; + } + totalIn += innerTx.vout[vin.vout].value; + } + const totalOut = transaction.vout.reduce((prev, output) => prev + output.value, 0); + transaction.fee = parseFloat((totalIn - totalOut).toFixed(8)); + return transaction; + } + +} + +export default BitcoinApi; diff --git a/backend/src/api/bitcoin/bitcoin-base.api.ts b/backend/src/api/bitcoin/bitcoin-base.api.ts new file mode 100644 index 000000000..22c8daaed --- /dev/null +++ b/backend/src/api/bitcoin/bitcoin-base.api.ts @@ -0,0 +1,40 @@ +import config from '../../config'; +import * as bitcoin from '@mempool/bitcoin'; +import { IBitcoinApi } from './bitcoin-api.interface'; + +class BitcoinBaseApi { + bitcoindClient: any; + + constructor() { + this.bitcoindClient = new bitcoin.Client({ + host: config.BITCOIND.HOST, + port: config.BITCOIND.PORT, + user: config.BITCOIND.USERNAME, + pass: config.BITCOIND.PASSWORD, + timeout: 60000, + }); + } + + $getMempoolInfo(): Promise { + return this.bitcoindClient.getMempoolInfo(); + } + + $getRawTransaction(txId: string): Promise { + return this.bitcoindClient.getRawTransaction(txId, true); + } + + $getMempoolEntry(txid: string): Promise { + return this.bitcoindClient.getMempoolEntry(txid); + } + + $getRawMempoolVerbose(): Promise { + return this.bitcoindClient.getRawMemPool(true); + } + + $validateAddress(address: string): Promise { + return this.bitcoindClient.validateAddress(address); + } + +} + +export default new BitcoinBaseApi(); diff --git a/backend/src/api/bitcoin/bitcoind-api.ts b/backend/src/api/bitcoin/bitcoind-api.ts deleted file mode 100644 index d008e95b2..000000000 --- a/backend/src/api/bitcoin/bitcoind-api.ts +++ /dev/null @@ -1,101 +0,0 @@ -import config from '../../config'; -import { Transaction, Block, MempoolInfo, RpcBlock, MempoolEntries, MempoolEntry, Address, - AddressInformation, ScriptHashBalance, ScriptHashHistory } from '../../interfaces'; -import * as bitcoin from '@mempool/bitcoin'; -import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; - -class BitcoindApi implements AbstractBitcoinApi { - bitcoindClient: any; - - constructor() { - this.bitcoindClient = new bitcoin.Client({ - host: config.BITCOIND.HOST, - port: config.BITCOIND.PORT, - user: config.BITCOIND.USERNAME, - pass: config.BITCOIND.PASSWORD, - timeout: 60000, - }); - } - - $getMempoolInfo(): Promise { - return this.bitcoindClient.getMempoolInfo(); - } - - $getRawMempool(): Promise { - return this.bitcoindClient.getRawMemPool(); - } - - $getRawMempoolVerbose(): Promise { - return this.bitcoindClient.getRawMemPool(true); - } - - $getMempoolEntry(txid: string): Promise { - return this.bitcoindClient.getMempoolEntry(txid); - } - - $getRawTransaction(txId: string): Promise { - return this.bitcoindClient.getRawTransaction(txId, true) - .then((transaction: Transaction) => { - transaction.vout.forEach((vout) => vout.value = vout.value * 100000000); - return transaction; - }); - } - - $getBlockHeightTip(): Promise { - return this.bitcoindClient.getChainTips() - .then((result) => result[0].height); - } - - $getTxIdsForBlock(hash: string): Promise { - return this.bitcoindClient.getBlock(hash, 1) - .then((rpcBlock: RpcBlock) => { - return rpcBlock.tx; - }); - } - - $getBlockHash(height: number): Promise { - return this.bitcoindClient.getBlockHash(height); - } - - $getBlock(hash: string): Promise { - return this.bitcoindClient.getBlock(hash) - .then((rpcBlock: RpcBlock) => { - return { - id: rpcBlock.hash, - height: rpcBlock.height, - version: rpcBlock.version, - timestamp: rpcBlock.time, - bits: rpcBlock.bits, - nonce: rpcBlock.nonce, - difficulty: rpcBlock.difficulty, - merkle_root: rpcBlock.merkleroot, - tx_count: rpcBlock.nTx, - size: rpcBlock.size, - weight: rpcBlock.weight, - previousblockhash: rpcBlock.previousblockhash, - }; - }); - } - - $getRawTransactionBitcond(txId: string): Promise { - throw new Error('Method not implemented.'); - } - - $getAddress(address: string): Promise
{ - throw new Error('Method not implemented.'); - } - - $validateAddress(address: string): Promise { - return this.bitcoindClient.validateAddress(address); - } - - $getScriptHashBalance(scriptHash: string): Promise { - throw new Error('Method not implemented.'); - } - - $getScriptHashHistory(scriptHash: string): Promise { - throw new Error('Method not implemented.'); - } -} - -export default BitcoindApi; diff --git a/backend/src/api/bitcoin/bitcoind-electrs-api.ts b/backend/src/api/bitcoin/bitcoind-electrs-api.ts deleted file mode 100644 index cbbdec8f2..000000000 --- a/backend/src/api/bitcoin/bitcoind-electrs-api.ts +++ /dev/null @@ -1,141 +0,0 @@ -import config from '../../config'; -import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; -import { Transaction, Block, MempoolInfo, RpcBlock, MempoolEntries, MempoolEntry, Address, - AddressInformation, ScriptHashBalance, ScriptHashHistory } from '../../interfaces'; -import * as bitcoin from '@mempool/bitcoin'; -import * as ElectrumClient from '@codewarriorr/electrum-client-js'; -import logger from '../../logger'; -import transactionUtils from '../transaction-utils'; -import * as sha256 from 'crypto-js/sha256'; -import * as hexEnc from 'crypto-js/enc-hex'; -class BitcoindElectrsApi implements AbstractBitcoinApi { - bitcoindClient: any; - electrumClient: any; - - constructor() { - this.bitcoindClient = new bitcoin.Client({ - host: config.BITCOIND.HOST, - port: config.BITCOIND.PORT, - user: config.BITCOIND.USERNAME, - pass: config.BITCOIND.PASSWORD, - timeout: 60000, - }); - - this.electrumClient = new ElectrumClient( - config.ELECTRS.HOST, - config.ELECTRS.PORT, - 'ssl' - ); - - this.electrumClient.connect( - 'electrum-client-js', - '1.4' - ); - } - - $getMempoolInfo(): Promise { - return this.bitcoindClient.getMempoolInfo(); - } - - $getRawMempool(): Promise { - return this.bitcoindClient.getRawMemPool(); - } - - $getRawMempoolVerbose(): Promise { - return this.bitcoindClient.getRawMemPool(true); - } - - $getMempoolEntry(txid: string): Promise { - return this.bitcoindClient.getMempoolEntry(txid); - } - - async $getRawTransaction(txId: string): Promise { - try { - const transaction: Transaction = await this.electrumClient.blockchain_transaction_get(txId, true); - if (!transaction) { - throw new Error(txId + ' not found!'); - } - transactionUtils.bitcoindToElectrsTransaction(transaction); - return transaction; - } catch (e) { - logger.debug('getRawTransaction error: ' + (e.message || e)); - throw new Error(e); - } - } - - $getRawTransactionBitcond(txId: string): Promise { - return this.bitcoindClient.getRawTransaction(txId, true) - .then((transaction: Transaction) => { - transactionUtils.bitcoindToElectrsTransaction(transaction); - return transaction; - }); - } - - $getBlockHeightTip(): Promise { - return this.bitcoindClient.getChainTips() - .then((result) => result[0].height); - } - - $getTxIdsForBlock(hash: string): Promise { - return this.bitcoindClient.getBlock(hash, 1) - .then((rpcBlock: RpcBlock) => { - return rpcBlock.tx; - }); - } - - $getBlockHash(height: number): Promise { - return this.bitcoindClient.getBlockHash(height); - } - - $getBlock(hash: string): Promise { - return this.bitcoindClient.getBlock(hash) - .then((rpcBlock: RpcBlock) => { - return { - id: rpcBlock.hash, - height: rpcBlock.height, - version: rpcBlock.version, - timestamp: rpcBlock.time, - bits: rpcBlock.bits, - nonce: rpcBlock.nonce, - difficulty: rpcBlock.difficulty, - merkle_root: rpcBlock.merkleroot, - tx_count: rpcBlock.nTx, - size: rpcBlock.size, - weight: rpcBlock.weight, - previousblockhash: rpcBlock.previousblockhash, - }; - }); - } - - async $getAddress(address: string): Promise
{ - try { - const addressInfo: Address = await this.electrumClient.blockchain_scripthash_getBalance(address); - if (!address) { - throw new Error('not found'); - } - return addressInfo; - } catch (e) { - logger.debug('getRawTransaction error: ' + (e.message || e)); - throw new Error(e); - } - } - - $validateAddress(address: string): Promise { - return this.bitcoindClient.validateAddress(address); - } - - $getScriptHashBalance(scriptHash: string): Promise { - return this.electrumClient.blockchain_scripthash_getBalance(this.encodeScriptHash(scriptHash)); - } - - $getScriptHashHistory(scriptHash: string): Promise { - return this.electrumClient.blockchain_scripthash_getHistory(this.encodeScriptHash(scriptHash)); - } - - private encodeScriptHash(scriptPubKey: string): string { - const addrScripthash = hexEnc.stringify(sha256(hexEnc.parse(scriptPubKey))); - return addrScripthash.match(/.{2}/g).reverse().join(''); - } -} - -export default BitcoindElectrsApi; diff --git a/backend/src/api/bitcoin/electrs-api.ts b/backend/src/api/bitcoin/electrs-api.ts deleted file mode 100644 index 521ff7ab5..000000000 --- a/backend/src/api/bitcoin/electrs-api.ts +++ /dev/null @@ -1,82 +0,0 @@ -import config from '../../config'; -import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; -import { Transaction, Block, MempoolInfo, MempoolEntry, MempoolEntries, Address, - AddressInformation, ScriptHashBalance, ScriptHashHistory } from '../../interfaces'; -import axios from 'axios'; - -class ElectrsApi implements AbstractBitcoinApi { - - constructor() { - } - - $getMempoolInfo(): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/mempool', { timeout: 10000 }) - .then((response) => { - return { - size: response.data.count, - bytes: response.data.vsize, - }; - }); - } - - $getRawMempool(): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/mempool/txids') - .then((response) => response.data); - } - - $getRawTransaction(txId: string): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/tx/' + txId) - .then((response) => response.data); - } - - $getBlockHeightTip(): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/blocks/tip/height') - .then((response) => response.data); - } - - $getTxIdsForBlock(hash: string): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/block/' + hash + '/txids') - .then((response) => response.data); - } - - $getBlockHash(height: number): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/block-height/' + height) - .then((response) => response.data); - } - - $getBlock(hash: string): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/block/' + hash) - .then((response) => response.data); - } - - $getRawMempoolVerbose(): Promise { - throw new Error('Method not implemented.'); - } - - $getMempoolEntry(): Promise { - throw new Error('Method not implemented.'); - } - - $getRawTransactionBitcond(txId: string): Promise { - throw new Error('Method not implemented.'); - } - - $getAddress(address: string): Promise
{ - throw new Error('Method not implemented.'); - } - - $getScriptHashBalance(scriptHash: string): Promise { - throw new Error('Method not implemented.'); - } - - $getScriptHashHistory(scriptHash: string): Promise { - throw new Error('Method not implemented.'); - } - - $validateAddress(address: string): Promise { - throw new Error('Method not implemented.'); - } - -} - -export default ElectrsApi; diff --git a/backend/src/api/bitcoin/electrum-api.interface.ts b/backend/src/api/bitcoin/electrum-api.interface.ts new file mode 100644 index 000000000..633de3cbc --- /dev/null +++ b/backend/src/api/bitcoin/electrum-api.interface.ts @@ -0,0 +1,12 @@ +export namespace IElectrumApi { + export interface ScriptHashBalance { + confirmed: number; + unconfirmed: number; + } + + export interface ScriptHashHistory { + height: number; + tx_hash: string; + fee?: number; + } +} diff --git a/backend/src/api/bitcoin/electrum-api.ts b/backend/src/api/bitcoin/electrum-api.ts new file mode 100644 index 000000000..697db749e --- /dev/null +++ b/backend/src/api/bitcoin/electrum-api.ts @@ -0,0 +1,119 @@ +import config from '../../config'; +import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; +import * as ElectrumClient from '@codewarriorr/electrum-client-js'; +import { IBitcoinApi } from './bitcoin-api.interface'; +import { IEsploraApi } from './esplora-api.interface'; +import { IElectrumApi } from './electrum-api.interface'; +import * as sha256 from 'crypto-js/sha256'; +import * as hexEnc from 'crypto-js/enc-hex'; +import BitcoinApi from './bitcoin-api'; +import bitcoinBaseApi from './bitcoin-base.api'; +class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { + private electrumClient: any; + + constructor() { + super(); + + this.electrumClient = new ElectrumClient( + config.ELECTRS.HOST, + config.ELECTRS.PORT, + 'ssl' + ); + + this.electrumClient.connect( + 'electrum-client-js', + '1.4' + ); + } + + async $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise { + const transaction: IBitcoinApi.Transaction = await this.electrumClient.blockchain_transaction_get(txId, true); + if (!transaction) { + throw new Error('Unable to get transaction: ' + txId); + } + if (skipConversion) { + // @ts-ignore + return transaction; + } + return this.$convertTransaction(transaction, addPrevout); + } + + async $getAddress(address: string): Promise { + const addressInfo = await bitcoinBaseApi.$validateAddress(address); + if (!addressInfo || !addressInfo.isvalid) { + return ({ + 'address': address, + 'chain_stats': { + 'funded_txo_count': 0, + 'funded_txo_sum': 0, + 'spent_txo_count': 0, + 'spent_txo_sum': 0, + 'tx_count': 0 + }, + 'mempool_stats': { + 'funded_txo_count': 0, + 'funded_txo_sum': 0, + 'spent_txo_count': 0, + 'spent_txo_sum': 0, + 'tx_count': 0 + } + }); + } + + const balance = await this.$getScriptHashBalance(addressInfo.scriptPubKey); + const history = await this.$getScriptHashHistory(addressInfo.scriptPubKey); + + const unconfirmed = history.filter((h) => h.fee).length; + + return { + 'address': addressInfo.address, + 'chain_stats': { + 'funded_txo_count': 0, + 'funded_txo_sum': balance.confirmed ? balance.confirmed : 0, + 'spent_txo_count': 0, + 'spent_txo_sum': balance.confirmed < 0 ? balance.confirmed : 0, + 'tx_count': history.length - unconfirmed, + }, + 'mempool_stats': { + 'funded_txo_count': 0, + 'funded_txo_sum': balance.unconfirmed > 0 ? balance.unconfirmed : 0, + 'spent_txo_count': 0, + 'spent_txo_sum': balance.unconfirmed < 0 ? -balance.unconfirmed : 0, + 'tx_count': unconfirmed, + } + }; + + } + + async $getAddressTransactions(address: string, lastSeenTxId: string): Promise { + const addressInfo = await bitcoinBaseApi.$validateAddress(address); + if (!addressInfo || !addressInfo.isvalid) { + return []; + } + const history = await this.$getScriptHashHistory(addressInfo.scriptPubKey); + const transactions: IEsploraApi.Transaction[] = []; + for (const h of history) { + const tx = await this.$getRawTransaction(h.tx_hash); + if (tx) { + transactions.push(tx); + } + } + return transactions; + } + + private $getScriptHashBalance(scriptHash: string): Promise { + return this.electrumClient.blockchain_scripthash_getBalance(this.encodeScriptHash(scriptHash)); + } + + private $getScriptHashHistory(scriptHash: string): Promise { + return this.electrumClient.blockchain_scripthash_getHistory(this.encodeScriptHash(scriptHash)); + } + + private encodeScriptHash(scriptPubKey: string): string { + const addrScripthash = hexEnc.stringify(sha256(hexEnc.parse(scriptPubKey))); + return addrScripthash.match(/.{2}/g).reverse().join(''); + } + +} + +export default BitcoindElectrsApi; diff --git a/backend/src/api/bitcoin/esplora-api.interface.ts b/backend/src/api/bitcoin/esplora-api.interface.ts new file mode 100644 index 000000000..cfabe40bc --- /dev/null +++ b/backend/src/api/bitcoin/esplora-api.interface.ts @@ -0,0 +1,168 @@ +export namespace IEsploraApi { + export interface Transaction { + txid: string; + version: number; + locktime: number; + size: number; + weight: number; + fee: number; + vin: Vin[]; + vout: Vout[]; + status: Status; + } + + export interface Recent { + txid: string; + fee: number; + vsize: number; + value: number; + } + + export interface Vin { + txid: string; + vout: number; + is_coinbase: boolean; + scriptsig: string; + scriptsig_asm: string; + inner_redeemscript_asm?: string; + inner_witnessscript_asm?: string; + sequence: any; + witness?: string[]; + prevout: Vout | null; + // Elements + is_pegin?: boolean; + issuance?: Issuance; + } + + interface Issuance { + asset_id: string; + is_reissuance: string; + asset_blinding_nonce: string; + asset_entropy: string; + contract_hash: string; + assetamount?: number; + assetamountcommitment?: string; + tokenamount?: number; + tokenamountcommitment?: string; + } + + export interface Vout { + scriptpubkey: string; + scriptpubkey_asm: string; + scriptpubkey_type: string; + scriptpubkey_address: string; + value: number; + // Elements + valuecommitment?: number; + asset?: string; + pegout?: Pegout; + } + + interface Pegout { + genesis_hash: string; + scriptpubkey: string; + scriptpubkey_asm: string; + scriptpubkey_address: string; + } + + export interface Status { + confirmed: boolean; + block_height?: number; + block_hash?: string; + block_time?: number; + } + + export interface Block { + id: string; + height: number; + version: number; + timestamp: number; + bits: number; + nonce: number; + difficulty: number; + merkle_root: string; + tx_count: number; + size: number; + weight: number; + previousblockhash: string; + } + + export interface Address { + address: string; + chain_stats: ChainStats; + mempool_stats: MempoolStats; + } + + export interface ChainStats { + funded_txo_count: number; + funded_txo_sum: number; + spent_txo_count: number; + spent_txo_sum: number; + tx_count: number; + } + + export interface MempoolStats { + funded_txo_count: number; + funded_txo_sum: number; + spent_txo_count: number; + spent_txo_sum: number; + tx_count: number; + } + + export interface Outspend { + spent: boolean; + txid: string; + vin: number; + status: Status; + } + + export interface Asset { + asset_id: string; + issuance_txin: IssuanceTxin; + issuance_prevout: IssuancePrevout; + reissuance_token: string; + contract_hash: string; + status: Status; + chain_stats: AssetStats; + mempool_stats: AssetStats; + } + + export interface AssetExtended extends Asset { + name: string; + ticker: string; + precision: number; + entity: Entity; + version: number; + issuer_pubkey: string; + } + + export interface Entity { + domain: string; + } + + interface IssuanceTxin { + txid: string; + vin: number; + } + + interface IssuancePrevout { + txid: string; + vout: number; + } + + interface AssetStats { + tx_count: number; + issuance_count: number; + issued_amount: number; + burned_amount: number; + has_blinded_issuances: boolean; + reissuance_tokens: number; + burned_reissuance_tokens: number; + peg_in_count: number; + peg_in_amount: number; + peg_out_count: number; + peg_out_amount: number; + burn_count: number; + } + +} diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts new file mode 100644 index 000000000..a7bedbbec --- /dev/null +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -0,0 +1,55 @@ +import config from '../../config'; +import axios from 'axios'; +import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; +import { IEsploraApi } from './esplora-api.interface'; + +class ElectrsApi implements AbstractBitcoinApi { + + constructor() { } + + $getRawMempool(): Promise { + return axios.get(config.ELECTRS.REST_API_URL + '/mempool/txids') + .then((response) => response.data); + } + + $getRawTransaction(txId: string): Promise { + return axios.get(config.ELECTRS.REST_API_URL + '/tx/' + txId) + .then((response) => response.data); + } + + $getBlockHeightTip(): Promise { + return axios.get(config.ELECTRS.REST_API_URL + '/blocks/tip/height') + .then((response) => response.data); + } + + $getTxIdsForBlock(hash: string): Promise { + return axios.get(config.ELECTRS.REST_API_URL + '/block/' + hash + '/txids') + .then((response) => response.data); + } + + $getBlockHash(height: number): Promise { + return axios.get(config.ELECTRS.REST_API_URL + '/block-height/' + height) + .then((response) => response.data); + } + + $getBlock(hash: string): Promise { + return axios.get(config.ELECTRS.REST_API_URL + '/block/' + hash) + .then((response) => response.data); + } + + $getAddress(address: string): Promise { + throw new Error('Method getAddress not implemented.'); + } + + $getAddressTransactions(address: string, txId?: string): Promise { + throw new Error('Method getAddressTransactions not implemented.'); + } + + $getRawTransactionBitcoind(txId: string): Promise { + return axios.get(config.ELECTRS.REST_API_URL + '/tx/' + txId) + .then((response) => response.data); + } + +} + +export default ElectrsApi; diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 76213f86b..80c77b7c5 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -2,29 +2,29 @@ import config from '../config'; import bitcoinApi from './bitcoin/bitcoin-api-factory'; import logger from '../logger'; import memPool from './mempool'; -import { Block, TransactionExtended, TransactionMinerInfo } from '../interfaces'; +import { BlockExtended, TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces'; import { Common } from './common'; import diskCache from './disk-cache'; import transactionUtils from './transaction-utils'; class Blocks { private static KEEP_BLOCK_AMOUNT = 8; - private blocks: Block[] = []; + private blocks: BlockExtended[] = []; private currentBlockHeight = 0; private lastDifficultyAdjustmentTime = 0; - private newBlockCallbacks: ((block: Block, txIds: string[], transactions: TransactionExtended[]) => void)[] = []; + private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = []; constructor() { } - public getBlocks(): Block[] { + public getBlocks(): BlockExtended[] { return this.blocks; } - public setBlocks(blocks: Block[]) { + public setBlocks(blocks: BlockExtended[]) { this.blocks = blocks; } - public setNewBlockCallback(fn: (block: Block, txIds: string[], transactions: TransactionExtended[]) => void) { + public setNewBlockCallback(fn: (block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void) { this.newBlockCallbacks.push(fn); } @@ -68,9 +68,8 @@ class Blocks { for (let i = 0; i < txIds.length; i++) { // When using bitcoind, just fetch the coinbase tx for now - if ((config.MEMPOOL.BACKEND === 'bitcoind' || - config.MEMPOOL.BACKEND === 'bitcoind-electrs') && i === 0) { - const tx = await transactionUtils.getTransactionExtended(txIds[i], true); + if (config.MEMPOOL.BACKEND !== 'none' && i === 0) { + const tx = await transactionUtils.$getTransactionExtended(txIds[i]); if (tx) { transactions.push(tx); } @@ -78,9 +77,9 @@ class Blocks { if (mempool[txIds[i]]) { transactions.push(mempool[txIds[i]]); found++; - } else if (config.MEMPOOL.BACKEND === 'electrs') { + } else if (config.MEMPOOL.BACKEND === 'esplora') { logger.debug(`Fetching block tx ${i} of ${txIds.length}`); - const tx = await transactionUtils.getTransactionExtended(txIds[i]); + const tx = await transactionUtils.$getTransactionExtended(txIds[i]); if (tx) { transactions.push(tx); } @@ -89,23 +88,24 @@ class Blocks { logger.debug(`${found} of ${txIds.length} found in mempool. ${txIds.length - found} not found.`); - block.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); - block.coinbaseTx = this.stripCoinbaseTransaction(transactions[0]); + const blockExtended: BlockExtended = Object.assign({}, block); + blockExtended.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); + blockExtended.coinbaseTx = this.stripCoinbaseTransaction(transactions[0]); transactions.sort((a, b) => b.feePerVsize - a.feePerVsize); - block.medianFee = transactions.length > 1 ? Common.median(transactions.map((tx) => tx.feePerVsize)) : 0; - block.feeRange = transactions.length > 1 ? Common.getFeesInRange(transactions.slice(0, transactions.length - 1), 8) : [0, 0]; + blockExtended.medianFee = transactions.length > 1 ? Common.median(transactions.map((tx) => tx.feePerVsize)) : 0; + blockExtended.feeRange = transactions.length > 1 ? Common.getFeesInRange(transactions.slice(0, transactions.length - 1), 8) : [0, 0]; if (block.height % 2016 === 0) { this.lastDifficultyAdjustmentTime = block.timestamp; } - this.blocks.push(block); + this.blocks.push(blockExtended); if (this.blocks.length > Blocks.KEEP_BLOCK_AMOUNT) { this.blocks = this.blocks.slice(-Blocks.KEEP_BLOCK_AMOUNT); } if (this.newBlockCallbacks.length) { - this.newBlockCallbacks.forEach((cb) => cb(block, txIds, transactions)); + this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions)); } diskCache.$saveCacheToDisk(); } diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index fdd50a11a..0bb90c6df 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -1,4 +1,4 @@ -import { TransactionExtended, TransactionStripped } from '../interfaces'; +import { TransactionExtended, TransactionStripped } from '../mempool.interfaces'; export class Common { static median(numbers: number[]) { diff --git a/backend/src/api/fee-api.ts b/backend/src/api/fee-api.ts index 551501240..8293520bf 100644 --- a/backend/src/api/fee-api.ts +++ b/backend/src/api/fee-api.ts @@ -1,5 +1,5 @@ import config from '../config'; -import { MempoolBlock } from '../interfaces'; +import { MempoolBlock } from '../mempool.interfaces'; import projectedBlocks from './mempool-blocks'; class FeeApi { diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 681c4f26e..d3dbfcb16 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,4 +1,4 @@ -import { MempoolBlock, TransactionExtended, MempoolBlockWithTransactions } from '../interfaces'; +import { MempoolBlock, TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces'; import { Common } from './common'; class MempoolBlocks { diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index cb13fb874..7f066f1a6 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -1,15 +1,18 @@ import config from '../config'; import bitcoinApi from './bitcoin/bitcoin-api-factory'; -import { MempoolInfo, TransactionExtended, Transaction, VbytesPerSecond, MempoolEntry, MempoolEntries } from '../interfaces'; +import { TransactionExtended, VbytesPerSecond } from '../mempool.interfaces'; import logger from '../logger'; import { Common } from './common'; import transactionUtils from './transaction-utils'; +import { IBitcoinApi } from './bitcoin/bitcoin-api.interface'; +import bitcoinBaseApi from './bitcoin/bitcoin-base.api'; class Mempool { private inSync: boolean = false; private mempoolCache: { [txId: string]: TransactionExtended } = {}; - private mempoolInfo: MempoolInfo = { size: 0, bytes: 0 }; - private mempoolChangedCallback: ((newMempool: { [txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[], + private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, + maxmempool: 0, mempoolminfee: 0, minrelaytxfee: 0 }; + private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => void) | undefined; private txPerSecondArray: number[] = []; @@ -49,10 +52,10 @@ class Mempool { } public async $updateMemPoolInfo() { - this.mempoolInfo = await bitcoinApi.$getMempoolInfo(); + this.mempoolInfo = await bitcoinBaseApi.$getMempoolInfo(); } - public getMempoolInfo(): MempoolInfo | undefined { + public getMempoolInfo(): IBitcoinApi.MempoolInfo | undefined { return this.mempoolInfo; } @@ -67,8 +70,9 @@ class Mempool { public getFirstSeenForTransactions(txIds: string[]): number[] { const txTimes: number[] = []; txIds.forEach((txId: string) => { - if (this.mempoolCache[txId]) { - txTimes.push(this.mempoolCache[txId].firstSeen); + const tx = this.mempoolCache[txId]; + if (tx && tx.firstSeen) { + txTimes.push(tx.firstSeen); } else { txTimes.push(0); } @@ -88,7 +92,7 @@ class Mempool { for (const txid of transactions) { if (!this.mempoolCache[txid]) { - const transaction = await transactionUtils.getTransactionExtended(txid, false, true); + const transaction = await transactionUtils.$getTransactionExtended(txid, true); if (transaction) { this.mempoolCache[txid] = transaction; txCount++; @@ -118,6 +122,7 @@ class Mempool { // Prevent mempool from clear on bitcoind restart by delaying the deletion if (this.mempoolProtection === 0 + && config.MEMPOOL.BACKEND === 'esplora' && currentMempoolSize > 20000 && transactions.length / currentMempoolSize <= 0.80 ) { diff --git a/backend/src/api/statistics.ts b/backend/src/api/statistics.ts index ebdd85a9b..2681933ee 100644 --- a/backend/src/api/statistics.ts +++ b/backend/src/api/statistics.ts @@ -2,7 +2,7 @@ import memPool from './mempool'; import { DB } from '../database'; import logger from '../logger'; -import { Statistic, TransactionExtended, OptimizedStatistic } from '../interfaces'; +import { Statistic, TransactionExtended, OptimizedStatistic } from '../mempool.interfaces'; class Statistics { protected intervalTimer: NodeJS.Timer | undefined; diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index fc150d40b..9d0ff990c 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -1,16 +1,15 @@ import bitcoinApi from './bitcoin/bitcoin-api-factory'; -import { MempoolEntries, MempoolEntry, Transaction, TransactionExtended, TransactionMinerInfo } from '../interfaces'; -import config from '../config'; import logger from '../logger'; -import mempool from './mempool'; -import blocks from './blocks'; +import { TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces'; +import { IEsploraApi } from './bitcoin/esplora-api.interface'; class TransactionUtils { - private mempoolEntriesCache: MempoolEntries | null = null; - constructor() { } public async $addPrevoutsToTransaction(transaction: TransactionExtended): Promise { + if (transaction.vin[0].is_coinbase) { + return transaction; + } for (const vin of transaction.vin) { const innerTx = await bitcoinApi.$getRawTransaction(vin.txid); vin.prevout = innerTx.vout[vin.vout]; @@ -18,26 +17,10 @@ class TransactionUtils { return transaction; } - public async $calculateFeeFromInputs(transaction: Transaction): Promise { - if (transaction.vin[0]['coinbase']) { - transaction.fee = 0; - // @ts-ignore - return transaction; - } - let totalIn = 0; - for (const vin of transaction.vin) { - const innerTx = await bitcoinApi.$getRawTransaction(vin.txid); - totalIn += innerTx.vout[vin.vout].value; - } - const totalOut = transaction.vout.reduce((prev, output) => prev + output.value, 0); - transaction.fee = parseFloat((totalIn - totalOut).toFixed(8)); - return this.extendTransaction(transaction); - } - - public extendTransaction(transaction: Transaction): TransactionExtended { + public extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended { transaction['vsize'] = Math.round(transaction.weight / 4); transaction['feePerVsize'] = Math.max(1, (transaction.fee || 0) / (transaction.weight / 4)); - if (!transaction.in_active_chain) { + if (!transaction.status.confirmed) { transaction['firstSeen'] = Math.round((new Date().getTime() / 1000)); } // @ts-ignore @@ -58,20 +41,13 @@ class TransactionUtils { }; } - public async getTransactionExtended(txId: string, isCoinbase = false, inMempool = false): Promise { + public async $getTransactionExtended(txId: string, inMempool = false, addPrevouts = false): Promise { try { - let transaction: Transaction; + let transaction: IEsploraApi.Transaction; if (inMempool) { - transaction = await bitcoinApi.$getRawTransactionBitcond(txId); + transaction = await bitcoinApi.$getRawTransactionBitcoind(txId, false, addPrevouts); } else { - transaction = await bitcoinApi.$getRawTransaction(txId); - } - if (config.MEMPOOL.BACKEND !== 'electrs' && !isCoinbase) { - if (inMempool) { - transaction = await this.$appendFeeData(transaction); - } else { - transaction = await this.$calculateFeeFromInputs(transaction); - } + transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts); } return this.extendTransaction(transaction); } catch (e) { @@ -81,64 +57,6 @@ class TransactionUtils { } } - public bitcoindToElectrsTransaction(transaction: any): void { - try { - transaction.vout = transaction.vout.map((vout) => { - return { - value: vout.value * 100000000, - scriptpubkey: vout.scriptPubKey.hex, - scriptpubkey_address: vout.scriptPubKey && vout.scriptPubKey.addresses ? vout.scriptPubKey.addresses[0] : null, - scriptpubkey_asm: vout.scriptPubKey.asm, - scriptpubkey_type: this.translateScriptPubKeyType(vout.scriptPubKey.type), - }; - }); - if (transaction.confirmations) { - transaction['status'] = { - confirmed: true, - block_height: blocks.getCurrentBlockHeight() - transaction.confirmations, - block_hash: transaction.blockhash, - block_time: transaction.blocktime, - }; - } else { - transaction['status'] = { confirmed: false }; - } - } catch (e) { - console.log('augment failed: ' + (e.message || e)); - } - } - - private translateScriptPubKeyType(outputType: string): string { - const map = { - 'pubkey': 'p2pk', - 'pubkeyhash': 'p2pkh', - 'scripthash': 'p2sh', - 'witness_v0_keyhash': 'v0_p2wpkh', - 'witness_v0_scripthash': 'v0_p2wsh', - 'witness_v1_taproot': 'v1_p2tr', - 'nonstandard': 'nonstandard', - 'nulldata': 'nulldata' - }; - - if (map[outputType]) { - return map[outputType]; - } else { - return ''; - } - } - - private async $appendFeeData(transaction: Transaction): Promise { - let mempoolEntry: MempoolEntry; - if (!mempool.isInSync() && !this.mempoolEntriesCache) { - this.mempoolEntriesCache = await bitcoinApi.$getRawMempoolVerbose(); - } - if (this.mempoolEntriesCache && this.mempoolEntriesCache[transaction.txid]) { - mempoolEntry = this.mempoolEntriesCache[transaction.txid]; - } else { - mempoolEntry = await bitcoinApi.$getMempoolEntry(transaction.txid); - } - transaction.fee = mempoolEntry.fees.base * 100000000; - return transaction; - } } export default new TransactionUtils(); diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 76fb91583..e696cad51 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -1,6 +1,6 @@ import logger from '../logger'; import * as WebSocket from 'ws'; -import { Block, TransactionExtended, WebsocketResponse, MempoolBlock, OptimizedStatistic } from '../interfaces'; +import { BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, OptimizedStatistic } from '../mempool.interfaces'; import blocks from './blocks'; import memPool from './mempool'; import backendInfo from './backend-info'; @@ -117,7 +117,7 @@ class WebsocketHandler { }); } - getInitData(_blocks?: Block[]) { + getInitData(_blocks?: BlockExtended[]) { if (!_blocks) { _blocks = blocks.getBlocks(); } @@ -256,7 +256,7 @@ class WebsocketHandler { }); } - handleNewBlock(block: Block, txIds: string[], transactions: TransactionExtended[]) { + handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) { if (!this.wss) { throw new Error('WebSocket.Server is not set'); } diff --git a/backend/src/config.ts b/backend/src/config.ts index 6e2e3dc9d..b8af02efc 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -3,7 +3,7 @@ const configFile = require('../mempool-config.json'); interface IConfig { MEMPOOL: { NETWORK: 'mainnet' | 'testnet' | 'liquid'; - BACKEND: 'electrs' | 'bitcoind' | 'bitcoind-electrs'; + BACKEND: 'esplora' | 'electrum' | 'none'; HTTP_PORT: number; SPAWN_CLUSTER_PROCS: number; API_URL_PREFIX: string; @@ -20,7 +20,7 @@ interface IConfig { PORT: number; USERNAME: string; PASSWORD: string; - }, + }; DATABASE: { ENABLED: boolean; HOST: string, @@ -53,7 +53,7 @@ interface IConfig { const defaults: IConfig = { 'MEMPOOL': { 'NETWORK': 'mainnet', - 'BACKEND': 'electrs', + 'BACKEND': 'none', 'HTTP_PORT': 8999, 'SPAWN_CLUSTER_PROCS': 0, 'API_URL_PREFIX': '/api/v1/', @@ -66,10 +66,10 @@ const defaults: IConfig = { 'PORT': 3306 }, 'BITCOIND': { - 'HOST': "127.0.0.1", + 'HOST': '127.0.0.1', 'PORT': 8332, - 'USERNAME': "mempoo", - 'PASSWORD': "mempool" + 'USERNAME': 'mempool', + 'PASSWORD': 'mempool' }, 'DATABASE': { 'ENABLED': true, diff --git a/backend/src/index.ts b/backend/src/index.ts index 11878460f..0ffb2d1e4 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -209,7 +209,7 @@ class Server { }); } - if (config.MEMPOOL.BACKEND === 'bitcoind' || config.MEMPOOL.BACKEND === 'bitcoind-electrs') { + if (config.MEMPOOL.BACKEND !== 'esplora') { this.app .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId', routes.getTransaction) .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', routes.getTransactionOutspends) @@ -219,7 +219,7 @@ class Server { .get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', routes.getBlockHeight) .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', routes.getAddress) .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', routes.getAddressTransactions) - .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', routes.getAdressTxChain) + .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', routes.getAddressTransactions) .get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', routes.getAddressPrefix) ; } diff --git a/backend/src/interfaces.ts b/backend/src/interfaces.ts deleted file mode 100644 index e7a77f670..000000000 --- a/backend/src/interfaces.ts +++ /dev/null @@ -1,320 +0,0 @@ -export interface MempoolInfo { - size: number; - bytes: number; - usage?: number; - maxmempool?: number; - mempoolminfee?: number; - minrelaytxfee?: number; -} - -export interface MempoolBlock { - blockSize: number; - blockVSize: number; - nTx: number; - medianFee: number; - totalFees: number; - feeRange: number[]; -} - -export interface MempoolBlockWithTransactions extends MempoolBlock { - transactionIds: string[]; -} - -export interface Transaction { - txid: string; - version: number; - locktime: number; - fee: number; - size: number; - weight: number; - vin: Vin[]; - vout: Vout[]; - status: Status; - - // bitcoind (temp?) - in_active_chain?: boolean; -} - -export interface TransactionMinerInfo { - vin: VinStrippedToScriptsig[]; - vout: VoutStrippedToScriptPubkey[]; -} - -interface VinStrippedToScriptsig { - scriptsig: string; -} - -interface VoutStrippedToScriptPubkey { - scriptpubkey_address: string | undefined; - value: number; -} - -export interface TransactionExtended extends Transaction { - vsize: number; - feePerVsize: number; - firstSeen: number; -} - -export interface TransactionStripped { - txid: string; - fee: number; - weight: number; - value: number; -} - -export interface Vin { - txid: string; - vout: number; - is_coinbase: boolean; - scriptsig: string; - scriptsig_asm: string; - inner_redeemscript_asm?: string; - inner_witnessscript_asm?: string; - sequence: any; - witness?: string[]; - prevout: Vout; - // Elements - is_pegin?: boolean; - issuance?: Issuance; -} - -interface Issuance { - asset_id: string; - is_reissuance: string; - asset_blinding_nonce: string; - asset_entropy: string; - contract_hash: string; - assetamount?: number; - assetamountcommitment?: string; - tokenamount?: number; - tokenamountcommitment?: string; -} - -export interface Vout { - scriptpubkey: string; - scriptpubkey_asm: string; - scriptpubkey_type: string; - scriptpubkey_address: string; - value: number; - // Elements - valuecommitment?: number; - asset?: string; - pegout?: Pegout; -} - -interface Pegout { - genesis_hash: string; - scriptpubkey: string; - scriptpubkey_asm: string; - scriptpubkey_address: string; -} - -export interface Status { - confirmed: boolean; - block_height?: number; - block_hash?: string; - block_time?: number; -} - -export interface Block { - id: string; - height: number; - version: number; - timestamp: number; - bits: number; - nonce: number; - difficulty: number; - merkle_root: string; - tx_count: number; - size: number; - weight: number; - previousblockhash: string; - - // Custom properties - medianFee?: number; - feeRange?: number[]; - reward?: number; - coinbaseTx?: TransactionMinerInfo; - matchRate?: number; -} - -export interface RpcBlock { - hash: string; - confirmations: number; - size: number; - strippedsize: number; - weight: number; - height: number; - version: number; - versionHex: string; - merkleroot: string; - tx: Transaction[]; - time: number; - mediantime: number; - nonce: number; - bits: number; - difficulty: number; - chainwork: string; - nTx: number; - previousblockhash: string; - nextblockhash: string; -} - -export interface MempoolEntries { [txId: string]: MempoolEntry; } - -export interface MempoolEntry { - fees: Fees; - vsize: number; - weight: number; - fee: number; - modifiedfee: number; - time: number; - height: number; - descendantcount: number; - descendantsize: number; - descendantfees: number; - ancestorcount: number; - ancestorsize: number; - ancestorfees: number; - wtxid: string; - depends: any[]; - spentby: any[]; - 'bip125-replaceable': boolean; -} - -export interface Fees { - base: number; - modified: number; - ancestor: number; - descendant: number; -} - -export interface Address { - address: string; - chain_stats: ChainStats; - mempool_stats: MempoolStats; -} - -export interface ChainStats { - funded_txo_count: number; - funded_txo_sum: number; - spent_txo_count: number; - spent_txo_sum: number; - tx_count: number; -} - -export interface MempoolStats { - funded_txo_count: number; - funded_txo_sum: number; - spent_txo_count: number; - spent_txo_sum: number; - tx_count: number; -} - -export interface Statistic { - id?: number; - added: string; - unconfirmed_transactions: number; - tx_per_second: number; - vbytes_per_second: number; - total_fee: number; - mempool_byte_weight: number; - fee_data: string; - - vsize_1: number; - vsize_2: number; - vsize_3: number; - vsize_4: number; - vsize_5: number; - vsize_6: number; - vsize_8: number; - vsize_10: number; - vsize_12: number; - vsize_15: number; - vsize_20: number; - vsize_30: number; - vsize_40: number; - vsize_50: number; - vsize_60: number; - vsize_70: number; - vsize_80: number; - vsize_90: number; - vsize_100: number; - vsize_125: number; - vsize_150: number; - vsize_175: number; - vsize_200: number; - vsize_250: number; - vsize_300: number; - vsize_350: number; - vsize_400: number; - vsize_500: number; - vsize_600: number; - vsize_700: number; - vsize_800: number; - vsize_900: number; - vsize_1000: number; - vsize_1200: number; - vsize_1400: number; - vsize_1600: number; - vsize_1800: number; - vsize_2000: number; -} - -export interface OptimizedStatistic { - id: number; - added: string; - unconfirmed_transactions: number; - tx_per_second: number; - vbytes_per_second: number; - total_fee: number; - mempool_byte_weight: number; - vsizes: number[]; -} - -export interface Outspend { - spent: boolean; - txid: string; - vin: number; - status: Status; -} -export interface WebsocketResponse { - action: string; - data: string[]; - 'track-tx': string; - 'track-address': string; - 'watch-mempool': boolean; -} - -export interface VbytesPerSecond { - unixTime: number; - vSize: number; -} - -export interface RequiredSpec { [name: string]: RequiredParams; } - -interface RequiredParams { - required: boolean; - types: ('@string' | '@number' | '@boolean' | string)[]; -} - -export interface AddressInformation { - isvalid: boolean; - address: string; - scriptPubKey: string; - isscript: boolean; - iswitness: boolean; - witness_version?: boolean; - witness_program: string; -} - -export interface ScriptHashBalance { - confirmed: number; - unconfirmed: number; -} - -export interface ScriptHashHistory { - height: number; - tx_hash: string; - fee?: number; -} diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts new file mode 100644 index 000000000..1058df7f0 --- /dev/null +++ b/backend/src/mempool.interfaces.ts @@ -0,0 +1,138 @@ +import { IEsploraApi } from './api/bitcoin/esplora-api.interface'; + +export interface MempoolBlock { + blockSize: number; + blockVSize: number; + nTx: number; + medianFee: number; + totalFees: number; + feeRange: number[]; +} + +export interface MempoolBlockWithTransactions extends MempoolBlock { + transactionIds: string[]; +} + +interface VinStrippedToScriptsig { + scriptsig: string; +} + +interface VoutStrippedToScriptPubkey { + scriptpubkey_address: string | undefined; + value: number; +} + +export interface TransactionExtended extends IEsploraApi.Transaction { + vsize: number; + feePerVsize: number; + firstSeen?: number; +} + +export interface TransactionStripped { + txid: string; + fee: number; + weight: number; + value: number; +} +export interface BlockExtended extends IEsploraApi.Block { + medianFee?: number; + feeRange?: number[]; + reward?: number; + coinbaseTx?: TransactionMinerInfo; + matchRate?: number; +} + +export interface TransactionMinerInfo { + vin: VinStrippedToScriptsig[]; + vout: VoutStrippedToScriptPubkey[]; +} + +export interface MempoolStats { + funded_txo_count: number; + funded_txo_sum: number; + spent_txo_count: number; + spent_txo_sum: number; + tx_count: number; +} + +export interface Statistic { + id?: number; + added: string; + unconfirmed_transactions: number; + tx_per_second: number; + vbytes_per_second: number; + total_fee: number; + mempool_byte_weight: number; + fee_data: string; + + vsize_1: number; + vsize_2: number; + vsize_3: number; + vsize_4: number; + vsize_5: number; + vsize_6: number; + vsize_8: number; + vsize_10: number; + vsize_12: number; + vsize_15: number; + vsize_20: number; + vsize_30: number; + vsize_40: number; + vsize_50: number; + vsize_60: number; + vsize_70: number; + vsize_80: number; + vsize_90: number; + vsize_100: number; + vsize_125: number; + vsize_150: number; + vsize_175: number; + vsize_200: number; + vsize_250: number; + vsize_300: number; + vsize_350: number; + vsize_400: number; + vsize_500: number; + vsize_600: number; + vsize_700: number; + vsize_800: number; + vsize_900: number; + vsize_1000: number; + vsize_1200: number; + vsize_1400: number; + vsize_1600: number; + vsize_1800: number; + vsize_2000: number; +} + +export interface OptimizedStatistic { + id: number; + added: string; + unconfirmed_transactions: number; + tx_per_second: number; + vbytes_per_second: number; + total_fee: number; + mempool_byte_weight: number; + vsizes: number[]; +} + +export interface WebsocketResponse { + action: string; + data: string[]; + 'track-tx': string; + 'track-address': string; + 'watch-mempool': boolean; +} + +export interface VbytesPerSecond { + unixTime: number; + vSize: number; +} + +export interface RequiredSpec { [name: string]: RequiredParams; } + +interface RequiredParams { + required: boolean; + types: ('@string' | '@number' | '@boolean' | string)[]; +} + diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 0a2c39465..a130e6025 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -1,5 +1,5 @@ import config from './config'; -import { json, Request, Response } from 'express'; +import { Request, Response } from 'express'; import statistics from './api/statistics'; import feeApi from './api/fee-api'; import backendInfo from './api/backend-info'; @@ -8,7 +8,7 @@ import mempool from './api/mempool'; import bisq from './api/bisq/bisq'; import websocketHandler from './api/websocket-handler'; import bisqMarket from './api/bisq/markets-api'; -import { OptimizedStatistic, RequiredSpec, Transaction, TransactionExtended } from './interfaces'; +import { OptimizedStatistic, RequiredSpec, TransactionExtended } from './mempool.interfaces'; import { MarketsApiError } from './api/bisq/interfaces'; import donations from './api/donations'; import logger from './logger'; @@ -533,10 +533,10 @@ class Routes { if (txInMempool) { transaction = txInMempool; } else { - transaction = await transactionUtils.getTransactionExtended(req.params.txId); + transaction = await transactionUtils.$getTransactionExtended(req.params.txId, false, true); } + if (transaction) { - transaction = await transactionUtils.$addPrevoutsToTransaction(transaction); res.json(transaction); } else { res.status(500).send('Error fetching transaction.'); @@ -560,7 +560,20 @@ class Routes { } public async getBlockTransactions(req: Request, res: Response) { - res.status(404).send('Not implemented'); + try { + const txIds = await bitcoinApi.$getTxIdsForBlock(req.params.hash); + const transactions: TransactionExtended[] = []; + + for (let i = 0; i < Math.min(15, txIds.length); i++) { + const transaction = await transactionUtils.$getTransactionExtended(txIds[i], false, true); + if (transaction) { + transactions.push(transaction); + } + } + res.json(transactions); + } catch (e) { + res.status(500).send(e.message); + } } public async getBlockHeight(req: Request, res: Response) { @@ -568,81 +581,27 @@ class Routes { } public async getAddress(req: Request, res: Response) { - if (config.MEMPOOL.BACKEND === 'bitcoind') { + if (config.MEMPOOL.BACKEND === 'none') { res.status(405).send('Address lookups cannot be used with bitcoind as backend.'); return; } + try { - const addressInfo = await bitcoinApi.$validateAddress(req.params.address); - if (!addressInfo || !addressInfo.isvalid) { - res.json({ - 'address': req.params.address, - 'chain_stats': { - 'funded_txo_count': 0, - 'funded_txo_sum': 0, - 'spent_txo_count': 0, - 'spent_txo_sum': 0, - 'tx_count': 0 - }, - 'mempool_stats': { - 'funded_txo_count': 0, - 'funded_txo_sum': 0, - 'spent_txo_count': 0, - 'spent_txo_sum': 0, - 'tx_count': 0 - } - }); - return; - } - - const balance = await bitcoinApi.$getScriptHashBalance(addressInfo.scriptPubKey); - const history = await bitcoinApi.$getScriptHashHistory(addressInfo.scriptPubKey); - - const unconfirmed = history.filter((h) => h.fee).length; - - res.json({ - 'address': addressInfo.address, - 'chain_stats': { - 'funded_txo_count': 0, - 'funded_txo_sum': balance.confirmed ? balance.confirmed : 0, - 'spent_txo_count': 0, - 'spent_txo_sum': balance.confirmed < 0 ? balance.confirmed : 0, - 'tx_count': history.length - unconfirmed, - }, - 'mempool_stats': { - 'funded_txo_count': 0, - 'funded_txo_sum': balance.unconfirmed > 0 ? balance.unconfirmed : 0, - 'spent_txo_count': 0, - 'spent_txo_sum': balance.unconfirmed < 0 ? -balance.unconfirmed : 0, - 'tx_count': unconfirmed, - } - }); - + const addressData = await bitcoinApi.$getAddress(req.params.address); + res.json(addressData); } catch (e) { res.status(500).send(e.message); } } public async getAddressTransactions(req: Request, res: Response) { - if (config.MEMPOOL.BACKEND === 'bitcoind') { + if (config.MEMPOOL.BACKEND === 'none') { res.status(405).send('Address lookups cannot be used with bitcoind as backend.'); return; } try { - const addressInfo = await bitcoinApi.$validateAddress(req.params.address); - if (!addressInfo || !addressInfo.isvalid) { - res.json([]); - } - const history = await bitcoinApi.$getScriptHashHistory(addressInfo.scriptPubKey); - const transactions: TransactionExtended[] = []; - for (const h of history) { - let tx = await transactionUtils.getTransactionExtended(h.tx_hash); - if (tx) { - tx = await transactionUtils.$addPrevoutsToTransaction(tx); - transactions.push(tx); - } - } + const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, req.params.txId); res.json(transactions); } catch (e) { res.status(500).send(e.message); From 3c0fa71a104c44ed9c8e2633d9316d2f390f62a7 Mon Sep 17 00:00:00 2001 From: softsimon Date: Mon, 28 Dec 2020 20:17:32 +0700 Subject: [PATCH 05/31] Updates for general transaction and block fetching. --- backend/src/api/bitcoin/bitcoin-api.ts | 27 +++++++++++++++++- backend/src/api/bitcoin/electrum-api.ts | 7 ++++- backend/src/api/blocks.ts | 37 +++++++++++-------------- backend/src/api/common.ts | 8 ++++++ backend/src/api/transaction-utils.ts | 36 ++++++++---------------- backend/src/routes.ts | 18 ++++++------ 6 files changed, 77 insertions(+), 56 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index c5452a533..3b3c704c9 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -6,6 +6,7 @@ import { IEsploraApi } from './esplora-api.interface'; import blocks from '../blocks'; import bitcoinBaseApi from './bitcoin-base.api'; import mempool from '../mempool'; +import { TransactionExtended } from '../../mempool.interfaces'; class BitcoinApi implements AbstractBitcoinApi { private rawMempoolCache: IBitcoinApi.RawMempool | null = null; @@ -32,6 +33,11 @@ class BitcoinApi implements AbstractBitcoinApi { } $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise { + // If the transaction is in the mempool we also already fetched the fee, just prevouts are missing + const txInMempool = mempool.getMempool()[txId]; + if (txInMempool && addPrevout) { + return this.$addPrevouts(txInMempool); + } return this.bitcoindClient.getRawTransaction(txId, true) .then((transaction: IBitcoinApi.Transaction) => { if (skipConversion) { @@ -55,7 +61,12 @@ class BitcoinApi implements AbstractBitcoinApi { return this.bitcoindClient.getBlockHash(height); } - $getBlock(hash: string): Promise { + async $getBlock(hash: string): Promise { + const foundBlock = blocks.getBlocks().find((block) => block.id === hash); + if (foundBlock) { + return foundBlock; + } + return this.bitcoindClient.getBlock(hash) .then((block: IBitcoinApi.Block) => this.convertBlock(block)); } @@ -163,6 +174,9 @@ class BitcoinApi implements AbstractBitcoinApi { } private async $appendMempoolFeeData(transaction: IEsploraApi.Transaction): Promise { + if (transaction.fee) { + return transaction; + } let mempoolEntry: IBitcoinApi.MempoolEntry; if (!mempool.isInSync() && !this.rawMempoolCache) { this.rawMempoolCache = await bitcoinBaseApi.$getRawMempoolVerbose(); @@ -176,6 +190,17 @@ class BitcoinApi implements AbstractBitcoinApi { return transaction; } + protected async $addPrevouts(transaction: TransactionExtended): Promise { + for (const vin of transaction.vin) { + if (vin.prevout) { + continue; + } + const innerTx = await this.$getRawTransaction(vin.txid, false); + vin.prevout = innerTx.vout[vin.vout]; + } + return transaction; + } + private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean): Promise { if (transaction.vin[0].is_coinbase) { transaction.fee = 0; diff --git a/backend/src/api/bitcoin/electrum-api.ts b/backend/src/api/bitcoin/electrum-api.ts index 697db749e..68c8b1b41 100644 --- a/backend/src/api/bitcoin/electrum-api.ts +++ b/backend/src/api/bitcoin/electrum-api.ts @@ -8,6 +8,7 @@ import * as sha256 from 'crypto-js/sha256'; import * as hexEnc from 'crypto-js/enc-hex'; import BitcoinApi from './bitcoin-api'; import bitcoinBaseApi from './bitcoin-base.api'; +import mempool from '../mempool'; class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { private electrumClient: any; @@ -27,6 +28,10 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { } async $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise { + const txInMempool = mempool.getMempool()[txId]; + if (txInMempool && addPrevout) { + return this.$addPrevouts(txInMempool); + } const transaction: IBitcoinApi.Transaction = await this.electrumClient.blockchain_transaction_get(txId, true); if (!transaction) { throw new Error('Unable to get transaction: ' + txId); @@ -93,7 +98,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { const history = await this.$getScriptHashHistory(addressInfo.scriptPubKey); const transactions: IEsploraApi.Transaction[] = []; for (const h of history) { - const tx = await this.$getRawTransaction(h.tx_hash); + const tx = await this.$getRawTransaction(h.tx_hash, false, true); if (tx) { transactions.push(tx); } diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 80c77b7c5..1c5774640 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -64,19 +64,28 @@ class Blocks { const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); const mempool = memPool.getMempool(); - let found = 0; + let transactionsFound = 0; for (let i = 0; i < txIds.length; i++) { // When using bitcoind, just fetch the coinbase tx for now if (config.MEMPOOL.BACKEND !== 'none' && i === 0) { - const tx = await transactionUtils.$getTransactionExtended(txIds[i]); - if (tx) { - transactions.push(tx); + let txFound = false; + let findCoinbaseTxTries = 0; + // It takes Electrum Server a few seconds to index the transaction after a block is found + while (findCoinbaseTxTries < 5 && !txFound) { + const tx = await transactionUtils.$getTransactionExtended(txIds[i]); + if (tx) { + txFound = true; + transactions.push(tx); + } else { + await Common.sleep(1000); + findCoinbaseTxTries++; + } } } if (mempool[txIds[i]]) { transactions.push(mempool[txIds[i]]); - found++; + transactionsFound++; } else if (config.MEMPOOL.BACKEND === 'esplora') { logger.debug(`Fetching block tx ${i} of ${txIds.length}`); const tx = await transactionUtils.$getTransactionExtended(txIds[i]); @@ -86,11 +95,11 @@ class Blocks { } } - logger.debug(`${found} of ${txIds.length} found in mempool. ${txIds.length - found} not found.`); + logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${txIds.length - transactionsFound} not found.`); const blockExtended: BlockExtended = Object.assign({}, block); blockExtended.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); - blockExtended.coinbaseTx = this.stripCoinbaseTransaction(transactions[0]); + blockExtended.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); transactions.sort((a, b) => b.feePerVsize - a.feePerVsize); blockExtended.medianFee = transactions.length > 1 ? Common.median(transactions.map((tx) => tx.feePerVsize)) : 0; blockExtended.feeRange = transactions.length > 1 ? Common.getFeesInRange(transactions.slice(0, transactions.length - 1), 8) : [0, 0]; @@ -118,20 +127,6 @@ class Blocks { public getCurrentBlockHeight(): number { return this.currentBlockHeight; } - - private stripCoinbaseTransaction(tx: TransactionExtended): TransactionMinerInfo { - return { - vin: [{ - scriptsig: tx.vin[0].scriptsig || tx.vin[0]['coinbase'] - }], - vout: tx.vout - .map((vout) => ({ - scriptpubkey_address: vout.scriptpubkey_address, - value: vout.value - })) - .filter((vout) => vout.value) - }; - } } export default new Blocks(); diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 0bb90c6df..3d6fb7161 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -56,4 +56,12 @@ export class Common { value: tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0), }; } + + static sleep(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, ms); + }); + } } diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index 9d0ff990c..d4ca8124b 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -6,27 +6,6 @@ import { IEsploraApi } from './bitcoin/esplora-api.interface'; class TransactionUtils { constructor() { } - public async $addPrevoutsToTransaction(transaction: TransactionExtended): Promise { - if (transaction.vin[0].is_coinbase) { - return transaction; - } - for (const vin of transaction.vin) { - const innerTx = await bitcoinApi.$getRawTransaction(vin.txid); - vin.prevout = innerTx.vout[vin.vout]; - } - return transaction; - } - - public extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended { - transaction['vsize'] = Math.round(transaction.weight / 4); - transaction['feePerVsize'] = Math.max(1, (transaction.fee || 0) / (transaction.weight / 4)); - if (!transaction.status.confirmed) { - transaction['firstSeen'] = Math.round((new Date().getTime() / 1000)); - } - // @ts-ignore - return transaction; - } - public stripCoinbaseTransaction(tx: TransactionExtended): TransactionMinerInfo { return { vin: [{ @@ -41,10 +20,10 @@ class TransactionUtils { }; } - public async $getTransactionExtended(txId: string, inMempool = false, addPrevouts = false): Promise { + public async $getTransactionExtended(txId: string, forceBitcoind = false, addPrevouts = false): Promise { try { let transaction: IEsploraApi.Transaction; - if (inMempool) { + if (forceBitcoind) { transaction = await bitcoinApi.$getRawTransactionBitcoind(txId, false, addPrevouts); } else { transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts); @@ -52,11 +31,20 @@ class TransactionUtils { return this.extendTransaction(transaction); } catch (e) { logger.debug('getTransactionExtended error: ' + (e.message || e)); - console.log(e); return null; } } + private extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended { + const transactionExtended: TransactionExtended = Object.assign({ + vsize: Math.round(transaction.weight / 4), + feePerVsize: Math.max(1, (transaction.fee || 0) / (transaction.weight / 4)), + }, transaction); + if (!transaction.status.confirmed) { + transactionExtended.firstSeen = Math.round((new Date().getTime() / 1000)); + } + return transactionExtended; + } } export default new TransactionUtils(); diff --git a/backend/src/routes.ts b/backend/src/routes.ts index a130e6025..ef39a596d 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -528,13 +528,7 @@ class Routes { public async getTransaction(req: Request, res: Response) { try { - let transaction: TransactionExtended | null; - const txInMempool = mempool.getMempool()[req.params.txId]; - if (txInMempool) { - transaction = txInMempool; - } else { - transaction = await transactionUtils.$getTransactionExtended(req.params.txId, false, true); - } + const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, false, true); if (transaction) { res.json(transaction); @@ -563,8 +557,9 @@ class Routes { try { const txIds = await bitcoinApi.$getTxIdsForBlock(req.params.hash); const transactions: TransactionExtended[] = []; + const startingIndex = Math.max(0, parseInt(req.params.index, 10)); - for (let i = 0; i < Math.min(15, txIds.length); i++) { + for (let i = startingIndex; i < Math.min(startingIndex + 10, txIds.length); i++) { const transaction = await transactionUtils.$getTransactionExtended(txIds[i], false, true); if (transaction) { transactions.push(transaction); @@ -577,7 +572,12 @@ class Routes { } public async getBlockHeight(req: Request, res: Response) { - res.status(404).send('Not implemented'); + try { + const blockHash = await bitcoinApi.$getBlockHash(parseInt(req.params.height, 10)); + res.send(blockHash); + } catch (e) { + res.status(500).send(e.message); + } } public async getAddress(req: Request, res: Response) { From a3644e23a785909c55d9360a3279929bdd9f2c9e Mon Sep 17 00:00:00 2001 From: softsimon Date: Tue, 29 Dec 2020 00:41:02 +0700 Subject: [PATCH 06/31] Switching Electrum Client lib supporting auto reconnect. --- backend/package.json | 1 + backend/src/api/bitcoin/electrum-api.ts | 40 ++++++++++++++++++------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/backend/package.json b/backend/package.json index 3796c340e..6a409a75a 100644 --- a/backend/package.json +++ b/backend/package.json @@ -29,6 +29,7 @@ "dependencies": { "@codewarriorr/electrum-client-js": "^0.1.1", "@mempool/bitcoin": "^3.0.2", + "@mempool/electrum-client": "^1.1.7", "axios": "^0.21.0", "crypto-js": "^4.0.0", "express": "^4.17.1", diff --git a/backend/src/api/bitcoin/electrum-api.ts b/backend/src/api/bitcoin/electrum-api.ts index 68c8b1b41..164607d30 100644 --- a/backend/src/api/bitcoin/electrum-api.ts +++ b/backend/src/api/bitcoin/electrum-api.ts @@ -1,6 +1,5 @@ import config from '../../config'; import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; -import * as ElectrumClient from '@codewarriorr/electrum-client-js'; import { IBitcoinApi } from './bitcoin-api.interface'; import { IEsploraApi } from './esplora-api.interface'; import { IElectrumApi } from './electrum-api.interface'; @@ -9,22 +8,43 @@ import * as hexEnc from 'crypto-js/enc-hex'; import BitcoinApi from './bitcoin-api'; import bitcoinBaseApi from './bitcoin-base.api'; import mempool from '../mempool'; +import logger from '../../logger'; + +// @ts-ignore +global.net = require('net'); +// @ts-ignore +global.tls = require('tls'); +import * as ElectrumClient from 'electrum-client'; + class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { private electrumClient: any; constructor() { super(); + const electrumConfig = { client: 'mempool-v2', version: '1.4' }; + const electrumPersistencePolicy = { retryPeriod: 10000, maxRetry: 1000, callback: null }; + + const electrumCallbacks = { + onConnect: (client, versionInfo) => { logger.info(`Connected to Electrum Server at ${config.ELECTRS.HOST}:${config.ELECTRS.PORT} (${JSON.stringify(versionInfo)})`); }, + onClose: (client) => { logger.info(`Disconnected from Electrum Server at ${config.ELECTRS.HOST}:${config.ELECTRS.PORT}`); }, + onError: (err) => { logger.err(`Electrum error: ${JSON.stringify(err)}`); }, + onLog: (str) => { logger.debug(str); }, + }; + this.electrumClient = new ElectrumClient( - config.ELECTRS.HOST, config.ELECTRS.PORT, - 'ssl' + config.ELECTRS.HOST, + config.ELECTRS.PORT === 50001 ? 'tcp' : 'tls', + null, + electrumCallbacks ); - this.electrumClient.connect( - 'electrum-client-js', - '1.4' - ); + this.electrumClient.initElectrum(electrumConfig, electrumPersistencePolicy) + .then(() => {}) + .catch((err) => { + logger.err(`Error connecting to Electrum Server at ${config.ELECTRS.HOST}:${config.ELECTRS.PORT}`); + }); } async $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise { @@ -32,7 +52,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { if (txInMempool && addPrevout) { return this.$addPrevouts(txInMempool); } - const transaction: IBitcoinApi.Transaction = await this.electrumClient.blockchain_transaction_get(txId, true); + const transaction: IBitcoinApi.Transaction = await this.electrumClient.blockchainTransaction_get(txId, true); if (!transaction) { throw new Error('Unable to get transaction: ' + txId); } @@ -107,11 +127,11 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { } private $getScriptHashBalance(scriptHash: string): Promise { - return this.electrumClient.blockchain_scripthash_getBalance(this.encodeScriptHash(scriptHash)); + return this.electrumClient.blockchainScripthash_getBalance(this.encodeScriptHash(scriptHash)); } private $getScriptHashHistory(scriptHash: string): Promise { - return this.electrumClient.blockchain_scripthash_getHistory(this.encodeScriptHash(scriptHash)); + return this.electrumClient.blockchainScripthash_getHistory(this.encodeScriptHash(scriptHash)); } private encodeScriptHash(scriptPubKey: string): string { From ae87694bc34267460562ac8a61f39f7fd2fd54d0 Mon Sep 17 00:00:00 2001 From: softsimon Date: Tue, 29 Dec 2020 14:14:34 +0700 Subject: [PATCH 07/31] Use @mempool/electrum-client lib as Electrum Client lib. --- backend/src/api/bitcoin/electrum-api.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/backend/src/api/bitcoin/electrum-api.ts b/backend/src/api/bitcoin/electrum-api.ts index 164607d30..ab805ea70 100644 --- a/backend/src/api/bitcoin/electrum-api.ts +++ b/backend/src/api/bitcoin/electrum-api.ts @@ -3,18 +3,13 @@ import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; import { IBitcoinApi } from './bitcoin-api.interface'; import { IEsploraApi } from './esplora-api.interface'; import { IElectrumApi } from './electrum-api.interface'; -import * as sha256 from 'crypto-js/sha256'; -import * as hexEnc from 'crypto-js/enc-hex'; import BitcoinApi from './bitcoin-api'; import bitcoinBaseApi from './bitcoin-base.api'; import mempool from '../mempool'; import logger from '../../logger'; - -// @ts-ignore -global.net = require('net'); -// @ts-ignore -global.tls = require('tls'); -import * as ElectrumClient from 'electrum-client'; +import * as ElectrumClient from '@mempool/electrum-client'; +import * as sha256 from 'crypto-js/sha256'; +import * as hexEnc from 'crypto-js/enc-hex'; class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { private electrumClient: any; From 62c78f5b0802de6b2b7755ff90f0ab7e69fc9527 Mon Sep 17 00:00:00 2001 From: softsimon Date: Tue, 29 Dec 2020 20:41:16 +0700 Subject: [PATCH 08/31] Adding getBlocks support. --- backend/src/index.ts | 1 + backend/src/routes.ts | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index 0ffb2d1e4..775372580 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -214,6 +214,7 @@ class Server { .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId', routes.getTransaction) .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', routes.getTransactionOutspends) .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', routes.getBlock) + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', routes.getBlocks) .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', routes.getBlocks) .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs/:index', routes.getBlockTransactions) .get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', routes.getBlockHeight) diff --git a/backend/src/routes.ts b/backend/src/routes.ts index ef39a596d..97aa56769 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -10,10 +10,12 @@ import websocketHandler from './api/websocket-handler'; import bisqMarket from './api/bisq/markets-api'; import { OptimizedStatistic, RequiredSpec, TransactionExtended } from './mempool.interfaces'; import { MarketsApiError } from './api/bisq/interfaces'; +import { IEsploraApi } from './api/bitcoin/esplora-api.interface'; import donations from './api/donations'; import logger from './logger'; import bitcoinApi from './api/bitcoin/bitcoin-api-factory'; import transactionUtils from './api/transaction-utils'; +import blocks from './api/blocks'; class Routes { private cache: { [date: string]: OptimizedStatistic[] } = { @@ -550,7 +552,38 @@ class Routes { } public async getBlocks(req: Request, res: Response) { - res.status(404).send('Not implemented'); + try { + const returnBlocks: IEsploraApi.Block[] = []; + const latestBlockHeight = blocks.getCurrentBlockHeight(); + const fromHeight = parseInt(req.params.height, 10) || latestBlockHeight; + const localBlocks = blocks.getBlocks(); + + // See if block hight exist in local cache to skip the hash lookup + const blockByHeight = localBlocks.find((b) => b.height === fromHeight); + let startFromHash: string | null = null; + if (blockByHeight) { + startFromHash = blockByHeight.id; + } else { + startFromHash = await bitcoinApi.$getBlockHash(fromHeight); + } + + let nextHash = startFromHash; + for (let i = 0; i < 10; i++) { + const localBlock = localBlocks.find((b) => b.id === nextHash); + if (localBlock) { + returnBlocks.push(localBlock); + nextHash = localBlock.previousblockhash; + } else { + const block = await bitcoinApi.$getBlock(nextHash); + returnBlocks.push(block); + nextHash = block.previousblockhash; + } + } + + res.json(returnBlocks); + } catch (e) { + res.status(500).send(e.message); + } } public async getBlockTransactions(req: Request, res: Response) { From 5390629e416fe8252f4102cfc9b00913da4eccd7 Mon Sep 17 00:00:00 2001 From: softsimon Date: Wed, 30 Dec 2020 01:47:07 +0700 Subject: [PATCH 09/31] Improved block fetching performance. --- backend/src/api/blocks.ts | 16 ++++++++-------- backend/src/api/websocket-handler.ts | 4 ++-- backend/src/routes.ts | 10 ++++------ 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 1c5774640..a6b181c1e 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -2,13 +2,13 @@ import config from '../config'; import bitcoinApi from './bitcoin/bitcoin-api-factory'; import logger from '../logger'; import memPool from './mempool'; -import { BlockExtended, TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces'; +import { BlockExtended, TransactionExtended } from '../mempool.interfaces'; import { Common } from './common'; import diskCache from './disk-cache'; import transactionUtils from './transaction-utils'; class Blocks { - private static KEEP_BLOCK_AMOUNT = 8; + private static INITIAL_BLOCK_AMOUNT = 8; private blocks: BlockExtended[] = []; private currentBlockHeight = 0; private lastDifficultyAdjustmentTime = 0; @@ -32,14 +32,14 @@ class Blocks { const blockHeightTip = await bitcoinApi.$getBlockHeightTip(); if (this.blocks.length === 0) { - this.currentBlockHeight = blockHeightTip - Blocks.KEEP_BLOCK_AMOUNT; + this.currentBlockHeight = blockHeightTip - Blocks.INITIAL_BLOCK_AMOUNT; } else { this.currentBlockHeight = this.blocks[this.blocks.length - 1].height; } - if (blockHeightTip - this.currentBlockHeight > Blocks.KEEP_BLOCK_AMOUNT * 2) { - logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${Blocks.KEEP_BLOCK_AMOUNT} recent blocks`); - this.currentBlockHeight = blockHeightTip - Blocks.KEEP_BLOCK_AMOUNT; + if (blockHeightTip - this.currentBlockHeight > Blocks.INITIAL_BLOCK_AMOUNT * 2) { + logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${Blocks.INITIAL_BLOCK_AMOUNT} recent blocks`); + this.currentBlockHeight = blockHeightTip - Blocks.INITIAL_BLOCK_AMOUNT; } if (!this.lastDifficultyAdjustmentTime) { @@ -109,8 +109,8 @@ class Blocks { } this.blocks.push(blockExtended); - if (this.blocks.length > Blocks.KEEP_BLOCK_AMOUNT) { - this.blocks = this.blocks.slice(-Blocks.KEEP_BLOCK_AMOUNT); + if (this.blocks.length > Blocks.INITIAL_BLOCK_AMOUNT * 4) { + this.blocks = this.blocks.slice(-Blocks.INITIAL_BLOCK_AMOUNT * 4); } if (this.newBlockCallbacks.length) { diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index e696cad51..898cca125 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -77,7 +77,7 @@ class WebsocketHandler { } if (parsedMessage.action === 'init') { - const _blocks = blocks.getBlocks(); + const _blocks = blocks.getBlocks().slice(-8); if (!_blocks) { return; } @@ -119,7 +119,7 @@ class WebsocketHandler { getInitData(_blocks?: BlockExtended[]) { if (!_blocks) { - _blocks = blocks.getBlocks(); + _blocks = blocks.getBlocks().slice(-8); } return { 'mempoolInfo': memPool.getMempoolInfo(), diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 97aa56769..8dfeffa04 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -554,12 +554,10 @@ class Routes { public async getBlocks(req: Request, res: Response) { try { const returnBlocks: IEsploraApi.Block[] = []; - const latestBlockHeight = blocks.getCurrentBlockHeight(); - const fromHeight = parseInt(req.params.height, 10) || latestBlockHeight; - const localBlocks = blocks.getBlocks(); + const fromHeight = parseInt(req.params.height, 10) || blocks.getCurrentBlockHeight(); - // See if block hight exist in local cache to skip the hash lookup - const blockByHeight = localBlocks.find((b) => b.height === fromHeight); + // Check if block height exist in local cache to skip the hash lookup + const blockByHeight = blocks.getBlocks().find((b) => b.height === fromHeight); let startFromHash: string | null = null; if (blockByHeight) { startFromHash = blockByHeight.id; @@ -569,7 +567,7 @@ class Routes { let nextHash = startFromHash; for (let i = 0; i < 10; i++) { - const localBlock = localBlocks.find((b) => b.id === nextHash); + const localBlock = blocks.getBlocks().find((b) => b.id === nextHash); if (localBlock) { returnBlocks.push(localBlock); nextHash = localBlock.previousblockhash; From 89b4de2484f9cb99d3e465b0d817cbc237c4dbaa Mon Sep 17 00:00:00 2001 From: softsimon Date: Wed, 30 Dec 2020 02:27:34 +0700 Subject: [PATCH 10/31] Fixed addressTransactions with pagination in electrum API. --- backend/src/api/bitcoin/electrum-api.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/backend/src/api/bitcoin/electrum-api.ts b/backend/src/api/bitcoin/electrum-api.ts index ab805ea70..ee7fe2276 100644 --- a/backend/src/api/bitcoin/electrum-api.ts +++ b/backend/src/api/bitcoin/electrum-api.ts @@ -110,14 +110,25 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { if (!addressInfo || !addressInfo.isvalid) { return []; } - const history = await this.$getScriptHashHistory(addressInfo.scriptPubKey); const transactions: IEsploraApi.Transaction[] = []; - for (const h of history) { - const tx = await this.$getRawTransaction(h.tx_hash, false, true); + const history = await this.$getScriptHashHistory(addressInfo.scriptPubKey); + history.reverse(); + + let startingIndex = 0; + if (lastSeenTxId) { + const pos = history.findIndex((historicalTx) => historicalTx.tx_hash === lastSeenTxId); + if (pos) { + startingIndex = pos + 1; + } + } + + for (let i = startingIndex; i < Math.min(startingIndex + 10, history.length); i++) { + const tx = await this.$getRawTransaction(history[i].tx_hash, false, true); if (tx) { transactions.push(tx); } } + return transactions; } From a25125091dcb3f75ce2ba1819187049dc362b59d Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 2 Jan 2021 04:40:10 +0700 Subject: [PATCH 11/31] Hack to make it possible to load the Coinbase transaction from Bitcoin Core. --- backend/src/api/bitcoin/bitcoin-api.ts | 17 ++++++++++++++++- backend/src/api/blocks.ts | 2 +- backend/src/api/transaction-utils.ts | 1 + backend/src/index.ts | 1 + 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 3b3c704c9..0e7bf22c4 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -33,11 +33,17 @@ class BitcoinApi implements AbstractBitcoinApi { } $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise { - // If the transaction is in the mempool we also already fetched the fee, just prevouts are missing + // If the transaction is in the mempool we already converted and fetched the fee. Only prevouts are missing const txInMempool = mempool.getMempool()[txId]; if (txInMempool && addPrevout) { return this.$addPrevouts(txInMempool); } + + // Special case to fetch the Coinbase transaction + if (txId === '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b') { + return this.$returnCoinbaseTransaction(); + } + return this.bitcoindClient.getRawTransaction(txId, true) .then((transaction: IBitcoinApi.Transaction) => { if (skipConversion) { @@ -201,6 +207,15 @@ class BitcoinApi implements AbstractBitcoinApi { return transaction; } + protected $returnCoinbaseTransaction(): Promise { + return this.bitcoindClient.getBlock('000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', 2) + .then((block: IBitcoinApi.Block) => { + return this.$convertTransaction(Object.assign(block.tx[0], { + confirmations: blocks.getCurrentBlockHeight() + 1, + blocktime: 1231006505 }), false); + }); + } + private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean): Promise { if (transaction.vin[0].is_coinbase) { transaction.fee = 0; diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index a6b181c1e..4258b5260 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -68,7 +68,7 @@ class Blocks { for (let i = 0; i < txIds.length; i++) { // When using bitcoind, just fetch the coinbase tx for now - if (config.MEMPOOL.BACKEND !== 'none' && i === 0) { + if (config.MEMPOOL.BACKEND !== 'esplora' && i === 0) { let txFound = false; let findCoinbaseTxTries = 0; // It takes Electrum Server a few seconds to index the transaction after a block is found diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index d4ca8124b..1b7fda068 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -31,6 +31,7 @@ class TransactionUtils { return this.extendTransaction(transaction); } catch (e) { logger.debug('getTransactionExtended error: ' + (e.message || e)); + logger.debug(JSON.stringify(e)); return null; } } diff --git a/backend/src/index.ts b/backend/src/index.ts index 775372580..4c753fea7 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -119,6 +119,7 @@ class Server { } else { logger.debug(loggerMsg); } + logger.debug(JSON.stringify(e)); setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.retryOnElectrsErrorAfterSeconds); this.retryOnElectrsErrorAfterSeconds *= 2; this.retryOnElectrsErrorAfterSeconds = Math.min(this.retryOnElectrsErrorAfterSeconds, 60); From 7729ad8b791513312d0c61d0e830340a8ad462aa Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 3 Jan 2021 22:02:10 +0700 Subject: [PATCH 12/31] Convert sighash to match esplora. --- backend/src/api/bitcoin/bitcoin-api.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 0e7bf22c4..04e5be511 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -107,7 +107,7 @@ class BitcoinApi implements AbstractBitcoinApi { value: vout.value * 100000000, scriptpubkey: vout.scriptPubKey.hex, scriptpubkey_address: vout.scriptPubKey && vout.scriptPubKey.addresses ? vout.scriptPubKey.addresses[0] : '', - scriptpubkey_asm: vout.scriptPubKey.asm, + scriptpubkey_asm: vout.scriptPubKey.asm ? this.convertScriptSigAsm(vout.scriptPubKey.asm) : '', scriptpubkey_type: this.translateScriptPubKeyType(vout.scriptPubKey.type), }; }); @@ -117,7 +117,7 @@ class BitcoinApi implements AbstractBitcoinApi { is_coinbase: !!vin.coinbase, prevout: null, scriptsig: vin.scriptSig && vin.scriptSig.hex || vin.coinbase || '', - scriptsig_asm: vin.scriptSig && vin.scriptSig.asm || '', + scriptsig_asm: vin.scriptSig && this.convertScriptSigAsm(vin.scriptSig.asm) || '', sequence: vin.sequence, txid: vin.txid || '', vout: vin.vout || 0, @@ -216,6 +216,20 @@ class BitcoinApi implements AbstractBitcoinApi { }); } + private convertScriptSigAsm(str: string): string { + const a = str.split(' '); + const b: string[] = []; + a.forEach((chunk) => { + if (chunk.substr(0, 3) === 'OP_') { + b.push(chunk); + } else { + chunk = chunk.replace('[ALL]', '01'); + b.push('OP_PUSHBYTES_' + Math.round(chunk.length / 2) + ' ' + chunk); + } + }); + return b.join(' '); + } + private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean): Promise { if (transaction.vin[0].is_coinbase) { transaction.fee = 0; From 1a6c2e79e62033a2cacf60f909ecb8e4d614f08c Mon Sep 17 00:00:00 2001 From: softsimon Date: Tue, 5 Jan 2021 03:06:57 +0700 Subject: [PATCH 13/31] Electrum protocol and tx lookup setting. --- backend/mempool-config.sample.json | 8 ++++++-- backend/src/api/bitcoin/electrum-api.ts | 15 +++++++++------ backend/src/config.ts | 14 ++++++++++++-- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 786aaddc8..694bb180b 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -9,9 +9,13 @@ }, "ELECTRS": { "REST_API_URL": "http://127.0.0.1:3000", - "POLL_RATE_MS": 2000, + "POLL_RATE_MS": 2000 + }, + "ELECTRUM": { "HOST": "127.0.0.1", - "PORT": 50002 + "PORT": 50002, + "PROTOCOL": "tcl", + "TX_LOOKUPS": false }, "BITCOIND": { "HOST": "127.0.0.1", diff --git a/backend/src/api/bitcoin/electrum-api.ts b/backend/src/api/bitcoin/electrum-api.ts index ee7fe2276..f2d498b00 100644 --- a/backend/src/api/bitcoin/electrum-api.ts +++ b/backend/src/api/bitcoin/electrum-api.ts @@ -21,16 +21,16 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { const electrumPersistencePolicy = { retryPeriod: 10000, maxRetry: 1000, callback: null }; const electrumCallbacks = { - onConnect: (client, versionInfo) => { logger.info(`Connected to Electrum Server at ${config.ELECTRS.HOST}:${config.ELECTRS.PORT} (${JSON.stringify(versionInfo)})`); }, - onClose: (client) => { logger.info(`Disconnected from Electrum Server at ${config.ELECTRS.HOST}:${config.ELECTRS.PORT}`); }, + onConnect: (client, versionInfo) => { logger.info(`Connected to Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT} (${JSON.stringify(versionInfo)})`); }, + onClose: (client) => { logger.info(`Disconnected from Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT}`); }, onError: (err) => { logger.err(`Electrum error: ${JSON.stringify(err)}`); }, onLog: (str) => { logger.debug(str); }, }; this.electrumClient = new ElectrumClient( - config.ELECTRS.PORT, - config.ELECTRS.HOST, - config.ELECTRS.PORT === 50001 ? 'tcp' : 'tls', + config.ELECTRUM.PORT, + config.ELECTRUM.HOST, + config.ELECTRUM.PROTOCOL, null, electrumCallbacks ); @@ -38,11 +38,14 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { this.electrumClient.initElectrum(electrumConfig, electrumPersistencePolicy) .then(() => {}) .catch((err) => { - logger.err(`Error connecting to Electrum Server at ${config.ELECTRS.HOST}:${config.ELECTRS.PORT}`); + logger.err(`Error connecting to Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT}`); }); } async $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise { + if (!config.ELECTRUM.TX_LOOKUPS) { + return super.$getRawTransaction(txId, skipConversion, addPrevout); + } const txInMempool = mempool.getMempool()[txId]; if (txInMempool && addPrevout) { return this.$addPrevouts(txInMempool); diff --git a/backend/src/config.ts b/backend/src/config.ts index b8af02efc..20132fdd8 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -12,8 +12,12 @@ interface IConfig { ELECTRS: { REST_API_URL: string; POLL_RATE_MS: number; + }; + ELECTRUM: { HOST: string; PORT: number; + PROTOCOL: 'tls' | 'tcp'; + TX_LOOKUPS: boolean; }; BITCOIND: { HOST: string; @@ -61,9 +65,13 @@ const defaults: IConfig = { }, 'ELECTRS': { 'REST_API_URL': 'http://127.0.0.1:3000', - 'POLL_RATE_MS': 2000, + 'POLL_RATE_MS': 2000 + }, + 'ELECTRUM': { 'HOST': '127.0.0.1', - 'PORT': 3306 + 'PORT': 3306, + 'PROTOCOL': 'tls', + 'TX_LOOKUPS': false }, 'BITCOIND': { 'HOST': '127.0.0.1', @@ -103,6 +111,7 @@ const defaults: IConfig = { class Config implements IConfig { MEMPOOL: IConfig['MEMPOOL']; ELECTRS: IConfig['ELECTRS']; + ELECTRUM: IConfig['ELECTRUM']; BITCOIND: IConfig['BITCOIND']; DATABASE: IConfig['DATABASE']; STATISTICS: IConfig['STATISTICS']; @@ -114,6 +123,7 @@ class Config implements IConfig { const configs = this.merge(configFile, defaults); this.MEMPOOL = configs.MEMPOOL; this.ELECTRS = configs.ELECTRS; + this.ELECTRUM = configs.ELECTRUM; this.BITCOIND = configs.BITCOIND; this.DATABASE = configs.DATABASE; this.STATISTICS = configs.STATISTICS; From f46728080df8ad98b87155a195c1a0a55d081bc2 Mon Sep 17 00:00:00 2001 From: softsimon Date: Tue, 5 Jan 2021 17:30:53 +0700 Subject: [PATCH 14/31] Inform client when Electrum server limit exceeded --- backend/src/api/bitcoin/electrum-api.ts | 84 +++++++++++-------- backend/src/routes.ts | 20 +++-- .../components/address/address.component.html | 8 ++ 3 files changed, 70 insertions(+), 42 deletions(-) diff --git a/backend/src/api/bitcoin/electrum-api.ts b/backend/src/api/bitcoin/electrum-api.ts index f2d498b00..2929c2da4 100644 --- a/backend/src/api/bitcoin/electrum-api.ts +++ b/backend/src/api/bitcoin/electrum-api.ts @@ -83,29 +83,35 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { }); } - const balance = await this.$getScriptHashBalance(addressInfo.scriptPubKey); - const history = await this.$getScriptHashHistory(addressInfo.scriptPubKey); + try { + const balance = await this.$getScriptHashBalance(addressInfo.scriptPubKey); + const history = await this.$getScriptHashHistory(addressInfo.scriptPubKey); - const unconfirmed = history.filter((h) => h.fee).length; + const unconfirmed = history.filter((h) => h.fee).length; - return { - 'address': addressInfo.address, - 'chain_stats': { - 'funded_txo_count': 0, - 'funded_txo_sum': balance.confirmed ? balance.confirmed : 0, - 'spent_txo_count': 0, - 'spent_txo_sum': balance.confirmed < 0 ? balance.confirmed : 0, - 'tx_count': history.length - unconfirmed, - }, - 'mempool_stats': { - 'funded_txo_count': 0, - 'funded_txo_sum': balance.unconfirmed > 0 ? balance.unconfirmed : 0, - 'spent_txo_count': 0, - 'spent_txo_sum': balance.unconfirmed < 0 ? -balance.unconfirmed : 0, - 'tx_count': unconfirmed, + return { + 'address': addressInfo.address, + 'chain_stats': { + 'funded_txo_count': 0, + 'funded_txo_sum': balance.confirmed ? balance.confirmed : 0, + 'spent_txo_count': 0, + 'spent_txo_sum': balance.confirmed < 0 ? balance.confirmed : 0, + 'tx_count': history.length - unconfirmed, + }, + 'mempool_stats': { + 'funded_txo_count': 0, + 'funded_txo_sum': balance.unconfirmed > 0 ? balance.unconfirmed : 0, + 'spent_txo_count': 0, + 'spent_txo_sum': balance.unconfirmed < 0 ? -balance.unconfirmed : 0, + 'tx_count': unconfirmed, + } + }; + } catch (e) { + if (e === 'failed to get confirmed status') { + e = 'The number of transactions on this address exceeds the Electrum server limit'; } - }; - + throw new Error(e); + } } async $getAddressTransactions(address: string, lastSeenTxId: string): Promise { @@ -113,26 +119,34 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { if (!addressInfo || !addressInfo.isvalid) { return []; } - const transactions: IEsploraApi.Transaction[] = []; - const history = await this.$getScriptHashHistory(addressInfo.scriptPubKey); - history.reverse(); - let startingIndex = 0; - if (lastSeenTxId) { - const pos = history.findIndex((historicalTx) => historicalTx.tx_hash === lastSeenTxId); - if (pos) { - startingIndex = pos + 1; + try { + const transactions: IEsploraApi.Transaction[] = []; + const history = await this.$getScriptHashHistory(addressInfo.scriptPubKey); + history.reverse(); + + let startingIndex = 0; + if (lastSeenTxId) { + const pos = history.findIndex((historicalTx) => historicalTx.tx_hash === lastSeenTxId); + if (pos) { + startingIndex = pos + 1; + } } - } - for (let i = startingIndex; i < Math.min(startingIndex + 10, history.length); i++) { - const tx = await this.$getRawTransaction(history[i].tx_hash, false, true); - if (tx) { - transactions.push(tx); + for (let i = startingIndex; i < Math.min(startingIndex + 10, history.length); i++) { + const tx = await this.$getRawTransaction(history[i].tx_hash, false, true); + if (tx) { + transactions.push(tx); + } } - } - return transactions; + return transactions; + } catch (e) { + if (e === 'failed to get confirmed status') { + e = 'The number of transactions on this address exceeds the Electrum server limit'; + } + throw new Error(e); + } } private $getScriptHashBalance(scriptHash: string): Promise { diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 8dfeffa04..d770aea8d 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -538,7 +538,7 @@ class Routes { res.status(500).send('Error fetching transaction.'); } } catch (e) { - res.status(500).send(e.message); + res.status(500).send(e.message || e); } } @@ -547,7 +547,7 @@ class Routes { const result = await bitcoinApi.$getBlock(req.params.hash); res.json(result); } catch (e) { - res.status(500).send(e.message); + res.status(500).send(e.message || e); } } @@ -580,7 +580,7 @@ class Routes { res.json(returnBlocks); } catch (e) { - res.status(500).send(e.message); + res.status(500).send(e.message || e); } } @@ -598,7 +598,7 @@ class Routes { } res.json(transactions); } catch (e) { - res.status(500).send(e.message); + res.status(500).send(e.message || e); } } @@ -607,7 +607,7 @@ class Routes { const blockHash = await bitcoinApi.$getBlockHash(parseInt(req.params.height, 10)); res.send(blockHash); } catch (e) { - res.status(500).send(e.message); + res.status(500).send(e.message || e); } } @@ -621,7 +621,10 @@ class Routes { const addressData = await bitcoinApi.$getAddress(req.params.address); res.json(addressData); } catch (e) { - res.status(500).send(e.message); + if (e.message && e.message.indexOf('exceeds') > 0) { + return res.status(413).send(e.message); + } + res.status(500).send(e.message || e); } } @@ -635,7 +638,10 @@ class Routes { const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, req.params.txId); res.json(transactions); } catch (e) { - res.status(500).send(e.message); + if (e.message && e.message.indexOf('exceeds') > 0) { + return res.status(413).send(e.message); + } + res.status(500).send(e.message || e); } } diff --git a/frontend/src/app/components/address/address.component.html b/frontend/src/app/components/address/address.component.html index 41728c083..e5a0ffd0a 100644 --- a/frontend/src/app/components/address/address.component.html +++ b/frontend/src/app/components/address/address.component.html @@ -105,6 +105,14 @@ Error loading address data.
{{ error.error }} + +

+ Consider view this address on the official Mempool website instead: +
+ https://mempool.space/address/{{ addressString }} +
+ http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/address/{{ addressString }} + From 632c243b34f4c7fc0dbacebd7d6c1ff681039351 Mon Sep 17 00:00:00 2001 From: softsimon Date: Tue, 5 Jan 2021 18:57:06 +0700 Subject: [PATCH 15/31] Loading indicator service indicating mempool sync status. --- backend/src/api/loading-indicators.ts | 32 +++++++++++++++++++ backend/src/api/mempool.ts | 6 ++++ backend/src/api/websocket-handler.ts | 17 +++++++++- backend/src/index.ts | 2 ++ backend/src/mempool.interfaces.ts | 1 + .../app/dashboard/dashboard.component.html | 4 +-- .../src/app/dashboard/dashboard.component.ts | 4 +++ .../src/app/interfaces/websocket.interface.ts | 2 ++ frontend/src/app/services/state.service.ts | 3 ++ .../src/app/services/websocket.service.ts | 4 +++ 10 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 backend/src/api/loading-indicators.ts diff --git a/backend/src/api/loading-indicators.ts b/backend/src/api/loading-indicators.ts new file mode 100644 index 000000000..c2d682d1c --- /dev/null +++ b/backend/src/api/loading-indicators.ts @@ -0,0 +1,32 @@ +import { ILoadingIndicators } from '../mempool.interfaces'; + +class LoadingIndicators { + private loadingIndicators: ILoadingIndicators = { + 'mempool': 0, + }; + private progressChangedCallback: ((loadingIndicators: ILoadingIndicators) => void) | undefined; + + constructor() { } + + public setProgressChangedCallback(fn: (loadingIndicators: ILoadingIndicators) => void) { + this.progressChangedCallback = fn; + } + + public setProgress(name: string, progressPercent: number) { + const newProgress = Math.round(progressPercent); + if (newProgress >= 100) { + delete this.loadingIndicators[name]; + } else { + this.loadingIndicators[name] = newProgress; + } + if (this.progressChangedCallback) { + this.progressChangedCallback(this.loadingIndicators); + } + } + + public getLoadingIndicators() { + return this.loadingIndicators; + } +} + +export default new LoadingIndicators(); diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 7f066f1a6..27cd94794 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -6,6 +6,7 @@ import { Common } from './common'; import transactionUtils from './transaction-utils'; import { IBitcoinApi } from './bitcoin/bitcoin-api.interface'; import bitcoinBaseApi from './bitcoin/bitcoin-base.api'; +import loadingIndicators from './loading-indicators'; class Mempool { private inSync: boolean = false; @@ -90,6 +91,10 @@ class Mempool { const diff = transactions.length - currentMempoolSize; const newTransactions: TransactionExtended[] = []; + if (!this.inSync) { + loadingIndicators.setProgress('mempool', Object.keys(this.mempoolCache).length / transactions.length * 100); + } + for (const txid of transactions) { if (!this.mempoolCache[txid]) { const transaction = await transactionUtils.$getTransactionExtended(txid, true); @@ -162,6 +167,7 @@ class Mempool { if (!this.inSync && transactions.length === Object.keys(newMempool).length) { this.inSync = true; logger.info('The mempool is now in sync!'); + loadingIndicators.setProgress('mempool', 100); } if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) { diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 898cca125..2d4e75d6f 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -1,12 +1,13 @@ import logger from '../logger'; import * as WebSocket from 'ws'; -import { BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, OptimizedStatistic } from '../mempool.interfaces'; +import { BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, OptimizedStatistic, ILoadingIndicators } from '../mempool.interfaces'; import blocks from './blocks'; import memPool from './mempool'; import backendInfo from './backend-info'; import mempoolBlocks from './mempool-blocks'; import fiatConversion from './fiat-conversion'; import { Common } from './common'; +import loadingIndicators from './loading-indicators'; class WebsocketHandler { private wss: WebSocket.Server | undefined; @@ -117,6 +118,19 @@ class WebsocketHandler { }); } + handleLoadingChanged(indicators: ILoadingIndicators) { + if (!this.wss) { + throw new Error('WebSocket.Server is not set'); + } + + this.wss.clients.forEach((client: WebSocket) => { + if (client.readyState !== WebSocket.OPEN) { + return; + } + client.send(JSON.stringify({ loadingIndicators: indicators })); + }); + } + getInitData(_blocks?: BlockExtended[]) { if (!_blocks) { _blocks = blocks.getBlocks().slice(-8); @@ -131,6 +145,7 @@ class WebsocketHandler { 'transactions': memPool.getLatestTransactions(), 'git-commit': backendInfo.gitCommitHash, 'hostname': backendInfo.hostname, + 'loadingIndicators': loadingIndicators.getLoadingIndicators(), ...this.extraInitProperties }; } diff --git a/backend/src/index.ts b/backend/src/index.ts index 4c753fea7..d2db74b74 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -20,6 +20,7 @@ import bisqMarkets from './api/bisq/markets'; import donations from './api/donations'; import logger from './logger'; import backendInfo from './api/backend-info'; +import loadingIndicators from './api/loading-indicators'; class Server { private wss: WebSocket.Server | undefined; @@ -135,6 +136,7 @@ class Server { blocks.setNewBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler)); memPool.setMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler)); donations.setNotfyDonationStatusCallback(websocketHandler.handleNewDonation.bind(websocketHandler)); + loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler)); } setUpHttpApiRoutes() { diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 1058df7f0..34d1bf6b6 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -136,3 +136,4 @@ interface RequiredParams { types: ('@string' | '@number' | '@boolean' | string)[]; } +export interface ILoadingIndicators { [name: string]: number; } diff --git a/frontend/src/app/dashboard/dashboard.component.html b/frontend/src/app/dashboard/dashboard.component.html index dcc7df9dd..a78092376 100644 --- a/frontend/src/app/dashboard/dashboard.component.html +++ b/frontend/src/app/dashboard/dashboard.component.html @@ -179,8 +179,8 @@
Incoming transactions
- -  Backend is synchronizing + +  Backend is synchronizing ({{ mempoolLoadingStatus$ | async }}%)
diff --git a/frontend/src/app/dashboard/dashboard.component.ts b/frontend/src/app/dashboard/dashboard.component.ts index 6e419712e..6f573aac2 100644 --- a/frontend/src/app/dashboard/dashboard.component.ts +++ b/frontend/src/app/dashboard/dashboard.component.ts @@ -50,6 +50,7 @@ export class DashboardComponent implements OnInit { mempoolBlocksData$: Observable; mempoolInfoData$: Observable; difficultyEpoch$: Observable; + mempoolLoadingStatus$: Observable; vBytesPerSecondLimit = 1667; blocks$: Observable; transactions$: Observable; @@ -77,6 +78,9 @@ export class DashboardComponent implements OnInit { this.websocketService.want(['blocks', 'stats', 'mempool-blocks', 'live-2h-chart']); this.network$ = merge(of(''), this.stateService.networkChanged$); this.collapseLevel = this.storageService.getValue('dashboard-collapsed') || 'one'; + this.mempoolLoadingStatus$ = this.stateService.loadingIndicators$.pipe( + map((indicators) => indicators.mempool !== undefined ? indicators.mempool : 100) + ); this.languageForm = this.formBuilder.group({ language: [''] diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index 893dc1f99..e6d21267b 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -1,3 +1,4 @@ +import { ILoadingIndicators } from '../services/state.service'; import { Block, Transaction } from './electrs.interface'; export interface WebsocketResponse { @@ -15,6 +16,7 @@ export interface WebsocketResponse { rbfTransaction?: Transaction; transactions?: TransactionStripped[]; donationConfirmed?: boolean; + loadingIndicators?: ILoadingIndicators; 'track-tx'?: string; 'track-address'?: string; 'track-asset'?: string; diff --git a/frontend/src/app/services/state.service.ts b/frontend/src/app/services/state.service.ts index 29b501f1e..289a2caa0 100644 --- a/frontend/src/app/services/state.service.ts +++ b/frontend/src/app/services/state.service.ts @@ -13,6 +13,8 @@ interface MarkBlockState { txFeePerVSize?: number; } +export interface ILoadingIndicators { [name: string]: number; } + export interface Env { TESTNET_ENABLED: boolean; LIQUID_ENABLED: boolean; @@ -63,6 +65,7 @@ export class StateService { lastDifficultyAdjustment$ = new ReplaySubject(1); gitCommit$ = new ReplaySubject(1); donationConfirmed$ = new Subject(); + loadingIndicators$ = new ReplaySubject(1); live2Chart$ = new Subject(); diff --git a/frontend/src/app/services/websocket.service.ts b/frontend/src/app/services/websocket.service.ts index 71bbe5263..66bb70b54 100644 --- a/frontend/src/app/services/websocket.service.ts +++ b/frontend/src/app/services/websocket.service.ts @@ -270,6 +270,10 @@ export class WebsocketService { this.stateService.live2Chart$.next(response['live-2h-chart']); } + if (response.loadingIndicators) { + this.stateService.loadingIndicators$.next(response.loadingIndicators); + } + if (response.mempoolInfo) { this.stateService.mempoolInfo$.next(response.mempoolInfo); } From 9e1ef1b747614844973f4ac62e8fc504d85fde1f Mon Sep 17 00:00:00 2001 From: softsimon Date: Wed, 6 Jan 2021 01:36:28 +0700 Subject: [PATCH 16/31] Adding bitcoinJS-lib and parse P2SH and P2WSH scripts. --- backend/package.json | 1 + backend/src/api/bitcoin/bitcoin-api.ts | 55 +++++++++++++++++++------- 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/backend/package.json b/backend/package.json index 6a409a75a..b7ac9a333 100644 --- a/backend/package.json +++ b/backend/package.json @@ -31,6 +31,7 @@ "@mempool/bitcoin": "^3.0.2", "@mempool/electrum-client": "^1.1.7", "axios": "^0.21.0", + "bitcoinjs-lib": "^5.2.0", "crypto-js": "^4.0.0", "express": "^4.17.1", "locutus": "^2.0.12", diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 04e5be511..3f2b83041 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -1,5 +1,6 @@ import config from '../../config'; import * as bitcoin from '@mempool/bitcoin'; +import * as bitcoinjs from 'bitcoinjs-lib'; import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; import { IBitcoinApi } from './bitcoin-api.interface'; import { IEsploraApi } from './esplora-api.interface'; @@ -203,6 +204,7 @@ class BitcoinApi implements AbstractBitcoinApi { } const innerTx = await this.$getRawTransaction(vin.txid, false); vin.prevout = innerTx.vout[vin.vout]; + this.addInnerScriptsToVin(vin); } return transaction; } @@ -216,20 +218,6 @@ class BitcoinApi implements AbstractBitcoinApi { }); } - private convertScriptSigAsm(str: string): string { - const a = str.split(' '); - const b: string[] = []; - a.forEach((chunk) => { - if (chunk.substr(0, 3) === 'OP_') { - b.push(chunk); - } else { - chunk = chunk.replace('[ALL]', '01'); - b.push('OP_PUSHBYTES_' + Math.round(chunk.length / 2) + ' ' + chunk); - } - }); - return b.join(' '); - } - private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean): Promise { if (transaction.vin[0].is_coinbase) { transaction.fee = 0; @@ -240,14 +228,51 @@ class BitcoinApi implements AbstractBitcoinApi { const innerTx = await this.$getRawTransaction(vin.txid, !addPrevout); if (addPrevout) { vin.prevout = innerTx.vout[vin.vout]; + this.addInnerScriptsToVin(vin); } totalIn += innerTx.vout[vin.vout].value; } - const totalOut = transaction.vout.reduce((prev, output) => prev + output.value, 0); + const totalOut = transaction.vout.reduce((p, output) => p + output.value, 0); transaction.fee = parseFloat((totalIn - totalOut).toFixed(8)); return transaction; } + private convertScriptSigAsm(str: string): string { + const a = str.split(' '); + const b: string[] = []; + a.forEach((chunk) => { + if (chunk.substr(0, 3) === 'OP_') { + chunk = chunk.replace(/^OP_(\d+)/, 'OP_PUSHNUM_$1'); + chunk = chunk.replace('OP_CHECKSEQUENCEVERIFY', 'OP_CSV'); + b.push(chunk); + } else { + chunk = chunk.replace('[ALL]', '01'); + if (chunk === '0') { + b.push('OP_0'); + } else { + b.push('OP_PUSHBYTES_' + Math.round(chunk.length / 2) + ' ' + chunk); + } + } + }); + return b.join(' '); + } + + private addInnerScriptsToVin(vin: IEsploraApi.Vin): void { + if (!vin.prevout) { + return; + } + + if (vin.prevout.scriptpubkey_type === 'p2sh') { + const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0]; + vin.inner_redeemscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(redeemScript, 'hex'))); + } + + if (vin.prevout.scriptpubkey_type === 'v0_p2wsh' && vin.witness) { + const witnessScript = vin.witness[vin.witness.length - 1]; + vin.inner_witnessscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(witnessScript, 'hex'))); + } + } + } export default BitcoinApi; From 29dd6e5d8d6dd833dad8ec7d48421fe9bf86c236 Mon Sep 17 00:00:00 2001 From: softsimon Date: Wed, 6 Jan 2021 03:09:31 +0700 Subject: [PATCH 17/31] Never run statistics when mempool not in sync. --- backend/src/api/statistics.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/api/statistics.ts b/backend/src/api/statistics.ts index 2681933ee..619331b77 100644 --- a/backend/src/api/statistics.ts +++ b/backend/src/api/statistics.ts @@ -25,15 +25,15 @@ class Statistics { setTimeout(() => { this.runStatistics(); this.intervalTimer = setInterval(() => { - if (!memPool.isInSync()) { - return; - } this.runStatistics(); }, 1 * 60 * 1000); }, difference); } private async runStatistics(): Promise { + if (!memPool.isInSync()) { + return; + } const currentMempool = memPool.getMempool(); const txPerSecond = memPool.getTxPerSecond(); const vBytesPerSecond = memPool.getVBytesPerSecond(); From dc63fd94284d2f8fd5fe1eb01cf9ece446c2b1a4 Mon Sep 17 00:00:00 2001 From: softsimon Date: Wed, 6 Jan 2021 22:49:28 +0700 Subject: [PATCH 18/31] Config file updates. electrs -> esplora --- backend/mempool-config.sample.json | 9 ++++----- backend/src/api/bitcoin/electrum-api.ts | 2 +- backend/src/api/bitcoin/esplora-api.ts | 14 +++++++------- backend/src/api/mempool.ts | 3 ++- backend/src/config.ts | 22 ++++++++++------------ backend/src/index.ts | 16 ++++++++-------- 6 files changed, 32 insertions(+), 34 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 694bb180b..f51aa3cb9 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -5,16 +5,15 @@ "HTTP_PORT": 8999, "SPAWN_CLUSTER_PROCS": 0, "API_URL_PREFIX": "/api/v1/", - "WEBSOCKET_REFRESH_RATE_MS": 2000 - }, - "ELECTRS": { - "REST_API_URL": "http://127.0.0.1:3000", "POLL_RATE_MS": 2000 }, + "ESPLORA": { + "REST_API_URL": "http://127.0.0.1:3000" + }, "ELECTRUM": { "HOST": "127.0.0.1", "PORT": 50002, - "PROTOCOL": "tcl", + "TLS_ENABLED": true, "TX_LOOKUPS": false }, "BITCOIND": { diff --git a/backend/src/api/bitcoin/electrum-api.ts b/backend/src/api/bitcoin/electrum-api.ts index 2929c2da4..b8bcc90d7 100644 --- a/backend/src/api/bitcoin/electrum-api.ts +++ b/backend/src/api/bitcoin/electrum-api.ts @@ -30,7 +30,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { this.electrumClient = new ElectrumClient( config.ELECTRUM.PORT, config.ELECTRUM.HOST, - config.ELECTRUM.PROTOCOL, + config.ELECTRUM.TLS_ENABLED ? 'tls' : 'tcp', null, electrumCallbacks ); diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index a7bedbbec..df566daac 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -8,32 +8,32 @@ class ElectrsApi implements AbstractBitcoinApi { constructor() { } $getRawMempool(): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/mempool/txids') + return axios.get(config.ESPLORA.REST_API_URL + '/mempool/txids') .then((response) => response.data); } $getRawTransaction(txId: string): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/tx/' + txId) + return axios.get(config.ESPLORA.REST_API_URL + '/tx/' + txId) .then((response) => response.data); } $getBlockHeightTip(): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/blocks/tip/height') + return axios.get(config.ESPLORA.REST_API_URL + '/blocks/tip/height') .then((response) => response.data); } $getTxIdsForBlock(hash: string): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/block/' + hash + '/txids') + return axios.get(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids') .then((response) => response.data); } $getBlockHash(height: number): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/block-height/' + height) + return axios.get(config.ESPLORA.REST_API_URL + '/block-height/' + height) .then((response) => response.data); } $getBlock(hash: string): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/block/' + hash) + return axios.get(config.ESPLORA.REST_API_URL + '/block/' + hash) .then((response) => response.data); } @@ -46,7 +46,7 @@ class ElectrsApi implements AbstractBitcoinApi { } $getRawTransactionBitcoind(txId: string): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/tx/' + txId) + return axios.get(config.ESPLORA.REST_API_URL + '/tx/' + txId) .then((response) => response.data); } diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 27cd94794..c00d5c10c 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -9,6 +9,7 @@ import bitcoinBaseApi from './bitcoin/bitcoin-base.api'; import loadingIndicators from './loading-indicators'; class Mempool { + private static WEBSOCKET_REFRESH_RATE_MS = 10000; private inSync: boolean = false; private mempoolCache: { [txId: string]: TransactionExtended } = {}; private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, @@ -120,7 +121,7 @@ class Mempool { } } - if ((new Date().getTime()) - start > config.MEMPOOL.WEBSOCKET_REFRESH_RATE_MS * 10) { + if ((new Date().getTime()) - start > Mempool.WEBSOCKET_REFRESH_RATE_MS) { break; } } diff --git a/backend/src/config.ts b/backend/src/config.ts index 20132fdd8..1886ae880 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -7,16 +7,15 @@ interface IConfig { HTTP_PORT: number; SPAWN_CLUSTER_PROCS: number; API_URL_PREFIX: string; - WEBSOCKET_REFRESH_RATE_MS: number; - }; - ELECTRS: { - REST_API_URL: string; POLL_RATE_MS: number; }; + ESPLORA: { + REST_API_URL: string; + }; ELECTRUM: { HOST: string; PORT: number; - PROTOCOL: 'tls' | 'tcp'; + TLS_ENABLED: boolean; TX_LOOKUPS: boolean; }; BITCOIND: { @@ -61,16 +60,15 @@ const defaults: IConfig = { 'HTTP_PORT': 8999, 'SPAWN_CLUSTER_PROCS': 0, 'API_URL_PREFIX': '/api/v1/', - 'WEBSOCKET_REFRESH_RATE_MS': 2000 - }, - 'ELECTRS': { - 'REST_API_URL': 'http://127.0.0.1:3000', 'POLL_RATE_MS': 2000 }, + 'ESPLORA': { + 'REST_API_URL': 'http://127.0.0.1:3000', + }, 'ELECTRUM': { 'HOST': '127.0.0.1', 'PORT': 3306, - 'PROTOCOL': 'tls', + 'TLS_ENABLED': true, 'TX_LOOKUPS': false }, 'BITCOIND': { @@ -110,7 +108,7 @@ const defaults: IConfig = { class Config implements IConfig { MEMPOOL: IConfig['MEMPOOL']; - ELECTRS: IConfig['ELECTRS']; + ESPLORA: IConfig['ESPLORA']; ELECTRUM: IConfig['ELECTRUM']; BITCOIND: IConfig['BITCOIND']; DATABASE: IConfig['DATABASE']; @@ -122,7 +120,7 @@ class Config implements IConfig { constructor() { const configs = this.merge(configFile, defaults); this.MEMPOOL = configs.MEMPOOL; - this.ELECTRS = configs.ELECTRS; + this.ESPLORA = configs.ESPLORA; this.ELECTRUM = configs.ELECTRUM; this.BITCOIND = configs.BITCOIND; this.DATABASE = configs.DATABASE; diff --git a/backend/src/index.ts b/backend/src/index.ts index d2db74b74..9d88453fb 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -26,7 +26,7 @@ class Server { private wss: WebSocket.Server | undefined; private server: https.Server | http.Server | undefined; private app: Express; - private retryOnElectrsErrorAfterSeconds = 5; + private currentBackendRetryInterval = 5; constructor() { this.app = express(); @@ -111,19 +111,19 @@ class Server { await memPool.$updateMemPoolInfo(); await blocks.$updateBlocks(); await memPool.$updateMempool(); - setTimeout(this.runMainUpdateLoop.bind(this), config.ELECTRS.POLL_RATE_MS); - this.retryOnElectrsErrorAfterSeconds = 5; + setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS); + this.currentBackendRetryInterval = 5; } catch (e) { - const loggerMsg = `runMainLoop error: ${(e.message || e)}. Retrying in ${this.retryOnElectrsErrorAfterSeconds} sec.`; - if (this.retryOnElectrsErrorAfterSeconds > 5) { + const loggerMsg = `runMainLoop error: ${(e.message || e)}. Retrying in ${this.currentBackendRetryInterval} sec.`; + if (this.currentBackendRetryInterval > 5) { logger.warn(loggerMsg); } else { logger.debug(loggerMsg); } logger.debug(JSON.stringify(e)); - setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.retryOnElectrsErrorAfterSeconds); - this.retryOnElectrsErrorAfterSeconds *= 2; - this.retryOnElectrsErrorAfterSeconds = Math.min(this.retryOnElectrsErrorAfterSeconds, 60); + setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval); + this.currentBackendRetryInterval *= 2; + this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60); } } From b91516a1c1454cf00c6a16db2ce833e8b047a9f8 Mon Sep 17 00:00:00 2001 From: softsimon Date: Wed, 6 Jan 2021 23:31:33 +0700 Subject: [PATCH 19/31] Push new conversion rate updates to the clients. --- backend/src/api/fiat-conversion.ts | 11 ++++++++++- backend/src/api/websocket-handler.ts | 15 ++++++++++++++- backend/src/index.ts | 1 + backend/src/mempool.interfaces.ts | 1 + 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/backend/src/api/fiat-conversion.ts b/backend/src/api/fiat-conversion.ts index 89916c91d..8de9f10bb 100644 --- a/backend/src/api/fiat-conversion.ts +++ b/backend/src/api/fiat-conversion.ts @@ -1,13 +1,19 @@ import logger from '../logger'; import axios from 'axios'; +import { IConversionRates } from '../mempool.interfaces'; class FiatConversion { - private conversionRates = { + private conversionRates: IConversionRates = { 'USD': 0 }; + private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined; constructor() { } + public setProgressChangedCallback(fn: (rates: IConversionRates) => void) { + this.ratesChangedCallback = fn; + } + public startService() { logger.info('Starting currency rates service'); setInterval(this.updateCurrency.bind(this), 1000 * 60 * 60); @@ -25,6 +31,9 @@ class FiatConversion { this.conversionRates = { 'USD': usd.price, }; + if (this.ratesChangedCallback) { + this.ratesChangedCallback(this.conversionRates); + } } catch (e) { logger.err('Error updating fiat conversion rates: ' + e); } diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 2d4e75d6f..af46968bc 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -1,6 +1,6 @@ import logger from '../logger'; import * as WebSocket from 'ws'; -import { BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, OptimizedStatistic, ILoadingIndicators } from '../mempool.interfaces'; +import { BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, OptimizedStatistic, ILoadingIndicators, IConversionRates } from '../mempool.interfaces'; import blocks from './blocks'; import memPool from './mempool'; import backendInfo from './backend-info'; @@ -131,6 +131,19 @@ class WebsocketHandler { }); } + handleNewConversionRates(conversionRates: IConversionRates) { + if (!this.wss) { + throw new Error('WebSocket.Server is not set'); + } + + this.wss.clients.forEach((client: WebSocket) => { + if (client.readyState !== WebSocket.OPEN) { + return; + } + client.send(JSON.stringify({ conversions: conversionRates })); + }); + } + getInitData(_blocks?: BlockExtended[]) { if (!_blocks) { _blocks = blocks.getBlocks().slice(-8); diff --git a/backend/src/index.ts b/backend/src/index.ts index 9d88453fb..d9e8722d1 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -136,6 +136,7 @@ class Server { blocks.setNewBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler)); memPool.setMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler)); donations.setNotfyDonationStatusCallback(websocketHandler.handleNewDonation.bind(websocketHandler)); + fiatConversion.setProgressChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler)); loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler)); } diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 34d1bf6b6..30227ba08 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -137,3 +137,4 @@ interface RequiredParams { } export interface ILoadingIndicators { [name: string]: number; } +export interface IConversionRates { [currency: string]: number; } From 6a5871769420e8a3a60c84fb70603b444b877e78 Mon Sep 17 00:00:00 2001 From: softsimon Date: Fri, 8 Jan 2021 21:44:36 +0700 Subject: [PATCH 20/31] Loading progressbar for loading address, block transactions and blocks --- backend/src/api/bitcoin/electrum-api.ts | 12 ++++++++---- backend/src/routes.ts | 12 +++++++++++- .../app/components/address/address.component.html | 8 ++++++++ .../src/app/components/address/address.component.ts | 13 ++++++++++--- .../src/app/components/block/block.component.html | 7 +++++++ .../src/app/components/block/block.component.ts | 11 +++++++++-- .../latest-blocks/latest-blocks.component.html | 11 ++++++++++- .../latest-blocks/latest-blocks.component.ts | 7 +++++++ 8 files changed, 70 insertions(+), 11 deletions(-) diff --git a/backend/src/api/bitcoin/electrum-api.ts b/backend/src/api/bitcoin/electrum-api.ts index b8bcc90d7..f63516d8a 100644 --- a/backend/src/api/bitcoin/electrum-api.ts +++ b/backend/src/api/bitcoin/electrum-api.ts @@ -10,6 +10,7 @@ import logger from '../../logger'; import * as ElectrumClient from '@mempool/electrum-client'; import * as sha256 from 'crypto-js/sha256'; import * as hexEnc from 'crypto-js/enc-hex'; +import loadingIndicators from '../loading-indicators'; class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { private electrumClient: any; @@ -121,6 +122,8 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { } try { + loadingIndicators.setProgress('address-' + address, 0); + const transactions: IEsploraApi.Transaction[] = []; const history = await this.$getScriptHashHistory(addressInfo.scriptPubKey); history.reverse(); @@ -132,16 +135,17 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { startingIndex = pos + 1; } } + const endIndex = Math.min(startingIndex + 10, history.length); - for (let i = startingIndex; i < Math.min(startingIndex + 10, history.length); i++) { + for (let i = startingIndex; i < endIndex; i++) { const tx = await this.$getRawTransaction(history[i].tx_hash, false, true); - if (tx) { - transactions.push(tx); - } + transactions.push(tx); + loadingIndicators.setProgress('address-' + address, (i + 1) / endIndex * 100); } return transactions; } catch (e) { + loadingIndicators.setProgress('address-' + address, 100); if (e === 'failed to get confirmed status') { e = 'The number of transactions on this address exceeds the Electrum server limit'; } diff --git a/backend/src/routes.ts b/backend/src/routes.ts index d770aea8d..390d11ddd 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -16,6 +16,7 @@ import logger from './logger'; import bitcoinApi from './api/bitcoin/bitcoin-api-factory'; import transactionUtils from './api/transaction-utils'; import blocks from './api/blocks'; +import loadingIndicators from './api/loading-indicators'; class Routes { private cache: { [date: string]: OptimizedStatistic[] } = { @@ -553,6 +554,8 @@ class Routes { public async getBlocks(req: Request, res: Response) { try { + loadingIndicators.setProgress('blocks', 0); + const returnBlocks: IEsploraApi.Block[] = []; const fromHeight = parseInt(req.params.height, 10) || blocks.getCurrentBlockHeight(); @@ -576,28 +579,35 @@ class Routes { returnBlocks.push(block); nextHash = block.previousblockhash; } + loadingIndicators.setProgress('blocks', i / 10 * 100); } res.json(returnBlocks); } catch (e) { + loadingIndicators.setProgress('blocks', 100); res.status(500).send(e.message || e); } } public async getBlockTransactions(req: Request, res: Response) { try { + loadingIndicators.setProgress('blocktxs-' + req.params.hash, 0); + const txIds = await bitcoinApi.$getTxIdsForBlock(req.params.hash); const transactions: TransactionExtended[] = []; const startingIndex = Math.max(0, parseInt(req.params.index, 10)); - for (let i = startingIndex; i < Math.min(startingIndex + 10, txIds.length); i++) { + const endIndex = Math.min(startingIndex + 10, txIds.length); + for (let i = startingIndex; i < endIndex; i++) { const transaction = await transactionUtils.$getTransactionExtended(txIds[i], false, true); if (transaction) { transactions.push(transaction); + loadingIndicators.setProgress('blocktxs-' + req.params.hash, (i + 1) / endIndex * 100); } } res.json(transactions); } catch (e) { + loadingIndicators.setProgress('blocktxs-' + req.params.hash, 100); res.status(500).send(e.message || e); } } diff --git a/frontend/src/app/components/address/address.component.html b/frontend/src/app/components/address/address.component.html index e5a0ffd0a..49280f471 100644 --- a/frontend/src/app/components/address/address.component.html +++ b/frontend/src/app/components/address/address.component.html @@ -67,6 +67,14 @@
+ + +
+
+
+
+ + diff --git a/frontend/src/app/components/address/address.component.ts b/frontend/src/app/components/address/address.component.ts index f1d21d3b9..601d422d6 100644 --- a/frontend/src/app/components/address/address.component.ts +++ b/frontend/src/app/components/address/address.component.ts @@ -1,13 +1,13 @@ import { Component, OnInit, OnDestroy } from '@angular/core'; import { ActivatedRoute, ParamMap } from '@angular/router'; import { ElectrsApiService } from '../../services/electrs-api.service'; -import { switchMap, filter, catchError } from 'rxjs/operators'; +import { switchMap, filter, catchError, map, tap } from 'rxjs/operators'; import { Address, Transaction } from '../../interfaces/electrs.interface'; import { WebsocketService } from 'src/app/services/websocket.service'; import { StateService } from 'src/app/services/state.service'; import { AudioService } from 'src/app/services/audio.service'; import { ApiService } from 'src/app/services/api.service'; -import { of, merge, Subscription } from 'rxjs'; +import { of, merge, Subscription, Observable } from 'rxjs'; import { SeoService } from 'src/app/services/seo.service'; @Component({ @@ -25,6 +25,7 @@ export class AddressComponent implements OnInit, OnDestroy { isLoadingTransactions = true; error: any; mainSubscription: Subscription; + addressLoadingStatus$: Observable; totalConfirmedTxCount = 0; loadedConfirmedTxCount = 0; @@ -48,7 +49,13 @@ export class AddressComponent implements OnInit, OnDestroy { ngOnInit() { this.stateService.networkChanged$.subscribe((network) => this.network = network); - this.websocketService.want(['blocks', 'mempool-blocks']); + this.websocketService.want(['blocks']); + + this.addressLoadingStatus$ = this.route.paramMap + .pipe( + switchMap(() => this.stateService.loadingIndicators$), + map((indicators) => indicators['address-' + this.addressString] !== undefined ? indicators['address-' + this.addressString] : 0) + ); this.mainSubscription = this.route.paramMap .pipe( diff --git a/frontend/src/app/components/block/block.component.html b/frontend/src/app/components/block/block.component.html index 8b0f852c9..85daa10bd 100644 --- a/frontend/src/app/components/block/block.component.html +++ b/frontend/src/app/components/block/block.component.html @@ -112,6 +112,13 @@ + + +
+
+
+
+ diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 3c9b3bf08..99b66de77 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -2,9 +2,9 @@ import { Component, OnInit, OnDestroy } from '@angular/core'; import { Location } from '@angular/common'; import { ActivatedRoute, ParamMap, Router } from '@angular/router'; import { ElectrsApiService } from '../../services/electrs-api.service'; -import { switchMap, tap, debounceTime, catchError } from 'rxjs/operators'; +import { switchMap, tap, debounceTime, catchError, map } from 'rxjs/operators'; import { Block, Transaction, Vout } from '../../interfaces/electrs.interface'; -import { of, Subscription } from 'rxjs'; +import { Observable, of, Subscription } from 'rxjs'; import { StateService } from '../../services/state.service'; import { SeoService } from 'src/app/services/seo.service'; import { WebsocketService } from 'src/app/services/websocket.service'; @@ -31,6 +31,7 @@ export class BlockComponent implements OnInit, OnDestroy { coinbaseTx: Transaction; page = 1; itemsPerPage: number; + txsLoadingStatus$: Observable; constructor( private route: ActivatedRoute, @@ -48,6 +49,12 @@ export class BlockComponent implements OnInit, OnDestroy { this.network = this.stateService.network; this.itemsPerPage = this.stateService.env.ELECTRS_ITEMS_PER_PAGE; + this.txsLoadingStatus$ = this.route.paramMap + .pipe( + switchMap(() => this.stateService.loadingIndicators$), + map((indicators) => indicators['blocktxs-' + this.blockHash] !== undefined ? indicators['blocktxs-' + this.blockHash] : 0) + ); + this.subscription = this.route.paramMap .pipe( switchMap((params: ParamMap) => { diff --git a/frontend/src/app/components/latest-blocks/latest-blocks.component.html b/frontend/src/app/components/latest-blocks/latest-blocks.component.html index 5c8853158..882442ec4 100644 --- a/frontend/src/app/components/latest-blocks/latest-blocks.component.html +++ b/frontend/src/app/components/latest-blocks/latest-blocks.component.html @@ -33,7 +33,16 @@ - + + + +
+
+
+ + + + diff --git a/frontend/src/app/components/latest-blocks/latest-blocks.component.ts b/frontend/src/app/components/latest-blocks/latest-blocks.component.ts index ccc898355..eb8b8236d 100644 --- a/frontend/src/app/components/latest-blocks/latest-blocks.component.ts +++ b/frontend/src/app/components/latest-blocks/latest-blocks.component.ts @@ -5,6 +5,7 @@ import { Block } from '../../interfaces/electrs.interface'; import { Subscription, Observable, merge, of } from 'rxjs'; import { SeoService } from '../../services/seo.service'; import { WebsocketService } from 'src/app/services/websocket.service'; +import { map } from 'rxjs/operators'; @Component({ selector: 'app-latest-blocks', @@ -19,6 +20,7 @@ export class LatestBlocksComponent implements OnInit, OnDestroy { blockSubscription: Subscription; isLoading = true; interval: any; + blocksLoadingStatus$: Observable; latestBlockHeight: number; @@ -39,6 +41,11 @@ export class LatestBlocksComponent implements OnInit, OnDestroy { this.network$ = merge(of(''), this.stateService.networkChanged$); + this.blocksLoadingStatus$ = this.stateService.loadingIndicators$ + .pipe( + map((indicators) => indicators['blocks'] !== undefined ? indicators['blocks'] : 0) + ); + this.blockSubscription = this.stateService.blocks$ .subscribe(([block]) => { if (block === null || !this.blocks.length) { From 065c21da1fc252f79d0c5bee36b0d200819a8d7f Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 10 Jan 2021 17:38:59 +0700 Subject: [PATCH 21/31] Bitcoind: Push full transactions to address page and RBF mode. --- backend/src/api/websocket-handler.ts | 48 +++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index af46968bc..35d11c37c 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -1,6 +1,7 @@ import logger from '../logger'; import * as WebSocket from 'ws'; -import { BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, OptimizedStatistic, ILoadingIndicators, IConversionRates } from '../mempool.interfaces'; +import { BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, + OptimizedStatistic, ILoadingIndicators, IConversionRates } from '../mempool.interfaces'; import blocks from './blocks'; import memPool from './mempool'; import backendInfo from './backend-info'; @@ -8,6 +9,8 @@ import mempoolBlocks from './mempool-blocks'; import fiatConversion from './fiat-conversion'; import { Common } from './common'; import loadingIndicators from './loading-indicators'; +import config from '../config'; +import transactionUtils from './transaction-utils'; class WebsocketHandler { private wss: WebSocket.Server | undefined; @@ -195,7 +198,7 @@ class WebsocketHandler { const vBytesPerSecond = memPool.getVBytesPerSecond(); const rbfTransactions = Common.findRbfTransactions(newTransactions, deletedTransactions); - this.wss.clients.forEach((client: WebSocket) => { + this.wss.clients.forEach(async (client: WebSocket) => { if (client.readyState !== WebSocket.OPEN) { return; } @@ -215,7 +218,14 @@ class WebsocketHandler { if (client['track-mempool-tx']) { const tx = newTransactions.find((t) => t.txid === client['track-mempool-tx']); if (tx) { - response['tx'] = tx; + if (config.MEMPOOL.BACKEND !== 'esplora') { + const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, false, true); + if (fullTx) { + response['tx'] = fullTx; + } + } else { + response['tx'] = tx; + } client['track-mempool-tx'] = null; } } @@ -223,17 +233,31 @@ class WebsocketHandler { if (client['track-address']) { const foundTransactions: TransactionExtended[] = []; - newTransactions.forEach((tx) => { + for (const tx of newTransactions) { const someVin = tx.vin.some((vin) => !!vin.prevout && vin.prevout.scriptpubkey_address === client['track-address']); if (someVin) { - foundTransactions.push(tx); + if (config.MEMPOOL.BACKEND !== 'esplora') { + const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, false, true); + if (fullTx) { + foundTransactions.push(fullTx); + } + } else { + foundTransactions.push(tx); + } return; } const someVout = tx.vout.some((vout) => vout.scriptpubkey_address === client['track-address']); if (someVout) { - foundTransactions.push(tx); + if (config.MEMPOOL.BACKEND !== 'esplora') { + const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, false, true); + if (fullTx) { + foundTransactions.push(fullTx); + } + } else { + foundTransactions.push(tx); + } } - }); + } if (foundTransactions.length) { response['address-transactions'] = foundTransactions; @@ -272,7 +296,15 @@ class WebsocketHandler { if (client['track-tx'] && rbfTransactions[client['track-tx']]) { for (const rbfTransaction in rbfTransactions) { if (client['track-tx'] === rbfTransaction) { - response['rbfTransaction'] = rbfTransactions[rbfTransaction]; + const rbfTx = rbfTransactions[rbfTransaction]; + if (config.MEMPOOL.BACKEND !== 'esplora') { + const fullTx = await transactionUtils.$getTransactionExtended(rbfTransaction, false, true); + if (fullTx) { + response['rbfTransaction'] = fullTx; + } + } else { + response['rbfTransaction'] = rbfTx; + } break; } } From 9a23d2c6b02d4b8ed0075f400eea9dd1cdd1316d Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 10 Jan 2021 17:40:05 +0700 Subject: [PATCH 22/31] Electrum: Sort address transactions correctly by confirmed and unconfirmed. --- backend/src/api/bitcoin/electrum-api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/bitcoin/electrum-api.ts b/backend/src/api/bitcoin/electrum-api.ts index f63516d8a..d6b8c76cf 100644 --- a/backend/src/api/bitcoin/electrum-api.ts +++ b/backend/src/api/bitcoin/electrum-api.ts @@ -126,7 +126,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { const transactions: IEsploraApi.Transaction[] = []; const history = await this.$getScriptHashHistory(addressInfo.scriptPubKey); - history.reverse(); + history.sort((a, b) => (b.height || 9999999) - (a.height || 9999999)); let startingIndex = 0; if (lastSeenTxId) { From c4d1fad85377ea3d1de01545a2d1cb48e50b1d34 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 10 Jan 2021 19:58:55 +0700 Subject: [PATCH 23/31] Cache electrum address history fetch a couple of seconds to prevent double requests on address page load. --- backend/src/api/bitcoin/electrum-api.ts | 11 ++++++- backend/src/api/memory-cache.ts | 38 +++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 backend/src/api/memory-cache.ts diff --git a/backend/src/api/bitcoin/electrum-api.ts b/backend/src/api/bitcoin/electrum-api.ts index d6b8c76cf..ee8395178 100644 --- a/backend/src/api/bitcoin/electrum-api.ts +++ b/backend/src/api/bitcoin/electrum-api.ts @@ -11,6 +11,7 @@ import * as ElectrumClient from '@mempool/electrum-client'; import * as sha256 from 'crypto-js/sha256'; import * as hexEnc from 'crypto-js/enc-hex'; import loadingIndicators from '../loading-indicators'; +import memoryCache from '../memory-cache'; class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { private electrumClient: any; @@ -158,7 +159,15 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { } private $getScriptHashHistory(scriptHash: string): Promise { - return this.electrumClient.blockchainScripthash_getHistory(this.encodeScriptHash(scriptHash)); + const fromCache = memoryCache.get('Scripthash_getHistory', scriptHash); + if (fromCache) { + return Promise.resolve(fromCache); + } + return this.electrumClient.blockchainScripthash_getHistory(this.encodeScriptHash(scriptHash)) + .then((history) => { + memoryCache.set('Scripthash_getHistory', scriptHash, history, 2); + return history; + }); } private encodeScriptHash(scriptPubKey: string): string { diff --git a/backend/src/api/memory-cache.ts b/backend/src/api/memory-cache.ts new file mode 100644 index 000000000..fe4162420 --- /dev/null +++ b/backend/src/api/memory-cache.ts @@ -0,0 +1,38 @@ +interface ICache { + type: string; + id: string; + expires: Date; + data: any; +} + +class MemoryCache { + private cache: ICache[] = []; + constructor() { + setInterval(this.cleanup.bind(this), 1000); + } + + public set(type: string, id: string, data: any, secondsExpiry: number) { + const expiry = new Date(); + expiry.setSeconds(expiry.getSeconds() + secondsExpiry); + this.cache.push({ + type: type, + id: id, + data: data, + expires: expiry, + }); + } + + public get(type: string, id: string): T | null { + const found = this.cache.find((cache) => cache.type === type && cache.id === id); + if (found) { + return found.data; + } + return null; + } + + private cleanup() { + this.cache = this.cache.filter((cache) => cache.expires < (new Date())); + } +} + +export default new MemoryCache(); From 9689ccf2ac80954375285ff635ada6cec6b6474d Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 10 Jan 2021 22:24:36 +0700 Subject: [PATCH 24/31] Bitcoind: Display the suggestion message on address page when address lookups is not implemented. --- frontend/src/app/components/address/address.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/components/address/address.component.html b/frontend/src/app/components/address/address.component.html index 49280f471..6c06aec2d 100644 --- a/frontend/src/app/components/address/address.component.html +++ b/frontend/src/app/components/address/address.component.html @@ -113,7 +113,7 @@ Error loading address data.
{{ error.error }} - +

Consider view this address on the official Mempool website instead:
From e36646ac7c2dae8def776eded744a32fa6eb24db Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 10 Jan 2021 22:41:52 +0700 Subject: [PATCH 25/31] Adding missing error output on /blocks page. --- .../latest-blocks/latest-blocks.component.html | 8 ++++++++ .../latest-blocks/latest-blocks.component.ts | 16 +++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/latest-blocks/latest-blocks.component.html b/frontend/src/app/components/latest-blocks/latest-blocks.component.html index 882442ec4..ce9496657 100644 --- a/frontend/src/app/components/latest-blocks/latest-blocks.component.html +++ b/frontend/src/app/components/latest-blocks/latest-blocks.component.html @@ -46,4 +46,12 @@ + +
+ Error loading blocks +
+ {{ error.error }} +
+ + diff --git a/frontend/src/app/components/latest-blocks/latest-blocks.component.ts b/frontend/src/app/components/latest-blocks/latest-blocks.component.ts index eb8b8236d..15b2843bd 100644 --- a/frontend/src/app/components/latest-blocks/latest-blocks.component.ts +++ b/frontend/src/app/components/latest-blocks/latest-blocks.component.ts @@ -15,7 +15,7 @@ import { map } from 'rxjs/operators'; }) export class LatestBlocksComponent implements OnInit, OnDestroy { network$: Observable; - + error: any; blocks: any[] = []; blockSubscription: Subscription; isLoading = true; @@ -86,6 +86,7 @@ export class LatestBlocksComponent implements OnInit, OnDestroy { .subscribe((blocks) => { this.blocks = blocks; this.isLoading = false; + this.error = undefined; this.latestBlockHeight = blocks[0].height; @@ -95,6 +96,12 @@ export class LatestBlocksComponent implements OnInit, OnDestroy { this.loadMore(chunks); } this.cd.markForCheck(); + }, + (error) => { + console.log(error); + this.error = error; + this.isLoading = false; + this.cd.markForCheck(); }); } @@ -107,12 +114,19 @@ export class LatestBlocksComponent implements OnInit, OnDestroy { .subscribe((blocks) => { this.blocks = this.blocks.concat(blocks); this.isLoading = false; + this.error = undefined; const chunksLeft = chunks - 1; if (chunksLeft > 0) { this.loadMore(chunksLeft); } this.cd.markForCheck(); + }, + (error) => { + console.log(error); + this.error = error; + this.isLoading = false; + this.cd.markForCheck(); }); } From 38d534caeef06f53cd253e0f8ea6b84c3ad94d07 Mon Sep 17 00:00:00 2001 From: softsimon Date: Mon, 11 Jan 2021 00:26:36 +0700 Subject: [PATCH 26/31] Return not implemented error instead of empty array. --- backend/src/routes.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 390d11ddd..fe6a82a8a 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -656,15 +656,15 @@ class Routes { } public async getAdressTxChain(req: Request, res: Response) { - res.status(404).send('Not implemented'); + res.status(501).send('Not implemented'); } public async getAddressPrefix(req: Request, res: Response) { - res.json([]); + res.status(501).send('Not implemented'); } public getTransactionOutspends(req: Request, res: Response) { - res.json([]); + res.status(501).send('Not implemented'); } } From 905ddbb363b3f9172eecef9d7d15f2d3dc678193 Mon Sep 17 00:00:00 2001 From: softsimon Date: Mon, 11 Jan 2021 01:51:57 +0700 Subject: [PATCH 27/31] Bitcoind: Use mempool as address index when doing address prefix search. --- .../api/bitcoin/bitcoin-api-abstract-factory.ts | 1 + backend/src/api/bitcoin/bitcoin-api.ts | 16 ++++++++++++++++ backend/src/api/bitcoin/esplora-api.ts | 3 +++ backend/src/routes.ts | 7 ++++++- backend/tslint.json | 2 +- 5 files changed, 27 insertions(+), 2 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index 4dcd4b4d4..951d9576c 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -10,4 +10,5 @@ export interface AbstractBitcoinApi { $getBlock(hash: string): Promise; $getAddress(address: string): Promise; $getAddressTransactions(address: string, lastSeenTxId: string): Promise; + $getAddressPrefix(prefix: string): string[]; } diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 3f2b83041..b77b63385 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -90,6 +90,22 @@ class BitcoinApi implements AbstractBitcoinApi { return this.bitcoindClient.getRawMemPool(); } + $getAddressPrefix(prefix: string): string[] { + const found: string[] = []; + const mp = mempool.getMempool(); + for (const tx in mp) { + for (const vout of mp[tx].vout) { + if (vout.scriptpubkey_address.indexOf(prefix) === 0) { + found.push(vout.scriptpubkey_address); + if (found.length >= 10) { + return found; + } + } + } + } + return found; + } + protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean): Promise { let esploraTransaction: IEsploraApi.Transaction = { txid: transaction.txid, diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index df566daac..24220da3e 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -50,6 +50,9 @@ class ElectrsApi implements AbstractBitcoinApi { .then((response) => response.data); } + $getAddressPrefix(prefix: string): string[] { + throw new Error('Method not implemented.'); + } } export default ElectrsApi; diff --git a/backend/src/routes.ts b/backend/src/routes.ts index fe6a82a8a..e35164ab0 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -660,7 +660,12 @@ class Routes { } public async getAddressPrefix(req: Request, res: Response) { - res.status(501).send('Not implemented'); + try { + const blockHash = await bitcoinApi.$getAddressPrefix(req.params.prefix); + res.send(blockHash); + } catch (e) { + res.status(500).send(e.message || e); + } } public getTransactionOutspends(req: Request, res: Response) { diff --git a/backend/tslint.json b/backend/tslint.json index 65ac58f4b..945512322 100644 --- a/backend/tslint.json +++ b/backend/tslint.json @@ -12,7 +12,7 @@ "severity": "warn" }, "eofline": true, - "forin": true, + "forin": false, "import-blacklist": [ true, "rxjs", From 20406fa522c6253fe112b6f999f0931111ca72ce Mon Sep 17 00:00:00 2001 From: softsimon Date: Mon, 11 Jan 2021 14:30:58 +0700 Subject: [PATCH 28/31] Moved method $validateAddress --- backend/src/api/bitcoin/bitcoin-api.ts | 4 ++++ backend/src/api/bitcoin/bitcoin-base.api.ts | 4 ---- backend/src/api/bitcoin/electrum-api.ts | 5 ++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index b77b63385..7fc8075ff 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -234,6 +234,10 @@ class BitcoinApi implements AbstractBitcoinApi { }); } + protected $validateAddress(address: string): Promise { + return this.bitcoindClient.validateAddress(address); + } + private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean): Promise { if (transaction.vin[0].is_coinbase) { transaction.fee = 0; diff --git a/backend/src/api/bitcoin/bitcoin-base.api.ts b/backend/src/api/bitcoin/bitcoin-base.api.ts index 22c8daaed..a79a1a311 100644 --- a/backend/src/api/bitcoin/bitcoin-base.api.ts +++ b/backend/src/api/bitcoin/bitcoin-base.api.ts @@ -31,10 +31,6 @@ class BitcoinBaseApi { return this.bitcoindClient.getRawMemPool(true); } - $validateAddress(address: string): Promise { - return this.bitcoindClient.validateAddress(address); - } - } export default new BitcoinBaseApi(); diff --git a/backend/src/api/bitcoin/electrum-api.ts b/backend/src/api/bitcoin/electrum-api.ts index ee8395178..6c275356b 100644 --- a/backend/src/api/bitcoin/electrum-api.ts +++ b/backend/src/api/bitcoin/electrum-api.ts @@ -4,7 +4,6 @@ import { IBitcoinApi } from './bitcoin-api.interface'; import { IEsploraApi } from './esplora-api.interface'; import { IElectrumApi } from './electrum-api.interface'; import BitcoinApi from './bitcoin-api'; -import bitcoinBaseApi from './bitcoin-base.api'; import mempool from '../mempool'; import logger from '../../logger'; import * as ElectrumClient from '@mempool/electrum-client'; @@ -64,7 +63,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { } async $getAddress(address: string): Promise { - const addressInfo = await bitcoinBaseApi.$validateAddress(address); + const addressInfo = await this.$validateAddress(address); if (!addressInfo || !addressInfo.isvalid) { return ({ 'address': address, @@ -117,7 +116,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { } async $getAddressTransactions(address: string, lastSeenTxId: string): Promise { - const addressInfo = await bitcoinBaseApi.$validateAddress(address); + const addressInfo = await this.$validateAddress(address); if (!addressInfo || !addressInfo.isvalid) { return []; } From e76ee93bbb50626c45742ce0d1f8e6098a8f4888 Mon Sep 17 00:00:00 2001 From: softsimon Date: Mon, 11 Jan 2021 14:53:18 +0700 Subject: [PATCH 29/31] Config updates. Renamed BITCOIND to CORE_RPC. --- backend/mempool-config.sample.json | 14 +++++++------- backend/src/api/bitcoin/bitcoin-api.ts | 8 ++++---- backend/src/api/bitcoin/bitcoin-base.api.ts | 8 ++++---- backend/src/config.ts | 8 ++++---- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index f51aa3cb9..9b55905f4 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -7,8 +7,11 @@ "API_URL_PREFIX": "/api/v1/", "POLL_RATE_MS": 2000 }, - "ESPLORA": { - "REST_API_URL": "http://127.0.0.1:3000" + "CORE_RPC": { + "HOST": "127.0.0.1", + "PORT": 8332, + "USERNAME": "mempool", + "PASSWORD": "mempool" }, "ELECTRUM": { "HOST": "127.0.0.1", @@ -16,11 +19,8 @@ "TLS_ENABLED": true, "TX_LOOKUPS": false }, - "BITCOIND": { - "HOST": "127.0.0.1", - "PORT": 3306, - "USERNAME": "mempool", - "PASSWORD": "mempool" + "ESPLORA": { + "REST_API_URL": "http://127.0.0.1:3000" }, "DATABASE": { "ENABLED": true, diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 7fc8075ff..c40ab031c 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -15,10 +15,10 @@ class BitcoinApi implements AbstractBitcoinApi { constructor() { this.bitcoindClient = new bitcoin.Client({ - host: config.BITCOIND.HOST, - port: config.BITCOIND.PORT, - user: config.BITCOIND.USERNAME, - pass: config.BITCOIND.PASSWORD, + host: config.CORE_RPC.HOST, + port: config.CORE_RPC.PORT, + user: config.CORE_RPC.USERNAME, + pass: config.CORE_RPC.PASSWORD, timeout: 60000, }); } diff --git a/backend/src/api/bitcoin/bitcoin-base.api.ts b/backend/src/api/bitcoin/bitcoin-base.api.ts index a79a1a311..71e7c9093 100644 --- a/backend/src/api/bitcoin/bitcoin-base.api.ts +++ b/backend/src/api/bitcoin/bitcoin-base.api.ts @@ -7,10 +7,10 @@ class BitcoinBaseApi { constructor() { this.bitcoindClient = new bitcoin.Client({ - host: config.BITCOIND.HOST, - port: config.BITCOIND.PORT, - user: config.BITCOIND.USERNAME, - pass: config.BITCOIND.PASSWORD, + host: config.CORE_RPC.HOST, + port: config.CORE_RPC.PORT, + user: config.CORE_RPC.USERNAME, + pass: config.CORE_RPC.PASSWORD, timeout: 60000, }); } diff --git a/backend/src/config.ts b/backend/src/config.ts index 1886ae880..2b7367507 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -18,7 +18,7 @@ interface IConfig { TLS_ENABLED: boolean; TX_LOOKUPS: boolean; }; - BITCOIND: { + CORE_RPC: { HOST: string; PORT: number; USERNAME: string; @@ -71,7 +71,7 @@ const defaults: IConfig = { 'TLS_ENABLED': true, 'TX_LOOKUPS': false }, - 'BITCOIND': { + 'CORE_RPC': { 'HOST': '127.0.0.1', 'PORT': 8332, 'USERNAME': 'mempool', @@ -110,7 +110,7 @@ class Config implements IConfig { MEMPOOL: IConfig['MEMPOOL']; ESPLORA: IConfig['ESPLORA']; ELECTRUM: IConfig['ELECTRUM']; - BITCOIND: IConfig['BITCOIND']; + CORE_RPC: IConfig['CORE_RPC']; DATABASE: IConfig['DATABASE']; STATISTICS: IConfig['STATISTICS']; BISQ_BLOCKS: IConfig['BISQ_BLOCKS']; @@ -122,7 +122,7 @@ class Config implements IConfig { this.MEMPOOL = configs.MEMPOOL; this.ESPLORA = configs.ESPLORA; this.ELECTRUM = configs.ELECTRUM; - this.BITCOIND = configs.BITCOIND; + this.CORE_RPC = configs.CORE_RPC; this.DATABASE = configs.DATABASE; this.STATISTICS = configs.STATISTICS; this.BISQ_BLOCKS = configs.BISQ_BLOCKS; From 0a0e7fad3ac5ab03652a026f1e4f6c8602677552 Mon Sep 17 00:00:00 2001 From: softsimon Date: Mon, 11 Jan 2021 15:23:59 +0700 Subject: [PATCH 30/31] Update backend/package.json Co-authored-by: wiz --- backend/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/package.json b/backend/package.json index b7ac9a333..0d6576fd3 100644 --- a/backend/package.json +++ b/backend/package.json @@ -27,7 +27,6 @@ "test": "echo \"Error: no test specified\" && exit 1" }, "dependencies": { - "@codewarriorr/electrum-client-js": "^0.1.1", "@mempool/bitcoin": "^3.0.2", "@mempool/electrum-client": "^1.1.7", "axios": "^0.21.0", From 3e2a49c08e3340940072d03407de96619aa806eb Mon Sep 17 00:00:00 2001 From: wiz Date: Mon, 11 Jan 2021 17:26:16 +0900 Subject: [PATCH 31/31] Update production/mempool-config.*.json files for new backend config --- production/mempool-config.bisq.json | 13 ++++++++----- production/mempool-config.liquid.json | 12 ++++++++---- production/mempool-config.mainnet.json | 13 ++++++++----- production/mempool-config.testnet.json | 14 +++++++++----- 4 files changed, 33 insertions(+), 19 deletions(-) diff --git a/production/mempool-config.bisq.json b/production/mempool-config.bisq.json index bd00bd0bf..de1c32104 100644 --- a/production/mempool-config.bisq.json +++ b/production/mempool-config.bisq.json @@ -1,17 +1,20 @@ { "MEMPOOL": { "NETWORK": "bisq", + "BACKEND": "esplora", "HTTP_PORT": 8996, "MINED_BLOCKS_CACHE": 144, "SPAWN_CLUSTER_PROCS": 4, "API_URL_PREFIX": "/api/v1/", - "WEBSOCKET_REFRESH_RATE_MS": 2000 - }, - "ELECTRS": { - "ENABLED": true, - "REST_API_URL": "http://[::1]:3000", "POLL_RATE_MS": 2000 }, + "CORE_RPC": { + "USERNAME": "foo", + "PASSWORD": "bar" + }, + "ESPLORA": { + "REST_API_URL": "http://[::1]:3000" + }, "DATABASE": { "ENABLED": false, "HOST": "localhost", diff --git a/production/mempool-config.liquid.json b/production/mempool-config.liquid.json index af2ce910a..5902dee68 100644 --- a/production/mempool-config.liquid.json +++ b/production/mempool-config.liquid.json @@ -1,16 +1,20 @@ { "MEMPOOL": { "NETWORK": "liquid", + "BACKEND": "esplora", "HTTP_PORT": 8998, "MINED_BLOCKS_CACHE": 144, "SPAWN_CLUSTER_PROCS": 0, "API_URL_PREFIX": "/api/v1/", "WEBSOCKET_REFRESH_RATE_MS": 2000 }, - "ELECTRS": { - "ENABLED": true, - "REST_API_URL": "http://[::1]:3001", - "POLL_RATE_MS": 2000 + "CORE_RPC": { + "PORT": 7041, + "USERNAME": "foo", + "PASSWORD": "bar" + }, + "ESPLORA": { + "REST_API_URL": "http://[::1]:3001" }, "DATABASE": { "ENABLED": true, diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index aef3838fc..bb6f7d2e1 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -1,17 +1,20 @@ { "MEMPOOL": { "NETWORK": "mainnet", + "BACKEND": "esplora", "HTTP_PORT": 8999, "MINED_BLOCKS_CACHE": 144, "SPAWN_CLUSTER_PROCS": 0, "API_URL_PREFIX": "/api/v1/", - "WEBSOCKET_REFRESH_RATE_MS": 2000 - }, - "ELECTRS": { - "ENABLED": true, - "REST_API_URL": "http://[::1]:3000", "POLL_RATE_MS": 2000 }, + "CORE_RPC": { + "USERNAME": "foo", + "PASSWORD": "bar" + }, + "ESPLORA": { + "REST_API_URL": "http://[::1]:3000" + }, "DATABASE": { "ENABLED": true, "HOST": "localhost", diff --git a/production/mempool-config.testnet.json b/production/mempool-config.testnet.json index 563c0b9a2..8e0c013ed 100644 --- a/production/mempool-config.testnet.json +++ b/production/mempool-config.testnet.json @@ -1,17 +1,21 @@ { "MEMPOOL": { "NETWORK": "testnet", + "BACKEND": "esplora", "HTTP_PORT": 8997, "MINED_BLOCKS_CACHE": 144, "SPAWN_CLUSTER_PROCS": 0, "API_URL_PREFIX": "/api/v1/", - "WEBSOCKET_REFRESH_RATE_MS": 2000 - }, - "ELECTRS": { - "ENABLED": true, - "REST_API_URL": "http://[::1]:3002", "POLL_RATE_MS": 2000 }, + "CORE_RPC": { + "PORT": 18332, + "USERNAME": "foo", + "PASSWORD": "bar" + }, + "ESPLORA": { + "REST_API_URL": "http://[::1]:3002" + }, "DATABASE": { "ENABLED": true, "HOST": "localhost",