From 3be67ea023800dc601213925c1451e89e81f6ce1 Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 12 May 2022 03:28:34 +0400 Subject: [PATCH 01/12] Reset outspends cache when switching to new tx page. fixes #1613 fixes #1164 --- frontend/src/app/components/block/block.component.html | 2 +- .../transactions-list/transactions-list.component.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/block/block.component.html b/frontend/src/app/components/block/block.component.html index 8b511b30c..6a227f29e 100644 --- a/frontend/src/app/components/block/block.component.html +++ b/frontend/src/app/components/block/block.component.html @@ -173,7 +173,7 @@
- +
diff --git a/frontend/src/app/components/transactions-list/transactions-list.component.ts b/frontend/src/app/components/transactions-list/transactions-list.component.ts index e9385c501..b4ef42d84 100644 --- a/frontend/src/app/components/transactions-list/transactions-list.component.ts +++ b/frontend/src/app/components/transactions-list/transactions-list.component.ts @@ -22,6 +22,7 @@ export class TransactionsListComponent implements OnInit, OnChanges { @Input() showConfirmations = false; @Input() transactionPage = false; @Input() errorUnblinded = false; + @Input() paginated = false; @Input() outputIndex: number; @Input() address: string = ''; @@ -84,6 +85,9 @@ export class TransactionsListComponent implements OnInit, OnChanges { if (!this.transactions || !this.transactions.length) { return; } + if (this.paginated) { + this.outspends = []; + } if (this.outputIndex) { setTimeout(() => { const assetBoxElements = document.getElementsByClassName('assetBox'); From a8d8a360ecd6e6f5c255e322dec3af132bdffb78 Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 12 May 2022 15:21:27 +0400 Subject: [PATCH 02/12] Tweaking the websocket retry timers fixes 1560 --- frontend/src/app/services/websocket.service.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/services/websocket.service.ts b/frontend/src/app/services/websocket.service.ts index 464b4dc1a..0a183557b 100644 --- a/frontend/src/app/services/websocket.service.ts +++ b/frontend/src/app/services/websocket.service.ts @@ -9,9 +9,9 @@ import { take } from 'rxjs/operators'; import { TransferState, makeStateKey } from '@angular/platform-browser'; import { BlockExtended } from '../interfaces/node-api.interface'; -const OFFLINE_RETRY_AFTER_MS = 10000; -const OFFLINE_PING_CHECK_AFTER_MS = 30000; -const EXPECT_PING_RESPONSE_AFTER_MS = 4000; +const OFFLINE_RETRY_AFTER_MS = 1000; +const OFFLINE_PING_CHECK_AFTER_MS = 10000; +const EXPECT_PING_RESPONSE_AFTER_MS = 5000; const initData = makeStateKey('/api/v1/init-data'); From c505ed5f0b00a5ce56b691c0aaca21733e841808 Mon Sep 17 00:00:00 2001 From: wiz Date: Tue, 17 May 2022 17:11:20 +0900 Subject: [PATCH 03/12] [installer] Configure nvm to install Node.js v16.15.0 / npm v8.5.5 --- production/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/install b/production/install index 0fc9becb6..b40d868bf 100755 --- a/production/install +++ b/production/install @@ -854,7 +854,7 @@ echo "[*] Installing nvm.sh from GitHub" osSudo "${MEMPOOL_USER}" sh -c 'curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | zsh' echo "[*] Building NodeJS via nvm.sh" -osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm install v16.10.0' +osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm install v16.15.0' #################### # Tor installation # From d2e041ec6515c039b84125a7a460ad7843b21f10 Mon Sep 17 00:00:00 2001 From: wiz Date: Tue, 17 May 2022 17:12:13 +0900 Subject: [PATCH 04/12] [installer] Fix electrs patch sed pattern typo introduced in #1628 --- production/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/install b/production/install index b40d868bf..8c46f568e 100755 --- a/production/install +++ b/production/install @@ -1016,7 +1016,7 @@ osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --releas echo "[*] Patching Bitcoin Electrs code for FreeBSD" osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_HOME}/.cargo/registry/src/github.com-1ecc6299db9ec823/sysconf-0.3.4\" && patch -p1 < \"${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/freebsd/sysconf.patch\"" osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_ELECTRS_HOME}/src/new_index/\" && sed -i .bak -e s/Snappy/None/ db.rs && rm db.rs.bak" -osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_ELECTRS_HOME}/src/bin/\" && sed -i .bak -e s/from_secs(5)/from_secs(1)/ electrs.rs && rm electrs.rs.bak" +osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_ELECTRS_HOME}/src/bin/\" && sed -i .bak -e 's/from_secs(5)/from_secs(1)/' electrs.rs && rm electrs.rs.bak" echo "[*] Building Bitcoin Electrs release binary" osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --release --bin electrs -- --version" From 5698468fbf1c9613cffc7f7c23e03b09e2bfd86d Mon Sep 17 00:00:00 2001 From: wiz Date: Tue, 17 May 2022 17:25:10 +0900 Subject: [PATCH 05/12] [installer] Checkout top-level mempool using master branch --- production/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/install b/production/install index 8c46f568e..68978399e 100755 --- a/production/install +++ b/production/install @@ -276,7 +276,7 @@ MINFEE_HOME=/minfee MEMPOOL_REPO_URL=https://github.com/mempool/mempool MEMPOOL_REPO_NAME=mempool MEMPOOL_REPO_BRANCH=master -MEMPOOL_LATEST_RELEASE=v2.3.1 +MEMPOOL_LATEST_RELEASE=master BITCOIN_REPO_URL=https://github.com/bitcoin/bitcoin BITCOIN_REPO_NAME=bitcoin From 30337095cf18ec736e5d1cfb49288e5feed09b43 Mon Sep 17 00:00:00 2001 From: wiz Date: Tue, 17 May 2022 17:26:05 +0900 Subject: [PATCH 06/12] [prod] Clarify syslog.conf syntax for priorities to keybase --- production/syslog.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/production/syslog.conf b/production/syslog.conf index 9a2a382fe..1a36100dd 100644 --- a/production/syslog.conf +++ b/production/syslog.conf @@ -1,4 +1,4 @@ local7.>=notice |/usr/local/bin/sudo -u mempool /usr/local/bin/mempool-logger mempool.ops alerts -local7.info |/usr/local/bin/sudo -u mempool /usr/local/bin/mempool-logger mempool.ops node100 -local7.info /var/log/mempool +local7.>=info |/usr/local/bin/sudo -u mempool /usr/local/bin/mempool-logger mempool.ops node100 +local7.>=info /var/log/mempool local7.* /var/log/mempool.debug From ece1cbbaa9c50f57d3b01a0eb5ca46030abaa28a Mon Sep 17 00:00:00 2001 From: wiz Date: Tue, 17 May 2022 17:43:35 +0900 Subject: [PATCH 07/12] [installer] Fix mempool checkout for bisq instance --- production/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/install b/production/install index 68978399e..f2cb17025 100755 --- a/production/install +++ b/production/install @@ -1299,7 +1299,7 @@ if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" fi -if [ "${BISQ_ENABLE}" = ON ];then +if [ "${BISQ_INSTALL}" = ON ];then echo "[*] Creating Mempool instance for Bisq" osSudo "${MEMPOOL_USER}" git config --global advice.detachedHead false osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/bisq" From 47bb073b9a567ce72e5828d75524356424389944 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 17 May 2022 14:40:18 +0200 Subject: [PATCH 08/12] Define avg block time to 10 min until we have at least 146 block in the current epoch --- backend/src/api/difficulty-adjustment.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/difficulty-adjustment.ts b/backend/src/api/difficulty-adjustment.ts index eea0e9b42..1f85fdb80 100644 --- a/backend/src/api/difficulty-adjustment.ts +++ b/backend/src/api/difficulty-adjustment.ts @@ -32,7 +32,7 @@ class DifficultyAdjustmentApi { } } - let timeAvgMins = blocksInEpoch ? diff / blocksInEpoch / 60 : 10; + let timeAvgMins = blocksInEpoch && blocksInEpoch > 146 ? diff / blocksInEpoch / 60 : 10; // Testnet difficulty is set to 1 after 20 minutes of no blocks, // therefore the time between blocks will always be below 20 minutes (1200s). From 0b351b9fcb56f0c9b3e5cda891070ed703fba1c1 Mon Sep 17 00:00:00 2001 From: Ayanami Date: Sun, 15 May 2022 20:53:04 +0900 Subject: [PATCH 09/12] pools-updater: Support secure Tor connection to sync data with Github Use Axios instead of native https --- backend/src/tasks/pools-updater.ts | 94 +++++++++++++++++------------- 1 file changed, 55 insertions(+), 39 deletions(-) diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index aee786ff9..aac301256 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -1,8 +1,10 @@ -const https = require('https'); +import axios from 'axios'; import poolsParser from '../api/pools-parser'; import config from '../config'; import DB from '../database'; import logger from '../logger'; +import { SocksProxyAgent } from 'socks-proxy-agent'; +import * as https from 'https'; /** * Maintain the most recent version of pools.json @@ -28,6 +30,13 @@ class PoolsUpdater { this.lastRun = now; + logger.info('Updating latest mining pools from Github'); + if (config.SOCKS5PROXY.ENABLED) { + logger.info('List of public pools will be queried over the Tor network'); + } else { + logger.info('List of public pools will be queried over clearnet'); + } + try { const dbSha = await this.getShaFromDb(); const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github @@ -41,7 +50,10 @@ class PoolsUpdater { } logger.warn('Pools.json is outdated, fetch latest from github'); - const poolsJson = await this.fetchPools(); + const poolsJson = await this.query('https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json'); + if (poolsJson === undefined) { + return; + } await poolsParser.migratePoolsJson(poolsJson); await this.updateDBSha(githubSha); logger.notice('PoolsUpdater completed'); @@ -52,14 +64,6 @@ class PoolsUpdater { } } - /** - * Fetch pools.json from github repo - */ - private async fetchPools(): Promise { - const response = await this.query('/repos/mempool/mining-pools/contents/pools.json'); - return JSON.parse(Buffer.from(response['content'], 'base64').toString('utf8')); - } - /** * Fetch our latest pools.json sha from the db */ @@ -90,11 +94,13 @@ class PoolsUpdater { * Fetch our latest pools.json sha from github */ private async fetchPoolsSha(): Promise { - const response = await this.query('/repos/mempool/mining-pools/git/trees/master'); + const response = await this.query('https://api.github.com/repos/mempool/mining-pools/git/trees/master'); - for (const file of response['tree']) { - if (file['path'] === 'pools.json') { - return file['sha']; + if (response !== undefined) { + for (const file of response['tree']) { + if (file['path'] === 'pools.json') { + return file['sha']; + } } } @@ -105,35 +111,45 @@ class PoolsUpdater { /** * Http request wrapper */ - private query(path): Promise { - return new Promise((resolve, reject) => { - const options = { - host: 'api.github.com', - path: path, - method: 'GET', - headers: { 'user-agent': 'node.js' } + private async query(path): Promise { + type axiosOptions = { + httpsAgent?: https.Agent; + } + const setDelay = (secs: number = 1): Promise => new Promise(resolve => setTimeout(() => resolve(), secs * 1000)); + const axiosOptions: axiosOptions = {}; + let retry = 0; + + if (config.SOCKS5PROXY.ENABLED) { + const socksOptions: any = { + agentOptions: { + keepAlive: true, + }, + hostname: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT }; - logger.debug('Querying: api.github.com' + path); + if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { + socksOptions.username = config.SOCKS5PROXY.USERNAME; + socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } - const request = https.get(options, (response) => { - const chunks_of_data: any[] = []; - response.on('data', (fragments) => { - chunks_of_data.push(fragments); - }); - response.on('end', () => { - resolve(JSON.parse(Buffer.concat(chunks_of_data).toString())); - }); - response.on('error', (error) => { - reject(error); - }); - }); + axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); + } - request.on('error', (error) => { - logger.err('Github API query failed. Reason: ' + error); - reject(error); - }); - }); + while(retry < 5) { + try { + const data = await axios.get(path, axiosOptions); + if (data.statusText !== 'OK' || !data.data) { + throw new Error(`Could not fetch data from Github, Error: ${data.status}`); + } + return data.data; + } catch (e) { + logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e)); + retry++; + } + await setDelay(); + } + return undefined; } } From d1671c4f1b3539dd3dda3fd4f3e6cf7d2fe51309 Mon Sep 17 00:00:00 2001 From: Ayanami Date: Sun, 15 May 2022 22:17:50 +0900 Subject: [PATCH 10/12] add contributor sign for ayanamidev --- contributors/ayanamidev.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 contributors/ayanamidev.txt diff --git a/contributors/ayanamidev.txt b/contributors/ayanamidev.txt new file mode 100644 index 000000000..c397f7286 --- /dev/null +++ b/contributors/ayanamidev.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of May 15, 2022. + +Signed: ayanamidev From fdb035c0d244de15e512906ea8c3e8227311cf9e Mon Sep 17 00:00:00 2001 From: Ayanami Date: Wed, 18 May 2022 11:23:51 +0900 Subject: [PATCH 11/12] Remove unused config from nginx --- nginx-mempool.conf | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/nginx-mempool.conf b/nginx-mempool.conf index 336f4efde..58d45f3cc 100644 --- a/nginx-mempool.conf +++ b/nginx-mempool.conf @@ -44,25 +44,6 @@ try_files $uri $uri/ /en-US/index.html =404; } - # mainnet API - location /api/v1/donations { - proxy_pass https://mempool.space; - } - location /api/v1/donations/images { - proxy_pass https://mempool.space; - } - location /api/v1/contributors { - proxy_pass https://mempool.space; - } - location /api/v1/contributors/images { - proxy_pass https://mempool.space; - } - location /api/v1/translators { - proxy_pass https://mempool.space; - } - location /api/v1/translators/images { - proxy_pass https://mempool.space; - } location /api/v1/ws { proxy_pass http://127.0.0.1:8999/; proxy_http_version 1.1; From 2dad8ba8ece37a577c9281b6150953f171d3406b Mon Sep 17 00:00:00 2001 From: softsimon Date: Wed, 18 May 2022 04:45:42 +0400 Subject: [PATCH 12/12] Fix for transactions being fetched recursively --- .../bitcoin/bitcoin-api-abstract-factory.ts | 2 +- backend/src/api/bitcoin/bitcoin-api.ts | 62 +++++++++---------- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index 266be5f1e..53c731e1f 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -2,7 +2,7 @@ import { IEsploraApi } from './esplora-api.interface'; export interface AbstractBitcoinApi { $getRawMempool(): Promise; - $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, blockHash?: string): Promise; + $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean): Promise; $getBlockHeightTip(): Promise; $getTxIdsForBlock(hash: string): Promise; $getBlockHash(height: number): Promise; diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 51ed99b6c..48368a128 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -14,14 +14,31 @@ class BitcoinApi implements AbstractBitcoinApi { this.bitcoindClient = bitcoinClient; } - $getRawTransaction(txId: string, skipConversion = false, addPrevout = false, blockHash?: string): Promise { + static convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block { + return { + id: block.hash, + height: block.height, + version: block.version, + timestamp: block.time, + bits: parseInt(block.bits, 16), + nonce: block.nonce, + difficulty: block.difficulty, + merkle_root: block.merkleroot, + tx_count: block.nTx, + size: block.size, + weight: block.weight, + previousblockhash: block.previousblockhash, + }; + } + + $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise { // If the transaction is in the mempool we already converted and fetched the fee. Only prevouts are missing const txInMempool = mempool.getMempool()[txId]; if (txInMempool && addPrevout) { return this.$addPrevouts(txInMempool); } - return this.bitcoindClient.getRawTransaction(txId, true, blockHash) + return this.bitcoindClient.getRawTransaction(txId, true) .then((transaction: IBitcoinApi.Transaction) => { if (skipConversion) { transaction.vout.forEach((vout) => { @@ -174,35 +191,18 @@ class BitcoinApi implements AbstractBitcoinApi { }; } - if (transaction.confirmations) { - esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, addPrevout); - } else { - esploraTransaction = await this.$appendMempoolFeeData(esploraTransaction); - if (addPrevout) { - esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, addPrevout); + if (addPrevout) { + if (transaction.confirmations) { + esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction); + } else { + esploraTransaction = await this.$appendMempoolFeeData(esploraTransaction); + esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction); } } return esploraTransaction; } - static convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block { - return { - id: block.hash, - height: block.height, - version: block.version, - timestamp: block.time, - bits: parseInt(block.bits, 16), - nonce: block.nonce, - difficulty: block.difficulty, - merkle_root: block.merkleroot, - tx_count: block.nTx, - size: block.size, - weight: block.weight, - previousblockhash: block.previousblockhash, - }; - } - private translateScriptPubKeyType(outputType: string): string { const map = { 'pubkey': 'p2pk', @@ -245,7 +245,7 @@ class BitcoinApi implements AbstractBitcoinApi { if (vin.prevout) { continue; } - const innerTx = await this.$getRawTransaction(vin.txid, false); + const innerTx = await this.$getRawTransaction(vin.txid, false, false); vin.prevout = innerTx.vout[vin.vout]; this.addInnerScriptsToVin(vin); } @@ -271,18 +271,16 @@ class BitcoinApi implements AbstractBitcoinApi { return this.bitcoindClient.getRawMemPool(true); } - private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean): Promise { + private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction): Promise { if (transaction.vin[0].is_coinbase) { transaction.fee = 0; return transaction; } let totalIn = 0; for (const vin of transaction.vin) { - const innerTx = await this.$getRawTransaction(vin.txid, !addPrevout); - if (addPrevout) { - vin.prevout = innerTx.vout[vin.vout]; - this.addInnerScriptsToVin(vin); - } + const innerTx = await this.$getRawTransaction(vin.txid, false, false); + vin.prevout = innerTx.vout[vin.vout]; + this.addInnerScriptsToVin(vin); totalIn += innerTx.vout[vin.vout].value; } const totalOut = transaction.vout.reduce((p, output) => p + output.value, 0);