From 4bb812a7e60d86607b2806259c05af7f07ebad28 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 1 Dec 2022 12:05:23 +0100 Subject: [PATCH 01/13] Make sure exchange API response format is valid before using it --- backend/src/tasks/price-feeds/bitfinex-api.ts | 6 +++++- backend/src/tasks/price-feeds/bitflyer-api.ts | 6 +++++- backend/src/tasks/price-feeds/coinbase-api.ts | 6 +++++- backend/src/tasks/price-feeds/gemini-api.ts | 6 +++++- backend/src/tasks/price-feeds/kraken-api.ts | 9 ++++++++- backend/src/tasks/price-updater.ts | 6 +++++- 6 files changed, 33 insertions(+), 6 deletions(-) diff --git a/backend/src/tasks/price-feeds/bitfinex-api.ts b/backend/src/tasks/price-feeds/bitfinex-api.ts index 04bd47732..0e06c3af7 100644 --- a/backend/src/tasks/price-feeds/bitfinex-api.ts +++ b/backend/src/tasks/price-feeds/bitfinex-api.ts @@ -13,7 +13,11 @@ class BitfinexApi implements PriceFeed { public async $fetchPrice(currency): Promise { const response = await query(this.url + currency); - return response ? parseInt(response['last_price'], 10) : -1; + if (response && response['last_price']) { + return parseInt(response['last_price'], 10); + } else { + return -1; + } } public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { diff --git a/backend/src/tasks/price-feeds/bitflyer-api.ts b/backend/src/tasks/price-feeds/bitflyer-api.ts index 143fbe8d9..72b2e6adf 100644 --- a/backend/src/tasks/price-feeds/bitflyer-api.ts +++ b/backend/src/tasks/price-feeds/bitflyer-api.ts @@ -13,7 +13,11 @@ class BitflyerApi implements PriceFeed { public async $fetchPrice(currency): Promise { const response = await query(this.url + currency); - return response ? parseInt(response['ltp'], 10) : -1; + if (response && response['ltp']) { + return parseInt(response['ltp'], 10); + } else { + return -1; + } } public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { diff --git a/backend/src/tasks/price-feeds/coinbase-api.ts b/backend/src/tasks/price-feeds/coinbase-api.ts index ef28b0d80..424ac8867 100644 --- a/backend/src/tasks/price-feeds/coinbase-api.ts +++ b/backend/src/tasks/price-feeds/coinbase-api.ts @@ -13,7 +13,11 @@ class CoinbaseApi implements PriceFeed { public async $fetchPrice(currency): Promise { const response = await query(this.url + currency); - return response ? parseInt(response['data']['amount'], 10) : -1; + if (response && response['data'] && response['data']['amount']) { + return parseInt(response['data']['amount'], 10); + } else { + return -1; + } } public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { diff --git a/backend/src/tasks/price-feeds/gemini-api.ts b/backend/src/tasks/price-feeds/gemini-api.ts index abd8e0939..fc86dc0a3 100644 --- a/backend/src/tasks/price-feeds/gemini-api.ts +++ b/backend/src/tasks/price-feeds/gemini-api.ts @@ -13,7 +13,11 @@ class GeminiApi implements PriceFeed { public async $fetchPrice(currency): Promise { const response = await query(this.url + currency); - return response ? parseInt(response['last'], 10) : -1; + if (response && response['last']) { + return parseInt(response['last'], 10); + } else { + return -1; + } } public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { diff --git a/backend/src/tasks/price-feeds/kraken-api.ts b/backend/src/tasks/price-feeds/kraken-api.ts index ddb3c4f65..cc8fa132c 100644 --- a/backend/src/tasks/price-feeds/kraken-api.ts +++ b/backend/src/tasks/price-feeds/kraken-api.ts @@ -23,7 +23,14 @@ class KrakenApi implements PriceFeed { public async $fetchPrice(currency): Promise { const response = await query(this.url + currency); - return response ? parseInt(response['result'][this.getTicker(currency)]['c'][0], 10) : -1; + const ticker = this.getTicker(currency); + if (response && response['result'] && response['result'][ticker] && + response['result'][ticker]['c'] && response['result'][ticker]['c'].length > 0 + ) { + return parseInt(response['result'][ticker]['c'][0], 10); + } else { + return -1; + } } public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index e069e4db4..e2d00ec7b 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -127,7 +127,11 @@ class PriceUpdater { // Compute average price, non weighted prices = prices.filter(price => price > 0); - this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length); + if (prices.length === 0) { + this.latestPrices[currency] = -1; + } else { + this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length); + } } logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`); From bf25ee478dfd9122f6d45b5e57a92624bc97506c Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Thu, 2 Feb 2023 14:23:08 -0500 Subject: [PATCH 02/13] Update cpfp faq for stored relationships --- frontend/src/app/docs/api-docs/api-docs.component.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/docs/api-docs/api-docs.component.html b/frontend/src/app/docs/api-docs/api-docs.component.html index 48e9b48eb..4e99ce469 100644 --- a/frontend/src/app/docs/api-docs/api-docs.component.html +++ b/frontend/src/app/docs/api-docs/api-docs.component.html @@ -214,8 +214,8 @@

Mempool aims to show you the effective feerate range for blocks—how much would you actually need to pay to get a transaction included in a block.

-

A transaction's effective feerate is not always the same as the feerate explicitly set for it. For example, if you see a 1 s/vb transaction in a block with a displayed feerate range of 5 s/vb to 72 s/vb, chances are that 1 s/vb transaction had a high-feerate child transaction that boosted its effective feerate to 5 s/vb or higher (this is how CPFP fee-bumping works). In such a case, it would be misleading to use 1 s/vb as the lower bound of the block's feerate range because it actually required more than 1 s/vb to confirm that transaction in that block.

-

For unconfirmed CPFP transactions, Mempool will show the effective feerate (along with descendent & ancestor transaction information) on the transaction page. For confirmed transactions, CPFP relationships are not stored, so this additional information is not shown.

+

A transaction's effective feerate is not always the same as the feerate explicitly set for it. For example, if you see a 1 s/vb transaction in a block with a displayed feerate range of 5 s/vb to 72 s/vb, chances are that 1 s/vb transaction had a high-feerate child transaction that boosted its effective feerate to 5 s/vb or higher (this is how CPFP fee-bumping works). In such a case, it would be misleading to use 1 s/vb as the lower bound of the block's feerate range since it actually required more than 1 s/vb to confirm that transaction in that block.

+

You can find a transaction's feerate on its transaction details page. If the transaction has any CPFP relationships, the page will also show the transaction's effective feerate along with links to descendent and/or ancestor transactions.

From 7755b7b4a34633aca49b4e80de4fbf635e3054ae Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Mon, 13 Feb 2023 14:58:35 +0900 Subject: [PATCH 03/13] Add `--reindex` doc to backend README --- backend/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/backend/README.md b/backend/README.md index 3d7c23eaa..d00dc1812 100644 --- a/backend/README.md +++ b/backend/README.md @@ -218,3 +218,21 @@ Generate block at regular interval (every 10 seconds in this example): ``` watch -n 10 "./src/bitcoin-cli -regtest -rpcport=8332 generatetoaddress 1 $address" ``` + +### Re-index tables + +You can manually force the nodejs backend to drop all data from a specified set of tables for future re-index. This is mostly useful for the mining dashboard and the lightning explorer. + +Use the `--reindex` command to specify a list of comma separated table which will be truncated at start. Note that a 5 seconds delay will be observed before truncating tables in order to give you a chance to cancel (CTRL+C) in case of misuse of the command. + +Usage: +``` +npm run start --reindex=blocks,hashrates +``` +Example output: +``` +Feb 13 14:55:27 [63246] WARN: Indexed data for "hashrates" tables will be erased in 5 seconds (using '--reindex') +Feb 13 14:55:32 [63246] NOTICE: Table hashrates has been truncated +``` + +Reference: https://github.com/mempool/mempool/pull/1269 \ No newline at end of file From 76b899031a1c2c43df9c2997be9323a68c427691 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Mon, 13 Feb 2023 15:50:22 +0900 Subject: [PATCH 04/13] Fix missing fee range in blocks api when querying non indexed blocks --- backend/src/utils/blocks-utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/utils/blocks-utils.ts b/backend/src/utils/blocks-utils.ts index b933d6ae7..43a2fc964 100644 --- a/backend/src/utils/blocks-utils.ts +++ b/backend/src/utils/blocks-utils.ts @@ -17,7 +17,7 @@ export function prepareBlock(block: any): BlockExtended { extras: { coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw, medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee, - feeRange: block.feeRange ?? block.fee_span, + feeRange: block.feeRange ?? block?.extras?.feeRange ?? block.fee_span, reward: block.reward ?? block?.extras?.reward, totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees, avgFee: block?.extras?.avgFee ?? block.avg_fee, From 158d02b60d56e8fdc997bab97d1a22365ed36435 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Tue, 14 Feb 2023 22:12:43 +0900 Subject: [PATCH 05/13] Always return fully extended block in blocks API even if indexing is disabled --- backend/src/api/blocks.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 83de897ca..1d45d6c58 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -609,7 +609,9 @@ class Blocks { const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); - await blocksRepository.$saveBlockInDatabase(blockExtended); + if (Common.indexingEnabled()) { + await blocksRepository.$saveBlockInDatabase(blockExtended); + } return prepareBlock(blockExtended); } @@ -713,7 +715,7 @@ class Blocks { block = await this.$indexBlock(currentHeight); returnBlocks.push(block); } else if (nextHash != null) { - block = prepareBlock(await bitcoinClient.getBlock(nextHash)); + block = await this.$indexBlock(currentHeight); nextHash = block.previousblockhash; returnBlocks.push(block); } From 87202ea9f831fce47885260bbc72c755a94fc9c9 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 15 Feb 2023 09:26:12 -0600 Subject: [PATCH 06/13] fix liquid address table overflow --- .../components/address/address.component.html | 4 ++-- .../components/address/address.component.scss | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/components/address/address.component.html b/frontend/src/app/components/address/address.component.html index d3b23315a..ae2d7ba9c 100644 --- a/frontend/src/app/components/address/address.component.html +++ b/frontend/src/app/components/address/address.component.html @@ -1,4 +1,4 @@ -
+

Address