From a671bfc226e32bdd296f7706aa95f30d48404f19 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 1 Dec 2022 12:05:23 +0100 Subject: [PATCH 01/21] Make sure exchange API response format is valid before using it --- backend/src/tasks/price-feeds/bitfinex-api.ts | 6 +++++- backend/src/tasks/price-feeds/bitflyer-api.ts | 6 +++++- backend/src/tasks/price-feeds/coinbase-api.ts | 6 +++++- backend/src/tasks/price-feeds/gemini-api.ts | 6 +++++- backend/src/tasks/price-feeds/kraken-api.ts | 9 ++++++++- backend/src/tasks/price-updater.ts | 6 +++++- 6 files changed, 33 insertions(+), 6 deletions(-) diff --git a/backend/src/tasks/price-feeds/bitfinex-api.ts b/backend/src/tasks/price-feeds/bitfinex-api.ts index 04bd47732..0e06c3af7 100644 --- a/backend/src/tasks/price-feeds/bitfinex-api.ts +++ b/backend/src/tasks/price-feeds/bitfinex-api.ts @@ -13,7 +13,11 @@ class BitfinexApi implements PriceFeed { public async $fetchPrice(currency): Promise { const response = await query(this.url + currency); - return response ? parseInt(response['last_price'], 10) : -1; + if (response && response['last_price']) { + return parseInt(response['last_price'], 10); + } else { + return -1; + } } public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { diff --git a/backend/src/tasks/price-feeds/bitflyer-api.ts b/backend/src/tasks/price-feeds/bitflyer-api.ts index 143fbe8d9..72b2e6adf 100644 --- a/backend/src/tasks/price-feeds/bitflyer-api.ts +++ b/backend/src/tasks/price-feeds/bitflyer-api.ts @@ -13,7 +13,11 @@ class BitflyerApi implements PriceFeed { public async $fetchPrice(currency): Promise { const response = await query(this.url + currency); - return response ? parseInt(response['ltp'], 10) : -1; + if (response && response['ltp']) { + return parseInt(response['ltp'], 10); + } else { + return -1; + } } public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { diff --git a/backend/src/tasks/price-feeds/coinbase-api.ts b/backend/src/tasks/price-feeds/coinbase-api.ts index ef28b0d80..424ac8867 100644 --- a/backend/src/tasks/price-feeds/coinbase-api.ts +++ b/backend/src/tasks/price-feeds/coinbase-api.ts @@ -13,7 +13,11 @@ class CoinbaseApi implements PriceFeed { public async $fetchPrice(currency): Promise { const response = await query(this.url + currency); - return response ? parseInt(response['data']['amount'], 10) : -1; + if (response && response['data'] && response['data']['amount']) { + return parseInt(response['data']['amount'], 10); + } else { + return -1; + } } public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { diff --git a/backend/src/tasks/price-feeds/gemini-api.ts b/backend/src/tasks/price-feeds/gemini-api.ts index abd8e0939..fc86dc0a3 100644 --- a/backend/src/tasks/price-feeds/gemini-api.ts +++ b/backend/src/tasks/price-feeds/gemini-api.ts @@ -13,7 +13,11 @@ class GeminiApi implements PriceFeed { public async $fetchPrice(currency): Promise { const response = await query(this.url + currency); - return response ? parseInt(response['last'], 10) : -1; + if (response && response['last']) { + return parseInt(response['last'], 10); + } else { + return -1; + } } public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { diff --git a/backend/src/tasks/price-feeds/kraken-api.ts b/backend/src/tasks/price-feeds/kraken-api.ts index ddb3c4f65..cc8fa132c 100644 --- a/backend/src/tasks/price-feeds/kraken-api.ts +++ b/backend/src/tasks/price-feeds/kraken-api.ts @@ -23,7 +23,14 @@ class KrakenApi implements PriceFeed { public async $fetchPrice(currency): Promise { const response = await query(this.url + currency); - return response ? parseInt(response['result'][this.getTicker(currency)]['c'][0], 10) : -1; + const ticker = this.getTicker(currency); + if (response && response['result'] && response['result'][ticker] && + response['result'][ticker]['c'] && response['result'][ticker]['c'].length > 0 + ) { + return parseInt(response['result'][ticker]['c'][0], 10); + } else { + return -1; + } } public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index e069e4db4..e2d00ec7b 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -127,7 +127,11 @@ class PriceUpdater { // Compute average price, non weighted prices = prices.filter(price => price > 0); - this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length); + if (prices.length === 0) { + this.latestPrices[currency] = -1; + } else { + this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length); + } } logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`); From b269e99cfb7a48d5ccb32e2f177f3fae6656454b Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Thu, 2 Feb 2023 14:23:08 -0500 Subject: [PATCH 02/21] Update cpfp faq for stored relationships --- frontend/src/app/docs/api-docs/api-docs.component.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/docs/api-docs/api-docs.component.html b/frontend/src/app/docs/api-docs/api-docs.component.html index 48e9b48eb..4e99ce469 100644 --- a/frontend/src/app/docs/api-docs/api-docs.component.html +++ b/frontend/src/app/docs/api-docs/api-docs.component.html @@ -214,8 +214,8 @@

Mempool aims to show you the effective feerate range for blocks—how much would you actually need to pay to get a transaction included in a block.

-

A transaction's effective feerate is not always the same as the feerate explicitly set for it. For example, if you see a 1 s/vb transaction in a block with a displayed feerate range of 5 s/vb to 72 s/vb, chances are that 1 s/vb transaction had a high-feerate child transaction that boosted its effective feerate to 5 s/vb or higher (this is how CPFP fee-bumping works). In such a case, it would be misleading to use 1 s/vb as the lower bound of the block's feerate range because it actually required more than 1 s/vb to confirm that transaction in that block.

-

For unconfirmed CPFP transactions, Mempool will show the effective feerate (along with descendent & ancestor transaction information) on the transaction page. For confirmed transactions, CPFP relationships are not stored, so this additional information is not shown.

+

A transaction's effective feerate is not always the same as the feerate explicitly set for it. For example, if you see a 1 s/vb transaction in a block with a displayed feerate range of 5 s/vb to 72 s/vb, chances are that 1 s/vb transaction had a high-feerate child transaction that boosted its effective feerate to 5 s/vb or higher (this is how CPFP fee-bumping works). In such a case, it would be misleading to use 1 s/vb as the lower bound of the block's feerate range since it actually required more than 1 s/vb to confirm that transaction in that block.

+

You can find a transaction's feerate on its transaction details page. If the transaction has any CPFP relationships, the page will also show the transaction's effective feerate along with links to descendent and/or ancestor transactions.

From 0176afa394b52c6a9b6b01272a4c882c95cc6ac2 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Mon, 13 Feb 2023 14:58:35 +0900 Subject: [PATCH 03/21] Add `--reindex` doc to backend README --- backend/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/backend/README.md b/backend/README.md index 3d7c23eaa..d00dc1812 100644 --- a/backend/README.md +++ b/backend/README.md @@ -218,3 +218,21 @@ Generate block at regular interval (every 10 seconds in this example): ``` watch -n 10 "./src/bitcoin-cli -regtest -rpcport=8332 generatetoaddress 1 $address" ``` + +### Re-index tables + +You can manually force the nodejs backend to drop all data from a specified set of tables for future re-index. This is mostly useful for the mining dashboard and the lightning explorer. + +Use the `--reindex` command to specify a list of comma separated table which will be truncated at start. Note that a 5 seconds delay will be observed before truncating tables in order to give you a chance to cancel (CTRL+C) in case of misuse of the command. + +Usage: +``` +npm run start --reindex=blocks,hashrates +``` +Example output: +``` +Feb 13 14:55:27 [63246] WARN: Indexed data for "hashrates" tables will be erased in 5 seconds (using '--reindex') +Feb 13 14:55:32 [63246] NOTICE: Table hashrates has been truncated +``` + +Reference: https://github.com/mempool/mempool/pull/1269 \ No newline at end of file From cf720c4beff98b8619816eccebd788976f5be513 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Mon, 13 Feb 2023 15:50:22 +0900 Subject: [PATCH 04/21] Fix missing fee range in blocks api when querying non indexed blocks --- backend/src/utils/blocks-utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/utils/blocks-utils.ts b/backend/src/utils/blocks-utils.ts index b933d6ae7..43a2fc964 100644 --- a/backend/src/utils/blocks-utils.ts +++ b/backend/src/utils/blocks-utils.ts @@ -17,7 +17,7 @@ export function prepareBlock(block: any): BlockExtended { extras: { coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw, medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee, - feeRange: block.feeRange ?? block.fee_span, + feeRange: block.feeRange ?? block?.extras?.feeRange ?? block.fee_span, reward: block.reward ?? block?.extras?.reward, totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees, avgFee: block?.extras?.avgFee ?? block.avg_fee, From 771825f2242b8ce70f1c9a2bec3ed745cdadd149 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Mon, 13 Feb 2023 16:57:39 -0800 Subject: [PATCH 05/21] Add offset to blockchain blocks classes and locators --- .../blockchain-blocks.component.html | 66 ++++++++++++------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html index 29df378a4..17c5709fd 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html @@ -1,64 +1,86 @@ -
+
-
+
 
-
- ~{{ block?.extras?.medianFee | number:feeRounding }} sat/vB +
+ ~{{ block?.extras?.medianFee | number:feeRounding }} sat/vB
-
- {{ block?.extras?.feeRange?.[1] | number:feeRounding }} - {{ block?.extras?.feeRange[block?.extras?.feeRange?.length - 1] | number:feeRounding }} sat/vB +
+ {{ block?.extras?.feeRange?.[1] | number:feeRounding }} - {{ + block?.extras?.feeRange[block?.extras?.feeRange?.length - 1] | number:feeRounding }} sat/vB
-
+
 
-
+
-
+
- - {{ i }} transaction - {{ i }} transactions + + {{ i }} + transaction + {{ i }} + transactions
-
+
+
-
- +
+
-
+
-
+
-
-
+
+
- From 635fadd13f39b83deba8953fe7cdf58b958645d0 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Mon, 13 Feb 2023 16:58:25 -0800 Subject: [PATCH 06/21] Update mainnet tests: increase blocks to 22, update locators and skip a test --- frontend/cypress/e2e/mainnet/mainnet.spec.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/frontend/cypress/e2e/mainnet/mainnet.spec.ts b/frontend/cypress/e2e/mainnet/mainnet.spec.ts index d6fe94dac..5ab3f9ce9 100644 --- a/frontend/cypress/e2e/mainnet/mainnet.spec.ts +++ b/frontend/cypress/e2e/mainnet/mainnet.spec.ts @@ -64,7 +64,7 @@ describe('Mainnet', () => { it('loads the status screen', () => { cy.visit('/status'); cy.get('#mempool-block-0').should('be.visible'); - cy.get('[id^="bitcoin-block-"]').should('have.length', 8); + cy.get('[id^="bitcoin-block-"]').should('have.length', 22); cy.get('.footer').should('be.visible'); cy.get('.row > :nth-child(1)').invoke('text').then((text) => { expect(text).to.match(/Incoming transactions.* vB\/s/); @@ -219,11 +219,11 @@ describe('Mainnet', () => { describe('blocks navigation', () => { describe('keyboard events', () => { - it('loads first blockchain blocks visible and keypress arrow right', () => { + it('loads first blockchain block visible and keypress arrow right', () => { cy.viewport('macbook-16'); cy.visit('/'); cy.waitForSkeletonGone(); - cy.get('.blockchain-blocks-0 > a').click().then(() => { + cy.get('[data-cy="bitcoin-block-offset-0-index-0"]').click().then(() => { cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('not.exist'); cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible'); cy.waitForPageIdle(); @@ -233,11 +233,11 @@ describe('Mainnet', () => { }); }); - it('loads first blockchain blocks visible and keypress arrow left', () => { + it('loads first blockchain block visible and keypress arrow left', () => { cy.viewport('macbook-16'); cy.visit('/'); cy.waitForSkeletonGone(); - cy.get('.blockchain-blocks-0 > a').click().then(() => { + cy.get('[data-cy="bitcoin-block-offset-0-index-0"]').click().then(() => { cy.waitForPageIdle(); cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('not.exist'); cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible'); @@ -246,11 +246,11 @@ describe('Mainnet', () => { }); }); - it('loads last blockchain blocks and keypress arrow right', () => { + it.skip('loads last blockchain block and keypress arrow right', () => { //Skip for now as "last" doesn't really work with infinite scrolling cy.viewport('macbook-16'); cy.visit('/'); cy.waitForSkeletonGone(); - cy.get('.blockchain-blocks-4 > a').click().then(() => { + cy.get('bitcoin-block-offset-0-index-7').click().then(() => { cy.waitForPageIdle(); // block 6 @@ -309,7 +309,7 @@ describe('Mainnet', () => { cy.viewport('macbook-16'); cy.visit('/'); cy.waitForSkeletonGone(); - cy.get('.blockchain-blocks-0 > a').click().then(() => { + cy.get('[data-cy="bitcoin-block-offset-0-index-0"]').click().then(() => { cy.waitForPageIdle(); cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('not.exist'); cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible'); From 56dad33fce4aa6c887be5ac9e9b9d3328ae89d7b Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Tue, 14 Feb 2023 22:12:43 +0900 Subject: [PATCH 07/21] Always return fully extended block in blocks API even if indexing is disabled --- backend/src/api/blocks.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 83de897ca..1d45d6c58 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -609,7 +609,9 @@ class Blocks { const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); - await blocksRepository.$saveBlockInDatabase(blockExtended); + if (Common.indexingEnabled()) { + await blocksRepository.$saveBlockInDatabase(blockExtended); + } return prepareBlock(blockExtended); } @@ -713,7 +715,7 @@ class Blocks { block = await this.$indexBlock(currentHeight); returnBlocks.push(block); } else if (nextHash != null) { - block = prepareBlock(await bitcoinClient.getBlock(nextHash)); + block = await this.$indexBlock(currentHeight); nextHash = block.previousblockhash; returnBlocks.push(block); } From cc5873f9956933e349e4426d5ce5b4f8f3830905 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 15 Feb 2023 09:26:12 -0600 Subject: [PATCH 08/21] fix liquid address table overflow --- .../components/address/address.component.html | 4 ++-- .../components/address/address.component.scss | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/components/address/address.component.html b/frontend/src/app/components/address/address.component.html index d3b23315a..ae2d7ba9c 100644 --- a/frontend/src/app/components/address/address.component.html +++ b/frontend/src/app/components/address/address.component.html @@ -1,4 +1,4 @@ -
+

Address