diff --git a/.github/workflows/on-tag.yml b/.github/workflows/on-tag.yml index 32d4b085b..5d8d71104 100644 --- a/.github/workflows/on-tag.yml +++ b/.github/workflows/on-tag.yml @@ -31,7 +31,7 @@ jobs: run: | sudo swapoff /mnt/swapfile sudo rm -v /mnt/swapfile - sudo fallocate -l 10G /mnt/swapfile + sudo fallocate -l 13G /mnt/swapfile sudo chmod 600 /mnt/swapfile sudo mkswap /mnt/swapfile sudo swapon /mnt/swapfile @@ -68,24 +68,24 @@ jobs: run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin - name: Checkout project - uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 + uses: actions/checkout@v3 - name: Init repo for Dockerization run: docker/init.sh "$TAG" - name: Set up QEMU - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0 + uses: docker/setup-qemu-action@v2 id: qemu - name: Setup Docker buildx action - uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v2.2.1 + uses: docker/setup-buildx-action@v2 id: buildx - name: Available platforms run: echo ${{ steps.buildx.outputs.platforms }} - name: Cache Docker layers - uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11 + uses: actions/cache@v3 id: cache with: path: /tmp/.buildx-cache diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index dbbc8412d..1f64214ce 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -27,7 +27,7 @@ "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master", "ADVANCED_GBT_AUDIT": false, "ADVANCED_GBT_MEMPOOL": false, - "TRANSACTION_INDEXING": false + "CPFP_INDEXING": false }, "CORE_RPC": { "HOST": "127.0.0.1", diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 2e9221c7a..e699c9458 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -26,9 +26,9 @@ "INDEXING_BLOCKS_AMOUNT": 14, "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__", "POOLS_JSON_URL": "__POOLS_JSON_URL__", - "ADVANCED_GBT_AUDIT": "__ADVANCED_GBT_AUDIT__", - "ADVANCED_GBT_MEMPOOL": "__ADVANCED_GBT_MEMPOOL__", - "TRANSACTION_INDEXING": "__TRANSACTION_INDEXING__" + "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__", + "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__", + "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__" }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 58cf3a214..4158d3df1 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -40,7 +40,7 @@ describe('Mempool Backend Config', () => { POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json', ADVANCED_GBT_AUDIT: false, ADVANCED_GBT_MEMPOOL: false, - TRANSACTION_INDEXING: false, + CPFP_INDEXING: false, }); expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index 6aafc9ded..b6b36dbdc 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -1,10 +1,5 @@ import config from '../config'; -import bitcoinApi from './bitcoin/bitcoin-api-factory'; -import { Common } from './common'; -import { TransactionExtended, MempoolBlockWithTransactions, AuditScore } from '../mempool.interfaces'; -import blocksRepository from '../repositories/BlocksRepository'; -import blocksAuditsRepository from '../repositories/BlocksAuditsRepository'; -import blocks from '../api/blocks'; +import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners diff --git a/backend/src/api/backend-info.ts b/backend/src/api/backend-info.ts index 57bb5fe13..fc3181524 100644 --- a/backend/src/api/backend-info.ts +++ b/backend/src/api/backend-info.ts @@ -2,6 +2,7 @@ import fs from 'fs'; import path from 'path'; import os from 'os'; import { IBackendInfo } from '../mempool.interfaces'; +import config from '../config'; class BackendInfo { private backendInfo: IBackendInfo; @@ -22,7 +23,8 @@ class BackendInfo { this.backendInfo = { hostname: os.hostname(), version: versionInfo.version, - gitCommit: versionInfo.gitCommit + gitCommit: versionInfo.gitCommit, + lightning: config.LIGHTNING.ENABLED }; } diff --git a/backend/src/api/bitcoin/bitcoin-api-factory.ts b/backend/src/api/bitcoin/bitcoin-api-factory.ts index f89d07b50..24916b97b 100644 --- a/backend/src/api/bitcoin/bitcoin-api-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-factory.ts @@ -17,4 +17,6 @@ function bitcoinApiFactory(): AbstractBitcoinApi { } } +export const bitcoinCoreApi = new BitcoinApi(bitcoinClient); + export default bitcoinApiFactory(); diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index 55500d0c9..2d77969a1 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -402,7 +402,8 @@ class BitcoinRoutes { private async getLegacyBlocks(req: Request, res: Response) { try { const returnBlocks: IEsploraApi.Block[] = []; - const fromHeight = parseInt(req.params.height, 10) || blocks.getCurrentBlockHeight(); + const tip = blocks.getCurrentBlockHeight(); + const fromHeight = Math.min(parseInt(req.params.height, 10) || tip, tip); // Check if block height exist in local cache to skip the hash lookup const blockByHeight = blocks.getBlocks().find((b) => b.height === fromHeight); diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 472ef48ef..83de897ca 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -22,12 +22,10 @@ import poolsParser from './pools-parser'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository'; import cpfpRepository from '../repositories/CpfpRepository'; -import transactionRepository from '../repositories/TransactionRepository'; import mining from './mining/mining'; import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; import PricesRepository from '../repositories/PricesRepository'; import priceUpdater from '../tasks/price-updater'; -import { Block } from 'bitcoinjs-lib'; class Blocks { private blocks: BlockExtended[] = []; @@ -101,12 +99,23 @@ class Blocks { transactions.push(tx); transactionsFetched++; } catch (e) { - if (i === 0) { - const msg = `Cannot fetch coinbase tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e); - logger.err(msg); - throw new Error(msg); - } else { - logger.err(`Cannot fetch tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e)); + try { + if (config.MEMPOOL.BACKEND === 'esplora') { + // Try again with core + const tx = await transactionUtils.$getTransactionExtended(txIds[i], false, false, true); + transactions.push(tx); + transactionsFetched++; + } else { + throw e; + } + } catch (e) { + if (i === 0) { + const msg = `Cannot fetch coinbase tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e); + logger.err(msg); + throw new Error(msg); + } else { + logger.err(`Cannot fetch tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e)); + } } } } @@ -329,9 +338,10 @@ class Blocks { try { // Get all indexed block hash - const unindexedBlocks = await blocksRepository.$getCPFPUnindexedBlocks(); + const unindexedBlockHeights = await blocksRepository.$getCPFPUnindexedBlocks(); + logger.info(`Indexing cpfp data for ${unindexedBlockHeights.length} blocks`); - if (!unindexedBlocks?.length) { + if (!unindexedBlockHeights?.length) { return; } @@ -340,30 +350,26 @@ class Blocks { let countThisRun = 0; let timer = new Date().getTime() / 1000; const startedAt = new Date().getTime() / 1000; - - for (const block of unindexedBlocks) { + for (const height of unindexedBlockHeights) { // Logging + const hash = await bitcoinApi.$getBlockHash(height); const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); if (elapsedSeconds > 5) { const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = Math.max(1, countThisRun / elapsedSeconds); - const progress = Math.round(count / unindexedBlocks.length * 10000) / 100; - logger.debug(`Indexing cpfp clusters for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`); + const blockPerSeconds = (countThisRun / elapsedSeconds); + const progress = Math.round(count / unindexedBlockHeights.length * 10000) / 100; + logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor} seconds`); timer = new Date().getTime() / 1000; countThisRun = 0; } - await this.$indexCPFP(block.hash, block.height); // Calculate and save CPFP data for transactions in this block + await this.$indexCPFP(hash, height); // Calculate and save CPFP data for transactions in this block // Logging count++; countThisRun++; } - if (count > 0) { - logger.notice(`CPFP indexing completed: indexed ${count} blocks`); - } else { - logger.debug(`CPFP indexing completed: indexed ${count} blocks`); - } + logger.notice(`CPFP indexing completed: indexed ${count} blocks`); } catch (e) { logger.err(`CPFP indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`); throw e; @@ -519,7 +525,7 @@ class Blocks { for (let i = 10; i >= 0; --i) { const newBlock = await this.$indexBlock(lastBlock['height'] - i); await this.$getStrippedBlockTransactions(newBlock.id, true, true); - if (config.MEMPOOL.TRANSACTION_INDEXING) { + if (config.MEMPOOL.CPFP_INDEXING) { await this.$indexCPFP(newBlock.id, lastBlock['height'] - i); } } @@ -547,7 +553,7 @@ class Blocks { if (Common.blocksSummariesIndexingEnabled() === true) { await this.$getStrippedBlockTransactions(blockExtended.id, true); } - if (config.MEMPOOL.TRANSACTION_INDEXING) { + if (config.MEMPOOL.CPFP_INDEXING) { this.$indexCPFP(blockExtended.id, this.currentBlockHeight); } } @@ -677,7 +683,12 @@ class Blocks { } public async $getBlocks(fromHeight?: number, limit: number = 15): Promise { + let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight; + if (currentHeight > this.currentBlockHeight) { + limit -= currentHeight - this.currentBlockHeight; + currentHeight = this.currentBlockHeight; + } const returnBlocks: BlockExtended[] = []; if (currentHeight < 0) { @@ -741,34 +752,15 @@ class Blocks { } public async $indexCPFP(hash: string, height: number): Promise { - let transactions; - if (Common.blocksSummariesIndexingEnabled()) { - transactions = await this.$getStrippedBlockTransactions(hash); - const rawBlock = await bitcoinApi.$getRawBlock(hash); - const block = Block.fromBuffer(rawBlock); - const txMap = {}; - for (const tx of block.transactions || []) { - txMap[tx.getId()] = tx; - } - for (const tx of transactions) { - // convert from bitcoinjs to esplora vin format - if (txMap[tx.txid]?.ins) { - tx.vin = txMap[tx.txid].ins.map(vin => { - return { - txid: vin.hash.slice().reverse().toString('hex') - }; - }); - } - } - } else { - const block = await bitcoinClient.getBlock(hash, 2); - transactions = block.tx.map(tx => { - tx.vsize = tx.weight / 4; - tx.fee *= 100_000_000; - return tx; - }); - } - + const block = await bitcoinClient.getBlock(hash, 2); + const transactions = block.tx.map(tx => { + tx.vsize = tx.weight / 4; + tx.fee *= 100_000_000; + return tx; + }); + + const clusters: any[] = []; + let cluster: TransactionStripped[] = []; let ancestors: { [txid: string]: boolean } = {}; for (let i = transactions.length - 1; i >= 0; i--) { @@ -782,10 +774,12 @@ class Blocks { }); const effectiveFeePerVsize = totalFee / totalVSize; if (cluster.length > 1) { - await cpfpRepository.$saveCluster(height, cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }), effectiveFeePerVsize); - for (const tx of cluster) { - await transactionRepository.$setCluster(tx.txid, cluster[0].txid); - } + clusters.push({ + root: cluster[0].txid, + height, + txs: cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }), + effectiveFeePerVsize, + }); } cluster = []; ancestors = {}; @@ -795,7 +789,10 @@ class Blocks { ancestors[vin.txid] = true; }); } - await blocksRepository.$setCPFPIndexed(hash); + const result = await cpfpRepository.$batchSaveClusters(clusters); + if (!result) { + await cpfpRepository.$insertProgressMarker(height); + } } } diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 621f021ba..f0c5c6b88 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -190,7 +190,7 @@ export class Common { static cpfpIndexingEnabled(): boolean { return ( Common.indexingEnabled() && - config.MEMPOOL.TRANSACTION_INDEXING === true + config.MEMPOOL.CPFP_INDEXING === true ); } diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 6e0e95699..42f223417 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -2,9 +2,12 @@ import config from '../config'; import DB from '../database'; import logger from '../logger'; import { Common } from './common'; +import blocksRepository from '../repositories/BlocksRepository'; +import cpfpRepository from '../repositories/CpfpRepository'; +import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 49; + private static currentVersion = 52; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -442,6 +445,29 @@ class DatabaseMigration { await this.$executeQuery('TRUNCATE TABLE `blocks_audits`'); await this.updateToSchemaVersion(49); } + + if (databaseSchemaVersion < 50) { + await this.$executeQuery('ALTER TABLE `blocks` DROP COLUMN `cpfp_indexed`'); + await this.updateToSchemaVersion(50); + } + + if (databaseSchemaVersion < 51) { + await this.$executeQuery('ALTER TABLE `cpfp_clusters` ADD INDEX `height` (`height`)'); + await this.updateToSchemaVersion(51); + } + + if (databaseSchemaVersion < 52) { + await this.$executeQuery(this.getCreateCompactCPFPTableQuery(), await this.$checkIfTableExists('compact_cpfp_clusters')); + await this.$executeQuery(this.getCreateCompactTransactionsTableQuery(), await this.$checkIfTableExists('compact_transactions')); + try { + await this.$convertCompactCpfpTables(); + await this.$executeQuery('DROP TABLE IF EXISTS `cpfp_clusters`'); + await this.$executeQuery('DROP TABLE IF EXISTS `transactions`'); + await this.updateToSchemaVersion(52); + } catch(e) { + logger.warn('' + (e instanceof Error ? e.message : e)); + } + } } /** @@ -913,6 +939,25 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getCreateCompactCPFPTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS compact_cpfp_clusters ( + root binary(32) NOT NULL, + height int(10) NOT NULL, + txs BLOB DEFAULT NULL, + fee_rate float unsigned, + PRIMARY KEY (root), + INDEX (height) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + + private getCreateCompactTransactionsTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS compact_transactions ( + txid binary(32) NOT NULL, + cluster binary(32) DEFAULT NULL, + PRIMARY KEY (txid) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + public async $truncateIndexedData(tables: string[]) { const allowedTables = ['blocks', 'hashrates', 'prices']; @@ -933,6 +978,49 @@ class DatabaseMigration { logger.warn(`Unable to erase indexed data`); } } + + private async $convertCompactCpfpTables(): Promise { + try { + const batchSize = 250; + const maxHeight = await blocksRepository.$mostRecentBlockHeight() || 0; + const [minHeightRows]: any = await DB.query(`SELECT MIN(height) AS minHeight from cpfp_clusters`); + const minHeight = (minHeightRows.length && minHeightRows[0].minHeight != null) ? minHeightRows[0].minHeight : maxHeight; + let height = maxHeight; + + // Logging + let timer = new Date().getTime() / 1000; + const startedAt = new Date().getTime() / 1000; + + while (height > minHeight) { + const [rows] = await DB.query( + ` + SELECT * from cpfp_clusters + WHERE height <= ? AND height > ? + ORDER BY height + `, + [height, height - batchSize] + ) as RowDataPacket[][]; + if (rows?.length) { + await cpfpRepository.$batchSaveClusters(rows.map(row => { + return { + root: row.root, + height: row.height, + txs: JSON.parse(row.txs), + effectiveFeePerVsize: row.fee_rate, + }; + })); + } + + const elapsed = new Date().getTime() / 1000 - timer; + const runningFor = new Date().getTime() / 1000 - startedAt; + logger.debug(`Migrated cpfp data from block ${height} to ${height - batchSize} in ${elapsed.toFixed(2)} seconds | total elapsed: ${runningFor.toFixed(2)} seconds`); + timer = new Date().getTime() / 1000; + height -= batchSize; + } + } catch (e) { + logger.warn(`Failed to migrate cpfp transaction data`); + } + } } export default new DatabaseMigration(); diff --git a/backend/src/api/explorer/nodes.routes.ts b/backend/src/api/explorer/nodes.routes.ts index c19bde236..e2dbcb0b6 100644 --- a/backend/src/api/explorer/nodes.routes.ts +++ b/backend/src/api/explorer/nodes.routes.ts @@ -41,13 +41,70 @@ class NodesRoutes { let nodes: any[] = []; switch (config.MEMPOOL.NETWORK) { case 'testnet': - nodesList = ['032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b', '025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7', '0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55', '032ab2028c0b614c6d87824e2373529652fd7e4221b4c70cc4da7c7005c49afcf0', '029001b22fe70b48bee12d014df91982eb85ff1bd404ec772d5c83c4ee3e88d2c3', '0212e2848d79f928411da5f2ff0a8c95ec6ccb5a09d2031b6f71e91309dcde63af', '03e871a2229523d34f76e6311ff197cfe7f26c2fbec13554b93a46f4e710c47dab', '032202ec98d976b0e928bd1d91924e8bd3eab07231fc39feb3737b010071073df8', '02fa7c5a948d03d563a9f36940c2205a814e594d17c0042ced242c71a857d72605', '039c14fdec2d958e3d14cebf657451bbd9e039196615785e82c917f274e3fb2205', '033589bbcb233ffc416cefd5437c7f37e9d7cb7942d405e39e72c4c846d9b37f18', '029293110441c6e2eacb57e1255bf6ef05c41a6a676fe474922d33c19f98a7d584']; + nodesList = [ + '032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b', + '025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7', + '0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55', + '032ab2028c0b614c6d87824e2373529652fd7e4221b4c70cc4da7c7005c49afcf0', + '029001b22fe70b48bee12d014df91982eb85ff1bd404ec772d5c83c4ee3e88d2c3', + '0212e2848d79f928411da5f2ff0a8c95ec6ccb5a09d2031b6f71e91309dcde63af', + '03e871a2229523d34f76e6311ff197cfe7f26c2fbec13554b93a46f4e710c47dab', + '032202ec98d976b0e928bd1d91924e8bd3eab07231fc39feb3737b010071073df8', + '02fa7c5a948d03d563a9f36940c2205a814e594d17c0042ced242c71a857d72605', + '039c14fdec2d958e3d14cebf657451bbd9e039196615785e82c917f274e3fb2205', + '033589bbcb233ffc416cefd5437c7f37e9d7cb7942d405e39e72c4c846d9b37f18', + '029293110441c6e2eacb57e1255bf6ef05c41a6a676fe474922d33c19f98a7d584', + '0235ad0b56ed8c42c4354444c24e971c05e769ec0b5fb0ccea42880095dc02ea2c', + '029700819a37afea630f80e6cc461f3fd3c4ace2598a21cfbbe64d1c78d0ee69a5', + '02c2d8b2dbf87c7894af2f1d321290e2fe6db5446cd35323987cee98f06e2e0075', + '030b0ca1ea7b1075716d2a555630e6fd47ef11bc7391fe68963ec06cf370a5e382', + '031adb9eb2d66693f85fa31a4adca0319ba68219f3ad5f9a2ef9b34a6b40755fa1', + '02ccd07faa47eda810ecf5591ccf5ca50f6c1034d0d175052898d32a00b9bae24f', + ]; break; case 'signet': - nodesList = ['03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956', '033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de', '02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781', '025196512905b8a3f1597428b867bec63ec9a95e5089eb7dc7e63e2d2691669029', '027c625aa1fbe3768db68ebcb05b53b6dc0ce68b7b54b8900d326d167363e684fe', '03f1629af3101fcc56b7aac2667016be84e3defbf3d0c8719f836c9b41c9a57a43', '02dfb81e2f7a3c4c9e8a51b70ef82b4a24549cc2fab1f5b2fd636501774a918991', '02d01ccf832944c68f10d39006093769c5b8bda886d561b128534e313d729fdb34', '02499ed23027d4698a6904ff4ec1b6085a61f10b9a6937f90438f9947e38e8ea86', '038310e3a786340f2bd7770704c7ccfe560fd163d9a1c99d67894597419d12cbf7', '03e5e9d879b72c7d67ecd483bae023bd33e695bb32b981a4021260f7b9d62bc761', '028d16e1a0ace4c0c0a421536d8d32ce484dfe6e2f726b7b0e7c30f12a195f8cc7']; + nodesList = [ + '03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956', + '033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de', + '02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781', + '025196512905b8a3f1597428b867bec63ec9a95e5089eb7dc7e63e2d2691669029', + '027c625aa1fbe3768db68ebcb05b53b6dc0ce68b7b54b8900d326d167363e684fe', + '03f1629af3101fcc56b7aac2667016be84e3defbf3d0c8719f836c9b41c9a57a43', + '02dfb81e2f7a3c4c9e8a51b70ef82b4a24549cc2fab1f5b2fd636501774a918991', + '02d01ccf832944c68f10d39006093769c5b8bda886d561b128534e313d729fdb34', + '02499ed23027d4698a6904ff4ec1b6085a61f10b9a6937f90438f9947e38e8ea86', + '038310e3a786340f2bd7770704c7ccfe560fd163d9a1c99d67894597419d12cbf7', + '03e5e9d879b72c7d67ecd483bae023bd33e695bb32b981a4021260f7b9d62bc761', + '028d16e1a0ace4c0c0a421536d8d32ce484dfe6e2f726b7b0e7c30f12a195f8cc7', + '02ff690d06c187ab994bf83c5a2114fe5bf50112c2c817af0f788f736be9fa2070', + '02a9f570c51a2526a5ee85802e88f9281bed771eb66a0c8a7d898430dd5d0eae45', + '038c3de773255d3bd7a50e31e58d423baac5c90826a74d75e64b74c95475de1097', + '0242c7f7d315095f37ad1421ae0a2fc967d4cbe65b61b079c5395a769436959853', + '02a909e70eb03742f12666ebb1f56ac42a5fbaab0c0e8b5b1df4aa9f10f8a09240', + '03a26efa12489803c07f3ac2f1dba63812e38f0f6e866ce3ebb34df7de1f458cd2', + ]; break; default: - nodesList = ['03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61', '03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437', '03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144', '0238bd27f02d67d6c51e269692bc8c9a32357a00e7777cba7f4f1f18a2a700b108', '03f983dcabed6baa1eab5b56c8b2e8fdc846ab3fd931155377897335e85a9fa57c', '03e399589533581e48796e29a825839a010036a61b20744fda929d6709fcbffcc5', '021f5288b5f72c42cd0d8801086af7ce09a816d8ee9a4c47a4b436399b26cb601a', '032b01b7585f781420cd4148841a82831ba37fa952342052cec16750852d4f2dd9', '02848036488d4b8fb1f1c4064261ec36151f43b085f0b51bd239ade3ddfc940c34', '02b6b1640fe029e304c216951af9fbefdb23b0bdc9baaf327540d31b6107841fdf', '03694289827203a5b3156d753071ddd5bf92e371f5a462943f9555eef6d2d6606c', '0283d850db7c3e8ea7cc9c4abc7afaab12bbdf72b677dcba1d608350d2537d7d43']; + nodesList = [ + '03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61', + '03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437', + '03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144', + '0238bd27f02d67d6c51e269692bc8c9a32357a00e7777cba7f4f1f18a2a700b108', + '03f983dcabed6baa1eab5b56c8b2e8fdc846ab3fd931155377897335e85a9fa57c', + '03e399589533581e48796e29a825839a010036a61b20744fda929d6709fcbffcc5', + '021f5288b5f72c42cd0d8801086af7ce09a816d8ee9a4c47a4b436399b26cb601a', + '032b01b7585f781420cd4148841a82831ba37fa952342052cec16750852d4f2dd9', + '02848036488d4b8fb1f1c4064261ec36151f43b085f0b51bd239ade3ddfc940c34', + '02b6b1640fe029e304c216951af9fbefdb23b0bdc9baaf327540d31b6107841fdf', + '03694289827203a5b3156d753071ddd5bf92e371f5a462943f9555eef6d2d6606c', + '0283d850db7c3e8ea7cc9c4abc7afaab12bbdf72b677dcba1d608350d2537d7d43', + '02521287789f851268a39c9eccc9d6180d2c614315b583c9e6ae0addbd6d79df06', + '0258c2a7b7f8af2585b4411b1ec945f70988f30412bb1df179de941f14d0b1bc3e', + '03c3389ff1a896f84d921ed01a19fc99c6724ce8dc4b960cd3b7b2362b62cd60d7', + '038d118996b3eaa15dcd317b32a539c9ecfdd7698f204acf8a087336af655a9192', + '02a928903d93d78877dacc3642b696128a3636e9566dd42d2d132325b2c8891c09', + '0328cd17f3a9d3d90b532ade0d1a67e05eb8a51835b3dce0a2e38eac04b5a62a57', + ]; } for (let pubKey of nodesList) { diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index 5b92cea5f..fb5aeea42 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -1,8 +1,7 @@ -import bitcoinApi from './bitcoin/bitcoin-api-factory'; import { TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces'; import { IEsploraApi } from './bitcoin/esplora-api.interface'; -import config from '../config'; import { Common } from './common'; +import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; class TransactionUtils { constructor() { } @@ -21,8 +20,19 @@ class TransactionUtils { }; } - public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false): Promise { - const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts); + /** + * @param txId + * @param addPrevouts + * @param lazyPrevouts + * @param forceCore - See https://github.com/mempool/mempool/issues/2904 + */ + public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise { + let transaction: IEsploraApi.Transaction; + if (forceCore === true) { + transaction = await bitcoinCoreApi.$getRawTransaction(txId, true); + } else { + transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts); + } return this.extendTransaction(transaction); } diff --git a/backend/src/config.ts b/backend/src/config.ts index e97deb5e5..fb06c84fb 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -31,7 +31,7 @@ interface IConfig { POOLS_JSON_TREE_URL: string, ADVANCED_GBT_AUDIT: boolean; ADVANCED_GBT_MEMPOOL: boolean; - TRANSACTION_INDEXING: boolean; + CPFP_INDEXING: boolean; }; ESPLORA: { REST_API_URL: string; @@ -152,7 +152,7 @@ const defaults: IConfig = { 'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', 'ADVANCED_GBT_AUDIT': false, 'ADVANCED_GBT_MEMPOOL': false, - 'TRANSACTION_INDEXING': false, + 'CPFP_INDEXING': false, }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 046083322..f79786279 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -274,6 +274,7 @@ export interface IBackendInfo { hostname: string; gitCommit: string; version: string; + lightning: boolean; } export interface IDifficultyAdjustment { @@ -337,4 +338,4 @@ export interface IOldestNodes { updatedAt?: number, city?: any, country?: any, -} \ No newline at end of file +} diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 78a8fcce2..df98719b9 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -8,6 +8,8 @@ import HashratesRepository from './HashratesRepository'; import { escape } from 'mysql2'; import BlocksSummariesRepository from './BlocksSummariesRepository'; import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository'; +import bitcoinClient from '../api/bitcoin/bitcoin-client'; +import config from '../config'; class BlocksRepository { /** @@ -667,16 +669,32 @@ class BlocksRepository { */ public async $getCPFPUnindexedBlocks(): Promise { try { - const [rows]: any = await DB.query(`SELECT height, hash FROM blocks WHERE cpfp_indexed = 0 ORDER BY height DESC`); - return rows; + const blockchainInfo = await bitcoinClient.getBlockchainInfo(); + const currentBlockHeight = blockchainInfo.blocks; + let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, currentBlockHeight); + if (indexingBlockAmount <= -1) { + indexingBlockAmount = currentBlockHeight + 1; + } + const minHeight = Math.max(0, currentBlockHeight - indexingBlockAmount + 1); + + const [rows]: any[] = await DB.query(` + SELECT height + FROM compact_cpfp_clusters + WHERE height <= ? AND height >= ? + ORDER BY height DESC; + `, [currentBlockHeight, minHeight]); + + const indexedHeights = {}; + rows.forEach((row) => { indexedHeights[row.height] = true; }); + const allHeights: number[] = Array.from(Array(currentBlockHeight - minHeight + 1).keys(), n => n + minHeight).reverse(); + const unindexedHeights = allHeights.filter(x => !indexedHeights[x]); + + return unindexedHeights; } catch (e) { logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e)); throw e; } - } - - public async $setCPFPIndexed(hash: string): Promise { - await DB.query(`UPDATE blocks SET cpfp_indexed = 1 WHERE hash = ?`, [hash]); + return []; } /** diff --git a/backend/src/repositories/CpfpRepository.ts b/backend/src/repositories/CpfpRepository.ts index 563e6ede1..ce7432d5b 100644 --- a/backend/src/repositories/CpfpRepository.ts +++ b/backend/src/repositories/CpfpRepository.ts @@ -1,34 +1,151 @@ +import cluster, { Cluster } from 'cluster'; +import { RowDataPacket } from 'mysql2'; import DB from '../database'; import logger from '../logger'; import { Ancestor } from '../mempool.interfaces'; +import transactionRepository from '../repositories/TransactionRepository'; class CpfpRepository { - public async $saveCluster(height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise { + public async $saveCluster(clusterRoot: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise { + if (!txs[0]) { + return false; + } + // skip clusters of transactions with the same fees + const roundedEffectiveFee = Math.round(effectiveFeePerVsize * 100) / 100; + const equalFee = txs.reduce((acc, tx) => { + return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee); + }, true); + if (equalFee) { + return false; + } + try { - const txsJson = JSON.stringify(txs); + const packedTxs = Buffer.from(this.pack(txs)); await DB.query( ` - INSERT INTO cpfp_clusters(root, height, txs, fee_rate) - VALUE (?, ?, ?, ?) + INSERT INTO compact_cpfp_clusters(root, height, txs, fee_rate) + VALUE (UNHEX(?), ?, ?, ?) ON DUPLICATE KEY UPDATE height = ?, txs = ?, fee_rate = ? `, - [txs[0].txid, height, txsJson, effectiveFeePerVsize, height, txsJson, effectiveFeePerVsize, height] + [clusterRoot, height, packedTxs, effectiveFeePerVsize, height, packedTxs, effectiveFeePerVsize] ); + const maxChunk = 10; + let chunkIndex = 0; + while (chunkIndex < txs.length) { + const chunk = txs.slice(chunkIndex, chunkIndex + maxChunk).map(tx => { + return { txid: tx.txid, cluster: clusterRoot }; + }); + await transactionRepository.$batchSetCluster(chunk); + chunkIndex += maxChunk; + } + return true; } catch (e: any) { logger.err(`Cannot save cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e)); throw e; } } + public async $batchSaveClusters(clusters: { root: string, height: number, txs: any, effectiveFeePerVsize: number}[]): Promise { + try { + const clusterValues: any[] = []; + const txs: any[] = []; + + for (const cluster of clusters) { + if (cluster.txs?.length > 1) { + const roundedEffectiveFee = Math.round(cluster.effectiveFeePerVsize * 100) / 100; + const equalFee = cluster.txs.reduce((acc, tx) => { + return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee); + }, true); + if (!equalFee) { + clusterValues.push([ + cluster.root, + cluster.height, + Buffer.from(this.pack(cluster.txs)), + cluster.effectiveFeePerVsize + ]); + for (const tx of cluster.txs) { + txs.push({ txid: tx.txid, cluster: cluster.root }); + } + } + } + } + + if (!clusterValues.length) { + return false; + } + + const maxChunk = 100; + let chunkIndex = 0; + // insert transactions in batches of up to 100 rows + while (chunkIndex < txs.length) { + const chunk = txs.slice(chunkIndex, chunkIndex + maxChunk); + await transactionRepository.$batchSetCluster(chunk); + chunkIndex += maxChunk; + } + + chunkIndex = 0; + // insert clusters in batches of up to 100 rows + while (chunkIndex < clusterValues.length) { + const chunk = clusterValues.slice(chunkIndex, chunkIndex + maxChunk); + let query = ` + INSERT IGNORE INTO compact_cpfp_clusters(root, height, txs, fee_rate) + VALUES + `; + query += chunk.map(chunk => { + return (' (UNHEX(?), ?, ?, ?)'); + }) + ';'; + const values = chunk.flat(); + await DB.query( + query, + values + ); + chunkIndex += maxChunk; + } + return true; + } catch (e: any) { + logger.err(`Cannot save cpfp clusters into db. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + + public async $getCluster(clusterRoot: string): Promise { + const [clusterRows]: any = await DB.query( + ` + SELECT * + FROM compact_cpfp_clusters + WHERE root = UNHEX(?) + `, + [clusterRoot] + ); + const cluster = clusterRows[0]; + cluster.txs = this.unpack(cluster.txs); + return cluster; + } + public async $deleteClustersFrom(height: number): Promise { logger.info(`Delete newer cpfp clusters from height ${height} from the database`); try { + const [rows] = await DB.query( + ` + SELECT txs, height, root from compact_cpfp_clusters + WHERE height >= ? + `, + [height] + ) as RowDataPacket[][]; + if (rows?.length) { + for (let clusterToDelete of rows) { + const txs = this.unpack(clusterToDelete.txs); + for (let tx of txs) { + await transactionRepository.$removeTransaction(tx.txid); + } + } + } await DB.query( ` - DELETE from cpfp_clusters + DELETE from compact_cpfp_clusters WHERE height >= ? `, [height] @@ -38,6 +155,70 @@ class CpfpRepository { throw e; } } + + // insert a dummy row to mark that we've indexed as far as this block + public async $insertProgressMarker(height: number): Promise { + try { + const [rows]: any = await DB.query( + ` + SELECT root + FROM compact_cpfp_clusters + WHERE height = ? + `, + [height] + ); + if (!rows?.length) { + const rootBuffer = Buffer.alloc(32); + rootBuffer.writeInt32LE(height); + await DB.query( + ` + INSERT INTO compact_cpfp_clusters(root, height, fee_rate) + VALUE (?, ?, ?) + `, + [rootBuffer, height, 0] + ); + } + } catch (e: any) { + logger.err(`Cannot insert cpfp progress marker. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + + public pack(txs: Ancestor[]): ArrayBuffer { + const buf = new ArrayBuffer(44 * txs.length); + const view = new DataView(buf); + txs.forEach((tx, i) => { + const offset = i * 44; + for (let x = 0; x < 32; x++) { + // store txid in little-endian + view.setUint8(offset + (31 - x), parseInt(tx.txid.slice(x * 2, (x * 2) + 2), 16)); + } + view.setUint32(offset + 32, tx.weight); + view.setBigUint64(offset + 36, BigInt(Math.round(tx.fee))); + }); + return buf; + } + + public unpack(buf: Buffer): Ancestor[] { + if (!buf) { + return []; + } + + const arrayBuffer = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength); + const txs: Ancestor[] = []; + const view = new DataView(arrayBuffer); + for (let offset = 0; offset < arrayBuffer.byteLength; offset += 44) { + const txid = Array.from(new Uint8Array(arrayBuffer, offset, 32)).reverse().map(b => b.toString(16).padStart(2, '0')).join(''); + const weight = view.getUint32(offset + 32); + const fee = Number(view.getBigUint64(offset + 36)); + txs.push({ + txid, + weight, + fee + }); + } + return txs; + } } export default new CpfpRepository(); \ No newline at end of file diff --git a/backend/src/repositories/TransactionRepository.ts b/backend/src/repositories/TransactionRepository.ts index 74debb833..061617451 100644 --- a/backend/src/repositories/TransactionRepository.ts +++ b/backend/src/repositories/TransactionRepository.ts @@ -1,6 +1,7 @@ import DB from '../database'; import logger from '../logger'; import { Ancestor, CpfpInfo } from '../mempool.interfaces'; +import cpfpRepository from './CpfpRepository'; interface CpfpSummary { txid: string; @@ -12,20 +13,20 @@ interface CpfpSummary { } class TransactionRepository { - public async $setCluster(txid: string, cluster: string): Promise { + public async $setCluster(txid: string, clusterRoot: string): Promise { try { await DB.query( ` - INSERT INTO transactions + INSERT INTO compact_transactions ( txid, cluster ) - VALUE (?, ?) + VALUE (UNHEX(?), UNHEX(?)) ON DUPLICATE KEY UPDATE - cluster = ? + cluster = UNHEX(?) ;`, - [txid, cluster, cluster] + [txid, clusterRoot, clusterRoot] ); } catch (e: any) { logger.err(`Cannot save transaction cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e)); @@ -33,20 +34,45 @@ class TransactionRepository { } } - public async $getCpfpInfo(txid: string): Promise { + public async $batchSetCluster(txs): Promise { try { let query = ` - SELECT * - FROM transactions - LEFT JOIN cpfp_clusters AS cluster ON cluster.root = transactions.cluster - WHERE transactions.txid = ? + INSERT IGNORE INTO compact_transactions + ( + txid, + cluster + ) + VALUES `; - const [rows]: any = await DB.query(query, [txid]); - if (rows.length) { - rows[0].txs = JSON.parse(rows[0].txs) as Ancestor[]; - if (rows[0]?.txs?.length) { - return this.convertCpfp(rows[0]); - } + query += txs.map(tx => { + return (' (UNHEX(?), UNHEX(?))'); + }) + ';'; + const values = txs.map(tx => [tx.txid, tx.cluster]).flat(); + await DB.query( + query, + values + ); + } catch (e: any) { + logger.err(`Cannot save cpfp transactions into db. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + + public async $getCpfpInfo(txid: string): Promise { + try { + const [txRows]: any = await DB.query( + ` + SELECT HEX(txid) as id, HEX(cluster) as root + FROM compact_transactions + WHERE txid = UNHEX(?) + `, + [txid] + ); + if (txRows.length && txRows[0].root != null) { + const txid = txRows[0].id.toLowerCase(); + const clusterId = txRows[0].root.toLowerCase(); + const cluster = await cpfpRepository.$getCluster(clusterId); + return this.convertCpfp(txid, cluster); } } catch (e) { logger.err('Cannot get transaction cpfp info from db. Reason: ' + (e instanceof Error ? e.message : e)); @@ -54,12 +80,23 @@ class TransactionRepository { } } - private convertCpfp(cpfp: CpfpSummary): CpfpInfo { + public async $removeTransaction(txid: string): Promise { + await DB.query( + ` + DELETE FROM compact_transactions + WHERE txid = UNHEX(?) + `, + [txid] + ); + } + + private convertCpfp(txid, cluster): CpfpInfo { const descendants: Ancestor[] = []; const ancestors: Ancestor[] = []; let matched = false; - for (const tx of cpfp.txs) { - if (tx.txid === cpfp.txid) { + + for (const tx of cluster.txs) { + if (tx.txid === txid) { matched = true; } else if (!matched) { descendants.push(tx); @@ -70,7 +107,6 @@ class TransactionRepository { return { descendants, ancestors, - effectiveFeePerVsize: cpfp.fee_rate }; } } diff --git a/docker/README.md b/docker/README.md index 4061f420c..0c53cee97 100644 --- a/docker/README.md +++ b/docker/README.md @@ -100,12 +100,18 @@ Below we list all settings from `mempool-config.json` and the corresponding over "BLOCK_WEIGHT_UNITS": 4000000, "INITIAL_BLOCKS_AMOUNT": 8, "MEMPOOL_BLOCKS_AMOUNT": 8, + "BLOCKS_SUMMARIES_INDEXING": false, "PRICE_FEED_UPDATE_INTERVAL": 600, "USE_SECOND_NODE_FOR_MINFEE": false, "EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"], "STDOUT_LOG_MIN_PRIORITY": "info", + "INDEXING_BLOCKS_AMOUNT": false, + "AUTOMATIC_BLOCK_REINDEXING": false, "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json", - "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master" + "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master", + "ADVANCED_GBT_AUDIT": false, + "ADVANCED_GBT_MEMPOOL": false, + "CPFP_INDEXING": false, }, ``` @@ -125,15 +131,25 @@ Corresponding `docker-compose.yml` overrides: MEMPOOL_BLOCK_WEIGHT_UNITS: "" MEMPOOL_INITIAL_BLOCKS_AMOUNT: "" MEMPOOL_MEMPOOL_BLOCKS_AMOUNT: "" + MEMPOOL_BLOCKS_SUMMARIES_INDEXING: "" MEMPOOL_PRICE_FEED_UPDATE_INTERVAL: "" MEMPOOL_USE_SECOND_NODE_FOR_MINFEE: "" MEMPOOL_EXTERNAL_ASSETS: "" MEMPOOL_STDOUT_LOG_MIN_PRIORITY: "" + MEMPOOL_INDEXING_BLOCKS_AMOUNT: "" + MEMPOOL_AUTOMATIC_BLOCK_REINDEXING: "" MEMPOOL_POOLS_JSON_URL: "" MEMPOOL_POOLS_JSON_TREE_URL: "" + MEMPOOL_ADVANCED_GBT_AUDIT: "" + MEMPOOL_ADVANCED_GBT_MEMPOOL: "" + MEMPOOL_CPFP_INDEXING: "" ... ``` +`ADVANCED_GBT_AUDIT` AND `ADVANCED_GBT_MEMPOOL` enable a more accurate (but slower) block prediction algorithm for the block audit feature and the projected mempool-blocks respectively. + +`CPFP_INDEXING` enables indexing CPFP (Child Pays For Parent) information for the last `INDEXING_BLOCKS_AMOUNT` blocks. +
`mempool-config.json`: diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 378bba8db..2e3826f1d 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -22,7 +22,10 @@ "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__", "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__, "BLOCKS_SUMMARIES_INDEXING": __MEMPOOL_BLOCKS_SUMMARIES_INDEXING__, - "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__ + "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__, + "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__, + "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__, + "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__ }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/docker/backend/start.sh b/docker/backend/start.sh index c8e2a1502..b5b6e863d 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -27,6 +27,9 @@ __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false} __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false} __MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json} __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master} +__MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false} +__MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false} +__MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} @@ -136,6 +139,8 @@ sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT_ sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json +sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json +sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json diff --git a/docker/frontend/entrypoint.sh b/docker/frontend/entrypoint.sh index 6a263de99..3e2210360 100644 --- a/docker/frontend/entrypoint.sh +++ b/docker/frontend/entrypoint.sh @@ -31,6 +31,9 @@ __LIQUID_WEBSITE_URL__=${LIQUID_WEBSITE_URL:=https://liquid.network} __BISQ_WEBSITE_URL__=${BISQ_WEBSITE_URL:=https://bisq.markets} __MINING_DASHBOARD__=${MINING_DASHBOARD:=true} __LIGHTNING__=${LIGHTNING:=false} +__MAINNET_BLOCK_AUDIT_START_HEIGHT__=${MAINNET_BLOCK_AUDIT_START_HEIGHT:=0} +__TESTNET_BLOCK_AUDIT_START_HEIGHT__=${TESTNET_BLOCK_AUDIT_START_HEIGHT:=0} +__SIGNET_BLOCK_AUDIT_START_HEIGHT__=${SIGNET_BLOCK_AUDIT_START_HEIGHT:=0} # Export as environment variables to be used by envsubst export __TESTNET_ENABLED__ @@ -52,6 +55,9 @@ export __LIQUID_WEBSITE_URL__ export __BISQ_WEBSITE_URL__ export __MINING_DASHBOARD__ export __LIGHTNING__ +export __MAINNET_BLOCK_AUDIT_START_HEIGHT__ +export __TESTNET_BLOCK_AUDIT_START_HEIGHT__ +export __SIGNET_BLOCK_AUDIT_START_HEIGHT__ folder=$(find /var/www/mempool -name "config.js" | xargs dirname) echo ${folder} diff --git a/frontend/cypress/e2e/liquid/liquid.spec.ts b/frontend/cypress/e2e/liquid/liquid.spec.ts index 1e7c4649d..e24b19fad 100644 --- a/frontend/cypress/e2e/liquid/liquid.spec.ts +++ b/frontend/cypress/e2e/liquid/liquid.spec.ts @@ -7,7 +7,6 @@ describe('Liquid', () => { cy.intercept('/liquid/api/blocks/').as('blocks'); cy.intercept('/liquid/api/tx/**/outspends').as('outspends'); cy.intercept('/liquid/api/block/**/txs/**').as('block-txs'); - cy.intercept('/resources/pools.json').as('pools'); Cypress.Commands.add('waitForBlockData', () => { cy.wait('@socket'); diff --git a/frontend/cypress/e2e/liquidtestnet/liquidtestnet.spec.ts b/frontend/cypress/e2e/liquidtestnet/liquidtestnet.spec.ts index 3ca23425e..5cf6cf331 100644 --- a/frontend/cypress/e2e/liquidtestnet/liquidtestnet.spec.ts +++ b/frontend/cypress/e2e/liquidtestnet/liquidtestnet.spec.ts @@ -7,7 +7,6 @@ describe('Liquid Testnet', () => { cy.intercept('/liquidtestnet/api/blocks/').as('blocks'); cy.intercept('/liquidtestnet/api/tx/**/outspends').as('outspends'); cy.intercept('/liquidtestnet/api/block/**/txs/**').as('block-txs'); - cy.intercept('/resources/pools.json').as('pools'); Cypress.Commands.add('waitForBlockData', () => { cy.wait('@socket'); diff --git a/frontend/cypress/e2e/mainnet/mainnet.spec.ts b/frontend/cypress/e2e/mainnet/mainnet.spec.ts index 4cc9a64c9..d6fe94dac 100644 --- a/frontend/cypress/e2e/mainnet/mainnet.spec.ts +++ b/frontend/cypress/e2e/mainnet/mainnet.spec.ts @@ -41,7 +41,6 @@ describe('Mainnet', () => { // cy.intercept('/api/v1/block/*/summary').as('block-summary'); // cy.intercept('/api/v1/outspends/*').as('outspends'); // cy.intercept('/api/tx/*/outspends').as('tx-outspends'); - // cy.intercept('/resources/pools.json').as('pools'); // Search Auto Complete cy.intercept('/api/address-prefix/1wiz').as('search-1wiz'); diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 72f9e7afa..bab0817b2 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -31,7 +31,6 @@ "bootstrap": "~4.6.1", "browserify": "^17.0.0", "clipboard": "^2.0.11", - "cypress": "^12.1.0", "domino": "^2.1.6", "echarts": "~5.4.0", "echarts-gl": "^2.0.9", diff --git a/frontend/proxy.conf.js b/frontend/proxy.conf.js index ab2240c03..f5384bef0 100644 --- a/frontend/proxy.conf.js +++ b/frontend/proxy.conf.js @@ -76,7 +76,7 @@ PROXY_CONFIG = [ if (configContent && configContent.BASE_MODULE == "liquid") { PROXY_CONFIG.push({ - context: ['/resources/pools.json', + context: [ '/resources/assets.json', '/resources/assets.minimal.json', '/resources/assets-testnet.json', '/resources/assets-testnet.minimal.json'], target: "https://liquid.network", @@ -85,7 +85,7 @@ if (configContent && configContent.BASE_MODULE == "liquid") { }); } else { PROXY_CONFIG.push({ - context: ['/resources/pools.json', '/resources/assets.json', '/resources/assets.minimal.json', '/resources/worldmap.json'], + context: ['/resources/assets.json', '/resources/assets.minimal.json', '/resources/worldmap.json'], target: "https://mempool.space", secure: false, changeOrigin: true, diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index 6ed7c43f9..b7bd1526f 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -6,6 +6,7 @@ import { AppRoutingModule } from './app-routing.module'; import { AppComponent } from './components/app/app.component'; import { ElectrsApiService } from './services/electrs-api.service'; import { StateService } from './services/state.service'; +import { CacheService } from './services/cache.service'; import { EnterpriseService } from './services/enterprise.service'; import { WebsocketService } from './services/websocket.service'; import { AudioService } from './services/audio.service'; @@ -23,6 +24,7 @@ import { AppPreloadingStrategy } from './app.preloading-strategy'; const providers = [ ElectrsApiService, StateService, + CacheService, WebsocketService, AudioService, SeoService, diff --git a/frontend/src/app/components/address-labels/address-labels.component.ts b/frontend/src/app/components/address-labels/address-labels.component.ts index be5bdeead..f4b3d0ca5 100644 --- a/frontend/src/app/components/address-labels/address-labels.component.ts +++ b/frontend/src/app/components/address-labels/address-labels.component.ts @@ -36,7 +36,9 @@ export class AddressLabelsComponent implements OnChanges { handleChannel() { const type = this.vout ? 'open' : 'close'; - this.label = `Channel ${type}: ${this.channel.node_left.alias} <> ${this.channel.node_right.alias}`; + const leftNodeName = this.channel.node_left.alias || this.channel.node_left.public_key.substring(0, 10); + const rightNodeName = this.channel.node_right.alias || this.channel.node_right.public_key.substring(0, 10); + this.label = `Channel ${type}: ${leftNodeName} <> ${rightNodeName}`; } handleVin() { diff --git a/frontend/src/app/components/app/app.component.ts b/frontend/src/app/components/app/app.component.ts index d9d6f77d6..c7ca798ae 100644 --- a/frontend/src/app/components/app/app.component.ts +++ b/frontend/src/app/components/app/app.component.ts @@ -42,6 +42,10 @@ export class AppComponent implements OnInit { if (event.target instanceof HTMLInputElement) { return; } + // prevent arrow key horizontal scrolling + if(["ArrowLeft","ArrowRight"].indexOf(event.code) > -1) { + event.preventDefault(); + } this.stateService.keyNavigation$.next(event); } diff --git a/frontend/src/app/components/block/block.component.html b/frontend/src/app/components/block/block.component.html index 7a958735c..08ea04ca9 100644 --- a/frontend/src/app/components/block/block.component.html +++ b/frontend/src/app/components/block/block.component.html @@ -112,6 +112,7 @@ [flip]="false" (txClickEvent)="onTxClick($event)" > + @@ -213,15 +214,21 @@

Projected Block

- +
+ + +

Actual Block

- +
+ + +
@@ -343,5 +350,17 @@ + + + + Why is this block empty? + + +

diff --git a/frontend/src/app/components/block/block.component.scss b/frontend/src/app/components/block/block.component.scss index 69002de79..931912e4e 100644 --- a/frontend/src/app/components/block/block.component.scss +++ b/frontend/src/app/components/block/block.component.scss @@ -202,4 +202,24 @@ h1 { &.active, &:hover { border-color: white; } +} + +.block-graph-wrapper { + position: relative; +} + +.info-bubble-link { + position: absolute; + display: block; + top: 2em; + left: 50%; + margin: auto; + text-align: center; + padding: 0.5em 1em; + font-size: 80%; + transform: translateX(-50%); + + .ng-fa-icon { + margin-right: 1em; + } } \ No newline at end of file diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index e92e44937..f04b4ec9c 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -138,7 +138,6 @@ export class BlockComponent implements OnInit, OnDestroy { this.page = 1; this.error = undefined; this.fees = undefined; - this.stateService.markBlock$.next({}); this.auditDataMissing = false; if (history.state.data && history.state.data.blockHeight) { diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html index 6bd617435..29df378a4 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html @@ -1,36 +1,55 @@ -
-
-
-   -
- {{ block.height }} +
+
+ +
+   + +
+
+ ~{{ block?.extras?.medianFee | number:feeRounding }} sat/vB +
+
+ {{ block?.extras?.feeRange?.[1] | number:feeRounding }} - {{ block?.extras?.feeRange[block?.extras?.feeRange?.length - 1] | number:feeRounding }} sat/vB +
+
+   +
+
+ +
+
+
+ + {{ i }} transaction + {{ i }} transactions +
+
+
+
-
-
- ~{{ block?.extras?.medianFee | number:feeRounding }} sat/vB + + + +
+
-
- {{ block?.extras?.feeRange[1] | number:feeRounding }} - {{ block?.extras?.feeRange[block?.extras?.feeRange.length - 1] | number:feeRounding }} sat/vB + + + + +
+
-
- -
-
-
- - {{ i }} transaction - {{ i }} transactions -
-
-
- -
+ +
-
+
diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.scss b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.scss index adde4a945..64bfd2379 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.scss +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.scss @@ -25,6 +25,10 @@ transition: background 2s, left 2s, transform 1s; } +.mined-block.placeholder-block { + background: none !important; +} + .block-size { font-size: 16px; font-weight: bold; @@ -96,6 +100,16 @@ transform-origin: top; } +.bitcoin-block.placeholder-block::after { + content: none; + background: 0; +} + +.bitcoin-block.placeholder-block::before { + content: none; + background: 0; +} + .black-background { background-color: #11131f; z-index: 100; diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.ts b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.ts index 8ac925eaf..fd8819a6f 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.ts +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.ts @@ -1,10 +1,16 @@ -import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, ChangeDetectorRef } from '@angular/core'; +import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, ChangeDetectorRef, Input, OnChanges, SimpleChanges } from '@angular/core'; import { Observable, Subscription } from 'rxjs'; import { StateService } from '../../services/state.service'; import { specialBlocks } from '../../app.constants'; import { BlockExtended } from '../../interfaces/node-api.interface'; import { Location } from '@angular/common'; import { config } from 'process'; +import { CacheService } from 'src/app/services/cache.service'; + +interface BlockchainBlock extends BlockExtended { + placeholder?: boolean; + loading?: boolean; +} @Component({ selector: 'app-blockchain-blocks', @@ -12,13 +18,19 @@ import { config } from 'process'; styleUrls: ['./blockchain-blocks.component.scss'], changeDetection: ChangeDetectionStrategy.OnPush, }) -export class BlockchainBlocksComponent implements OnInit, OnDestroy { +export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy { + @Input() static: boolean = false; + @Input() offset: number = 0; + @Input() height: number = 0; + @Input() count: number = 8; + specialBlocks = specialBlocks; network = ''; - blocks: BlockExtended[] = []; + blocks: BlockchainBlock[] = []; emptyBlocks: BlockExtended[] = this.mountEmptyBlocks(); markHeight: number; blocksSubscription: Subscription; + blockPageSubscription: Subscription; networkSubscription: Subscription; tabHiddenSubscription: Subscription; markBlockSubscription: Subscription; @@ -31,7 +43,7 @@ export class BlockchainBlocksComponent implements OnInit, OnDestroy { arrowVisible = false; arrowLeftPx = 30; blocksFilled = false; - transition = '1s'; + arrowTransition = '1s'; showMiningInfo = false; timeLtrSubscription: Subscription; timeLtr: boolean; @@ -47,6 +59,7 @@ export class BlockchainBlocksComponent implements OnInit, OnDestroy { constructor( public stateService: StateService, + public cacheService: CacheService, private cd: ChangeDetectorRef, private location: Location, ) { @@ -75,44 +88,52 @@ export class BlockchainBlocksComponent implements OnInit, OnDestroy { this.loadingBlocks$ = this.stateService.isLoadingWebSocket$; this.networkSubscription = this.stateService.networkChanged$.subscribe((network) => this.network = network); this.tabHiddenSubscription = this.stateService.isTabHidden$.subscribe((tabHidden) => this.tabHidden = tabHidden); - this.blocksSubscription = this.stateService.blocks$ - .subscribe(([block, txConfirmed]) => { - if (this.blocks.some((b) => b.height === block.height)) { - return; - } + if (!this.static) { + this.blocksSubscription = this.stateService.blocks$ + .subscribe(([block, txConfirmed]) => { + if (this.blocks.some((b) => b.height === block.height)) { + return; + } - if (this.blocks.length && block.height !== this.blocks[0].height + 1) { - this.blocks = []; - this.blocksFilled = false; - } + if (this.blocks.length && block.height !== this.blocks[0].height + 1) { + this.blocks = []; + this.blocksFilled = false; + } - this.blocks.unshift(block); - this.blocks = this.blocks.slice(0, this.stateService.env.KEEP_BLOCKS_AMOUNT); + this.blocks.unshift(block); + this.blocks = this.blocks.slice(0, this.stateService.env.KEEP_BLOCKS_AMOUNT); - if (this.blocksFilled && !this.tabHidden && block.extras) { - block.extras.stage = block.extras.matchRate >= 66 ? 1 : 2; - } + if (txConfirmed) { + this.markHeight = block.height; + this.moveArrowToPosition(true, true); + } else { + this.moveArrowToPosition(true, false); + } - if (txConfirmed) { - this.markHeight = block.height; - this.moveArrowToPosition(true, true); - } else { - this.moveArrowToPosition(true, false); - } - - this.blockStyles = []; - this.blocks.forEach((b) => this.blockStyles.push(this.getStyleForBlock(b))); - setTimeout(() => { this.blockStyles = []; - this.blocks.forEach((b) => this.blockStyles.push(this.getStyleForBlock(b))); - this.cd.markForCheck(); - }, 50); + if (this.blocksFilled) { + this.blocks.forEach((b, i) => this.blockStyles.push(this.getStyleForBlock(b, i, i ? -155 : -205))); + setTimeout(() => { + this.blockStyles = []; + this.blocks.forEach((b, i) => this.blockStyles.push(this.getStyleForBlock(b, i))); + this.cd.markForCheck(); + }, 50); + } else { + this.blocks.forEach((b, i) => this.blockStyles.push(this.getStyleForBlock(b, i))); + } - if (this.blocks.length === this.stateService.env.KEEP_BLOCKS_AMOUNT) { - this.blocksFilled = true; + if (this.blocks.length === this.stateService.env.KEEP_BLOCKS_AMOUNT) { + this.blocksFilled = true; + } + this.cd.markForCheck(); + }); + } else { + this.blockPageSubscription = this.cacheService.loadedBlocks$.subscribe((block) => { + if (block.height <= this.height && block.height > this.height - this.count) { + this.onBlockLoaded(block); } - this.cd.markForCheck(); }); + } this.markBlockSubscription = this.stateService.markBlock$ .subscribe((state) => { @@ -123,10 +144,26 @@ export class BlockchainBlocksComponent implements OnInit, OnDestroy { this.moveArrowToPosition(false); this.cd.markForCheck(); }); + + if (this.static) { + this.updateStaticBlocks(); + } + } + + ngOnChanges(changes: SimpleChanges): void { + if (this.static) { + const animateSlide = changes.height && (changes.height.currentValue === changes.height.previousValue + 1); + this.updateStaticBlocks(animateSlide); + } } ngOnDestroy() { - this.blocksSubscription.unsubscribe(); + if (this.blocksSubscription) { + this.blocksSubscription.unsubscribe(); + } + if (this.blockPageSubscription) { + this.blockPageSubscription.unsubscribe(); + } this.networkSubscription.unsubscribe(); this.tabHiddenSubscription.unsubscribe(); this.markBlockSubscription.unsubscribe(); @@ -142,13 +179,13 @@ export class BlockchainBlocksComponent implements OnInit, OnDestroy { const blockindex = this.blocks.findIndex((b) => b.height === this.markHeight); if (blockindex > -1) { if (!animate) { - this.transition = 'inherit'; + this.arrowTransition = 'inherit'; } this.arrowVisible = true; if (newBlockFromLeft) { this.arrowLeftPx = blockindex * 155 + 30 - 205; setTimeout(() => { - this.transition = '2s'; + this.arrowTransition = '2s'; this.arrowLeftPx = blockindex * 155 + 30; this.cd.markForCheck(); }, 50); @@ -156,45 +193,117 @@ export class BlockchainBlocksComponent implements OnInit, OnDestroy { this.arrowLeftPx = blockindex * 155 + 30; if (!animate) { setTimeout(() => { - this.transition = '2s'; + this.arrowTransition = '2s'; this.cd.markForCheck(); - }); + }, 50); } } + } else { + this.arrowVisible = false; } } - trackByBlocksFn(index: number, item: BlockExtended) { + trackByBlocksFn(index: number, item: BlockchainBlock) { return item.height; } - getStyleForBlock(block: BlockExtended) { + updateStaticBlocks(animateSlide: boolean = false) { + // reset blocks + this.blocks = []; + this.blockStyles = []; + while (this.blocks.length < this.count) { + const height = this.height - this.blocks.length; + let block; + if (height >= 0) { + this.cacheService.loadBlock(height); + block = this.cacheService.getCachedBlock(height) || null; + } + this.blocks.push(block || { + placeholder: height < 0, + loading: height >= 0, + id: '', + height, + version: 0, + timestamp: 0, + bits: 0, + nonce: 0, + difficulty: 0, + merkle_root: '', + tx_count: 0, + size: 0, + weight: 0, + previousblockhash: '', + }); + } + this.blocks = this.blocks.slice(0, this.count); + this.blockStyles = []; + this.blocks.forEach((b, i) => this.blockStyles.push(this.getStyleForBlock(b, i, animateSlide ? -155 : 0))); + this.cd.markForCheck(); + if (animateSlide) { + // animate blocks slide right + setTimeout(() => { + this.blockStyles = []; + this.blocks.forEach((b, i) => this.blockStyles.push(this.getStyleForBlock(b, i))); + this.cd.markForCheck(); + }, 50); + this.moveArrowToPosition(true, true); + } else { + this.moveArrowToPosition(false, false); + } + } + + onBlockLoaded(block: BlockExtended) { + const blockIndex = this.height - block.height; + if (blockIndex >= 0 && blockIndex < this.blocks.length) { + this.blocks[blockIndex] = block; + this.blockStyles[blockIndex] = this.getStyleForBlock(block, blockIndex); + } + this.cd.markForCheck(); + } + + getStyleForBlock(block: BlockchainBlock, index: number, animateEnterFrom: number = 0) { + if (!block || block.placeholder) { + return this.getStyleForPlaceholderBlock(index, animateEnterFrom); + } else if (block.loading) { + return this.getStyleForLoadingBlock(index, animateEnterFrom); + } const greenBackgroundHeight = 100 - (block.weight / this.stateService.env.BLOCK_WEIGHT_UNITS) * 100; let addLeft = 0; - if (block?.extras?.stage === 1) { - block.extras.stage = 2; - addLeft = -205; + if (animateEnterFrom) { + addLeft = animateEnterFrom || 0; } return { - left: addLeft + 155 * this.blocks.indexOf(block) + 'px', + left: addLeft + 155 * index + 'px', background: `repeating-linear-gradient( #2d3348, #2d3348 ${greenBackgroundHeight}%, ${this.gradientColors[this.network][0]} ${Math.max(greenBackgroundHeight, 0)}%, ${this.gradientColors[this.network][1]} 100% )`, + transition: animateEnterFrom ? 'background 2s, transform 1s' : null, }; } - getStyleForEmptyBlock(block: BlockExtended) { - let addLeft = 0; + getStyleForLoadingBlock(index: number, animateEnterFrom: number = 0) { + const addLeft = animateEnterFrom || 0; - if (block?.extras?.stage === 1) { - block.extras.stage = 2; - addLeft = -205; - } + return { + left: addLeft + (155 * index) + 'px', + background: "#2d3348", + }; + } + + getStyleForPlaceholderBlock(index: number, animateEnterFrom: number = 0) { + const addLeft = animateEnterFrom || 0; + return { + left: addLeft + (155 * index) + 'px', + }; + } + + getStyleForEmptyBlock(block: BlockExtended, animateEnterFrom: number = 0) { + const addLeft = animateEnterFrom || 0; return { left: addLeft + 155 * this.emptyBlocks.indexOf(block) + 'px', @@ -219,7 +328,6 @@ export class BlockchainBlocksComponent implements OnInit, OnDestroy { weight: 0, previousblockhash: '', matchRate: 0, - stage: 0, }); } return emptyBlocks; diff --git a/frontend/src/app/components/blockchain/blockchain.component.html b/frontend/src/app/components/blockchain/blockchain.component.html index 66ae8dd43..ad2e5e86a 100644 --- a/frontend/src/app/components/blockchain/blockchain.component.html +++ b/frontend/src/app/components/blockchain/blockchain.component.html @@ -2,10 +2,14 @@
- - +
+ + + + +
-
+
diff --git a/frontend/src/app/components/blockchain/blockchain.component.scss b/frontend/src/app/components/blockchain/blockchain.component.scss index df609ff40..63ca22626 100644 --- a/frontend/src/app/components/blockchain/blockchain.component.scss +++ b/frontend/src/app/components/blockchain/blockchain.component.scss @@ -72,6 +72,15 @@ position: relative; } +.scroll-spacer { + position: absolute; + top: 0; + left: 0; + width: 1px; + height: 1px; + pointer-events: none; +} + .loading-block { position: absolute; text-align: center; diff --git a/frontend/src/app/components/blockchain/blockchain.component.ts b/frontend/src/app/components/blockchain/blockchain.component.ts index e99b3532d..0ad3625ea 100644 --- a/frontend/src/app/components/blockchain/blockchain.component.ts +++ b/frontend/src/app/components/blockchain/blockchain.component.ts @@ -1,4 +1,4 @@ -import { Component, OnInit, OnDestroy, ChangeDetectionStrategy } from '@angular/core'; +import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, Input, OnChanges, SimpleChanges } from '@angular/core'; import { Subscription } from 'rxjs'; import { StateService } from '../../services/state.service'; @@ -9,6 +9,11 @@ import { StateService } from '../../services/state.service'; changeDetection: ChangeDetectionStrategy.OnPush, }) export class BlockchainComponent implements OnInit, OnDestroy { + @Input() pages: any[] = []; + @Input() pageIndex: number; + @Input() blocksPerPage: number = 8; + @Input() minScrollWidth: number = 0; + network: string; timeLtrSubscription: Subscription; timeLtr: boolean = this.stateService.timeLtr.value; @@ -29,6 +34,10 @@ export class BlockchainComponent implements OnInit, OnDestroy { this.timeLtrSubscription.unsubscribe(); } + trackByPageFn(index: number, item: { index: number }) { + return item.index; + } + toggleTimeDirection() { this.ltrTransitionEnabled = true; this.stateService.timeLtr.next(!this.timeLtr); diff --git a/frontend/src/app/components/search-form/search-form.component.html b/frontend/src/app/components/search-form/search-form.component.html index 4e38ea6e0..b881c6ea7 100644 --- a/frontend/src/app/components/search-form/search-form.component.html +++ b/frontend/src/app/components/search-form/search-form.component.html @@ -5,7 +5,7 @@
-
diff --git a/frontend/src/app/components/search-form/search-form.component.scss b/frontend/src/app/components/search-form/search-form.component.scss index 7c8161196..d59acadb9 100644 --- a/frontend/src/app/components/search-form/search-form.component.scss +++ b/frontend/src/app/components/search-form/search-form.component.scss @@ -43,9 +43,6 @@ form { @media (min-width: 1200px) { min-width: 300px; } - input { - border: 0px; - } .btn { width: 100px; } diff --git a/frontend/src/app/components/start/start.component.html b/frontend/src/app/components/start/start.component.html index 89b6efdc3..c3277cb9a 100644 --- a/frontend/src/app/components/start/start.component.html +++ b/frontend/src/app/components/start/start.component.html @@ -11,8 +11,9 @@
- +
diff --git a/frontend/src/app/components/start/start.component.ts b/frontend/src/app/components/start/start.component.ts index 37c94baa3..558e6f909 100644 --- a/frontend/src/app/components/start/start.component.ts +++ b/frontend/src/app/components/start/start.component.ts @@ -19,16 +19,51 @@ export class StartComponent implements OnInit, OnDestroy { blockchainScrollLeftInit: number; timeLtrSubscription: Subscription; timeLtr: boolean = this.stateService.timeLtr.value; + chainTipSubscription: Subscription; + chainTip: number = -1; + markBlockSubscription: Subscription; @ViewChild('blockchainContainer') blockchainContainer: ElementRef; + isMobile: boolean = false; + blockWidth = 155; + blocksPerPage: number = 1; + pageWidth: number; + firstPageWidth: number; + minScrollWidth: number; + pageIndex: number = 0; + pages: any[] = []; + pendingMark: number | void = null; + constructor( private stateService: StateService, ) { } ngOnInit() { + this.firstPageWidth = 40 + (this.blockWidth * this.stateService.env.KEEP_BLOCKS_AMOUNT); + this.onResize(); + this.updatePages(); this.timeLtrSubscription = this.stateService.timeLtr.subscribe((ltr) => { this.timeLtr = !!ltr; }); + this.chainTipSubscription = this.stateService.chainTip$.subscribe((height) => { + this.chainTip = height; + this.updatePages(); + if (this.pendingMark != null) { + this.scrollToBlock(this.pendingMark); + this.pendingMark = null; + } + }); + this.markBlockSubscription = this.stateService.markBlock$.subscribe((mark) => { + if (mark?.blockHeight != null) { + if (this.chainTip >=0) { + if (!this.blockInViewport(mark.blockHeight)) { + this.scrollToBlock(mark.blockHeight); + } + } else { + this.pendingMark = mark.blockHeight; + } + } + }); this.stateService.blocks$ .subscribe((blocks: any) => { if (this.stateService.network !== '') { @@ -55,6 +90,34 @@ export class StartComponent implements OnInit, OnDestroy { }); } + @HostListener('window:resize', ['$event']) + onResize(): void { + this.isMobile = window.innerWidth <= 767.98; + let firstVisibleBlock; + let offset; + if (this.blockchainContainer?.nativeElement != null) { + this.pages.forEach(page => { + const left = page.offset - this.getConvertedScrollOffset(); + const right = left + this.pageWidth; + if (left <= 0 && right > 0) { + const blockIndex = Math.max(0, Math.floor(left / -this.blockWidth)); + firstVisibleBlock = page.height - blockIndex; + offset = left + (blockIndex * this.blockWidth); + } + }); + } + + this.blocksPerPage = Math.ceil(window.innerWidth / this.blockWidth); + this.pageWidth = this.blocksPerPage * this.blockWidth; + this.minScrollWidth = this.firstPageWidth + (this.pageWidth * 2); + + if (firstVisibleBlock != null) { + this.scrollToBlock(firstVisibleBlock, offset); + } else { + this.updatePages(); + } + } + onMouseDown(event: MouseEvent) { this.mouseDragStartX = event.clientX; this.blockchainScrollLeftInit = this.blockchainContainer.nativeElement.scrollLeft; @@ -70,7 +133,7 @@ export class StartComponent implements OnInit, OnDestroy { if (this.mouseDragStartX != null) { this.stateService.setBlockScrollingInProgress(true); this.blockchainContainer.nativeElement.scrollLeft = - this.blockchainScrollLeftInit + this.mouseDragStartX - event.clientX + this.blockchainScrollLeftInit + this.mouseDragStartX - event.clientX; } } @HostListener('document:mouseup', []) @@ -79,7 +142,149 @@ export class StartComponent implements OnInit, OnDestroy { this.stateService.setBlockScrollingInProgress(false); } + onScroll(e) { + const middlePage = this.pageIndex === 0 ? this.pages[0] : this.pages[1]; + // compensate for css transform + const translation = (this.isMobile ? window.innerWidth * 0.95 : window.innerWidth * 0.5); + const backThreshold = middlePage.offset + (this.pageWidth * 0.5) + translation; + const forwardThreshold = middlePage.offset - (this.pageWidth * 0.5) + translation; + const scrollLeft = this.getConvertedScrollOffset(); + if (scrollLeft > backThreshold) { + if (this.shiftPagesBack()) { + this.addConvertedScrollOffset(-this.pageWidth); + this.blockchainScrollLeftInit -= this.pageWidth; + } + } else if (scrollLeft < forwardThreshold) { + if (this.shiftPagesForward()) { + this.addConvertedScrollOffset(this.pageWidth); + this.blockchainScrollLeftInit += this.pageWidth; + } + } + } + + scrollToBlock(height, blockOffset = 0) { + if (!this.blockchainContainer?.nativeElement) { + setTimeout(() => { this.scrollToBlock(height, blockOffset); }, 50); + return; + } + const targetHeight = this.isMobile ? height - 1 : height; + const viewingPageIndex = this.getPageIndexOf(targetHeight); + const pages = []; + this.pageIndex = Math.max(viewingPageIndex - 1, 0); + let viewingPage = this.getPageAt(viewingPageIndex); + const isLastPage = viewingPage.height < this.blocksPerPage; + if (isLastPage) { + this.pageIndex = Math.max(viewingPageIndex - 2, 0); + viewingPage = this.getPageAt(viewingPageIndex); + } + const left = viewingPage.offset - this.getConvertedScrollOffset(); + const blockIndex = viewingPage.height - targetHeight; + const targetOffset = (this.blockWidth * blockIndex) + left; + let deltaOffset = targetOffset - blockOffset; + + if (isLastPage) { + pages.push(this.getPageAt(viewingPageIndex - 2)); + } + if (viewingPageIndex > 1) { + pages.push(this.getPageAt(viewingPageIndex - 1)); + } + if (viewingPageIndex > 0) { + pages.push(viewingPage); + } + if (!isLastPage) { + pages.push(this.getPageAt(viewingPageIndex + 1)); + } + if (viewingPageIndex === 0) { + pages.push(this.getPageAt(viewingPageIndex + 2)); + } + + this.pages = pages; + this.addConvertedScrollOffset(deltaOffset); + } + + updatePages() { + const pages = []; + if (this.pageIndex > 0) { + pages.push(this.getPageAt(this.pageIndex)); + } + pages.push(this.getPageAt(this.pageIndex + 1)); + pages.push(this.getPageAt(this.pageIndex + 2)); + this.pages = pages; + } + + shiftPagesBack(): boolean { + const nextPage = this.getPageAt(this.pageIndex + 3); + if (nextPage.height >= 0) { + this.pageIndex++; + this.pages.forEach(page => page.offset -= this.pageWidth); + if (this.pageIndex !== 1) { + this.pages.shift(); + } + this.pages.push(this.getPageAt(this.pageIndex + 2)); + return true; + } else { + return false; + } + } + + shiftPagesForward(): boolean { + if (this.pageIndex > 0) { + this.pageIndex--; + this.pages.forEach(page => page.offset += this.pageWidth); + this.pages.pop(); + if (this.pageIndex) { + this.pages.unshift(this.getPageAt(this.pageIndex)); + } + return true; + } + return false; + } + + getPageAt(index: number) { + const height = this.chainTip - 8 - ((index - 1) * this.blocksPerPage) + return { + offset: this.firstPageWidth + (this.pageWidth * (index - 1 - this.pageIndex)), + height: height, + depth: this.chainTip - height, + index: index, + }; + } + + getPageIndexOf(height: number): number { + const delta = this.chainTip - 8 - height; + return Math.max(0, Math.floor(delta / this.blocksPerPage) + 1); + } + + blockInViewport(height: number): boolean { + const firstHeight = this.pages[0].height; + const translation = (this.isMobile ? window.innerWidth * 0.95 : window.innerWidth * 0.5); + const firstX = this.pages[0].offset - this.getConvertedScrollOffset() + translation; + const xPos = firstX + ((firstHeight - height) * 155); + return xPos > -55 && xPos < (window.innerWidth - 100); + } + + getConvertedScrollOffset(): number { + if (this.timeLtr) { + return -this.blockchainContainer?.nativeElement?.scrollLeft || 0; + } else { + return this.blockchainContainer?.nativeElement?.scrollLeft || 0; + } + } + + addConvertedScrollOffset(offset: number): void { + if (!this.blockchainContainer?.nativeElement) { + return; + } + if (this.timeLtr) { + this.blockchainContainer.nativeElement.scrollLeft -= offset; + } else { + this.blockchainContainer.nativeElement.scrollLeft += offset; + } + } + ngOnDestroy() { this.timeLtrSubscription.unsubscribe(); + this.chainTipSubscription.unsubscribe(); + this.markBlockSubscription.unsubscribe(); } } diff --git a/frontend/src/app/components/transaction/transaction-preview.component.ts b/frontend/src/app/components/transaction/transaction-preview.component.ts index 9d2d502b4..6db0e588c 100644 --- a/frontend/src/app/components/transaction/transaction-preview.component.ts +++ b/frontend/src/app/components/transaction/transaction-preview.component.ts @@ -11,6 +11,7 @@ import { import { Transaction, Vout } from '../../interfaces/electrs.interface'; import { of, merge, Subscription, Observable, Subject, from } from 'rxjs'; import { StateService } from '../../services/state.service'; +import { CacheService } from '../../services/cache.service'; import { OpenGraphService } from '../../services/opengraph.service'; import { ApiService } from '../../services/api.service'; import { SeoService } from '../../services/seo.service'; @@ -45,6 +46,7 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy { private route: ActivatedRoute, private electrsApiService: ElectrsApiService, private stateService: StateService, + private cacheService: CacheService, private apiService: ApiService, private seoService: SeoService, private openGraphService: OpenGraphService, @@ -97,7 +99,7 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy { }), switchMap(() => { let transactionObservable$: Observable; - const cached = this.stateService.getTxFromCache(this.txId); + const cached = this.cacheService.getTxFromCache(this.txId); if (cached && cached.fee !== -1) { transactionObservable$ = of(cached); } else { diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index 575c00637..cd85d0f4f 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -13,6 +13,7 @@ import { import { Transaction } from '../../interfaces/electrs.interface'; import { of, merge, Subscription, Observable, Subject, timer, combineLatest, from, throwError } from 'rxjs'; import { StateService } from '../../services/state.service'; +import { CacheService } from '../../services/cache.service'; import { WebsocketService } from '../../services/websocket.service'; import { AudioService } from '../../services/audio.service'; import { ApiService } from '../../services/api.service'; @@ -74,6 +75,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { private relativeUrlPipe: RelativeUrlPipe, private electrsApiService: ElectrsApiService, private stateService: StateService, + private cacheService: CacheService, private websocketService: WebsocketService, private audioService: AudioService, private apiService: ApiService, @@ -131,26 +133,20 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.cpfpInfo = null; return; } - if (cpfpInfo.effectiveFeePerVsize) { - this.tx.effectiveFeePerVsize = cpfpInfo.effectiveFeePerVsize; - } else { - const lowerFeeParents = cpfpInfo.ancestors.filter( - (parent) => parent.fee / (parent.weight / 4) < this.tx.feePerVsize - ); - let totalWeight = - this.tx.weight + - lowerFeeParents.reduce((prev, val) => prev + val.weight, 0); - let totalFees = - this.tx.fee + - lowerFeeParents.reduce((prev, val) => prev + val.fee, 0); - - if (cpfpInfo?.bestDescendant) { - totalWeight += cpfpInfo?.bestDescendant.weight; - totalFees += cpfpInfo?.bestDescendant.fee; - } - - this.tx.effectiveFeePerVsize = totalFees / (totalWeight / 4); + // merge ancestors/descendants + const relatives = [...(cpfpInfo.ancestors || []), ...(cpfpInfo.descendants || [])]; + if (cpfpInfo.bestDescendant && !cpfpInfo.descendants?.length) { + relatives.push(cpfpInfo.bestDescendant); } + let totalWeight = + this.tx.weight + + relatives.reduce((prev, val) => prev + val.weight, 0); + let totalFees = + this.tx.fee + + relatives.reduce((prev, val) => prev + val.fee, 0); + + this.tx.effectiveFeePerVsize = totalFees / (totalWeight / 4); + if (!this.tx.status.confirmed) { this.stateService.markBlock$.next({ txFeePerVSize: this.tx.effectiveFeePerVsize, @@ -203,7 +199,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { }), switchMap(() => { let transactionObservable$: Observable; - const cached = this.stateService.getTxFromCache(this.txId); + const cached = this.cacheService.getTxFromCache(this.txId); if (cached && cached.fee !== -1) { transactionObservable$ = of(cached); } else { @@ -302,7 +298,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.waitingForTransaction = false; } this.rbfTransaction = rbfTransaction; - this.stateService.setTxCache([this.rbfTransaction]); + this.cacheService.setTxCache([this.rbfTransaction]); }); this.queryParamsSubscription = this.route.queryParams.subscribe((params) => { diff --git a/frontend/src/app/components/transactions-list/transactions-list.component.ts b/frontend/src/app/components/transactions-list/transactions-list.component.ts index 8b4fabf6e..67df2daa2 100644 --- a/frontend/src/app/components/transactions-list/transactions-list.component.ts +++ b/frontend/src/app/components/transactions-list/transactions-list.component.ts @@ -1,5 +1,6 @@ import { Component, OnInit, Input, ChangeDetectionStrategy, OnChanges, Output, EventEmitter, ChangeDetectorRef } from '@angular/core'; import { StateService } from '../../services/state.service'; +import { CacheService } from '../../services/cache.service'; import { Observable, ReplaySubject, BehaviorSubject, merge, Subscription } from 'rxjs'; import { Outspend, Transaction, Vin, Vout } from '../../interfaces/electrs.interface'; import { ElectrsApiService } from '../../services/electrs-api.service'; @@ -44,6 +45,7 @@ export class TransactionsListComponent implements OnInit, OnChanges { constructor( public stateService: StateService, + private cacheService: CacheService, private electrsApiService: ElectrsApiService, private apiService: ApiService, private assetsService: AssetsService, @@ -123,7 +125,7 @@ export class TransactionsListComponent implements OnInit, OnChanges { } this.transactionsLength = this.transactions.length; - this.stateService.setTxCache(this.transactions); + this.cacheService.setTxCache(this.transactions); this.transactions.forEach((tx) => { tx['@voutLimit'] = true; diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 2e6b94988..c35eb8098 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -24,7 +24,6 @@ export interface CpfpInfo { ancestors: Ancestor[]; descendants?: Ancestor[]; bestDescendant?: BestDescendant | null; - effectiveFeePerVsize?: number; } export interface DifficultyAdjustment { @@ -122,8 +121,6 @@ export interface BlockExtension { name: string; slug: string; } - - stage?: number; // Frontend only } export interface BlockExtended extends Block { diff --git a/frontend/src/app/services/cache.service.ts b/frontend/src/app/services/cache.service.ts new file mode 100644 index 000000000..be37164dd --- /dev/null +++ b/frontend/src/app/services/cache.service.ts @@ -0,0 +1,105 @@ +import { Injectable } from '@angular/core'; +import { firstValueFrom, Subject, Subscription} from 'rxjs'; +import { Transaction } from '../interfaces/electrs.interface'; +import { BlockExtended } from '../interfaces/node-api.interface'; +import { StateService } from './state.service'; +import { ApiService } from './api.service'; + +const BLOCK_CACHE_SIZE = 500; +const KEEP_RECENT_BLOCKS = 50; + +@Injectable({ + providedIn: 'root' +}) +export class CacheService { + loadedBlocks$ = new Subject(); + tip: number = 0; + + txCache: { [txid: string]: Transaction } = {}; + + blockCache: { [height: number]: BlockExtended } = {}; + blockLoading: { [height: number]: boolean } = {}; + copiesInBlockQueue: { [height: number]: number } = {}; + blockPriorities: number[] = []; + + constructor( + private stateService: StateService, + private apiService: ApiService, + ) { + this.stateService.blocks$.subscribe(([block]) => { + this.addBlockToCache(block); + this.clearBlocks(); + }); + this.stateService.chainTip$.subscribe((height) => { + this.tip = height; + }); + } + + setTxCache(transactions) { + this.txCache = {}; + transactions.forEach(tx => { + this.txCache[tx.txid] = tx; + }); + } + + getTxFromCache(txid) { + if (this.txCache && this.txCache[txid]) { + return this.txCache[txid]; + } else { + return null; + } + } + + addBlockToCache(block: BlockExtended) { + this.blockCache[block.height] = block; + this.bumpBlockPriority(block.height); + } + + async loadBlock(height) { + if (!this.blockCache[height] && !this.blockLoading[height]) { + const chunkSize = 10; + const maxHeight = Math.ceil(height / chunkSize) * chunkSize; + for (let i = 0; i < chunkSize; i++) { + this.blockLoading[maxHeight - i] = true; + } + const result = await firstValueFrom(this.apiService.getBlocks$(maxHeight)); + for (let i = 0; i < chunkSize; i++) { + delete this.blockLoading[maxHeight - i]; + } + if (result && result.length) { + result.forEach(block => { + this.addBlockToCache(block); + this.loadedBlocks$.next(block); + }); + } + this.clearBlocks(); + } else { + this.bumpBlockPriority(height); + } + } + + // increase the priority of a block, to delay removal + bumpBlockPriority(height) { + this.blockPriorities.push(height); + this.copiesInBlockQueue[height] = (this.copiesInBlockQueue[height] || 0) + 1; + } + + // remove lowest priority blocks from the cache + clearBlocks() { + while (Object.keys(this.blockCache).length > (BLOCK_CACHE_SIZE + KEEP_RECENT_BLOCKS) && this.blockPriorities.length > KEEP_RECENT_BLOCKS) { + const height = this.blockPriorities.shift(); + if (this.copiesInBlockQueue[height] > 1) { + this.copiesInBlockQueue[height]--; + } else if ((this.tip - height) < KEEP_RECENT_BLOCKS) { + this.bumpBlockPriority(height); + } else { + delete this.blockCache[height]; + delete this.copiesInBlockQueue[height]; + } + } + } + + getCachedBlock(height) { + return this.blockCache[height]; + } +} \ No newline at end of file diff --git a/frontend/src/app/services/state.service.ts b/frontend/src/app/services/state.service.ts index 8a87b97e5..86efa57f8 100644 --- a/frontend/src/app/services/state.service.ts +++ b/frontend/src/app/services/state.service.ts @@ -104,6 +104,7 @@ export class StateService { backendInfo$ = new ReplaySubject(1); loadingIndicators$ = new ReplaySubject(1); recommendedFees$ = new ReplaySubject(1); + chainTip$ = new ReplaySubject(-1); live2Chart$ = new Subject(); @@ -111,15 +112,13 @@ export class StateService { connectionState$ = new BehaviorSubject<0 | 1 | 2>(2); isTabHidden$: Observable; - markBlock$ = new ReplaySubject(); + markBlock$ = new BehaviorSubject({}); keyNavigation$ = new Subject(); blockScrolling$: Subject = new Subject(); timeLtr: BehaviorSubject; hideFlow: BehaviorSubject; - txCache: { [txid: string]: Transaction } = {}; - constructor( @Inject(PLATFORM_ID) private platformId: any, @Inject(LOCALE_ID) private locale: string, @@ -274,18 +273,15 @@ export class StateService { return this.network === 'liquid' || this.network === 'liquidtestnet'; } - setTxCache(transactions) { - this.txCache = {}; - transactions.forEach(tx => { - this.txCache[tx.txid] = tx; - }); + resetChainTip() { + this.latestBlockHeight = -1; + this.chainTip$.next(-1); } - - getTxFromCache(txid) { - if (this.txCache && this.txCache[txid]) { - return this.txCache[txid]; - } else { - return null; + + updateChainTip(height) { + if (height > this.latestBlockHeight) { + this.latestBlockHeight = height; + this.chainTip$.next(height); } } } diff --git a/frontend/src/app/services/websocket.service.ts b/frontend/src/app/services/websocket.service.ts index 7cb279a08..d58ab58c9 100644 --- a/frontend/src/app/services/websocket.service.ts +++ b/frontend/src/app/services/websocket.service.ts @@ -70,7 +70,7 @@ export class WebsocketService { clearTimeout(this.onlineCheckTimeout); clearTimeout(this.onlineCheckTimeoutTwo); - this.stateService.latestBlockHeight = -1; + this.stateService.resetChainTip(); this.websocketSubject.complete(); this.subscription.unsubscribe(); @@ -224,12 +224,14 @@ export class WebsocketService { handleResponse(response: WebsocketResponse) { if (response.blocks && response.blocks.length) { const blocks = response.blocks; + let maxHeight = 0; blocks.forEach((block: BlockExtended) => { if (block.height > this.stateService.latestBlockHeight) { - this.stateService.latestBlockHeight = block.height; + maxHeight = Math.max(maxHeight, block.height); this.stateService.blocks$.next([block, false]); } }); + this.stateService.updateChainTip(maxHeight); } if (response.tx) { @@ -238,7 +240,7 @@ export class WebsocketService { if (response.block) { if (response.block.height > this.stateService.latestBlockHeight) { - this.stateService.latestBlockHeight = response.block.height; + this.stateService.updateChainTip(response.block.height); this.stateService.blocks$.next([response.block, !!response.txConfirmed]); } diff --git a/frontend/src/styles.scss b/frontend/src/styles.scss index d3d16d12e..b0cdb5ef6 100644 --- a/frontend/src/styles.scss +++ b/frontend/src/styles.scss @@ -115,10 +115,38 @@ body { } .form-control { - color: #495057; + color: #fff; + background-color: #2d3348; + border: 1px solid rgba(17, 19, 31, 0.2); } + .form-control:focus { - color: #000; + color: #fff; + background-color: #2d3348; +} + +.btn-purple { + background-color: #653b9c; + border-color: #653b9c; +} + +.btn-purple:not(:disabled):not(.disabled):active, .btn-purple:not(:disabled):not(.disabled).active, .show > .btn-purple.dropdown-toggle { + color: #fff; + background-color: #4d2d77; + border-color: #472a6e; +} + +.btn-purple:focus, .btn-purple.focus { + color: #fff; + background-color: #533180; + border-color: #4d2d77; + box-shadow: 0 0 0 0.2rem rgb(124 88 171 / 50%); +} + +.btn-purple:hover { + color: #fff; + background-color: #533180; + border-color: #4d2d77; } .form-control.form-control-secondary { diff --git a/frontend/sync-assets.js b/frontend/sync-assets.js index 9c447bf7d..879a7fba4 100644 --- a/frontend/sync-assets.js +++ b/frontend/sync-assets.js @@ -54,9 +54,13 @@ function downloadMiningPoolLogos() { response.on('end', () => { let response_body = Buffer.concat(chunks_of_data); - const poolLogos = JSON.parse(response_body.toString()); - for (const poolLogo of poolLogos) { - download(`${PATH}/mining-pools/${poolLogo.name}`, poolLogo.download_url); + try { + const poolLogos = JSON.parse(response_body.toString()); + for (const poolLogo of poolLogos) { + download(`${PATH}/mining-pools/${poolLogo.name}`, poolLogo.download_url); + } + } catch (e) { + console.error(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); } }); @@ -66,7 +70,6 @@ function downloadMiningPoolLogos() { }) } -const poolsJsonUrl = 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json'; let assetsJsonUrl = 'https://raw.githubusercontent.com/mempool/asset_registry_db/master/index.json'; let assetsMinimalJsonUrl = 'https://raw.githubusercontent.com/mempool/asset_registry_db/master/index.minimal.json'; @@ -82,8 +85,6 @@ console.log('Downloading assets'); download(PATH + 'assets.json', assetsJsonUrl); console.log('Downloading assets minimal'); download(PATH + 'assets.minimal.json', assetsMinimalJsonUrl); -console.log('Downloading mining pools info'); -download(PATH + 'pools.json', poolsJsonUrl); console.log('Downloading testnet assets'); download(PATH + 'assets-testnet.json', testnetAssetsJsonUrl); console.log('Downloading testnet assets minimal'); diff --git a/production/bitcoin.conf b/production/bitcoin.conf index 0fa1e943f..99fbaeed1 100644 --- a/production/bitcoin.conf +++ b/production/bitcoin.conf @@ -36,6 +36,12 @@ zmqpubrawtx=tcp://127.0.0.1:8335 #addnode=[2401:b140:2::92:204]:8333 #addnode=[2401:b140:2::92:205]:8333 #addnode=[2401:b140:2::92:206]:8333 +#addnode=[2401:b140:3::92:201]:8333 +#addnode=[2401:b140:3::92:202]:8333 +#addnode=[2401:b140:3::92:203]:8333 +#addnode=[2401:b140:3::92:204]:8333 +#addnode=[2401:b140:3::92:205]:8333 +#addnode=[2401:b140:3::92:206]:8333 [test] daemon=1 @@ -57,6 +63,12 @@ zmqpubrawtx=tcp://127.0.0.1:18335 #addnode=[2401:b140:2::92:204]:18333 #addnode=[2401:b140:2::92:205]:18333 #addnode=[2401:b140:2::92:206]:18333 +#addnode=[2401:b140:3::92:201]:18333 +#addnode=[2401:b140:3::92:202]:18333 +#addnode=[2401:b140:3::92:203]:18333 +#addnode=[2401:b140:3::92:204]:18333 +#addnode=[2401:b140:3::92:205]:18333 +#addnode=[2401:b140:3::92:206]:18333 [signet] daemon=1 @@ -78,3 +90,9 @@ zmqpubrawtx=tcp://127.0.0.1:38335 #addnode=[2401:b140:2::92:204]:38333 #addnode=[2401:b140:2::92:205]:38333 #addnode=[2401:b140:2::92:206]:38333 +#addnode=[2401:b140:3::92:201]:38333 +#addnode=[2401:b140:3::92:202]:38333 +#addnode=[2401:b140:3::92:203]:38333 +#addnode=[2401:b140:3::92:204]:38333 +#addnode=[2401:b140:3::92:205]:38333 +#addnode=[2401:b140:3::92:206]:38333 diff --git a/production/install b/production/install index ae28c2cdc..240af8dfc 100755 --- a/production/install +++ b/production/install @@ -251,6 +251,7 @@ MEMPOOL_BISQ_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') MEMPOOL_HOME=/mempool MEMPOOL_USER=mempool MEMPOOL_GROUP=mempool +MEMPOOL_MYSQL_CREDENTIALS="${MEMPOOL_HOME}/.mysql_credentials" # name of Tor hidden service in torrc MEMPOOL_TOR_HS=mempool @@ -1009,6 +1010,7 @@ osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_ osSudo "${MEMPOOL_USER}" ln -s mempool/production/mempool-build-all upgrade osSudo "${MEMPOOL_USER}" ln -s mempool/production/mempool-kill-all stop osSudo "${MEMPOOL_USER}" ln -s mempool/production/mempool-start-all start +osSudo "${MEMPOOL_USER}" ln -s mempool/production/mempool-reset-all reset case $OS in @@ -1869,7 +1871,7 @@ grant all on mempool_bisq.* to '${MEMPOOL_BISQ_USER}'@'localhost' identified by _EOF_ echo "[*] save MySQL credentials" -cat > ${MEMPOOL_HOME}/mysql_credentials << _EOF_ +cat > "${MEMPOOL_MYSQL_CREDENTIALS}" << _EOF_ declare -x MEMPOOL_MAINNET_USER="${MEMPOOL_MAINNET_USER}" declare -x MEMPOOL_MAINNET_PASS="${MEMPOOL_MAINNET_PASS}" declare -x MEMPOOL_TESTNET_USER="${MEMPOOL_TESTNET_USER}" @@ -1889,6 +1891,7 @@ declare -x MEMPOOL_LIQUIDTESTNET_PASS="${MEMPOOL_LIQUIDTESTNET_PASS}" declare -x MEMPOOL_BISQ_USER="${MEMPOOL_BISQ_USER}" declare -x MEMPOOL_BISQ_PASS="${MEMPOOL_BISQ_PASS}" _EOF_ +chown "${MEMPOOL_USER}:${MEMPOOL_GROUP}" "${MEMPOOL_MYSQL_CREDENTIALS}" ##### nginx diff --git a/production/mempool-build-all b/production/mempool-build-all index 048aeefdc..a4cb650d7 100755 --- a/production/mempool-build-all +++ b/production/mempool-build-all @@ -12,7 +12,7 @@ ELEMENTS_RPC_USER=$(grep '^rpcuser' /elements/elements.conf | cut -d '=' -f2) ELEMENTS_RPC_PASS=$(grep '^rpcpassword' /elements/elements.conf | cut -d '=' -f2) # get mysql credentials -MYSQL_CRED_FILE=${HOME}/mempool/mysql_credentials +MYSQL_CRED_FILE=${HOME}/.mysql_credentials if [ -f "${MYSQL_CRED_FILE}" ];then . ${MYSQL_CRED_FILE} fi diff --git a/production/mempool-frontend-config.mainnet.json b/production/mempool-frontend-config.mainnet.json index 525cdb115..e612c0440 100644 --- a/production/mempool-frontend-config.mainnet.json +++ b/production/mempool-frontend-config.mainnet.json @@ -9,5 +9,6 @@ "MEMPOOL_WEBSITE_URL": "https://mempool.space", "LIQUID_WEBSITE_URL": "https://liquid.network", "BISQ_WEBSITE_URL": "https://bisq.markets", - "ITEMS_PER_PAGE": 25 + "ITEMS_PER_PAGE": 25, + "LIGHTNING": true } diff --git a/production/mempool-reset-all b/production/mempool-reset-all new file mode 100755 index 000000000..22f004610 --- /dev/null +++ b/production/mempool-reset-all @@ -0,0 +1,3 @@ +#!/usr/bin/env zsh +rm $HOME/*/backend/mempool-config.json +rm $HOME/*/frontend/mempool-frontend-config.json