diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index b057e6141..7cace626c 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -6,86 +6,53 @@ on: jobs: cypress: if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')" - runs-on: ${{ matrix.os }} + runs-on: "ubuntu-latest" strategy: fail-fast: false matrix: - containers: [1, 2, 3, 4, 5] - os: ["ubuntu-latest"] - browser: [chrome] - name: E2E tests on ${{ matrix.browser }} - ${{ matrix.os }} + module: ["mempool", "liquid", "bisq"] + include: + - module: "mempool" + spec: | + cypress/e2e/mainnet/*.spec.ts + cypress/e2e/signet/*.spec.ts + cypress/e2e/testnet/*.spec.ts + - module: "liquid" + spec: | + cypress/e2e/liquid/liquid.spec.ts + cypress/e2e/liquidtestnet/liquidtestnet.spec.ts + - module: "bisq" + spec: | + cypress/e2e/bisq/bisq.spec.ts + + name: E2E tests for ${{ matrix.module }} steps: - name: Checkout uses: actions/checkout@v2 + with: + path: ${{ matrix.module }} + - name: Setup node uses: actions/setup-node@v2 with: node-version: 16.15.0 cache: 'npm' - cache-dependency-path: frontend/package-lock.json - - name: ${{ matrix.browser }} browser tests (Mempool) - uses: cypress-io/github-action@v4 - with: - tag: ${{ github.event_name }} - working-directory: frontend - build: npm run config:defaults:mempool - start: npm run start:local-staging - wait-on: 'http://localhost:4200' - wait-on-timeout: 120 - record: true - parallel: true - spec: | - cypress/e2e/mainnet/*.spec.ts - cypress/e2e/signet/*.spec.ts - cypress/e2e/testnet/*.spec.ts - group: Tests on ${{ matrix.browser }} (Mempool) - browser: ${{ matrix.browser }} - ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}' - env: - COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }} - CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} + cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json - - name: ${{ matrix.browser }} browser tests (Liquid) + - name: Chrome browser tests (${{ matrix.module }}) uses: cypress-io/github-action@v4 - if: always() with: tag: ${{ github.event_name }} - working-directory: frontend - build: npm run config:defaults:liquid + working-directory: ${{ matrix.module }}/frontend + build: npm run config:defaults:${{ matrix.module }} start: npm run start:local-staging wait-on: 'http://localhost:4200' wait-on-timeout: 120 record: true parallel: true - spec: | - cypress/e2e/liquid/liquid.spec.ts - cypress/e2e/liquidtestnet/liquidtestnet.spec.ts - group: Tests on ${{ matrix.browser }} (Liquid) - browser: ${{ matrix.browser }} - ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}' - env: - COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }} - CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} - - - name: ${{ matrix.browser }} browser tests (Bisq) - uses: cypress-io/github-action@v4 - if: always() - with: - tag: ${{ github.event_name }} - working-directory: frontend - build: npm run config:defaults:bisq - start: npm run start:local-staging - wait-on: 'http://localhost:4200' - wait-on-timeout: 120 - record: true - parallel: true - spec: cypress/e2e/bisq/bisq.spec.ts - group: Tests on ${{ matrix.browser }} (Bisq) - browser: ${{ matrix.browser }} + spec: ${{ matrix.spec }} + group: Tests on Chrome (${{ matrix.module }}) + browser: "chrome" ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}' env: COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }} diff --git a/backend/.gitignore b/backend/.gitignore index 5476d633c..67fb162c6 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -1,9 +1,9 @@ # See http://help.github.com/ignore-files/ for more about ignoring files. # production config and external assets -*.json -!mempool-config.sample.json +mempool-config.json +pools.json icons.json # compiled output diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index c8c6f4a98..e40977c6c 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -578,7 +578,7 @@ class Blocks { // Index the response if needed if (Common.blocksSummariesIndexingEnabled() === true) { - await BlocksSummariesRepository.$saveSummary(block.height, summary); + await BlocksSummariesRepository.$saveSummary({height: block.height, mined: summary}); } return summary.transactions; diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index e0a592b01..d9be6e1e7 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 31; + private static currentVersion = 33; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -297,7 +297,14 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `prices` ADD `id` int NULL AUTO_INCREMENT UNIQUE'); await this.$executeQuery('DROP TABLE IF EXISTS `blocks_prices`'); await this.$executeQuery(this.getCreateBlocksPricesTableQuery(), await this.$checkIfTableExists('blocks_prices')); + } + if (databaseSchemaVersion < 32 && isBitcoin == true) { + await this.$executeQuery('ALTER TABLE `blocks_summaries` ADD `template` JSON DEFAULT "[]"'); + } + + if (databaseSchemaVersion < 33 && isBitcoin == true) { + await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization", "country_iso_code") NOT NULL'); } } diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index a1cd6a41e..f0d7dc56b 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -13,6 +13,30 @@ class ChannelsApi { } } + public async $getAllChannelsGeo(): Promise { + try { + const query = `SELECT nodes_1.public_key as node1_public_key, nodes_1.alias AS node1_alias, + nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude, + nodes_2.public_key as node2_public_key, nodes_2.alias AS node2_alias, + nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude, + channels.capacity + FROM channels + JOIN nodes AS nodes_1 on nodes_1.public_key = channels.node1_public_key + JOIN nodes AS nodes_2 on nodes_2.public_key = channels.node2_public_key + WHERE nodes_1.latitude IS NOT NULL AND nodes_1.longitude IS NOT NULL + AND nodes_2.latitude IS NOT NULL AND nodes_2.longitude IS NOT NULL + `; + const [rows]: any = await DB.query(query); + return rows.map((row) => [ + row.node1_public_key, row.node1_alias, row.node1_longitude, row.node1_latitude, + row.node2_public_key, row.node2_alias, row.node2_longitude, row.node2_latitude, + row.capacity]); + } catch (e) { + logger.err('$getAllChannelsGeo error: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } + public async $searchChannelsById(search: string): Promise { try { const searchStripped = search.replace('%', '') + '%'; diff --git a/backend/src/api/explorer/channels.routes.ts b/backend/src/api/explorer/channels.routes.ts index 5ad1d8743..c6df30802 100644 --- a/backend/src/api/explorer/channels.routes.ts +++ b/backend/src/api/explorer/channels.routes.ts @@ -11,6 +11,7 @@ class ChannelsRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/search/:search', this.$searchChannelsById) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/:short_id', this.$getChannel) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels', this.$getChannelsForNode) + .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels-geo', this.$getChannelsGeo) ; } @@ -93,6 +94,15 @@ class ChannelsRoutes { } } + private async $getChannelsGeo(req: Request, res: Response) { + try { + const channels = await channelsApi.$getAllChannelsGeo(); + res.json(channels); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + } export default new ChannelsRoutes(); diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 590ed1f20..a14f7336f 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -93,6 +93,132 @@ class NodesApi { throw e; } } + + public async $getNodesISP() { + try { + let query = `SELECT nodes.as_number as ispId, geo_names.names as names, COUNT(DISTINCT nodes.public_key) as nodesCount, SUM(capacity) as capacity + FROM nodes + JOIN geo_names ON geo_names.id = nodes.as_number + JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key + GROUP BY as_number + ORDER BY COUNT(DISTINCT nodes.public_key) DESC + `; + const [nodesCountPerAS]: any = await DB.query(query); + + query = `SELECT COUNT(*) as total FROM nodes WHERE as_number IS NOT NULL`; + const [nodesWithAS]: any = await DB.query(query); + + const nodesPerAs: any[] = []; + for (const as of nodesCountPerAS) { + nodesPerAs.push({ + ispId: as.ispId, + name: JSON.parse(as.names), + count: as.nodesCount, + share: Math.floor(as.nodesCount / nodesWithAS[0].total * 10000) / 100, + capacity: as.capacity, + }) + } + + return nodesPerAs; + } catch (e) { + logger.err(`Cannot get nodes grouped by AS. Reason: ${e instanceof Error ? e.message : e}`); + throw e; + } + } + + public async $getNodesPerCountry(countryId: string) { + try { + const query = ` + SELECT node_stats.public_key, node_stats.capacity, node_stats.channels, nodes.alias, + UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at, + geo_names_city.names as city + FROM node_stats + JOIN ( + SELECT public_key, MAX(added) as last_added + FROM node_stats + GROUP BY public_key + ) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added + JOIN nodes ON nodes.public_key = node_stats.public_key + JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' + LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' + WHERE geo_names_country.id = ? + ORDER BY capacity DESC + `; + + const [rows]: any = await DB.query(query, [countryId]); + for (let i = 0; i < rows.length; ++i) { + rows[i].city = JSON.parse(rows[i].city); + } + return rows; + } catch (e) { + logger.err(`Cannot get nodes for country id ${countryId}. Reason: ${e instanceof Error ? e.message : e}`); + throw e; + } + } + + public async $getNodesPerISP(ISPId: string) { + try { + const query = ` + SELECT node_stats.public_key, node_stats.capacity, node_stats.channels, nodes.alias, + UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at, + geo_names_city.names as city, geo_names_country.names as country + FROM node_stats + JOIN ( + SELECT public_key, MAX(added) as last_added + FROM node_stats + GROUP BY public_key + ) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added + JOIN nodes ON nodes.public_key = node_stats.public_key + JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' + LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' + WHERE nodes.as_number = ? + ORDER BY capacity DESC + `; + + const [rows]: any = await DB.query(query, [ISPId]); + for (let i = 0; i < rows.length; ++i) { + rows[i].country = JSON.parse(rows[i].country); + rows[i].city = JSON.parse(rows[i].city); + } + return rows; + } catch (e) { + logger.err(`Cannot get nodes for ISP id ${ISPId}. Reason: ${e instanceof Error ? e.message : e}`); + throw e; + } + } + + public async $getNodesCountries() { + try { + let query = `SELECT geo_names.names as names, geo_names_iso.names as iso_code, COUNT(DISTINCT nodes.public_key) as nodesCount, SUM(capacity) as capacity + FROM nodes + JOIN geo_names ON geo_names.id = nodes.country_id AND geo_names.type = 'country' + JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code' + JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key + GROUP BY country_id + ORDER BY COUNT(DISTINCT nodes.public_key) DESC + `; + const [nodesCountPerCountry]: any = await DB.query(query); + + query = `SELECT COUNT(*) as total FROM nodes WHERE country_id IS NOT NULL`; + const [nodesWithAS]: any = await DB.query(query); + + const nodesPerCountry: any[] = []; + for (const country of nodesCountPerCountry) { + nodesPerCountry.push({ + name: JSON.parse(country.names), + iso: country.iso_code, + count: country.nodesCount, + share: Math.floor(country.nodesCount / nodesWithAS[0].total * 10000) / 100, + capacity: country.capacity, + }) + } + + return nodesPerCountry; + } catch (e) { + logger.err(`Cannot get nodes grouped by AS. Reason: ${e instanceof Error ? e.message : e}`); + throw e; + } + } } export default new NodesApi(); diff --git a/backend/src/api/explorer/nodes.routes.ts b/backend/src/api/explorer/nodes.routes.ts index 6c79c8201..bbc8efb5a 100644 --- a/backend/src/api/explorer/nodes.routes.ts +++ b/backend/src/api/explorer/nodes.routes.ts @@ -1,13 +1,19 @@ import config from '../../config'; import { Application, Request, Response } from 'express'; import nodesApi from './nodes.api'; +import DB from '../../database'; + class NodesRoutes { constructor() { } public initRoutes(app: Application) { app + .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/country/:country', this.$getNodesPerCountry) + .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp/:isp', this.$getNodesPerISP) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/search/:search', this.$searchNode) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/top', this.$getTopNodes) + .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp', this.$getNodesISP) + .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/countries', this.$getNodesCountries) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/statistics', this.$getHistoricalNodeStats) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key', this.$getNode) ; @@ -56,6 +62,85 @@ class NodesRoutes { res.status(500).send(e instanceof Error ? e.message : e); } } + + private async $getNodesISP(req: Request, res: Response) { + try { + const nodesPerAs = await nodesApi.$getNodesISP(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString()); + res.json(nodesPerAs); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getNodesPerCountry(req: Request, res: Response) { + try { + const [country]: any[] = await DB.query( + `SELECT geo_names.id, geo_names_country.names as country_names + FROM geo_names + JOIN geo_names geo_names_country on geo_names.id = geo_names_country.id AND geo_names_country.type = 'country' + WHERE geo_names.type = 'country_iso_code' AND geo_names.names = ?`, + [req.params.country] + ); + + if (country.length === 0) { + res.status(404).send(`This country does not exist or does not host any lightning nodes on clearnet`); + return; + } + + const nodes = await nodesApi.$getNodesPerCountry(country[0].id); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json({ + country: JSON.parse(country[0].country_names), + nodes: nodes, + }); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getNodesPerISP(req: Request, res: Response) { + try { + const [isp]: any[] = await DB.query( + `SELECT geo_names.names as isp_name + FROM geo_names + WHERE geo_names.type = 'as_organization' AND geo_names.id = ?`, + [req.params.isp] + ); + + if (isp.length === 0) { + res.status(404).send(`This ISP does not exist or does not host any lightning nodes on clearnet`); + return; + } + + const nodes = await nodesApi.$getNodesPerISP(req.params.isp); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json({ + isp: JSON.parse(isp[0].isp_name), + nodes: nodes, + }); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getNodesCountries(req: Request, res: Response) { + try { + const nodesPerAs = await nodesApi.$getNodesCountries(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString()); + res.json(nodesPerAs); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } } export default new NodesRoutes(); diff --git a/backend/src/api/mining/mining-routes.ts b/backend/src/api/mining/mining-routes.ts index 02262353f..f52d42d1f 100644 --- a/backend/src/api/mining/mining-routes.ts +++ b/backend/src/api/mining/mining-routes.ts @@ -26,7 +26,8 @@ class MiningRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', this.$getHistoricalBlockSizeAndWeight) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', this.$getDifficultyAdjustments) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/predictions/:interval', this.$getHistoricalBlockPrediction) - ; + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit) + ; } private async $getPool(req: Request, res: Response): Promise { @@ -233,6 +234,18 @@ class MiningRoutes { res.status(500).send(e instanceof Error ? e.message : e); } } + + public async $getBlockAudit(req: Request, res: Response) { + try { + const audit = await BlocksAuditsRepository.$getBlockAudit(req.params.hash); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString()); + res.json(audit); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } } export default new MiningRoutes(); diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 300341ef5..4896ee058 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -17,6 +17,7 @@ import rbfCache from './rbf-cache'; import difficultyAdjustment from './difficulty-adjustment'; import feeApi from './fee-api'; import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository'; +import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; class WebsocketHandler { private wss: WebSocket.Server | undefined; @@ -442,6 +443,22 @@ class WebsocketHandler { mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); if (Common.indexingEnabled()) { + const stripped = _mempoolBlocks[0].transactions.map((tx) => { + return { + txid: tx.txid, + vsize: tx.vsize, + fee: tx.fee ? Math.round(tx.fee) : 0, + value: tx.value, + }; + }); + BlocksSummariesRepository.$saveSummary({ + height: block.height, + template: { + id: block.id, + transactions: stripped + } + }); + BlocksAuditsRepository.$saveAudit({ time: block.timestamp, height: block.height, diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index 31d8ec785..54b723959 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -1,3 +1,4 @@ +import transactionUtils from '../api/transaction-utils'; import DB from '../database'; import logger from '../logger'; import { BlockAudit } from '../mempool.interfaces'; @@ -45,6 +46,30 @@ class BlocksAuditRepositories { throw e; } } + + public async $getBlockAudit(hash: string): Promise { + try { + const [rows]: any[] = await DB.query( + `SELECT blocks.height, blocks.hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.size, + blocks.weight, blocks.tx_count, + transactions, template, missing_txs as missingTxs, added_txs as addedTxs, match_rate as matchRate + FROM blocks_audits + JOIN blocks ON blocks.hash = blocks_audits.hash + JOIN blocks_summaries ON blocks_summaries.id = blocks_audits.hash + WHERE blocks_audits.hash = "${hash}" + `); + + rows[0].missingTxs = JSON.parse(rows[0].missingTxs); + rows[0].addedTxs = JSON.parse(rows[0].addedTxs); + rows[0].transactions = JSON.parse(rows[0].transactions); + rows[0].template = JSON.parse(rows[0].template); + + return rows[0]; + } catch (e: any) { + logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } } export default new BlocksAuditRepositories(); diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index 66c6b97f2..28b3cc7eb 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -17,14 +17,24 @@ class BlocksSummariesRepository { return undefined; } - public async $saveSummary(height: number, summary: BlockSummary) { + public async $saveSummary(params: { height: number, mined?: BlockSummary, template?: BlockSummary}) { + const blockId = params.mined?.id ?? params.template?.id; try { - await DB.query(`INSERT INTO blocks_summaries VALUE (?, ?, ?)`, [height, summary.id, JSON.stringify(summary.transactions)]); + const [dbSummary]: any[] = await DB.query(`SELECT * FROM blocks_summaries WHERE id = "${blockId}"`); + if (dbSummary.length === 0) { // First insertion + await DB.query(`INSERT INTO blocks_summaries VALUE (?, ?, ?, ?)`, [ + params.height, blockId, JSON.stringify(params.mined?.transactions ?? []), JSON.stringify(params.template?.transactions ?? []) + ]); + } else if (params.mined !== undefined) { // Update mined block summary + await DB.query(`UPDATE blocks_summaries SET transactions = ? WHERE id = "${params.mined.id}"`, [JSON.stringify(params.mined.transactions)]); + } else if (params.template !== undefined) { // Update template block summary + await DB.query(`UPDATE blocks_summaries SET template = ? WHERE id = "${params.template.id}"`, [JSON.stringify(params.template?.transactions)]); + } } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart - logger.debug(`Cannot save block summary for ${summary.id} because it has already been indexed, ignoring`); + logger.debug(`Cannot save block summary for ${blockId} because it has already been indexed, ignoring`); } else { - logger.debug(`Cannot save block summary for ${summary.id}. Reason: ${e instanceof Error ? e.message : e}`); + logger.debug(`Cannot save block summary for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`); throw e; } } @@ -44,7 +54,7 @@ class BlocksSummariesRepository { /** * Delete blocks from the database from blockHeight */ - public async $deleteBlocksFrom(blockHeight: number) { + public async $deleteBlocksFrom(blockHeight: number) { logger.info(`Delete newer blocks summary from height ${blockHeight} from the database`); try { diff --git a/backend/src/tasks/lightning/sync-tasks/node-locations.ts b/backend/src/tasks/lightning/sync-tasks/node-locations.ts index 444bd6557..e503190a0 100644 --- a/backend/src/tasks/lightning/sync-tasks/node-locations.ts +++ b/backend/src/tasks/lightning/sync-tasks/node-locations.ts @@ -39,6 +39,13 @@ export async function $lookupNodeLocation(): Promise { [city.country?.geoname_id, JSON.stringify(city.country?.names)]); } + // Store Country ISO code + if (city.country?.iso_code) { + await DB.query( + `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`, + [city.country?.geoname_id, city.country?.iso_code]); + } + // Store Division if (city.subdivisions && city.subdivisions[0]) { await DB.query( diff --git a/backend/src/tasks/price-feeds/kraken-api.ts b/backend/src/tasks/price-feeds/kraken-api.ts index ce76d62c2..ddb3c4f65 100644 --- a/backend/src/tasks/price-feeds/kraken-api.ts +++ b/backend/src/tasks/price-feeds/kraken-api.ts @@ -62,7 +62,7 @@ class KrakenApi implements PriceFeed { // CHF weekly price history goes back to timestamp 1575504000 (December 5, 2019) // AUD weekly price history goes back to timestamp 1591833600 (June 11, 2020) - const priceHistory: any = {}; // map: timestamp -> Prices + let priceHistory: any = {}; // map: timestamp -> Prices for (const currency of this.currencies) { const response = await query(this.urlHist.replace('{GRANULARITY}', '10080') + currency); @@ -83,6 +83,10 @@ class KrakenApi implements PriceFeed { } for (const time in priceHistory) { + if (priceHistory[time].USD === -1) { + delete priceHistory[time]; + continue; + } await PricesRepository.$savePrices(parseInt(time, 10), priceHistory[time]); } diff --git a/frontend/cypress/e2e/mainnet/mainnet.spec.ts b/frontend/cypress/e2e/mainnet/mainnet.spec.ts index 14f2c88de..4cc9a64c9 100644 --- a/frontend/cypress/e2e/mainnet/mainnet.spec.ts +++ b/frontend/cypress/e2e/mainnet/mainnet.spec.ts @@ -35,21 +35,23 @@ const getRectangle = ($el) => $el[0].getBoundingClientRect(); describe('Mainnet', () => { beforeEach(() => { //cy.intercept('/sockjs-node/info*').as('socket'); - cy.intercept('/api/block-height/*').as('block-height'); - cy.intercept('/api/block/*').as('block'); - cy.intercept('/api/block/*/txs/0').as('block-txs'); - cy.intercept('/api/tx/*/outspends').as('tx-outspends'); - cy.intercept('/resources/pools.json').as('pools'); + // cy.intercept('/api/block-height/*').as('block-height'); + // cy.intercept('/api/v1/block/*').as('block'); + // cy.intercept('/api/block/*/txs/0').as('block-txs'); + // cy.intercept('/api/v1/block/*/summary').as('block-summary'); + // cy.intercept('/api/v1/outspends/*').as('outspends'); + // cy.intercept('/api/tx/*/outspends').as('tx-outspends'); + // cy.intercept('/resources/pools.json').as('pools'); // Search Auto Complete cy.intercept('/api/address-prefix/1wiz').as('search-1wiz'); cy.intercept('/api/address-prefix/1wizS').as('search-1wizS'); cy.intercept('/api/address-prefix/1wizSA').as('search-1wizSA'); - Cypress.Commands.add('waitForBlockData', () => { - cy.wait('@tx-outspends'); - cy.wait('@pools'); - }); + // Cypress.Commands.add('waitForBlockData', () => { + // cy.wait('@tx-outspends'); + // cy.wait('@pools'); + // }); }); if (baseModule === 'mempool') { @@ -409,7 +411,7 @@ describe('Mainnet', () => { it('loads the tv screen - desktop', () => { cy.viewport('macbook-16'); - cy.visit('/'); + cy.visit('/graphs/mempool'); cy.waitForSkeletonGone(); cy.get('#btn-tv').click().then(() => { cy.viewport('macbook-16'); diff --git a/frontend/cypress/e2e/signet/signet.spec.ts b/frontend/cypress/e2e/signet/signet.spec.ts index d2bbd1196..2f09bc4b8 100644 --- a/frontend/cypress/e2e/signet/signet.spec.ts +++ b/frontend/cypress/e2e/signet/signet.spec.ts @@ -60,10 +60,10 @@ describe('Signet', () => { }); }); - describe('tv mode', () => { + describe.skip('tv mode', () => { it('loads the tv screen - desktop', () => { cy.viewport('macbook-16'); - cy.visit('/signet'); + cy.visit('/signet/graphs'); cy.waitForSkeletonGone(); cy.get('#btn-tv').click().then(() => { cy.get('.chart-holder').should('be.visible'); @@ -73,19 +73,17 @@ describe('Signet', () => { }); it('loads the tv screen - mobile', () => { - cy.visit('/signet'); + cy.visit('/signet/graphs'); cy.waitForSkeletonGone(); cy.get('#btn-tv').click().then(() => { cy.viewport('iphone-8'); cy.get('.chart-holder').should('be.visible'); cy.get('.tv-only').should('not.exist'); - //TODO: Remove comment when the bug is fixed - //cy.get('#mempool-block-0').should('be.visible'); + cy.get('#mempool-block-0').should('be.visible'); }); }); }); - it('loads the api screen', () => { cy.visit('/signet'); cy.waitForSkeletonGone(); diff --git a/frontend/cypress/e2e/testnet/testnet.spec.ts b/frontend/cypress/e2e/testnet/testnet.spec.ts index c0c07aa74..b05229a28 100644 --- a/frontend/cypress/e2e/testnet/testnet.spec.ts +++ b/frontend/cypress/e2e/testnet/testnet.spec.ts @@ -63,18 +63,17 @@ describe('Testnet', () => { describe('tv mode', () => { it('loads the tv screen - desktop', () => { cy.viewport('macbook-16'); - cy.visit('/testnet'); + cy.visit('/testnet/graphs'); cy.waitForSkeletonGone(); cy.get('#btn-tv').click().then(() => { cy.wait(1000); cy.get('.tv-only').should('not.exist'); - //TODO: Remove comment when the bug is fixed - //cy.get('#mempool-block-0').should('be.visible'); + cy.get('#mempool-block-0').should('be.visible'); }); }); it('loads the tv screen - mobile', () => { - cy.visit('/testnet'); + cy.visit('/testnet/graphs'); cy.waitForSkeletonGone(); cy.get('#btn-tv').click().then(() => { cy.viewport('iphone-6'); diff --git a/frontend/package-lock.json b/frontend/package-lock.json index c72038f38..04682aac5 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -34,6 +34,7 @@ "clipboard": "^2.0.10", "domino": "^2.1.6", "echarts": "~5.3.2", + "echarts-gl": "^2.0.9", "express": "^4.17.1", "lightweight-charts": "~3.8.0", "ngx-echarts": "8.0.1", @@ -6396,6 +6397,11 @@ "webpack": ">=4.0.1" } }, + "node_modules/claygl": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/claygl/-/claygl-1.3.0.tgz", + "integrity": "sha512-+gGtJjT6SSHD2l2yC3MCubW/sCV40tZuSs5opdtn79vFSGUgp/lH139RNEQ6Jy078/L0aV8odCw8RSrUcMfLaQ==" + }, "node_modules/clean-stack": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", @@ -8107,6 +8113,18 @@ "zrender": "5.3.1" } }, + "node_modules/echarts-gl": { + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/echarts-gl/-/echarts-gl-2.0.9.tgz", + "integrity": "sha512-oKeMdkkkpJGWOzjgZUsF41DOh6cMsyrGGXimbjK2l6Xeq/dBQu4ShG2w2Dzrs/1bD27b2pLTGSaUzouY191gzA==", + "dependencies": { + "claygl": "^1.2.1", + "zrender": "^5.1.1" + }, + "peerDependencies": { + "echarts": "^5.1.2" + } + }, "node_modules/echarts/node_modules/tslib": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz", @@ -22520,6 +22538,11 @@ "integrity": "sha512-g38K9Cm5WRwlaH6g03B9OEz/0qRizI+2I7n+Gz+L5DxXJAPAiWQvwlYNm1V1jkdpUv95bOe/ASm2vfi/G560jQ==", "requires": {} }, + "claygl": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/claygl/-/claygl-1.3.0.tgz", + "integrity": "sha512-+gGtJjT6SSHD2l2yC3MCubW/sCV40tZuSs5opdtn79vFSGUgp/lH139RNEQ6Jy078/L0aV8odCw8RSrUcMfLaQ==" + }, "clean-stack": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", @@ -23866,6 +23889,15 @@ } } }, + "echarts-gl": { + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/echarts-gl/-/echarts-gl-2.0.9.tgz", + "integrity": "sha512-oKeMdkkkpJGWOzjgZUsF41DOh6cMsyrGGXimbjK2l6Xeq/dBQu4ShG2w2Dzrs/1bD27b2pLTGSaUzouY191gzA==", + "requires": { + "claygl": "^1.2.1", + "zrender": "^5.1.1" + } + }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index f2d54135e..d2f7f2f6c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -88,6 +88,7 @@ "clipboard": "^2.0.10", "domino": "^2.1.6", "echarts": "~5.3.2", + "echarts-gl": "^2.0.9", "express": "^4.17.1", "lightweight-charts": "~3.8.0", "ngx-echarts": "8.0.1", diff --git a/frontend/proxy.conf.js b/frontend/proxy.conf.js index 77a77bb5a..ab2240c03 100644 --- a/frontend/proxy.conf.js +++ b/frontend/proxy.conf.js @@ -85,7 +85,7 @@ if (configContent && configContent.BASE_MODULE == "liquid") { }); } else { PROXY_CONFIG.push({ - context: ['/resources/pools.json', '/resources/assets.json', '/resources/assets.minimal.json'], + context: ['/resources/pools.json', '/resources/assets.json', '/resources/assets.minimal.json', '/resources/worldmap.json'], target: "https://mempool.space", secure: false, changeOrigin: true, diff --git a/frontend/src/app/app-routing.module.ts b/frontend/src/app/app-routing.module.ts index 564b8653b..c9f7e19d4 100644 --- a/frontend/src/app/app-routing.module.ts +++ b/frontend/src/app/app-routing.module.ts @@ -3,8 +3,11 @@ import { Routes, RouterModule, PreloadAllModules } from '@angular/router'; import { StartComponent } from './components/start/start.component'; import { TransactionComponent } from './components/transaction/transaction.component'; import { BlockComponent } from './components/block/block.component'; +import { BlockAuditComponent } from './components/block-audit/block-audit.component'; +import { BlockPreviewComponent } from './components/block/block-preview.component'; import { AddressComponent } from './components/address/address.component'; import { MasterPageComponent } from './components/master-page/master-page.component'; +import { MasterPagePreviewComponent } from './components/master-page-preview/master-page-preview.component'; import { AboutComponent } from './components/about/about.component'; import { StatusViewComponent } from './components/status-view/status-view.component'; import { TermsOfServiceComponent } from './components/terms-of-service/terms-of-service.component'; @@ -22,7 +25,7 @@ import { AssetComponent } from './components/asset/asset.component'; import { AssetsNavComponent } from './components/assets/assets-nav/assets-nav.component'; let routes: Routes = [ - { + { path: 'testnet', children: [ { @@ -88,6 +91,15 @@ let routes: Routes = [ }, ], }, + { + path: 'block-audit', + children: [ + { + path: ':id', + component: BlockAuditComponent, + }, + ], + }, { path: 'docs', loadChildren: () => import('./docs/docs.module').then(m => m.DocsModule) @@ -182,6 +194,15 @@ let routes: Routes = [ }, ], }, + { + path: 'block-audit', + children: [ + { + path: ':id', + component: BlockAuditComponent, + }, + ], + }, { path: 'docs', loadChildren: () => import('./docs/docs.module').then(m => m.DocsModule) @@ -273,6 +294,15 @@ let routes: Routes = [ }, ], }, + { + path: 'block-audit', + children: [ + { + path: ':id', + component: BlockAuditComponent + }, + ], + }, { path: 'docs', loadChildren: () => import('./docs/docs.module').then(m => m.DocsModule) @@ -287,6 +317,16 @@ let routes: Routes = [ }, ], }, + { + path: 'preview', + component: MasterPagePreviewComponent, + children: [ + { + path: 'block/:id', + component: BlockPreviewComponent + }, + ], + }, { path: 'status', component: StatusViewComponent @@ -548,4 +588,3 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') { })], }) export class AppRoutingModule { } - diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index 8f95920f3..97c8f9957 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -6,6 +6,7 @@ import { AppRoutingModule } from './app-routing.module'; import { AppComponent } from './components/app/app.component'; import { ElectrsApiService } from './services/electrs-api.service'; import { StateService } from './services/state.service'; +import { EnterpriseService } from './services/enterprise.service'; import { WebsocketService } from './services/websocket.service'; import { AudioService } from './services/audio.service'; import { SeoService } from './services/seo.service'; @@ -36,6 +37,7 @@ import { CapAddressPipe } from './shared/pipes/cap-address-pipe/cap-address-pipe AudioService, SeoService, StorageService, + EnterpriseService, LanguageService, ShortenStringPipe, FiatShortenerPipe, diff --git a/frontend/src/app/components/block-audit/block-audit.component.html b/frontend/src/app/components/block-audit/block-audit.component.html new file mode 100644 index 000000000..0ee6bef44 --- /dev/null +++ b/frontend/src/app/components/block-audit/block-audit.component.html @@ -0,0 +1,111 @@ +
+ +
+
+

+ + Block +   + {{ blockAudit.height }} +   + Template vs Mined + +

+ +
+ + +
+ + +
+
+ +
+ + + + + + + + + + + + + + + + + + + +
Hash{{ blockAudit.id | shortenString : 13 }} + +
Timestamp + ‎{{ blockAudit.timestamp * 1000 | date:'yyyy-MM-dd HH:mm' }} +
+ ( + ) +
+
Size
Weight
+
+ + +
+ + + + + + + + + + + + + + + + + + + +
Transactions{{ blockAudit.tx_count }}
Match rate{{ blockAudit.matchRate }}%
Missing txs{{ blockAudit.missingTxs.length }}
Added txs{{ blockAudit.addedTxs.length }}
+
+
+
+ + +
+ Missing + Added +
+
+ + +
+
+ +
+ +
+ + +
+ +
+
+
+ + + +
\ No newline at end of file diff --git a/frontend/src/app/components/block-audit/block-audit.component.scss b/frontend/src/app/components/block-audit/block-audit.component.scss new file mode 100644 index 000000000..7ec503891 --- /dev/null +++ b/frontend/src/app/components/block-audit/block-audit.component.scss @@ -0,0 +1,40 @@ +.title-block { + border-top: none; +} + +.table { + tr td { + &:last-child { + text-align: right; + @media (min-width: 768px) { + text-align: left; + } + } + } +} + +.block-tx-title { + display: flex; + justify-content: space-between; + flex-direction: column; + position: relative; + @media (min-width: 550px) { + flex-direction: row; + } + h2 { + line-height: 1; + margin: 0; + position: relative; + padding-bottom: 10px; + @media (min-width: 550px) { + padding-bottom: 0px; + align-self: end; + } + } +} + +.menu-button { + @media (min-width: 768px) { + max-width: 150px; + } +} \ No newline at end of file diff --git a/frontend/src/app/components/block-audit/block-audit.component.ts b/frontend/src/app/components/block-audit/block-audit.component.ts new file mode 100644 index 000000000..044552a3b --- /dev/null +++ b/frontend/src/app/components/block-audit/block-audit.component.ts @@ -0,0 +1,120 @@ +import { Component, OnDestroy, OnInit, ViewChild } from '@angular/core'; +import { ActivatedRoute, ParamMap, Router } from '@angular/router'; +import { Observable } from 'rxjs'; +import { map, share, switchMap, tap } from 'rxjs/operators'; +import { BlockAudit, TransactionStripped } from 'src/app/interfaces/node-api.interface'; +import { ApiService } from 'src/app/services/api.service'; +import { StateService } from 'src/app/services/state.service'; +import { detectWebGL } from 'src/app/shared/graphs.utils'; +import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.pipe'; +import { BlockOverviewGraphComponent } from '../block-overview-graph/block-overview-graph.component'; + +@Component({ + selector: 'app-block-audit', + templateUrl: './block-audit.component.html', + styleUrls: ['./block-audit.component.scss'], + styles: [` + .loadingGraphs { + position: absolute; + top: 50%; + left: calc(50% - 15px); + z-index: 100; + } + `], +}) +export class BlockAuditComponent implements OnInit, OnDestroy { + blockAudit: BlockAudit = undefined; + transactions: string[]; + auditObservable$: Observable; + + paginationMaxSize: number; + page = 1; + itemsPerPage: number; + + mode: 'missing' | 'added' = 'missing'; + isLoading = true; + webGlEnabled = true; + isMobile = window.innerWidth <= 767.98; + + @ViewChild('blockGraphTemplate') blockGraphTemplate: BlockOverviewGraphComponent; + @ViewChild('blockGraphMined') blockGraphMined: BlockOverviewGraphComponent; + + constructor( + private route: ActivatedRoute, + public stateService: StateService, + private router: Router, + private apiService: ApiService + ) { + this.webGlEnabled = detectWebGL(); + } + + ngOnDestroy(): void { + } + + ngOnInit(): void { + this.paginationMaxSize = window.matchMedia('(max-width: 670px)').matches ? 3 : 5; + this.itemsPerPage = this.stateService.env.ITEMS_PER_PAGE; + + this.auditObservable$ = this.route.paramMap.pipe( + switchMap((params: ParamMap) => { + const blockHash: string = params.get('id') || ''; + return this.apiService.getBlockAudit$(blockHash) + .pipe( + map((response) => { + const blockAudit = response.body; + for (let i = 0; i < blockAudit.template.length; ++i) { + if (blockAudit.missingTxs.includes(blockAudit.template[i].txid)) { + blockAudit.template[i].status = 'missing'; + } else if (blockAudit.addedTxs.includes(blockAudit.template[i].txid)) { + blockAudit.template[i].status = 'added'; + } else { + blockAudit.template[i].status = 'found'; + } + } + for (let i = 0; i < blockAudit.transactions.length; ++i) { + if (blockAudit.missingTxs.includes(blockAudit.transactions[i].txid)) { + blockAudit.transactions[i].status = 'missing'; + } else if (blockAudit.addedTxs.includes(blockAudit.transactions[i].txid)) { + blockAudit.transactions[i].status = 'added'; + } else { + blockAudit.transactions[i].status = 'found'; + } + } + return blockAudit; + }), + tap((blockAudit) => { + this.changeMode(this.mode); + if (this.blockGraphTemplate) { + this.blockGraphTemplate.destroy(); + this.blockGraphTemplate.setup(blockAudit.template); + } + if (this.blockGraphMined) { + this.blockGraphMined.destroy(); + this.blockGraphMined.setup(blockAudit.transactions); + } + this.isLoading = false; + }), + ); + }), + share() + ); + } + + onResize(event: any) { + this.isMobile = event.target.innerWidth <= 767.98; + this.paginationMaxSize = event.target.innerWidth < 670 ? 3 : 5; + } + + changeMode(mode: 'missing' | 'added') { + this.router.navigate([], { fragment: mode }); + this.mode = mode; + } + + onTxClick(event: TransactionStripped): void { + const url = new RelativeUrlPipe(this.stateService).transform(`/tx/${event.txid}`); + this.router.navigate([url]); + } + + pageChange(page: number, target: HTMLElement) { + } +} diff --git a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html index 2dbe4d569..e694f5676 100644 --- a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html +++ b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html @@ -2,10 +2,13 @@
- Block Fee Rates - +
+ Block Fee Rates + +
+