diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e0aee68c5..a824900f4 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,6 +1,7 @@ version: 2 updates: - package-ecosystem: npm + versioning-strategy: increase directory: "/backend" schedule: interval: daily @@ -14,6 +15,7 @@ updates: - package-ecosystem: npm directory: "/frontend" + versioning-strategy: increase schedule: interval: daily open-pull-requests-limit: 10 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6947a0f00..6b9b1594b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: - name: Unit Tests if: ${{ matrix.flavor == 'dev'}} - run: npm run test + run: npm run test:ci working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend - name: Build diff --git a/backend/package.json b/backend/package.json index d1cdce286..500cbf93c 100644 --- a/backend/package.json +++ b/backend/package.json @@ -31,6 +31,7 @@ "reindex-updated-pools": "npm run start-production --update-pools", "reindex-all-blocks": "npm run start-production --update-pools --reindex-blocks", "test": "./node_modules/.bin/jest --coverage", + "test:ci": "CI=true ./node_modules/.bin/jest --coverage", "lint": "./node_modules/.bin/eslint . --ext .ts", "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix", "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\"", diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 655898917..2370fe7a1 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -186,7 +186,9 @@ describe('Mempool Backend Config', () => { for (const [key, value] of Object.entries(jsonObj)) { // We have a few cases where we can't follow the pattern if (root === 'MEMPOOL' && key === 'HTTP_PORT') { - console.log('skipping check for MEMPOOL_HTTP_PORT'); + if (process.env.CI) { + console.log('skipping check for MEMPOOL_HTTP_PORT'); + } continue; } switch (typeof value) { @@ -208,13 +210,17 @@ describe('Mempool Backend Config', () => { //The string used as the default value, to be checked as a regex, i.e, __MEMPOOL_ENABLED__=${MEMPOOL_ENABLED:=(.*)} const defaultEntry = replaceStr + '=' + '\\${' + envVarStr + ':=(.*)' + '}'; - console.log(`looking for ${defaultEntry} in the start.sh script`); + if (process.env.CI) { + console.log(`looking for ${defaultEntry} in the start.sh script`); + } const re = new RegExp(defaultEntry); expect(startSh).toMatch(re); //The string that actually replaces the values in the config file const sedStr = 'sed -i "s!' + replaceStr + '!${' + replaceStr + '}!g" mempool-config.json'; - console.log(`looking for ${sedStr} in the start.sh script`); + if (process.env.CI) { + console.log(`looking for ${sedStr} in the start.sh script`); + } expect(startSh).toContain(sedStr); break; } diff --git a/backend/src/api/mining/mining-routes.ts b/backend/src/api/mining/mining-routes.ts index 1c9a0de30..bb78de44a 100644 --- a/backend/src/api/mining/mining-routes.ts +++ b/backend/src/api/mining/mining-routes.ts @@ -12,6 +12,7 @@ import PricesRepository from '../../repositories/PricesRepository'; class MiningRoutes { public initRoutes(app: Application) { app + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools', this.$listPools) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/:interval', this.$getPools) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/hashrate', this.$getPoolHistoricalHashrate) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks', this.$getPoolBlocks) @@ -41,6 +42,10 @@ class MiningRoutes { res.header('Pragma', 'public'); res.header('Cache-control', 'public'); res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); + if (['testnet', 'signet', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) { + res.status(400).send('Prices are not available on testnets.'); + return; + } if (req.query.timestamp) { res.status(200).send(await PricesRepository.$getNearestHistoricalPrice( parseInt(req.query.timestamp ?? 0, 10) @@ -88,6 +93,29 @@ class MiningRoutes { } } + private async $listPools(req: Request, res: Response): Promise { + try { + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + + const pools = await mining.$listPools(); + if (!pools) { + res.status(500).end(); + return; + } + + res.header('X-total-count', pools.length.toString()); + if (pools.length === 0) { + res.status(204).send(); + } else { + res.json(pools); + } + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + private async $getPools(req: Request, res: Response) { try { const stats = await mining.$getPoolsStats(req.params.interval); diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index 6e87d70b8..d9d5995da 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -26,7 +26,7 @@ class Mining { /** * Get historical blocks health */ - public async $getBlocksHealthHistory(interval: string | null = null): Promise { + public async $getBlocksHealthHistory(interval: string | null = null): Promise { return await BlocksAuditsRepository.$getBlocksHealthHistory( this.getTimeRange(interval), Common.getSqlInterval(interval) @@ -56,7 +56,7 @@ class Mining { /** * Get historical block fee rates percentiles */ - public async $getHistoricalBlockFeeRates(interval: string | null = null): Promise { + public async $getHistoricalBlockFeeRates(interval: string | null = null): Promise { return await BlocksRepository.$getHistoricalBlockFeeRates( this.getTimeRange(interval), Common.getSqlInterval(interval) @@ -66,7 +66,7 @@ class Mining { /** * Get historical block sizes */ - public async $getHistoricalBlockSizes(interval: string | null = null): Promise { + public async $getHistoricalBlockSizes(interval: string | null = null): Promise { return await BlocksRepository.$getHistoricalBlockSizes( this.getTimeRange(interval), Common.getSqlInterval(interval) @@ -76,7 +76,7 @@ class Mining { /** * Get historical block weights */ - public async $getHistoricalBlockWeights(interval: string | null = null): Promise { + public async $getHistoricalBlockWeights(interval: string | null = null): Promise { return await BlocksRepository.$getHistoricalBlockWeights( this.getTimeRange(interval), Common.getSqlInterval(interval) @@ -595,6 +595,20 @@ class Mining { } } + /** + * List existing mining pools + */ + public async $listPools(): Promise<{name: string, slug: string, unique_id: number}[] | null> { + const [rows] = await database.query(` + SELECT + name, + slug, + unique_id + FROM pools` + ); + return rows as {name: string, slug: string, unique_id: number}[]; + } + private getDateMidnight(date: Date): Date { date.setUTCHours(0); date.setUTCMinutes(0); diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 8ade49288..9cb24df10 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -198,18 +198,14 @@ class WebsocketHandler { matchedAddress = matchedAddress.toLowerCase(); } if (/^04[a-fA-F0-9]{128}$/.test(parsedMessage['track-address'])) { - client['track-address'] = null; - client['track-scriptpubkey'] = '41' + matchedAddress + 'ac'; - } else if (/^|(02|03)[a-fA-F0-9]{64}$/.test(parsedMessage['track-address'])) { - client['track-address'] = null; - client['track-scriptpubkey'] = '21' + matchedAddress + 'ac'; + client['track-address'] = '41' + matchedAddress + 'ac'; + } else if (/^(02|03)[a-fA-F0-9]{64}$/.test(parsedMessage['track-address'])) { + client['track-address'] = '21' + matchedAddress + 'ac'; } else { client['track-address'] = matchedAddress; - client['track-scriptpubkey'] = null; } } else { client['track-address'] = null; - client['track-scriptpubkey'] = null; } } @@ -488,6 +484,9 @@ class WebsocketHandler { } } + // pre-compute address transactions + const addressCache = this.makeAddressCache(newTransactions); + this.wss.clients.forEach(async (client) => { if (client.readyState !== WebSocket.OPEN) { return; @@ -527,78 +526,13 @@ class WebsocketHandler { } if (client['track-address']) { - const foundTransactions: TransactionExtended[] = []; + const foundTransactions = Array.from(addressCache[client['track-address']]?.values() || []); + // txs may be missing prevouts in non-esplora backends + // so fetch the full transactions now + const fullTransactions = (config.MEMPOOL.BACKEND !== 'esplora') ? await this.getFullTransactions(foundTransactions) : foundTransactions; - for (const tx of newTransactions) { - const someVin = tx.vin.some((vin) => !!vin.prevout && vin.prevout.scriptpubkey_address === client['track-address']); - if (someVin) { - if (config.MEMPOOL.BACKEND !== 'esplora') { - try { - const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true); - foundTransactions.push(fullTx); - } catch (e) { - logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e)); - } - } else { - foundTransactions.push(tx); - } - return; - } - const someVout = tx.vout.some((vout) => vout.scriptpubkey_address === client['track-address']); - if (someVout) { - if (config.MEMPOOL.BACKEND !== 'esplora') { - try { - const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true); - foundTransactions.push(fullTx); - } catch (e) { - logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e)); - } - } else { - foundTransactions.push(tx); - } - } - } - - if (foundTransactions.length) { - response['address-transactions'] = JSON.stringify(foundTransactions); - } - } - - if (client['track-scriptpubkey']) { - const foundTransactions: TransactionExtended[] = []; - - for (const tx of newTransactions) { - const someVin = tx.vin.some((vin) => !!vin.prevout && vin.prevout.scriptpubkey_type === 'p2pk' && vin.prevout.scriptpubkey === client['track-scriptpubkey']); - if (someVin) { - if (config.MEMPOOL.BACKEND !== 'esplora') { - try { - const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true); - foundTransactions.push(fullTx); - } catch (e) { - logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e)); - } - } else { - foundTransactions.push(tx); - } - return; - } - const someVout = tx.vout.some((vout) => vout.scriptpubkey_type === 'p2pk' && vout.scriptpubkey === client['track-scriptpubkey']); - if (someVout) { - if (config.MEMPOOL.BACKEND !== 'esplora') { - try { - const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true); - foundTransactions.push(fullTx); - } catch (e) { - logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e)); - } - } else { - foundTransactions.push(tx); - } - } - } - - if (foundTransactions.length) { - response['address-transactions'] = JSON.stringify(foundTransactions); + if (fullTransactions.length) { + response['address-transactions'] = JSON.stringify(fullTransactions); } } @@ -606,7 +540,6 @@ class WebsocketHandler { const foundTransactions: TransactionExtended[] = []; newTransactions.forEach((tx) => { - if (client['track-asset'] === Common.nativeAssetId) { if (tx.vin.some((vin) => !!vin.is_pegin)) { foundTransactions.push(tx); @@ -805,6 +738,9 @@ class WebsocketHandler { const fees = feeApi.getRecommendedFee(); const mempoolInfo = memPool.getMempoolInfo(); + // pre-compute address transactions + const addressCache = this.makeAddressCache(transactions); + // update init data this.updateSocketDataFields({ 'mempoolInfo': mempoolInfo, @@ -867,44 +803,7 @@ class WebsocketHandler { } if (client['track-address']) { - const foundTransactions: TransactionExtended[] = []; - - transactions.forEach((tx) => { - if (tx.vin && tx.vin.some((vin) => !!vin.prevout && vin.prevout.scriptpubkey_address === client['track-address'])) { - foundTransactions.push(tx); - return; - } - if (tx.vout && tx.vout.some((vout) => vout.scriptpubkey_address === client['track-address'])) { - foundTransactions.push(tx); - } - }); - - if (foundTransactions.length) { - foundTransactions.forEach((tx) => { - tx.status = { - confirmed: true, - block_height: block.height, - block_hash: block.id, - block_time: block.timestamp, - }; - }); - - response['block-transactions'] = JSON.stringify(foundTransactions); - } - } - - if (client['track-scriptpubkey']) { - const foundTransactions: TransactionExtended[] = []; - - transactions.forEach((tx) => { - if (tx.vin && tx.vin.some((vin) => !!vin.prevout && vin.prevout.scriptpubkey_type === 'p2pk' && vin.prevout.scriptpubkey === client['track-scriptpubkey'])) { - foundTransactions.push(tx); - return; - } - if (tx.vout && tx.vout.some((vout) => vout.scriptpubkey_type === 'p2pk' && vout.scriptpubkey === client['track-scriptpubkey'])) { - foundTransactions.push(tx); - } - }); + const foundTransactions: TransactionExtended[] = Array.from(addressCache[client['track-address']]?.values() || []); if (foundTransactions.length) { foundTransactions.forEach((tx) => { @@ -982,6 +881,52 @@ class WebsocketHandler { + '}'; } + private makeAddressCache(transactions: MempoolTransactionExtended[]): { [address: string]: Set } { + const addressCache: { [address: string]: Set } = {}; + for (const tx of transactions) { + for (const vin of tx.vin) { + if (vin?.prevout?.scriptpubkey_address) { + if (!addressCache[vin.prevout.scriptpubkey_address]) { + addressCache[vin.prevout.scriptpubkey_address] = new Set(); + } + addressCache[vin.prevout.scriptpubkey_address].add(tx); + } + if (vin?.prevout?.scriptpubkey) { + if (!addressCache[vin.prevout.scriptpubkey]) { + addressCache[vin.prevout.scriptpubkey] = new Set(); + } + addressCache[vin.prevout.scriptpubkey].add(tx); + } + } + for (const vout of tx.vout) { + if (vout?.scriptpubkey_address) { + if (!addressCache[vout?.scriptpubkey_address]) { + addressCache[vout?.scriptpubkey_address] = new Set(); + } + addressCache[vout?.scriptpubkey_address].add(tx); + } + if (vout?.scriptpubkey) { + if (!addressCache[vout.scriptpubkey]) { + addressCache[vout.scriptpubkey] = new Set(); + } + addressCache[vout.scriptpubkey].add(tx); + } + } + } + return addressCache; + } + + private async getFullTransactions(transactions: MempoolTransactionExtended[]): Promise { + for (let i = 0; i < transactions.length; i++) { + try { + transactions[i] = await transactionUtils.$getMempoolTransactionExtended(transactions[i].txid, true); + } catch (e) { + logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e)); + } + } + return transactions; + } + private printLogs(): void { if (this.wss) { const count = this.wss?.clients?.size || 0; diff --git a/frontend/src/app/components/fiat-selector/fiat-selector.component.html b/frontend/src/app/components/fiat-selector/fiat-selector.component.html index dd32b1815..eec6f4b0a 100644 --- a/frontend/src/app/components/fiat-selector/fiat-selector.component.html +++ b/frontend/src/app/components/fiat-selector/fiat-selector.component.html @@ -1,5 +1,5 @@
-
diff --git a/frontend/src/app/components/language-selector/language-selector.component.html b/frontend/src/app/components/language-selector/language-selector.component.html index 22839441c..41e0efb0e 100644 --- a/frontend/src/app/components/language-selector/language-selector.component.html +++ b/frontend/src/app/components/language-selector/language-selector.component.html @@ -1,5 +1,5 @@
-
diff --git a/frontend/src/app/components/rate-unit-selector/rate-unit-selector.component.html b/frontend/src/app/components/rate-unit-selector/rate-unit-selector.component.html index 016d1b555..a2be9df87 100644 --- a/frontend/src/app/components/rate-unit-selector/rate-unit-selector.component.html +++ b/frontend/src/app/components/rate-unit-selector/rate-unit-selector.component.html @@ -1,5 +1,5 @@
-
diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 1ed9d2f5c..798df72c1 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@angular/core'; import { HttpClient, HttpParams, HttpResponse } from '@angular/common/http'; import { CpfpInfo, OptimizedMempoolStats, AddressInformation, LiquidPegs, ITranslators, PoolStat, BlockExtended, TransactionStripped, RewardStats, AuditScore, BlockSizesAndWeights, RbfTree, BlockAudit } from '../interfaces/node-api.interface'; -import { Observable } from 'rxjs'; +import { Observable, of } from 'rxjs'; import { StateService } from './state.service'; import { WebsocketResponse } from '../interfaces/websocket.interface'; import { Outspend, Transaction } from '../interfaces/electrs.interface'; @@ -312,6 +312,19 @@ export class ApiService { } getHistoricalPrice$(timestamp: number | undefined): Observable { + if (this.stateService.isAnyTestnet()) { + return of({ + prices: [], + exchangeRates: { + USDEUR: 0, + USDGBP: 0, + USDCAD: 0, + USDCHF: 0, + USDAUD: 0, + USDJPY: 0, + } + }); + } return this.httpClient.get( this.apiBaseUrl + this.apiBasePath + '/api/v1/historical-price' + (timestamp ? `?timestamp=${timestamp}` : '') diff --git a/frontend/src/app/services/state.service.ts b/frontend/src/app/services/state.service.ts index 675cf88d1..91e4d7475 100644 --- a/frontend/src/app/services/state.service.ts +++ b/frontend/src/app/services/state.service.ts @@ -339,6 +339,10 @@ export class StateService { return this.network === 'liquid' || this.network === 'liquidtestnet'; } + isAnyTestnet(): boolean { + return ['testnet', 'signet', 'liquidtestnet'].includes(this.network); + } + resetChainTip() { this.latestBlockHeight = -1; this.chainTip$.next(-1); diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index d4f303221..0b9c20387 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -1,20 +1,21 @@