From e88e56421bc62945e8f1618086fa8663b70d630e Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 20 May 2023 18:00:22 -0400 Subject: [PATCH 01/38] Enable fullrbf --- production/bitcoin.conf | 1 + production/bitcoin.minfee.conf | 1 + 2 files changed, 2 insertions(+) diff --git a/production/bitcoin.conf b/production/bitcoin.conf index 501f49f50..d9f3fc252 100644 --- a/production/bitcoin.conf +++ b/production/bitcoin.conf @@ -8,6 +8,7 @@ par=16 dbcache=8192 maxmempool=4096 mempoolexpiry=999999 +mempoolfullrbf=1 maxconnections=42 onion=127.0.0.1:9050 rpcallowip=127.0.0.1 diff --git a/production/bitcoin.minfee.conf b/production/bitcoin.minfee.conf index d8ff97258..0bd7f2ed1 100644 --- a/production/bitcoin.minfee.conf +++ b/production/bitcoin.minfee.conf @@ -4,6 +4,7 @@ txindex=0 listen=1 daemon=1 prune=1337 +mempoolfullrbf=1 rpcallowip=127.0.0.1 rpcuser=__BITCOIN_RPC_USER__ rpcpassword=__BITCOIN_RPC_PASS__ From 5ef592f53eac5266a5b7631f749182824d7478b7 Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 13 Jul 2023 16:57:36 +0900 Subject: [PATCH 02/38] Load more mempool transactions --- .../app/components/address/address.component.ts | 6 +++++- frontend/src/app/services/electrs-api.service.ts | 14 +++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/frontend/src/app/components/address/address.component.ts b/frontend/src/app/components/address/address.component.ts index 2ae9a962b..57439f983 100644 --- a/frontend/src/app/components/address/address.component.ts +++ b/frontend/src/app/components/address/address.component.ts @@ -207,7 +207,7 @@ export class AddressComponent implements OnInit, OnDestroy { } this.isLoadingTransactions = true; this.retryLoadMore = false; - this.electrsApiService.getAddressTransactionsFromHash$(this.address.address, this.lastTransactionTxId) + this.electrsApiService.getAddressTransactions$(this.address.address, this.lastTransactionTxId) .subscribe((transactions: Transaction[]) => { this.lastTransactionTxId = transactions[transactions.length - 1].txid; this.loadedConfirmedTxCount += transactions.length; @@ -217,6 +217,10 @@ export class AddressComponent implements OnInit, OnDestroy { (error) => { this.isLoadingTransactions = false; this.retryLoadMore = true; + // In the unlikely event of the txid wasn't found in the mempool anymore and we must reload the page. + if (error.status === 422) { + window.location.reload(); + } }); } diff --git a/frontend/src/app/services/electrs-api.service.ts b/frontend/src/app/services/electrs-api.service.ts index 5756f7fb2..c87018741 100644 --- a/frontend/src/app/services/electrs-api.service.ts +++ b/frontend/src/app/services/electrs-api.service.ts @@ -1,5 +1,5 @@ import { Injectable } from '@angular/core'; -import { HttpClient } from '@angular/common/http'; +import { HttpClient, HttpParams } from '@angular/common/http'; import { Observable } from 'rxjs'; import { Transaction, Address, Outspend, Recent, Asset } from '../interfaces/electrs.interface'; import { StateService } from './state.service'; @@ -65,12 +65,12 @@ export class ElectrsApiService { return this.httpClient.get
(this.apiBaseUrl + this.apiBasePath + '/api/address/' + address); } - getAddressTransactions$(address: string): Observable { - return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/address/' + address + '/txs'); - } - - getAddressTransactionsFromHash$(address: string, txid: string): Observable { - return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/address/' + address + '/txs/chain/' + txid); + getAddressTransactions$(address: string, txid?: string): Observable { + let params = new HttpParams(); + if (txid) { + params = params.append('after_txid', txid); + } + return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/address/' + address + '/txs', { params }); } getAsset$(assetId: string): Observable { From 15a8c8d42062bd81da20e007a98f3934db9155b2 Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 13 Jul 2023 17:59:02 +0900 Subject: [PATCH 03/38] Support for romanz/electrs --- backend/src/api/bitcoin/bitcoin.routes.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index 17ebc9275..babc0aa53 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -121,7 +121,6 @@ class BitcoinRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', this.getBlockHeight) .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', this.getAddress) .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', this.getAddressTransactions) - .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', this.getAddressTransactions) .get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', this.getAddressPrefix) ; } @@ -546,27 +545,28 @@ class BitcoinRoutes { } } - private async getAddressTransactions(req: Request, res: Response) { + private async getAddressTransactions(req: Request, res: Response): Promise { if (config.MEMPOOL.BACKEND === 'none') { res.status(405).send('Address lookups cannot be used with bitcoind as backend.'); return; } try { - const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, req.params.txId); + let lastTxId: string = ''; + if (req.query.after_txid && typeof req.query.after_txid === 'string') { + lastTxId = req.query.after_txid; + } + const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, lastTxId); res.json(transactions); } catch (e) { if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) { - return res.status(413).send(e instanceof Error ? e.message : e); + res.status(413).send(e instanceof Error ? e.message : e); + return; } res.status(500).send(e instanceof Error ? e.message : e); } } - private async getAdressTxChain(req: Request, res: Response) { - res.status(501).send('Not implemented'); - } - private async getAddressPrefix(req: Request, res: Response) { try { const blockHash = await bitcoinApi.$getAddressPrefix(req.params.prefix); From 69e6b164b98d4f87ecf10bb7f367027be35a7961 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 20 Jun 2023 16:21:54 -0400 Subject: [PATCH 04/38] Add audit data replication service --- backend/mempool-config.sample.json | 11 ++ .../__fixtures__/mempool-config.template.json | 6 + backend/src/__tests__/config.test.ts | 7 + backend/src/config.ts | 14 ++ backend/src/indexer.ts | 2 + backend/src/mempool.interfaces.ts | 9 ++ backend/src/replication/AuditReplication.ts | 123 ++++++++++++++++++ backend/src/replication/replicator.ts | 70 ++++++++++ production/mempool-config.mainnet.json | 25 ++++ 9 files changed, 267 insertions(+) create mode 100644 backend/src/replication/AuditReplication.ts create mode 100644 backend/src/replication/replicator.ts diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index c0a2d9d62..e3df7d2fe 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -125,5 +125,16 @@ "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1", "BISQ_URL": "https://bisq.markets/api", "BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api" + }, + "REPLICATION": { + "ENABLED": false, + "AUDIT": false, + "AUDIT_START_HEIGHT": 774000, + "SERVERS": [ + "list", + "of", + "trusted", + "servers" + ] } } diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 776f01de1..4213f0ffb 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -121,5 +121,11 @@ }, "CLIGHTNING": { "SOCKET": "__CLIGHTNING_SOCKET__" + }, + "REPLICATION": { + "ENABLED": false, + "AUDIT": false, + "AUDIT_START_HEIGHT": 774000, + "SERVERS": [] } } diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index fdd8a02de..dc1beaa46 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -120,6 +120,13 @@ describe('Mempool Backend Config', () => { GEOLITE2_ASN: '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb', GEOIP2_ISP: '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb' }); + + expect(config.REPLICATION).toStrictEqual({ + ENABLED: false, + AUDIT: false, + AUDIT_START_HEIGHT: 774000, + SERVERS: [] + }); }); }); diff --git a/backend/src/config.ts b/backend/src/config.ts index 40b407a57..09d279537 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -132,6 +132,12 @@ interface IConfig { GEOLITE2_ASN: string; GEOIP2_ISP: string; }, + REPLICATION: { + ENABLED: boolean; + AUDIT: boolean; + AUDIT_START_HEIGHT: number; + SERVERS: string[]; + } } const defaults: IConfig = { @@ -264,6 +270,12 @@ const defaults: IConfig = { 'GEOLITE2_ASN': '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb', 'GEOIP2_ISP': '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb' }, + 'REPLICATION': { + 'ENABLED': false, + 'AUDIT': false, + 'AUDIT_START_HEIGHT': 774000, + 'SERVERS': [], + } }; class Config implements IConfig { @@ -283,6 +295,7 @@ class Config implements IConfig { PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER']; EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER']; MAXMIND: IConfig['MAXMIND']; + REPLICATION: IConfig['REPLICATION']; constructor() { const configs = this.merge(configFromFile, defaults); @@ -302,6 +315,7 @@ class Config implements IConfig { this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER; this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER; this.MAXMIND = configs.MAXMIND; + this.REPLICATION = configs.REPLICATION; } merge = (...objects: object[]): IConfig => { diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 88f44d587..d89a2647f 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -7,6 +7,7 @@ import bitcoinClient from './api/bitcoin/bitcoin-client'; import priceUpdater from './tasks/price-updater'; import PricesRepository from './repositories/PricesRepository'; import config from './config'; +import auditReplicator from './replication/AuditReplication'; export interface CoreIndex { name: string; @@ -136,6 +137,7 @@ class Indexer { await blocks.$generateBlocksSummariesDatabase(); await blocks.$generateCPFPDatabase(); await blocks.$generateAuditStats(); + await auditReplicator.$sync(); } catch (e) { this.indexerRunning = false; logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e)); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index a051eea4f..1971234f8 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -236,6 +236,15 @@ export interface BlockSummary { transactions: TransactionStripped[]; } +export interface AuditSummary extends BlockAudit { + timestamp?: number, + size?: number, + weight?: number, + tx_count?: number, + transactions: TransactionStripped[]; + template?: TransactionStripped[]; +} + export interface BlockPrice { height: number; priceId: number; diff --git a/backend/src/replication/AuditReplication.ts b/backend/src/replication/AuditReplication.ts new file mode 100644 index 000000000..b950acb6c --- /dev/null +++ b/backend/src/replication/AuditReplication.ts @@ -0,0 +1,123 @@ +import DB from '../database'; +import logger from '../logger'; +import { AuditSummary } from '../mempool.interfaces'; +import blocksAuditsRepository from '../repositories/BlocksAuditsRepository'; +import blocksSummariesRepository from '../repositories/BlocksSummariesRepository'; +import { $sync } from './replicator'; +import config from '../config'; +import { Common } from '../api/common'; + +const BATCH_SIZE = 16; + +/** + * Syncs missing block template and audit data from trusted servers + */ +class AuditReplication { + inProgress: boolean = false; + skip: Set = new Set(); + + public async $sync(): Promise { + if (!config.REPLICATION.ENABLED || !config.REPLICATION.AUDIT) { + // replication not enabled + return; + } + if (this.inProgress) { + logger.info(`AuditReplication sync already in progress`, 'Replication'); + return; + } + this.inProgress = true; + + const missingAudits = await this.$getMissingAuditBlocks(); + + logger.debug(`Fetching missing audit data for ${missingAudits.length} blocks from trusted servers`, 'Replication'); + + let totalSynced = 0; + let totalMissed = 0; + let loggerTimer = Date.now(); + // process missing audits in batches of + for (let i = 0; i < missingAudits.length; i += BATCH_SIZE) { + const results = await Promise.all(missingAudits.slice(i * BATCH_SIZE, (i + 1) * BATCH_SIZE).map(hash => this.$syncAudit(hash))); + const synced = results.reduce((total, status) => status ? total + 1 : total, 0); + totalSynced += synced; + totalMissed += (BATCH_SIZE - synced); + if (Date.now() - loggerTimer > 10000) { + loggerTimer = Date.now(); + logger.info(`Found ${totalSynced} / ${totalSynced + totalMissed} of ${missingAudits.length} missing audits`, 'Replication'); + } + await Common.sleep$(1000); + } + + logger.debug(`Fetched ${totalSynced} audits, ${totalMissed} still missing`, 'Replication'); + + this.inProgress = false; + } + + private async $syncAudit(hash: string): Promise { + if (this.skip.has(hash)) { + // we already know none of our trusted servers have this audit + return false; + } + + let success = false; + // start with a random server so load is uniformly spread + const syncResult = await $sync(`/api/v1/block/${hash}/audit-summary`); + if (syncResult) { + if (syncResult.data?.template?.length) { + await this.$saveAuditData(hash, syncResult.data); + success = true; + } + if (!syncResult.data && !syncResult.exists) { + this.skip.add(hash); + } + } + + return success; + } + + private async $getMissingAuditBlocks(): Promise { + try { + const startHeight = config.REPLICATION.AUDIT_START_HEIGHT || 0; + const [rows]: any[] = await DB.query(` + SELECT auditable.hash, auditable.height + FROM ( + SELECT hash, height + FROM blocks + WHERE height >= ? + ) AS auditable + LEFT JOIN blocks_audits ON auditable.hash = blocks_audits.hash + WHERE blocks_audits.hash IS NULL + ORDER BY auditable.height DESC + `, [startHeight]); + return rows.map(row => row.hash); + } catch (e: any) { + logger.err(`Cannot fetch missing audit blocks from db. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + + private async $saveAuditData(blockHash: string, auditSummary: AuditSummary): Promise { + // save audit & template to DB + await blocksSummariesRepository.$saveTemplate({ + height: auditSummary.height, + template: { + id: blockHash, + transactions: auditSummary.template || [] + } + }); + await blocksAuditsRepository.$saveAudit({ + hash: blockHash, + height: auditSummary.height, + time: auditSummary.timestamp || auditSummary.time, + missingTxs: auditSummary.missingTxs || [], + addedTxs: auditSummary.addedTxs || [], + freshTxs: auditSummary.freshTxs || [], + sigopTxs: auditSummary.sigopTxs || [], + matchRate: auditSummary.matchRate, + expectedFees: auditSummary.expectedFees, + expectedWeight: auditSummary.expectedWeight, + }); + } +} + +export default new AuditReplication(); + diff --git a/backend/src/replication/replicator.ts b/backend/src/replication/replicator.ts new file mode 100644 index 000000000..60dfa8a2d --- /dev/null +++ b/backend/src/replication/replicator.ts @@ -0,0 +1,70 @@ +import config from '../config'; +import backendInfo from '../api/backend-info'; +import axios, { AxiosResponse } from 'axios'; +import { SocksProxyAgent } from 'socks-proxy-agent'; +import * as https from 'https'; + +export async function $sync(path): Promise<{ data?: any, exists: boolean }> { + // start with a random server so load is uniformly spread + let allMissing = true; + const offset = Math.floor(Math.random() * config.REPLICATION.SERVERS.length); + for (let i = 0; i < config.REPLICATION.SERVERS.length; i++) { + const server = config.REPLICATION.SERVERS[(i + offset) % config.REPLICATION.SERVERS.length]; + // don't query ourself + if (server === backendInfo.getBackendInfo().hostname) { + continue; + } + + try { + const result = await query(`https://${server}${path}`); + if (result) { + return { data: result, exists: true }; + } + } catch (e: any) { + if (e?.response?.status === 404) { + // this server is also missing this data + } else { + // something else went wrong + allMissing = false; + } + } + } + + return { exists: !allMissing }; +} + +export async function query(path): Promise { + type axiosOptions = { + headers: { + 'User-Agent': string + }; + timeout: number; + httpsAgent?: https.Agent; + }; + const axiosOptions: axiosOptions = { + headers: { + 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` + }, + timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000 + }; + + if (config.SOCKS5PROXY.ENABLED) { + const socksOptions = { + agentOptions: { + keepAlive: true, + }, + hostname: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT, + username: config.SOCKS5PROXY.USERNAME || 'circuit0', + password: config.SOCKS5PROXY.PASSWORD, + }; + + axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); + } + + const data: AxiosResponse = await axios.get(path, axiosOptions); + if (data.statusText === 'error' || !data.data) { + throw new Error(`${data.status}`); + } + return data.data; +} \ No newline at end of file diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index a76053913..5e25bcb76 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -48,5 +48,30 @@ "STATISTICS": { "ENABLED": true, "TX_PER_SECOND_SAMPLE_PERIOD": 150 + }, + "REPLICATION": { + "ENABLED": true, + "AUDIT": true, + "AUDIT_START_HEIGHT": 774000, + "SERVERS": [ + "node201.fmt.mempool.space", + "node202.fmt.mempool.space", + "node203.fmt.mempool.space", + "node204.fmt.mempool.space", + "node205.fmt.mempool.space", + "node206.fmt.mempool.space", + "node201.fra.mempool.space", + "node202.fra.mempool.space", + "node203.fra.mempool.space", + "node204.fra.mempool.space", + "node205.fra.mempool.space", + "node206.fra.mempool.space", + "node201.tk7.mempool.space", + "node202.tk7.mempool.space", + "node203.tk7.mempool.space", + "node204.tk7.mempool.space", + "node205.tk7.mempool.space", + "node206.tk7.mempool.space" + ] } } From 736b997104c592673fd3e558f94d3eac1be2071d Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 20 Jun 2023 17:03:24 -0400 Subject: [PATCH 05/38] Add missing audit data to cached blocks --- backend/src/replication/AuditReplication.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/src/replication/AuditReplication.ts b/backend/src/replication/AuditReplication.ts index b950acb6c..c762df201 100644 --- a/backend/src/replication/AuditReplication.ts +++ b/backend/src/replication/AuditReplication.ts @@ -6,6 +6,7 @@ import blocksSummariesRepository from '../repositories/BlocksSummariesRepository import { $sync } from './replicator'; import config from '../config'; import { Common } from '../api/common'; +import blocks from '../api/blocks'; const BATCH_SIZE = 16; @@ -116,6 +117,13 @@ class AuditReplication { expectedFees: auditSummary.expectedFees, expectedWeight: auditSummary.expectedWeight, }); + // add missing data to cached blocks + const cachedBlock = blocks.getBlocks().find(block => block.id === blockHash); + if (cachedBlock) { + cachedBlock.extras.matchRate = auditSummary.matchRate; + cachedBlock.extras.expectedFees = auditSummary.expectedFees || null; + cachedBlock.extras.expectedWeight = auditSummary.expectedWeight || null; + } } } From 7f6d17fc0ece0bfc57945fafca338a96ce0e64c1 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 20 Jun 2023 17:07:14 -0400 Subject: [PATCH 06/38] Fix audit sync progress logging --- backend/src/replication/AuditReplication.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/replication/AuditReplication.ts b/backend/src/replication/AuditReplication.ts index c762df201..89c514347 100644 --- a/backend/src/replication/AuditReplication.ts +++ b/backend/src/replication/AuditReplication.ts @@ -37,10 +37,11 @@ class AuditReplication { let loggerTimer = Date.now(); // process missing audits in batches of for (let i = 0; i < missingAudits.length; i += BATCH_SIZE) { - const results = await Promise.all(missingAudits.slice(i * BATCH_SIZE, (i + 1) * BATCH_SIZE).map(hash => this.$syncAudit(hash))); + const slice = missingAudits.slice(i * BATCH_SIZE, (i + 1) * BATCH_SIZE); + const results = await Promise.all(slice.map(hash => this.$syncAudit(hash))); const synced = results.reduce((total, status) => status ? total + 1 : total, 0); totalSynced += synced; - totalMissed += (BATCH_SIZE - synced); + totalMissed += (slice.length - synced); if (Date.now() - loggerTimer > 10000) { loggerTimer = Date.now(); logger.info(`Found ${totalSynced} / ${totalSynced + totalMissed} of ${missingAudits.length} missing audits`, 'Replication'); From bccc6b3680e4bd6aeeb25bc229983de1f21b8c91 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 21 Jun 2023 09:19:59 -0400 Subject: [PATCH 07/38] Add missing replication docker config --- docker/backend/mempool-config.json | 6 ++++++ docker/backend/start.sh | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index d070d8010..2ff76d5dd 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -127,5 +127,11 @@ "GEOLITE2_CITY": "__MAXMIND_GEOLITE2_CITY__", "GEOLITE2_ASN": "__MAXMIND_GEOLITE2_ASN__", "GEOIP2_ISP": "__MAXMIND_GEOIP2_ISP__" + }, + "REPLICATION": { + "ENABLED": __REPLICATION_ENABLED__, + "AUDIT": __REPLICATION_AUDIT__, + "AUDIT_START_HEIGHT": __REPLICATION_AUDIT_START_HEIGHT__, + "SERVERS": __REPLICATION_SERVERS__ } } diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 7241444fb..c34d804b4 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -130,6 +130,12 @@ __MAXMIND_GEOLITE2_CITY__=${MAXMIND_GEOLITE2_CITY:="/backend/GeoIP/GeoLite2-City __MAXMIND_GEOLITE2_ASN__=${MAXMIND_GEOLITE2_ASN:="/backend/GeoIP/GeoLite2-ASN.mmdb"} __MAXMIND_GEOIP2_ISP__=${MAXMIND_GEOIP2_ISP:=""} +# REPLICATION +__REPLICATION_ENABLED__=${REPLICATION_ENABLED:=true} +__REPLICATION_AUDIT__=${REPLICATION_AUDIT:=true} +__REPLICATION_AUDIT_START_HEIGHT__=${REPLICATION_AUDIT_START_HEIGHT:=774000} +__REPLICATION_SERVERS__=${REPLICATION_SERVERS:=[]} + mkdir -p "${__MEMPOOL_CACHE_DIR__}" @@ -250,5 +256,10 @@ sed -i "s!__MAXMIND_GEOLITE2_CITY__!${__MAXMIND_GEOLITE2_CITY__}!g" mempool-conf sed -i "s!__MAXMIND_GEOLITE2_ASN__!${__MAXMIND_GEOLITE2_ASN__}!g" mempool-config.json sed -i "s!__MAXMIND_GEOIP2_ISP__!${__MAXMIND_GEOIP2_ISP__}!g" mempool-config.json +# REPLICATION +sed -i "s!__REPLICATION_ENABLED__!${__REPLICATION_ENABLED__}!g" mempool-config.json +sed -i "s!__REPLICATION_AUDIT__!${__REPLICATION_AUDIT__}!g" mempool-config.json +sed -i "s!__REPLICATION_AUDIT_START_HEIGHT__!${__REPLICATION_AUDIT_START_HEIGHT__}!g" mempool-config.json +sed -i "s!__REPLICATION_SERVERS__!${__REPLICATION_SERVERS__}!g" mempool-config.json node /backend/package/index.js From e59a9d38ff238850fc9e727102a56ff0f5c0e05e Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 14 Jul 2023 16:47:58 +0900 Subject: [PATCH 08/38] fix audit replication merge conflicts --- backend/src/replication/AuditReplication.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/replication/AuditReplication.ts b/backend/src/replication/AuditReplication.ts index 89c514347..2043532db 100644 --- a/backend/src/replication/AuditReplication.ts +++ b/backend/src/replication/AuditReplication.ts @@ -37,7 +37,7 @@ class AuditReplication { let loggerTimer = Date.now(); // process missing audits in batches of for (let i = 0; i < missingAudits.length; i += BATCH_SIZE) { - const slice = missingAudits.slice(i * BATCH_SIZE, (i + 1) * BATCH_SIZE); + const slice = missingAudits.slice(i, i + BATCH_SIZE); const results = await Promise.all(slice.map(hash => this.$syncAudit(hash))); const synced = results.reduce((total, status) => status ? total + 1 : total, 0); totalSynced += synced; @@ -114,6 +114,7 @@ class AuditReplication { addedTxs: auditSummary.addedTxs || [], freshTxs: auditSummary.freshTxs || [], sigopTxs: auditSummary.sigopTxs || [], + fullrbfTxs: auditSummary.fullrbfTxs || [], matchRate: auditSummary.matchRate, expectedFees: auditSummary.expectedFees, expectedWeight: auditSummary.expectedWeight, From 1abd2a23cce04a8913791eb88a0818814f35d106 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 14 Jul 2023 16:48:11 +0900 Subject: [PATCH 09/38] Add audit replication success logging --- backend/src/replication/AuditReplication.ts | 1 + backend/src/replication/replicator.ts | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/replication/AuditReplication.ts b/backend/src/replication/AuditReplication.ts index 2043532db..26bf6dad7 100644 --- a/backend/src/replication/AuditReplication.ts +++ b/backend/src/replication/AuditReplication.ts @@ -66,6 +66,7 @@ class AuditReplication { if (syncResult) { if (syncResult.data?.template?.length) { await this.$saveAuditData(hash, syncResult.data); + logger.info(`Imported audit data from ${syncResult.server} for block ${syncResult.data.height} (${hash})`); success = true; } if (!syncResult.data && !syncResult.exists) { diff --git a/backend/src/replication/replicator.ts b/backend/src/replication/replicator.ts index 60dfa8a2d..ac204efcc 100644 --- a/backend/src/replication/replicator.ts +++ b/backend/src/replication/replicator.ts @@ -4,7 +4,7 @@ import axios, { AxiosResponse } from 'axios'; import { SocksProxyAgent } from 'socks-proxy-agent'; import * as https from 'https'; -export async function $sync(path): Promise<{ data?: any, exists: boolean }> { +export async function $sync(path): Promise<{ data?: any, exists: boolean, server?: string }> { // start with a random server so load is uniformly spread let allMissing = true; const offset = Math.floor(Math.random() * config.REPLICATION.SERVERS.length); @@ -18,7 +18,7 @@ export async function $sync(path): Promise<{ data?: any, exists: boolean }> { try { const result = await query(`https://${server}${path}`); if (result) { - return { data: result, exists: true }; + return { data: result, exists: true, server }; } } catch (e: any) { if (e?.response?.status === 404) { From 9ffd4cc38d804cb498c422c9c481760e99fec4e3 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 15 Jul 2023 12:18:55 +0900 Subject: [PATCH 10/38] Calculator mobile margin --- .../src/app/components/calculator/calculator.component.scss | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/calculator/calculator.component.scss b/frontend/src/app/components/calculator/calculator.component.scss index 5c0cbb5b1..81f74f9ee 100644 --- a/frontend/src/app/components/calculator/calculator.component.scss +++ b/frontend/src/app/components/calculator/calculator.component.scss @@ -23,4 +23,8 @@ .sats { font-size: 20px; margin-left: 5px; -} \ No newline at end of file +} + +.row { + margin: auto; +} From 992196c91f7600578c079eefeaa77eead08e2a51 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 15 Jul 2023 15:09:41 +0900 Subject: [PATCH 11/38] Calculator validation improvements --- .../calculator/calculator.component.ts | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/calculator/calculator.component.ts b/frontend/src/app/components/calculator/calculator.component.ts index 838afbbd4..d99302f40 100644 --- a/frontend/src/app/components/calculator/calculator.component.ts +++ b/frontend/src/app/components/calculator/calculator.component.ts @@ -54,6 +54,9 @@ export class CalculatorComponent implements OnInit { ]).subscribe(([price, value]) => { const rate = (value / price).toFixed(8); const satsRate = Math.round(value / price * 100_000_000); + if (isNaN(value)) { + return; + } this.form.get('bitcoin').setValue(rate, { emitEvent: false }); this.form.get('satoshis').setValue(satsRate, { emitEvent: false } ); }); @@ -63,6 +66,9 @@ export class CalculatorComponent implements OnInit { this.form.get('bitcoin').valueChanges ]).subscribe(([price, value]) => { const rate = parseFloat((value * price).toFixed(8)); + if (isNaN(value)) { + return; + } this.form.get('fiat').setValue(rate, { emitEvent: false } ); this.form.get('satoshis').setValue(Math.round(value * 100_000_000), { emitEvent: false } ); }); @@ -73,6 +79,9 @@ export class CalculatorComponent implements OnInit { ]).subscribe(([price, value]) => { const rate = parseFloat((value / 100_000_000 * price).toFixed(8)); const bitcoinRate = (value / 100_000_000).toFixed(8); + if (isNaN(value)) { + return; + } this.form.get('fiat').setValue(rate, { emitEvent: false } ); this.form.get('bitcoin').setValue(bitcoinRate, { emitEvent: false }); }); @@ -88,7 +97,16 @@ export class CalculatorComponent implements OnInit { if (value === '.') { value = '0'; } - const sanitizedValue = this.removeExtraDots(value); + let sanitizedValue = this.removeExtraDots(value); + if (name === 'bitcoin' && this.countDecimals(sanitizedValue) > 8) { + sanitizedValue = this.toFixedWithoutRounding(sanitizedValue, 8); + } + if (sanitizedValue === '') { + sanitizedValue = '0'; + } + if (name === 'satoshis') { + sanitizedValue = parseFloat(sanitizedValue).toFixed(0); + } formControl.setValue(sanitizedValue, {emitEvent: true}); } @@ -100,4 +118,16 @@ export class CalculatorComponent implements OnInit { const afterDotReplaced = afterDot.replace(/\./g, ''); return `${beforeDot}.${afterDotReplaced}`; } + + countDecimals(numberString: string): number { + const decimalPos = numberString.indexOf('.'); + if (decimalPos === -1) return 0; + return numberString.length - decimalPos - 1; + } + + toFixedWithoutRounding(numStr: string, fixed: number): string { + const re = new RegExp(`^-?\\d+(?:.\\d{0,${(fixed || -1)}})?`); + const result = numStr.match(re); + return result ? result[0] : numStr; + } } From 73d9b4ef2873bda5a82316c734517407bb3373b0 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 15 Jul 2023 17:29:29 +0900 Subject: [PATCH 12/38] [price updater] update latestPrices timestamp before pushing to websocket --- backend/src/tasks/price-updater.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 3b9dad30e..fafe2b913 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -153,6 +153,7 @@ class PriceUpdater { try { const p = 60 * 60 * 1000; // milliseconds in an hour const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042 + this.latestPrices.time = nowRounded.getTime() / 1000; await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices); } catch (e) { this.lastRun = previousRun + 5 * 60; From b39f01471a492f8dcee347299db152fb09c32009 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 15 Jul 2023 17:47:36 +0900 Subject: [PATCH 13/38] Select all input box text on click --- .../src/app/components/calculator/calculator.component.html | 6 +++--- .../src/app/components/calculator/calculator.component.ts | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/components/calculator/calculator.component.html b/frontend/src/app/components/calculator/calculator.component.html index df2146760..bdbfdd0cd 100644 --- a/frontend/src/app/components/calculator/calculator.component.html +++ b/frontend/src/app/components/calculator/calculator.component.html @@ -12,7 +12,7 @@
{{ currency$ | async }}
- + @@ -20,7 +20,7 @@
BTC
- + @@ -28,7 +28,7 @@
sats
- + diff --git a/frontend/src/app/components/calculator/calculator.component.ts b/frontend/src/app/components/calculator/calculator.component.ts index d99302f40..a6f10c049 100644 --- a/frontend/src/app/components/calculator/calculator.component.ts +++ b/frontend/src/app/components/calculator/calculator.component.ts @@ -130,4 +130,8 @@ export class CalculatorComponent implements OnInit { const result = numStr.match(re); return result ? result[0] : numStr; } + + selectAll(event): void { + event.target.select(); + } } From b6a6fcd4e2e929f2f50d5f649fe0989e394c9048 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 16 Jul 2023 12:53:55 +0900 Subject: [PATCH 14/38] Fix tx.ancestors undefined bug --- .../src/app/components/transaction/transaction.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index 0faa328c0..e856f34eb 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -379,7 +379,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { ancestors: tx.ancestors, bestDescendant: tx.bestDescendant, }; - const hasRelatives = !!(tx.ancestors.length || tx.bestDescendant); + const hasRelatives = !!(tx.ancestors?.length || tx.bestDescendant); this.hasEffectiveFeeRate = hasRelatives || (tx.effectiveFeePerVsize && (Math.abs(tx.effectiveFeePerVsize - tx.feePerVsize) > 0.01)); } else { this.fetchCpfp$.next(this.tx.txid); From b33ea4679ddf0e0b7c9df69908296afd17cd0039 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 16 Jul 2023 13:49:33 +0900 Subject: [PATCH 15/38] Add "recently cpfpd" exception to audits --- backend/src/api/audit.ts | 11 +++++++---- backend/src/api/mempool-blocks.ts | 1 + backend/src/mempool.interfaces.ts | 1 + .../app/components/block-overview-graph/tx-view.ts | 3 ++- .../block-overview-tooltip.component.html | 1 + frontend/src/app/components/block/block.component.ts | 6 +++++- frontend/src/app/interfaces/node-api.interface.ts | 3 ++- frontend/src/app/interfaces/websocket.interface.ts | 2 +- 8 files changed, 20 insertions(+), 8 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index e79196a7a..f7aecfca8 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -1,12 +1,12 @@ import config from '../config'; import logger from '../logger'; -import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces'; +import { MempoolTransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces'; import rbfCache from './rbf-cache'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners class Audit { - auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended }) + auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }) : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], score: number, similarity: number } { if (!projectedBlocks?.[0]?.transactionIds || !mempool) { return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], score: 0, similarity: 1 }; @@ -14,7 +14,7 @@ class Audit { const matches: string[] = []; // present in both mined block and template const added: string[] = []; // present in mined block, not in template - const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN + const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN const fullrbf: string[] = []; // either missing or present, and part of a fullrbf replacement const isCensored = {}; // missing, without excuse const isDisplaced = {}; @@ -36,10 +36,13 @@ class Audit { // look for transactions that were expected in the template, but missing from the mined block for (const txid of projectedBlocks[0].transactionIds) { if (!inBlock[txid]) { - // tx is recent, may have reached the miner too late for inclusion if (rbfCache.isFullRbf(txid)) { fullrbf.push(txid); } else if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) { + // tx is recent, may have reached the miner too late for inclusion + fresh.push(txid); + } else if (mempool[txid]?.lastBoosted != null && (now - (mempool[txid]?.lastBoosted || 0)) <= PROPAGATION_MARGIN) { + // tx was recently cpfp'd, miner may not have the latest effective rate fresh.push(txid); } else { isCensored[txid] = true; diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index d5538854a..08508310d 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -457,6 +457,7 @@ class MempoolBlocks { }; if (matched) { descendants.push(relative); + mempoolTx.lastBoosted = Math.max(mempoolTx.lastBoosted || 0, mempool[txid].firstSeen || 0); } else { ancestors.push(relative); } diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index a051eea4f..3dad451ac 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -100,6 +100,7 @@ export interface MempoolTransactionExtended extends TransactionExtended { adjustedVsize: number; adjustedFeePerVsize: number; inputs?: number[]; + lastBoosted?: number; } export interface AuditTransaction { diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index 77f5a182a..452bb38f5 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -38,7 +38,7 @@ export default class TxView implements TransactionStripped { value: number; feerate: number; rate?: number; - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'fullrbf'; context?: 'projected' | 'actual'; scene?: BlockScene; @@ -210,6 +210,7 @@ export default class TxView implements TransactionStripped { case 'fullrbf': return marginalFeeColors[feeLevelIndex] || marginalFeeColors[mempoolFeeColors.length - 1]; case 'fresh': + case 'freshcpfp': return auditColors.missing; case 'added': return auditColors.added; diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html index 5ebd8fceb..59450326b 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html @@ -50,6 +50,7 @@ Marginal fee rate High sigop count Recently broadcasted + Recently CPFP'd Added Marginal fee rate Full RBF diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 0d733ff6b..4be6e3aff 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -370,7 +370,11 @@ export class BlockComponent implements OnInit, OnDestroy { tx.status = 'found'; } else { if (isFresh[tx.txid]) { - tx.status = 'fresh'; + if (tx.rate - (tx.fee / tx.vsize) >= 0.1) { + tx.status = 'freshcpfp'; + } else { + tx.status = 'fresh'; + } } else if (isSigop[tx.txid]) { tx.status = 'sigop'; } else if (isFullRbf[tx.txid]) { diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 7a8ab3f06..ad97d5f3d 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -173,7 +173,8 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf'; + rate?: number; // effective fee rate + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'fullrbf'; context?: 'projected' | 'actual'; } diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index 991fe2680..15d97fa8d 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -89,7 +89,7 @@ export interface TransactionStripped { vsize: number; value: number; rate?: number; // effective fee rate - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'fullrbf'; context?: 'projected' | 'actual'; } From 1f003cc2923123846d260e547dba70dad788bd59 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 5 Jul 2023 11:06:51 +0200 Subject: [PATCH 16/38] [lightning] save node features as stringified json array in db --- backend/src/api/database-migration.ts | 7 +- backend/src/api/explorer/nodes.api.ts | 17 ++- .../clightning/clightning-convert.ts | 112 +++++++++++++++++- .../tasks/lightning/network-sync.service.ts | 1 - backend/src/utils/format.ts | 33 ++++++ 5 files changed, 164 insertions(+), 6 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index a9266a016..7c7608aff 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 63; + private static currentVersion = 64; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -543,6 +543,11 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fullrbf_txs JSON DEFAULT "[]"'); await this.updateToSchemaVersion(63); } + + if (databaseSchemaVersion < 64 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `nodes` ADD features text NULL'); + await this.updateToSchemaVersion(64); + } } /** diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index d429299e1..f5c87fbd1 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -656,10 +656,19 @@ class NodesApi { alias_search, color, sockets, - status + status, + features ) - VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, ?, 1) - ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, alias_search = ?, color = ?, sockets = ?, status = 1`; + VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, ?, 1, ?) + ON DUPLICATE KEY UPDATE + updated_at = FROM_UNIXTIME(?), + alias = ?, + alias_search = ?, + color = ?, + sockets = ?, + status = 1, + features = ? + `; await DB.query(query, [ node.pub_key, @@ -668,11 +677,13 @@ class NodesApi { this.aliasToSearchText(node.alias), node.color, sockets, + JSON.stringify(node.features), node.last_update, node.alias, this.aliasToSearchText(node.alias), node.color, sockets, + JSON.stringify(node.features), ]); } catch (e) { logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e)); diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts index 084965383..75296ded2 100644 --- a/backend/src/api/lightning/clightning/clightning-convert.ts +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -2,8 +2,94 @@ import { ILightningApi } from '../lightning-api.interface'; import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher'; import logger from '../../../logger'; import { Common } from '../../common'; +import { hex2bin } from '../../../utils/format'; import config from '../../../config'; +// https://github.com/lightningnetwork/lnd/blob/master/lnwire/features.go +enum FeatureBits { + DataLossProtectRequired = 0, + DataLossProtectOptional = 1, + InitialRoutingSync = 3, + UpfrontShutdownScriptRequired = 4, + UpfrontShutdownScriptOptional = 5, + GossipQueriesRequired = 6, + GossipQueriesOptional = 7, + TLVOnionPayloadRequired = 8, + TLVOnionPayloadOptional = 9, + StaticRemoteKeyRequired = 12, + StaticRemoteKeyOptional = 13, + PaymentAddrRequired = 14, + PaymentAddrOptional = 15, + MPPRequired = 16, + MPPOptional = 17, + WumboChannelsRequired = 18, + WumboChannelsOptional = 19, + AnchorsRequired = 20, + AnchorsOptional = 21, + AnchorsZeroFeeHtlcTxRequired = 22, + AnchorsZeroFeeHtlcTxOptional = 23, + ShutdownAnySegwitRequired = 26, + ShutdownAnySegwitOptional = 27, + AMPRequired = 30, + AMPOptional = 31, + ExplicitChannelTypeRequired = 44, + ExplicitChannelTypeOptional = 45, + ScidAliasRequired = 46, + ScidAliasOptional = 47, + PaymentMetadataRequired = 48, + PaymentMetadataOptional = 49, + ZeroConfRequired = 50, + ZeroConfOptional = 51, + KeysendRequired = 54, + KeysendOptional = 55, + ScriptEnforcedLeaseRequired = 2022, + ScriptEnforcedLeaseOptional = 2023, + MaxBolt11Feature = 5114, +}; + +// Features is a mapping of known feature bits to a descriptive name. All known +// feature bits must be assigned a name in this mapping, and feature bit pairs +// must be assigned together for correct behavior. +const FeaturesMap = new Map([ + [FeatureBits.DataLossProtectRequired, 'data-loss-protect'], + [FeatureBits.DataLossProtectOptional, 'data-loss-protect'], + [FeatureBits.InitialRoutingSync, 'initial-routing-sync'], + [FeatureBits.UpfrontShutdownScriptRequired, 'upfront-shutdown-script'], + [FeatureBits.UpfrontShutdownScriptOptional, 'upfront-shutdown-script'], + [FeatureBits.GossipQueriesRequired, 'gossip-queries'], + [FeatureBits.GossipQueriesOptional, 'gossip-queries'], + [FeatureBits.TLVOnionPayloadRequired, 'tlv-onion'], + [FeatureBits.TLVOnionPayloadOptional, 'tlv-onion'], + [FeatureBits.StaticRemoteKeyOptional, 'static-remote-key'], + [FeatureBits.StaticRemoteKeyRequired, 'static-remote-key'], + [FeatureBits.PaymentAddrOptional, 'payment-addr'], + [FeatureBits.PaymentAddrRequired, 'payment-addr'], + [FeatureBits.MPPOptional, 'multi-path-payments'], + [FeatureBits.MPPRequired, 'multi-path-payments'], + [FeatureBits.AnchorsRequired, 'anchor-commitments'], + [FeatureBits.AnchorsOptional, 'anchor-commitments'], + [FeatureBits.AnchorsZeroFeeHtlcTxRequired, 'anchors-zero-fee-htlc-tx'], + [FeatureBits.AnchorsZeroFeeHtlcTxOptional, 'anchors-zero-fee-htlc-tx'], + [FeatureBits.WumboChannelsRequired, 'wumbo-channels'], + [FeatureBits.WumboChannelsOptional, 'wumbo-channels'], + [FeatureBits.AMPRequired, 'amp'], + [FeatureBits.AMPOptional, 'amp'], + [FeatureBits.PaymentMetadataOptional, 'payment-metadata'], + [FeatureBits.PaymentMetadataRequired, 'payment-metadata'], + [FeatureBits.ExplicitChannelTypeOptional, 'explicit-commitment-type'], + [FeatureBits.ExplicitChannelTypeRequired, 'explicit-commitment-type'], + [FeatureBits.KeysendOptional, 'keysend'], + [FeatureBits.KeysendRequired, 'keysend'], + [FeatureBits.ScriptEnforcedLeaseRequired, 'script-enforced-lease'], + [FeatureBits.ScriptEnforcedLeaseOptional, 'script-enforced-lease'], + [FeatureBits.ScidAliasRequired, 'scid-alias'], + [FeatureBits.ScidAliasOptional, 'scid-alias'], + [FeatureBits.ZeroConfRequired, 'zero-conf'], + [FeatureBits.ZeroConfOptional, 'zero-conf'], + [FeatureBits.ShutdownAnySegwitRequired, 'shutdown-any-segwit'], + [FeatureBits.ShutdownAnySegwitOptional, 'shutdown-any-segwit'], +]); + /** * Convert a clightning "listnode" entry to a lnd node entry */ @@ -17,10 +103,34 @@ export function convertNode(clNode: any): ILightningApi.Node { custom_records = undefined; } } + + const nodeFeatures: ILightningApi.Feature[] = []; + const nodeFeaturesBinary = hex2bin(clNode.features).split('').reverse().join(''); + + for (let i = 0; i < nodeFeaturesBinary.length; i++) { + if (nodeFeaturesBinary[i] === '0') { + continue; + } + const feature = FeaturesMap.get(i); + if (!feature) { + nodeFeatures.push({ + name: 'unknown', + is_required: i % 2 === 0, + is_known: false + }); + } else { + nodeFeatures.push({ + name: feature, + is_required: i % 2 === 0, + is_known: true + }); + } + } + return { alias: clNode.alias ?? '', color: `#${clNode.color ?? ''}`, - features: [], // TODO parse and return clNode.feature + features: nodeFeatures, pub_key: clNode.nodeid, addresses: clNode.addresses?.map((addr) => { let address = addr.address; diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index 6785b0e2d..963b9e8c2 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -3,7 +3,6 @@ import logger from '../../logger'; import channelsApi from '../../api/explorer/channels.api'; import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory'; import config from '../../config'; -import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface'; import { ILightningApi } from '../../api/lightning/lightning-api.interface'; import { $lookupNodeLocation } from './sync-tasks/node-locations'; import lightningApi from '../../api/lightning/lightning-api-factory'; diff --git a/backend/src/utils/format.ts b/backend/src/utils/format.ts index a18ce1892..9017f349f 100644 --- a/backend/src/utils/format.ts +++ b/backend/src/utils/format.ts @@ -26,4 +26,37 @@ export function formatBytes(bytes: number, toUnit: string, skipUnit = false): st } return `${bytes.toFixed(2)}${skipUnit ? '' : ' ' + byteUnits[unitIndex]}`; +} + +// https://stackoverflow.com/a/64235212 +export function hex2bin(hex): string { + if (!hex) { + return ''; + } + + hex = hex.replace('0x', '').toLowerCase(); + let out = ''; + + for (const c of hex) { + switch (c) { + case '0': out += '0000'; break; + case '1': out += '0001'; break; + case '2': out += '0010'; break; + case '3': out += '0011'; break; + case '4': out += '0100'; break; + case '5': out += '0101'; break; + case '6': out += '0110'; break; + case '7': out += '0111'; break; + case '8': out += '1000'; break; + case '9': out += '1001'; break; + case 'a': out += '1010'; break; + case 'b': out += '1011'; break; + case 'c': out += '1100'; break; + case 'd': out += '1101'; break; + case 'e': out += '1110'; break; + case 'f': out += '1111'; break; + default: return ''; + } + } + return out; } \ No newline at end of file From 32d46ad7ac4f2434219712d9024abe1e89400c3c Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 5 Jul 2023 11:17:04 +0200 Subject: [PATCH 17/38] [lightning] save bit number when converting features from clightning --- backend/src/api/explorer/nodes.api.ts | 5 ++++- backend/src/api/lightning/clightning/clightning-convert.ts | 2 ++ backend/src/api/lightning/lightning-api.interface.ts | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index f5c87fbd1..dc359f914 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -56,7 +56,8 @@ class NodesApi { UNIX_TIMESTAMP(updated_at) AS updated_at, color, sockets as sockets, as_number, city_id, country_id, subdivision_id, longitude, latitude, geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city, - geo_names_country.names as country, geo_names_subdivision.names as subdivision + geo_names_country.names as country, geo_names_subdivision.names as subdivision, + features FROM nodes LEFT JOIN geo_names geo_names_as on geo_names_as.id = as_number LEFT JOIN geo_names geo_names_city on geo_names_city.id = city_id @@ -75,6 +76,8 @@ class NodesApi { node.subdivision = JSON.parse(node.subdivision); node.city = JSON.parse(node.city); node.country = JSON.parse(node.country); + + node.features = JSON.parse(node.features); // Active channels and capacity const activeChannelsStats: any = await this.$getActiveChannelsStats(public_key); diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts index 75296ded2..84510d0fd 100644 --- a/backend/src/api/lightning/clightning/clightning-convert.ts +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -114,12 +114,14 @@ export function convertNode(clNode: any): ILightningApi.Node { const feature = FeaturesMap.get(i); if (!feature) { nodeFeatures.push({ + bit: i, name: 'unknown', is_required: i % 2 === 0, is_known: false }); } else { nodeFeatures.push({ + bit: i, name: feature, is_required: i % 2 === 0, is_known: true diff --git a/backend/src/api/lightning/lightning-api.interface.ts b/backend/src/api/lightning/lightning-api.interface.ts index cd5cb973d..ef26646a0 100644 --- a/backend/src/api/lightning/lightning-api.interface.ts +++ b/backend/src/api/lightning/lightning-api.interface.ts @@ -79,6 +79,7 @@ export namespace ILightningApi { } export interface Feature { + bit: number; name: string; is_required: boolean; is_known: boolean; From 4d41d36fe768493ef98716502b741decdcd8799a Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 5 Jul 2023 11:28:13 +0200 Subject: [PATCH 18/38] [lightning] save feature bit number when using lnd describegraph --- backend/src/api/lightning/lnd/lnd-api.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/backend/src/api/lightning/lnd/lnd-api.ts b/backend/src/api/lightning/lnd/lnd-api.ts index b4c91d36e..eb48b5f96 100644 --- a/backend/src/api/lightning/lnd/lnd-api.ts +++ b/backend/src/api/lightning/lnd/lnd-api.ts @@ -41,8 +41,23 @@ class LndApi implements AbstractLightningApi { } async $getNetworkGraph(): Promise { - return axios.get(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig) + const graph = await axios.get(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig) .then((response) => response.data); + + for (const node of graph.nodes) { + const nodeFeatures: ILightningApi.Feature[] = []; + for (const bit in node.features) { + nodeFeatures.push({ + bit: parseInt(bit, 10), + name: node.features[bit].name, + is_required: node.features[bit].is_required, + is_known: node.features[bit].is_known, + }); + } + node.features = nodeFeatures; + } + + return graph; } } From 556eb65320d358d9d4812cac27b3ea497774dc4e Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 5 Jul 2023 14:00:30 +0200 Subject: [PATCH 19/38] [lightning] start integrating features bits in the node page --- backend/src/api/explorer/nodes.api.ts | 13 ++++++- .../clightning/clightning-convert.ts | 7 ++-- backend/src/api/lightning/lnd/lnd-api.ts | 4 +-- backend/src/utils/format.ts | 35 ++++++++++++++++++- .../app/lightning/node/node.component.html | 19 ++++++++-- 5 files changed, 67 insertions(+), 11 deletions(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index dc359f914..40e106345 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -3,6 +3,7 @@ import DB from '../../database'; import { ResultSetHeader } from 'mysql2'; import { ILightningApi } from '../lightning/lightning-api.interface'; import { ITopNodesPerCapacity, ITopNodesPerChannels } from '../../mempool.interfaces'; +import { bin2hex } from '../../utils/format'; class NodesApi { public async $getWorldNodes(): Promise { @@ -76,8 +77,18 @@ class NodesApi { node.subdivision = JSON.parse(node.subdivision); node.city = JSON.parse(node.city); node.country = JSON.parse(node.country); - + + // Features node.features = JSON.parse(node.features); + let maxBit = 0; + for (const feature of node.features) { + maxBit = Math.max(maxBit, feature.bit); + } + node.featuresBits = new Array(maxBit + 1).fill(0); + for (const feature of node.features) { + node.featuresBits[feature.bit] = 1; + } + node.featuresBits = bin2hex(node.featuresBits.reverse().join('')); // Active channels and capacity const activeChannelsStats: any = await this.$getActiveChannelsStats(public_key); diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts index 84510d0fd..771dabcd7 100644 --- a/backend/src/api/lightning/clightning/clightning-convert.ts +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -6,7 +6,7 @@ import { hex2bin } from '../../../utils/format'; import config from '../../../config'; // https://github.com/lightningnetwork/lnd/blob/master/lnwire/features.go -enum FeatureBits { +export enum FeatureBits { DataLossProtectRequired = 0, DataLossProtectOptional = 1, InitialRoutingSync = 3, @@ -47,10 +47,7 @@ enum FeatureBits { MaxBolt11Feature = 5114, }; -// Features is a mapping of known feature bits to a descriptive name. All known -// feature bits must be assigned a name in this mapping, and feature bit pairs -// must be assigned together for correct behavior. -const FeaturesMap = new Map([ +export const FeaturesMap = new Map([ [FeatureBits.DataLossProtectRequired, 'data-loss-protect'], [FeatureBits.DataLossProtectOptional, 'data-loss-protect'], [FeatureBits.InitialRoutingSync, 'initial-routing-sync'], diff --git a/backend/src/api/lightning/lnd/lnd-api.ts b/backend/src/api/lightning/lnd/lnd-api.ts index eb48b5f96..f4099e82b 100644 --- a/backend/src/api/lightning/lnd/lnd-api.ts +++ b/backend/src/api/lightning/lnd/lnd-api.ts @@ -46,10 +46,10 @@ class LndApi implements AbstractLightningApi { for (const node of graph.nodes) { const nodeFeatures: ILightningApi.Feature[] = []; - for (const bit in node.features) { + for (const bit in node.features) { nodeFeatures.push({ bit: parseInt(bit, 10), - name: node.features[bit].name, + name: node.features[bit].name, is_required: node.features[bit].is_required, is_known: node.features[bit].is_known, }); diff --git a/backend/src/utils/format.ts b/backend/src/utils/format.ts index 9017f349f..63dc07ae4 100644 --- a/backend/src/utils/format.ts +++ b/backend/src/utils/format.ts @@ -29,7 +29,7 @@ export function formatBytes(bytes: number, toUnit: string, skipUnit = false): st } // https://stackoverflow.com/a/64235212 -export function hex2bin(hex): string { +export function hex2bin(hex: string): string { if (!hex) { return ''; } @@ -58,5 +58,38 @@ export function hex2bin(hex): string { default: return ''; } } + return out; +} + +export function bin2hex(bin: string): string { + if (!bin) { + return ''; + } + + let out = ''; + + for (let i = 0; i < bin.length; i += 4) { + const c = bin.substring(i, i + 4); + switch (c) { + case '0000': out += '0'; break; + case '0001': out += '1'; break; + case '0010': out += '2'; break; + case '0011': out += '3'; break; + case '0100': out += '4'; break; + case '0101': out += '5'; break; + case '0110': out += '6'; break; + case '0111': out += '7'; break; + case '1000': out += '8'; break; + case '1001': out += '9'; break; + case '1010': out += 'a'; break; + case '1011': out += 'b'; break; + case '1100': out += 'c'; break; + case '1101': out += 'd'; break; + case '1110': out += 'e'; break; + case '1111': out += 'f'; break; + default: return ''; + } + } + return out; } \ No newline at end of file diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index 2a74a68aa..c3903e915 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -21,7 +21,6 @@
-
@@ -59,6 +58,9 @@ + + +
Avg channel distance {{ avgDistance | amountShortener: 1 }} km ·{{ kmToMiles(avgDistance) | amountShortener: 1 }} mi
@@ -100,13 +102,26 @@ + + + + + +
- + + Features + + {{ bits }} + ⊕ + + +
From 6336c529edcf297909142627b5f75be7ab55fe40 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 8 Jul 2023 10:43:37 +0200 Subject: [PATCH 20/38] [lightning] show decoded features in node page --- .../app/lightning/node/node.component.html | 28 ++++++++++++++++++- .../src/app/lightning/node/node.component.ts | 7 ++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index c3903e915..0ed1c78c9 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -118,10 +118,36 @@ Features {{ bits }} - ⊕ + ⊕ +
+
+
+
+
Raw bits
+ {{ node.featuresBits }} +
+
Decoded
+ + + + + + + + + + + + + +
BitNameRequired
{{ feature.bit }}{{ feature.name }}{{ feature.is_required }}
+
+
+
+
diff --git a/frontend/src/app/lightning/node/node.component.ts b/frontend/src/app/lightning/node/node.component.ts index 47f65007f..719136d79 100644 --- a/frontend/src/app/lightning/node/node.component.ts +++ b/frontend/src/app/lightning/node/node.component.ts @@ -37,7 +37,7 @@ export class NodeComponent implements OnInit { liquidityAd: ILiquidityAd; tlvRecords: CustomRecord[]; avgChannelDistance$: Observable; - + showFeatures = false; kmToMiles = kmToMiles; constructor( @@ -164,4 +164,9 @@ export class NodeComponent implements OnInit { onLoadingEvent(e) { this.channelListLoading = e; } + + toggleFeatures() { + this.showFeatures = !this.showFeatures; + return false; + } } From 8fb67a914cc3eb12e9d6e74b9f22b08b56ce419a Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 8 Jul 2023 10:59:43 +0200 Subject: [PATCH 21/38] [lightning] fix node features binary conversion --- backend/src/api/explorer/nodes.api.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 40e106345..7d78559c4 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -84,6 +84,8 @@ class NodesApi { for (const feature of node.features) { maxBit = Math.max(maxBit, feature.bit); } + maxBit = Math.ceil(maxBit / 4) * 4 - 1; + node.featuresBits = new Array(maxBit + 1).fill(0); for (const feature of node.features) { node.featuresBits[feature.bit] = 1; From 6fe32cdd191259f53d95a3448ef912a6d319ed89 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sun, 16 Jul 2023 18:24:42 +0900 Subject: [PATCH 22/38] [lightning] fix issue during initial node.features indexing --- backend/src/api/explorer/nodes.api.ts | 23 +++++++++++-------- .../app/lightning/node/node.component.html | 4 ++-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 7d78559c4..22f9ca48a 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -80,17 +80,20 @@ class NodesApi { // Features node.features = JSON.parse(node.features); - let maxBit = 0; - for (const feature of node.features) { - maxBit = Math.max(maxBit, feature.bit); + node.featuresBits = null; + if (node.features) { + let maxBit = 0; + for (const feature of node.features) { + maxBit = Math.max(maxBit, feature.bit); + } + maxBit = Math.ceil(maxBit / 4) * 4 - 1; + + node.featuresBits = new Array(maxBit + 1).fill(0); + for (const feature of node.features) { + node.featuresBits[feature.bit] = 1; + } + node.featuresBits = bin2hex(node.featuresBits.reverse().join('')); } - maxBit = Math.ceil(maxBit / 4) * 4 - 1; - - node.featuresBits = new Array(maxBit + 1).fill(0); - for (const feature of node.features) { - node.featuresBits[feature.bit] = 1; - } - node.featuresBits = bin2hex(node.featuresBits.reverse().join('')); // Active channels and capacity const activeChannelsStats: any = await this.$getActiveChannelsStats(public_key); diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index 0ed1c78c9..47b10184e 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -102,10 +102,10 @@ - + - + From 565336df21a6d2c0a821e73752aee9d8bf8c36be Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 16 Jul 2023 18:39:51 +0900 Subject: [PATCH 23/38] Set missing websocket init data --- backend/src/api/websocket-handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 48e9106f0..e31221dfd 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -236,7 +236,7 @@ class WebsocketHandler { } if (parsedMessage.action === 'init') { - if (!this.socketData['blocks']?.length || !this.socketData['da']) { + if (!this.socketData['blocks']?.length || !this.socketData['da'] || !this.socketData['backendInfo'] || !this.socketData['conversions']) { this.updateSocketData(); } if (!this.socketData['blocks']?.length) { From a7ec9138c3308e692625550396b004b119c6a780 Mon Sep 17 00:00:00 2001 From: wiz Date: Mon, 17 Jul 2023 01:14:37 +0900 Subject: [PATCH 24/38] ops: Bump elements tag to 22.1.1 --- production/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/install b/production/install index 1121f5b4f..9ea9b7a75 100755 --- a/production/install +++ b/production/install @@ -353,7 +353,7 @@ ELEMENTS_REPO_URL=https://github.com/ElementsProject/elements ELEMENTS_REPO_NAME=elements ELEMENTS_REPO_BRANCH=master #ELEMENTS_LATEST_RELEASE=$(curl -s https://api.github.com/repos/ElementsProject/elements/releases/latest|grep tag_name|head -1|cut -d '"' -f4) -ELEMENTS_LATEST_RELEASE=elements-22.1 +ELEMENTS_LATEST_RELEASE=elements-22.1.1 echo -n '.' BITCOIN_ELECTRS_REPO_URL=https://github.com/mempool/electrs From bf5a16b043cb5eb7b70d396bd25c4fd3fcbe649c Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 17 Jul 2023 11:02:28 +0900 Subject: [PATCH 25/38] always send 6 latest transactions to websocket clients --- backend/src/api/websocket-handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index e31221dfd..ab7dcf443 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -419,7 +419,7 @@ class WebsocketHandler { memPool.addToSpendMap(newTransactions); const recommendedFees = feeApi.getRecommendedFee(); - const latestTransactions = newTransactions.slice(0, 6).map((tx) => Common.stripTransaction(tx)); + const latestTransactions = memPool.getLatestTransactions(); // update init data const socketDataFields = { From 2c39e1e203786d5520018fc7fcbf275a026cf23d Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 17 Jul 2023 11:15:55 +0900 Subject: [PATCH 26/38] unbork mining pool blocks list --- frontend/src/app/components/pool/pool.component.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/components/pool/pool.component.ts b/frontend/src/app/components/pool/pool.component.ts index 139da5ef0..f2fc79ff2 100644 --- a/frontend/src/app/components/pool/pool.component.ts +++ b/frontend/src/app/components/pool/pool.component.ts @@ -37,7 +37,7 @@ export class PoolComponent implements OnInit { auditAvailable = false; - loadMoreSubject: BehaviorSubject = new BehaviorSubject(this.blocks[0]?.height); + loadMoreSubject: BehaviorSubject = new BehaviorSubject(this.blocks[this.blocks.length - 1]?.height); constructor( @Inject(LOCALE_ID) public locale: string, @@ -91,7 +91,7 @@ export class PoolComponent implements OnInit { if (this.slug === undefined) { return []; } - return this.apiService.getPoolBlocks$(this.slug, this.blocks[0]?.height); + return this.apiService.getPoolBlocks$(this.slug, this.blocks[this.blocks.length - 1]?.height); }), tap((newBlocks) => { this.blocks = this.blocks.concat(newBlocks); @@ -237,7 +237,7 @@ export class PoolComponent implements OnInit { } loadMore() { - this.loadMoreSubject.next(this.blocks[0]?.height); + this.loadMoreSubject.next(this.blocks[this.blocks.length - 1]?.height); } trackByBlock(index: number, block: BlockExtended) { From 7a059ba29476a0cad6a4fd704061abfd1e774e4a Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 16 Jul 2023 17:12:26 +0900 Subject: [PATCH 27/38] get chain tip direct from Bitcoin Core to avoid race conditions --- backend/src/api/bitcoin/bitcoin-api.ts | 10 ++-------- backend/src/api/blocks.ts | 15 +++++++++------ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index a0cc41770..cbcb2c571 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -65,17 +65,11 @@ class BitcoinApi implements AbstractBitcoinApi { } $getBlockHeightTip(): Promise { - return this.bitcoindClient.getChainTips() - .then((result: IBitcoinApi.ChainTips[]) => { - return result.find(tip => tip.status === 'active')!.height; - }); + return this.bitcoindClient.getBlockCount(); } $getBlockHashTip(): Promise { - return this.bitcoindClient.getChainTips() - .then((result: IBitcoinApi.ChainTips[]) => { - return result.find(tip => tip.status === 'active')!.hash; - }); + return this.bitcoindClient.getBestBlockHash(); } $getTxIdsForBlock(hash: string): Promise { diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 5939421a7..fdf32f438 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -76,11 +76,14 @@ class Blocks { blockHash: string, blockHeight: number, onlyCoinbase: boolean, + txIds: string[] | null = null, quiet: boolean = false, addMempoolData: boolean = false, ): Promise { const transactions: TransactionExtended[] = []; - const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); + if (!txIds) { + txIds = await bitcoinApi.$getTxIdsForBlock(blockHash); + } const mempool = memPool.getMempool(); let transactionsFound = 0; @@ -554,7 +557,7 @@ class Blocks { } const blockHash = await bitcoinApi.$getBlockHash(blockHeight); const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); - const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true); + const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, null, true); const blockExtended = await this.$getBlockExtended(block, transactions); newlyIndexed++; @@ -586,7 +589,7 @@ class Blocks { let fastForwarded = false; let handledBlocks = 0; - const blockHeightTip = await bitcoinApi.$getBlockHeightTip(); + const blockHeightTip = await bitcoinCoreApi.$getBlockHeightTip(); this.updateTimerProgress(timer, 'got block height tip'); if (this.blocks.length === 0) { @@ -639,11 +642,11 @@ class Blocks { } this.updateTimerProgress(timer, `getting block data for ${this.currentBlockHeight}`); - const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight); + const blockHash = await bitcoinCoreApi.$getBlockHash(this.currentBlockHeight); const verboseBlock = await bitcoinClient.getBlock(blockHash, 2); const block = BitcoinApi.convertBlock(verboseBlock); - const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); - const transactions = await this.$getTransactionsExtended(blockHash, block.height, false, false, true) as MempoolTransactionExtended[]; + const txIds: string[] = verboseBlock.tx.map(tx => tx.txid); + const transactions = await this.$getTransactionsExtended(blockHash, block.height, false, txIds, false, true) as MempoolTransactionExtended[]; if (config.MEMPOOL.BACKEND !== 'esplora') { // fill in missing transaction fee data from verboseBlock for (let i = 0; i < transactions.length; i++) { From 6ab3b89884c8d7631e45e23a0f1a91070e783556 Mon Sep 17 00:00:00 2001 From: softsimon Date: Mon, 17 Jul 2023 17:41:38 +0900 Subject: [PATCH 28/38] Change to a Details-button --- frontend/src/app/lightning/node/node.component.html | 2 +- frontend/src/styles.scss | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index 47b10184e..c6c693a3a 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -118,7 +118,7 @@ Features {{ bits }} - ⊕ + diff --git a/frontend/src/styles.scss b/frontend/src/styles.scss index ac299a547..428752d60 100644 --- a/frontend/src/styles.scss +++ b/frontend/src/styles.scss @@ -1164,3 +1164,10 @@ app-master-page, app-liquid-master-page, app-bisq-master-page { app-global-footer { margin-top: auto; } + +.btn-xs { + padding: 0.25rem 0.5rem; + font-size: 0.875rem; + line-height: 0.5; + border-radius: 0.2rem; +} From 4309bfd519e913db34b64c9e1846cc7e8af23e6e Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 17 Jul 2023 17:53:26 +0900 Subject: [PATCH 29/38] Fix websocket null data for undefined rbfSummary --- backend/src/api/websocket-handler.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index ab7dcf443..a0c031175 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -229,7 +229,9 @@ class WebsocketHandler { if (parsedMessage && parsedMessage['track-rbf-summary'] != null) { if (parsedMessage['track-rbf-summary']) { client['track-rbf-summary'] = true; - response['rbfLatestSummary'] = this.socketData['rbfSummary']; + if (this.socketData['rbfSummary'] != null) { + response['rbfLatestSummary'] = this.socketData['rbfSummary']; + } } else { client['track-rbf-summary'] = false; } From 2c1b9b9095cde80c4d1e7e960cf7fad4b2b4898d Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 17 Jul 2023 18:21:44 +0900 Subject: [PATCH 30/38] Fix mempool update poll delay --- backend/src/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index 81863a208..bbfaa9ff3 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -169,6 +169,7 @@ class Server { } async runMainUpdateLoop(): Promise { + const start = Date.now(); try { try { await memPool.$updateMemPoolInfo(); @@ -188,7 +189,9 @@ class Server { indexer.$run(); // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS - setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 1 : config.MEMPOOL.POLL_RATE_MS); + const elapsed = Date.now() - start; + const remainingTime = Math.max(0, config.MEMPOOL.POLL_RATE_MS - elapsed) + setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 0 : remainingTime); this.backendRetryCount = 0; } catch (e: any) { this.backendRetryCount++; From 6a16759e2025ac2aa75415922a29aa60dc1cfc24 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 17 Jul 2023 18:37:46 +0900 Subject: [PATCH 31/38] Hide confirmations badge if height unknown --- .../app/components/transaction/transaction.component.html | 1 + .../components/confirmations/confirmations.component.html | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index 203a5df5c..d4cd6913d 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -19,6 +19,7 @@
{{ i }} confirmations + + + - + \ No newline at end of file From c02eef352be0c36482c7123802110bf2bcc79897 Mon Sep 17 00:00:00 2001 From: wiz Date: Mon, 17 Jul 2023 22:56:16 +0900 Subject: [PATCH 32/38] ops: Increase bitcoin.conf maxconnections 42 -> 100 --- production/bitcoin.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/bitcoin.conf b/production/bitcoin.conf index d9f3fc252..fb8ad52e5 100644 --- a/production/bitcoin.conf +++ b/production/bitcoin.conf @@ -9,7 +9,7 @@ dbcache=8192 maxmempool=4096 mempoolexpiry=999999 mempoolfullrbf=1 -maxconnections=42 +maxconnections=100 onion=127.0.0.1:9050 rpcallowip=127.0.0.1 rpcuser=__BITCOIN_RPC_USER__ From 7e06c97f519d3fc4c74817574c9e60309eb939c6 Mon Sep 17 00:00:00 2001 From: wiz Date: Tue, 18 Jul 2023 07:50:14 +0900 Subject: [PATCH 33/38] ops: Bump NodeJS to v20.4.0 --- production/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/install b/production/install index 9ea9b7a75..2a5064b89 100755 --- a/production/install +++ b/production/install @@ -1045,7 +1045,7 @@ echo "[*] Installing nvm.sh from GitHub" osSudo "${MEMPOOL_USER}" sh -c 'curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | zsh' echo "[*] Building NodeJS via nvm.sh" -osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm install v16.16.0 --shared-zlib' +osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm install v20.4.0 --shared-zlib' #################### # Tor installation # From fa48c6f025cb1ade799baf82b5c21f2b75093bb3 Mon Sep 17 00:00:00 2001 From: wiz Date: Tue, 18 Jul 2023 08:52:51 +0900 Subject: [PATCH 34/38] ops: Use NodeJS v18 to build, v20 to run backend --- production/install | 5 ++++- production/mempool-start-all | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/production/install b/production/install index 2a5064b89..1af975d64 100755 --- a/production/install +++ b/production/install @@ -1044,8 +1044,11 @@ osSudo "${ROOT_USER}" crontab -u "${MEMPOOL_USER}" "${MEMPOOL_HOME}/${MEMPOOL_RE echo "[*] Installing nvm.sh from GitHub" osSudo "${MEMPOOL_USER}" sh -c 'curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | zsh' -echo "[*] Building NodeJS via nvm.sh" +echo "[*] Building NodeJS v20.4.0 via nvm.sh" osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm install v20.4.0 --shared-zlib' +echo "[*] Building NodeJS v18.16.1 via nvm.sh" +osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm install v18.16.1 --shared-zlib' +osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm alias default 18.16.1' #################### # Tor installation # diff --git a/production/mempool-start-all b/production/mempool-start-all index 13fd30430..306156660 100755 --- a/production/mempool-start-all +++ b/production/mempool-start-all @@ -1,6 +1,7 @@ #!/usr/bin/env zsh export NVM_DIR="$HOME/.nvm" source "$NVM_DIR/nvm.sh" +nvm use v20.4.0 # start all mempool backends that exist for site in mainnet mainnet-lightning testnet testnet-lightning signet signet-lightning bisq liquid liquidtestnet;do From 5e46176c4e9cf20377ffa4b8fa8c6ae9a7e68774 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 18 Jul 2023 10:52:47 +0900 Subject: [PATCH 35/38] Fix clock horizontal scroll bug --- frontend/src/app/components/clock/clock.component.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/app/components/clock/clock.component.scss b/frontend/src/app/components/clock/clock.component.scss index 20baf02ee..64929aa38 100644 --- a/frontend/src/app/components/clock/clock.component.scss +++ b/frontend/src/app/components/clock/clock.component.scss @@ -9,6 +9,7 @@ display: flex; flex-direction: column; justify-content: flex-start; + overflow: hidden; --chain-height: 60px; --clock-width: 300px; From 17866f80bd4ddb14e3469b9357043612fcc18b8c Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 18 Jul 2023 11:01:35 +0900 Subject: [PATCH 36/38] Remove frontend FULL_RBF_ENABLED flag --- docker/frontend/entrypoint.sh | 2 -- frontend/mempool-frontend-config.sample.json | 1 - frontend/src/app/components/rbf-list/rbf-list.component.html | 2 +- frontend/src/app/components/rbf-list/rbf-list.component.ts | 5 +---- frontend/src/app/services/state.service.ts | 2 -- 5 files changed, 2 insertions(+), 10 deletions(-) diff --git a/docker/frontend/entrypoint.sh b/docker/frontend/entrypoint.sh index 013b1ce53..b6946578b 100644 --- a/docker/frontend/entrypoint.sh +++ b/docker/frontend/entrypoint.sh @@ -39,7 +39,6 @@ __AUDIT__=${AUDIT:=false} __MAINNET_BLOCK_AUDIT_START_HEIGHT__=${MAINNET_BLOCK_AUDIT_START_HEIGHT:=0} __TESTNET_BLOCK_AUDIT_START_HEIGHT__=${TESTNET_BLOCK_AUDIT_START_HEIGHT:=0} __SIGNET_BLOCK_AUDIT_START_HEIGHT__=${SIGNET_BLOCK_AUDIT_START_HEIGHT:=0} -__FULL_RBF_ENABLED__=${FULL_RBF_ENABLED:=false} __HISTORICAL_PRICE__=${HISTORICAL_PRICE:=true} # Export as environment variables to be used by envsubst @@ -66,7 +65,6 @@ export __AUDIT__ export __MAINNET_BLOCK_AUDIT_START_HEIGHT__ export __TESTNET_BLOCK_AUDIT_START_HEIGHT__ export __SIGNET_BLOCK_AUDIT_START_HEIGHT__ -export __FULL_RBF_ENABLED__ export __HISTORICAL_PRICE__ folder=$(find /var/www/mempool -name "config.js" | xargs dirname) diff --git a/frontend/mempool-frontend-config.sample.json b/frontend/mempool-frontend-config.sample.json index c45425612..084cbd0ef 100644 --- a/frontend/mempool-frontend-config.sample.json +++ b/frontend/mempool-frontend-config.sample.json @@ -22,6 +22,5 @@ "TESTNET_BLOCK_AUDIT_START_HEIGHT": 0, "SIGNET_BLOCK_AUDIT_START_HEIGHT": 0, "LIGHTNING": false, - "FULL_RBF_ENABLED": false, "HISTORICAL_PRICE": true } diff --git a/frontend/src/app/components/rbf-list/rbf-list.component.html b/frontend/src/app/components/rbf-list/rbf-list.component.html index eebb7e152..5ce6254c4 100644 --- a/frontend/src/app/components/rbf-list/rbf-list.component.html +++ b/frontend/src/app/components/rbf-list/rbf-list.component.html @@ -2,7 +2,7 @@

RBF Replacements

-
+