diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 7948049fc..47ec6898a 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -32,7 +32,8 @@ "CPFP_INDEXING": false, "DISK_CACHE_BLOCK_INTERVAL": 6, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, - "ALLOW_UNREACHABLE": true + "ALLOW_UNREACHABLE": true, + "PRICE_UPDATES_PER_HOUR": 1 }, "CORE_RPC": { "HOST": "127.0.0.1", @@ -115,10 +116,6 @@ "USERNAME": "", "PASSWORD": "" }, - "PRICE_DATA_SERVER": { - "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", - "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" - }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "https://mempool.space/api/v1", "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1", @@ -137,5 +134,9 @@ "trusted", "servers" ] + }, + "MEMPOOL_SERVICES": { + "API": "https://mempool.space/api", + "ACCELERATIONS": false } } diff --git a/backend/npm_package.sh b/backend/npm_package.sh new file mode 100755 index 000000000..627a77360 --- /dev/null +++ b/backend/npm_package.sh @@ -0,0 +1,14 @@ +#/bin/sh +set -e + +npm run build +# Remove previous package folder +rm -rf package +# Move JS and deps +mv dist package +mv node_modules package +# Remove symlink for rust-gbt and insert real folder +rm package/node_modules/rust-gbt +mv rust-gbt package/node_modules +# Clean up deps +npm run package-rm-build-deps diff --git a/backend/npm_package_rm_build_deps.sh b/backend/npm_package_rm_build_deps.sh new file mode 100755 index 000000000..6b260d84d --- /dev/null +++ b/backend/npm_package_rm_build_deps.sh @@ -0,0 +1,12 @@ +#/bin/sh +set -e + +# Cleaning up inside the node_modules folder +cd package/node_modules +rm -r \ + typescript \ + @typescript-eslint \ + @napi-rs \ + ./rust-gbt/src \ + ./rust-gbt/Cargo.toml \ + ./rust-gbt/build.rs diff --git a/backend/package-lock.json b/backend/package-lock.json index 1a92552cb..f5452e908 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -17,9 +17,9 @@ "crypto-js": "~4.1.1", "express": "~4.18.2", "maxmind": "~4.3.11", - "mysql2": "~3.5.2", - "rust-gbt": "file:./rust-gbt", + "mysql2": "~3.6.0", "redis": "^4.6.6", + "rust-gbt": "file:./rust-gbt", "socks-proxy-agent": "~7.0.0", "typescript": "~4.9.3", "ws": "~8.13.0" @@ -6102,9 +6102,9 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/mysql2": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.5.2.tgz", - "integrity": "sha512-cptobmhYkYeTBIFp2c0piw2+gElpioga1rUw5UidHvo8yaHijMZoo8A3zyBVoo/K71f7ZFvrShA9iMIy9dCzCA==", + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.6.0.tgz", + "integrity": "sha512-EWUGAhv6SphezurlfI2Fpt0uJEWLmirrtQR7SkbTHFC+4/mJBrPiSzHESHKAWKG7ALVD6xaG/NBjjd1DGJGQQQ==", "dependencies": { "denque": "^2.1.0", "generate-function": "^2.3.1", @@ -12212,9 +12212,9 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "mysql2": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.5.2.tgz", - "integrity": "sha512-cptobmhYkYeTBIFp2c0piw2+gElpioga1rUw5UidHvo8yaHijMZoo8A3zyBVoo/K71f7ZFvrShA9iMIy9dCzCA==", + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.6.0.tgz", + "integrity": "sha512-EWUGAhv6SphezurlfI2Fpt0uJEWLmirrtQR7SkbTHFC+4/mJBrPiSzHESHKAWKG7ALVD6xaG/NBjjd1DGJGQQQ==", "requires": { "denque": "^2.1.0", "generate-function": "^2.3.1", diff --git a/backend/package.json b/backend/package.json index 24da55e17..e21924207 100644 --- a/backend/package.json +++ b/backend/package.json @@ -22,10 +22,10 @@ "main": "index.ts", "scripts": { "tsc": "./node_modules/typescript/bin/tsc -p tsconfig.build.json", - "build": "npm run build-rust && npm run tsc && npm run create-resources", + "build": "npm run tsc && npm run create-resources", "create-resources": "cp ./src/tasks/price-feeds/mtgox-weekly.json ./dist/tasks && node dist/api/fetch-version.js", - "package": "npm run build && rm -rf package && mv dist package && mv node_modules package && mv rust-gbt package && npm run package-rm-build-deps", - "package-rm-build-deps": "(cd package/node_modules; rm -r typescript @typescript-eslint @napi-rs ../rust-gbt/target ../rust-gbt/node_modules ../rust-gbt/src)", + "package": "./npm_package.sh", + "package-rm-build-deps": "./npm_package_rm_build_deps.sh", "start": "node --max-old-space-size=2048 dist/index.js", "start-production": "node --max-old-space-size=16384 dist/index.js", "reindex-updated-pools": "npm run start-production --update-pools", @@ -33,8 +33,7 @@ "test": "./node_modules/.bin/jest --coverage", "lint": "./node_modules/.bin/eslint . --ext .ts", "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix", - "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\"", - "build-rust": "cd rust-gbt && npm install" + "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\"" }, "dependencies": { "@babel/core": "^7.21.3", @@ -45,7 +44,7 @@ "crypto-js": "~4.1.1", "express": "~4.18.2", "maxmind": "~4.3.11", - "mysql2": "~3.5.2", + "mysql2": "~3.6.0", "rust-gbt": "file:./rust-gbt", "redis": "^4.6.6", "socks-proxy-agent": "~7.0.0", diff --git a/backend/rust-gbt/index.d.ts b/backend/rust-gbt/index.d.ts index 33ae32bdf..2bd8a620a 100644 --- a/backend/rust-gbt/index.d.ts +++ b/backend/rust-gbt/index.d.ts @@ -12,6 +12,10 @@ export interface ThreadTransaction { effectiveFeePerVsize: number inputs: Array } +export interface ThreadAcceleration { + uid: number + delta: number +} export class GbtGenerator { constructor() /** @@ -19,13 +23,13 @@ export class GbtGenerator { * * Rejects if the thread panics or if the Mutex is poisoned. */ - make(mempool: Array, maxUid: number): Promise + make(mempool: Array, accelerations: Array, maxUid: number): Promise /** * # Errors * * Rejects if the thread panics or if the Mutex is poisoned. */ - update(newTxs: Array, removeTxs: Array, maxUid: number): Promise + update(newTxs: Array, removeTxs: Array, accelerations: Array, maxUid: number): Promise } /** * The result from calling the gbt function. diff --git a/backend/rust-gbt/src/audit_transaction.rs b/backend/rust-gbt/src/audit_transaction.rs index 3e25a18a0..fe20e5a14 100644 --- a/backend/rust-gbt/src/audit_transaction.rs +++ b/backend/rust-gbt/src/audit_transaction.rs @@ -1,6 +1,6 @@ use crate::{ u32_hasher_types::{u32hashset_new, U32HasherState}, - ThreadTransaction, + ThreadTransaction, thread_acceleration::ThreadAcceleration, }; use std::{ cmp::Ordering, @@ -88,44 +88,49 @@ impl Ord for AuditTransaction { } #[inline] -fn calc_fee_rate(fee: f64, vsize: f64) -> f64 { - fee / (if vsize == 0.0 { 1.0 } else { vsize }) +fn calc_fee_rate(fee: u64, vsize: f64) -> f64 { + (fee as f64) / (if vsize == 0.0 { 1.0 } else { vsize }) } impl AuditTransaction { - pub fn from_thread_transaction(tx: &ThreadTransaction) -> Self { + pub fn from_thread_transaction(tx: &ThreadTransaction, maybe_acceleration: Option>) -> Self { + let fee_delta = match maybe_acceleration { + Some(Some(acceleration)) => acceleration.delta, + _ => 0.0 + }; + let fee = (tx.fee as u64) + (fee_delta as u64); // rounded up to the nearest integer let is_adjusted = tx.weight < (tx.sigops * 20); let sigop_adjusted_vsize = ((tx.weight + 3) / 4).max(tx.sigops * 5); let sigop_adjusted_weight = tx.weight.max(tx.sigops * 20); - let effective_fee_per_vsize = if is_adjusted { - calc_fee_rate(tx.fee, f64::from(sigop_adjusted_weight) / 4.0) + let effective_fee_per_vsize = if is_adjusted || fee_delta > 0.0 { + calc_fee_rate(fee, f64::from(sigop_adjusted_weight) / 4.0) } else { tx.effective_fee_per_vsize }; Self { uid: tx.uid, order: tx.order, - fee: tx.fee as u64, + fee, weight: tx.weight, sigop_adjusted_weight, sigop_adjusted_vsize, sigops: tx.sigops, - adjusted_fee_per_vsize: calc_fee_rate(tx.fee, f64::from(sigop_adjusted_vsize)), + adjusted_fee_per_vsize: calc_fee_rate(fee, f64::from(sigop_adjusted_vsize)), effective_fee_per_vsize, dependency_rate: f64::INFINITY, inputs: tx.inputs.clone(), relatives_set_flag: false, ancestors: u32hashset_new(), children: u32hashset_new(), - ancestor_fee: tx.fee as u64, + ancestor_fee: fee, ancestor_sigop_adjusted_weight: sigop_adjusted_weight, ancestor_sigop_adjusted_vsize: sigop_adjusted_vsize, ancestor_sigops: tx.sigops, score: 0.0, used: false, modified: false, - dirty: effective_fee_per_vsize != tx.effective_fee_per_vsize, + dirty: effective_fee_per_vsize != tx.effective_fee_per_vsize || fee_delta > 0.0, } } @@ -156,7 +161,7 @@ impl AuditTransaction { // grows, so if we think of 0 as "grew infinitely" then dependency_rate would be // the smaller of the two. If either side is NaN, the other side is returned. self.dependency_rate.min(calc_fee_rate( - self.ancestor_fee as f64, + self.ancestor_fee, f64::from(self.ancestor_sigop_adjusted_weight) / 4.0, )) } @@ -172,7 +177,7 @@ impl AuditTransaction { #[inline] fn calc_new_score(&mut self) { self.score = self.adjusted_fee_per_vsize.min(calc_fee_rate( - self.ancestor_fee as f64, + self.ancestor_fee, f64::from(self.ancestor_sigop_adjusted_vsize), )); } diff --git a/backend/rust-gbt/src/gbt.rs b/backend/rust-gbt/src/gbt.rs index 09b6377e6..0bf7f9999 100644 --- a/backend/rust-gbt/src/gbt.rs +++ b/backend/rust-gbt/src/gbt.rs @@ -5,7 +5,7 @@ use tracing::{info, trace}; use crate::{ audit_transaction::{partial_cmp_uid_score, AuditTransaction}, u32_hasher_types::{u32hashset_new, u32priority_queue_with_capacity, U32HasherState}, - GbtResult, ThreadTransactionsMap, + GbtResult, ThreadTransactionsMap, thread_acceleration::ThreadAcceleration, }; const MAX_BLOCK_WEIGHT_UNITS: u32 = 4_000_000 - 4_000; @@ -53,7 +53,13 @@ impl Ord for TxPriority { // TODO: Make gbt smaller to fix these lints. #[allow(clippy::too_many_lines)] #[allow(clippy::cognitive_complexity)] -pub fn gbt(mempool: &mut ThreadTransactionsMap, max_uid: usize) -> GbtResult { +pub fn gbt(mempool: &mut ThreadTransactionsMap, accelerations: &[ThreadAcceleration], max_uid: usize) -> GbtResult { + let mut indexed_accelerations = Vec::with_capacity(max_uid + 1); + indexed_accelerations.resize(max_uid + 1, None); + for acceleration in accelerations { + indexed_accelerations[acceleration.uid as usize] = Some(acceleration); + } + let mempool_len = mempool.len(); let mut audit_pool: AuditPool = Vec::with_capacity(max_uid + 1); audit_pool.resize(max_uid + 1, None); @@ -63,7 +69,8 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap, max_uid: usize) -> GbtResult { info!("Initializing working structs"); for (uid, tx) in &mut *mempool { - let audit_tx = AuditTransaction::from_thread_transaction(tx); + let acceleration = indexed_accelerations.get(*uid as usize); + let audit_tx = AuditTransaction::from_thread_transaction(tx, acceleration.copied()); // Safety: audit_pool and mempool_stack must always contain the same transactions audit_pool[*uid as usize] = Some(ManuallyDrop::new(audit_tx)); mempool_stack.push(*uid); diff --git a/backend/rust-gbt/src/lib.rs b/backend/rust-gbt/src/lib.rs index 516a26402..53db0ba21 100644 --- a/backend/rust-gbt/src/lib.rs +++ b/backend/rust-gbt/src/lib.rs @@ -9,6 +9,7 @@ use napi::bindgen_prelude::Result; use napi_derive::napi; use thread_transaction::ThreadTransaction; +use thread_acceleration::ThreadAcceleration; use tracing::{debug, info, trace}; use tracing_log::LogTracer; use tracing_subscriber::{EnvFilter, FmtSubscriber}; @@ -19,6 +20,7 @@ use std::sync::{Arc, Mutex}; mod audit_transaction; mod gbt; mod thread_transaction; +mod thread_acceleration; mod u32_hasher_types; use u32_hasher_types::{u32hashmap_with_capacity, U32HasherState}; @@ -74,10 +76,11 @@ impl GbtGenerator { /// /// Rejects if the thread panics or if the Mutex is poisoned. #[napi] - pub async fn make(&self, mempool: Vec, max_uid: u32) -> Result { + pub async fn make(&self, mempool: Vec, accelerations: Vec, max_uid: u32) -> Result { trace!("make: Current State {:#?}", self.thread_transactions); run_task( Arc::clone(&self.thread_transactions), + accelerations, max_uid as usize, move |map| { for tx in mempool { @@ -96,11 +99,13 @@ impl GbtGenerator { &self, new_txs: Vec, remove_txs: Vec, + accelerations: Vec, max_uid: u32, ) -> Result { trace!("update: Current State {:#?}", self.thread_transactions); run_task( Arc::clone(&self.thread_transactions), + accelerations, max_uid as usize, move |map| { for tx in new_txs { @@ -141,6 +146,7 @@ pub struct GbtResult { /// to the `HashMap` as the only argument. (A move closure is recommended to meet the bounds) async fn run_task( thread_transactions: Arc>, + accelerations: Vec, max_uid: usize, callback: F, ) -> Result @@ -159,7 +165,7 @@ where callback(&mut map); info!("Starting gbt algorithm for {} elements...", map.len()); - let result = gbt::gbt(&mut map, max_uid); + let result = gbt::gbt(&mut map, &accelerations, max_uid); info!("Finished gbt algorithm for {} elements...", map.len()); debug!( diff --git a/backend/rust-gbt/src/thread_acceleration.rs b/backend/rust-gbt/src/thread_acceleration.rs new file mode 100644 index 000000000..618cac3db --- /dev/null +++ b/backend/rust-gbt/src/thread_acceleration.rs @@ -0,0 +1,8 @@ +use napi_derive::napi; + +#[derive(Debug)] +#[napi(object)] +pub struct ThreadAcceleration { + pub uid: u32, + pub delta: f64, // fee delta +} diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index ab700c466..658b1a6c2 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -23,8 +23,8 @@ "USER_AGENT": "__MEMPOOL_USER_AGENT__", "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__", "INDEXING_BLOCKS_AMOUNT": 14, - "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__", - "POOLS_JSON_URL": "__POOLS_JSON_URL__", + "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", + "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", "AUDIT": true, "ADVANCED_GBT_AUDIT": true, "ADVANCED_GBT_MEMPOOL": true, @@ -33,7 +33,8 @@ "MAX_BLOCKS_BULK_QUERY": 999, "DISK_CACHE_BLOCK_INTERVAL": 999, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, - "ALLOW_UNREACHABLE": true + "ALLOW_UNREACHABLE": true, + "PRICE_UPDATES_PER_HOUR": 1 }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", @@ -92,10 +93,6 @@ "USERNAME": "__SOCKS5PROXY_USERNAME__", "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, - "PRICE_DATA_SERVER": { - "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", - "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" - }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__", "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__", @@ -129,6 +126,10 @@ "AUDIT_START_HEIGHT": 774000, "SERVERS": [] }, + "MEMPOOL_SERVICES": { + "API": "", + "ACCELERATIONS": false + }, "REDIS": { "ENABLED": false, "UNIX_SOCKET_PATH": "/tmp/redis.sock" diff --git a/backend/src/__tests__/api/difficulty-adjustment.test.ts b/backend/src/__tests__/api/difficulty-adjustment.test.ts index da1ac0d2c..c3e8e1a88 100644 --- a/backend/src/__tests__/api/difficulty-adjustment.test.ts +++ b/backend/src/__tests__/api/difficulty-adjustment.test.ts @@ -1,4 +1,8 @@ -import { calcDifficultyAdjustment, DifficultyAdjustment } from '../../api/difficulty-adjustment'; +import { + calcBitsDifference, + calcDifficultyAdjustment, + DifficultyAdjustment, +} from '../../api/difficulty-adjustment'; describe('Mempool Difficulty Adjustment', () => { test('should calculate Difficulty Adjustments properly', () => { @@ -86,4 +90,46 @@ describe('Mempool Difficulty Adjustment', () => { expect(result).toStrictEqual(vector[1]); } }); + + test('should calculate Difficulty change from bits fields of two blocks', () => { + // Check same exponent + check min max for output + expect(calcBitsDifference(0x1d000200, 0x1d000100)).toEqual(100); + expect(calcBitsDifference(0x1d000400, 0x1d000100)).toEqual(300); + expect(calcBitsDifference(0x1d000800, 0x1d000100)).toEqual(300); // Actually 700 + expect(calcBitsDifference(0x1d000100, 0x1d000200)).toEqual(-50); + expect(calcBitsDifference(0x1d000100, 0x1d000400)).toEqual(-75); + expect(calcBitsDifference(0x1d000100, 0x1d000800)).toEqual(-75); // Actually -87.5 + // Check new higher exponent + expect(calcBitsDifference(0x1c000200, 0x1d000001)).toEqual(100); + expect(calcBitsDifference(0x1c000400, 0x1d000001)).toEqual(300); + expect(calcBitsDifference(0x1c000800, 0x1d000001)).toEqual(300); + expect(calcBitsDifference(0x1c000100, 0x1d000002)).toEqual(-50); + expect(calcBitsDifference(0x1c000100, 0x1d000004)).toEqual(-75); + expect(calcBitsDifference(0x1c000100, 0x1d000008)).toEqual(-75); + // Check new lower exponent + expect(calcBitsDifference(0x1d000002, 0x1c000100)).toEqual(100); + expect(calcBitsDifference(0x1d000004, 0x1c000100)).toEqual(300); + expect(calcBitsDifference(0x1d000008, 0x1c000100)).toEqual(300); + expect(calcBitsDifference(0x1d000001, 0x1c000200)).toEqual(-50); + expect(calcBitsDifference(0x1d000001, 0x1c000400)).toEqual(-75); + expect(calcBitsDifference(0x1d000001, 0x1c000800)).toEqual(-75); + // Check error when exponents are too far apart + expect(() => calcBitsDifference(0x1d000001, 0x1a000800)).toThrow( + /Impossible exponent difference/ + ); + // Check invalid inputs + expect(() => calcBitsDifference(0x7f000001, 0x1a000800)).toThrow( + /Invalid bits/ + ); + expect(() => calcBitsDifference(0, 0x1a000800)).toThrow(/Invalid bits/); + expect(() => calcBitsDifference(100.2783, 0x1a000800)).toThrow( + /Invalid bits/ + ); + expect(() => calcBitsDifference(0x00800000, 0x1a000800)).toThrow( + /Invalid bits/ + ); + expect(() => calcBitsDifference(0x1c000000, 0x1a000800)).toThrow( + /Invalid bits/ + ); + }); }); diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 0c06b03e1..23ad0e4a6 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -47,6 +47,7 @@ describe('Mempool Backend Config', () => { DISK_CACHE_BLOCK_INTERVAL: 6, MAX_PUSH_TX_SIZE_WEIGHT: 400000, ALLOW_UNREACHABLE: true, + PRICE_UPDATES_PER_HOUR: 1, }); expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); @@ -101,11 +102,6 @@ describe('Mempool Backend Config', () => { PASSWORD: '' }); - expect(config.PRICE_DATA_SERVER).toStrictEqual({ - TOR_URL: 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', - CLEARNET_URL: 'https://price.bisq.wiz.biz/getAllMarketPrices' - }); - expect(config.EXTERNAL_DATA_SERVER).toStrictEqual({ MEMPOOL_API: 'https://mempool.space/api/v1', MEMPOOL_ONION: 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1', @@ -129,6 +125,11 @@ describe('Mempool Backend Config', () => { SERVERS: [] }); + expect(config.MEMPOOL_SERVICES).toStrictEqual({ + API: "", + ACCELERATIONS: false, + }); + expect(config.REDIS).toStrictEqual({ ENABLED: false, UNIX_SOCKET_PATH: '' @@ -163,10 +164,10 @@ describe('Mempool Backend Config', () => { expect(config.SOCKS5PROXY).toStrictEqual(fixture.SOCKS5PROXY); - expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER); - expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER); + expect(config.MEMPOOL_SERVICES).toStrictEqual(fixture.MEMPOOL_SERVICES); + expect(config.REDIS).toStrictEqual(fixture.REDIS); }); }); @@ -181,12 +182,12 @@ describe('Mempool Backend Config', () => { // We have a few cases where we can't follow the pattern if (root === 'MEMPOOL' && key === 'HTTP_PORT') { console.log('skipping check for MEMPOOL_HTTP_PORT'); - return; + continue; } switch (typeof value) { case 'object': { if (Array.isArray(value)) { - return; + continue; } else { parseJson(value, key); } diff --git a/backend/src/__tests__/gbt/gbt-tests.ts b/backend/src/__tests__/gbt/gbt-tests.ts index 0651faac4..8a3995f71 100644 --- a/backend/src/__tests__/gbt/gbt-tests.ts +++ b/backend/src/__tests__/gbt/gbt-tests.ts @@ -1,5 +1,5 @@ import fs from 'fs'; -import { GbtGenerator, ThreadTransaction } from '../../../rust-gbt'; +import { GbtGenerator, ThreadTransaction } from 'rust-gbt'; import path from 'path'; const baseline = require('./test-data/target-template.json'); @@ -15,7 +15,7 @@ describe('Rust GBT', () => { test('should produce the same template as getBlockTemplate from Bitcoin Core', async () => { const rustGbt = new GbtGenerator(); const { mempool, maxUid } = mempoolFromArrayBuffer(vectorBuffer.buffer); - const result = await rustGbt.make(mempool, maxUid); + const result = await rustGbt.make(mempool, [], maxUid); const blocks: [string, number][][] = result.blocks.map(block => { return block.map(uid => [vectorUidMap.get(uid) || 'missing', uid]); diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index a909fc2b6..e78b2796f 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -6,16 +6,17 @@ import rbfCache from './rbf-cache'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners class Audit { - auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }) - : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], score: number, similarity: number } { + auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, useAccelerations: boolean = false) + : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } { if (!projectedBlocks?.[0]?.transactionIds || !mempool) { - return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], score: 0, similarity: 1 }; + return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 0, similarity: 1 }; } const matches: string[] = []; // present in both mined block and template const added: string[] = []; // present in mined block, not in template const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN const rbf: string[] = []; // either missing or present, and either part of a full-rbf replacement, or a conflict with the mined block + const accelerated: string[] = []; // prioritized by the mempool accelerator const isCensored = {}; // missing, without excuse const isDisplaced = {}; let displacedWeight = 0; @@ -28,6 +29,9 @@ class Audit { const now = Math.round((Date.now() / 1000)); for (const tx of transactions) { inBlock[tx.txid] = tx; + if (mempool[tx.txid] && mempool[tx.txid].acceleration) { + accelerated.push(tx.txid); + } } // coinbase is always expected if (transactions[0]) { @@ -149,6 +153,7 @@ class Audit { fresh, sigop: [], fullrbf: rbf, + accelerated, score, similarity, }; diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index 7f4a5e53a..f14c5525d 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -3,7 +3,8 @@ import { IEsploraApi } from './esplora-api.interface'; export interface AbstractBitcoinApi { $getRawMempool(): Promise; $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise; - $getMempoolTransactions(lastTxid: string); + $getMempoolTransactions(txids: string[]): Promise; + $getAllMempoolTransactions(lastTxid: string); $getTransactionHex(txId: string): Promise; $getBlockHeightTip(): Promise; $getBlockHashTip(): Promise; diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 132cda91a..b315ed0f7 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -60,8 +60,13 @@ class BitcoinApi implements AbstractBitcoinApi { }); } - $getMempoolTransactions(lastTxid: string): Promise { - return Promise.resolve([]); + $getMempoolTransactions(txids: string[]): Promise { + throw new Error('Method getMempoolTransactions not supported by the Bitcoin RPC API.'); + } + + $getAllMempoolTransactions(lastTxid: string): Promise { + throw new Error('Method getAllMempoolTransactions not supported by the Bitcoin RPC API.'); + } async $getTransactionHex(txId: string): Promise { diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index d2a77c995..90a31ecae 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -6,7 +6,7 @@ import websocketHandler from '../websocket-handler'; import mempool from '../mempool'; import feeApi from '../fee-api'; import mempoolBlocks from '../mempool-blocks'; -import bitcoinApi, { bitcoinCoreApi } from './bitcoin-api-factory'; +import bitcoinApi from './bitcoin-api-factory'; import { Common } from '../common'; import backendInfo from '../backend-info'; import transactionUtils from '../transaction-utils'; @@ -214,6 +214,7 @@ class BitcoinRoutes { effectiveFeePerVsize: tx.effectiveFeePerVsize || null, sigops: tx.sigops, adjustedVsize: tx.adjustedVsize, + acceleration: tx.acceleration }); return; } @@ -483,7 +484,7 @@ class BitcoinRoutes { returnBlocks.push(localBlock); nextHash = localBlock.previousblockhash; } else { - const block = await bitcoinCoreApi.$getBlock(nextHash); + const block = await bitcoinApi.$getBlock(nextHash); returnBlocks.push(block); nextHash = block.previousblockhash; } diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index ff10751e0..77c6d80fc 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -61,6 +61,25 @@ class ElectrsApi implements AbstractBitcoinApi { }); } + $postWrapper(url, body, responseType = 'json', params: any = undefined): Promise { + return axiosConnection.post(url, body, { ...this.activeAxiosConfig, responseType: responseType, params }) + .then((response) => response.data) + .catch((e) => { + if (e?.code === 'ECONNREFUSED') { + this.fallbackToTcpSocket(); + // Retry immediately + return axiosConnection.post(url, body, this.activeAxiosConfig) + .then((response) => response.data) + .catch((e) => { + logger.warn(`Cannot query esplora through the unix socket nor the tcp socket. Exception ${e}`); + throw e; + }); + } else { + throw e; + } + }); + } + $getRawMempool(): Promise { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/mempool/txids'); } @@ -69,7 +88,11 @@ class ElectrsApi implements AbstractBitcoinApi { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId); } - async $getMempoolTransactions(lastSeenTxid?: string): Promise { + async $getMempoolTransactions(txids: string[]): Promise { + return this.$postWrapper(config.ESPLORA.REST_API_URL + '/mempool/txs', txids, 'json'); + } + + async $getAllMempoolTransactions(lastSeenTxid?: string): Promise { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/mempool/txs' + (lastSeenTxid ? '/' + lastSeenTxid : '')); } diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index f86bc53e9..33196b052 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -28,12 +28,13 @@ import chainTips from './chain-tips'; import websocketHandler from './websocket-handler'; import redisCache from './redis-cache'; import rbfCache from './rbf-cache'; +import { calcBitsDifference } from './difficulty-adjustment'; class Blocks { private blocks: BlockExtended[] = []; private blockSummaries: BlockSummary[] = []; private currentBlockHeight = 0; - private currentDifficulty = 0; + private currentBits = 0; private lastDifficultyAdjustmentTime = 0; private previousDifficultyRetarget = 0; private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = []; @@ -419,8 +420,8 @@ class Blocks { let newlyIndexed = 0; let totalIndexed = indexedBlockSummariesHashesArray.length; let indexedThisRun = 0; - let timer = new Date().getTime() / 1000; - const startedAt = new Date().getTime() / 1000; + let timer = Date.now() / 1000; + const startedAt = Date.now() / 1000; for (const block of indexedBlocks) { if (indexedBlockSummariesHashes[block.hash] === true) { @@ -428,17 +429,24 @@ class Blocks { } // Logging - const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); + const elapsedSeconds = (Date.now() / 1000) - timer; if (elapsedSeconds > 5) { - const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = indexedThisRun / elapsedSeconds; const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100; - logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining); - timer = new Date().getTime() / 1000; + logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`, logger.tags.mining); + timer = Date.now() / 1000; indexedThisRun = 0; } - await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary + + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendTransaction(tx)); + const cpfpSummary = await this.$indexCPFP(block.hash, block.height, txs); + await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary + } else { + await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary + } // Logging indexedThisRun++; @@ -477,18 +485,18 @@ class Blocks { // Logging let count = 0; let countThisRun = 0; - let timer = new Date().getTime() / 1000; - const startedAt = new Date().getTime() / 1000; + let timer = Date.now() / 1000; + const startedAt = Date.now() / 1000; for (const height of unindexedBlockHeights) { // Logging const hash = await bitcoinApi.$getBlockHash(height); - const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); + const elapsedSeconds = (Date.now() / 1000) - timer; if (elapsedSeconds > 5) { - const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = (countThisRun / elapsedSeconds); + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = countThisRun / elapsedSeconds; const progress = Math.round(count / unindexedBlockHeights.length * 10000) / 100; - logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor} seconds`); - timer = new Date().getTime() / 1000; + logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`); + timer = Date.now() / 1000; countThisRun = 0; } @@ -567,8 +575,8 @@ class Blocks { let totalIndexed = await blocksRepository.$blockCountBetweenHeight(currentBlockHeight, lastBlockToIndex); let indexedThisRun = 0; let newlyIndexed = 0; - const startedAt = new Date().getTime() / 1000; - let timer = new Date().getTime() / 1000; + const startedAt = Date.now() / 1000; + let timer = Date.now() / 1000; while (currentBlockHeight >= lastBlockToIndex) { const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1); @@ -588,18 +596,18 @@ class Blocks { } ++indexedThisRun; ++totalIndexed; - const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); + const elapsedSeconds = (Date.now() / 1000) - timer; if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) { - const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = indexedThisRun / elapsedSeconds; const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100; - logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining); - timer = new Date().getTime() / 1000; + logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress.toFixed(2)}%) | elapsed: ${runningFor.toFixed(2)} seconds`, logger.tags.mining); + timer = Date.now() / 1000; indexedThisRun = 0; loadingIndicators.setProgress('block-indexing', progress, false); } const blockHash = await bitcoinApi.$getBlockHash(blockHeight); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, null, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -656,17 +664,17 @@ class Blocks { const heightDiff = blockHeightTip % 2016; const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff); this.updateTimerProgress(timer, 'got block hash for initial difficulty adjustment'); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); this.updateTimerProgress(timer, 'got block for initial difficulty adjustment'); this.lastDifficultyAdjustmentTime = block.timestamp; - this.currentDifficulty = block.difficulty; + this.currentBits = block.bits; if (blockHeightTip >= 2016) { const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016); this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment'); - const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash); + const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash); this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment'); - this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100; + this.previousDifficultyRetarget = calcBitsDifference(previousPeriodBlock.bits, block.bits); logger.debug(`Initial difficulty adjustment data set.`); } } else { @@ -694,7 +702,7 @@ class Blocks { // fill in missing transaction fee data from verboseBlock for (let i = 0; i < transactions.length; i++) { if (!transactions[i].fee && transactions[i].txid === verboseBlock.tx[i].txid) { - transactions[i].fee = verboseBlock.tx[i].fee * 100_000_000; + transactions[i].fee = (verboseBlock.tx[i].fee * 100_000_000) || 0; } } @@ -779,14 +787,18 @@ class Blocks { time: block.timestamp, height: block.height, difficulty: block.difficulty, - adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise + adjustment: Math.round( + // calcBitsDifference returns +- percentage, +100 returns to positive, /100 returns to ratio. + // Instead of actually doing /100, just reduce the multiplier. + (calcBitsDifference(this.currentBits, block.bits) + 100) * 10000 + ) / 1000000, // Remove float point noise }); this.updateTimerProgress(timer, `saved difficulty adjustment for ${this.currentBlockHeight}`); } - this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100; + this.previousDifficultyRetarget = calcBitsDifference(this.currentBits, block.bits); this.lastDifficultyAdjustmentTime = block.timestamp; - this.currentDifficulty = block.difficulty; + this.currentBits = block.bits; } // wait for pending async callbacks to finish @@ -862,7 +874,7 @@ class Blocks { } const blockHash = await bitcoinApi.$getBlockHash(height); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -874,7 +886,7 @@ class Blocks { } public async $indexStaleBlock(hash: string): Promise { - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash); const transactions = await this.$getTransactionsExtended(hash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -899,7 +911,7 @@ class Blocks { } // Bitcoin network, add our custom data on top - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash); if (block.stale) { return await this.$indexStaleBlock(hash); } else { @@ -934,7 +946,7 @@ class Blocks { transactions: cpfpSummary.transactions.map(tx => { return { txid: tx.txid, - fee: tx.fee, + fee: tx.fee || 0, vsize: tx.vsize, value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0)), rate: tx.effectiveFeePerVsize @@ -942,10 +954,15 @@ class Blocks { }), }; } else { - // Call Core RPC - const block = await bitcoinClient.getBlock(hash, 2); - summary = this.summarizeBlock(block); - height = block.height; + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx)); + summary = this.summarizeBlockTransactions(hash, txs); + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(hash, 2); + summary = this.summarizeBlock(block); + height = block.height; + } } if (height == null) { const block = await bitcoinApi.$getBlock(hash); @@ -1068,8 +1085,17 @@ class Blocks { if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) { cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); if (cleanBlock.fee_amt_percentiles === null) { - const block = await bitcoinClient.getBlock(cleanBlock.hash, 2); - const summary = this.summarizeBlock(block); + + let summary; + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(cleanBlock.hash)).map(tx => transactionUtils.extendTransaction(tx)); + summary = this.summarizeBlockTransactions(cleanBlock.hash, txs); + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(cleanBlock.hash, 2); + summary = this.summarizeBlock(block); + } + await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions); cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); } @@ -1129,19 +1155,29 @@ class Blocks { return this.currentBlockHeight; } - public async $indexCPFP(hash: string, height: number): Promise { - const block = await bitcoinClient.getBlock(hash, 2); - const transactions = block.tx.map(tx => { - tx.fee *= 100_000_000; - return tx; - }); + public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise { + let transactions = txs; + if (!transactions) { + if (config.MEMPOOL.BACKEND === 'esplora') { + transactions = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx)); + } + if (!transactions) { + const block = await bitcoinClient.getBlock(hash, 2); + transactions = block.tx.map(tx => { + tx.fee *= 100_000_000; + return tx; + }); + } + } - const summary = Common.calculateCpfp(height, transactions); + const summary = Common.calculateCpfp(height, transactions as TransactionExtended[]); await this.$saveCpfp(hash, height, summary); const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions); await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats); + + return summary; } public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise { diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index cd9da3d2a..64bac2036 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -108,9 +108,10 @@ export class Common { static stripTransaction(tx: TransactionExtended): TransactionStripped { return { txid: tx.txid, - fee: tx.fee, + fee: tx.fee || 0, vsize: tx.weight / 4, value: tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0), + acc: tx.acceleration || undefined, rate: tx.effectiveFeePerVsize, }; } @@ -460,7 +461,7 @@ export class Common { }; } - static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string }[]): EffectiveFeeStats { + static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string, acceleration?: boolean }[]): EffectiveFeeStats { const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate); let weightCount = 0; diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 7c7608aff..b7dc39493 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 64; + private static currentVersion = 65; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -548,6 +548,11 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `nodes` ADD features text NULL'); await this.updateToSchemaVersion(64); } + + if (databaseSchemaVersion < 65 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `blocks_audits` ADD accelerated_txs JSON DEFAULT "[]"'); + await this.updateToSchemaVersion(65); + } } /** diff --git a/backend/src/api/difficulty-adjustment.ts b/backend/src/api/difficulty-adjustment.ts index 1f37d8be9..23d0c33de 100644 --- a/backend/src/api/difficulty-adjustment.ts +++ b/backend/src/api/difficulty-adjustment.ts @@ -16,6 +16,68 @@ export interface DifficultyAdjustment { expectedBlocks: number; // Block count } +/** + * Calculate the difficulty increase/decrease by using the `bits` integer contained in two + * block headers. + * + * Warning: Only compare `bits` from blocks in two adjacent difficulty periods. This code + * assumes the maximum difference is x4 or /4 (as per the protocol) and will throw an + * error if an exponent difference of 2 or more is seen. + * + * @param {number} oldBits The 32 bit `bits` integer from a block header. + * @param {number} newBits The 32 bit `bits` integer from a block header in the next difficulty period. + * @returns {number} A floating point decimal of the difficulty change from old to new. + * (ie. 21.3 means 21.3% increase in difficulty, -21.3 is a 21.3% decrease in difficulty) + */ +export function calcBitsDifference(oldBits: number, newBits: number): number { + // Must be + // - integer + // - highest exponent is 0x1f, so max value (as integer) is 0x1f0000ff + // - min value is 1 (exponent = 0) + // - highest bit of the number-part is +- sign, it must not be 1 + const verifyBits = (bits: number): void => { + if ( + Math.floor(bits) !== bits || + bits > 0x1f0000ff || + bits < 1 || + (bits & 0x00800000) !== 0 || + (bits & 0x007fffff) === 0 + ) { + throw new Error('Invalid bits'); + } + }; + verifyBits(oldBits); + verifyBits(newBits); + + // No need to mask exponents because we checked the bounds above + const oldExp = oldBits >> 24; + const newExp = newBits >> 24; + const oldNum = oldBits & 0x007fffff; + const newNum = newBits & 0x007fffff; + // The diff can only possibly be 1, 0, -1 + // (because maximum difficulty change is x4 or /4 (2 bits up or down)) + let result: number; + switch (newExp - oldExp) { + // New less than old, target lowered, difficulty increased + case -1: + result = ((oldNum << 8) * 100) / newNum - 100; + break; + // Same exponent, compare numbers as is. + case 0: + result = (oldNum * 100) / newNum - 100; + break; + // Old less than new, target raised, difficulty decreased + case 1: + result = (oldNum * 100) / (newNum << 8) - 100; + break; + default: + throw new Error('Impossible exponent difference'); + } + + // Min/Max values + return result > 300 ? 300 : result < -75 ? -75 : result; +} + export function calcDifficultyAdjustment( DATime: number, nowSeconds: number, diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 08508310d..1de4bbee7 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,10 +1,11 @@ -import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction } from '../../rust-gbt'; +import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from 'rust-gbt'; import logger from '../logger'; -import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats } from '../mempool.interfaces'; +import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag } from '../mempool.interfaces'; import { Common, OnlineFeeStatsCalculator } from './common'; import config from '../config'; import { Worker } from 'worker_threads'; import path from 'path'; +import mempool from './mempool'; const MAX_UINT32 = Math.pow(2, 32) - 1; @@ -170,7 +171,7 @@ class MempoolBlocks { for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) { let added: TransactionStripped[] = []; let removed: string[] = []; - const changed: { txid: string, rate: number | undefined }[] = []; + const changed: { txid: string, rate: number | undefined, acc: boolean | undefined }[] = []; if (mempoolBlocks[i] && !prevBlocks[i]) { added = mempoolBlocks[i].transactions; } else if (!mempoolBlocks[i] && prevBlocks[i]) { @@ -192,8 +193,8 @@ class MempoolBlocks { mempoolBlocks[i].transactions.forEach(tx => { if (!prevIds[tx.txid]) { added.push(tx); - } else if (tx.rate !== prevIds[tx.txid].rate) { - changed.push({ txid: tx.txid, rate: tx.rate }); + } else if (tx.rate !== prevIds[tx.txid].rate || tx.acc !== prevIds[tx.txid].acc) { + changed.push({ txid: tx.txid, rate: tx.rate, acc: tx.acc }); } }); } @@ -206,14 +207,19 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise { + public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { const start = Date.now(); // reset mempool short ids - this.resetUids(); - for (const tx of Object.values(newMempool)) { - this.setUid(tx); + if (saveResults) { + this.resetUids(); } + // set missing short ids + for (const tx of Object.values(newMempool)) { + this.setUid(tx, !saveResults); + } + + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread @@ -222,7 +228,7 @@ class MempoolBlocks { if (entry.uid !== null && entry.uid !== undefined) { const stripped = { uid: entry.uid, - fee: entry.fee, + fee: entry.fee + (useAccelerations && (!accelerationPool || accelerations[entry.txid]?.pools?.includes(accelerationPool)) ? (accelerations[entry.txid]?.feeDelta || 0) : 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -262,7 +268,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, accelerationPool, saveResults); logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); @@ -273,25 +279,29 @@ class MempoolBlocks { return this.mempoolBlocks; } - public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], saveResults: boolean = false): Promise { + public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise { if (!this.txSelectionWorker) { // need to reset the worker - await this.$makeBlockTemplates(newMempool, saveResults); + await this.$makeBlockTemplates(newMempool, saveResults, useAccelerations); return; } const start = Date.now(); - for (const tx of Object.values(added)) { + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; + const addedAndChanged: MempoolTransactionExtended[] = useAccelerations ? accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added) : added; + + for (const tx of addedAndChanged) { this.setUid(tx, true); } - const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[]; + const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => uid != null) as number[]; + // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread - const addedStripped: CompactThreadTransaction[] = added.filter(entry => (entry.uid !== null && entry.uid !== undefined)).map(entry => { + const addedStripped: CompactThreadTransaction[] = addedAndChanged.filter(entry => entry.uid != null).map(entry => { return { uid: entry.uid || 0, - fee: entry.fee, + fee: entry.fee + (useAccelerations ? (accelerations[entry.txid]?.feeDelta || 0) : 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -318,7 +328,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), saveResults); + this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, null, saveResults); logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`); } catch (e) { logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); @@ -330,7 +340,7 @@ class MempoolBlocks { this.rustGbtGenerator = new GbtGenerator(); } - private async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise { + public async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { const start = Date.now(); // reset mempool short ids @@ -346,16 +356,25 @@ class MempoolBlocks { tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[]; } + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; + const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]); + const convertedAccelerations = acceleratedList.map(acc => { + return { + uid: this.getUid(newMempool[acc.txid]), + delta: acc.feeDelta, + }; + }); + // run the block construction algorithm in a separate thread, and wait for a result const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator(); try { const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids( - await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], this.nextUid), + await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid), ); if (saveResults) { this.rustInitialized = true; } - const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, saveResults); logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); return processed; } catch (e) { @@ -367,20 +386,20 @@ class MempoolBlocks { return this.mempoolBlocks; } - public async $oneOffRustBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }): Promise { - return this.$rustMakeBlockTemplates(newMempool, false); + public async $oneOffRustBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, useAccelerations: boolean, accelerationPool?: number): Promise { + return this.$rustMakeBlockTemplates(newMempool, false, useAccelerations, accelerationPool); } - public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[]): Promise { + public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], useAccelerations: boolean, accelerationPool?: number): Promise { // GBT optimization requires that uids never get too sparse // as a sanity check, we should also explicitly prevent uint32 uid overflow if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * mempoolSize), MAX_UINT32)) { this.resetRustGbt(); } + if (!this.rustInitialized) { // need to reset the worker - await this.$rustMakeBlockTemplates(newMempool, true); - return; + return this.$rustMakeBlockTemplates(newMempool, true, useAccelerations, accelerationPool); } const start = Date.now(); @@ -394,12 +413,22 @@ class MempoolBlocks { } const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[]; + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; + const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]); + const convertedAccelerations = acceleratedList.map(acc => { + return { + uid: this.getUid(newMempool[acc.txid]), + delta: acc.feeDelta, + }; + }); + // run the block construction algorithm in a separate thread, and wait for a result try { const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids( await this.rustGbtGenerator.update( added as RustThreadTransaction[], removedUids, + convertedAccelerations as RustThreadAcceleration[], this.nextUid, ), ); @@ -407,17 +436,19 @@ class MempoolBlocks { if (mempoolSize !== resultMempoolSize) { throw new Error('GBT returned wrong number of transactions, cache is probably out of sync'); } else { - this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, true); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, true); + this.removeUids(removedUids); + logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); + return processed; } - this.removeUids(removedUids); - logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); } catch (e) { logger.err('RUST updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); this.resetRustGbt(); + return this.mempoolBlocks; } } - private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], saveResults): MempoolBlockWithTransactions[] { + private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] { for (const [txid, rate] of rates) { if (txid in mempool) { mempool[txid].effectiveFeePerVsize = rate; @@ -468,6 +499,8 @@ class MempoolBlocks { } } + const isAccelerated : { [txid: string]: boolean } = {}; + const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2; // update this thread's mempool with the results let mempoolTx: MempoolTransactionExtended; @@ -496,6 +529,17 @@ class MempoolBlocks { mempoolTx.cpfpChecked = true; } + const acceleration = accelerations[txid]; + if (isAccelerated[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) { + mempoolTx.acceleration = true; + for (const ancestor of mempoolTx.ancestors || []) { + mempool[ancestor.txid].acceleration = true; + isAccelerated[ancestor.txid] = true; + } + } else { + delete mempoolTx.acceleration; + } + // online calculation of stack-of-blocks fee stats if (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) { feeStatsCalculator.processNext(mempoolTx); @@ -532,7 +576,7 @@ class MempoolBlocks { private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions { if (!feeStats) { - feeStats = Common.calcEffectiveFeeStatistics(transactions); + feeStats = Common.calcEffectiveFeeStatistics(transactions.filter(tx => !tx.acceleration)); } return { blockSize: totalSize, diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index d5214de5d..73260dc9e 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -1,5 +1,5 @@ import config from '../config'; -import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond } from '../mempool.interfaces'; import logger from '../logger'; import { Common } from './common'; @@ -9,6 +9,7 @@ import loadingIndicators from './loading-indicators'; import bitcoinClient from './bitcoin/bitcoin-client'; import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; import rbfCache from './rbf-cache'; +import accelerationApi, { Acceleration } from './services/acceleration'; import redisCache from './redis-cache'; class Mempool { @@ -19,9 +20,11 @@ class Mempool { private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0, maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 }; private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[], - deletedTransactions: MempoolTransactionExtended[]) => void) | undefined; + deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void) | undefined; private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[], - deletedTransactions: MempoolTransactionExtended[]) => Promise) | undefined; + deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise) | undefined; + + private accelerations: { [txId: string]: Acceleration } = {}; private txPerSecondArray: number[] = []; private txPerSecond: number = 0; @@ -66,12 +69,12 @@ class Mempool { } public setMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, - newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]) => void): void { + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void): void { this.mempoolChangedCallback = fn; } public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, mempoolSize: number, - newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]) => Promise): void { + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise): void { this.$asyncMempoolChangedCallback = fn; } @@ -107,10 +110,10 @@ class Mempool { logger.debug(`Finished migrating cache transactions in ${((Date.now() - redisTimer) / 1000).toFixed(2)} seconds`); } if (this.mempoolChangedCallback) { - this.mempoolChangedCallback(this.mempoolCache, [], []); + this.mempoolChangedCallback(this.mempoolCache, [], [], []); } if (this.$asyncMempoolChangedCallback) { - await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], []); + await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], [], []); } this.addToSpendMap(Object.values(this.mempoolCache)); } @@ -123,7 +126,7 @@ class Mempool { loadingIndicators.setProgress('mempool', count / expectedCount * 100); while (!done) { try { - const result = await bitcoinApi.$getMempoolTransactions(last_txid); + const result = await bitcoinApi.$getAllMempoolTransactions(last_txid); if (result) { for (const tx of result) { const extendedTransaction = transactionUtils.extendMempoolTransaction(tx); @@ -182,7 +185,7 @@ class Mempool { return txTimes; } - public async $updateMempool(transactions: string[]): Promise { + public async $updateMempool(transactions: string[], pollRate: number): Promise { logger.debug(`Updating mempool...`); // warn if this run stalls the main loop for more than 2 minutes @@ -231,34 +234,40 @@ class Mempool { } if (!loaded) { - for (const txid of transactions) { - if (!this.mempoolCache[txid]) { - try { - const transaction = await transactionUtils.$getMempoolTransactionExtended(txid, false, false, false); - this.updateTimerProgress(timer, 'fetched new transaction'); - this.mempoolCache[txid] = transaction; - if (this.inSync) { - this.txPerSecondArray.push(new Date().getTime()); - this.vBytesPerSecondArray.push({ - unixTime: new Date().getTime(), - vSize: transaction.vsize, - }); - } - hasChange = true; - newTransactions.push(transaction); + const remainingTxids = transactions.filter(txid => !this.mempoolCache[txid]); + const sliceLength = 10000; + for (let i = 0; i < Math.ceil(remainingTxids.length / sliceLength); i++) { + const slice = remainingTxids.slice(i * sliceLength, (i + 1) * sliceLength); + const txs = await transactionUtils.$getMempoolTransactionsExtended(slice, false, false, false); + logger.debug(`fetched ${txs.length} transactions`); + this.updateTimerProgress(timer, 'fetched new transactions'); - if (config.REDIS.ENABLED) { - await redisCache.$addTransaction(transaction); - } - } catch (e: any) { - if (config.MEMPOOL.BACKEND === 'esplora' && e.response?.status === 404) { - this.missingTxCount++; - } - logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e)); + for (const transaction of txs) { + this.mempoolCache[transaction.txid] = transaction; + if (this.inSync) { + this.txPerSecondArray.push(new Date().getTime()); + this.vBytesPerSecondArray.push({ + unixTime: new Date().getTime(), + vSize: transaction.vsize, + }); + } + hasChange = true; + newTransactions.push(transaction); + + if (config.REDIS.ENABLED) { + await redisCache.$addTransaction(transaction); } } - if (Date.now() - intervalTimer > 5_000) { + if (txs.length < slice.length) { + const missing = slice.length - txs.length; + if (config.MEMPOOL.BACKEND === 'esplora') { + this.missingTxCount += missing; + } + logger.debug(`Error finding ${missing} transactions in the mempool: `); + } + + if (Date.now() - intervalTimer > Math.max(pollRate * 2, 5_000)) { if (this.inSync) { // Break and restart mempool loop if we spend too much time processing // new transactions that may lead to falling behind on block height @@ -270,7 +279,7 @@ class Mempool { if (Math.floor(progress) < 100) { loadingIndicators.setProgress('mempool', progress); } - intervalTimer = Date.now() + intervalTimer = Date.now(); } } } @@ -321,14 +330,19 @@ class Mempool { const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx)); this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6); + const accelerationDelta = await this.$updateAccelerations(); + if (accelerationDelta.length) { + hasChange = true; + } + this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize); if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) { - this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions); + this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions, accelerationDelta); } if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) { this.updateTimerProgress(timer, 'running async mempool callback'); - await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions); + await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions, accelerationDelta); this.updateTimerProgress(timer, 'completed async mempool callback'); } @@ -352,6 +366,70 @@ class Mempool { this.clearTimer(timer); } + public getAccelerations(): { [txid: string]: Acceleration } { + return this.accelerations; + } + + public async $updateAccelerations(): Promise { + if (!config.MEMPOOL_SERVICES.ACCELERATIONS) { + return []; + } + + try { + const newAccelerations = await accelerationApi.$fetchAccelerations(); + + const changed: string[] = []; + + const newAccelerationMap: { [txid: string]: Acceleration } = {}; + for (const acceleration of newAccelerations) { + newAccelerationMap[acceleration.txid] = acceleration; + if (this.accelerations[acceleration.txid] == null) { + // new acceleration + changed.push(acceleration.txid); + } else { + if (this.accelerations[acceleration.txid].feeDelta !== acceleration.feeDelta) { + // feeDelta changed + changed.push(acceleration.txid); + } else if (this.accelerations[acceleration.txid].pools?.length) { + let poolsChanged = false; + const pools = new Set(); + this.accelerations[acceleration.txid].pools.forEach(pool => { + pools.add(pool); + }); + acceleration.pools.forEach(pool => { + if (!pools.has(pool)) { + poolsChanged = true; + } else { + pools.delete(pool); + } + }); + if (pools.size > 0) { + poolsChanged = true; + } + if (poolsChanged) { + // pools changed + changed.push(acceleration.txid); + } + } + } + } + + for (const oldTxid of Object.keys(this.accelerations)) { + if (!newAccelerationMap[oldTxid]) { + // removed + changed.push(oldTxid); + } + } + + this.accelerations = newAccelerationMap; + + return changed; + } catch (e: any) { + logger.debug(`Failed to update accelerations: ` + (e instanceof Error ? e.message : e)); + return []; + } + } + private startTimer() { const state: any = { start: Date.now(), diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index e190492b8..6e87d70b8 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -11,7 +11,7 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust import config from '../../config'; import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository'; import PricesRepository from '../../repositories/PricesRepository'; -import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory'; +import bitcoinApi from '../bitcoin/bitcoin-api-factory'; import { IEsploraApi } from '../bitcoin/esplora-api.interface'; import database from '../../database'; @@ -107,6 +107,7 @@ class Mining { slug: poolInfo.slug, avgMatchRate: poolInfo.avgMatchRate !== null ? Math.round(100 * poolInfo.avgMatchRate) / 100 : null, avgFeeDelta: poolInfo.avgFeeDelta, + poolUniqueId: poolInfo.poolUniqueId }; poolsStats.push(poolStat); }); @@ -201,7 +202,7 @@ class Mining { try { const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); const genesisTimestamp = genesisBlock.timestamp * 1000; const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps(); @@ -312,7 +313,7 @@ class Mining { const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; try { - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); const genesisTimestamp = genesisBlock.timestamp * 1000; const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); const lastMidnight = this.getDateMidnight(new Date()); @@ -421,8 +422,9 @@ class Mining { } const blocks: any = await BlocksRepository.$getBlocksDifficulty(); - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); let currentDifficulty = genesisBlock.difficulty; + let currentBits = genesisBlock.bits; let totalIndexed = 0; if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) { @@ -436,6 +438,7 @@ class Mining { const oldestConsecutiveBlock = await BlocksRepository.$getOldestConsecutiveBlock(); if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== -1) { + currentBits = oldestConsecutiveBlock.bits; currentDifficulty = oldestConsecutiveBlock.difficulty; } @@ -443,10 +446,11 @@ class Mining { let timer = new Date().getTime() / 1000; for (const block of blocks) { - if (block.difficulty !== currentDifficulty) { + if (block.bits !== currentBits) { if (indexedHeights[block.height] === true) { // Already indexed if (block.height >= oldestConsecutiveBlock.height) { currentDifficulty = block.difficulty; + currentBits = block.bits; } continue; } @@ -464,6 +468,7 @@ class Mining { totalIndexed++; if (block.height >= oldestConsecutiveBlock.height) { currentDifficulty = block.difficulty; + currentBits = block.bits; } } diff --git a/backend/src/api/prices/prices.routes.ts b/backend/src/api/prices/prices.routes.ts new file mode 100644 index 000000000..b46331b73 --- /dev/null +++ b/backend/src/api/prices/prices.routes.ts @@ -0,0 +1,19 @@ +import { Application, Request, Response } from 'express'; +import config from '../../config'; +import pricesUpdater from '../../tasks/price-updater'; + +class PricesRoutes { + public initRoutes(app: Application): void { + app.get(config.MEMPOOL.API_URL_PREFIX + 'prices', this.$getCurrentPrices.bind(this)); + } + + private $getCurrentPrices(req: Request, res: Response): void { + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR).toUTCString()); + + res.json(pricesUpdater.getLatestPrices()); + } +} + +export default new PricesRoutes(); diff --git a/backend/src/api/services/acceleration.ts b/backend/src/api/services/acceleration.ts new file mode 100644 index 000000000..635dc8300 --- /dev/null +++ b/backend/src/api/services/acceleration.ts @@ -0,0 +1,30 @@ +import { query } from '../../utils/axios-query'; +import config from '../../config'; +import { BlockExtended, PoolTag } from '../../mempool.interfaces'; + +export interface Acceleration { + txid: string, + feeDelta: number, + pools: number[], +} + +class AccelerationApi { + public async $fetchAccelerations(): Promise { + if (config.MEMPOOL_SERVICES.ACCELERATIONS) { + const response = await query(`${config.MEMPOOL_SERVICES.API}/accelerator/accelerations`); + return (response as Acceleration[]) || []; + } else { + return []; + } + } + + public isAcceleratedBlock(block: BlockExtended, accelerations: Acceleration[]): boolean { + let anyAccelerated = false; + for (let i = 0; i < accelerations.length && !anyAccelerated; i++) { + anyAccelerated = anyAccelerated || accelerations[i].pools?.includes(block.extras.pool.id); + } + return anyAccelerated; + } +} + +export default new AccelerationApi(); \ No newline at end of file diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index e141a6076..02ee7c055 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -4,6 +4,7 @@ import { Common } from './common'; import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; import * as bitcoinjs from 'bitcoinjs-lib'; import logger from '../logger'; +import config from '../config'; class TransactionUtils { constructor() { } @@ -71,6 +72,24 @@ class TransactionUtils { return (await this.$getTransactionExtended(txId, addPrevouts, lazyPrevouts, forceCore, true)) as MempoolTransactionExtended; } + public async $getMempoolTransactionsExtended(txids: string[], addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise { + if (forceCore || config.MEMPOOL.BACKEND !== 'esplora') { + const results = await Promise.allSettled(txids.map(txid => this.$getTransactionExtended(txid, addPrevouts, lazyPrevouts, forceCore, true))); + return (results.filter(r => r.status === 'fulfilled') as PromiseFulfilledResult[]).map(r => r.value); + } else { + const transactions = await bitcoinApi.$getMempoolTransactions(txids); + return transactions.map(transaction => { + if (Common.isLiquid()) { + if (!isFinite(Number(transaction.fee))) { + transaction.fee = Object.values(transaction.fee || {}).reduce((total, output) => total + output, 0); + } + } + + return this.extendMempoolTransaction(transaction); + }); + } + } + public extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended { // @ts-ignore if (transaction.vsize) { diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 0d0332523..8ade49288 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -21,6 +21,8 @@ import Audit from './audit'; import { deepClone } from '../utils/clone'; import priceUpdater from '../tasks/price-updater'; import { ApiPrice } from '../repositories/PricesRepository'; +import accelerationApi from './services/acceleration'; +import mempool from './mempool'; // valid 'want' subscriptions const wantable = [ @@ -172,9 +174,15 @@ class WebsocketHandler { } const tx = memPool.getMempool()[trackTxid]; if (tx && tx.position) { + const position: { block: number, vsize: number, accelerated?: boolean } = { + ...tx.position + }; + if (tx.acceleration) { + position.accelerated = tx.acceleration; + } response['txPosition'] = JSON.stringify({ txid: trackTxid, - position: tx.position, + position }); } } else { @@ -390,7 +398,7 @@ class WebsocketHandler { } async $handleMempoolChange(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, - newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]): Promise { + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]): Promise { if (!this.wss) { throw new Error('WebSocket.Server is not set'); } @@ -399,9 +407,9 @@ class WebsocketHandler { if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { if (config.MEMPOOL.RUST_GBT) { - await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions); + await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions, config.MEMPOOL_SERVICES.ACCELERATIONS); } else { - await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, true); + await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } } else { mempoolBlocks.updateMempoolBlocks(newMempool, true); @@ -647,7 +655,10 @@ class WebsocketHandler { if (mempoolTx && mempoolTx.position) { response['txPosition'] = JSON.stringify({ txid: trackTxid, - position: mempoolTx.position, + position: { + ...mempoolTx.position, + accelerated: mempoolTx.acceleration || undefined, + } }); } } @@ -695,6 +706,7 @@ class WebsocketHandler { if (config.MEMPOOL.AUDIT && memPool.isInSync()) { let projectedBlocks; let auditMempool = _memPool; + const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); // template calculation functions have mempool side effects, so calculate audits using // a cloned copy of the mempool if we're running a different algorithm for mempool updates const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL; @@ -702,19 +714,27 @@ class WebsocketHandler { auditMempool = deepClone(_memPool); if (config.MEMPOOL.ADVANCED_GBT_AUDIT) { if (config.MEMPOOL.RUST_GBT) { - projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool); + projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool, isAccelerated, block.extras.pool.id); } else { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false); + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id); } } else { projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false); } } else { - projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); + if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) { + if (config.MEMPOOL.RUST_GBT) { + projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(auditMempool, Object.keys(auditMempool).length, [], [], isAccelerated, block.extras.pool.id); + } else { + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id); + } + } else { + projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); + } } if (Common.indexingEnabled()) { - const { censored, added, fresh, sigop, fullrbf, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool); + const { censored, added, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool); const matchRate = Math.round(score * 100 * 100) / 100; const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : []; @@ -743,6 +763,7 @@ class WebsocketHandler { freshTxs: fresh, sigopTxs: sigop, fullrbfTxs: fullrbf, + acceleratedTxs: accelerated, matchRate: matchRate, expectedFees: totalFees, expectedWeight: totalWeight, @@ -770,9 +791,9 @@ class WebsocketHandler { if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { if (config.MEMPOOL.RUST_GBT) { - await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions); + await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions, true); } else { - await mempoolBlocks.$makeBlockTemplates(_memPool, true); + await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } } else { mempoolBlocks.updateMempoolBlocks(_memPool, true); @@ -836,7 +857,10 @@ class WebsocketHandler { if (mempoolTx && mempoolTx.position) { response['txPosition'] = JSON.stringify({ txid: trackTxid, - position: mempoolTx.position, + position: { + ...mempoolTx.position, + accelerated: mempoolTx.acceleration || undefined, + } }); } } diff --git a/backend/src/config.ts b/backend/src/config.ts index 3a028d0cd..982e17b34 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -38,6 +38,7 @@ interface IConfig { DISK_CACHE_BLOCK_INTERVAL: number; MAX_PUSH_TX_SIZE_WEIGHT: number; ALLOW_UNREACHABLE: boolean; + PRICE_UPDATES_PER_HOUR: number; }; ESPLORA: { REST_API_URL: string; @@ -115,10 +116,6 @@ interface IConfig { USERNAME: string; PASSWORD: string; }; - PRICE_DATA_SERVER: { - TOR_URL: string; - CLEARNET_URL: string; - }; EXTERNAL_DATA_SERVER: { MEMPOOL_API: string; MEMPOOL_ONION: string; @@ -139,6 +136,10 @@ interface IConfig { AUDIT_START_HEIGHT: number; SERVERS: string[]; }, + MEMPOOL_SERVICES: { + API: string; + ACCELERATIONS: boolean; + }, REDIS: { ENABLED: boolean; UNIX_SOCKET_PATH: string; @@ -181,6 +182,7 @@ const defaults: IConfig = { 'DISK_CACHE_BLOCK_INTERVAL': 6, 'MAX_PUSH_TX_SIZE_WEIGHT': 400000, 'ALLOW_UNREACHABLE': true, + 'PRICE_UPDATES_PER_HOUR': 1, }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', @@ -258,10 +260,6 @@ const defaults: IConfig = { 'USERNAME': '', 'PASSWORD': '' }, - 'PRICE_DATA_SERVER': { - 'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', - 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices' - }, 'EXTERNAL_DATA_SERVER': { 'MEMPOOL_API': 'https://mempool.space/api/v1', 'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1', @@ -282,6 +280,10 @@ const defaults: IConfig = { 'AUDIT_START_HEIGHT': 774000, 'SERVERS': [], }, + 'MEMPOOL_SERVICES': { + 'API': '', + 'ACCELERATIONS': false, + }, 'REDIS': { 'ENABLED': false, 'UNIX_SOCKET_PATH': '', @@ -302,10 +304,10 @@ class Config implements IConfig { LND: IConfig['LND']; CLIGHTNING: IConfig['CLIGHTNING']; SOCKS5PROXY: IConfig['SOCKS5PROXY']; - PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER']; EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER']; MAXMIND: IConfig['MAXMIND']; REPLICATION: IConfig['REPLICATION']; + MEMPOOL_SERVICES: IConfig['MEMPOOL_SERVICES']; REDIS: IConfig['REDIS']; constructor() { @@ -323,10 +325,10 @@ class Config implements IConfig { this.LND = configs.LND; this.CLIGHTNING = configs.CLIGHTNING; this.SOCKS5PROXY = configs.SOCKS5PROXY; - this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER; this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER; this.MAXMIND = configs.MAXMIND; this.REPLICATION = configs.REPLICATION; + this.MEMPOOL_SERVICES = configs.MEMPOOL_SERVICES; this.REDIS = configs.REDIS; } diff --git a/backend/src/index.ts b/backend/src/index.ts index 51d407f6f..adb3f2e02 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -30,6 +30,7 @@ import generalLightningRoutes from './api/explorer/general.routes'; import lightningStatsUpdater from './tasks/lightning/stats-updater.service'; import networkSyncService from './tasks/lightning/network-sync.service'; import statisticsRoutes from './api/statistics/statistics.routes'; +import pricesRoutes from './api/prices/prices.routes'; import miningRoutes from './api/mining/mining-routes'; import bisqRoutes from './api/bisq/bisq.routes'; import liquidRoutes from './api/liquid/liquid.routes'; @@ -188,14 +189,16 @@ class Server { } const newMempool = await bitcoinApi.$getRawMempool(); const numHandledBlocks = await blocks.$updateBlocks(); + const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerRunning ? 10 : 1); if (numHandledBlocks === 0) { - await memPool.$updateMempool(newMempool); + await memPool.$updateMempool(newMempool, pollRate); } indexer.$run(); + priceUpdater.$run(); // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS const elapsed = Date.now() - start; - const remainingTime = Math.max(0, config.MEMPOOL.POLL_RATE_MS - elapsed) + const remainingTime = Math.max(0, pollRate - elapsed); setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 0 : remainingTime); this.backendRetryCount = 0; } catch (e: any) { @@ -260,6 +263,7 @@ class Server { setUpHttpApiRoutes(): void { bitcoinRoutes.initRoutes(this.app); + pricesRoutes.initRoutes(this.app); if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && config.MEMPOOL.ENABLED) { statisticsRoutes.initRoutes(this.app); } diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index d89a2647f..7ec65d9c9 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -105,6 +105,12 @@ class Indexer { return; } + try { + await priceUpdater.$run(); + } catch (e) { + logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e)); + } + // Do not attempt to index anything unless Bitcoin Core is fully synced const blockchainInfo = await bitcoinClient.getBlockchainInfo(); if (blockchainInfo.blocks !== blockchainInfo.headers) { @@ -119,8 +125,6 @@ class Indexer { await this.checkAvailableCoreIndexes(); try { - await priceUpdater.$run(); - const chainValid = await blocks.$generateBlockDatabase(); if (chainValid === false) { // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 25e7f0387..c08846191 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -20,6 +20,7 @@ export interface PoolInfo { slug: string; avgMatchRate: number | null; avgFeeDelta: number | null; + poolUniqueId: number; } export interface PoolStats extends PoolInfo { @@ -36,6 +37,7 @@ export interface BlockAudit { sigopTxs: string[], fullrbfTxs: string[], addedTxs: string[], + acceleratedTxs: string[], matchRate: number, expectedFees?: number, expectedWeight?: number, @@ -91,6 +93,7 @@ export interface TransactionExtended extends IEsploraApi.Transaction { block: number, vsize: number, }; + acceleration?: boolean; uid?: number; } @@ -182,6 +185,7 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; + acc?: boolean; rate?: number; // effective fee rate } diff --git a/backend/src/replication/AuditReplication.ts b/backend/src/replication/AuditReplication.ts index 26bf6dad7..5de9de0da 100644 --- a/backend/src/replication/AuditReplication.ts +++ b/backend/src/replication/AuditReplication.ts @@ -116,6 +116,7 @@ class AuditReplication { freshTxs: auditSummary.freshTxs || [], sigopTxs: auditSummary.sigopTxs || [], fullrbfTxs: auditSummary.fullrbfTxs || [], + acceleratedTxs: auditSummary.acceleratedTxs || [], matchRate: auditSummary.matchRate, expectedFees: auditSummary.expectedFees, expectedWeight: auditSummary.expectedWeight, diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index f7a2a59b5..c17958d2b 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -6,9 +6,9 @@ import { BlockAudit, AuditScore } from '../mempool.interfaces'; class BlocksAuditRepositories { public async $saveAudit(audit: BlockAudit): Promise { try { - await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, fullrbf_txs, match_rate, expected_fees, expected_weight) - VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), - JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]); + await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight) + VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), + JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), JSON.stringify(audit.acceleratedTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]); } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`); @@ -69,6 +69,7 @@ class BlocksAuditRepositories { fresh_txs as freshTxs, sigop_txs as sigopTxs, fullrbf_txs as fullrbfTxs, + accelerated_txs as acceleratedTxs, match_rate as matchRate, expected_fees as expectedFees, expected_weight as expectedWeight @@ -83,6 +84,7 @@ class BlocksAuditRepositories { rows[0].freshTxs = JSON.parse(rows[0].freshTxs); rows[0].sigopTxs = JSON.parse(rows[0].sigopTxs); rows[0].fullrbfTxs = JSON.parse(rows[0].fullrbfTxs); + rows[0].acceleratedTxs = JSON.parse(rows[0].acceleratedTxs); rows[0].template = JSON.parse(rows[0].template); return rows[0]; diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 078b85a03..0953f9b84 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -1,3 +1,4 @@ +import bitcoinApi from '../api/bitcoin/bitcoin-api-factory'; import { BlockExtended, BlockExtension, BlockPrice, EffectiveFeeStats } from '../mempool.interfaces'; import DB from '../database'; import logger from '../logger'; @@ -12,6 +13,7 @@ import config from '../config'; import chainTips from '../api/chain-tips'; import blocks from '../api/blocks'; import BlocksAuditsRepository from './BlocksAuditsRepository'; +import transactionUtils from '../api/transaction-utils'; interface DatabaseBlock { id: string; @@ -539,7 +541,7 @@ class BlocksRepository { */ public async $getBlocksDifficulty(): Promise { try { - const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty FROM blocks`); + const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty, bits FROM blocks`); return rows; } catch (e) { logger.err('Cannot get blocks difficulty list from the db. Reason: ' + (e instanceof Error ? e.message : e)); @@ -848,7 +850,7 @@ class BlocksRepository { */ public async $getOldestConsecutiveBlock(): Promise { try { - const [rows]: any = await DB.query(`SELECT height, UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty FROM blocks ORDER BY height DESC`); + const [rows]: any = await DB.query(`SELECT height, UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty, bits FROM blocks ORDER BY height DESC`); for (let i = 0; i < rows.length - 1; ++i) { if (rows[i].height - rows[i + 1].height > 1) { return rows[i]; @@ -1036,8 +1038,17 @@ class BlocksRepository { { extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); if (extras.feePercentiles === null) { - const block = await bitcoinClient.getBlock(dbBlk.id, 2); - const summary = blocks.summarizeBlock(block); + + let summary; + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(dbBlk.id)).map(tx => transactionUtils.extendTransaction(tx)); + summary = blocks.summarizeBlockTransactions(dbBlk.id, txs); + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(dbBlk.id, 2); + summary = blocks.summarizeBlock(block); + } + await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.id, summary.transactions); extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); } diff --git a/backend/src/repositories/PoolsRepository.ts b/backend/src/repositories/PoolsRepository.ts index 899712266..eda792bb3 100644 --- a/backend/src/repositories/PoolsRepository.ts +++ b/backend/src/repositories/PoolsRepository.ts @@ -40,7 +40,8 @@ class PoolsRepository { pools.link AS link, slug, AVG(blocks_audits.match_rate) AS avgMatchRate, - AVG((CAST(blocks.fees as SIGNED) - CAST(blocks_audits.expected_fees as SIGNED)) / NULLIF(CAST(blocks_audits.expected_fees as SIGNED), 0)) AS avgFeeDelta + AVG((CAST(blocks.fees as SIGNED) - CAST(blocks_audits.expected_fees as SIGNED)) / NULLIF(CAST(blocks_audits.expected_fees as SIGNED), 0)) AS avgFeeDelta, + unique_id as poolUniqueId FROM blocks JOIN pools on pools.id = pool_id LEFT JOIN blocks_audits ON blocks_audits.height = blocks.height diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index fafe2b913..fd799fb87 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -25,7 +25,10 @@ export interface PriceHistory { class PriceUpdater { public historyInserted = false; - private lastRun = 0; + private timeBetweenUpdatesMs = 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR; + private cyclePosition = -1; + private firstRun = true; + private lastTime = -1; private lastHistoricalRun = 0; private running = false; private feeds: PriceFeed[] = []; @@ -41,6 +44,8 @@ class PriceUpdater { this.feeds.push(new CoinbaseApi()); this.feeds.push(new BitfinexApi()); this.feeds.push(new GeminiApi()); + + this.setCyclePosition(); } public getLatestPrices(): ApiPrice { @@ -100,22 +105,48 @@ class PriceUpdater { this.running = false; } + private getMillisecondsSinceBeginningOfHour(): number { + const now = new Date(); + const beginningOfHour = new Date(now); + beginningOfHour.setMinutes(0, 0, 0); + return now.getTime() - beginningOfHour.getTime(); + } + + private setCyclePosition(): void { + const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour(); + for (let i = 0; i < config.MEMPOOL.PRICE_UPDATES_PER_HOUR; i++) { + if (this.timeBetweenUpdatesMs * i > millisecondsSinceBeginningOfHour) { + this.cyclePosition = i; + return; + } + } + this.cyclePosition = config.MEMPOOL.PRICE_UPDATES_PER_HOUR; + } + /** * Fetch last BTC price from exchanges, average them, and save it in the database once every hour */ private async $updatePrice(): Promise { - if (this.lastRun === 0 && config.DATABASE.ENABLED === true) { - this.lastRun = await PricesRepository.$getLatestPriceTime(); + let forceUpdate = false; + if (this.firstRun === true && config.DATABASE.ENABLED === true) { + const lastUpdate = await PricesRepository.$getLatestPriceTime(); + if (new Date().getTime() / 1000 - lastUpdate > this.timeBetweenUpdatesMs / 1000) { + forceUpdate = true; + } + this.firstRun = false; } - if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < 3600) { - // Refresh only once every hour + const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour(); + + // Reset the cycle on new hour + if (this.lastTime > millisecondsSinceBeginningOfHour) { + this.cyclePosition = 0; + } + this.lastTime = millisecondsSinceBeginningOfHour; + if (millisecondsSinceBeginningOfHour < this.timeBetweenUpdatesMs * this.cyclePosition && !forceUpdate && this.cyclePosition !== 0) { return; } - const previousRun = this.lastRun; - this.lastRun = new Date().getTime() / 1000; - for (const currency of this.currencies) { let prices: number[] = []; @@ -146,26 +177,27 @@ class PriceUpdater { } } - logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`); - - if (config.DATABASE.ENABLED === true) { + if (config.DATABASE.ENABLED === true && this.cyclePosition === 0) { // Save everything in db try { const p = 60 * 60 * 1000; // milliseconds in an hour const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042 - this.latestPrices.time = nowRounded.getTime() / 1000; await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices); } catch (e) { - this.lastRun = previousRun + 5 * 60; logger.err(`Cannot save latest prices into db. Trying again in 5 minutes. Reason: ${(e instanceof Error ? e.message : e)}`); } } + this.latestPrices.time = Math.round(new Date().getTime() / 1000); + logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`); + if (this.ratesChangedCallback) { this.ratesChangedCallback(this.latestPrices); } - this.lastRun = new Date().getTime() / 1000; + if (!forceUpdate) { + this.cyclePosition++; + } if (this.latestPrices.USD === -1) { this.latestPrices = await PricesRepository.$getLatestConversionRates(); diff --git a/contributors/andrewtoth.txt b/contributors/andrewtoth.txt new file mode 100644 index 000000000..bd2ed3d06 --- /dev/null +++ b/contributors/andrewtoth.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of August 2, 2023. + +Signed: andrewtoth diff --git a/contributors/bguillaumat.txt b/contributors/bguillaumat.txt new file mode 100644 index 000000000..ac14a07c7 --- /dev/null +++ b/contributors/bguillaumat.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022. + +Signed: bguillaumat diff --git a/docker/README.md b/docker/README.md index d95bc7aee..13bda7ec6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -113,7 +113,8 @@ Below we list all settings from `mempool-config.json` and the corresponding over "ADVANCED_GBT_MEMPOOL": false, "CPFP_INDEXING": false, "MAX_BLOCKS_BULK_QUERY": 0, - "DISK_CACHE_BLOCK_INTERVAL": 6 + "DISK_CACHE_BLOCK_INTERVAL": 6, + "PRICE_UPDATES_PER_HOUR": 1 }, ``` @@ -146,6 +147,7 @@ Corresponding `docker-compose.yml` overrides: MEMPOOL_CPFP_INDEXING: "" MEMPOOL_MAX_BLOCKS_BULK_QUERY: "" MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: "" + MEMPOOL_PRICE_UPDATES_PER_HOUR: "" ... ``` @@ -363,25 +365,6 @@ Corresponding `docker-compose.yml` overrides:
-`mempool-config.json`: -```json - "PRICE_DATA_SERVER": { - "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", - "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" - } -``` - -Corresponding `docker-compose.yml` overrides: -```yaml - api: - environment: - PRICE_DATA_SERVER_TOR_URL: "" - PRICE_DATA_SERVER_CLEARNET_URL: "" - ... -``` - -
- `mempool-config.json`: ```json "LIGHTNING": { diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 8b47d53b8..70ff0d283 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -33,7 +33,8 @@ "MAX_PUSH_TX_SIZE_WEIGHT": __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__, "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__, "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", - "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__" + "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", + "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__ }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", @@ -111,10 +112,6 @@ "USERNAME": "__SOCKS5PROXY_USERNAME__", "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, - "PRICE_DATA_SERVER": { - "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", - "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" - }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__", "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__", @@ -135,6 +132,10 @@ "AUDIT_START_HEIGHT": __REPLICATION_AUDIT_START_HEIGHT__, "SERVERS": __REPLICATION_SERVERS__ }, + "MEMPOOL_SERVICES": { + "API": "__MEMPOOL_SERVICES_API__", + "ACCELERATIONS": __MEMPOOL_SERVICES_ACCELERATIONS__ + }, "REDIS": { "ENABLED": __REDIS_ENABLED__, "UNIX_SOCKET_PATH": "__REDIS_UNIX_SOCKET_PATH__" diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 7071493fa..681872681 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -35,7 +35,7 @@ __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0} __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6} __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__=${MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT:=4000000} __MEMPOOL_ALLOW_UNREACHABLE__=${MEMPOOL_ALLOW_UNREACHABLE:=true} - +__MEMPOOL_PRICE_UPDATES_PER_HOUR__=${MEMPOOL_PRICE_UPDATES_PER_HOUR:=1} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} @@ -94,10 +94,6 @@ __SOCKS5PROXY_PORT__=${SOCKS5PROXY_PORT:=9050} __SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""} __SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""} -# PRICE_DATA_SERVER -__PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices} -__PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices} - # EXTERNAL_DATA_SERVER __EXTERNAL_DATA_SERVER_MEMPOOL_API__=${EXTERNAL_DATA_SERVER_MEMPOOL_API:=https://mempool.space/api/v1} __EXTERNAL_DATA_SERVER_MEMPOOL_ONION__=${EXTERNAL_DATA_SERVER_MEMPOOL_ONION:=http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1} @@ -137,6 +133,10 @@ __REPLICATION_AUDIT__=${REPLICATION_AUDIT:=true} __REPLICATION_AUDIT_START_HEIGHT__=${REPLICATION_AUDIT_START_HEIGHT:=774000} __REPLICATION_SERVERS__=${REPLICATION_SERVERS:=[]} +# MEMPOOL_SERVICES +__MEMPOOL_SERVICES_API__=${MEMPOOL_SERVICES_API:=""} +__MEMPOOL_SERVICES_ACCELERATIONS__=${MEMPOOL_SERVICES_ACCELERATIONS:=false} + # REDIS __REDIS_ENABLED__=${REDIS_ENABLED:=true} __REDIS_UNIX_SOCKET_PATH__=${REDIS_UNIX_SOCKET_PATH:=true} @@ -170,13 +170,14 @@ sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-co sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json -sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_GBT__}!g" mempool-config.json +sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_RUST_GBT__}!g" mempool-config.json sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json sed -i "s!__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__!${__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__}!g" mempool-config.json sed -i "s!__MEMPOOL_ALLOW_UNREACHABLE__!${__MEMPOOL_ALLOW_UNREACHABLE__}!g" mempool-config.json +sed -i "s!__MEMPOOL_PRICE_UPDATES_PER_HOUR__!${__MEMPOOL_PRICE_UPDATES_PER_HOUR__}!g" mempool-config.json sed -i "s!__CORE_RPC_HOST__!${__CORE_RPC_HOST__}!g" mempool-config.json sed -i "s!__CORE_RPC_PORT__!${__CORE_RPC_PORT__}!g" mempool-config.json @@ -226,9 +227,6 @@ sed -i "s!__SOCKS5PROXY_PORT__!${__SOCKS5PROXY_PORT__}!g" mempool-config.json sed -i "s!__SOCKS5PROXY_USERNAME__!${__SOCKS5PROXY_USERNAME__}!g" mempool-config.json sed -i "s!__SOCKS5PROXY_PASSWORD__!${__SOCKS5PROXY_PASSWORD__}!g" mempool-config.json -sed -i "s!__PRICE_DATA_SERVER_TOR_URL__!${__PRICE_DATA_SERVER_TOR_URL__}!g" mempool-config.json -sed -i "s!__PRICE_DATA_SERVER_CLEARNET_URL__!${__PRICE_DATA_SERVER_CLEARNET_URL__}!g" mempool-config.json - sed -i "s!__EXTERNAL_DATA_SERVER_MEMPOOL_API__!${__EXTERNAL_DATA_SERVER_MEMPOOL_API__}!g" mempool-config.json sed -i "s!__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__!${__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__}!g" mempool-config.json sed -i "s!__EXTERNAL_DATA_SERVER_LIQUID_API__!${__EXTERNAL_DATA_SERVER_LIQUID_API__}!g" mempool-config.json @@ -267,6 +265,10 @@ sed -i "s!__REPLICATION_AUDIT__!${__REPLICATION_AUDIT__}!g" mempool-config.json sed -i "s!__REPLICATION_AUDIT_START_HEIGHT__!${__REPLICATION_AUDIT_START_HEIGHT__}!g" mempool-config.json sed -i "s!__REPLICATION_SERVERS__!${__REPLICATION_SERVERS__}!g" mempool-config.json +# MEMPOOL_SERVICES +sed -i "s!__MEMPOOL_SERVICES_API__!${__MEMPOOL_SERVICES_API__}!g" mempool-config.json +sed -i "s!__MEMPOOL_SERVICES_ACCELERATIONS__!${__MEMPOOL_SERVICES_ACCELERATIONS__}!g" mempool-config.json + # REDIS sed -i "s!__REDIS_ENABLED__!${__REDIS_ENABLED__}!g" mempool-config.json sed -i "s!__REDIS_UNIX_SOCKET_PATH__!${__REDIS_UNIX_SOCKET_PATH__}!g" mempool-config.json diff --git a/frontend/mempool-frontend-config.sample.json b/frontend/mempool-frontend-config.sample.json index 084cbd0ef..79cce7686 100644 --- a/frontend/mempool-frontend-config.sample.json +++ b/frontend/mempool-frontend-config.sample.json @@ -22,5 +22,6 @@ "TESTNET_BLOCK_AUDIT_START_HEIGHT": 0, "SIGNET_BLOCK_AUDIT_START_HEIGHT": 0, "LIGHTNING": false, - "HISTORICAL_PRICE": true + "HISTORICAL_PRICE": true, + "ACCELERATOR": false } diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 23f50b544..e9d5ec3b2 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -31,6 +31,14 @@ + +

Sponsor the project

+
+ Community + Enterprise +
+ +

Enterprise Sponsors 🚀

@@ -191,16 +199,41 @@
-
-

Community Sponsors ❤️

+ +
+
+

Whale Sponsors

+
+ + + + + + + +
+
+
+

Chad Sponsors

+
+ + + + + +
+
+
+ + +
+

OG Sponsors ❤️

- - - - - - + + + +
@@ -340,7 +373,7 @@
- +
@@ -354,7 +387,7 @@
- + {{ contributor.name }} @@ -366,7 +399,7 @@
- + {{ contributor.name }} diff --git a/frontend/src/app/components/about/about.component.scss b/frontend/src/app/components/about/about.component.scss index e1d6c829a..2a5710ca1 100644 --- a/frontend/src/app/components/about/about.component.scss +++ b/frontend/src/app/components/about/about.component.scss @@ -10,6 +10,9 @@ margin: 25px; line-height: 32px; } + .unknown { + border: 1px solid #b4b4b4; + } .image.not-rounded { border-radius: 0; diff --git a/frontend/src/app/components/about/about.component.ts b/frontend/src/app/components/about/about.component.ts index 176490add..4bf7869de 100644 --- a/frontend/src/app/components/about/about.component.ts +++ b/frontend/src/app/components/about/about.component.ts @@ -6,7 +6,7 @@ import { Observable } from 'rxjs'; import { ApiService } from '../../services/api.service'; import { IBackendInfo } from '../../interfaces/websocket.interface'; import { Router, ActivatedRoute } from '@angular/router'; -import { map, tap } from 'rxjs/operators'; +import { map, share, tap } from 'rxjs/operators'; import { ITranslators } from '../../interfaces/node-api.interface'; import { DOCUMENT } from '@angular/common'; @@ -19,14 +19,16 @@ import { DOCUMENT } from '@angular/common'; export class AboutComponent implements OnInit { @ViewChild('promoVideo') promoVideo: ElementRef; backendInfo$: Observable; - sponsors$: Observable; - translators$: Observable; - allContributors$: Observable; frontendGitCommitHash = this.stateService.env.GIT_COMMIT_HASH; packetJsonVersion = this.stateService.env.PACKAGE_JSON_VERSION; officialMempoolSpace = this.stateService.env.OFFICIAL_MEMPOOL_SPACE; showNavigateToSponsor = false; + profiles$: Observable; + translators$: Observable; + allContributors$: Observable; + ogs$: Observable; + constructor( private websocketService: WebsocketService, private seoService: SeoService, @@ -43,10 +45,13 @@ export class AboutComponent implements OnInit { this.seoService.setTitle($localize`:@@004b222ff9ef9dd4771b777950ca1d0e4cd4348a:About`); this.websocketService.want(['blocks']); - this.sponsors$ = this.apiService.getDonation$() - .pipe( - tap(() => this.goToAnchor()) - ); + this.profiles$ = this.apiService.getAboutPageProfiles$().pipe( + tap(() => { + this.goToAnchor() + }), + share(), + ) + this.translators$ = this.apiService.getTranslators$() .pipe( map((translators) => { @@ -59,6 +64,9 @@ export class AboutComponent implements OnInit { }), tap(() => this.goToAnchor()) ); + + this.ogs$ = this.apiService.getOgs$(); + this.allContributors$ = this.apiService.getContributor$().pipe( map((contributors) => { return { diff --git a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts index 49da16d55..634d0f524 100644 --- a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts +++ b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts @@ -147,7 +147,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On } } - update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { + update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: boolean | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { if (this.scene) { this.scene.update(add, remove, change, direction, resetLayout); this.start(); diff --git a/frontend/src/app/components/block-overview-graph/block-scene.ts b/frontend/src/app/components/block-overview-graph/block-scene.ts index 510803f03..cb0537e2a 100644 --- a/frontend/src/app/components/block-overview-graph/block-scene.ts +++ b/frontend/src/app/components/block-overview-graph/block-scene.ts @@ -150,7 +150,7 @@ export default class BlockScene { this.updateAll(startTime, 200, direction); } - update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { + update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: boolean | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { const startTime = performance.now(); const removed = this.removeBatch(remove, startTime, direction); @@ -175,6 +175,7 @@ export default class BlockScene { // update effective rates change.forEach(tx => { if (this.txs[tx.txid]) { + this.txs[tx.txid].acc = tx.acc; this.txs[tx.txid].feerate = tx.rate || (this.txs[tx.txid].fee / this.txs[tx.txid].vsize); this.txs[tx.txid].rate = tx.rate; this.txs[tx.txid].dirty = true; diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index 1b8c88704..db2c4f6ae 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -17,6 +17,7 @@ const auditColors = { missing: darken(desaturate(hexToColor('f344df'), 0.3), 0.7), added: hexToColor('0099ff'), selected: darken(desaturate(hexToColor('0099ff'), 0.3), 0.7), + accelerated: hexToColor('8F5FF6'), }; // convert from this class's update format to TxSprite's update format @@ -37,8 +38,9 @@ export default class TxView implements TransactionStripped { vsize: number; value: number; feerate: number; + acc?: boolean; rate?: number; - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; scene?: BlockScene; @@ -63,6 +65,7 @@ export default class TxView implements TransactionStripped { this.vsize = tx.vsize; this.value = tx.value; this.feerate = tx.rate || (tx.fee / tx.vsize); // sort by effective fee rate where available + this.acc = tx.acc; this.rate = tx.rate; this.status = tx.status; this.initialised = false; @@ -199,6 +202,11 @@ export default class TxView implements TransactionStripped { const feeLevelColor = feeColors[feeLevelIndex] || feeColors[mempoolFeeColors.length - 1]; // Normal mode if (!this.scene?.highlightingEnabled) { + if (this.acc) { + return auditColors.accelerated; + } else { + return feeLevelColor; + } return feeLevelColor; } // Block audit @@ -216,6 +224,8 @@ export default class TxView implements TransactionStripped { return auditColors.added; case 'selected': return marginalFeeColors[feeLevelIndex] || marginalFeeColors[mempoolFeeColors.length - 1]; + case 'accelerated': + return auditColors.accelerated; case 'found': if (this.context === 'projected') { return auditFeeColors[feeLevelIndex] || auditFeeColors[mempoolFeeColors.length - 1]; @@ -223,7 +233,11 @@ export default class TxView implements TransactionStripped { return feeLevelColor; } default: - return feeLevelColor; + if (this.acc) { + return auditColors.accelerated; + } else { + return feeLevelColor; + } } } } diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html index c62779b69..a53cfdc9c 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html @@ -29,7 +29,8 @@ - Effective fee rate + Effective fee rate + Accelerated fee rate @@ -54,6 +55,7 @@ Added Marginal fee rate Conflicting + Accelerated diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts index 61c294263..65d0f984c 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts @@ -21,6 +21,7 @@ export class BlockOverviewTooltipComponent implements OnChanges { vsize = 1; feeRate = 0; effectiveRate; + acceleration; tooltipPosition: Position = { x: 0, y: 0 }; @@ -53,6 +54,7 @@ export class BlockOverviewTooltipComponent implements OnChanges { this.vsize = tx.vsize || 1; this.feeRate = this.fee / this.vsize; this.effectiveRate = tx.rate; + this.acceleration = tx.acc; } } } diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index ec9a49504..1345717bd 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -340,12 +340,16 @@ export class BlockComponent implements OnInit, OnDestroy { const isFresh = {}; const isSigop = {}; const isRbf = {}; + const isAccelerated = {}; this.numMissing = 0; this.numUnexpected = 0; if (blockAudit?.template) { for (const tx of blockAudit.template) { inTemplate[tx.txid] = true; + if (tx.acc) { + isAccelerated[tx.txid] = true; + } } for (const tx of transactions) { inBlock[tx.txid] = true; @@ -365,6 +369,9 @@ export class BlockComponent implements OnInit, OnDestroy { for (const txid of blockAudit.fullrbfTxs || []) { isRbf[txid] = true; } + for (const txid of blockAudit.acceleratedTxs || []) { + isAccelerated[txid] = true; + } // set transaction statuses for (const tx of blockAudit.template) { tx.context = 'projected'; @@ -389,6 +396,9 @@ export class BlockComponent implements OnInit, OnDestroy { isMissing[tx.txid] = true; this.numMissing++; } + if (isAccelerated[tx.txid]) { + tx.status = 'accelerated'; + } } for (const [index, tx] of transactions.entries()) { tx.context = 'actual'; @@ -405,6 +415,9 @@ export class BlockComponent implements OnInit, OnDestroy { isSelected[tx.txid] = true; this.numUnexpected++; } + if (isAccelerated[tx.txid]) { + tx.status = 'accelerated'; + } } for (const tx of transactions) { inBlock[tx.txid] = true; diff --git a/frontend/src/app/components/difficulty/difficulty-tooltip.component.html b/frontend/src/app/components/difficulty/difficulty-tooltip.component.html index d06bb5e91..7e4b421e1 100644 --- a/frontend/src/app/components/difficulty/difficulty-tooltip.component.html +++ b/frontend/src/app/components/difficulty/difficulty-tooltip.component.html @@ -4,38 +4,56 @@ class="difficulty-tooltip" [style.visibility]="status ? 'visible' : 'hidden'" [style.left]="tooltipPosition.x + 'px'" - [style.top]="tooltipPosition.y + 'px'" + [style.top]="tooltipPosition.y + (isMobile ? -60 : 0) + 'px'" > - + - - {{ i }} blocks expected - {{ i }} block expected + - - {{ i }} blocks mined - {{ i }} block mined + - - {{ i }} blocks remaining - {{ i }} block remaining + - - {{ i }} blocks ahead - {{ i }} block ahead + - - {{ i }} blocks behind - {{ i }} block behind + Next Block -
\ No newline at end of file + + + + + + + +
+ +
+ + + + + + + +
+ +{{ i }} blocks expected +{{ i }} block expected +{{ i }} blocks mined +{{ i }} block mined +{{ i }} blocks remaining +{{ i }} block remaining +{{ i }} blocks ahead +{{ i }} block ahead +{{ i }} blocks behind +{{ i }} block behind \ No newline at end of file diff --git a/frontend/src/app/components/difficulty/difficulty-tooltip.component.ts b/frontend/src/app/components/difficulty/difficulty-tooltip.component.ts index c7d26f61a..42d2a61b4 100644 --- a/frontend/src/app/components/difficulty/difficulty-tooltip.component.ts +++ b/frontend/src/app/components/difficulty/difficulty-tooltip.component.ts @@ -1,4 +1,4 @@ -import { Component, ElementRef, ViewChild, Input, OnChanges } from '@angular/core'; +import { Component, ElementRef, ViewChild, Input, OnChanges, HostListener } from '@angular/core'; interface EpochProgress { base: string; @@ -35,12 +35,15 @@ export class DifficultyTooltipComponent implements OnChanges { remaining: number; isAhead: boolean; isBehind: boolean; + isMobile: boolean; tooltipPosition = { x: 0, y: 0 }; @ViewChild('tooltip') tooltipElement: ElementRef; - constructor() {} + constructor() { + this.onResize(); + } ngOnChanges(changes): void { if (changes.cursorPosition && changes.cursorPosition.currentValue) { @@ -63,4 +66,9 @@ export class DifficultyTooltipComponent implements OnChanges { this.isBehind = this.behind > 0; } } + + @HostListener('window:resize', ['$event']) + onResize(): void { + this.isMobile = window.innerWidth <= 767.98; + } } diff --git a/frontend/src/app/components/difficulty/difficulty.component.html b/frontend/src/app/components/difficulty/difficulty.component.html index 27cddc043..f08ea06f5 100644 --- a/frontend/src/app/components/difficulty/difficulty.component.html +++ b/frontend/src/app/components/difficulty/difficulty.component.html @@ -4,7 +4,7 @@
- + @@ -22,7 +22,7 @@ class="rect {{rect.status}}" [class.hover]="hoverSection && rect.status === hoverSection.status" (pointerover)="onHover($event, rect);" - (pointerout)="onBlur($event);" + (pointerout)="onBlur();" > ; isLoadingWebSocket$: Observable; difficultyEpoch$: Observable; @@ -191,21 +193,26 @@ export class DifficultyComponent implements OnInit { } @HostListener('pointerdown', ['$event']) - onPointerDown(event) { - this.onPointerMove(event); + onPointerDown(event): void { + if (this.epochSvgElement.nativeElement?.contains(event.target)) { + this.onPointerMove(event); + event.preventDefault(); + } } @HostListener('pointermove', ['$event']) - onPointerMove(event) { - this.tooltipPosition = { x: event.clientX, y: event.clientY }; - this.cd.markForCheck(); + onPointerMove(event): void { + if (this.epochSvgElement.nativeElement?.contains(event.target)) { + this.tooltipPosition = { x: event.clientX, y: event.clientY }; + this.cd.markForCheck(); + } } - onHover(event, rect): void { + onHover(_, rect): void { this.hoverSection = rect; } - onBlur(event): void { + onBlur(): void { this.hoverSection = null; } } diff --git a/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts b/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts index f275588a1..010466952 100644 --- a/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts +++ b/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts @@ -64,7 +64,7 @@ export class FeeDistributionGraphComponent implements OnInit, OnChanges, OnDestr return; } const samples = []; - const txs = this.transactions.map(tx => { return { vsize: tx.vsize, rate: tx.rate || (tx.fee / tx.vsize) }; }).sort((a, b) => { return b.rate - a.rate; }); + const txs = this.transactions.filter(tx => !tx.acc).map(tx => { return { vsize: tx.vsize, rate: tx.rate || (tx.fee / tx.vsize) }; }).sort((a, b) => { return b.rate - a.rate; }); const maxBlockVSize = this.stateService.env.BLOCK_WEIGHT_UNITS / 4; const sampleInterval = maxBlockVSize / this.numSamples; let cumVSize = 0; @@ -74,14 +74,14 @@ export class FeeDistributionGraphComponent implements OnInit, OnChanges, OnDestr this.labelInterval = this.numSamples / this.numLabels; while (nextSample <= maxBlockVSize) { if (txIndex >= txs.length) { - samples.push([(1 - (sampleIndex / this.numSamples)) * 100, 0]); + samples.push([(1 - (sampleIndex / this.numSamples)) * 100, 0.000001]); nextSample += sampleInterval; sampleIndex++; continue; } while (txs[txIndex] && nextSample < cumVSize + txs[txIndex].vsize) { - samples.push([(1 - (sampleIndex / this.numSamples)) * 100, txs[txIndex].rate]); + samples.push([(1 - (sampleIndex / this.numSamples)) * 100, txs[txIndex].rate || 0.000001]); nextSample += sampleInterval; sampleIndex++; } @@ -118,7 +118,9 @@ export class FeeDistributionGraphComponent implements OnInit, OnChanges, OnDestr }, }, yAxis: { - type: 'value', + type: 'log', + min: 1, + max: this.data.reduce((min, val) => Math.max(min, val[1]), 1), // name: 'Effective Fee Rate s/vb', // nameLocation: 'middle', splitLine: { @@ -129,12 +131,16 @@ export class FeeDistributionGraphComponent implements OnInit, OnChanges, OnDestr } }, axisLabel: { + show: true, formatter: (value: number): string => { const unitValue = this.weightMode ? value / 4 : value; const selectedPowerOfTen = selectPowerOfTen(unitValue); const newVal = Math.round(unitValue / selectedPowerOfTen.divider); return `${newVal}${selectedPowerOfTen.unit}`; }, + }, + axisTick: { + show: true, } }, series: [{ diff --git a/frontend/src/app/components/master-page/master-page.component.html b/frontend/src/app/components/master-page/master-page.component.html index 79daf7de6..f12fbc960 100644 --- a/frontend/src/app/components/master-page/master-page.component.html +++ b/frontend/src/app/components/master-page/master-page.component.html @@ -1,5 +1,5 @@ -
+
-
+
diff --git a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.scss b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.scss index 40f43a015..606699d93 100644 --- a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.scss +++ b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.scss @@ -169,4 +169,34 @@ transform: translate(calc(-0.2 * var(--block-size)), calc(1.1 * var(--block-size))); border-radius: 2px; z-index: -1; +} + +.blink{ + width:400px; + height:400px; + border-bottom: 35px solid #FFF; + animation: blink 0.2s infinite; +} +@keyframes blink{ + 0% { + border-bottom: 35px solid green; + } + 50% { + border-bottom: 35px solid yellow; + } + 100% { + border-bottom: 35px solid orange; + } +} + +@-webkit-keyframes blink{ + 0% { + border-bottom: 35px solid green; + } + 50% { + border-bottom: 35px solid yellow; + } + 100% { + border-bottom: 35px solid orange; + } } \ No newline at end of file diff --git a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts index cedcf03f4..2269d38a9 100644 --- a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts +++ b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts @@ -26,6 +26,7 @@ import { animate, style, transition, trigger } from '@angular/animations'; export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy { @Input() minimal: boolean = false; @Input() blockWidth: number = 125; + @Input() containerWidth: number = null; @Input() count: number = null; @Input() spotlight: number = 0; @Input() getHref?: (index) => string = (index) => `/mempool-block/${index}`; diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index d4cd6913d..9584cecfd 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -99,14 +99,20 @@ - In several hours (or more) + + In several hours (or more) + Accelerate + - + + + Accelerate + @@ -488,7 +494,8 @@ - Effective fee rate + Accelerated fee rate + Effective fee rate
diff --git a/frontend/src/app/components/transaction/transaction.component.scss b/frontend/src/app/components/transaction/transaction.component.scss index bea8e82bc..5bef401d7 100644 --- a/frontend/src/app/components/transaction/transaction.component.scss +++ b/frontend/src/app/components/transaction/transaction.component.scss @@ -216,4 +216,23 @@ .alert-link { display: block; } +} + +.eta { + display: flex; + justify-content: end; + flex-wrap: wrap; + align-content: center; + @media (min-width: 850px) { + justify-content: space-between; + } +} + +.accelerate { + align-self: auto; + margin-top: 3px; + @media (min-width: 850px) { + justify-self: start; + margin-left: 0px; + } } \ No newline at end of file diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index e856f34eb..26e39515e 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -97,7 +97,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { private router: Router, private relativeUrlPipe: RelativeUrlPipe, private electrsApiService: ElectrsApiService, - private stateService: StateService, + public stateService: StateService, private cacheService: CacheService, private websocketService: WebsocketService, private audioService: AudioService, @@ -183,6 +183,9 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { } else { this.tx.effectiveFeePerVsize = cpfpInfo.effectiveFeePerVsize; } + if (cpfpInfo.acceleration) { + this.tx.acceleration = cpfpInfo.acceleration; + } this.cpfpInfo = cpfpInfo; this.hasEffectiveFeeRate = hasRelatives || (this.tx.effectiveFeePerVsize && (Math.abs(this.tx.effectiveFeePerVsize - this.tx.feePerVsize) > 0.01)); diff --git a/frontend/src/app/docs/api-docs/api-docs.component.html b/frontend/src/app/docs/api-docs/api-docs.component.html index 9945a3f05..40a7ae486 100644 --- a/frontend/src/app/docs/api-docs/api-docs.component.html +++ b/frontend/src/app/docs/api-docs/api-docs.component.html @@ -10,8 +10,8 @@
-

mempool.space merely provides data about the Bitcoin network. It cannot help you with retrieving funds, confirming your transaction quicker, etc.

For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).

-

mempool.space merely provides data about the Bitcoin network. It cannot help you with retrieving funds, confirming your transaction quicker, etc.

For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).

+

mempool.space merely provides data about the Bitcoin network. It cannot help you with retrieving funds, wallet issues, etc.

For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).

+

mempool.space merely provides data about the Bitcoin network. It cannot help you with retrieving funds, wallet issues, etc.

For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).

diff --git a/frontend/src/app/interfaces/electrs.interface.ts b/frontend/src/app/interfaces/electrs.interface.ts index df19f7491..5c15b0ae4 100644 --- a/frontend/src/app/interfaces/electrs.interface.ts +++ b/frontend/src/app/interfaces/electrs.interface.ts @@ -19,6 +19,7 @@ export interface Transaction { ancestors?: Ancestor[]; bestDescendant?: BestDescendant | null; cpfpChecked?: boolean; + acceleration?: number; deleteAfter?: number; _unblinded?: any; _deduced?: boolean; diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 59dff8e90..fbf86aeb4 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -27,6 +27,7 @@ export interface CpfpInfo { effectiveFeePerVsize?: number; sigops?: number; adjustedVsize?: number; + acceleration?: number; } export interface RbfInfo { @@ -111,6 +112,7 @@ export interface PoolInfo { addresses: string; // JSON array emptyBlocks: number; slug: string; + poolUniqueId: number; } export interface PoolStat { pool: PoolInfo; @@ -159,6 +161,7 @@ export interface BlockAudit extends BlockExtended { freshTxs: string[], sigopTxs: string[], fullrbfTxs: string[], + acceleratedTxs: string[], matchRate: number, expectedFees: number, expectedWeight: number, @@ -175,7 +178,8 @@ export interface TransactionStripped { vsize: number; value: number; rate?: number; // effective fee rate - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf'; + acc?: boolean; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; } @@ -187,6 +191,7 @@ export interface RbfTransaction extends TransactionStripped { export interface MempoolPosition { block: number, vsize: number, + accelerated?: boolean } export interface RewardStats { diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index e0ecdfeda..43ab1e5f4 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -70,7 +70,7 @@ export interface MempoolBlockWithTransactions extends MempoolBlock { export interface MempoolBlockDelta { added: TransactionStripped[], removed: string[], - changed?: { txid: string, rate: number | undefined }[]; + changed?: { txid: string, rate: number | undefined, acc: boolean | undefined }[]; } export interface MempoolInfo { @@ -88,8 +88,9 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; + acc?: boolean; // is accelerated? rate?: number; // effective fee rate - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; } diff --git a/frontend/src/app/lightning/lightning.module.ts b/frontend/src/app/lightning/lightning.module.ts index 0b824ad78..f0154a15f 100644 --- a/frontend/src/app/lightning/lightning.module.ts +++ b/frontend/src/app/lightning/lightning.module.ts @@ -34,6 +34,7 @@ import { OldestNodes } from '../lightning/nodes-ranking/oldest-nodes/oldest-node import { NodesRankingsDashboard } from '../lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component'; import { NodeChannels } from '../lightning/nodes-channels/node-channels.component'; import { GroupComponent } from './group/group.component'; +import { NodeOwnerComponent } from './node-owner/node-owner.component'; @NgModule({ declarations: [ @@ -66,6 +67,7 @@ import { GroupComponent } from './group/group.component'; NodesRankingsDashboard, NodeChannels, GroupComponent, + NodeOwnerComponent, ], imports: [ CommonModule, @@ -103,6 +105,7 @@ import { GroupComponent } from './group/group.component'; OldestNodes, NodesRankingsDashboard, NodeChannels, + NodeOwnerComponent, ], providers: [ LightningApiService, diff --git a/frontend/src/app/lightning/node-owner/node-owner.component.html b/frontend/src/app/lightning/node-owner/node-owner.component.html new file mode 100644 index 000000000..e37b1e027 --- /dev/null +++ b/frontend/src/app/lightning/node-owner/node-owner.component.html @@ -0,0 +1,17 @@ +
+ +
+ +
+ + + +
+ +
+ Claim +
+ +
+ +
\ No newline at end of file diff --git a/frontend/src/app/lightning/node-owner/node-owner.component.scss b/frontend/src/app/lightning/node-owner/node-owner.component.scss new file mode 100644 index 000000000..6734168cf --- /dev/null +++ b/frontend/src/app/lightning/node-owner/node-owner.component.scss @@ -0,0 +1,4 @@ +.profile-photo { + width: 31px; + height: 31px; +} \ No newline at end of file diff --git a/frontend/src/app/lightning/node-owner/node-owner.component.ts b/frontend/src/app/lightning/node-owner/node-owner.component.ts new file mode 100644 index 000000000..a03c04901 --- /dev/null +++ b/frontend/src/app/lightning/node-owner/node-owner.component.ts @@ -0,0 +1,20 @@ +import { ChangeDetectionStrategy, Component, Input } from '@angular/core'; +import { Observable } from 'rxjs'; +import { StateService } from '../../services/state.service'; + +@Component({ + selector: 'app-node-owner', + templateUrl: './node-owner.component.html', + styleUrls: ['./node-owner.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class NodeOwnerComponent{ + @Input() publicKey: string = ''; + @Input() alias: string = ''; + @Input() nodeOwner$: Observable; + + constructor( + public stateService: StateService + ) { + } +} diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index c6c693a3a..11ddbc0eb 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -3,13 +3,17 @@
Lightning node
-

{{ node.alias }}

- +
+

{{ node.alias }}

+ +
+ +
diff --git a/frontend/src/app/lightning/node/node.component.scss b/frontend/src/app/lightning/node/node.component.scss index 272de4b09..117fc8a2c 100644 --- a/frontend/src/app/lightning/node/node.component.scss +++ b/frontend/src/app/lightning/node/node.component.scss @@ -111,3 +111,17 @@ app-fiat { margin: 0 0.25em; color: slategrey; } + +.claim-btn { + max-height: 32px; + @media (min-width: 850px) { + display: none; + } +} + +.claim-btn-mobile { + max-height: 32px; + @media (max-width: 850px) { + display: none; + } +} \ No newline at end of file diff --git a/frontend/src/app/lightning/node/node.component.ts b/frontend/src/app/lightning/node/node.component.ts index 719136d79..6447eb6bd 100644 --- a/frontend/src/app/lightning/node/node.component.ts +++ b/frontend/src/app/lightning/node/node.component.ts @@ -1,7 +1,7 @@ -import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; +import { ChangeDetectionStrategy, Component, OnInit, ChangeDetectorRef } from '@angular/core'; import { ActivatedRoute, ParamMap } from '@angular/router'; -import { Observable } from 'rxjs'; -import { catchError, map, switchMap, tap } from 'rxjs/operators'; +import { Observable, of, EMPTY } from 'rxjs'; +import { catchError, map, switchMap, tap, share } from 'rxjs/operators'; import { SeoService } from '../../services/seo.service'; import { ApiService } from '../../services/api.service'; import { LightningApiService } from '../lightning-api.service'; @@ -38,6 +38,7 @@ export class NodeComponent implements OnInit { tlvRecords: CustomRecord[]; avgChannelDistance$: Observable; showFeatures = false; + nodeOwner$: Observable; kmToMiles = kmToMiles; constructor( @@ -45,6 +46,7 @@ export class NodeComponent implements OnInit { private lightningApiService: LightningApiService, private activatedRoute: ActivatedRoute, private seoService: SeoService, + private cd: ChangeDetectorRef, ) { } ngOnInit(): void { @@ -147,6 +149,24 @@ export class NodeComponent implements OnInit { return null; }) ) as Observable; + + this.nodeOwner$ = this.activatedRoute.paramMap + .pipe( + switchMap((params: ParamMap) => { + return this.apiService.getNodeOwner$(params.get('public_key')).pipe( + switchMap((response) => { + if (response.status === 204) { + return of(false); + } + return of(response.body); + }), + catchError(() => { + return of(false); + }) + ) + }), + share(), + ); } toggleShowDetails(): void { diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index e2d3be9be..1ed9d2f5c 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -8,6 +8,8 @@ import { WebsocketResponse } from '../interfaces/websocket.interface'; import { Outspend, Transaction } from '../interfaces/electrs.interface'; import { Conversion } from './price.service'; +const SERVICES_API_PREFIX = `/api/v1/services`; + @Injectable({ providedIn: 'root' }) @@ -92,15 +94,11 @@ export class ApiService { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/outspends', { params }); } - requestDonation$(amount: number, orderId: string): Observable { - const params = { - amount: amount, - orderId: orderId, - }; - return this.httpClient.post(this.apiBaseUrl + '/api/v1/donations', params); + getAboutPageProfiles$(): Observable { + return this.httpClient.get(this.apiBaseUrl + '/api/v1/about-page'); } - getDonation$(): Observable { + getOgs$(): Observable { return this.httpClient.get(this.apiBaseUrl + '/api/v1/donations'); } @@ -112,10 +110,6 @@ export class ApiService { return this.httpClient.get(this.apiBaseUrl + '/api/v1/contributors'); } - checkDonation$(orderId: string): Observable { - return this.httpClient.get(this.apiBaseUrl + '/api/v1/donations/check?order_id=' + orderId); - } - getInitData$(): Observable { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/init-data'); } @@ -323,4 +317,13 @@ export class ApiService { (timestamp ? `?timestamp=${timestamp}` : '') ); } + + /** + * Services + */ + getNodeOwner$(publicKey: string) { + let params = new HttpParams() + .set('node_public_key', publicKey); + return this.httpClient.get(`${SERVICES_API_PREFIX}/lightning/claim/current`, { params, observe: 'response' }); + } } diff --git a/frontend/src/app/services/state.service.ts b/frontend/src/app/services/state.service.ts index 9ab8a7e93..675cf88d1 100644 --- a/frontend/src/app/services/state.service.ts +++ b/frontend/src/app/services/state.service.ts @@ -47,6 +47,7 @@ export interface Env { TESTNET_BLOCK_AUDIT_START_HEIGHT: number; SIGNET_BLOCK_AUDIT_START_HEIGHT: number; HISTORICAL_PRICE: boolean; + ACCELERATOR: boolean; } const defaultEnv: Env = { @@ -77,6 +78,7 @@ const defaultEnv: Env = { 'TESTNET_BLOCK_AUDIT_START_HEIGHT': 0, 'SIGNET_BLOCK_AUDIT_START_HEIGHT': 0, 'HISTORICAL_PRICE': true, + 'ACCELERATOR': false, }; @Injectable({ diff --git a/frontend/src/app/services/websocket.service.ts b/frontend/src/app/services/websocket.service.ts index 4bd20e987..af2a15e8c 100644 --- a/frontend/src/app/services/websocket.service.ts +++ b/frontend/src/app/services/websocket.service.ts @@ -28,8 +28,9 @@ export class WebsocketService { private isTrackingTx = false; private trackingTxId: string; private isTrackingMempoolBlock = false; - private isTrackingRbf = false; + private isTrackingRbf: 'all' | 'fullRbf' | false = false; private isTrackingRbfSummary = false; + private isTrackingAddress: string | false = false; private trackingMempoolBlock: number; private latestGitCommit = ''; private onlineCheckTimeout: number; @@ -110,6 +111,15 @@ export class WebsocketService { if (this.isTrackingMempoolBlock) { this.startTrackMempoolBlock(this.trackingMempoolBlock); } + if (this.isTrackingRbf) { + this.startTrackRbf(this.isTrackingRbf); + } + if (this.isTrackingRbfSummary) { + this.startTrackRbfSummary(); + } + if (this.isTrackingAddress) { + this.startTrackAddress(this.isTrackingAddress); + } this.stateService.connectionState$.next(2); } @@ -151,10 +161,12 @@ export class WebsocketService { startTrackAddress(address: string) { this.websocketSubject.next({ 'track-address': address }); + this.isTrackingAddress = address; } stopTrackingAddress() { this.websocketSubject.next({ 'track-address': 'stop' }); + this.isTrackingAddress = false; } startTrackAsset(asset: string) { @@ -178,7 +190,7 @@ export class WebsocketService { startTrackRbf(mode: 'all' | 'fullRbf') { this.websocketSubject.next({ 'track-rbf': mode }); - this.isTrackingRbf = true; + this.isTrackingRbf = mode; } stopTrackRbf() { diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index b1ba84109..d4f303221 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -1,8 +1,11 @@