machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into nymkappa/network-switch-align

Browse Source
This commit is contained in:
wiz 2023-07-18 15:40:57 +09:00 committed by GitHub
commit c1d0e802d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
56 changed files with 7427 additions and 8466 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
backend/mempool-config.sample.json
View File

@ -125,5 +125,16 @@
"LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1",
"BISQ_URL": "https://bisq.markets/api",
"BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api"
},
"REPLICATION": {
"ENABLED": false,
"AUDIT": false,
"AUDIT_START_HEIGHT": 774000,
"SERVERS": [
"list",
"of",
"trusted",
"servers"
]
}
}

6
backend/src/__fixtures__/mempool-config.template.json
View File

@ -121,5 +121,11 @@
},
"CLIGHTNING": {
"SOCKET": "__CLIGHTNING_SOCKET__"
},
"REPLICATION": {
"ENABLED": false,
"AUDIT": false,
"AUDIT_START_HEIGHT": 774000,
"SERVERS": []
}
}

7
backend/src/__tests__/config.test.ts
View File

@ -120,6 +120,13 @@ describe('Mempool Backend Config', () => {
GEOLITE2_ASN: '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
GEOIP2_ISP: '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
});
expect(config.REPLICATION).toStrictEqual({
ENABLED: false,
AUDIT: false,
AUDIT_START_HEIGHT: 774000,
SERVERS: []
});
});
});

11
backend/src/api/audit.ts
View File

@ -1,12 +1,12 @@
import config from '../config';
import logger from '../logger';
import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
import { MempoolTransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
import rbfCache from './rbf-cache';
const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
class Audit {
auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended })
auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended })
: { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], score: number, similarity: number } {
if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], score: 0, similarity: 1 };
@ -14,7 +14,7 @@ class Audit {
const matches: string[] = []; // present in both mined block and template
const added: string[] = []; // present in mined block, not in template
const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN
const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN
const fullrbf: string[] = []; // either missing or present, and part of a fullrbf replacement
const isCensored = {}; // missing, without excuse
const isDisplaced = {};
@ -36,10 +36,13 @@ class Audit {
// look for transactions that were expected in the template, but missing from the mined block
for (const txid of projectedBlocks[0].transactionIds) {
if (!inBlock[txid]) {
// tx is recent, may have reached the miner too late for inclusion
if (rbfCache.isFullRbf(txid)) {
fullrbf.push(txid);
} else if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
// tx is recent, may have reached the miner too late for inclusion
fresh.push(txid);
} else if (mempool[txid]?.lastBoosted != null && (now - (mempool[txid]?.lastBoosted || 0)) <= PROPAGATION_MARGIN) {
// tx was recently cpfp'd, miner may not have the latest effective rate
fresh.push(txid);
} else {
isCensored[txid] = true;

10
backend/src/api/bitcoin/bitcoin-api.ts
View File

@ -65,17 +65,11 @@ class BitcoinApi implements AbstractBitcoinApi {
}
$getBlockHeightTip(): Promise<number> {
return this.bitcoindClient.getChainTips()
.then((result: IBitcoinApi.ChainTips[]) => {
return result.find(tip => tip.status === 'active')!.height;
});
return this.bitcoindClient.getBlockCount();
}
$getBlockHashTip(): Promise<string> {
return this.bitcoindClient.getChainTips()
.then((result: IBitcoinApi.ChainTips[]) => {
return result.find(tip => tip.status === 'active')!.hash;
});
return this.bitcoindClient.getBestBlockHash();
}
$getTxIdsForBlock(hash: string): Promise<string[]> {

16
backend/src/api/bitcoin/bitcoin.routes.ts
View File

@ -121,7 +121,6 @@ class BitcoinRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', this.getBlockHeight)
.get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', this.getAddress)
.get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', this.getAddressTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', this.getAddressTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', this.getAddressPrefix)
;
}
@ -546,27 +545,28 @@ class BitcoinRoutes {
}
}
private async getAddressTransactions(req: Request, res: Response) {
private async getAddressTransactions(req: Request, res: Response): Promise<void> {
if (config.MEMPOOL.BACKEND === 'none') {
res.status(405).send('Address lookups cannot be used with bitcoind as backend.');
return;
}
try {
const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, req.params.txId);
let lastTxId: string = '';
if (req.query.after_txid && typeof req.query.after_txid === 'string') {
lastTxId = req.query.after_txid;
}
const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, lastTxId);
res.json(transactions);
} catch (e) {
if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
return res.status(413).send(e instanceof Error ? e.message : e);
res.status(413).send(e instanceof Error ? e.message : e);
return;
}
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async getAdressTxChain(req: Request, res: Response) {
res.status(501).send('Not implemented');
}
private async getAddressPrefix(req: Request, res: Response) {
try {
const blockHash = await bitcoinApi.$getAddressPrefix(req.params.prefix);

15
backend/src/api/blocks.ts
View File

@ -76,11 +76,14 @@ class Blocks {
blockHash: string,
blockHeight: number,
onlyCoinbase: boolean,
txIds: string[] | null = null,
quiet: boolean = false,
addMempoolData: boolean = false,
): Promise<TransactionExtended[]> {
const transactions: TransactionExtended[] = [];
const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
if (!txIds) {
txIds = await bitcoinApi.$getTxIdsForBlock(blockHash);
}
const mempool = memPool.getMempool();
let transactionsFound = 0;
@ -554,7 +557,7 @@ class Blocks {
}
const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, null, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
newlyIndexed++;
@ -586,7 +589,7 @@ class Blocks {
let fastForwarded = false;
let handledBlocks = 0;
const blockHeightTip = await bitcoinApi.$getBlockHeightTip();
const blockHeightTip = await bitcoinCoreApi.$getBlockHeightTip();
this.updateTimerProgress(timer, 'got block height tip');
if (this.blocks.length === 0) {
@ -639,11 +642,11 @@ class Blocks {
}
this.updateTimerProgress(timer, `getting block data for ${this.currentBlockHeight}`);
const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
const blockHash = await bitcoinCoreApi.$getBlockHash(this.currentBlockHeight);
const verboseBlock = await bitcoinClient.getBlock(blockHash, 2);
const block = BitcoinApi.convertBlock(verboseBlock);
const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, false, false, true) as MempoolTransactionExtended[];
const txIds: string[] = verboseBlock.tx.map(tx => tx.txid);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, false, txIds, false, true) as MempoolTransactionExtended[];
if (config.MEMPOOL.BACKEND !== 'esplora') {
// fill in missing transaction fee data from verboseBlock
for (let i = 0; i < transactions.length; i++) {

7
backend/src/api/database-migration.ts
View File

@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 63;
private static currentVersion = 64;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@ -543,6 +543,11 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fullrbf_txs JSON DEFAULT "[]"');
await this.updateToSchemaVersion(63);
}
if (databaseSchemaVersion < 64 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `nodes` ADD features text NULL');
await this.updateToSchemaVersion(64);
}
}
/**

38
backend/src/api/explorer/nodes.api.ts
View File

@ -3,6 +3,7 @@ import DB from '../../database';
import { ResultSetHeader } from 'mysql2';
import { ILightningApi } from '../lightning/lightning-api.interface';
import { ITopNodesPerCapacity, ITopNodesPerChannels } from '../../mempool.interfaces';
import { bin2hex } from '../../utils/format';
class NodesApi {
public async $getWorldNodes(): Promise<any> {
@ -56,7 +57,8 @@ class NodesApi {
UNIX_TIMESTAMP(updated_at) AS updated_at, color, sockets as sockets,
as_number, city_id, country_id, subdivision_id, longitude, latitude,
geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city,
geo_names_country.names as country, geo_names_subdivision.names as subdivision
geo_names_country.names as country, geo_names_subdivision.names as subdivision,
features
FROM nodes
LEFT JOIN geo_names geo_names_as on geo_names_as.id = as_number
LEFT JOIN geo_names geo_names_city on geo_names_city.id = city_id
@ -76,6 +78,23 @@ class NodesApi {
node.city = JSON.parse(node.city);
node.country = JSON.parse(node.country);
// Features
node.features = JSON.parse(node.features);
node.featuresBits = null;
if (node.features) {
let maxBit = 0;
for (const feature of node.features) {
maxBit = Math.max(maxBit, feature.bit);
}
maxBit = Math.ceil(maxBit / 4) * 4 - 1;
node.featuresBits = new Array(maxBit + 1).fill(0);
for (const feature of node.features) {
node.featuresBits[feature.bit] = 1;
}
node.featuresBits = bin2hex(node.featuresBits.reverse().join(''));
}
// Active channels and capacity
const activeChannelsStats: any = await this.$getActiveChannelsStats(public_key);
node.active_channel_count = activeChannelsStats.active_channel_count ?? 0;
@ -656,10 +675,19 @@ class NodesApi {
alias_search,
color,
sockets,
status
status,
features
)
VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, ?, 1)
ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, alias_search = ?, color = ?, sockets = ?, status = 1`;
VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, ?, 1, ?)
ON DUPLICATE KEY UPDATE
updated_at = FROM_UNIXTIME(?),
alias = ?,
alias_search = ?,
color = ?,
sockets = ?,
status = 1,
features = ?
`;
await DB.query(query, [
node.pub_key,
@ -668,11 +696,13 @@ class NodesApi {
this.aliasToSearchText(node.alias),
node.color,
sockets,
JSON.stringify(node.features),
node.last_update,
node.alias,
this.aliasToSearchText(node.alias),
node.color,
sockets,
JSON.stringify(node.features),
]);
} catch (e) {
logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));

111
backend/src/api/lightning/clightning/clightning-convert.ts
View File

@ -2,8 +2,91 @@ import { ILightningApi } from '../lightning-api.interface';
import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher';
import logger from '../../../logger';
import { Common } from '../../common';
import { hex2bin } from '../../../utils/format';
import config from '../../../config';
// https://github.com/lightningnetwork/lnd/blob/master/lnwire/features.go
export enum FeatureBits {
DataLossProtectRequired = 0,
DataLossProtectOptional = 1,
InitialRoutingSync = 3,
UpfrontShutdownScriptRequired = 4,
UpfrontShutdownScriptOptional = 5,
GossipQueriesRequired = 6,
GossipQueriesOptional = 7,
TLVOnionPayloadRequired = 8,
TLVOnionPayloadOptional = 9,
StaticRemoteKeyRequired = 12,
StaticRemoteKeyOptional = 13,
PaymentAddrRequired = 14,
PaymentAddrOptional = 15,
MPPRequired = 16,
MPPOptional = 17,
WumboChannelsRequired = 18,
WumboChannelsOptional = 19,
AnchorsRequired = 20,
AnchorsOptional = 21,
AnchorsZeroFeeHtlcTxRequired = 22,
AnchorsZeroFeeHtlcTxOptional = 23,
ShutdownAnySegwitRequired = 26,
ShutdownAnySegwitOptional = 27,
AMPRequired = 30,
AMPOptional = 31,
ExplicitChannelTypeRequired = 44,
ExplicitChannelTypeOptional = 45,
ScidAliasRequired = 46,
ScidAliasOptional = 47,
PaymentMetadataRequired = 48,
PaymentMetadataOptional = 49,
ZeroConfRequired = 50,
ZeroConfOptional = 51,
KeysendRequired = 54,
KeysendOptional = 55,
ScriptEnforcedLeaseRequired = 2022,
ScriptEnforcedLeaseOptional = 2023,
MaxBolt11Feature = 5114,
};
export const FeaturesMap = new Map<FeatureBits, string>([
[FeatureBits.DataLossProtectRequired, 'data-loss-protect'],
[FeatureBits.DataLossProtectOptional, 'data-loss-protect'],
[FeatureBits.InitialRoutingSync, 'initial-routing-sync'],
[FeatureBits.UpfrontShutdownScriptRequired, 'upfront-shutdown-script'],
[FeatureBits.UpfrontShutdownScriptOptional, 'upfront-shutdown-script'],
[FeatureBits.GossipQueriesRequired, 'gossip-queries'],
[FeatureBits.GossipQueriesOptional, 'gossip-queries'],
[FeatureBits.TLVOnionPayloadRequired, 'tlv-onion'],
[FeatureBits.TLVOnionPayloadOptional, 'tlv-onion'],
[FeatureBits.StaticRemoteKeyOptional, 'static-remote-key'],
[FeatureBits.StaticRemoteKeyRequired, 'static-remote-key'],
[FeatureBits.PaymentAddrOptional, 'payment-addr'],
[FeatureBits.PaymentAddrRequired, 'payment-addr'],
[FeatureBits.MPPOptional, 'multi-path-payments'],
[FeatureBits.MPPRequired, 'multi-path-payments'],
[FeatureBits.AnchorsRequired, 'anchor-commitments'],
[FeatureBits.AnchorsOptional, 'anchor-commitments'],
[FeatureBits.AnchorsZeroFeeHtlcTxRequired, 'anchors-zero-fee-htlc-tx'],
[FeatureBits.AnchorsZeroFeeHtlcTxOptional, 'anchors-zero-fee-htlc-tx'],
[FeatureBits.WumboChannelsRequired, 'wumbo-channels'],
[FeatureBits.WumboChannelsOptional, 'wumbo-channels'],
[FeatureBits.AMPRequired, 'amp'],
[FeatureBits.AMPOptional, 'amp'],
[FeatureBits.PaymentMetadataOptional, 'payment-metadata'],
[FeatureBits.PaymentMetadataRequired, 'payment-metadata'],
[FeatureBits.ExplicitChannelTypeOptional, 'explicit-commitment-type'],
[FeatureBits.ExplicitChannelTypeRequired, 'explicit-commitment-type'],
[FeatureBits.KeysendOptional, 'keysend'],
[FeatureBits.KeysendRequired, 'keysend'],
[FeatureBits.ScriptEnforcedLeaseRequired, 'script-enforced-lease'],
[FeatureBits.ScriptEnforcedLeaseOptional, 'script-enforced-lease'],
[FeatureBits.ScidAliasRequired, 'scid-alias'],
[FeatureBits.ScidAliasOptional, 'scid-alias'],
[FeatureBits.ZeroConfRequired, 'zero-conf'],
[FeatureBits.ZeroConfOptional, 'zero-conf'],
[FeatureBits.ShutdownAnySegwitRequired, 'shutdown-any-segwit'],
[FeatureBits.ShutdownAnySegwitOptional, 'shutdown-any-segwit'],
]);
/**
* Convert a clightning "listnode" entry to a lnd node entry
*/
@ -17,10 +100,36 @@ export function convertNode(clNode: any): ILightningApi.Node {
custom_records = undefined;
}
}
const nodeFeatures: ILightningApi.Feature[] = [];
const nodeFeaturesBinary = hex2bin(clNode.features).split('').reverse().join('');
for (let i = 0; i < nodeFeaturesBinary.length; i++) {
if (nodeFeaturesBinary[i] === '0') {
continue;
}
const feature = FeaturesMap.get(i);
if (!feature) {
nodeFeatures.push({
bit: i,
name: 'unknown',
is_required: i % 2 === 0,
is_known: false
});
} else {
nodeFeatures.push({
bit: i,
name: feature,
is_required: i % 2 === 0,
is_known: true
});
}
}
return {
alias: clNode.alias ?? '',
color: `#${clNode.color ?? ''}`,
features: [], // TODO parse and return clNode.feature
features: nodeFeatures,
pub_key: clNode.nodeid,
addresses: clNode.addresses?.map((addr) => {
let address = addr.address;

1
backend/src/api/lightning/lightning-api.interface.ts
View File

@ -79,6 +79,7 @@ export namespace ILightningApi {
}
export interface Feature {
bit: number;
name: string;
is_required: boolean;
is_known: boolean;

17
backend/src/api/lightning/lnd/lnd-api.ts
View File

@ -41,8 +41,23 @@ class LndApi implements AbstractLightningApi {
}
async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
return axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
const graph = await axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
.then((response) => response.data);
for (const node of graph.nodes) {
const nodeFeatures: ILightningApi.Feature[] = [];
for (const bit in node.features) {
nodeFeatures.push({
bit: parseInt(bit, 10),
name: node.features[bit].name,
is_required: node.features[bit].is_required,
is_known: node.features[bit].is_known,
});
}
node.features = nodeFeatures;
}
return graph;
}
}

1
backend/src/api/mempool-blocks.ts
View File

@ -457,6 +457,7 @@ class MempoolBlocks {
};
if (matched) {
descendants.push(relative);
mempoolTx.lastBoosted = Math.max(mempoolTx.lastBoosted || 0, mempool[txid].firstSeen || 0);
} else {
ancestors.push(relative);
}

6
backend/src/api/websocket-handler.ts
View File

@ -229,14 +229,16 @@ class WebsocketHandler {
if (parsedMessage && parsedMessage['track-rbf-summary'] != null) {
if (parsedMessage['track-rbf-summary']) {
client['track-rbf-summary'] = true;
if (this.socketData['rbfSummary'] != null) {
response['rbfLatestSummary'] = this.socketData['rbfSummary'];
}
} else {
client['track-rbf-summary'] = false;
}
}
if (parsedMessage.action === 'init') {
if (!this.socketData['blocks']?.length || !this.socketData['da']) {
if (!this.socketData['blocks']?.length || !this.socketData['da'] || !this.socketData['backendInfo'] || !this.socketData['conversions']) {
this.updateSocketData();
}
if (!this.socketData['blocks']?.length) {
@ -419,7 +421,7 @@ class WebsocketHandler {
memPool.addToSpendMap(newTransactions);
const recommendedFees = feeApi.getRecommendedFee();
const latestTransactions = newTransactions.slice(0, 6).map((tx) => Common.stripTransaction(tx));
const latestTransactions = memPool.getLatestTransactions();
// update init data
const socketDataFields = {

14
backend/src/config.ts
View File

@ -132,6 +132,12 @@ interface IConfig {
GEOLITE2_ASN: string;
GEOIP2_ISP: string;
},
REPLICATION: {
ENABLED: boolean;
AUDIT: boolean;
AUDIT_START_HEIGHT: number;
SERVERS: string[];
}
}
const defaults: IConfig = {
@ -264,6 +270,12 @@ const defaults: IConfig = {
'GEOLITE2_ASN': '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
'GEOIP2_ISP': '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
},
'REPLICATION': {
'ENABLED': false,
'AUDIT': false,
'AUDIT_START_HEIGHT': 774000,
'SERVERS': [],
}
};
class Config implements IConfig {
@ -283,6 +295,7 @@ class Config implements IConfig {
PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
MAXMIND: IConfig['MAXMIND'];
REPLICATION: IConfig['REPLICATION'];
constructor() {
const configs = this.merge(configFromFile, defaults);
@ -302,6 +315,7 @@ class Config implements IConfig {
this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
this.MAXMIND = configs.MAXMIND;
this.REPLICATION = configs.REPLICATION;
}
merge = (...objects: object[]): IConfig => {

5
backend/src/index.ts
View File

@ -169,6 +169,7 @@ class Server {
}
async runMainUpdateLoop(): Promise<void> {
const start = Date.now();
try {
try {
await memPool.$updateMemPoolInfo();
@ -188,7 +189,9 @@ class Server {
indexer.$run();
// rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS
setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 1 : config.MEMPOOL.POLL_RATE_MS);
const elapsed = Date.now() - start;
const remainingTime = Math.max(0, config.MEMPOOL.POLL_RATE_MS - elapsed)
setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 0 : remainingTime);
this.backendRetryCount = 0;
} catch (e: any) {
this.backendRetryCount++;

2
backend/src/indexer.ts
View File

@ -7,6 +7,7 @@ import bitcoinClient from './api/bitcoin/bitcoin-client';
import priceUpdater from './tasks/price-updater';
import PricesRepository from './repositories/PricesRepository';
import config from './config';
import auditReplicator from './replication/AuditReplication';
export interface CoreIndex {
name: string;
@ -136,6 +137,7 @@ class Indexer {
await blocks.$generateBlocksSummariesDatabase();
await blocks.$generateCPFPDatabase();
await blocks.$generateAuditStats();
await auditReplicator.$sync();
} catch (e) {
this.indexerRunning = false;
logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));

10
backend/src/mempool.interfaces.ts
View File

@ -100,6 +100,7 @@ export interface MempoolTransactionExtended extends TransactionExtended {
adjustedVsize: number;
adjustedFeePerVsize: number;
inputs?: number[];
lastBoosted?: number;
}
export interface AuditTransaction {
@ -236,6 +237,15 @@ export interface BlockSummary {
transactions: TransactionStripped[];
}
export interface AuditSummary extends BlockAudit {
timestamp?: number,
size?: number,
weight?: number,
tx_count?: number,
transactions: TransactionStripped[];
template?: TransactionStripped[];
}
export interface BlockPrice {
height: number;
priceId: number;

134
backend/src/replication/AuditReplication.ts Normal file
View File

@ -0,0 +1,134 @@
import DB from '../database';
import logger from '../logger';
import { AuditSummary } from '../mempool.interfaces';
import blocksAuditsRepository from '../repositories/BlocksAuditsRepository';
import blocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import { $sync } from './replicator';
import config from '../config';
import { Common } from '../api/common';
import blocks from '../api/blocks';
const BATCH_SIZE = 16;
/**
* Syncs missing block template and audit data from trusted servers
*/
class AuditReplication {
inProgress: boolean = false;
skip: Set<string> = new Set();
public async $sync(): Promise<void> {
if (!config.REPLICATION.ENABLED || !config.REPLICATION.AUDIT) {
// replication not enabled
return;
}
if (this.inProgress) {
logger.info(`AuditReplication sync already in progress`, 'Replication');
return;
}
this.inProgress = true;
const missingAudits = await this.$getMissingAuditBlocks();
logger.debug(`Fetching missing audit data for ${missingAudits.length} blocks from trusted servers`, 'Replication');
let totalSynced = 0;
let totalMissed = 0;
let loggerTimer = Date.now();
// process missing audits in batches of
for (let i = 0; i < missingAudits.length; i += BATCH_SIZE) {
const slice = missingAudits.slice(i, i + BATCH_SIZE);
const results = await Promise.all(slice.map(hash => this.$syncAudit(hash)));
const synced = results.reduce((total, status) => status ? total + 1 : total, 0);
totalSynced += synced;
totalMissed += (slice.length - synced);
if (Date.now() - loggerTimer > 10000) {
loggerTimer = Date.now();
logger.info(`Found ${totalSynced} / ${totalSynced + totalMissed} of ${missingAudits.length} missing audits`, 'Replication');
}
await Common.sleep$(1000);
}
logger.debug(`Fetched ${totalSynced} audits, ${totalMissed} still missing`, 'Replication');
this.inProgress = false;
}
private async $syncAudit(hash: string): Promise<boolean> {
if (this.skip.has(hash)) {
// we already know none of our trusted servers have this audit
return false;
}
let success = false;
// start with a random server so load is uniformly spread
const syncResult = await $sync(`/api/v1/block/${hash}/audit-summary`);
if (syncResult) {
if (syncResult.data?.template?.length) {
await this.$saveAuditData(hash, syncResult.data);
logger.info(`Imported audit data from ${syncResult.server} for block ${syncResult.data.height} (${hash})`);
success = true;
}
if (!syncResult.data && !syncResult.exists) {
this.skip.add(hash);
}
}
return success;
}
private async $getMissingAuditBlocks(): Promise<string[]> {
try {
const startHeight = config.REPLICATION.AUDIT_START_HEIGHT || 0;
const [rows]: any[] = await DB.query(`
SELECT auditable.hash, auditable.height
FROM (
SELECT hash, height
FROM blocks
WHERE height >= ?
) AS auditable
LEFT JOIN blocks_audits ON auditable.hash = blocks_audits.hash
WHERE blocks_audits.hash IS NULL
ORDER BY auditable.height DESC
`, [startHeight]);
return rows.map(row => row.hash);
} catch (e: any) {
logger.err(`Cannot fetch missing audit blocks from db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
private async $saveAuditData(blockHash: string, auditSummary: AuditSummary): Promise<void> {
// save audit & template to DB
await blocksSummariesRepository.$saveTemplate({
height: auditSummary.height,
template: {
id: blockHash,
transactions: auditSummary.template || []
}
});
await blocksAuditsRepository.$saveAudit({
hash: blockHash,
height: auditSummary.height,
time: auditSummary.timestamp || auditSummary.time,
missingTxs: auditSummary.missingTxs || [],
addedTxs: auditSummary.addedTxs || [],
freshTxs: auditSummary.freshTxs || [],
sigopTxs: auditSummary.sigopTxs || [],
fullrbfTxs: auditSummary.fullrbfTxs || [],
matchRate: auditSummary.matchRate,
expectedFees: auditSummary.expectedFees,
expectedWeight: auditSummary.expectedWeight,
});
// add missing data to cached blocks
const cachedBlock = blocks.getBlocks().find(block => block.id === blockHash);
if (cachedBlock) {
cachedBlock.extras.matchRate = auditSummary.matchRate;
cachedBlock.extras.expectedFees = auditSummary.expectedFees || null;
cachedBlock.extras.expectedWeight = auditSummary.expectedWeight || null;
}
}
}
export default new AuditReplication();

70
backend/src/replication/replicator.ts Normal file
View File

@ -0,0 +1,70 @@
import config from '../config';
import backendInfo from '../api/backend-info';
import axios, { AxiosResponse } from 'axios';
import { SocksProxyAgent } from 'socks-proxy-agent';
import * as https from 'https';
export async function $sync(path): Promise<{ data?: any, exists: boolean, server?: string }> {
// start with a random server so load is uniformly spread
let allMissing = true;
const offset = Math.floor(Math.random() * config.REPLICATION.SERVERS.length);
for (let i = 0; i < config.REPLICATION.SERVERS.length; i++) {
const server = config.REPLICATION.SERVERS[(i + offset) % config.REPLICATION.SERVERS.length];
// don't query ourself
if (server === backendInfo.getBackendInfo().hostname) {
continue;
}
try {
const result = await query(`https://${server}${path}`);
if (result) {
return { data: result, exists: true, server };
}
} catch (e: any) {
if (e?.response?.status === 404) {
// this server is also missing this data
} else {
// something else went wrong
allMissing = false;
}
}
}
return { exists: !allMissing };
}
export async function query(path): Promise<object> {
type axiosOptions = {
headers: {
'User-Agent': string
};
timeout: number;
httpsAgent?: https.Agent;
};
const axiosOptions: axiosOptions = {
headers: {
'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
},
timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
};
if (config.SOCKS5PROXY.ENABLED) {
const socksOptions = {
agentOptions: {
keepAlive: true,
},
hostname: config.SOCKS5PROXY.HOST,
port: config.SOCKS5PROXY.PORT,
username: config.SOCKS5PROXY.USERNAME || 'circuit0',
password: config.SOCKS5PROXY.PASSWORD,
};
axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
}
const data: AxiosResponse = await axios.get(path, axiosOptions);
if (data.statusText === 'error' || !data.data) {
throw new Error(`${data.status}`);
}
return data.data;
}

1
backend/src/tasks/lightning/network-sync.service.ts
View File

@ -3,7 +3,6 @@ import logger from '../../logger';
import channelsApi from '../../api/explorer/channels.api';
import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
import config from '../../config';
import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
import { ILightningApi } from '../../api/lightning/lightning-api.interface';
import { $lookupNodeLocation } from './sync-tasks/node-locations';
import lightningApi from '../../api/lightning/lightning-api-factory';

1
backend/src/tasks/price-updater.ts
View File

@ -153,6 +153,7 @@ class PriceUpdater {
try {
const p = 60 * 60 * 1000; // milliseconds in an hour
const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042
this.latestPrices.time = nowRounded.getTime() / 1000;
await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices);
} catch (e) {
this.lastRun = previousRun + 5 * 60;

66
backend/src/utils/format.ts
View File

@ -27,3 +27,69 @@ export function formatBytes(bytes: number, toUnit: string, skipUnit = false): st
return `${bytes.toFixed(2)}${skipUnit ? '' : ' ' + byteUnits[unitIndex]}`;
}
// https://stackoverflow.com/a/64235212
export function hex2bin(hex: string): string {
if (!hex) {
return '';
}
hex = hex.replace('0x', '').toLowerCase();
let out = '';
for (const c of hex) {
switch (c) {
case '0': out += '0000'; break;
case '1': out += '0001'; break;
case '2': out += '0010'; break;
case '3': out += '0011'; break;
case '4': out += '0100'; break;
case '5': out += '0101'; break;
case '6': out += '0110'; break;
case '7': out += '0111'; break;
case '8': out += '1000'; break;
case '9': out += '1001'; break;
case 'a': out += '1010'; break;
case 'b': out += '1011'; break;
case 'c': out += '1100'; break;
case 'd': out += '1101'; break;
case 'e': out += '1110'; break;
case 'f': out += '1111'; break;
default: return '';
}
}
return out;
}
export function bin2hex(bin: string): string {
if (!bin) {
return '';
}
let out = '';
for (let i = 0; i < bin.length; i += 4) {
const c = bin.substring(i, i + 4);
switch (c) {
case '0000': out += '0'; break;
case '0001': out += '1'; break;
case '0010': out += '2'; break;
case '0011': out += '3'; break;
case '0100': out += '4'; break;
case '0101': out += '5'; break;
case '0110': out += '6'; break;
case '0111': out += '7'; break;
case '1000': out += '8'; break;
case '1001': out += '9'; break;
case '1010': out += 'a'; break;
case '1011': out += 'b'; break;
case '1100': out += 'c'; break;
case '1101': out += 'd'; break;
case '1110': out += 'e'; break;
case '1111': out += 'f'; break;
default: return '';
}
}
return out;
}

6
docker/backend/mempool-config.json
View File

@ -127,5 +127,11 @@
"GEOLITE2_CITY": "__MAXMIND_GEOLITE2_CITY__",
"GEOLITE2_ASN": "__MAXMIND_GEOLITE2_ASN__",
"GEOIP2_ISP": "__MAXMIND_GEOIP2_ISP__"
},
"REPLICATION": {
"ENABLED": __REPLICATION_ENABLED__,
"AUDIT": __REPLICATION_AUDIT__,
"AUDIT_START_HEIGHT": __REPLICATION_AUDIT_START_HEIGHT__,
"SERVERS": __REPLICATION_SERVERS__
}
}

11
docker/backend/start.sh
View File

@ -130,6 +130,12 @@ __MAXMIND_GEOLITE2_CITY__=${MAXMIND_GEOLITE2_CITY:="/backend/GeoIP/GeoLite2-City
__MAXMIND_GEOLITE2_ASN__=${MAXMIND_GEOLITE2_ASN:="/backend/GeoIP/GeoLite2-ASN.mmdb"}
__MAXMIND_GEOIP2_ISP__=${MAXMIND_GEOIP2_ISP:=""}
# REPLICATION
__REPLICATION_ENABLED__=${REPLICATION_ENABLED:=true}
__REPLICATION_AUDIT__=${REPLICATION_AUDIT:=true}
__REPLICATION_AUDIT_START_HEIGHT__=${REPLICATION_AUDIT_START_HEIGHT:=774000}
__REPLICATION_SERVERS__=${REPLICATION_SERVERS:=[]}
mkdir -p "${__MEMPOOL_CACHE_DIR__}"
@ -250,5 +256,10 @@ sed -i "s!__MAXMIND_GEOLITE2_CITY__!${__MAXMIND_GEOLITE2_CITY__}!g" mempool-conf
sed -i "s!__MAXMIND_GEOLITE2_ASN__!${__MAXMIND_GEOLITE2_ASN__}!g" mempool-config.json
sed -i "s!__MAXMIND_GEOIP2_ISP__!${__MAXMIND_GEOIP2_ISP__}!g" mempool-config.json
# REPLICATION
sed -i "s!__REPLICATION_ENABLED__!${__REPLICATION_ENABLED__}!g" mempool-config.json
sed -i "s!__REPLICATION_AUDIT__!${__REPLICATION_AUDIT__}!g" mempool-config.json
sed -i "s!__REPLICATION_AUDIT_START_HEIGHT__!${__REPLICATION_AUDIT_START_HEIGHT__}!g" mempool-config.json
sed -i "s!__REPLICATION_SERVERS__!${__REPLICATION_SERVERS__}!g" mempool-config.json
node /backend/package/index.js

2
docker/frontend/entrypoint.sh
View File

@ -39,7 +39,6 @@ __AUDIT__=${AUDIT:=false}
__MAINNET_BLOCK_AUDIT_START_HEIGHT__=${MAINNET_BLOCK_AUDIT_START_HEIGHT:=0}
__TESTNET_BLOCK_AUDIT_START_HEIGHT__=${TESTNET_BLOCK_AUDIT_START_HEIGHT:=0}
__SIGNET_BLOCK_AUDIT_START_HEIGHT__=${SIGNET_BLOCK_AUDIT_START_HEIGHT:=0}
__FULL_RBF_ENABLED__=${FULL_RBF_ENABLED:=false}
__HISTORICAL_PRICE__=${HISTORICAL_PRICE:=true}
# Export as environment variables to be used by envsubst
@ -66,7 +65,6 @@ export __AUDIT__
export __MAINNET_BLOCK_AUDIT_START_HEIGHT__
export __TESTNET_BLOCK_AUDIT_START_HEIGHT__
export __SIGNET_BLOCK_AUDIT_START_HEIGHT__
export __FULL_RBF_ENABLED__
export __HISTORICAL_PRICE__
folder=$(find /var/www/mempool -name "config.js" | xargs dirname)

1
frontend/mempool-frontend-config.sample.json
View File

@ -22,6 +22,5 @@
"TESTNET_BLOCK_AUDIT_START_HEIGHT": 0,
"SIGNET_BLOCK_AUDIT_START_HEIGHT": 0,
"LIGHTNING": false,
"FULL_RBF_ENABLED": false,
"HISTORICAL_PRICE": true
}

15027
frontend/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

68
frontend/package.json
View File

@ -61,60 +61,60 @@
"cypress:run:ci:staging": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 && npm run generate-config && start-server-and-test serve:local-staging 4200 cypress:run:record"
},
"dependencies": {
"@angular-devkit/build-angular": "^14.2.10",
"@angular/animations": "^14.2.12",
"@angular/cli": "^14.2.10",
"@angular/common": "^14.2.12",
"@angular/compiler": "^14.2.12",
"@angular/core": "^14.2.12",
"@angular/forms": "^14.2.12",
"@angular/localize": "^14.2.12",
"@angular/platform-browser": "^14.2.12",
"@angular/platform-browser-dynamic": "^14.2.12",
"@angular/platform-server": "^14.2.12",
"@angular/router": "^14.2.12",
"@fortawesome/angular-fontawesome": "~0.11.1",
"@fortawesome/fontawesome-common-types": "~6.2.1",
"@fortawesome/fontawesome-svg-core": "~6.2.1",
"@fortawesome/free-solid-svg-icons": "~6.2.1",
"@angular-devkit/build-angular": "^16.1.4",
"@angular/animations": "^16.1.5",
"@angular/cli": "^16.1.4",
"@angular/common": "^16.1.5",
"@angular/compiler": "^16.1.5",
"@angular/core": "^16.1.5",
"@angular/forms": "^16.1.5",
"@angular/localize": "^16.1.5",
"@angular/platform-browser": "^16.1.5",
"@angular/platform-browser-dynamic": "^16.1.5",
"@angular/platform-server": "^16.1.5",
"@angular/router": "^16.1.5",
"@fortawesome/angular-fontawesome": "~0.13.0",
"@fortawesome/fontawesome-common-types": "~6.4.0",
"@fortawesome/fontawesome-svg-core": "~6.4.0",
"@fortawesome/free-solid-svg-icons": "~6.4.0",
"@mempool/mempool.js": "2.3.0",
"@ng-bootstrap/ng-bootstrap": "^13.1.1",
"@ng-bootstrap/ng-bootstrap": "^15.1.0",
"@types/qrcode": "~1.5.0",
"bootstrap": "~4.6.1",
"bootstrap": "~4.6.2",
"browserify": "^17.0.0",
"clipboard": "^2.0.11",
"domino": "^2.1.6",
"echarts": "~5.4.1",
"echarts": "~5.4.3",
"echarts-gl": "^2.0.9",
"lightweight-charts": "~3.8.0",
"ngx-echarts": "~14.0.0",
"ngx-infinite-scroll": "^14.0.1",
"ngx-echarts": "~16.0.0",
"ngx-infinite-scroll": "^16.0.0",
"qrcode": "1.5.1",
"rxjs": "~7.8.0",
"tinyify": "^3.1.0",
"rxjs": "~7.8.1",
"tinyify": "^4.0.0",
"tlite": "^0.1.9",
"tslib": "~2.4.1",
"zone.js": "~0.12.0"
"tslib": "~2.6.0",
"zone.js": "~0.13.1"
},
"devDependencies": {
"@angular/compiler-cli": "^14.2.12",
"@angular/language-service": "^14.2.12",
"@angular/compiler-cli": "^16.1.5",
"@angular/language-service": "^16.1.5",
"@types/node": "^18.11.9",
"@typescript-eslint/eslint-plugin": "^5.48.1",
"@typescript-eslint/parser": "^5.48.1",
"eslint": "^8.31.0",
"http-proxy-middleware": "~2.0.6",
"prettier": "^2.8.2",
"prettier": "^3.0.0",
"ts-node": "~10.9.1",
"typescript": "~4.6.4"
"typescript": "~4.9.3"
},
"optionalDependencies": {
"@cypress/schematic": "^2.4.0",
"cypress": "^12.7.0",
"cypress-fail-on-console-error": "~4.0.2",
"@cypress/schematic": "^2.5.0",
"cypress": "^12.17.1",
"cypress-fail-on-console-error": "~4.0.3",
"cypress-wait-until": "^1.7.2",
"mock-socket": "~9.1.5",
"start-server-and-test": "~1.14.0"
"mock-socket": "~9.2.1",
"start-server-and-test": "~2.0.0"
},
"scarfSettings": {
"enabled": false

5
frontend/src/app/app.module.ts
View File

@ -1,4 +1,4 @@
import { BrowserModule, BrowserTransferStateModule } from '@angular/platform-browser';
import { BrowserModule } from '@angular/platform-browser';
import { ModuleWithProviders, NgModule } from '@angular/core';
import { HttpClientModule, HTTP_INTERCEPTORS } from '@angular/common/http';
import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
@ -48,8 +48,7 @@ const providers = [
AppComponent,
],
imports: [
BrowserModule.withServerTransition({ appId: 'serverApp' }),
BrowserTransferStateModule,
BrowserModule,
AppRoutingModule,
HttpClientModule,
BrowserAnimationsModule,

6
frontend/src/app/components/address/address.component.ts
View File

@ -207,7 +207,7 @@ export class AddressComponent implements OnInit, OnDestroy {
}
this.isLoadingTransactions = true;
this.retryLoadMore = false;
this.electrsApiService.getAddressTransactionsFromHash$(this.address.address, this.lastTransactionTxId)
this.electrsApiService.getAddressTransactions$(this.address.address, this.lastTransactionTxId)
.subscribe((transactions: Transaction[]) => {
this.lastTransactionTxId = transactions[transactions.length - 1].txid;
this.loadedConfirmedTxCount += transactions.length;
@ -217,6 +217,10 @@ export class AddressComponent implements OnInit, OnDestroy {
(error) => {
this.isLoadingTransactions = false;
this.retryLoadMore = true;
// In the unlikely event of the txid wasn't found in the mempool anymore and we must reload the page.
if (error.status === 422) {
window.location.reload();
}
});
}

3
frontend/src/app/components/block-overview-graph/tx-view.ts
View File

@ -38,7 +38,7 @@ export default class TxView implements TransactionStripped {
value: number;
feerate: number;
rate?: number;
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf';
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'fullrbf';
context?: 'projected' | 'actual';
scene?: BlockScene;
@ -210,6 +210,7 @@ export default class TxView implements TransactionStripped {
case 'fullrbf':
return marginalFeeColors[feeLevelIndex] || marginalFeeColors[mempoolFeeColors.length - 1];
case 'fresh':
case 'freshcpfp':
return auditColors.missing;
case 'added':
return auditColors.added;

1
frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html
View File

@ -50,6 +50,7 @@
<td *ngSwitchCase="'missing'"><span class="badge badge-warning" i18n="transaction.audit.marginal">Marginal fee rate</span></td>
<td *ngSwitchCase="'sigop'"><span class="badge badge-warning" i18n="transaction.audit.sigop">High sigop count</span></td>
<td *ngSwitchCase="'fresh'"><span class="badge badge-warning" i18n="transaction.audit.recently-broadcasted">Recently broadcasted</span></td>
<td *ngSwitchCase="'freshcpfp'"><span class="badge badge-warning" i18n="transaction.audit.recently-cpfped">Recently CPFP'd</span></td>
<td *ngSwitchCase="'added'"><span class="badge badge-warning" i18n="transaction.audit.added">Added</span></td>
<td *ngSwitchCase="'selected'"><span class="badge badge-warning" i18n="transaction.audit.marginal">Marginal fee rate</span></td>
<td *ngSwitchCase="'fullrbf'"><span class="badge badge-warning" i18n="transaction.audit.fullrbf">Full RBF</span></td>

4
frontend/src/app/components/block/block.component.ts
View File

@ -370,7 +370,11 @@ export class BlockComponent implements OnInit, OnDestroy {
tx.status = 'found';
} else {
if (isFresh[tx.txid]) {
if (tx.rate - (tx.fee / tx.vsize) >= 0.1) {
tx.status = 'freshcpfp';
} else {
tx.status = 'fresh';
}
} else if (isSigop[tx.txid]) {
tx.status = 'sigop';
} else if (isFullRbf[tx.txid]) {

6
frontend/src/app/components/calculator/calculator.component.html
View File

@ -12,7 +12,7 @@
<div class="input-group-prepend">
<span class="input-group-text">{{ currency$ | async }}</span>
</div>
<input type="text" class="form-control" formControlName="fiat" (input)="transformInput('fiat')">
<input type="text" class="form-control" formControlName="fiat" (input)="transformInput('fiat')" (click)="selectAll($event)">
<app-clipboard [button]="true" [text]="form.get('fiat').value" [class]="'btn btn-lg btn-secondary ml-1'"></app-clipboard>
</div>
@ -20,7 +20,7 @@
<div class="input-group-prepend">
<span class="input-group-text">BTC</span>
</div>
<input type="text" class="form-control" formControlName="bitcoin" (input)="transformInput('bitcoin')">
<input type="text" class="form-control" formControlName="bitcoin" (input)="transformInput('bitcoin')" (click)="selectAll($event)">
<app-clipboard [button]="true" [text]="form.get('bitcoin').value" [class]="'btn btn-lg btn-secondary ml-1'"></app-clipboard>
</div>
@ -28,7 +28,7 @@
<div class="input-group-prepend">
<span class="input-group-text">sats</span>
</div>
<input type="text" class="form-control" formControlName="satoshis" (input)="transformInput('satoshis')">
<input type="text" class="form-control" formControlName="satoshis" (input)="transformInput('satoshis')" (click)="selectAll($event)">
<app-clipboard [button]="true" [text]="form.get('satoshis').value" [class]="'btn btn-lg btn-secondary ml-1'"></app-clipboard>
</div>
</form>

4
frontend/src/app/components/calculator/calculator.component.scss
View File

@ -24,3 +24,7 @@
font-size: 20px;
margin-left: 5px;
}
.row {
margin: auto;
}

36
frontend/src/app/components/calculator/calculator.component.ts
View File

@ -54,6 +54,9 @@ export class CalculatorComponent implements OnInit {
]).subscribe(([price, value]) => {
const rate = (value / price).toFixed(8);
const satsRate = Math.round(value / price * 100_000_000);
if (isNaN(value)) {
return;
}
this.form.get('bitcoin').setValue(rate, { emitEvent: false });
this.form.get('satoshis').setValue(satsRate, { emitEvent: false } );
});
@ -63,6 +66,9 @@ export class CalculatorComponent implements OnInit {
this.form.get('bitcoin').valueChanges
]).subscribe(([price, value]) => {
const rate = parseFloat((value * price).toFixed(8));
if (isNaN(value)) {
return;
}
this.form.get('fiat').setValue(rate, { emitEvent: false } );
this.form.get('satoshis').setValue(Math.round(value * 100_000_000), { emitEvent: false } );
});
@ -73,6 +79,9 @@ export class CalculatorComponent implements OnInit {
]).subscribe(([price, value]) => {
const rate = parseFloat((value / 100_000_000 * price).toFixed(8));
const bitcoinRate = (value / 100_000_000).toFixed(8);
if (isNaN(value)) {
return;
}
this.form.get('fiat').setValue(rate, { emitEvent: false } );
this.form.get('bitcoin').setValue(bitcoinRate, { emitEvent: false });
});
@ -88,7 +97,16 @@ export class CalculatorComponent implements OnInit {
if (value === '.') {
value = '0';
}
const sanitizedValue = this.removeExtraDots(value);
let sanitizedValue = this.removeExtraDots(value);
if (name === 'bitcoin' && this.countDecimals(sanitizedValue) > 8) {
sanitizedValue = this.toFixedWithoutRounding(sanitizedValue, 8);
}
if (sanitizedValue === '') {
sanitizedValue = '0';
}
if (name === 'satoshis') {
sanitizedValue = parseFloat(sanitizedValue).toFixed(0);
}
formControl.setValue(sanitizedValue, {emitEvent: true});
}
@ -100,4 +118,20 @@ export class CalculatorComponent implements OnInit {
const afterDotReplaced = afterDot.replace(/\./g, '');
return `${beforeDot}.${afterDotReplaced}`;
}
countDecimals(numberString: string): number {
const decimalPos = numberString.indexOf('.');
if (decimalPos === -1) return 0;
return numberString.length - decimalPos - 1;
}
toFixedWithoutRounding(numStr: string, fixed: number): string {
const re = new RegExp(`^-?\\d+(?:.\\d{0,${(fixed || -1)}})?`);
const result = numStr.match(re);
return result ? result[0] : numStr;
}
selectAll(event): void {
event.target.select();
}
}

1
frontend/src/app/components/clock/clock.component.scss
View File

@ -9,6 +9,7 @@
display: flex;
flex-direction: column;
justify-content: flex-start;
overflow: hidden;
--chain-height: 60px;
--clock-width: 300px;

6
frontend/src/app/components/pool/pool.component.ts
View File

@ -37,7 +37,7 @@ export class PoolComponent implements OnInit {
auditAvailable = false;
loadMoreSubject: BehaviorSubject<number> = new BehaviorSubject(this.blocks[0]?.height);
loadMoreSubject: BehaviorSubject<number> = new BehaviorSubject(this.blocks[this.blocks.length - 1]?.height);
constructor(
@Inject(LOCALE_ID) public locale: string,
@ -91,7 +91,7 @@ export class PoolComponent implements OnInit {
if (this.slug === undefined) {
return [];
}
return this.apiService.getPoolBlocks$(this.slug, this.blocks[0]?.height);
return this.apiService.getPoolBlocks$(this.slug, this.blocks[this.blocks.length - 1]?.height);
}),
tap((newBlocks) => {
this.blocks = this.blocks.concat(newBlocks);
@ -237,7 +237,7 @@ export class PoolComponent implements OnInit {
}
loadMore() {
this.loadMoreSubject.next(this.blocks[0]?.height);
this.loadMoreSubject.next(this.blocks[this.blocks.length - 1]?.height);
}
trackByBlock(index: number, block: BlockExtended) {

2
frontend/src/app/components/rbf-list/rbf-list.component.html
View File

@ -2,7 +2,7 @@
<h1 class="float-left" i18n="page.rbf-replacements">RBF Replacements</h1>
<div *ngIf="isLoading" class="spinner-border ml-3" role="status"></div>
<div class="mode-toggle float-right" *ngIf="fullRbfEnabled">
<div class="mode-toggle float-right">
<form class="formRadioGroup">
<div class="btn-group btn-group-toggle" name="radioBasic">
<label class="btn btn-primary btn-sm" [class.active]="!fullRbf">

5
frontend/src/app/components/rbf-list/rbf-list.component.ts
View File

@ -17,7 +17,6 @@ export class RbfList implements OnInit, OnDestroy {
rbfTrees$: Observable<RbfTree[]>;
nextRbfSubject = new BehaviorSubject(null);
urlFragmentSubscription: Subscription;
fullRbfEnabled: boolean;
fullRbf: boolean;
isLoading = true;
@ -27,9 +26,7 @@ export class RbfList implements OnInit, OnDestroy {
private apiService: ApiService,
public stateService: StateService,
private websocketService: WebsocketService,
) {
this.fullRbfEnabled = stateService.env.FULL_RBF_ENABLED;
}
) { }
ngOnInit(): void {
this.urlFragmentSubscription = this.route.fragment.subscribe((fragment) => {

1
frontend/src/app/components/transaction/transaction.component.html
View File

@ -19,6 +19,7 @@
<div class="container-buttons">
<app-confirmations
*ngIf="tx"
[chainTip]="latestBlock?.height"
[height]="tx?.status?.block_height"
[replaced]="replaced"

2
frontend/src/app/components/transaction/transaction.component.ts
View File

@ -379,7 +379,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
ancestors: tx.ancestors,
bestDescendant: tx.bestDescendant,
};
const hasRelatives = !!(tx.ancestors.length || tx.bestDescendant);
const hasRelatives = !!(tx.ancestors?.length || tx.bestDescendant);
this.hasEffectiveFeeRate = hasRelatives || (tx.effectiveFeePerVsize && (Math.abs(tx.effectiveFeePerVsize - tx.feePerVsize) > 0.01));
} else {
this.fetchCpfp$.next(this.tx.txid);

3
frontend/src/app/interfaces/node-api.interface.ts
View File

@ -173,7 +173,8 @@ export interface TransactionStripped {
fee: number;
vsize: number;
value: number;
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf';
rate?: number; // effective fee rate
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'fullrbf';
context?: 'projected' | 'actual';
}

2
frontend/src/app/interfaces/websocket.interface.ts
View File

@ -89,7 +89,7 @@ export interface TransactionStripped {
vsize: number;
value: number;
rate?: number; // effective fee rate
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf';
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'fullrbf';
context?: 'projected' | 'actual';
}

43
frontend/src/app/lightning/node/node.component.html
View File

@ -21,7 +21,6 @@
</div>
<div class="box" *ngIf="!error">
<div class="row">
<div class="col-md">
<table class="table table-borderless table-striped table-fixed">
@ -59,6 +58,9 @@
<td i18n="lightning.avg-distance" class="text-truncate">Avg channel distance</td>
<td class="direction-ltr">{{ avgDistance | amountShortener: 1 }} <span class="symbol">km</span> <span class="separator">·</span>{{ kmToMiles(avgDistance) | amountShortener: 1 }} <span class="symbol">mi</span></td>
</tr>
<tr *ngIf="!node.geolocation" class="d-none d-md-table-row">
<ng-container *ngTemplateOutlet="featurebits;context:{bits: node.featuresBits}"></ng-container>
</tr>
</tbody>
</table>
</div>
@ -100,11 +102,50 @@
</td>
</ng-template>
</tr>
<tr *ngIf="node.geolocation && node.featuresBits">
<ng-container *ngTemplateOutlet="featurebits;context:{bits: node.featuresBits}"></ng-container>
</tr>
<tr *ngIf="!node.geolocation && node.featuresBits" class="d-table-row d-md-none">
<ng-container *ngTemplateOutlet="featurebits;context:{bits: node.featuresBits}"></ng-container>
</tr>
</tbody>
</table>
</div>
</div>
</div>
<ng-template #featurebits let-bits="bits">
<td i18n="lightning.features" class="text-truncate label">Features</td>
<td class="d-flex justify-content-between">
<span class="text-truncate w-90">{{ bits }}</span>
<button type="button" class="btn btn-outline-info btn-xs" (click)="toggleFeatures()" i18n="transaction.details|Transaction Details">Details</button>
</td>
</ng-template>
<div class="box mt-2" *ngIf="!error && showFeatures">
<div class="row">
<div class="col-md">
<div class="mb-3">
<h5>Raw bits</h5>
<span class="text-wrap w-100"><small>{{ node.featuresBits }}</small></span>
</div>
<h5>Decoded</h5>
<table class="table table-borderless table-striped table-fixed">
<thead>
<th style="width: 13%">Bit</th>
<th>Name</th>
<th style="width: 25%; text-align: right">Required</th>
</thead>
<tbody>
<tr *ngFor="let feature of node.features">
<td style="width: 13%">{{ feature.bit }}</td>
<td>{{ feature.name }}</td>
<td style="width: 25%; text-align: right">{{ feature.is_required }}</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
<div class="input-group mt-3" *ngIf="!error && node.socketsObject.length">

7
frontend/src/app/lightning/node/node.component.ts
View File

@ -37,7 +37,7 @@ export class NodeComponent implements OnInit {
liquidityAd: ILiquidityAd;
tlvRecords: CustomRecord[];
avgChannelDistance$: Observable<number | null>;
showFeatures = false;
kmToMiles = kmToMiles;
constructor(
@ -164,4 +164,9 @@ export class NodeComponent implements OnInit {
onLoadingEvent(e) {
this.channelListLoading = e;
}
toggleFeatures() {
this.showFeatures = !this.showFeatures;
return false;
}
}

12
frontend/src/app/services/electrs-api.service.ts
View File

@ -1,5 +1,5 @@
import { Injectable } from '@angular/core';
import { HttpClient } from '@angular/common/http';
import { HttpClient, HttpParams } from '@angular/common/http';
import { Observable } from 'rxjs';
import { Transaction, Address, Outspend, Recent, Asset } from '../interfaces/electrs.interface';
import { StateService } from './state.service';
@ -65,12 +65,12 @@ export class ElectrsApiService {
return this.httpClient.get<Address>(this.apiBaseUrl + this.apiBasePath + '/api/address/' + address);
}
getAddressTransactions$(address: string): Observable<Transaction[]> {
return this.httpClient.get<Transaction[]>(this.apiBaseUrl + this.apiBasePath + '/api/address/' + address + '/txs');
getAddressTransactions$(address: string, txid?: string): Observable<Transaction[]> {
let params = new HttpParams();
if (txid) {
params = params.append('after_txid', txid);
}
getAddressTransactionsFromHash$(address: string, txid: string): Observable<Transaction[]> {
return this.httpClient.get<Transaction[]>(this.apiBaseUrl + this.apiBasePath + '/api/address/' + address + '/txs/chain/' + txid);
return this.httpClient.get<Transaction[]>(this.apiBaseUrl + this.apiBasePath + '/api/address/' + address + '/txs', { params });
}
getAsset$(assetId: string): Observable<Asset> {

2
frontend/src/app/services/state.service.ts
View File

@ -45,7 +45,6 @@ export interface Env {
MAINNET_BLOCK_AUDIT_START_HEIGHT: number;
TESTNET_BLOCK_AUDIT_START_HEIGHT: number;
SIGNET_BLOCK_AUDIT_START_HEIGHT: number;
FULL_RBF_ENABLED: boolean;
HISTORICAL_PRICE: boolean;
}
@ -76,7 +75,6 @@ const defaultEnv: Env = {
'MAINNET_BLOCK_AUDIT_START_HEIGHT': 0,
'TESTNET_BLOCK_AUDIT_START_HEIGHT': 0,
'SIGNET_BLOCK_AUDIT_START_HEIGHT': 0,
'FULL_RBF_ENABLED': false,
'HISTORICAL_PRICE': true,
};

5
frontend/src/app/shared/components/confirmations/confirmations.component.html
View File

@ -5,12 +5,15 @@
<ng-template #confirmationPlural let-i i18n="shared.confirmation-count.plural|Transaction plural confirmation count">{{ i }} confirmations</ng-template>
</button>
</ng-template>
<ng-template [ngIf]="!confirmations && height != null">
<button type="button" class="btn btn-sm btn-success {{buttonClass}}" i18n="transaction.confirmed|Transaction confirmed state">Confirmed</button>
</ng-template>
<ng-template [ngIf]="!hideUnconfirmed && !confirmations && replaced">
<button type="button" class="btn btn-sm btn-warning {{buttonClass}}" i18n="transaction.replaced|Transaction replaced state">Replaced</button>
</ng-template>
<ng-template [ngIf]="!hideUnconfirmed && !confirmations && !replaced && removed">
<button type="button" class="btn btn-sm btn-warning {{buttonClass}}" i18n="transaction.audit.removed|Transaction removed state">Removed</button>
</ng-template>
<ng-template [ngIf]="!hideUnconfirmed && !confirmations && !replaced && !removed">
<ng-template [ngIf]="!hideUnconfirmed && chainTip != null && !confirmations && !replaced && !removed">
<button type="button" class="btn btn-sm btn-danger {{buttonClass}}" i18n="transaction.unconfirmed|Transaction unconfirmed state">Unconfirmed</button>
</ng-template>

11
frontend/src/styles.scss
View File

@ -45,8 +45,8 @@ $dropdown-link-hover-bg: #11131f;
$dropdown-link-active-color: #fff;
$dropdown-link-active-bg: #11131f;
@import "~bootstrap/scss/bootstrap";
@import '~tlite/tlite.css';
@import "bootstrap/scss/bootstrap";
@import 'tlite/tlite.css';
html, body {
height: 100%;
@ -1164,3 +1164,10 @@ app-master-page, app-liquid-master-page, app-bisq-master-page {
app-global-footer {
margin-top: auto;
}
.btn-xs {
padding: 0.25rem 0.5rem;
font-size: 0.875rem;
line-height: 0.5;
border-radius: 0.2rem;
}

3
production/bitcoin.conf
View File

@ -8,7 +8,8 @@ par=16
dbcache=8192
maxmempool=4096
mempoolexpiry=999999
maxconnections=42
mempoolfullrbf=1
maxconnections=100
onion=127.0.0.1:9050
rpcallowip=127.0.0.1
rpcuser=__BITCOIN_RPC_USER__

1
production/bitcoin.minfee.conf
View File

@ -4,6 +4,7 @@ txindex=0
listen=1
daemon=1
prune=1337
mempoolfullrbf=1
rpcallowip=127.0.0.1
rpcuser=__BITCOIN_RPC_USER__
rpcpassword=__BITCOIN_RPC_PASS__

9
production/install
View File

@ -353,7 +353,7 @@ ELEMENTS_REPO_URL=https://github.com/ElementsProject/elements
ELEMENTS_REPO_NAME=elements
ELEMENTS_REPO_BRANCH=master
#ELEMENTS_LATEST_RELEASE=$(curl -s https://api.github.com/repos/ElementsProject/elements/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
ELEMENTS_LATEST_RELEASE=elements-22.1
ELEMENTS_LATEST_RELEASE=elements-22.1.1
echo -n '.'
BITCOIN_ELECTRS_REPO_URL=https://github.com/mempool/electrs
@ -1044,8 +1044,11 @@ osSudo "${ROOT_USER}" crontab -u "${MEMPOOL_USER}" "${MEMPOOL_HOME}/${MEMPOOL_RE
echo "[*] Installing nvm.sh from GitHub"
osSudo "${MEMPOOL_USER}" sh -c 'curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | zsh'
echo "[*] Building NodeJS via nvm.sh"
osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm install v16.16.0 --shared-zlib'
echo "[*] Building NodeJS v20.4.0 via nvm.sh"
osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm install v20.4.0 --shared-zlib'
echo "[*] Building NodeJS v18.16.1 via nvm.sh"
osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm install v18.16.1 --shared-zlib'
osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm alias default 18.16.1'
####################
# Tor installation #

25
production/mempool-config.mainnet.json
View File

@ -48,5 +48,30 @@
"STATISTICS": {
"ENABLED": true,
"TX_PER_SECOND_SAMPLE_PERIOD": 150
},
"REPLICATION": {
"ENABLED": true,
"AUDIT": true,
"AUDIT_START_HEIGHT": 774000,
"SERVERS": [
"node201.fmt.mempool.space",
"node202.fmt.mempool.space",
"node203.fmt.mempool.space",
"node204.fmt.mempool.space",
"node205.fmt.mempool.space",
"node206.fmt.mempool.space",
"node201.fra.mempool.space",
"node202.fra.mempool.space",
"node203.fra.mempool.space",
"node204.fra.mempool.space",
"node205.fra.mempool.space",
"node206.fra.mempool.space",
"node201.tk7.mempool.space",
"node202.tk7.mempool.space",
"node203.tk7.mempool.space",
"node204.tk7.mempool.space",
"node205.tk7.mempool.space",
"node206.tk7.mempool.space"
]
}
}

1
production/mempool-start-all
View File

@ -1,6 +1,7 @@
#!/usr/bin/env zsh
export NVM_DIR="$HOME/.nvm"
source "$NVM_DIR/nvm.sh"
nvm use v20.4.0
# start all mempool backends that exist
for site in mainnet mainnet-lightning testnet testnet-lightning signet signet-lightning bisq liquid liquidtestnet;do