machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into update_gha

Browse Source
This commit is contained in:
Felipe Knorr Kuhn 2022-12-05 20:15:14 -08:00 committed by GitHub
commit d17ccbc5ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 3095 additions and 2341 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backend/mempool-config.sample.json
View File

@ -25,7 +25,8 @@
"AUTOMATIC_BLOCK_REINDEXING": false,
"POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
"POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
"ADVANCED_TRANSACTION_SELECTION": false,
"ADVANCED_GBT_AUDIT": false,
"ADVANCED_GBT_MEMPOOL": false,
"TRANSACTION_INDEXING": false
},
"CORE_RPC": {

3593
backend/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

36
backend/package.json
View File

@ -34,35 +34,35 @@
"prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
},
"dependencies": {
"@babel/core": "^7.18.6",
"@babel/core": "^7.20.5",
"@mempool/electrum-client": "^1.1.7",
"@types/node": "^16.11.41",
"axios": "~0.27.2",
"bitcoinjs-lib": "6.0.2",
"crypto-js": "^4.0.0",
"express": "^4.18.0",
"maxmind": "^4.3.6",
"mysql2": "2.3.3",
"node-worker-threads-pool": "^1.5.1",
"bitcoinjs-lib": "~6.0.2",
"crypto-js": "~4.1.1",
"express": "~4.18.2",
"maxmind": "~4.3.8",
"mysql2": "~2.3.3",
"node-worker-threads-pool": "~1.5.1",
"socks-proxy-agent": "~7.0.0",
"typescript": "~4.7.4",
"ws": "~8.8.0"
"ws": "~8.11.0"
},
"devDependencies": {
"@babel/core": "^7.18.6",
"@babel/core": "^7.20.5",
"@babel/code-frame": "^7.18.6",
"@types/compression": "^1.7.2",
"@types/crypto-js": "^4.1.1",
"@types/express": "^4.17.13",
"@types/jest": "^28.1.4",
"@types/express": "^4.17.14",
"@types/jest": "^29.2.3",
"@types/ws": "~8.5.3",
"@typescript-eslint/eslint-plugin": "^5.30.5",
"@typescript-eslint/parser": "^5.30.5",
"eslint": "^8.19.0",
"@typescript-eslint/eslint-plugin": "^5.45.0",
"@typescript-eslint/parser": "^5.45.0",
"eslint": "^8.28.0",
"eslint-config-prettier": "^8.5.0",
"jest": "^28.1.2",
"prettier": "^2.7.1",
"ts-jest": "^28.0.5",
"ts-node": "^10.8.2"
"jest": "^29.3.1",
"prettier": "^2.8.0",
"ts-jest": "^29.0.3",
"ts-node": "^10.9.1"
}
}

3
backend/src/__fixtures__/mempool-config.template.json
View File

@ -26,7 +26,8 @@
"INDEXING_BLOCKS_AMOUNT": 14,
"POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
"POOLS_JSON_URL": "__POOLS_JSON_URL__",
"ADVANCED_TRANSACTION_SELECTION": "__ADVANCED_TRANSACTION_SELECTION__",
"ADVANCED_GBT_AUDIT": "__ADVANCED_GBT_AUDIT__",
"ADVANCED_GBT_MEMPOOL": "__ADVANCED_GBT_MEMPOOL__",
"TRANSACTION_INDEXING": "__TRANSACTION_INDEXING__"
},
"CORE_RPC": {

3
backend/src/__tests__/config.test.ts
View File

@ -38,7 +38,8 @@ describe('Mempool Backend Config', () => {
STDOUT_LOG_MIN_PRIORITY: 'debug',
POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
ADVANCED_TRANSACTION_SELECTION: false,
ADVANCED_GBT_AUDIT: false,
ADVANCED_GBT_MEMPOOL: false,
TRANSACTION_INDEXING: false,
});

75
backend/src/api/database-migration.ts
View File

@ -4,8 +4,8 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 47;
private queryTimeout = 900_000;
private static currentVersion = 49;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@ -107,18 +107,22 @@ class DatabaseMigration {
await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) {
await this.$executeQuery(`CREATE INDEX added ON statistics (added);`);
await this.updateToSchemaVersion(2);
}
if (databaseSchemaVersion < 3) {
await this.$executeQuery(this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
await this.updateToSchemaVersion(3);
}
if (databaseSchemaVersion < 4) {
await this.$executeQuery('DROP table IF EXISTS blocks;');
await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
await this.updateToSchemaVersion(4);
}
if (databaseSchemaVersion < 5 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
await this.updateToSchemaVersion(5);
}
if (databaseSchemaVersion < 6 && isBitcoin === true) {
@ -141,11 +145,13 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
await this.$executeQuery('ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
await this.updateToSchemaVersion(6);
}
if (databaseSchemaVersion < 7 && isBitcoin === true) {
await this.$executeQuery('DROP table IF EXISTS hashrates;');
await this.$executeQuery(this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates'));
await this.updateToSchemaVersion(7);
}
if (databaseSchemaVersion < 8 && isBitcoin === true) {
@ -155,6 +161,7 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
await this.$executeQuery('ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
await this.updateToSchemaVersion(8);
}
if (databaseSchemaVersion < 9 && isBitcoin === true) {
@ -162,10 +169,12 @@ class DatabaseMigration {
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
await this.updateToSchemaVersion(9);
}
if (databaseSchemaVersion < 10 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
await this.updateToSchemaVersion(10);
}
if (databaseSchemaVersion < 11 && isBitcoin === true) {
@ -178,11 +187,13 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
await this.updateToSchemaVersion(11);
}
if (databaseSchemaVersion < 12 && isBitcoin === true) {
// No need to re-index because the new data type can contain larger values
await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.updateToSchemaVersion(12);
}
if (databaseSchemaVersion < 13 && isBitcoin === true) {
@ -190,6 +201,7 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.updateToSchemaVersion(13);
}
if (databaseSchemaVersion < 14 && isBitcoin === true) {
@ -197,37 +209,45 @@ class DatabaseMigration {
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
await this.updateToSchemaVersion(14);
}
if (databaseSchemaVersion < 16 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps
await this.updateToSchemaVersion(16);
}
if (databaseSchemaVersion < 17 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `pools` ADD `slug` CHAR(50) NULL');
await this.updateToSchemaVersion(17);
}
if (databaseSchemaVersion < 18 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `hash` (`hash`);');
await this.updateToSchemaVersion(18);
}
if (databaseSchemaVersion < 19) {
await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates'));
await this.updateToSchemaVersion(19);
}
if (databaseSchemaVersion < 20 && isBitcoin === true) {
await this.$executeQuery(this.getCreateBlocksSummariesTableQuery(), await this.$checkIfTableExists('blocks_summaries'));
await this.updateToSchemaVersion(20);
}
if (databaseSchemaVersion < 21) {
await this.$executeQuery('DROP TABLE IF EXISTS `rates`');
await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices'));
await this.updateToSchemaVersion(21);
}
if (databaseSchemaVersion < 22 && isBitcoin === true) {
await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`');
await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments'));
await this.updateToSchemaVersion(22);
}
if (databaseSchemaVersion < 23) {
@ -240,11 +260,13 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `prices` ADD `CHF` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `AUD` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `JPY` float DEFAULT "0"');
await this.updateToSchemaVersion(23);
}
if (databaseSchemaVersion < 24 && isBitcoin == true) {
await this.$executeQuery('DROP TABLE IF EXISTS `blocks_audits`');
await this.$executeQuery(this.getCreateBlocksAuditsTableQuery(), await this.$checkIfTableExists('blocks_audits'));
await this.updateToSchemaVersion(24);
}
if (databaseSchemaVersion < 25 && isBitcoin === true) {
@ -252,6 +274,7 @@ class DatabaseMigration {
await this.$executeQuery(this.getCreateNodesQuery(), await this.$checkIfTableExists('nodes'));
await this.$executeQuery(this.getCreateChannelsQuery(), await this.$checkIfTableExists('channels'));
await this.$executeQuery(this.getCreateNodesStatsQuery(), await this.$checkIfTableExists('node_stats'));
await this.updateToSchemaVersion(25);
}
if (databaseSchemaVersion < 26 && isBitcoin === true) {
@ -262,6 +285,7 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD tor_nodes int(11) NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_nodes int(11) NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD unannounced_nodes int(11) NOT NULL DEFAULT "0"');
await this.updateToSchemaVersion(26);
}
if (databaseSchemaVersion < 27 && isBitcoin === true) {
@ -271,6 +295,7 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_capacity bigint(20) unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_fee_rate int(11) unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"');
await this.updateToSchemaVersion(27);
}
if (databaseSchemaVersion < 28 && isBitcoin === true) {
@ -280,6 +305,7 @@ class DatabaseMigration {
await this.$executeQuery(`TRUNCATE lightning_stats`);
await this.$executeQuery(`TRUNCATE node_stats`);
await this.$executeQuery(`ALTER TABLE lightning_stats MODIFY added DATE`);
await this.updateToSchemaVersion(28);
}
if (databaseSchemaVersion < 29 && isBitcoin === true) {
@ -291,41 +317,50 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `nodes` ADD subdivision_id int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD longitude double NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD latitude double NULL DEFAULT NULL');
await this.updateToSchemaVersion(29);
}
if (databaseSchemaVersion < 30 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization") NOT NULL');
await this.updateToSchemaVersion(30);
}
if (databaseSchemaVersion < 31 && isBitcoin == true) { // Link blocks to prices
await this.$executeQuery('ALTER TABLE `prices` ADD `id` int NULL AUTO_INCREMENT UNIQUE');
await this.$executeQuery('DROP TABLE IF EXISTS `blocks_prices`');
await this.$executeQuery(this.getCreateBlocksPricesTableQuery(), await this.$checkIfTableExists('blocks_prices'));
await this.updateToSchemaVersion(31);
}
if (databaseSchemaVersion < 32 && isBitcoin == true) {
await this.$executeQuery('ALTER TABLE `blocks_summaries` ADD `template` JSON DEFAULT "[]"');
await this.updateToSchemaVersion(32);
}
if (databaseSchemaVersion < 33 && isBitcoin == true) {
await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization", "country_iso_code") NOT NULL');
await this.updateToSchemaVersion(33);
}
if (databaseSchemaVersion < 34 && isBitcoin == true) {
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_tor_nodes int(11) NOT NULL DEFAULT "0"');
await this.updateToSchemaVersion(34);
}
if (databaseSchemaVersion < 35 && isBitcoin == true) {
await this.$executeQuery('DELETE from `lightning_stats` WHERE added > "2021-09-19"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD CONSTRAINT added_unique UNIQUE (added);');
await this.updateToSchemaVersion(35);
}
if (databaseSchemaVersion < 36 && isBitcoin == true) {
await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"');
await this.updateToSchemaVersion(36);
}
if (databaseSchemaVersion < 37 && isBitcoin == true) {
await this.$executeQuery(this.getCreateLNNodesSocketsTableQuery(), await this.$checkIfTableExists('nodes_sockets'));
await this.updateToSchemaVersion(37);
}
if (databaseSchemaVersion < 38 && isBitcoin == true) {
@ -336,50 +371,78 @@ class DatabaseMigration {
await this.$executeQuery(`TRUNCATE node_stats`);
await this.$executeQuery('ALTER TABLE `lightning_stats` CHANGE `added` `added` timestamp NULL');
await this.$executeQuery('ALTER TABLE `node_stats` CHANGE `added` `added` timestamp NULL');
await this.updateToSchemaVersion(38);
}
if (databaseSchemaVersion < 39 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `nodes` ADD alias_search TEXT NULL DEFAULT NULL AFTER `alias`');
await this.$executeQuery('ALTER TABLE nodes ADD FULLTEXT(alias_search)');
await this.updateToSchemaVersion(39);
}
if (databaseSchemaVersion < 40 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `nodes` ADD capacity bigint(20) unsigned DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD channels int(11) unsigned DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD INDEX `capacity` (`capacity`);');
await this.updateToSchemaVersion(40);
}
if (databaseSchemaVersion < 41 && isBitcoin === true) {
await this.$executeQuery('UPDATE channels SET closing_reason = NULL WHERE closing_reason = 1');
await this.updateToSchemaVersion(41);
}
if (databaseSchemaVersion < 42 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `channels` ADD closing_resolved tinyint(1) DEFAULT 0');
await this.updateToSchemaVersion(42);
}
if (databaseSchemaVersion < 43 && isBitcoin === true) {
await this.$executeQuery(this.getCreateLNNodeRecordsTableQuery(), await this.$checkIfTableExists('nodes_records'));
await this.updateToSchemaVersion(43);
}
if (databaseSchemaVersion < 44 && isBitcoin === true) {
await this.$executeQuery('TRUNCATE TABLE `blocks_audits`');
await this.$executeQuery('UPDATE blocks_summaries SET template = NULL');
await this.updateToSchemaVersion(44);
}
if (databaseSchemaVersion < 45 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fresh_txs JSON DEFAULT "[]"');
await this.updateToSchemaVersion(45);
}
if (databaseSchemaVersion < 46) {
await this.$executeQuery(`ALTER TABLE blocks MODIFY blockTimestamp timestamp NOT NULL DEFAULT 0`);
await this.updateToSchemaVersion(46);
}
if (databaseSchemaVersion < 47) {
await this.$executeQuery('ALTER TABLE `blocks` ADD cpfp_indexed tinyint(1) DEFAULT 0');
await this.$executeQuery(this.getCreateCPFPTableQuery(), await this.$checkIfTableExists('cpfp_clusters'));
await this.$executeQuery(this.getCreateTransactionsTableQuery(), await this.$checkIfTableExists('transactions'));
await this.updateToSchemaVersion(47);
}
if (databaseSchemaVersion < 48 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `channels` ADD source_checked tinyint(1) DEFAULT 0');
await this.$executeQuery('ALTER TABLE `channels` ADD closing_fee bigint(20) unsigned DEFAULT 0');
await this.$executeQuery('ALTER TABLE `channels` ADD node1_funding_balance bigint(20) unsigned DEFAULT 0');
await this.$executeQuery('ALTER TABLE `channels` ADD node2_funding_balance bigint(20) unsigned DEFAULT 0');
await this.$executeQuery('ALTER TABLE `channels` ADD node1_closing_balance bigint(20) unsigned DEFAULT 0');
await this.$executeQuery('ALTER TABLE `channels` ADD node2_closing_balance bigint(20) unsigned DEFAULT 0');
await this.$executeQuery('ALTER TABLE `channels` ADD funding_ratio float unsigned DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `channels` ADD closed_by varchar(66) DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `channels` ADD single_funded tinyint(1) DEFAULT 0');
await this.$executeQuery('ALTER TABLE `channels` ADD outputs JSON DEFAULT "[]"');
await this.updateToSchemaVersion(48);
}
if (databaseSchemaVersion < 49 && isBitcoin === true) {
await this.$executeQuery('TRUNCATE TABLE `blocks_audits`');
await this.updateToSchemaVersion(49);
}
}
}
/**
* Special case here for the `statistics` table - It appeared that somehow some dbs already had the `added` field indexed
@ -516,6 +579,10 @@ class DatabaseMigration {
return `UPDATE state SET number = ${DatabaseMigration.currentVersion} WHERE name = 'schema_version';`;
}
private async updateToSchemaVersion(version): Promise<void> {
await this.$executeQuery(`UPDATE state SET number = ${version} WHERE name = 'schema_version';`);
}
/**
* Print current database version
*/

127
backend/src/api/explorer/channels.api.ts
View File

@ -128,6 +128,21 @@ class ChannelsApi {
}
}
public async $getChannelsWithoutSourceChecked(): Promise<any[]> {
try {
const query = `
SELECT channels.*
FROM channels
WHERE channels.source_checked != 1
`;
const [rows]: any = await DB.query(query);
return rows;
} catch (e) {
logger.err('$getUnresolvedClosedChannels error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getChannelsWithoutCreatedDate(): Promise<any[]> {
try {
const query = `SELECT * FROM channels WHERE created IS NULL`;
@ -257,6 +272,108 @@ class ChannelsApi {
}
}
public async $getChannelByClosingId(transactionId: string): Promise<any> {
try {
const query = `
SELECT
channels.*
FROM channels
WHERE channels.closing_transaction_id = ?
`;
const [rows]: any = await DB.query(query, [transactionId]);
if (rows.length > 0) {
rows[0].outputs = JSON.parse(rows[0].outputs);
return rows[0];
}
} catch (e) {
logger.err('$getChannelByClosingId error: ' + (e instanceof Error ? e.message : e));
// don't throw - this data isn't essential
}
}
public async $getChannelsByOpeningId(transactionId: string): Promise<any> {
try {
const query = `
SELECT
channels.*
FROM channels
WHERE channels.transaction_id = ?
`;
const [rows]: any = await DB.query(query, [transactionId]);
if (rows.length > 0) {
return rows.map(row => {
row.outputs = JSON.parse(row.outputs);
return row;
});
}
} catch (e) {
logger.err('$getChannelsByOpeningId error: ' + (e instanceof Error ? e.message : e));
// don't throw - this data isn't essential
}
}
public async $updateClosingInfo(channelInfo: { id: string, node1_closing_balance: number, node2_closing_balance: number, closed_by: string | null, closing_fee: number, outputs: ILightningApi.ForensicOutput[]}): Promise<void> {
try {
const query = `
UPDATE channels SET
node1_closing_balance = ?,
node2_closing_balance = ?,
closed_by = ?,
closing_fee = ?,
outputs = ?
WHERE channels.id = ?
`;
await DB.query<ResultSetHeader>(query, [
channelInfo.node1_closing_balance || 0,
channelInfo.node2_closing_balance || 0,
channelInfo.closed_by,
channelInfo.closing_fee || 0,
JSON.stringify(channelInfo.outputs),
channelInfo.id,
]);
} catch (e) {
logger.err('$updateClosingInfo error: ' + (e instanceof Error ? e.message : e));
// don't throw - this data isn't essential
}
}
public async $updateOpeningInfo(channelInfo: { id: string, node1_funding_balance: number, node2_funding_balance: number, funding_ratio: number, single_funded: boolean | void }): Promise<void> {
try {
const query = `
UPDATE channels SET
node1_funding_balance = ?,
node2_funding_balance = ?,
funding_ratio = ?,
single_funded = ?
WHERE channels.id = ?
`;
await DB.query<ResultSetHeader>(query, [
channelInfo.node1_funding_balance || 0,
channelInfo.node2_funding_balance || 0,
channelInfo.funding_ratio,
channelInfo.single_funded ? 1 : 0,
channelInfo.id,
]);
} catch (e) {
logger.err('$updateOpeningInfo error: ' + (e instanceof Error ? e.message : e));
// don't throw - this data isn't essential
}
}
public async $markChannelSourceChecked(id: string): Promise<void> {
try {
const query = `
UPDATE channels
SET source_checked = 1
WHERE id = ?
`;
await DB.query<ResultSetHeader>(query, [id]);
} catch (e) {
logger.err('$markChannelSourceChecked error: ' + (e instanceof Error ? e.message : e));
// don't throw - this data isn't essential
}
}
public async $getChannelsForNode(public_key: string, index: number, length: number, status: string): Promise<any[]> {
try {
let channelStatusFilter;
@ -385,11 +502,15 @@ class ChannelsApi {
'transaction_id': channel.transaction_id,
'transaction_vout': channel.transaction_vout,
'closing_transaction_id': channel.closing_transaction_id,
'closing_fee': channel.closing_fee,
'closing_reason': channel.closing_reason,
'closing_date': channel.closing_date,
'updated_at': channel.updated_at,
'created': channel.created,
'status': channel.status,
'funding_ratio': channel.funding_ratio,
'closed_by': channel.closed_by,
'single_funded': !!channel.single_funded,
'node_left': {
'alias': channel.alias_left,
'public_key': channel.node1_public_key,
@ -404,6 +525,9 @@ class ChannelsApi {
'updated_at': channel.node1_updated_at,
'longitude': channel.node1_longitude,
'latitude': channel.node1_latitude,
'funding_balance': channel.node1_funding_balance,
'closing_balance': channel.node1_closing_balance,
'initiated_close': channel.closed_by === channel.node1_public_key ? true : undefined,
},
'node_right': {
'alias': channel.alias_right,
@ -419,6 +543,9 @@ class ChannelsApi {
'updated_at': channel.node2_updated_at,
'longitude': channel.node2_longitude,
'latitude': channel.node2_latitude,
'funding_balance': channel.node2_funding_balance,
'closing_balance': channel.node2_closing_balance,
'initiated_close': channel.closed_by === channel.node2_public_key ? true : undefined,
},
};
}

4
backend/src/api/explorer/nodes.api.ts
View File

@ -14,8 +14,8 @@ class NodesApi {
nodes.longitude, nodes.latitude,
geo_names_country.names as country, geo_names_iso.names as isoCode
FROM nodes
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
WHERE status = 1 AND nodes.as_number IS NOT NULL
ORDER BY capacity
`;

6
backend/src/api/lightning/lightning-api.interface.ts
View File

@ -83,4 +83,10 @@ export namespace ILightningApi {
is_required: boolean;
is_known: boolean;
}
export interface ForensicOutput {
node?: 1 | 2;
type: number;
value: number;
}
}

13
backend/src/api/websocket-handler.ts
View File

@ -250,12 +250,12 @@ class WebsocketHandler {
throw new Error('WebSocket.Server is not set');
}
if (config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) {
if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
await mempoolBlocks.makeBlockTemplates(newMempool, 8, null, true);
}
else {
} else {
mempoolBlocks.updateMempoolBlocks(newMempool);
}
const mBlocks = mempoolBlocks.getMempoolBlocks();
const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
const mempoolInfo = memPool.getMempoolInfo();
@ -417,9 +417,8 @@ class WebsocketHandler {
}
const _memPool = memPool.getMempool();
let matchRate;
if (config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) {
if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
await mempoolBlocks.makeBlockTemplates(_memPool, 2);
} else {
mempoolBlocks.updateMempoolBlocks(_memPool);
@ -429,7 +428,7 @@ class WebsocketHandler {
const projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
const { censored, added, fresh, score } = Audit.auditBlock(transactions, projectedBlocks, _memPool);
matchRate = Math.round(score * 100 * 100) / 100;
const matchRate = Math.round(score * 100 * 100) / 100;
const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
return {
@ -468,7 +467,7 @@ class WebsocketHandler {
delete _memPool[txId];
}
if (config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) {
if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
await mempoolBlocks.makeBlockTemplates(_memPool, 2);
} else {
mempoolBlocks.updateMempoolBlocks(_memPool);

6
backend/src/config.ts
View File

@ -29,7 +29,8 @@ interface IConfig {
AUTOMATIC_BLOCK_REINDEXING: boolean;
POOLS_JSON_URL: string,
POOLS_JSON_TREE_URL: string,
ADVANCED_TRANSACTION_SELECTION: boolean;
ADVANCED_GBT_AUDIT: boolean;
ADVANCED_GBT_MEMPOOL: boolean;
TRANSACTION_INDEXING: boolean;
};
ESPLORA: {
@ -148,7 +149,8 @@ const defaults: IConfig = {
'AUTOMATIC_BLOCK_REINDEXING': false,
'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
'ADVANCED_TRANSACTION_SELECTION': false,
'ADVANCED_GBT_AUDIT': false,
'ADVANCED_GBT_MEMPOOL': false,
'TRANSACTION_INDEXING': false,
},
'ESPLORA': {

269
backend/src/tasks/lightning/forensics.service.ts
View File

@ -5,13 +5,16 @@ import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
import config from '../../config';
import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
import { Common } from '../../api/common';
import { ILightningApi } from '../../api/lightning/lightning-api.interface';
const throttleDelay = 20; //ms
const tempCacheSize = 10000;
class ForensicsService {
loggerTimer = 0;
closedChannelsScanBlock = 0;
txCache: { [txid: string]: IEsploraApi.Transaction } = {};
tempCached: string[] = [];
constructor() {}
@ -29,6 +32,7 @@ class ForensicsService {
if (config.MEMPOOL.BACKEND === 'esplora') {
await this.$runClosedChannelsForensics(false);
await this.$runOpenedChannelsForensics();
}
} catch (e) {
@ -95,17 +99,10 @@ class ForensicsService {
const lightningScriptReasons: number[] = [];
for (const outspend of outspends) {
if (outspend.spent && outspend.txid) {
let spendingTx: IEsploraApi.Transaction | undefined = this.txCache[outspend.txid];
let spendingTx = await this.fetchTransaction(outspend.txid);
if (!spendingTx) {
try {
spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid);
await Common.sleep$(throttleDelay);
this.txCache[outspend.txid] = spendingTx;
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`);
continue;
}
}
cached.push(spendingTx.txid);
const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
lightningScriptReasons.push(lightningScript);
@ -124,17 +121,10 @@ class ForensicsService {
We can detect a commitment transaction (force close) by reading Sequence and Locktime
https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction
*/
let closingTx: IEsploraApi.Transaction | undefined = this.txCache[channel.closing_transaction_id];
let closingTx = await this.fetchTransaction(channel.closing_transaction_id, true);
if (!closingTx) {
try {
closingTx = await bitcoinApi.$getRawTransaction(channel.closing_transaction_id);
await Common.sleep$(throttleDelay);
this.txCache[channel.closing_transaction_id] = closingTx;
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id}. Reason ${e instanceof Error ? e.message : e}`);
continue;
}
}
cached.push(closingTx.txid);
const sequenceHex: string = closingTx.vin[0].sequence.toString(16);
const locktimeHex: string = closingTx.locktime.toString(16);
@ -174,7 +164,7 @@ class ForensicsService {
}
private findLightningScript(vin: IEsploraApi.Vin): number {
const topElement = vin.witness[vin.witness.length - 2];
const topElement = vin.witness?.length > 2 ? vin.witness[vin.witness.length - 2] : null;
if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) {
// https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs
if (topElement === '01') {
@ -193,7 +183,7 @@ class ForensicsService {
) {
// https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs
// https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs
if (topElement.length === 66) {
if (topElement?.length === 66) {
// top element is a public key
// 'Revoked Lightning HTLC'; Penalty force closed
return 4;
@ -220,6 +210,249 @@ class ForensicsService {
}
return 1;
}
// If a channel open tx spends funds from a another channel transaction,
// we can attribute that output to a specific counterparty
private async $runOpenedChannelsForensics(): Promise<void> {
const runTimer = Date.now();
let progress = 0;
try {
logger.info(`Started running open channel forensics...`);
const channels = await channelsApi.$getChannelsWithoutSourceChecked();
for (const openChannel of channels) {
let openTx = await this.fetchTransaction(openChannel.transaction_id, true);
if (!openTx) {
continue;
}
for (const input of openTx.vin) {
const closeChannel = await channelsApi.$getChannelByClosingId(input.txid);
if (closeChannel) {
// this input directly spends a channel close output
await this.$attributeChannelBalances(closeChannel, openChannel, input);
} else {
const prevOpenChannels = await channelsApi.$getChannelsByOpeningId(input.txid);
if (prevOpenChannels?.length) {
// this input spends a channel open change output
for (const prevOpenChannel of prevOpenChannels) {
await this.$attributeChannelBalances(prevOpenChannel, openChannel, input, null, null, true);
}
} else {
// check if this input spends any swept channel close outputs
await this.$attributeSweptChannelCloses(openChannel, input);
}
}
}
// calculate how much of the total input value is attributable to the channel open output
openChannel.funding_ratio = openTx.vout[openChannel.transaction_vout].value / ((openTx.vout.reduce((sum, v) => sum + v.value, 0) || 1) + openTx.fee);
// save changes to the opening channel, and mark it as checked
if (openTx?.vin?.length === 1) {
openChannel.single_funded = true;
}
if (openChannel.node1_funding_balance || openChannel.node2_funding_balance || openChannel.node1_closing_balance || openChannel.node2_closing_balance || openChannel.closed_by) {
await channelsApi.$updateOpeningInfo(openChannel);
}
await channelsApi.$markChannelSourceChecked(openChannel.id);
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating opened channel forensics ${progress}/${channels?.length}`);
this.loggerTimer = new Date().getTime() / 1000;
this.truncateTempCache();
}
if (Date.now() - runTimer > (config.LIGHTNING.FORENSICS_INTERVAL * 1000)) {
break;
}
}
logger.info(`Open channels forensics scan complete.`);
} catch (e) {
logger.err('$runOpenedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
} finally {
this.clearTempCache();
}
}
// Check if a channel open tx input spends the result of a swept channel close output
private async $attributeSweptChannelCloses(openChannel: ILightningApi.Channel, input: IEsploraApi.Vin): Promise<void> {
let sweepTx = await this.fetchTransaction(input.txid, true);
if (!sweepTx) {
logger.err(`couldn't find input transaction for channel forensics ${openChannel.channel_id} ${input.txid}`);
return;
}
const openContribution = sweepTx.vout[input.vout].value;
for (const sweepInput of sweepTx.vin) {
const lnScriptType = this.findLightningScript(sweepInput);
if (lnScriptType > 1) {
const closeChannel = await channelsApi.$getChannelByClosingId(sweepInput.txid);
if (closeChannel) {
const initiator = (lnScriptType === 2 || lnScriptType === 4) ? 'remote' : (lnScriptType === 3 ? 'local' : null);
await this.$attributeChannelBalances(closeChannel, openChannel, sweepInput, openContribution, initiator);
}
}
}
}
private async $attributeChannelBalances(
prevChannel, openChannel, input: IEsploraApi.Vin, openContribution: number | null = null,
initiator: 'remote' | 'local' | null = null, linkedOpenings: boolean = false
): Promise<void> {
// figure out which node controls the input/output
let openSide;
let prevLocal;
let prevRemote;
let matched = false;
let ambiguous = false; // if counterparties are the same in both channels, we can't tell them apart
if (openChannel.node1_public_key === prevChannel.node1_public_key) {
openSide = 1;
prevLocal = 1;
prevRemote = 2;
matched = true;
} else if (openChannel.node1_public_key === prevChannel.node2_public_key) {
openSide = 1;
prevLocal = 2;
prevRemote = 1;
matched = true;
}
if (openChannel.node2_public_key === prevChannel.node1_public_key) {
openSide = 2;
prevLocal = 1;
prevRemote = 2;
if (matched) {
ambiguous = true;
}
matched = true;
} else if (openChannel.node2_public_key === prevChannel.node2_public_key) {
openSide = 2;
prevLocal = 2;
prevRemote = 1;
if (matched) {
ambiguous = true;
}
matched = true;
}
if (matched && !ambiguous) {
// fetch closing channel transaction and perform forensics on the outputs
let prevChannelTx = await this.fetchTransaction(input.txid, true);
let outspends: IEsploraApi.Outspend[] | undefined;
try {
outspends = await bitcoinApi.$getOutspends(input.txid);
await Common.sleep$(throttleDelay);
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + input.txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
}
if (!outspends || !prevChannelTx) {
return;
}
if (!linkedOpenings) {
if (!prevChannel.outputs || !prevChannel.outputs.length) {
prevChannel.outputs = prevChannelTx.vout.map(vout => {
return {
type: 0,
value: vout.value,
};
});
}
for (let i = 0; i < outspends?.length; i++) {
const outspend = outspends[i];
const output = prevChannel.outputs[i];
if (outspend.spent && outspend.txid) {
try {
const spendingTx = await this.fetchTransaction(outspend.txid, true);
if (spendingTx) {
output.type = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
}
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`);
}
} else {
output.type = 0;
}
}
// attribute outputs to each counterparty, and sum up total known balances
prevChannel.outputs[input.vout].node = prevLocal;
const isPenalty = prevChannel.outputs.filter((out) => out.type === 2 || out.type === 4)?.length > 0;
const normalOutput = [1,3].includes(prevChannel.outputs[input.vout].type);
const mutualClose = ((prevChannel.status === 2 || prevChannel.status === 'closed') && prevChannel.closing_reason === 1);
let localClosingBalance = 0;
let remoteClosingBalance = 0;
for (const output of prevChannel.outputs) {
if (isPenalty) {
// penalty close, so local node takes everything
localClosingBalance += output.value;
} else if (output.node) {
// this output determinstically linked to one of the counterparties
if (output.node === prevLocal) {
localClosingBalance += output.value;
} else {
remoteClosingBalance += output.value;
}
} else if (normalOutput && (output.type === 1 || output.type === 3 || (mutualClose && prevChannel.outputs.length === 2))) {
// local node had one main output, therefore remote node takes the other
remoteClosingBalance += output.value;
}
}
prevChannel[`node${prevLocal}_closing_balance`] = localClosingBalance;
prevChannel[`node${prevRemote}_closing_balance`] = remoteClosingBalance;
prevChannel.closing_fee = prevChannelTx.fee;
if (initiator && !linkedOpenings) {
const initiatorSide = initiator === 'remote' ? prevRemote : prevLocal;
prevChannel.closed_by = prevChannel[`node${initiatorSide}_public_key`];
}
// save changes to the closing channel
await channelsApi.$updateClosingInfo(prevChannel);
} else {
if (prevChannelTx.vin.length <= 1) {
prevChannel[`node${prevLocal}_funding_balance`] = prevChannel.capacity;
prevChannel.single_funded = true;
prevChannel.funding_ratio = 1;
// save changes to the closing channel
await channelsApi.$updateOpeningInfo(prevChannel);
}
}
openChannel[`node${openSide}_funding_balance`] = openChannel[`node${openSide}_funding_balance`] + (openContribution || prevChannelTx?.vout[input.vout]?.value || 0);
}
}
async fetchTransaction(txid: string, temp: boolean = false): Promise<IEsploraApi.Transaction | null> {
let tx = this.txCache[txid];
if (!tx) {
try {
tx = await bitcoinApi.$getRawTransaction(txid);
this.txCache[txid] = tx;
if (temp) {
this.tempCached.push(txid);
}
await Common.sleep$(throttleDelay);
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
return null;
}
}
return tx;
}
clearTempCache(): void {
for (const txid of this.tempCached) {
delete this.txCache[txid];
}
this.tempCached = [];
}
truncateTempCache(): void {
if (this.tempCached.length > tempCacheSize) {
const removed = this.tempCached.splice(0, this.tempCached.length - tempCacheSize);
for (const txid of removed) {
delete this.txCache[txid];
}
}
}
}
export default new ForensicsService();

3
backend/src/tasks/lightning/network-sync.service.ts
View File

@ -31,6 +31,7 @@ class NetworkSyncService {
}
private async $runTasks(): Promise<void> {
const taskStartTime = Date.now();
try {
logger.info(`Updating nodes and channels`);
@ -57,7 +58,7 @@ class NetworkSyncService {
logger.err('$runTasks() error: ' + (e instanceof Error ? e.message : e));
}
setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL);
setTimeout(() => { this.$runTasks(); }, Math.max(1, (1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL) - (Date.now() - taskStartTime)));
}
/**

3
frontend/mempool-frontend-config.sample.json
View File

@ -17,5 +17,8 @@
"LIQUID_WEBSITE_URL": "https://liquid.network",
"BISQ_WEBSITE_URL": "https://bisq.markets",
"MINING_DASHBOARD": true,
"MAINNET_BLOCK_AUDIT_START_HEIGHT": 0,
"TESTNET_BLOCK_AUDIT_START_HEIGHT": 0,
"SIGNET_BLOCK_AUDIT_START_HEIGHT": 0,
"LIGHTNING": false
}

925
frontend/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

20
frontend/package.json
View File

@ -84,28 +84,28 @@
"browserify": "^17.0.0",
"clipboard": "^2.0.11",
"domino": "^2.1.6",
"echarts": "~5.3.2",
"echarts": "~5.4.0",
"echarts-gl": "^2.0.9",
"lightweight-charts": "~3.8.0",
"ngx-echarts": "8.0.1",
"ngx-echarts": "~14.0.0",
"ngx-infinite-scroll": "^14.0.1",
"qrcode": "1.5.0",
"qrcode": "1.5.1",
"rxjs": "~7.5.7",
"tinyify": "^3.1.0",
"tlite": "^0.1.9",
"tslib": "~2.4.1",
"zone.js": "~0.11.5"
"zone.js": "~0.12.0"
},
"devDependencies": {
"@angular/compiler-cli": "^14.2.12",
"@angular/language-service": "^14.2.12",
"@types/node": "^12.11.1",
"@typescript-eslint/eslint-plugin": "^5.30.5",
"@typescript-eslint/parser": "^5.30.5",
"eslint": "^8.19.0",
"@types/node": "^18.11.9",
"@typescript-eslint/eslint-plugin": "^5.45.0",
"@typescript-eslint/parser": "^5.45.0",
"eslint": "^8.28.0",
"http-proxy-middleware": "~2.0.6",
"prettier": "^2.7.1",
"ts-node": "~10.8.1",
"prettier": "^2.8.0",
"ts-node": "~10.9.1",
"typescript": "~4.6.4"
},
"optionalDependencies": {

2
frontend/src/app/components/about/about.component.html
View File

@ -129,7 +129,7 @@
<span>Gemini</span>
</a>
<a href="https://exodus.com/" target="_blank" title="Exodus">
<svg width="81" height="81" viewBox="0 0 500 500" fill="none" xmlns="http://www.w3.org/2000/svg">
<svg width="80" height="80" viewBox="0 0 500 500" fill="none" xmlns="http://www.w3.org/2000/svg">
<circle cx="250" cy="250" r="250" fill="#1F2033"/>
<g clip-path="url(#clip0_2_14)">
<path d="M411.042 178.303L271.79 87V138.048L361.121 196.097L350.612 229.351H271.79V271.648H350.612L361.121 304.903L271.79 362.952V414L411.042 322.989L388.271 250.646L411.042 178.303Z" fill="url(#paint0_linear_2_14)"/>

6
frontend/src/app/components/about/about.component.scss
View File

@ -3,8 +3,8 @@
text-align: center;
.image {
width: 81px;
height: 81px;
width: 80px;
height: 80px;
background-size: 100%, 100%;
border-radius: 50%;
margin: 25px;
@ -191,6 +191,6 @@
}
.community-integrations-sponsor {
max-width: 970px;
max-width: 965px;
margin: auto;
}

1
frontend/src/app/components/bisq-master-page/bisq-master-page.component.ts
View File

@ -30,7 +30,6 @@ export class BisqMasterPageComponent implements OnInit {
this.connectionState$ = this.stateService.connectionState$;
this.urlLanguage = this.languageService.getLanguageForUrl();
this.navigationService.subnetPaths.subscribe((paths) => {
console.log('network paths updated...');
this.networkPaths = paths;
});
}

12
frontend/src/app/components/block/block.component.html
View File

@ -61,7 +61,7 @@
<span *ngIf="blockAudit?.matchRate === null" i18n="unknown">Unknown</span>
</td>
</tr>
<ng-container *ngIf="!indexingAvailable && webGlEnabled">
<ng-container *ngIf="webGlEnabled && (auditDataMissing || !indexingAvailable)">
<tr *ngIf="isMobile && auditEnabled"></tr>
<tr *ngIf="network !== 'liquid' && network !== 'liquidtestnet'">
<td i18n="mempool-block.fee-span">Fee span</td>
@ -146,7 +146,7 @@
<tr *ngIf="network !== 'liquid' && network !== 'liquidtestnet'">
<td colspan="2"><span class="skeleton-loader"></span></td>
</tr>
<ng-container *ngIf="!indexingAvailable && webGlEnabled">
<ng-container *ngIf="webGlEnabled && (!indexingAvailable || auditDataMissing)">
<tr *ngIf="isMobile && !auditEnabled"></tr>
<tr>
<td class="td-width" colspan="2"><span class="skeleton-loader"></span></td>
@ -169,7 +169,7 @@
</div>
</ng-template>
<div class="col-sm">
<table class="table table-borderless table-striped" *ngIf="!isLoadingBlock && (indexingAvailable || !webGlEnabled)">
<table class="table table-borderless table-striped" *ngIf="!isLoadingBlock && (!auditDataMissing || indexingAvailable && !webGlEnabled)">
<tbody>
<tr *ngIf="isMobile && auditEnabled"></tr>
<tr *ngIf="network !== 'liquid' && network !== 'liquidtestnet'">
@ -233,7 +233,7 @@
</tr>
</tbody>
</table>
<table class="table table-borderless table-striped" *ngIf="isLoadingBlock && (indexingAvailable || !webGlEnabled)">
<table class="table table-borderless table-striped" *ngIf="isLoadingBlock && !auditDataMissing && (indexingAvailable || !webGlEnabled)">
<tbody>
<tr *ngIf="isMobile && !auditEnabled"></tr>
<tr>
@ -253,7 +253,7 @@
</tr>
</tbody>
</table>
<div class="col-sm chart-container" *ngIf="webGlEnabled && !indexingAvailable">
<div class="col-sm chart-container" *ngIf="webGlEnabled && (!indexingAvailable || auditDataMissing)">
<app-block-overview-graph
#blockGraphActual
[isLoading]="isLoadingOverview"
@ -273,7 +273,7 @@
<br>
<!-- VISUALIZATIONS -->
<div class="box" *ngIf="!error && webGlEnabled && indexingAvailable">
<div class="box" *ngIf="!error && webGlEnabled && indexingAvailable && !auditDataMissing">
<div class="nav nav-tabs" *ngIf="isMobile && auditEnabled">
<a class="nav-link" [class.active]="mode === 'projected'" i18n="block.projected"
fragment="projected" (click)="changeMode('projected')">Projected</a>

26
frontend/src/app/components/block/block.component.ts
View File

@ -58,6 +58,7 @@ export class BlockComponent implements OnInit, OnDestroy {
webGlEnabled = true;
indexingAvailable = false;
auditEnabled = true;
auditDataMissing: boolean;
isMobile = window.innerWidth <= 767.98;
hoverTx: string;
numMissing: number = 0;
@ -137,9 +138,11 @@ export class BlockComponent implements OnInit, OnDestroy {
this.error = undefined;
this.fees = undefined;
this.stateService.markBlock$.next({});
this.auditDataMissing = false;
if (history.state.data && history.state.data.blockHeight) {
this.blockHeight = history.state.data.blockHeight;
this.updateAuditDataMissingFromBlockHeight(this.blockHeight);
}
let isBlockHeight = false;
@ -152,6 +155,7 @@ export class BlockComponent implements OnInit, OnDestroy {
if (history.state.data && history.state.data.block) {
this.blockHeight = history.state.data.block.height;
this.updateAuditDataMissingFromBlockHeight(this.blockHeight);
return of(history.state.data.block);
} else {
this.isLoadingBlock = true;
@ -213,7 +217,7 @@ export class BlockComponent implements OnInit, OnDestroy {
this.apiService.getBlockAudit$(block.previousblockhash);
}, 100);
}
this.updateAuditDataMissingFromBlockHeight(block.height);
this.block = block;
this.blockHeight = block.height;
this.lastBlockHeight = this.blockHeight;
@ -363,6 +367,7 @@ export class BlockComponent implements OnInit, OnDestroy {
this.auditEnabled = true;
} else {
this.auditEnabled = false;
this.auditDataMissing = true;
}
return blockAudit;
}),
@ -582,4 +587,23 @@ export class BlockComponent implements OnInit, OnDestroy {
this.hoverTx = null;
}
}
updateAuditDataMissingFromBlockHeight(blockHeight: number): void {
switch (this.stateService.network) {
case 'testnet':
if (blockHeight < this.stateService.env.TESTNET_BLOCK_AUDIT_START_HEIGHT) {
this.auditDataMissing = true;
}
break;
case 'signet':
if (blockHeight < this.stateService.env.SIGNET_BLOCK_AUDIT_START_HEIGHT) {
this.auditDataMissing = true;
}
break;
default:
if (blockHeight < this.stateService.env.MAINNET_BLOCK_AUDIT_START_HEIGHT) {
this.auditDataMissing = true;
}
}
}
}

1
frontend/src/app/components/liquid-master-page/liquid-master-page.component.ts
View File

@ -33,7 +33,6 @@ export class LiquidMasterPageComponent implements OnInit {
this.network$ = merge(of(''), this.stateService.networkChanged$);
this.urlLanguage = this.languageService.getLanguageForUrl();
this.navigationService.subnetPaths.subscribe((paths) => {
console.log('network paths updated...');
this.networkPaths = paths;
});
}

1
frontend/src/app/components/master-page/master-page.component.ts
View File

@ -35,7 +35,6 @@ export class MasterPageComponent implements OnInit {
this.urlLanguage = this.languageService.getLanguageForUrl();
this.subdomain = this.enterpriseService.getSubdomain();
this.navigationService.subnetPaths.subscribe((paths) => {
console.log('network paths updated...');
this.networkPaths = paths;
});
}

36
frontend/src/app/components/transaction/transaction-preview.component.ts
View File

@ -63,40 +63,14 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy {
this.fetchCpfpSubscription = this.fetchCpfp$
.pipe(
switchMap((txId) =>
this.apiService
.getCpfpinfo$(txId)
.pipe(retryWhen((errors) => errors.pipe(delay(2000))))
this.apiService.getCpfpinfo$(txId).pipe(
catchError((err) => {
return of(null);
})
)
)
)
.subscribe((cpfpInfo) => {
if (!this.tx) {
return;
}
if (cpfpInfo.effectiveFeePerVsize) {
this.tx.effectiveFeePerVsize = cpfpInfo.effectiveFeePerVsize;
} else {
const lowerFeeParents = cpfpInfo.ancestors.filter(
(parent) => parent.fee / (parent.weight / 4) < this.tx.feePerVsize
);
let totalWeight =
this.tx.weight +
lowerFeeParents.reduce((prev, val) => prev + val.weight, 0);
let totalFees =
this.tx.fee +
lowerFeeParents.reduce((prev, val) => prev + val.fee, 0);
if (cpfpInfo?.bestDescendant) {
totalWeight += cpfpInfo?.bestDescendant.weight;
totalFees += cpfpInfo?.bestDescendant.fee;
}
this.tx.effectiveFeePerVsize = totalFees / (totalWeight / 4);
}
if (!this.tx.status.confirmed) {
this.stateService.markBlock$.next({
txFeePerVSize: this.tx.effectiveFeePerVsize,
});
}
this.cpfpInfo = cpfpInfo;
this.openGraphService.waitOver('cpfp-data-' + this.txId);
});

24
frontend/src/app/components/transaction/transaction.component.ts
View File

@ -7,10 +7,11 @@ import {
catchError,
retryWhen,
delay,
map
map,
mergeMap
} from 'rxjs/operators';
import { Transaction } from '../../interfaces/electrs.interface';
import { of, merge, Subscription, Observable, Subject, timer, combineLatest, from } from 'rxjs';
import { of, merge, Subscription, Observable, Subject, timer, combineLatest, from, throwError } from 'rxjs';
import { StateService } from '../../services/state.service';
import { WebsocketService } from '../../services/websocket.service';
import { AudioService } from '../../services/audio.service';
@ -110,11 +111,24 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
switchMap((txId) =>
this.apiService
.getCpfpinfo$(txId)
.pipe(retryWhen((errors) => errors.pipe(delay(2000))))
)
.pipe(retryWhen((errors) => errors.pipe(
mergeMap((error) => {
if (!this.tx?.status || this.tx.status.confirmed) {
return throwError(error);
} else {
return of(null);
}
}),
delay(2000)
)))
),
catchError(() => {
return of(null);
})
)
.subscribe((cpfpInfo) => {
if (!this.tx) {
if (!cpfpInfo || !this.tx) {
this.cpfpInfo = null;
return;
}
if (cpfpInfo.effectiveFeePerVsize) {

6
frontend/src/app/interfaces/node-api.interface.ts
View File

@ -217,8 +217,8 @@ export interface IChannel {
updated_at: string;
created: string;
status: number;
node_left: Node,
node_right: Node,
node_left: INode,
node_right: INode,
}
@ -236,4 +236,6 @@ export interface INode {
updated_at: string;
longitude: number;
latitude: number;
funding_balance?: number;
closing_balance?: number;
}

19
frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.html Normal file
View File

@ -0,0 +1,19 @@
<div class="box">
<table class="table table-borderless table-striped">
<tbody>
<tr></tr>
<tr>
<td i18n="lightning.starting-balance|Channel starting balance">Starting balance</td>
<td *ngIf="showStartingBalance && minStartingBalance === maxStartingBalance"><app-sats [satoshis]="minStartingBalance"></app-sats></td>
<td *ngIf="showStartingBalance && minStartingBalance !== maxStartingBalance">{{ minStartingBalance | number : '1.0-0' }} - {{ maxStartingBalance | number : '1.0-0' }}<app-sats [valueOverride]=" "></app-sats></td>
<td *ngIf="!showStartingBalance">?</td>
</tr>
<tr *ngIf="channel.status === 2">
<td i18n="lightning.closing-balance|Channel closing balance">Closing balance</td>
<td *ngIf="showClosingBalance && minClosingBalance === maxClosingBalance"><app-sats [satoshis]="minClosingBalance"></app-sats></td>
<td *ngIf="showClosingBalance && minClosingBalance !== maxClosingBalance">{{ minClosingBalance | number : '1.0-0' }} - {{ maxClosingBalance | number : '1.0-0' }}<app-sats [valueOverride]=" "></app-sats></td>
<td *ngIf="!showClosingBalance">?</td>
</tr>
</tbody>
</table>
</div>

9
frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.scss Normal file
View File

@ -0,0 +1,9 @@
.box {
margin-top: 20px;
}
@media (max-width: 768px) {
.box {
margin-bottom: 20px;
}
}

25
frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.spec.ts Normal file
View File

@ -0,0 +1,25 @@
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { ChannelCloseBoxComponent } from './channel-close-box.component';
describe('ChannelCloseBoxComponent', () => {
let component: ChannelCloseBoxComponent;
let fixture: ComponentFixture<ChannelCloseBoxComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
declarations: [ ChannelCloseBoxComponent ]
})
.compileComponents();
});
beforeEach(() => {
fixture = TestBed.createComponent(ChannelCloseBoxComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});
it('should create', () => {
expect(component).toBeTruthy();
});
});

58
frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.ts Normal file
View File

@ -0,0 +1,58 @@
import { ChangeDetectionStrategy, Component, Input, OnChanges, SimpleChanges } from '@angular/core';
@Component({
selector: 'app-channel-close-box',
templateUrl: './channel-close-box.component.html',
styleUrls: ['./channel-close-box.component.scss'],
changeDetection: ChangeDetectionStrategy.OnPush,
})
export class ChannelCloseBoxComponent implements OnChanges {
@Input() channel: any;
@Input() local: any;
@Input() remote: any;
showStartingBalance: boolean = false;
showClosingBalance: boolean = false;
minStartingBalance: number;
maxStartingBalance: number;
minClosingBalance: number;
maxClosingBalance: number;
constructor() { }
ngOnChanges(changes: SimpleChanges): void {
if (this.channel && this.local && this.remote) {
this.showStartingBalance = (this.local.funding_balance || this.remote.funding_balance) && this.channel.funding_ratio;
this.showClosingBalance = this.local.closing_balance || this.remote.closing_balance;
if (this.channel.single_funded) {
if (this.local.funding_balance) {
this.minStartingBalance = this.channel.capacity;
this.maxStartingBalance = this.channel.capacity;
} else if (this.remote.funding_balance) {
this.minStartingBalance = 0;
this.maxStartingBalance = 0;
}
} else {
this.minStartingBalance = clampRound(0, this.channel.capacity, this.local.funding_balance * this.channel.funding_ratio);
this.maxStartingBalance = clampRound(0, this.channel.capacity, this.channel.capacity - (this.remote.funding_balance * this.channel.funding_ratio));
}
const closingCapacity = this.channel.capacity - this.channel.closing_fee;
this.minClosingBalance = clampRound(0, closingCapacity, this.local.closing_balance);
this.maxClosingBalance = clampRound(0, closingCapacity, closingCapacity - this.remote.closing_balance);
// margin of error to account for 2 x 330 sat anchor outputs
if (Math.abs(this.minClosingBalance - this.maxClosingBalance) <= 660) {
this.maxClosingBalance = this.minClosingBalance;
}
} else {
this.showStartingBalance = false;
this.showClosingBalance = false;
}
}
}
function clampRound(min: number, max: number, value: number): number {
return Math.max(0, Math.min(max, Math.round(value)));
}

11
frontend/src/app/lightning/channel/channel.component.html
View File

@ -48,6 +48,15 @@
<td i18n="lightning.capacity">Capacity</td>
<td><app-sats [satoshis]="channel.capacity"></app-sats><app-fiat [value]="channel.capacity" digitsInfo="1.0-0"></app-fiat></td>
</tr>
<tr *ngIf="channel.closed_by">
<td i18n="lightning.closed_by">Closed by</td>
<td>
<a [routerLink]="['/lightning/node' | relativeUrl, channel.closed_by]" >
<ng-container *ngIf="channel.closed_by === channel.node_left.public_key">{{ channel.node_left.alias }}</ng-container>
<ng-container *ngIf="channel.closed_by === channel.node_right.public_key">{{ channel.node_right.alias }}</ng-container>
</a>
</td>
</tr>
</tbody>
</table>
</div>
@ -59,9 +68,11 @@
<div class="row row-cols-1 row-cols-md-2">
<div class="col">
<app-channel-box [channel]="channel.node_left"></app-channel-box>
<app-channel-close-box *ngIf="showCloseBoxes(channel)" [channel]="channel" [local]="channel.node_left" [remote]="channel.node_right"></app-channel-close-box>
</div>
<div class="col">
<app-channel-box [channel]="channel.node_right"></app-channel-box>
<app-channel-close-box *ngIf="showCloseBoxes(channel)" [channel]="channel" [local]="channel.node_right" [remote]="channel.node_left"></app-channel-close-box>
</div>
</div>

5
frontend/src/app/lightning/channel/channel.component.ts
View File

@ -78,4 +78,9 @@ export class ChannelComponent implements OnInit {
);
}
showCloseBoxes(channel: IChannel): boolean {
return !!(channel.node_left.funding_balance || channel.node_left.closing_balance
|| channel.node_right.funding_balance || channel.node_right.closing_balance);
}
}

3
frontend/src/app/lightning/lightning.module.ts
View File

@ -12,6 +12,7 @@ import { ChannelsListComponent } from './channels-list/channels-list.component';
import { ChannelComponent } from './channel/channel.component';
import { LightningWrapperComponent } from './lightning-wrapper/lightning-wrapper.component';
import { ChannelBoxComponent } from './channel/channel-box/channel-box.component';
import { ChannelCloseBoxComponent } from './channel/channel-close-box/channel-close-box.component';
import { ClosingTypeComponent } from './channel/closing-type/closing-type.component';
import { LightningStatisticsChartComponent } from './statistics-chart/lightning-statistics-chart.component';
import { NodeStatisticsChartComponent } from './node-statistics-chart/node-statistics-chart.component';
@ -45,6 +46,7 @@ import { GroupComponent } from './group/group.component';
ChannelComponent,
LightningWrapperComponent,
ChannelBoxComponent,
ChannelCloseBoxComponent,
ClosingTypeComponent,
LightningStatisticsChartComponent,
NodesNetworksChartComponent,
@ -81,6 +83,7 @@ import { GroupComponent } from './group/group.component';
ChannelComponent,
LightningWrapperComponent,
ChannelBoxComponent,
ChannelCloseBoxComponent,
ClosingTypeComponent,
LightningStatisticsChartComponent,
NodesNetworksChartComponent,

6
frontend/src/app/services/state.service.ts
View File

@ -39,6 +39,9 @@ export interface Env {
BISQ_WEBSITE_URL: string;
MINING_DASHBOARD: boolean;
LIGHTNING: boolean;
MAINNET_BLOCK_AUDIT_START_HEIGHT: number;
TESTNET_BLOCK_AUDIT_START_HEIGHT: number;
SIGNET_BLOCK_AUDIT_START_HEIGHT: number;
}
const defaultEnv: Env = {
@ -64,6 +67,9 @@ const defaultEnv: Env = {
'BISQ_WEBSITE_URL': 'https://bisq.markets',
'MINING_DASHBOARD': true,
'LIGHTNING': false,
'MAINNET_BLOCK_AUDIT_START_HEIGHT': 0,
'TESTNET_BLOCK_AUDIT_START_HEIGHT': 0,
'SIGNET_BLOCK_AUDIT_START_HEIGHT': 0,
};
@Injectable({

4
frontend/src/app/shared/graphs.utils.ts
View File

@ -85,10 +85,10 @@ export const download = (href, name) => {
document.body.removeChild(a);
};
export function detectWebGL() {
export function detectWebGL(): boolean {
const canvas = document.createElement('canvas');
const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
return (gl && gl instanceof WebGLRenderingContext);
return !!(gl && gl instanceof WebGLRenderingContext);
}
/**

6
production/install
View File

@ -1842,13 +1842,13 @@ create database mempool_signet;
grant all on mempool_signet.* to '${MEMPOOL_SIGNET_USER}'@'localhost' identified by '${MEMPOOL_SIGNET_PASS}';
create database mempool_mainnet_lightning;
grant all on mempool_mainnet_lightning.* to '${LN_MEMPOOL_MAINNET_USER}'@'%' identified by '${LN_MEMPOOL_MAINNET_PASS}';
grant all on mempool_mainnet_lightning.* to '${LN_MEMPOOL_MAINNET_USER}'@'localhost' identified by '${LN_MEMPOOL_MAINNET_PASS}';
create database mempool_testnet_lightning;
grant all on mempool_testnet_lightning.* to '${LN_MEMPOOL_TESTNET_USER}'@'%' identified by '${LN_MEMPOOL_TESTNET_PASS}';
grant all on mempool_testnet_lightning.* to '${LN_MEMPOOL_TESTNET_USER}'@'localhost' identified by '${LN_MEMPOOL_TESTNET_PASS}';
create database mempool_signet_lightning;
grant all on mempool_signet_lightning.* to '${LN_MEMPOOL_SIGNET_USER}'@'%' identified by '${LN_MEMPOOL_SIGNET_PASS}';
grant all on mempool_signet_lightning.* to '${LN_MEMPOOL_SIGNET_USER}'@'localhost' identified by '${LN_MEMPOOL_SIGNET_PASS}';
create database mempool_liquid;
grant all on mempool_liquid.* to '${MEMPOOL_LIQUID_USER}'@'localhost' identified by '${MEMPOOL_LIQUID_PASS}';

2
production/mempool-config.mainnet.json
View File

@ -10,6 +10,8 @@
"POLL_RATE_MS": 1000,
"INDEXING_BLOCKS_AMOUNT": -1,
"BLOCKS_SUMMARIES_INDEXING": true,
"ADVANCED_GBT_AUDIT": true,
"ADVANCED_GBT_MEMPOOL": false,
"USE_SECOND_NODE_FOR_MINFEE": true
},
"SYSLOG" : {

2
production/mempool-config.signet.json
View File

@ -7,6 +7,8 @@
"SPAWN_CLUSTER_PROCS": 0,
"API_URL_PREFIX": "/api/v1/",
"INDEXING_BLOCKS_AMOUNT": -1,
"ADVANCED_GBT_AUDIT": true,
"ADVANCED_GBT_MEMPOOL": false,
"POLL_RATE_MS": 1000
},
"SYSLOG" : {

2
production/mempool-config.testnet.json
View File

@ -7,6 +7,8 @@
"SPAWN_CLUSTER_PROCS": 0,
"API_URL_PREFIX": "/api/v1/",
"INDEXING_BLOCKS_AMOUNT": -1,
"ADVANCED_GBT_AUDIT": true,
"ADVANCED_GBT_MEMPOOL": false,
"POLL_RATE_MS": 1000
},
"SYSLOG" : {

2
production/nginx/http-basic.conf
View File

@ -35,3 +35,5 @@ gzip_types application/javascript application/json application/ld+json applicati
# limit request body size
client_max_body_size 10m;
# need to bump this up for about page sponsor images lol
http2_max_concurrent_streams 256;