diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 8635ee96f..b9cc1453c 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -228,34 +228,75 @@ export class Common { return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0]; } - static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket { + static findSocketNetwork(addr: string): {network: string | null, url: string} { let network: string | null = null; + let url = addr.split('://')[1]; - if (config.LIGHTNING.BACKEND === 'cln') { - network = socket.network; - } else if (config.LIGHTNING.BACKEND === 'lnd') { - if (socket.addr.indexOf('onion') !== -1) { - if (socket.addr.split('.')[0].length >= 56) { - network = 'torv3'; - } else { - network = 'torv2'; - } - } else if (socket.addr.indexOf('i2p') !== -1) { - network = 'i2p'; + if (!url) { + return { + network: null, + url: addr, + }; + } + + if (addr.indexOf('onion') !== -1) { + if (url.split('.')[0].length >= 56) { + network = 'torv3'; } else { - const ipv = isIP(socket.addr.split(':')[0]); - if (ipv === 4) { - network = 'ipv4'; - } else if (ipv === 6) { - network = 'ipv6'; - } + network = 'torv2'; } + } else if (addr.indexOf('i2p') !== -1) { + network = 'i2p'; + } else if (addr.indexOf('ipv4') !== -1) { + const ipv = isIP(url.split(':')[0]); + if (ipv === 4) { + network = 'ipv4'; + } else { + return { + network: null, + url: addr, + }; + } + } else if (addr.indexOf('ipv6') !== -1) { + url = url.split('[')[1].split(']')[0]; + const ipv = isIP(url); + if (ipv === 6) { + const parts = addr.split(':'); + network = 'ipv6'; + url = `[${url}]:${parts[parts.length - 1]}`; + } else { + return { + network: null, + url: addr, + }; + } + } else { + return { + network: null, + url: addr, + }; } return { - publicKey: publicKey, network: network, - addr: socket.addr, + url: url, }; } + + static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket { + if (config.LIGHTNING.BACKEND === 'cln') { + return { + publicKey: publicKey, + network: socket.network, + addr: socket.addr, + }; + } else /* if (config.LIGHTNING.BACKEND === 'lnd') */ { + const formatted = this.findSocketNetwork(socket.addr); + return { + publicKey: publicKey, + network: formatted.network, + addr: formatted.url, + }; + } + } } diff --git a/backend/src/api/explorer/channels.routes.ts b/backend/src/api/explorer/channels.routes.ts index eda3a6168..2b7f3fa6d 100644 --- a/backend/src/api/explorer/channels.routes.ts +++ b/backend/src/api/explorer/channels.routes.ts @@ -70,7 +70,7 @@ class ChannelsRoutes { } } - private async $getChannelsByTransactionIds(req: Request, res: Response) { + private async $getChannelsByTransactionIds(req: Request, res: Response): Promise { try { if (!Array.isArray(req.query.txId)) { res.status(400).send('Not an array'); @@ -83,27 +83,26 @@ class ChannelsRoutes { } } const channels = await channelsApi.$getChannelsByTransactionId(txIds); - const inputs: any[] = []; - const outputs: any[] = []; + const result: any[] = []; for (const txid of txIds) { - const foundChannelInputs = channels.find((channel) => channel.closing_transaction_id === txid); - if (foundChannelInputs) { - inputs.push(foundChannelInputs); - } else { - inputs.push(null); + const inputs: any = {}; + const outputs: any = {}; + // Assuming that we only have one lightning close input in each transaction. This may not be true in the future + const foundChannelsFromInput = channels.find((channel) => channel.closing_transaction_id === txid); + if (foundChannelsFromInput) { + inputs[0] = foundChannelsFromInput; } - const foundChannelOutputs = channels.find((channel) => channel.transaction_id === txid); - if (foundChannelOutputs) { - outputs.push(foundChannelOutputs); - } else { - outputs.push(null); + const foundChannelsFromOutputs = channels.filter((channel) => channel.transaction_id === txid); + for (const output of foundChannelsFromOutputs) { + outputs[output.transaction_vout] = output; } + result.push({ + inputs, + outputs, + }); } - res.json({ - inputs: inputs, - outputs: outputs, - }); + res.json(result); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); } diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index c49ed9ac5..fcd5b8815 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -412,6 +412,7 @@ class NodesApi { nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at, geo_names_city.names as city, geo_names_country.names as country, geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision + FROM nodes LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code' @@ -502,6 +503,18 @@ class NodesApi { } } + /** + * Update node sockets + */ + public async $updateNodeSockets(publicKey: string, sockets: {network: string; addr: string}[]): Promise { + const formattedSockets = (sockets.map(a => a.addr).join(',')) ?? ''; + try { + await DB.query(`UPDATE nodes SET sockets = ? WHERE public_key = ?`, [formattedSockets, publicKey]); + } catch (e) { + logger.err(`Cannot update node sockets for ${publicKey}. Reason: ${e instanceof Error ? e.message : e}`); + } + } + /** * Set all nodes not in `nodesPubkeys` as inactive (status = 0) */ diff --git a/backend/src/api/explorer/statistics.api.ts b/backend/src/api/explorer/statistics.api.ts index 7bf3d9107..558ee86fd 100644 --- a/backend/src/api/explorer/statistics.api.ts +++ b/backend/src/api/explorer/statistics.api.ts @@ -27,7 +27,7 @@ class StatisticsApi { public async $getLatestStatistics(): Promise { try { const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1`); - const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1 OFFSET 7`); + const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats WHERE DATE(added) = DATE(NOW() - INTERVAL 7 DAY)`); return { latest: rows[0], previous: rows2[0], diff --git a/backend/src/logger.ts b/backend/src/logger.ts index ea7e8cd3d..63774d513 100644 --- a/backend/src/logger.ts +++ b/backend/src/logger.ts @@ -74,7 +74,7 @@ class Logger { private getNetwork(): string { if (config.LIGHTNING.ENABLED) { - return 'lightning'; + return config.MEMPOOL.NETWORK === 'mainnet' ? 'lightning' : `${config.MEMPOOL.NETWORK}-lightning`; } if (config.BISQ.ENABLED) { return 'bisq'; diff --git a/backend/src/tasks/lightning/sync-tasks/node-locations.ts b/backend/src/tasks/lightning/sync-tasks/node-locations.ts index 30a6bfc2a..fbcc6c30d 100644 --- a/backend/src/tasks/lightning/sync-tasks/node-locations.ts +++ b/backend/src/tasks/lightning/sync-tasks/node-locations.ts @@ -4,6 +4,7 @@ import nodesApi from '../../../api/explorer/nodes.api'; import config from '../../../config'; import DB from '../../../database'; import logger from '../../../logger'; +import * as IPCheck from '../../../utils/ipcheck.js'; export async function $lookupNodeLocation(): Promise { let loggerTimer = new Date().getTime() / 1000; @@ -27,6 +28,11 @@ export async function $lookupNodeLocation(): Promise { const asn = lookupAsn.get(ip); const isp = lookupIsp.get(ip); + let asOverwrite: number | null = null; + if (asn && (IPCheck.match(ip, '170.75.160.0/20') || IPCheck.match(ip, '172.81.176.0/21'))) { + asOverwrite = 394745; + } + if (city && (asn || isp)) { const query = ` UPDATE nodes SET @@ -41,7 +47,7 @@ export async function $lookupNodeLocation(): Promise { `; const params = [ - isp?.autonomous_system_number ?? asn?.autonomous_system_number, + asOverwrite ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number, city.city?.geoname_id, city.country?.geoname_id, city.subdivisions ? city.subdivisions[0].geoname_id : null, @@ -91,7 +97,7 @@ export async function $lookupNodeLocation(): Promise { if (isp?.autonomous_system_organization ?? asn?.autonomous_system_organization) { await DB.query( `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'as_organization', ?)`, - [isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]); + [asOverwrite ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]); } } diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index 7ac1c5885..e3dfe6652 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -8,6 +8,7 @@ import { isIP } from 'net'; import { Common } from '../../../api/common'; import channelsApi from '../../../api/explorer/channels.api'; import nodesApi from '../../../api/explorer/nodes.api'; +import { ResultSetHeader } from 'mysql2'; const fsPromises = promises; @@ -19,7 +20,12 @@ class LightningStatsImporter { logger.info('Caching funding txs for currently existing channels'); await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id)); + if (config.MEMPOOL.NETWORK !== 'mainnet' || config.DATABASE.ENABLED === false) { + return; + } + await this.$importHistoricalLightningStats(); + await this.$cleanupIncorrectSnapshot(); } /** @@ -51,6 +57,8 @@ class LightningStatsImporter { features: node.features, }); nodesInDb[node.pub_key] = node; + } else { + await nodesApi.$updateNodeSockets(node.pub_key, node.addresses); } let hasOnion = false; @@ -363,10 +371,16 @@ class LightningStatsImporter { graph = JSON.parse(fileContent); graph = await this.cleanupTopology(graph); } catch (e) { - logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`); + logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content. Reason: ${e instanceof Error ? e.message : e}`); continue; } + if (this.isIncorrectSnapshot(timestamp, graph)) { + logger.debug(`Ignoring ${this.topologiesFolder}/${filename}, because we defined it as an incorrect snapshot`); + ++totalProcessed; + continue; + } + if (!logStarted) { logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`); logStarted = true; @@ -397,7 +411,7 @@ class LightningStatsImporter { } } - async cleanupTopology(graph) { + cleanupTopology(graph): ILightningApi.NetworkGraph { const newGraph = { nodes: [], edges: [], @@ -407,9 +421,10 @@ class LightningStatsImporter { const addressesParts = (node.addresses ?? '').split(','); const addresses: any[] = []; for (const address of addressesParts) { + const formatted = Common.findSocketNetwork(address); addresses.push({ - network: '', - addr: address + network: formatted.network, + addr: formatted.url }); } @@ -456,6 +471,69 @@ class LightningStatsImporter { return newGraph; } + + private isIncorrectSnapshot(timestamp, graph): boolean { + if (timestamp >= 1549065600 /* 2019-02-02 */ && timestamp <= 1550620800 /* 2019-02-20 */ && graph.nodes.length < 2600) { + return true; + } + if (timestamp >= 1552953600 /* 2019-03-19 */ && timestamp <= 1556323200 /* 2019-05-27 */ && graph.nodes.length < 4000) { + return true; + } + if (timestamp >= 1557446400 /* 2019-05-10 */ && timestamp <= 1560470400 /* 2019-06-14 */ && graph.nodes.length < 4000) { + return true; + } + if (timestamp >= 1561680000 /* 2019-06-28 */ && timestamp <= 1563148800 /* 2019-07-15 */ && graph.nodes.length < 4000) { + return true; + } + if (timestamp >= 1571270400 /* 2019-11-17 */ && timestamp <= 1580601600 /* 2020-02-02 */ && graph.nodes.length < 4500) { + return true; + } + if (timestamp >= 1591142400 /* 2020-06-03 */ && timestamp <= 1592006400 /* 2020-06-13 */ && graph.nodes.length < 5500) { + return true; + } + if (timestamp >= 1632787200 /* 2021-09-28 */ && timestamp <= 1633564800 /* 2021-10-07 */ && graph.nodes.length < 13000) { + return true; + } + if (timestamp >= 1634256000 /* 2021-10-15 */ && timestamp <= 1645401600 /* 2022-02-21 */ && graph.nodes.length < 17000) { + return true; + } + if (timestamp >= 1654992000 /* 2022-06-12 */ && timestamp <= 1661472000 /* 2022-08-26 */ && graph.nodes.length < 14000) { + return true; + } + + return false; + } + + private async $cleanupIncorrectSnapshot(): Promise { + // We do not run this one automatically because those stats are not supposed to be inserted in the first + // place, but I write them here to remind us we manually run those queries + + // DELETE FROM lightning_stats + // WHERE ( + // UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 AND node_count < 2600 OR + // UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 AND node_count < 4000 OR + // UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 AND node_count < 4000 OR + // UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 AND node_count < 4000 OR + // UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 AND node_count < 4500 OR + // UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 AND node_count < 5500 OR + // UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 AND node_count < 13000 OR + // UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 AND node_count < 17000 OR + // UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000 AND node_count < 14000 + // ) + + // DELETE FROM node_stats + // WHERE ( + // UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 OR + // UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 OR + // UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 OR + // UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 OR + // UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 OR + // UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 OR + // UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 OR + // UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 OR + // UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000 + // ) + } } export default new LightningStatsImporter; diff --git a/backend/src/utils/ipcheck.js b/backend/src/utils/ipcheck.js new file mode 100644 index 000000000..06d4a6f15 --- /dev/null +++ b/backend/src/utils/ipcheck.js @@ -0,0 +1,119 @@ +var net = require('net'); + +var IPCheck = module.exports = function(input) { + var self = this; + + if (!(self instanceof IPCheck)) { + return new IPCheck(input); + } + + self.input = input; + self.parse(); +}; + +IPCheck.prototype.parse = function() { + var self = this; + + if (!self.input || typeof self.input !== 'string') return self.valid = false; + + var ip; + + var pos = self.input.lastIndexOf('/'); + if (pos !== -1) { + ip = self.input.substring(0, pos); + self.mask = +self.input.substring(pos + 1); + } else { + ip = self.input; + self.mask = null; + } + + self.ipv = net.isIP(ip); + self.valid = !!self.ipv && !isNaN(self.mask); + + if (!self.valid) return; + + // default mask = 32 for ipv4 and 128 for ipv6 + if (self.mask === null) self.mask = self.ipv === 4 ? 32 : 128; + + if (self.ipv === 4) { + // difference between ipv4 and ipv6 masks + self.mask += 96; + } + + if (self.mask < 0 || self.mask > 128) { + self.valid = false; + return; + } + + self.address = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]; + + if(self.ipv === 4){ + self.parseIPv4(ip); + }else{ + self.parseIPv6(ip); + } +}; + +IPCheck.prototype.parseIPv4 = function(ip) { + var self = this; + + // ipv4 addresses live under ::ffff:0:0 + self.address[10] = self.address[11] = 0xff; + + var octets = ip.split('.'); + for (var i = 0; i < 4; i++) { + self.address[i + 12] = parseInt(octets[i], 10); + } +}; + + +var V6_TRANSITIONAL = /:(\d+\.\d+\.\d+\.\d+)$/; + +IPCheck.prototype.parseIPv6 = function(ip) { + var self = this; + + var transitionalMatch = V6_TRANSITIONAL.exec(ip); + if(transitionalMatch){ + self.parseIPv4(transitionalMatch[1]); + return; + } + + var bits = ip.split(':'); + if (bits.length < 8) { + ip = ip.replace('::', Array(11 - bits.length).join(':')); + bits = ip.split(':'); + } + + var j = 0; + for (var i = 0; i < bits.length; i += 1) { + var x = bits[i] ? parseInt(bits[i], 16) : 0; + self.address[j++] = x >> 8; + self.address[j++] = x & 0xff; + } +}; + +IPCheck.prototype.match = function(cidr) { + var self = this; + + if (!(cidr instanceof IPCheck)) cidr = new IPCheck(cidr); + if (!self.valid || !cidr.valid) return false; + + var mask = cidr.mask; + var i = 0; + + while (mask >= 8) { + if (self.address[i] !== cidr.address[i]) return false; + + i++; + mask -= 8; + } + + var shift = 8 - mask; + return (self.address[i] >>> shift) === (cidr.address[i] >>> shift); +}; + + +IPCheck.match = function(ip, cidr) { + ip = ip instanceof IPCheck ? ip : new IPCheck(ip); + return ip.match(cidr); +}; diff --git a/backend/tsconfig.json b/backend/tsconfig.json index 6a1970331..0670010e1 100644 --- a/backend/tsconfig.json +++ b/backend/tsconfig.json @@ -13,7 +13,8 @@ "node_modules/@types" ], "allowSyntheticDefaultImports": true, - "esModuleInterop": true + "esModuleInterop": true, + "allowJs": true, }, "include": [ "src/**/*.ts" diff --git a/frontend/src/app/app-routing.module.ts b/frontend/src/app/app-routing.module.ts index f4c6dbbc8..2d12bc2e7 100644 --- a/frontend/src/app/app-routing.module.ts +++ b/frontend/src/app/app-routing.module.ts @@ -1,5 +1,6 @@ import { NgModule } from '@angular/core'; -import { Routes, RouterModule, PreloadAllModules } from '@angular/router'; +import { Routes, RouterModule } from '@angular/router'; +import { AppPreloadingStrategy } from './app.preloading-strategy' import { StartComponent } from './components/start/start.component'; import { TransactionComponent } from './components/transaction/transaction.component'; import { TransactionPreviewComponent } from './components/transaction/transaction-preview.component'; @@ -25,6 +26,10 @@ import { AssetsComponent } from './components/assets/assets.component'; import { AssetComponent } from './components/asset/asset.component'; import { AssetsNavComponent } from './components/assets/assets-nav/assets-nav.component'; +const browserWindow = window || {}; +// @ts-ignore +const browserWindowEnv = browserWindow.__env || {}; + let routes: Routes = [ { path: 'testnet', @@ -32,7 +37,8 @@ let routes: Routes = [ { path: '', pathMatch: 'full', - loadChildren: () => import('./graphs/graphs.module').then(m => m.GraphsModule) + loadChildren: () => import('./graphs/graphs.module').then(m => m.GraphsModule), + data: { preload: true }, }, { path: '', @@ -109,7 +115,8 @@ let routes: Routes = [ }, { path: 'docs', - loadChildren: () => import('./docs/docs.module').then(m => m.DocsModule) + loadChildren: () => import('./docs/docs.module').then(m => m.DocsModule), + data: { preload: true }, }, { path: 'api', @@ -117,7 +124,8 @@ let routes: Routes = [ }, { path: 'lightning', - loadChildren: () => import('./lightning/lightning.module').then(m => m.LightningModule) + loadChildren: () => import('./lightning/lightning.module').then(m => m.LightningModule), + data: { preload: browserWindowEnv && browserWindowEnv.LIGHTNING === true }, }, ], }, @@ -410,10 +418,6 @@ let routes: Routes = [ }, ]; -const browserWindow = window || {}; -// @ts-ignore -const browserWindowEnv = browserWindow.__env || {}; - if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'bisq') { routes = [{ path: '', @@ -691,7 +695,7 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') { initialNavigation: 'enabled', scrollPositionRestoration: 'enabled', anchorScrolling: 'enabled', - preloadingStrategy: PreloadAllModules + preloadingStrategy: AppPreloadingStrategy })], }) export class AppRoutingModule { } diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index b6b8859f6..5ae0c6cb5 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -18,6 +18,7 @@ import { LanguageService } from './services/language.service'; import { FiatShortenerPipe } from './shared/pipes/fiat-shortener.pipe'; import { ShortenStringPipe } from './shared/pipes/shorten-string-pipe/shorten-string.pipe'; import { CapAddressPipe } from './shared/pipes/cap-address-pipe/cap-address-pipe'; +import { AppPreloadingStrategy } from './app.preloading-strategy'; @NgModule({ declarations: [ @@ -44,6 +45,7 @@ import { CapAddressPipe } from './shared/pipes/cap-address-pipe/cap-address-pipe ShortenStringPipe, FiatShortenerPipe, CapAddressPipe, + AppPreloadingStrategy, { provide: HTTP_INTERCEPTORS, useClass: HttpCacheInterceptor, multi: true } ], bootstrap: [AppComponent] diff --git a/frontend/src/app/app.preloading-strategy.ts b/frontend/src/app/app.preloading-strategy.ts new file mode 100644 index 000000000..f62d072da --- /dev/null +++ b/frontend/src/app/app.preloading-strategy.ts @@ -0,0 +1,10 @@ +import { PreloadingStrategy, Route } from '@angular/router'; +import { Observable, timer, mergeMap, of } from 'rxjs'; + +export class AppPreloadingStrategy implements PreloadingStrategy { + preload(route: Route, load: Function): Observable { + return route.data && route.data.preload + ? timer(1500).pipe(mergeMap(() => load())) + : of(null); + } +} diff --git a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss index 5aaf8a91b..c5a217983 100644 --- a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss +++ b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss @@ -61,7 +61,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss b/frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss index 5aaf8a91b..c5a217983 100644 --- a/frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss +++ b/frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss @@ -61,7 +61,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.scss b/frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.scss index 5aaf8a91b..c5a217983 100644 --- a/frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.scss +++ b/frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.scss @@ -61,7 +61,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss index 5aaf8a91b..c5a217983 100644 --- a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss +++ b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss @@ -61,7 +61,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss b/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss index a47f63923..7b1395d78 100644 --- a/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss +++ b/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss @@ -61,7 +61,7 @@ flex-direction: column; @media (min-width: 1130px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 1130px) { position: relative; diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss index 52b5b2c2f..3021cf689 100644 --- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss @@ -61,7 +61,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.scss b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.scss index b15df44fa..c382d9886 100644 --- a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.scss +++ b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.scss @@ -55,7 +55,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/components/pool-ranking/pool-ranking.component.scss b/frontend/src/app/components/pool-ranking/pool-ranking.component.scss index 855b4e65c..8cb82d92d 100644 --- a/frontend/src/app/components/pool-ranking/pool-ranking.component.scss +++ b/frontend/src/app/components/pool-ranking/pool-ranking.component.scss @@ -39,7 +39,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/components/statistics/statistics.component.html b/frontend/src/app/components/statistics/statistics.component.html index 83ec77acf..9f62fffce 100644 --- a/frontend/src/app/components/statistics/statistics.component.html +++ b/frontend/src/app/components/statistics/statistics.component.html @@ -11,7 +11,7 @@
+ [class]="(stateService.env.MINING_DASHBOARD || stateService.env.LIGHTNING) ? 'mining' : 'no-menu'" (click)="saveGraphPreference()">
diff --git a/frontend/src/app/lightning/channel/channel.component.html b/frontend/src/app/lightning/channel/channel.component.html index 65866273d..50bb52eaa 100644 --- a/frontend/src/app/lightning/channel/channel.component.html +++ b/frontend/src/app/lightning/channel/channel.component.html @@ -11,6 +11,7 @@ Inactive Active Closed +
@@ -65,13 +66,13 @@

Opening transaction

- +

Closing transaction

  
- + diff --git a/frontend/src/app/lightning/channel/channel.component.scss b/frontend/src/app/lightning/channel/channel.component.scss index f19215f87..bf080b644 100644 --- a/frontend/src/app/lightning/channel/channel.component.scss +++ b/frontend/src/app/lightning/channel/channel.component.scss @@ -30,6 +30,10 @@ font-size: 20px; } +.badge { + margin-right: 5px; +} + app-fiat { display: block; font-size: 13px; diff --git a/frontend/src/app/lightning/channel/channel.component.ts b/frontend/src/app/lightning/channel/channel.component.ts index bbf9be36d..553173052 100644 --- a/frontend/src/app/lightning/channel/channel.component.ts +++ b/frontend/src/app/lightning/channel/channel.component.ts @@ -2,6 +2,7 @@ import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; import { ActivatedRoute, ParamMap } from '@angular/router'; import { forkJoin, Observable, of, share, zip } from 'rxjs'; import { catchError, map, shareReplay, switchMap, tap } from 'rxjs/operators'; +import { IChannel } from 'src/app/interfaces/node-api.interface'; import { ApiService } from 'src/app/services/api.service'; import { ElectrsApiService } from 'src/app/services/electrs-api.service'; import { SeoService } from 'src/app/services/seo.service'; @@ -62,10 +63,15 @@ export class ChannelComponent implements OnInit { ); this.transactions$ = this.channel$.pipe( - switchMap((data) => { + switchMap((channel: IChannel) => { return zip([ - data.transaction_id ? this.electrsApiService.getTransaction$(data.transaction_id) : of(null), - data.closing_transaction_id ? this.electrsApiService.getTransaction$(data.closing_transaction_id) : of(null), + channel.transaction_id ? this.electrsApiService.getTransaction$(channel.transaction_id) : of(null), + channel.closing_transaction_id ? this.electrsApiService.getTransaction$(channel.closing_transaction_id).pipe( + map((tx) => { + tx._channels = { inputs: {0: channel}, outputs: {}}; + return tx; + }) + ) : of(null), ]); }), ); diff --git a/frontend/src/app/lightning/channels-list/channels-list.component.html b/frontend/src/app/lightning/channels-list/channels-list.component.html index 0dd2de183..af87cefa4 100644 --- a/frontend/src/app/lightning/channels-list/channels-list.component.html +++ b/frontend/src/app/lightning/channels-list/channels-list.component.html @@ -1,5 +1,5 @@ -
- +
+
- +
@@ -87,8 +87,6 @@ -

Channels

-
diff --git a/frontend/src/app/lightning/channels-list/channels-list.component.scss b/frontend/src/app/lightning/channels-list/channels-list.component.scss index ba7b0a3b5..b460e8a51 100644 --- a/frontend/src/app/lightning/channels-list/channels-list.component.scss +++ b/frontend/src/app/lightning/channels-list/channels-list.component.scss @@ -7,3 +7,20 @@ font-size: 12px; top: 0px; } + +.formRadioGroup { + @media (min-width: 435px) { + position: absolute; + right: 0; + top: -46px; + } + @media (max-width: 435px) { + display: flex; + } +} + +.btn-group { + @media (max-width: 435px) { + flex-grow: 1; + } +} \ No newline at end of file diff --git a/frontend/src/app/lightning/channels-list/channels-list.component.ts b/frontend/src/app/lightning/channels-list/channels-list.component.ts index 6a0732522..75b8263e2 100644 --- a/frontend/src/app/lightning/channels-list/channels-list.component.ts +++ b/frontend/src/app/lightning/channels-list/channels-list.component.ts @@ -14,6 +14,7 @@ import { LightningApiService } from '../lightning-api.service'; export class ChannelsListComponent implements OnInit, OnChanges { @Input() publicKey: string; @Output() channelsStatusChangedEvent = new EventEmitter(); + @Output() loadingEvent = new EventEmitter(false); channels$: Observable; // @ts-ignore @@ -26,6 +27,7 @@ export class ChannelsListComponent implements OnInit, OnChanges { defaultStatus = 'open'; status = 'open'; publicKeySize = 25; + isLoading = false; constructor( private lightningApiService: LightningApiService, @@ -56,6 +58,8 @@ export class ChannelsListComponent implements OnInit, OnChanges { ) .pipe( tap((val) => { + this.isLoading = true; + this.loadingEvent.emit(true); if (typeof val === 'string') { this.status = val; this.page = 1; @@ -64,10 +68,12 @@ export class ChannelsListComponent implements OnInit, OnChanges { } }), switchMap(() => { - this.channelsStatusChangedEvent.emit(this.status); - return this.lightningApiService.getChannelsByNodeId$(this.publicKey, (this.page - 1) * this.itemsPerPage, this.status); + this.channelsStatusChangedEvent.emit(this.status); + return this.lightningApiService.getChannelsByNodeId$(this.publicKey, (this.page - 1) * this.itemsPerPage, this.status); }), map((response) => { + this.isLoading = false; + this.loadingEvent.emit(false); return { channels: response.body, totalItems: parseInt(response.headers.get('x-total-count'), 10) diff --git a/frontend/src/app/lightning/channels-statistics/channels-statistics.component.html b/frontend/src/app/lightning/channels-statistics/channels-statistics.component.html index 033438cf3..31261a84b 100644 --- a/frontend/src/app/lightning/channels-statistics/channels-statistics.component.html +++ b/frontend/src/app/lightning/channels-statistics/channels-statistics.component.html @@ -9,44 +9,44 @@
-
+
Avg Capacity
-
+
{{ statistics.latest?.avg_capacity || 0 | number: '1.0-0' }} sats
- +
-
+
Avg Fee Rate
-
+
{{ statistics.latest?.avg_fee_rate || 0 | number: '1.0-0' }} ppm
- +
-
+
Avg Base Fee
-
+
{{ statistics.latest?.avg_base_fee_mtokens || 0 | number: '1.0-0' }} msats
- +
@@ -55,43 +55,45 @@
-
+
Med Capacity
-
+
{{ statistics.latest?.med_capacity || 0 | number: '1.0-0' }} sats
- +
-
+ +
Med Fee Rate
-
+
{{ statistics.latest?.med_fee_rate || 0 | number: '1.0-0' }} ppm
- +
-
+ +
Med Base Fee
-
+
{{ statistics.latest?.med_base_fee_mtokens || 0 | number: '1.0-0' }} msats
- +
diff --git a/frontend/src/app/lightning/channels-statistics/channels-statistics.component.scss b/frontend/src/app/lightning/channels-statistics/channels-statistics.component.scss index 372d9eb78..e97f0d0af 100644 --- a/frontend/src/app/lightning/channels-statistics/channels-statistics.component.scss +++ b/frontend/src/app/lightning/channels-statistics/channels-statistics.component.scss @@ -18,6 +18,10 @@ } } +.fee-estimation-wrapper { + min-height: 77px; +} + .fee-estimation-container { display: flex; justify-content: space-between; @@ -30,7 +34,10 @@ width: -webkit-fill-available; @media (min-width: 376px) { margin: 0 auto 0px; - } + } + &.more-padding { + padding-top: 10px; + } &:first-child{ display: none; @media (min-width: 485px) { @@ -57,6 +64,9 @@ margin: auto; line-height: 1.45; padding: 0px 2px; + &.no-border { + border-bottom: none; + } } .fiat { display: block; diff --git a/frontend/src/app/lightning/node-statistics/node-statistics.component.html b/frontend/src/app/lightning/node-statistics/node-statistics.component.html index 152ee2dbe..ae4ea3dd7 100644 --- a/frontend/src/app/lightning/node-statistics/node-statistics.component.html +++ b/frontend/src/app/lightning/node-statistics/node-statistics.component.html @@ -1,76 +1,64 @@
-
-
Capacity
-
-
+
+
Capacity
+
+
- +
-
-
Nodes
-
-
+
+
Nodes
+
+
{{ statistics.latest?.node_count || 0 | number }}
- +
-
-
Channels
-
-
+
+
Channels
+
+
{{ statistics.latest?.channel_count || 0 | number }}
- +
-
-
Nodes
+
Nodes
-
Channels
+
Channels
-
Average Channel
+
Average Channel
diff --git a/frontend/src/app/lightning/node-statistics/node-statistics.component.scss b/frontend/src/app/lightning/node-statistics/node-statistics.component.scss index acc4578f3..1532f9c4b 100644 --- a/frontend/src/app/lightning/node-statistics/node-statistics.component.scss +++ b/frontend/src/app/lightning/node-statistics/node-statistics.component.scss @@ -18,6 +18,10 @@ } } +.fee-estimation-wrapper { + min-height: 77px; +} + .fee-estimation-container { display: flex; justify-content: space-between; @@ -30,7 +34,10 @@ width: -webkit-fill-available; @media (min-width: 376px) { margin: 0 auto 0px; - } + } + &.more-padding { + padding-top: 10px; + } &:first-child{ display: none; @media (min-width: 485px) { @@ -57,6 +64,9 @@ margin: auto; line-height: 1.45; padding: 0px 2px; + &.no-border { + border-bottom: none; + } } .fiat { display: block; diff --git a/frontend/src/app/lightning/node/node-preview.component.html b/frontend/src/app/lightning/node/node-preview.component.html index 0bb7255a6..a94882161 100644 --- a/frontend/src/app/lightning/node/node-preview.component.html +++ b/frontend/src/app/lightning/node/node-preview.component.html @@ -52,7 +52,7 @@
- +
diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index 423b29afb..def6e28e6 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -1,7 +1,7 @@
Lightning node
-

{{ node.alias }}

+

{{ node.alias }}

{{ node.public_key | shortenString : publicKeySize }} @@ -131,10 +131,9 @@
-

Active channels map

-
+

Open channels ({{ node.opened_channel_count }}) @@ -143,10 +142,13 @@ Closed channels ({{ node.closed_channel_count }})

+
+ (channelsStatusChangedEvent)="onChannelsListStatusChanged($event)" + (loadingEvent)="onLoadingEvent($event)" + >
diff --git a/frontend/src/app/lightning/node/node.component.scss b/frontend/src/app/lightning/node/node.component.scss index 2b171416f..b2e6f573b 100644 --- a/frontend/src/app/lightning/node/node.component.scss +++ b/frontend/src/app/lightning/node/node.component.scss @@ -56,4 +56,17 @@ app-fiat { display: inline-block; margin-left: 10px; } +} + +.spinner-border { + @media (min-width: 768px) { + margin-top: 6.5px; + width: 1.75rem; + height: 1.75rem; + } + @media (max-width: 768px) { + margin-top: 2.3px; + width: 1.5rem; + height: 1.5rem; + } } \ No newline at end of file diff --git a/frontend/src/app/lightning/node/node.component.ts b/frontend/src/app/lightning/node/node.component.ts index 8ddaacf95..bfee9252d 100644 --- a/frontend/src/app/lightning/node/node.component.ts +++ b/frontend/src/app/lightning/node/node.component.ts @@ -22,7 +22,7 @@ export class NodeComponent implements OnInit { channelsListStatus: string; error: Error; publicKey: string; - + channelListLoading = false; publicKeySize = 99; constructor( @@ -97,4 +97,8 @@ export class NodeComponent implements OnInit { onChannelsListStatusChanged(e) { this.channelsListStatus = e; } + + onLoadingEvent(e) { + this.channelListLoading = e; + } } diff --git a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts index 91de48186..1f30e96f5 100644 --- a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts +++ b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts @@ -22,6 +22,7 @@ export class NodesChannelsMap implements OnInit { @Input() channel: any[] = []; @Input() fitContainer = false; @Input() hasLocation = true; + @Input() placeholder = false; @Output() readyEvent = new EventEmitter(); channelsObservable: Observable; @@ -201,11 +202,26 @@ export class NodesChannelsMap implements OnInit { prepareChartOptions(nodes, channels) { let title: object; - if (channels.length === 0) { + if (channels.length === 0 && !this.placeholder) { this.chartOptions = null; return; } + // empty map fallback + if (channels.length === 0 && this.placeholder) { + title = { + textStyle: { + color: 'white', + fontSize: 18 + }, + text: $localize`No geolocation data available`, + left: 'center', + top: 'center' + }; + this.zoom = 1.5; + this.center = [0, 20]; + } + this.chartOptions = { silent: this.style === 'widget', title: title ?? undefined, @@ -222,7 +238,7 @@ export class NodesChannelsMap implements OnInit { roam: this.style === 'widget' ? false : true, itemStyle: { borderColor: 'black', - color: '#ffffff44' + color: '#272b3f' }, scaleLimit: { min: 1.3, diff --git a/frontend/src/app/lightning/nodes-channels/node-channels.component.html b/frontend/src/app/lightning/nodes-channels/node-channels.component.html index 43a5fad60..8fc63793c 100644 --- a/frontend/src/app/lightning/nodes-channels/node-channels.component.html +++ b/frontend/src/app/lightning/nodes-channels/node-channels.component.html @@ -1,2 +1,9 @@ -
+
+

Active channels map

+
+
+
+ + diff --git a/frontend/src/app/lightning/nodes-channels/node-channels.component.scss b/frontend/src/app/lightning/nodes-channels/node-channels.component.scss index e69de29bb..4d7b4de0e 100644 --- a/frontend/src/app/lightning/nodes-channels/node-channels.component.scss +++ b/frontend/src/app/lightning/nodes-channels/node-channels.component.scss @@ -0,0 +1,9 @@ +.loading-spinner { + min-height: 455px; + z-index: 100; +} + +.spinner-border { + position: relative; + top: 225px; +} \ No newline at end of file diff --git a/frontend/src/app/lightning/nodes-channels/node-channels.component.ts b/frontend/src/app/lightning/nodes-channels/node-channels.component.ts index 9d6d7df2b..f675c81b5 100644 --- a/frontend/src/app/lightning/nodes-channels/node-channels.component.ts +++ b/frontend/src/app/lightning/nodes-channels/node-channels.component.ts @@ -1,8 +1,8 @@ import { formatNumber } from '@angular/common'; -import { ChangeDetectionStrategy, Component, Inject, Input, LOCALE_ID, NgZone, OnChanges, OnInit } from '@angular/core'; +import { ChangeDetectionStrategy, Component, Inject, Input, LOCALE_ID, NgZone, OnChanges } from '@angular/core'; import { Router } from '@angular/router'; import { ECharts, EChartsOption, TreemapSeriesOption } from 'echarts'; -import { Observable, tap } from 'rxjs'; +import { Observable, share, switchMap, tap } from 'rxjs'; import { lerpColor } from 'src/app/shared/graphs.utils'; import { AmountShortenerPipe } from 'src/app/shared/pipes/amount-shortener.pipe'; import { LightningApiService } from '../lightning-api.service'; @@ -25,7 +25,7 @@ export class NodeChannels implements OnChanges { }; channelsObservable$: Observable; - isLoading: true; + isLoading = true; constructor( @Inject(LOCALE_ID) public locale: string, @@ -41,9 +41,20 @@ export class NodeChannels implements OnChanges { this.channelsObservable$ = this.lightningApiService.getChannelsByNodeId$(this.publicKey, -1, 'active') .pipe( - tap((response) => { - const biggestCapacity = response.body[0].capacity; - this.prepareChartOptions(response.body.map(channel => { + switchMap((response) => { + this.isLoading = true; + if ((response.body?.length ?? 0) <= 0) { + this.isLoading = false; + return ['']; + } + return [response.body]; + }), + tap((body: any[]) => { + if (body.length === 0 || body[0].length === 0) { + return; + } + const biggestCapacity = body[0].capacity; + this.prepareChartOptions(body.map(channel => { return { name: channel.node.alias, value: channel.capacity, @@ -54,7 +65,9 @@ export class NodeChannels implements OnChanges { } }; })); - }) + this.isLoading = false; + }), + share(), ); } @@ -117,10 +130,6 @@ export class NodeChannels implements OnChanges { } onChartInit(ec: ECharts): void { - if (this.chartInstance !== undefined) { - return; - } - this.chartInstance = ec; this.chartInstance.on('click', (e) => { diff --git a/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.scss b/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.scss index 760e782ca..4ff4dbf3e 100644 --- a/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.scss +++ b/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.scss @@ -60,7 +60,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.ts b/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.ts index 22f46e8e7..0ef9e3cd5 100644 --- a/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.ts +++ b/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.ts @@ -300,7 +300,7 @@ export class NodesNetworksChartComponent implements OnInit { { offset: 1, color: '#D81B60AA' }, ]), - smooth: true, + smooth: false, }, { zlevel: 1, @@ -321,7 +321,7 @@ export class NodesNetworksChartComponent implements OnInit { { offset: 0, color: '#FFB300' }, { offset: 1, color: '#FFB300AA' }, ]), - smooth: true, + smooth: false, }, { zlevel: 1, @@ -342,7 +342,7 @@ export class NodesNetworksChartComponent implements OnInit { { offset: 0, color: '#7D4698' }, { offset: 1, color: '#7D4698AA' }, ]), - smooth: true, + smooth: false, }, ], dataZoom: this.widget ? null : [{ diff --git a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html index 25773a06e..ef1c71efd 100644 --- a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html +++ b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html @@ -28,7 +28,7 @@
- Top 100 ISP hosting LN nodes + Top 100 ISPs hosting LN nodes diff --git a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss index c6897cda9..54055de36 100644 --- a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss +++ b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss @@ -40,7 +40,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.scss b/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.scss index c2e00e520..c9b835054 100644 --- a/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.scss +++ b/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.scss @@ -60,7 +60,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.ts b/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.ts index 6fb8fd1e2..3a25367dc 100644 --- a/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.ts +++ b/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.ts @@ -274,7 +274,7 @@ export class LightningStatisticsChartComponent implements OnInit { width: 1, }, }, - smooth: true, + smooth: false, }, { zlevel: 0, @@ -288,7 +288,7 @@ export class LightningStatisticsChartComponent implements OnInit { opacity: 0.5, }, type: 'line', - smooth: true, + smooth: false, } ], dataZoom: this.widget ? null : [{ diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 5d89a168f..5f036c575 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -242,12 +242,12 @@ export class ApiService { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + `/api/v1/enterprise/info/` + name); } - getChannelByTxIds$(txIds: string[]): Observable<{ inputs: any[], outputs: any[] }> { + getChannelByTxIds$(txIds: string[]): Observable { let params = new HttpParams(); txIds.forEach((txId: string) => { params = params.append('txId[]', txId); }); - return this.httpClient.get<{ inputs: any[], outputs: any[] }>(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/channels/txids/', { params }); + return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/channels/txids/', { params }); } lightningSearch$(searchText: string): Observable { diff --git a/frontend/src/app/services/state.service.ts b/frontend/src/app/services/state.service.ts index 14d220fa1..b0e018941 100644 --- a/frontend/src/app/services/state.service.ts +++ b/frontend/src/app/services/state.service.ts @@ -153,7 +153,12 @@ export class StateService { if (this.env.BASE_MODULE !== 'mempool' && this.env.BASE_MODULE !== 'liquid') { return; } - const networkMatches = url.match(/^\/(bisq|testnet|liquidtestnet|liquid|signet)/); + // horrible network regex breakdown: + // /^\/ starts with a forward slash... + // (?:[a-z]{2}(?:-[A-Z]{2})?\/)? optional locale prefix (non-capturing) + // (?:preview\/)? optional "preview" prefix (non-capturing) + // (bisq|testnet|liquidtestnet|liquid|signet)/ network string (captured as networkMatches[1]) + const networkMatches = url.match(/^\/(?:[a-z]{2}(?:-[A-Z]{2})?\/)?(?:preview\/)?(bisq|testnet|liquidtestnet|liquid|signet)/); switch (networkMatches && networkMatches[1]) { case 'liquid': if (this.network !== 'liquid') { diff --git a/frontend/src/app/shared/shared.module.ts b/frontend/src/app/shared/shared.module.ts index f9de57834..c340fb50b 100644 --- a/frontend/src/app/shared/shared.module.ts +++ b/frontend/src/app/shared/shared.module.ts @@ -63,6 +63,7 @@ import { StatusViewComponent } from '../components/status-view/status-view.compo import { FeesBoxComponent } from '../components/fees-box/fees-box.component'; import { DifficultyComponent } from '../components/difficulty/difficulty.component'; import { TermsOfServiceComponent } from '../components/terms-of-service/terms-of-service.component'; +import { TxBowtieGraphComponent } from '../components/tx-bowtie-graph/tx-bowtie-graph.component'; import { PrivacyPolicyComponent } from '../components/privacy-policy/privacy-policy.component'; import { TrademarkPolicyComponent } from '../components/trademark-policy/trademark-policy.component'; import { PushTransactionComponent } from '../components/push-transaction/push-transaction.component'; @@ -138,6 +139,7 @@ import { GeolocationComponent } from '../shared/components/geolocation/geolocati StatusViewComponent, FeesBoxComponent, DifficultyComponent, + TxBowtieGraphComponent, TermsOfServiceComponent, PrivacyPolicyComponent, TrademarkPolicyComponent, @@ -242,6 +244,7 @@ import { GeolocationComponent } from '../shared/components/geolocation/geolocati StatusViewComponent, FeesBoxComponent, DifficultyComponent, + TxBowtieGraphComponent, TermsOfServiceComponent, PrivacyPolicyComponent, TrademarkPolicyComponent, diff --git a/frontend/src/index.bisq.html b/frontend/src/index.bisq.html index b5c65e8d4..8da1e77e0 100644 --- a/frontend/src/index.bisq.html +++ b/frontend/src/index.bisq.html @@ -5,7 +5,7 @@ mempool - Bisq Markets - + @@ -14,7 +14,7 @@ - + diff --git a/frontend/src/index.liquid.html b/frontend/src/index.liquid.html index 65d397aac..89a6984ba 100644 --- a/frontend/src/index.liquid.html +++ b/frontend/src/index.liquid.html @@ -5,7 +5,7 @@ mempool - Liquid Network - + @@ -14,7 +14,7 @@ - + diff --git a/frontend/src/index.mempool.html b/frontend/src/index.mempool.html index 05c8d8398..1176a3da2 100644 --- a/frontend/src/index.mempool.html +++ b/frontend/src/index.mempool.html @@ -5,7 +5,7 @@ mempool - Bitcoin Explorer - + @@ -14,7 +14,7 @@ - + diff --git a/frontend/src/resources/bisq/bisq-markets-preview.png b/frontend/src/resources/bisq/bisq-markets-preview.png index 2fff8f99e..2b5e1250b 100644 Binary files a/frontend/src/resources/bisq/bisq-markets-preview.png and b/frontend/src/resources/bisq/bisq-markets-preview.png differ diff --git a/frontend/src/resources/liquid/liquid-network-preview.png b/frontend/src/resources/liquid/liquid-network-preview.png index 5a6ed9eb1..72942110c 100644 Binary files a/frontend/src/resources/liquid/liquid-network-preview.png and b/frontend/src/resources/liquid/liquid-network-preview.png differ diff --git a/frontend/src/resources/previews/dashboard.png b/frontend/src/resources/previews/dashboard.png new file mode 100644 index 000000000..e11588994 Binary files /dev/null and b/frontend/src/resources/previews/dashboard.png differ diff --git a/frontend/src/resources/previews/lightning.png b/frontend/src/resources/previews/lightning.png new file mode 100644 index 000000000..4686b0ef0 Binary files /dev/null and b/frontend/src/resources/previews/lightning.png differ diff --git a/frontend/src/resources/previews/mining.png b/frontend/src/resources/previews/mining.png new file mode 100644 index 000000000..6a2aa1b41 Binary files /dev/null and b/frontend/src/resources/previews/mining.png differ diff --git a/frontend/src/styles.scss b/frontend/src/styles.scss index 2ef537456..5882af254 100644 --- a/frontend/src/styles.scss +++ b/frontend/src/styles.scss @@ -705,6 +705,10 @@ th { .locktime { color: #ff8c00 } .reserved { color: #ff8c00 } +.shortable-address { + font-family: monospace; +} + .rtl-layout { .navbar-brand { @@ -881,6 +885,7 @@ th { .shortable-address { direction: ltr; + font-family: monospace; } .lastest-blocks-table { diff --git a/production/install b/production/install index 6d50e8bf7..4eac1817b 100755 --- a/production/install +++ b/production/install @@ -48,6 +48,9 @@ BITCOIN_MAINNET_ENABLE=ON BITCOIN_MAINNET_MINFEE_ENABLE=ON BITCOIN_TESTNET_ENABLE=ON BITCOIN_SIGNET_ENABLE=ON +LN_BITCOIN_MAINNET_ENABLE=ON +LN_BITCOIN_TESTNET_ENABLE=ON +LN_BITCOIN_SIGNET_ENABLE=ON BISQ_MAINNET_ENABLE=ON ELEMENTS_LIQUID_ENABLE=ON ELEMENTS_LIQUIDTESTNET_ENABLE=ON @@ -227,6 +230,9 @@ MYSQL_GROUP=mysql MEMPOOL_MAINNET_USER='mempool' MEMPOOL_TESTNET_USER='mempool_testnet' MEMPOOL_SIGNET_USER='mempool_signet' +LN_MEMPOOL_MAINNET_USER='mempool_mainnet_lightning' +LN_MEMPOOL_TESTNET_USER='mempool_testnet_lightning' +LN_MEMPOOL_SIGNET_USER='mempool_signet_lightning' MEMPOOL_LIQUID_USER='mempool_liquid' MEMPOOL_LIQUIDTESTNET_USER='mempool_liquidtestnet' MEMPOOL_BISQ_USER='mempool_bisq' @@ -234,6 +240,9 @@ MEMPOOL_BISQ_USER='mempool_bisq' MEMPOOL_MAINNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') MEMPOOL_TESTNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') MEMPOOL_SIGNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') +LN_MEMPOOL_MAINNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') +LN_MEMPOOL_TESTNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') +LN_MEMPOOL_SIGNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') MEMPOOL_LIQUID_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') MEMPOOL_LIQUIDTESTNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') MEMPOOL_BISQ_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') @@ -391,6 +400,10 @@ FREEBSD_PKG+=(boost-libs autoconf automake gmake gcc libevent libtool pkgconf) FREEBSD_PKG+=(nginx rsync py39-certbot-nginx mariadb105-server keybase) FREEBSD_PKG+=(geoipupdate) +FREEBSD_UNFURL_PKG=() +FREEBSD_UNFURL_PKG+=(nvidia-driver-470-470.129.06 chromium xinit xterm twm ja-sourcehansans-otf) +FREEBSD_UNFURL_PKG+=(zh-sourcehansans-sc-otf ko-aleefonts-ttf lohit tlwg-ttf) + ############################# ##### utility functions ##### ############################# @@ -747,6 +760,9 @@ $CUT >$input <<-EOF Tor:Enable Tor v3 HS Onion:ON Mainnet:Enable Bitcoin Mainnet:ON Mainnet-Minfee:Enable Bitcoin Mainnet Minfee:ON +LN-Mainnet:Enable Bitcoin Mainnet Lightning:ON +LN-Testnet:Enable Bitcoin Testnet Lightning:ON +LN-Signet:Enable Bitcoin Signet Lightning:ON Testnet:Enable Bitcoin Testnet:ON Signet:Enable Bitcoin Signet:ON Liquid:Enable Elements Liquid:ON @@ -809,6 +825,24 @@ else BITCOIN_INSTALL=OFF fi +if grep LN-Mainnet $tempfile >/dev/null 2>&1;then + LN_BITCOIN_MAINNET_ENABLE=ON +else + LN_BITCOIN_MAINNET_ENABLE=OFF +fi + +if grep LN-Testnet $tempfile >/dev/null 2>&1;then + LN_BITCOIN_TESTNET_ENABLE=ON +else + LN_BITCOIN_TESTNET_ENABLE=OFF +fi + +if grep LN-Signet $tempfile >/dev/null 2>&1;then + LN_BITCOIN_SIGNET_ENABLE=ON +else + LN_BITCOIN_SIGNET_ENABLE=OFF +fi + if grep Liquid $tempfile >/dev/null 2>&1;then ELEMENTS_LIQUID_ENABLE=ON else @@ -831,6 +865,7 @@ if grep CoreLN $tempfile >/dev/null 2>&1;then CLN_INSTALL=ON else CLN_INSTALL=OFF +fi if [ "${BITCOIN_MAINNET_ENABLE}" = ON -o "${BITCOIN_TESTNET_ENABLE}" = ON -o "${BITCOIN_SIGNET_ENABLE}" = ON ];then BITCOIN_ELECTRS_INSTALL=ON @@ -1279,17 +1314,20 @@ case $OS in echo "[*] Creating Core Lightning user" osGroupCreate "${CLN_GROUP}" osUserCreate "${CLN_USER}" "${CLN_HOME}" "${CLN_GROUP}" + osSudo "${ROOT_USER}" pw usermod ${MEMPOOL_USER} -G "${CLN_GROUP}" osSudo "${ROOT_USER}" chsh -s `which zsh` "${CLN_USER}" echo "export PATH=$PATH:$HOME/.local/bin" >> "${CLN_HOME}/.zshrc" + osSudo "${ROOT_USER}" mkdir -p "${CLN_HOME}/.lightning/{bitcoin,signet,testnet}" + osSudo "${ROOT_USER}" chmod 750 "${CLN_HOME}" "${CLN_HOME}/.lightning" "${CLN_HOME}/.lightning/{bitcoin,signet,testnet}" osSudo "${ROOT_USER}" chown -R "${CLN_USER}:${CLN_GROUP}" "${CLN_HOME}" echo "[*] Installing Core Lightning package" osPackageInstall ${CLN_PKG} echo "[*] Installing Core Lightning mainnet Cronjob" - crontab_cln+='@reboot sleep 30 ; screen -dmS main lightningd --alias `hostname` --bitcoin-datadir /bitcoin\n' - crontab_cln+='@reboot sleep 60 ; screen -dmS sig lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network signet\n' + crontab_cln+='@reboot sleep 60 ; screen -dmS main lightningd --alias `hostname` --bitcoin-datadir /bitcoin\n' crontab_cln+='@reboot sleep 90 ; screen -dmS tes lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network testnet\n' + crontab_cln+='@reboot sleep 120 ; screen -dmS sig lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network signet\n' echo "${crontab_cln}" | crontab -u "${CLN_USER}" - ;; Debian) @@ -1397,7 +1435,42 @@ if [ "${UNFURL_INSTALL}" = ON ];then case $OS in FreeBSD) - echo "[*] FIXME: Unfurl must be installed manually on FreeBSD" + + if pciconf -lv|grep -i nvidia >/dev/null 2>&1;then + echo "[*] GPU detected: Installing packages for Unfurl" + osPackageInstall ${FREEBSD_UNFURL_PKG[@]} + + echo 'allowed_users = anybody' >> /usr/local/etc/X11/Xwrapper.config + echo 'kld_list="nvidia"' >> /etc/rc.conf + echo 'nvidia_xorg_enable="YES"' >> /etc/rc.conf + + echo "[*] Installing color emoji" + osSudo "${ROOT_USER}" curl "https://github.com/samuelngs/apple-emoji-linux/releases/download/ios-15.4/AppleColorEmoji.ttf" -o /usr/local/share/fonts/TTF/AppleColorEmoji.ttf + cat >> /usr/local/etc/fonts/conf.d/01-emoji.conf < + + + +sans-serif + +Apple Color Emoji + + + +serif + +Apple Color Emoji + + + +Apple Color Emoji + +Apple Color Emoji + + + +EOF + fi ;; Debian) @@ -1671,7 +1744,16 @@ if [ "${BITCOIN_MAINNET_ENABLE}" = ON -o "${BITCOIN_TESTNET_ENABLE}" = ON -o "${ osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/mainnet" echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Bitcoin Mainnet" - osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/mainnet && git checkout ${MEMPOOL_LATEST_RELEASE}" +fi + +if [ "${LN_BITCOIN_MAINNET_ENABLE}" = ON ];then + echo "[*] Creating Mempool instance for Lightning Network on Bitcoin Mainnet" + osSudo "${MEMPOOL_USER}" git config --global advice.detachedHead false + osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/mainnet-lightning" + + echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Lightning Network on Bitcoin Mainnet" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/mainnet-lightning && git checkout ${MEMPOOL_LATEST_RELEASE}" fi if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then @@ -1680,7 +1762,16 @@ if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/testnet" echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Bitcoin Testnet" - osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/testnet && git checkout ${MEMPOOL_LATEST_RELEASE}" +fi + +if [ "${LN_BITCOIN_TESTNET_ENABLE}" = ON ];then + echo "[*] Creating Mempool instance for Lightning Network on Bitcoin Testnet" + osSudo "${MEMPOOL_USER}" git config --global advice.detachedHead false + osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/testnet-lightning" + + echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Lightning Network on Bitcoin Testnet" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/testnet-lightning && git checkout ${MEMPOOL_LATEST_RELEASE}" fi if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then @@ -1689,7 +1780,16 @@ if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/signet" echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Bitcoin Signet" - osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/signet && git checkout ${MEMPOOL_LATEST_RELEASE}" +fi + +if [ "${LN_BITCOIN_SIGNET_ENABLE}" = ON ];then + echo "[*] Creating Mempool instance for Lightning Network on Bitcoin Signet" + osSudo "${MEMPOOL_USER}" git config --global advice.detachedHead false + osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/signet-lightning" + + echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Lightning Network on Bitcoin Signet" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/signet-lightning && git checkout ${MEMPOOL_LATEST_RELEASE}" fi if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then @@ -1698,7 +1798,7 @@ if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/liquid" echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Liquid" - osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/liquid && git checkout ${MEMPOOL_LATEST_RELEASE}" fi if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then @@ -1707,7 +1807,7 @@ if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/liquidtestnet" echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Liquid Testnet" - osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/liquidtestnet && git checkout ${MEMPOOL_LATEST_RELEASE}" fi if [ "${BISQ_INSTALL}" = ON ];then @@ -1716,7 +1816,7 @@ if [ "${BISQ_INSTALL}" = ON ];then osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/bisq" echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Bisq" - osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/bisq && git checkout ${MEMPOOL_LATEST_RELEASE}" fi ##### mariadb @@ -1742,6 +1842,15 @@ grant all on mempool_testnet.* to '${MEMPOOL_TESTNET_USER}'@'localhost' identifi create database mempool_signet; grant all on mempool_signet.* to '${MEMPOOL_SIGNET_USER}'@'localhost' identified by '${MEMPOOL_SIGNET_PASS}'; +create database mempool_mainnet_lightning; +grant all on mempool_mainnet_lightning.* to '${LN_MEMPOOL_MAINNET_USER}'@'%' identified by '${LN_MEMPOOL_MAINNET_PASS}'; + +create database mempool_testnet_lightning; +grant all on mempool_testnet_lightning.* to '${LN_MEMPOOL_TESTNET_USER}'@'%' identified by '${LN_MEMPOOL_TESTNET_PASS}'; + +create database mempool_signet_lightning; +grant all on mempool_signet_lightning.* to '${LN_MEMPOOL_SIGNET_USER}'@'%' identified by '${LN_MEMPOOL_SIGNET_PASS}'; + create database mempool_liquid; grant all on mempool_liquid.* to '${MEMPOOL_LIQUID_USER}'@'localhost' identified by '${MEMPOOL_LIQUID_PASS}'; @@ -1760,6 +1869,12 @@ declare -x MEMPOOL_TESTNET_USER="${MEMPOOL_TESTNET_USER}" declare -x MEMPOOL_TESTNET_PASS="${MEMPOOL_TESTNET_PASS}" declare -x MEMPOOL_SIGNET_USER="${MEMPOOL_SIGNET_USER}" declare -x MEMPOOL_SIGNET_PASS="${MEMPOOL_SIGNET_PASS}" +declare -x LN_MEMPOOL_MAINNET_USER="${LN_MEMPOOL_MAINNET_USER}" +declare -x LN_MEMPOOL_MAINNET_PASS="${LN_MEMPOOL_MAINNET_PASS}" +declare -x LN_MEMPOOL_TESTNET_USER="${LN_MEMPOOL_TESTNET_USER}" +declare -x LN_MEMPOOL_TESTNET_PASS="${LN_MEMPOOL_TESTNET_PASS}" +declare -x LN_MEMPOOL_SIGNET_USER="${LN_MEMPOOL_SIGNET_USER}" +declare -x LN_MEMPOOL_SIGNET_PASS="${LN_MEMPOOL_SIGNET_PASS}" declare -x MEMPOOL_LIQUID_USER="${MEMPOOL_LIQUID_USER}" declare -x MEMPOOL_LIQUID_PASS="${MEMPOOL_LIQUID_PASS}" declare -x MEMPOOL_LIQUIDTESTNET_USER="${MEMPOOL_LIQUIDTESTNET_USER}" @@ -1770,24 +1885,32 @@ _EOF_ ##### nginx -echo "[*] Adding Nginx configuration" -osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/nginx/nginx.conf" "${NGINX_CONFIGURATION}" -mkdir -p /var/cache/nginx/services /var/cache/nginx/api -chown ${NGINX_USER}: /var/cache/nginx/services /var/cache/nginx/api -ln -s /mempool/mempool /etc/nginx/mempool -osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_USER__!${NGINX_USER}!" "${NGINX_CONFIGURATION}" -osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_ETC_FOLDER__!${NGINX_ETC_FOLDER}!" "${NGINX_CONFIGURATION}" -if [ "${TOR_INSTALL}" = ON ];then -echo "[*] Read tor v3 onion hostnames" - NGINX_MEMPOOL_ONION=$(cat "${TOR_RESOURCES}/mempool/hostname") - NGINX_BISQ_ONION=$(cat "${TOR_RESOURCES}/bisq/hostname") - NGINX_LIQUID_ONION=$(cat "${TOR_RESOURCES}/liquid/hostname") - osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_MEMPOOL_ONION__!${NGINX_MEMPOOL_ONION%.onion}!" "${NGINX_CONFIGURATION}" - osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_BISQ_ONION__!${NGINX_BISQ_ONION%.onion}!" "${NGINX_CONFIGURATION}" - osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_LIQUID_ONION__!${NGINX_LIQUID_ONIONi%.onion}!" "${NGINX_CONFIGURATION}" -fi -echo "[*] Restarting Nginx" -osSudo "${ROOT_USER}" service nginx restart +case $OS in + + FreeBSD) + ;; + +Debian) + echo "[*] Adding Nginx configuration" + osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/nginx/nginx.conf" "${NGINX_CONFIGURATION}" + mkdir -p /var/cache/nginx/services /var/cache/nginx/api + chown ${NGINX_USER}: /var/cache/nginx/services /var/cache/nginx/api + ln -s /mempool/mempool /etc/nginx/mempool + osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_USER__!${NGINX_USER}!" "${NGINX_CONFIGURATION}" + osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_ETC_FOLDER__!${NGINX_ETC_FOLDER}!" "${NGINX_CONFIGURATION}" + if [ "${TOR_INSTALL}" = ON ];then + echo "[*] Read tor v3 onion hostnames" + NGINX_MEMPOOL_ONION=$(cat "${TOR_RESOURCES}/mempool/hostname") + NGINX_BISQ_ONION=$(cat "${TOR_RESOURCES}/bisq/hostname") + NGINX_LIQUID_ONION=$(cat "${TOR_RESOURCES}/liquid/hostname") + osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_MEMPOOL_ONION__!${NGINX_MEMPOOL_ONION%.onion}!" "${NGINX_CONFIGURATION}" + osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_BISQ_ONION__!${NGINX_BISQ_ONION%.onion}!" "${NGINX_CONFIGURATION}" + osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_LIQUID_ONION__!${NGINX_LIQUID_ONIONi%.onion}!" "${NGINX_CONFIGURATION}" + fi + echo "[*] Restarting Nginx" + osSudo "${ROOT_USER}" service nginx restart + ;; +esac ##### OS systemd diff --git a/production/mempool-build-all b/production/mempool-build-all index c0e9a2c2a..048aeefdc 100755 --- a/production/mempool-build-all +++ b/production/mempool-build-all @@ -12,7 +12,10 @@ ELEMENTS_RPC_USER=$(grep '^rpcuser' /elements/elements.conf | cut -d '=' -f2) ELEMENTS_RPC_PASS=$(grep '^rpcpassword' /elements/elements.conf | cut -d '=' -f2) # get mysql credentials -. /mempool/mysql_credentials +MYSQL_CRED_FILE=${HOME}/mempool/mysql_credentials +if [ -f "${MYSQL_CRED_FILE}" ];then + . ${MYSQL_CRED_FILE} +fi if [ -f "${LOCKFILE}" ];then echo "upgrade already running? check lockfile ${LOCKFILE}" @@ -63,6 +66,19 @@ build_frontend() npm run build || exit 1 } +build_unfurler() +{ + local site="$1" + echo "[*] Building unfurler for ${site}" + [ -z "${HASH}" ] && exit 1 + cd "$HOME/${site}/unfurler" || exit 1 + if [ ! -e "config.json" ];then + cp "${HOME}/mempool/production/unfurler-config.${site}.json" "config.json" + fi + PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true npm install || exit 1 + npm run build || exit 1 +} + build_backend() { local site="$1" @@ -82,6 +98,12 @@ build_backend() -e "s!__MEMPOOL_TESTNET_PASS__!${MEMPOOL_TESTNET_PASS}!" \ -e "s!__MEMPOOL_SIGNET_USER__!${MEMPOOL_SIGNET_USER}!" \ -e "s!__MEMPOOL_SIGNET_PASS__!${MEMPOOL_SIGNET_PASS}!" \ + -e "s!__LN_MEMPOOL_MAINNET_USER__!${LN_MEMPOOL_MAINNET_USER}!" \ + -e "s!__LN_MEMPOOL_MAINNET_PASS__!${LN_MEMPOOL_MAINNET_PASS}!" \ + -e "s!__LN_MEMPOOL_TESTNET_USER__!${LN_MEMPOOL_TESTNET_USER}!" \ + -e "s!__LN_MEMPOOL_TESTNET_PASS__!${LN_MEMPOOL_TESTNET_PASS}!" \ + -e "s!__LN_MEMPOOL_SIGNET_USER__!${LN_MEMPOOL_SIGNET_USER}!" \ + -e "s!__LN_MEMPOOL_SIGNET_PASS__!${LN_MEMPOOL_SIGNET_PASS}!" \ -e "s!__MEMPOOL_LIQUID_USER__!${MEMPOOL_LIQUID_USER}!" \ -e "s!__MEMPOOL_LIQUID_PASS__!${MEMPOOL_LIQUID_PASS}!" \ -e "s!__MEMPOOL_LIQUIDTESTNET_USER__!${LIQUIDTESTNET_USER}!" \ @@ -128,6 +150,11 @@ for repo in $backend_repos;do update_repo "${repo}" done +# build unfurlers +for repo in mainnet liquid bisq;do + build_unfurler "${repo}" +done + # build backends for repo in $backend_repos;do build_backend "${repo}" diff --git a/production/mempool-config.mainnet-lightning.json b/production/mempool-config.mainnet-lightning.json new file mode 100644 index 000000000..99ce8d518 --- /dev/null +++ b/production/mempool-config.mainnet-lightning.json @@ -0,0 +1,49 @@ +{ + "MEMPOOL": { + "NETWORK": "mainnet", + "BACKEND": "esplora", + "HTTP_PORT": 8993, + "INDEXING_BLOCKS_AMOUNT": 0, + "API_URL_PREFIX": "/api/v1/" + }, + "SYSLOG": { + "MIN_PRIORITY": "debug" + }, + "CORE_RPC": { + "PORT": 8332, + "USERNAME": "__BITCOIN_RPC_USER__", + "PASSWORD": "__BITCOIN_RPC_PASS__" + }, + "ESPLORA": { + "REST_API_URL": "http://127.0.0.1:4000" + }, + "LIGHTNING": { + "ENABLED": true, + "BACKEND": "cln", + "GRAPH_REFRESH_INTERVAL": 60, + "TOPOLOGY_FOLDER": "/cln/topology/output" + }, + "LND": { + "REST_API_URL": "https://127.0.0.1:8888", + "TLS_CERT_PATH": "/lnd/.lnd/tls.cert", + "MACAROON_PATH": "/lnd/.lnd/data/chain/bitcoin/mainnet/readonly.macaroon" + }, + "CLIGHTNING": { + "SOCKET": "/cln/.lightning/bitcoin/lightning-rpc" + }, + "MAXMIND": { + "ENABLED": true, + "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoIP2-City.mmdb" + }, + "STATISTICS": { + "ENABLED": false + }, + "DATABASE": { + "ENABLED": true, + "HOST": "127.0.0.1", + "PORT": 3306, + "DATABASE": "mempool_mainnet_lightning", + "USERNAME": "mempool_mainnet_lightning", + "PASSWORD": "mempool_mainnet_lightning" + } +} diff --git a/production/mempool-config.signet-lightning.json b/production/mempool-config.signet-lightning.json new file mode 100644 index 000000000..08250ffbd --- /dev/null +++ b/production/mempool-config.signet-lightning.json @@ -0,0 +1,44 @@ +{ + "MEMPOOL": { + "NETWORK": "signet", + "BACKEND": "esplora", + "HTTP_PORT": 8991, + "INDEXING_BLOCKS_AMOUNT": 0, + "API_URL_PREFIX": "/api/v1/" + }, + "SYSLOG": { + "MIN_PRIORITY": "debug" + }, + "CORE_RPC": { + "PORT": 38332, + "USERNAME": "__BITCOIN_RPC_USER__", + "PASSWORD": "__BITCOIN_RPC_PASS__" + }, + "ESPLORA": { + "REST_API_URL": "http://127.0.0.1:4003" + }, + "LIGHTNING": { + "ENABLED": true, + "BACKEND": "cln", + "GRAPH_REFRESH_INTERVAL": 60, + "TOPOLOGY_FOLDER": "" + }, + "CLIGHTNING": { + "SOCKET": "/cln/.lightning/signet/lightning-rpc" + }, + "MAXMIND": { + "ENABLED": true, + "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoIP2-City.mmdb" + }, + "STATISTICS": { + "ENABLED": false + }, + "DATABASE": { + "ENABLED": true, + "HOST": "127.0.0.1", + "PORT": 3306, + "USERNAME": "mempool_signet_lightning", + "PASSWORD": "mempool_signet_lightning", + "DATABASE": "mempool_signet_lightning" + } +} diff --git a/production/mempool-config.testnet-lightning.json b/production/mempool-config.testnet-lightning.json new file mode 100644 index 000000000..c2a42e4bc --- /dev/null +++ b/production/mempool-config.testnet-lightning.json @@ -0,0 +1,44 @@ +{ + "MEMPOOL": { + "NETWORK": "testnet", + "BACKEND": "esplora", + "HTTP_PORT": 8992, + "INDEXING_BLOCKS_AMOUNT": 0, + "API_URL_PREFIX": "/api/v1/" + }, + "SYSLOG": { + "MIN_PRIORITY": "debug" + }, + "CORE_RPC": { + "PORT": 18332, + "USERNAME": "__BITCOIN_RPC_USER__", + "PASSWORD": "__BITCOIN_RPC_PASS__" + }, + "ESPLORA": { + "REST_API_URL": "http://127.0.0.1:4002" + }, + "LIGHTNING": { + "ENABLED": true, + "BACKEND": "cln", + "GRAPH_REFRESH_INTERVAL": 60, + "TOPOLOGY_FOLDER": "" + }, + "CLIGHTNING": { + "SOCKET": "/cln/.lightning/testnet/lightning-rpc" + }, + "MAXMIND": { + "ENABLED": true, + "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoIP2-City.mmdb" + }, + "STATISTICS": { + "ENABLED": false + }, + "DATABASE": { + "ENABLED": true, + "HOST": "127.0.0.1", + "PORT": 3306, + "USERNAME": "mempool_testnet_lightning", + "PASSWORD": "mempool_testnet_lightning", + "DATABASE": "mempool_testnet_lightning" + } +} diff --git a/production/mempool-kill-all b/production/mempool-kill-all index ae48552c2..6abb1a935 100755 --- a/production/mempool-kill-all +++ b/production/mempool-kill-all @@ -1,2 +1,21 @@ #!/usr/bin/env zsh -killall sh node + +# kill "while true" loops +killall sh + +# kill actual node backends +killall node + +# kill unfurler chrome instances +killall chrome + +# kill xorg +killall xinit + +# kill nginx cache warmer scripts +for pid in `ps uaxww|grep warmer|grep zsh|awk '{print $2}'`;do + kill $pid +done + +# always exit successfully despite above errors +exit 0 diff --git a/production/mempool-start-all b/production/mempool-start-all index 94766d5ce..13fd30430 100755 --- a/production/mempool-start-all +++ b/production/mempool-start-all @@ -2,7 +2,31 @@ export NVM_DIR="$HOME/.nvm" source "$NVM_DIR/nvm.sh" +# start all mempool backends that exist for site in mainnet mainnet-lightning testnet testnet-lightning signet signet-lightning bisq liquid liquidtestnet;do cd "${HOME}/${site}/backend/" && \ + echo "starting mempool backend: ${site}" && \ screen -dmS "${site}" sh -c 'while true;do npm run start-production;sleep 1;done' done + +# only start xorg if GPU present +if pciconf -lv|grep -i nvidia >/dev/null 2>&1;then + export DISPLAY=:0 + screen -dmS x startx + sleep 3 +fi + +# start unfurlers for each frontend +for site in mainnet liquid bisq;do + cd "$HOME/${site}/unfurler" && \ + echo "starting mempool unfurler: ${site}" && \ + screen -dmS "unfurler-${site}" sh -c 'while true;do npm run unfurler;sleep 2;done' +done + +# start nginx warm cacher +for site in mainnet;do + echo "starting mempool cache warmer: ${site}" + screen -dmS "warmer-${site}" $HOME/mainnet/production/nginx-cache-warmer +done + +exit 0 diff --git a/production/mempool.crontab b/production/mempool.crontab index 08639362f..cc1bcd878 100644 --- a/production/mempool.crontab +++ b/production/mempool.crontab @@ -1,9 +1,6 @@ # start on reboot @reboot sleep 10 ; $HOME/start -# start cache warmer on reboot -@reboot sleep 180 ; /mempool/mempool/production/nginx-cache-warmer >/dev/null 2>&1 & - # daily backup 37 13 * * * sleep 30 ; /mempool/mempool.space/backup >/dev/null 2>&1 & diff --git a/production/nginx-cache-warmer b/production/nginx-cache-warmer index a4ece6e0b..27d1e3a8f 100755 --- a/production/nginx-cache-warmer +++ b/production/nginx-cache-warmer @@ -2,6 +2,12 @@ hostname=$(hostname) slugs=(`curl -sSL https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json | jq -r '.slugs[]'`) +warm() +{ + echo "$1" + curl -i -s "$1" | head -1 +} + while true do for url in / \ '/api/v1/blocks' \ @@ -79,16 +85,48 @@ do for url in / \ '/api/v1/mining/difficulty-adjustments/all' \ '/api/v1/lightning/channels-geo?style=widget' \ '/api/v1/lightning/channels-geo?style=graph' \ + '/api/v1/lightning/statistics/latest' \ + '/api/v1/lightning/statistics/1m' \ + '/api/v1/lightning/statistics/3m' \ + '/api/v1/lightning/statistics/6m' \ + '/api/v1/lightning/statistics/1y' \ + '/api/v1/lightning/statistics/2y' \ + '/api/v1/lightning/statistics/3y' \ + '/api/v1/lightning/statistics/all' \ + '/api/v1/lightning/nodes/isp-ranking' \ + '/api/v1/lightning/nodes/isp/396982,15169' `# Google` \ + '/api/v1/lightning/nodes/isp/14618,16509' `# Amazon` \ + '/api/v1/lightning/nodes/isp/39572' `# DataWeb` \ + '/api/v1/lightning/nodes/isp/14061' `# Digital Ocean` \ + '/api/v1/lightning/nodes/isp/24940,213230' `# Hetzner` \ + '/api/v1/lightning/nodes/isp/394745' `# LunaNode` \ + '/api/v1/lightning/nodes/isp/45102' `# Alibaba` \ + '/api/v1/lightning/nodes/isp/3209' `# Vodafone Germany` \ + '/api/v1/lightning/nodes/isp/7922' `# Comcast Cable` \ + '/api/v1/lightning/nodes/isp/34197' `# SHRD SARL` \ + '/api/v1/lightning/nodes/isp/42275' `# Three Fourteen SASU` \ + '/api/v1/lightning/nodes/isp/16276' `# OVH SAS` \ + '/api/v1/lightning/nodes/isp/11426,11427,20001,20115,11351,10796,33363,12271' `# Spectrum` \ + '/api/v1/lightning/nodes/isp/701' `# Verizon` \ + '/api/v1/lightning/nodes/isp/12876' `# Scaleway` \ + '/api/v1/lightning/nodes/isp/33915' `# Ziggo` \ + '/api/v1/lightning/nodes/isp/3320' `# Deutsche Telekom AG` \ + '/api/v1/lightning/nodes/isp/8075' `# Microsoft Azure` \ + '/api/v1/lightning/nodes/countries' \ + '/api/v1/lightning/nodes/rankings' \ + '/api/v1/lightning/nodes/rankings/liquidity' \ + '/api/v1/lightning/nodes/rankings/connectivity' \ + '/api/v1/lightning/nodes/rankings/age' \ do - curl -s "https://${hostname}${url}" >/dev/null + warm "https://${hostname}${url}" done for slug in $slugs do - curl -s "https://${hostname}/api/v1/mining/pool/${slug}" >/dev/null - curl -s "https://${hostname}/api/v1/mining/pool/${slug}/hashrate" >/dev/null - curl -s "https://${hostname}/api/v1/mining/pool/${slug}/blocks" >/dev/null + warm "https://${hostname}/api/v1/mining/pool/${slug}" + warm "https://${hostname}/api/v1/mining/pool/${slug}/hashrate" + warm "https://${hostname}/api/v1/mining/pool/${slug}/blocks" done sleep 10 diff --git a/production/nginx/location-api-v1-services.conf b/production/nginx/location-api-v1-services.conf index 869036c54..aae49727e 100644 --- a/production/nginx/location-api-v1-services.conf +++ b/production/nginx/location-api-v1-services.conf @@ -70,30 +70,6 @@ location /api/v1/translators { proxy_hide_header content-security-policy; proxy_hide_header x-frame-options; } -location /api/v1/enterprise/images { - proxy_pass $mempoolSpaceServices; - proxy_cache services; - proxy_cache_background_update on; - proxy_cache_use_stale updating; - proxy_cache_valid 200 10m; - expires 10m; - proxy_hide_header onion-location; - proxy_hide_header strict-transport-security; - proxy_hide_header content-security-policy; - proxy_hide_header x-frame-options; -} -location /api/v1/enterprise { - proxy_pass $mempoolSpaceServices; - proxy_cache services; - proxy_cache_background_update on; - proxy_cache_use_stale updating; - proxy_cache_valid 200 5m; - expires 5m; - proxy_hide_header onion-location; - proxy_hide_header strict-transport-security; - proxy_hide_header content-security-policy; - proxy_hide_header x-frame-options; -} location /api/v1/assets { proxy_pass $mempoolSpaceServices; proxy_cache services; diff --git a/production/nginx/location-testnet-api-v1-lightning.conf b/production/nginx/location-testnet-api-v1-lightning.conf index 5319004ee..cc7c617a6 100644 --- a/production/nginx/location-testnet-api-v1-lightning.conf +++ b/production/nginx/location-testnet-api-v1-lightning.conf @@ -4,7 +4,7 @@ location /testnet/api/v1/lightning { try_files /dev/null @mempool-testnet-api-v1-lightning; } location @mempool-testnet-api-v1-lightning { - proxy_pass $mempoolSignetLightning; + proxy_pass $mempoolTestnetLightning; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; diff --git a/production/nginx/nginx.conf b/production/nginx/nginx.conf index 83a794e51..5861067e8 100644 --- a/production/nginx/nginx.conf +++ b/production/nginx/nginx.conf @@ -22,6 +22,11 @@ http { include mempool/production/nginx/http-proxy-cache.conf; include mempool/production/nginx/http-language.conf; + # match preview/unfurl bot user-agents + map $http_user_agent $unfurlbot { + default 0; + } + # mempool configuration include mempool/production/nginx/upstream-mempool.conf; @@ -42,6 +47,7 @@ http { # for services from mempool.space like contributors on about page set $mempoolSpaceServices "https://mempool.space"; + set $mempoolSpaceUnfurler "http://127.0.0.1:8001"; # for mempool daemons, see upstream-mempool.conf set $mempoolMainnet "http://mempool-bitcoin-mainnet"; @@ -77,6 +83,7 @@ http { # for services from mempool.space like contributors on about page set $mempoolSpaceServices "https://mempool.space"; + set $mempoolSpaceUnfurler "http://127.0.0.1:8001"; # for mempool daemons, see upstream-mempool.conf set $mempoolBisq "http://mempool-bitcoin-bisq"; @@ -105,6 +112,7 @@ http { # for services from mempool.space like contributors on about page set $mempoolSpaceServices "https://mempool.space"; + set $mempoolSpaceUnfurler "http://127.0.0.1:8001"; # for mempool daemons, see upstream-mempool.conf set $mempoolMainnet "http://mempool-liquid-mainnet"; diff --git a/production/nginx/server-common.conf b/production/nginx/server-common.conf index f1b21c1e0..26e81f7fa 100644 --- a/production/nginx/server-common.conf +++ b/production/nginx/server-common.conf @@ -48,6 +48,9 @@ add_header Vary Cookie; # for exact / requests, redirect based on $lang # cache redirect for 5 minutes location = / { + if ($unfurlbot) { + proxy_pass $mempoolSpaceUnfurler; + } if ($lang != '') { return 302 $scheme://$host/$lang/; } @@ -59,7 +62,7 @@ location = / { # cache /resources/** for 1 week since they don't change often location ~ ^/[a-z][a-z]/resources/(.*) { try_files $uri /en-US/resources/$1 =404; - expires 1w; + expires 1w; } # cache //main.f40e91d908a068a2.js forever since they never change location ~ ^/([a-z][a-z])/(.+\..+\.(js|css)) { @@ -69,11 +72,14 @@ location ~ ^/([a-z][a-z])/(.+\..+\.(js|css)) { # cache everything else for 5 minutes location ~ ^/([a-z][a-z])$ { try_files $uri /$1/index.html /en-US/index.html =404; - expires 5m; + expires 5m; } location ~ ^/([a-z][a-z])/ { + if ($unfurlbot) { + proxy_pass $mempoolSpaceUnfurler; + } try_files $uri /$1/index.html /en-US/index.html =404; - expires 5m; + expires 5m; } # cache /resources/** for 1 week since they don't change often @@ -86,9 +92,24 @@ location ~* ^/.+\..+\.(js|css) { try_files /$lang/$uri /en-US/$uri =404; expires 1y; } + +# unfurl preview +location /preview { + try_files /$lang/$uri $uri /en-US/$uri /en-US/index.html =404; + expires 10m; +} +# unfurl renderer +location ^~ /render { + proxy_pass $mempoolSpaceUnfurler; + expires 10m; +} + # catch-all for all URLs i.e. /address/foo /tx/foo /block/000 # cache 5 minutes since they change frequently location / { + if ($unfurlbot) { + proxy_pass $mempoolSpaceUnfurler; + } try_files /$lang/$uri $uri /en-US/$uri /en-US/index.html =404; expires 5m; } diff --git a/production/unfurl-build b/production/unfurl-build deleted file mode 100755 index 5b838e0ae..000000000 --- a/production/unfurl-build +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env zsh -PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:$HOME/bin -HOSTNAME=$(hostname) -LOCATION=$(hostname|cut -d . -f2) -LOCKFILE="${HOME}/lock" -REF=$(echo "${1:=origin/master}"|sed -e 's!:!/!') - -if [ -f "${LOCKFILE}" ];then - echo "upgrade already running? check lockfile ${LOCKFILE}" - exit 1 -fi - -# on exit, remove lockfile but preserve exit code -trap "rv=\$?; rm -f "${LOCKFILE}"; exit \$rv" INT TERM EXIT - -# create lockfile -touch "${LOCKFILE}" - -# notify logged in users -echo "Upgrading unfurler to ${REF}" | wall - -update_repo() -{ - echo "[*] Upgrading unfurler to ${REF}" - cd "$HOME/unfurl/unfurler" || exit 1 - - git fetch origin || exit 1 - for remote in origin;do - git remote add "${remote}" "https://github.com/${remote}/mempool" >/dev/null 2>&1 - git fetch "${remote}" || exit 1 - done - - if [ $(git tag -l "${REF}") ];then - git reset --hard "tags/${REF}" || exit 1 - elif [ $(git branch -r -l "origin/${REF}") ];then - git reset --hard "origin/${REF}" || exit 1 - else - git reset --hard "${REF}" || exit 1 - fi - export HASH=$(git rev-parse HEAD) -} - -build_backend() -{ - echo "[*] Building backend for unfurler" - [ -z "${HASH}" ] && exit 1 - cd "$HOME/unfurl/unfurler" || exit 1 - if [ ! -e "config.json" ];then - cp "${HOME}/unfurl/production/mempool-config.unfurl.json" "config.json" - fi - npm install || exit 1 - npm run build || exit 1 -} - -update_repo -build_backend - -# notify everyone -echo "${HOSTNAME} unfurl updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general mempool.dev -echo "${HOSTNAME} unfurl updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general "mempool.ops.${LOCATION}" - -exit 0 diff --git a/production/unfurl-kill b/production/unfurl-kill deleted file mode 100755 index ae48552c2..000000000 --- a/production/unfurl-kill +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env zsh -killall sh node diff --git a/production/unfurl-start b/production/unfurl-start deleted file mode 100755 index 29b5ddf3e..000000000 --- a/production/unfurl-start +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env zsh -export NVM_DIR="$HOME/.nvm" -source "$NVM_DIR/nvm.sh" - -cd "${HOME}/unfurl/unfurler/" && \ -screen -dmS "unfurl" sh -c 'while true;do npm run start-production;sleep 1;done' diff --git a/production/unfurler-config.bisq.json b/production/unfurler-config.bisq.json new file mode 100644 index 000000000..9742743f0 --- /dev/null +++ b/production/unfurler-config.bisq.json @@ -0,0 +1,17 @@ +{ + "SERVER": { + "HOST": "https://bisq.fra.mempool.space", + "HTTP_PORT": 8002 + }, + "MEMPOOL": { + "HTTP_HOST": "http://127.0.0.1", + "HTTP_PORT": 82, + "NETWORK": "bisq" + }, + "PUPPETEER": { + "CLUSTER_SIZE": 8, + "EXEC_PATH": "/usr/local/bin/chrome", + "MAX_PAGE_AGE": 86400, + "RENDER_TIMEOUT": 3000 + } +} diff --git a/production/unfurler-config.liquid.json b/production/unfurler-config.liquid.json new file mode 100644 index 000000000..243af6ed3 --- /dev/null +++ b/production/unfurler-config.liquid.json @@ -0,0 +1,17 @@ +{ + "SERVER": { + "HOST": "https://liquid.fra.mempool.space", + "HTTP_PORT": 8003 + }, + "MEMPOOL": { + "HTTP_HOST": "http://127.0.0.1", + "HTTP_PORT": 83, + "NETWORK": "bitcoin" + }, + "PUPPETEER": { + "CLUSTER_SIZE": 8, + "EXEC_PATH": "/usr/local/bin/chrome", + "MAX_PAGE_AGE": 86400, + "RENDER_TIMEOUT": 3000 + } +} diff --git a/production/unfurler-config.mainnet.json b/production/unfurler-config.mainnet.json new file mode 100644 index 000000000..77df23704 --- /dev/null +++ b/production/unfurler-config.mainnet.json @@ -0,0 +1,17 @@ +{ + "SERVER": { + "HOST": "https://mempool.fra.mempool.space", + "HTTP_PORT": 8001 + }, + "MEMPOOL": { + "HTTP_HOST": "http://127.0.0.1", + "HTTP_PORT": 81, + "NETWORK": "bitcoin" + }, + "PUPPETEER": { + "CLUSTER_SIZE": 8, + "EXEC_PATH": "/usr/local/bin/chrome", + "MAX_PAGE_AGE": 86400, + "RENDER_TIMEOUT": 3000 + } +} diff --git a/unfurler/config.sample.json b/unfurler/config.sample.json index e080ee68a..64f56c1f7 100644 --- a/unfurler/config.sample.json +++ b/unfurler/config.sample.json @@ -9,6 +9,7 @@ "NETWORK": "bitcoin" // "bitcoin" | "liquid" | "bisq" (optional - defaults to "bitcoin") }, "PUPPETEER": { + "DISABLE": false, // optional, boolean, disables puppeteer and /render endpoints "CLUSTER_SIZE": 2, "EXEC_PATH": "/usr/local/bin/chrome", // optional "MAX_PAGE_AGE": 86400, // maximum lifetime of a page session (in seconds) diff --git a/unfurler/package-lock.json b/unfurler/package-lock.json index 3da33c69f..40520d413 100644 --- a/unfurler/package-lock.json +++ b/unfurler/package-lock.json @@ -1,12 +1,12 @@ { "name": "mempool-unfurl", - "version": "0.0.1", + "version": "0.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "mempool-unfurl", - "version": "0.0.1", + "version": "0.1.0", "dependencies": { "@types/node": "^16.11.41", "express": "^4.18.0", diff --git a/unfurler/package.json b/unfurler/package.json index 2d353bfdf..59d48aa50 100644 --- a/unfurler/package.json +++ b/unfurler/package.json @@ -1,6 +1,6 @@ { "name": "mempool-unfurl", - "version": "0.0.2", + "version": "0.1.0", "description": "Renderer for mempool open graph link preview images", "repository": { "type": "git", @@ -11,7 +11,7 @@ "tsc": "./node_modules/typescript/bin/tsc", "build": "npm run tsc", "start": "node --max-old-space-size=2048 dist/index.js", - "start-production": "node --max-old-space-size=4096 dist/index.js", + "unfurler": "node --max-old-space-size=4096 dist/index.js", "lint": "./node_modules/.bin/eslint . --ext .ts", "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix", "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\"" diff --git a/unfurler/puppeteer.config.json b/unfurler/puppeteer.config.json index 3de7b0652..b3a9b7fc4 100644 --- a/unfurler/puppeteer.config.json +++ b/unfurler/puppeteer.config.json @@ -41,6 +41,6 @@ "--use-mock-keychain", "--ignore-gpu-blacklist", "--ignore-gpu-blocklist", - "--use-gl=swiftshader" + "--use-gl=egl" ] } diff --git a/unfurler/src/config.ts b/unfurler/src/config.ts index a65d48f6f..3c4a4e422 100644 --- a/unfurler/src/config.ts +++ b/unfurler/src/config.ts @@ -11,6 +11,7 @@ interface IConfig { NETWORK?: string; }; PUPPETEER: { + DISABLE: boolean; CLUSTER_SIZE: number; EXEC_PATH?: string; MAX_PAGE_AGE?: number; @@ -28,6 +29,7 @@ const defaults: IConfig = { 'HTTP_PORT': 4200, }, 'PUPPETEER': { + 'DISABLE': false, 'CLUSTER_SIZE': 1, }, }; diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 08dff3964..eab4723b7 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -1,10 +1,12 @@ import express from "express"; import { Application, Request, Response, NextFunction } from 'express'; import * as http from 'http'; +import * as https from 'https'; import config from './config'; import { Cluster } from 'puppeteer-cluster'; import ReusablePage from './concurrency/ReusablePage'; import { parseLanguageUrl } from './language/lang'; +import { matchRoute } from './routes'; const puppeteerConfig = require('../puppeteer.config.json'); if (config.PUPPETEER.EXEC_PATH) { @@ -17,13 +19,13 @@ class Server { cluster?: Cluster; mempoolHost: string; network: string; - defaultImageUrl: string; + secureHost = true; constructor() { this.app = express(); this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : ''); + this.secureHost = this.mempoolHost.startsWith('https'); this.network = config.MEMPOOL.NETWORK || 'bitcoin'; - this.defaultImageUrl = this.getDefaultImageUrl(); this.startServer(); } @@ -37,12 +39,14 @@ class Server { .use(express.text()) ; - this.cluster = await Cluster.launch({ - concurrency: ReusablePage, - maxConcurrency: config.PUPPETEER.CLUSTER_SIZE, - puppeteerOptions: puppeteerConfig, - }); - await this.cluster?.task(async (args) => { return this.clusterTask(args) }); + if (!config.PUPPETEER.DISABLE) { + this.cluster = await Cluster.launch({ + concurrency: ReusablePage, + maxConcurrency: config.PUPPETEER.CLUSTER_SIZE, + puppeteerOptions: puppeteerConfig, + }); + await this.cluster?.task(async (args) => { return this.clusterTask(args) }); + } this.setUpRoutes(); @@ -64,7 +68,11 @@ class Server { } setUpRoutes() { - this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) }) + if (!config.PUPPETEER.DISABLE) { + this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) }) + } else { + this.app.get('/render*', async (req, res) => { return this.renderDisabled(req, res) }) + } this.app.get('*', (req, res) => { return this.renderHTML(req, res) }) } @@ -111,13 +119,31 @@ class Server { } } + async renderDisabled(req, res) { + res.status(500).send("preview rendering disabled"); + } + async renderPreview(req, res) { try { - const path = req.params[0] - const img = await this.cluster?.execute({ url: this.mempoolHost + path, path: path, action: 'screenshot' }); + const rawPath = req.params[0]; + + let img = null; + + const { lang, path } = parseLanguageUrl(rawPath); + const matchedRoute = matchRoute(this.network, path); + + // don't bother unless the route is definitely renderable + if (rawPath.includes('/preview/') && matchedRoute.render) { + img = await this.cluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'screenshot' }); + } if (!img) { - res.status(500).send('failed to render page preview'); + // proxy fallback image from the frontend + if (this.secureHost) { + https.get(config.SERVER.HOST + matchedRoute.fallbackImg, (got) => got.pipe(res)); + } else { + http.get(config.SERVER.HOST + matchedRoute.fallbackImg, (got) => got.pipe(res)); + } } else { res.contentType('image/png'); res.send(img); @@ -137,50 +163,14 @@ class Server { return; } - let previewSupported = true; - let mode = 'mainnet' - let ogImageUrl = this.defaultImageUrl; - let ogTitle; const { lang, path } = parseLanguageUrl(rawPath); - const parts = path.slice(1).split('/'); + const matchedRoute = matchRoute(this.network, path); + let ogImageUrl = config.SERVER.HOST + (matchedRoute.staticImg || matchedRoute.fallbackImg); + let ogTitle = 'The Mempool Open Source Project™'; - // handle network mode modifiers - if (['testnet', 'signet'].includes(parts[0])) { - mode = parts.shift(); - } - - // handle supported preview routes - switch (parts[0]) { - case 'block': - ogTitle = `Block: ${parts[1]}`; - break; - case 'address': - ogTitle = `Address: ${parts[1]}`; - break; - case 'tx': - ogTitle = `Transaction: ${parts[1]}`; - break; - case 'lightning': - switch (parts[1]) { - case 'node': - ogTitle = `Lightning Node: ${parts[2]}`; - break; - case 'channel': - ogTitle = `Lightning Channel: ${parts[2]}`; - break; - default: - previewSupported = false; - } - break; - default: - previewSupported = false; - } - - if (previewSupported) { + if (matchedRoute.render) { ogImageUrl = `${config.SERVER.HOST}/render/${lang || 'en'}/preview${path}`; - ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${mode !== 'mainnet' ? capitalize(mode) + ' ' : ''}${ogTitle}`; - } else { - ogTitle = 'The Mempool Open Source Project™'; + ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${matchedRoute.networkMode !== 'mainnet' ? capitalize(matchedRoute.networkMode) + ' ' : ''}${matchedRoute.title}`; } res.send(` @@ -189,34 +179,23 @@ class Server { ${ogTitle} - + - - + + - + `); } - - getDefaultImageUrl() { - switch (this.network) { - case 'liquid': - return this.mempoolHost + '/resources/liquid/liquid-network-preview.png'; - case 'bisq': - return this.mempoolHost + '/resources/bisq/bisq-markets-preview.png'; - default: - return this.mempoolHost + '/resources/mempool-space-preview.png'; - } - } } const server = new Server(); diff --git a/unfurler/src/routes.ts b/unfurler/src/routes.ts new file mode 100644 index 000000000..24922c85d --- /dev/null +++ b/unfurler/src/routes.ts @@ -0,0 +1,124 @@ +interface Match { + render: boolean; + title: string; + fallbackImg: string; + staticImg?: string; + networkMode: string; +} + +const routes = { + block: { + render: true, + params: 1, + getTitle(path) { + return `Block: ${path[0]}`; + } + }, + address: { + render: true, + params: 1, + getTitle(path) { + return `Address: ${path[0]}`; + } + }, + tx: { + render: true, + params: 1, + getTitle(path) { + return `Transaction: ${path[0]}`; + } + }, + lightning: { + title: "Lightning", + fallbackImg: '/resources/previews/lightning.png', + routes: { + node: { + render: true, + params: 1, + getTitle(path) { + return `Lightning Node: ${path[0]}`; + } + }, + channel: { + render: true, + params: 1, + getTitle(path) { + return `Lightning Channel: ${path[0]}`; + } + }, + } + }, + mining: { + title: "Mining", + fallbackImg: '/resources/previews/mining.png' + } +}; + +const networks = { + bitcoin: { + fallbackImg: '/resources/mempool-space-preview.png', + staticImg: '/resources/previews/dashboard.png', + routes: { + ...routes // all routes supported + } + }, + liquid: { + fallbackImg: '/resources/liquid/liquid-network-preview.png', + routes: { // only block, address & tx routes supported + block: routes.block, + address: routes.address, + tx: routes.tx + } + }, + bisq: { + fallbackImg: '/resources/bisq/bisq-markets-preview.png', + routes: {} // no routes supported + } +}; + +export function matchRoute(network: string, path: string): Match { + const match: Match = { + render: false, + title: '', + fallbackImg: '', + networkMode: 'mainnet' + } + + const parts = path.slice(1).split('/').filter(p => p.length); + + if (parts[0] === 'preview') { + parts.shift(); + } + if (['testnet', 'signet'].includes(parts[0])) { + match.networkMode = parts.shift() || 'mainnet'; + } + + let route = networks[network] || networks.bitcoin; + match.fallbackImg = route.fallbackImg; + + // traverse the route tree until we run out of route or tree, or hit a renderable match + while (!route.render && route.routes && parts.length && route.routes[parts[0]]) { + route = route.routes[parts[0]]; + parts.shift(); + if (route.fallbackImg) { + match.fallbackImg = route.fallbackImg; + } + } + + // enough route parts left for title & rendering + if (route.render && parts.length >= route.params) { + match.render = true; + } + // only use set a static image for exact matches + if (!parts.length && route.staticImg) { + match.staticImg = route.staticImg; + } + // apply the title function if present + if (route.getTitle && typeof route.getTitle === 'function') { + match.title = route.getTitle(parts); + } else { + match.title = route.title; + } + + return match; +} \ No newline at end of file