diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 8635ee96f..b9cc1453c 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -228,34 +228,75 @@ export class Common { return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0]; } - static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket { + static findSocketNetwork(addr: string): {network: string | null, url: string} { let network: string | null = null; + let url = addr.split('://')[1]; - if (config.LIGHTNING.BACKEND === 'cln') { - network = socket.network; - } else if (config.LIGHTNING.BACKEND === 'lnd') { - if (socket.addr.indexOf('onion') !== -1) { - if (socket.addr.split('.')[0].length >= 56) { - network = 'torv3'; - } else { - network = 'torv2'; - } - } else if (socket.addr.indexOf('i2p') !== -1) { - network = 'i2p'; + if (!url) { + return { + network: null, + url: addr, + }; + } + + if (addr.indexOf('onion') !== -1) { + if (url.split('.')[0].length >= 56) { + network = 'torv3'; } else { - const ipv = isIP(socket.addr.split(':')[0]); - if (ipv === 4) { - network = 'ipv4'; - } else if (ipv === 6) { - network = 'ipv6'; - } + network = 'torv2'; } + } else if (addr.indexOf('i2p') !== -1) { + network = 'i2p'; + } else if (addr.indexOf('ipv4') !== -1) { + const ipv = isIP(url.split(':')[0]); + if (ipv === 4) { + network = 'ipv4'; + } else { + return { + network: null, + url: addr, + }; + } + } else if (addr.indexOf('ipv6') !== -1) { + url = url.split('[')[1].split(']')[0]; + const ipv = isIP(url); + if (ipv === 6) { + const parts = addr.split(':'); + network = 'ipv6'; + url = `[${url}]:${parts[parts.length - 1]}`; + } else { + return { + network: null, + url: addr, + }; + } + } else { + return { + network: null, + url: addr, + }; } return { - publicKey: publicKey, network: network, - addr: socket.addr, + url: url, }; } + + static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket { + if (config.LIGHTNING.BACKEND === 'cln') { + return { + publicKey: publicKey, + network: socket.network, + addr: socket.addr, + }; + } else /* if (config.LIGHTNING.BACKEND === 'lnd') */ { + const formatted = this.findSocketNetwork(socket.addr); + return { + publicKey: publicKey, + network: formatted.network, + addr: formatted.url, + }; + } + } } diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 56cfebe9f..fcd5b8815 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -503,6 +503,18 @@ class NodesApi { } } + /** + * Update node sockets + */ + public async $updateNodeSockets(publicKey: string, sockets: {network: string; addr: string}[]): Promise { + const formattedSockets = (sockets.map(a => a.addr).join(',')) ?? ''; + try { + await DB.query(`UPDATE nodes SET sockets = ? WHERE public_key = ?`, [formattedSockets, publicKey]); + } catch (e) { + logger.err(`Cannot update node sockets for ${publicKey}. Reason: ${e instanceof Error ? e.message : e}`); + } + } + /** * Set all nodes not in `nodesPubkeys` as inactive (status = 0) */ diff --git a/backend/src/api/explorer/statistics.api.ts b/backend/src/api/explorer/statistics.api.ts index 7bf3d9107..558ee86fd 100644 --- a/backend/src/api/explorer/statistics.api.ts +++ b/backend/src/api/explorer/statistics.api.ts @@ -27,7 +27,7 @@ class StatisticsApi { public async $getLatestStatistics(): Promise { try { const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1`); - const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1 OFFSET 7`); + const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats WHERE DATE(added) = DATE(NOW() - INTERVAL 7 DAY)`); return { latest: rows[0], previous: rows2[0], diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index dd958a6e3..e3dfe6652 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -57,6 +57,8 @@ class LightningStatsImporter { features: node.features, }); nodesInDb[node.pub_key] = node; + } else { + await nodesApi.$updateNodeSockets(node.pub_key, node.addresses); } let hasOnion = false; @@ -369,7 +371,7 @@ class LightningStatsImporter { graph = JSON.parse(fileContent); graph = await this.cleanupTopology(graph); } catch (e) { - logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`); + logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content. Reason: ${e instanceof Error ? e.message : e}`); continue; } @@ -419,9 +421,10 @@ class LightningStatsImporter { const addressesParts = (node.addresses ?? '').split(','); const addresses: any[] = []; for (const address of addressesParts) { + const formatted = Common.findSocketNetwork(address); addresses.push({ - network: '', - addr: address + network: formatted.network, + addr: formatted.url }); } diff --git a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss index 5aaf8a91b..c5a217983 100644 --- a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss +++ b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss @@ -61,7 +61,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss b/frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss index 5aaf8a91b..c5a217983 100644 --- a/frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss +++ b/frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss @@ -61,7 +61,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.scss b/frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.scss index 5aaf8a91b..c5a217983 100644 --- a/frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.scss +++ b/frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.scss @@ -61,7 +61,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss index 5aaf8a91b..c5a217983 100644 --- a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss +++ b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss @@ -61,7 +61,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss b/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss index a47f63923..7b1395d78 100644 --- a/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss +++ b/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss @@ -61,7 +61,7 @@ flex-direction: column; @media (min-width: 1130px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 1130px) { position: relative; diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss index 52b5b2c2f..3021cf689 100644 --- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss @@ -61,7 +61,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.scss b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.scss index b15df44fa..c382d9886 100644 --- a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.scss +++ b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.scss @@ -55,7 +55,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/components/pool-ranking/pool-ranking.component.scss b/frontend/src/app/components/pool-ranking/pool-ranking.component.scss index 855b4e65c..8cb82d92d 100644 --- a/frontend/src/app/components/pool-ranking/pool-ranking.component.scss +++ b/frontend/src/app/components/pool-ranking/pool-ranking.component.scss @@ -39,7 +39,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/components/statistics/statistics.component.html b/frontend/src/app/components/statistics/statistics.component.html index 83ec77acf..9f62fffce 100644 --- a/frontend/src/app/components/statistics/statistics.component.html +++ b/frontend/src/app/components/statistics/statistics.component.html @@ -11,7 +11,7 @@
+ [class]="(stateService.env.MINING_DASHBOARD || stateService.env.LIGHTNING) ? 'mining' : 'no-menu'" (click)="saveGraphPreference()">
- +
diff --git a/frontend/src/app/lightning/channels-list/channels-list.component.ts b/frontend/src/app/lightning/channels-list/channels-list.component.ts index 6a0732522..75b8263e2 100644 --- a/frontend/src/app/lightning/channels-list/channels-list.component.ts +++ b/frontend/src/app/lightning/channels-list/channels-list.component.ts @@ -14,6 +14,7 @@ import { LightningApiService } from '../lightning-api.service'; export class ChannelsListComponent implements OnInit, OnChanges { @Input() publicKey: string; @Output() channelsStatusChangedEvent = new EventEmitter(); + @Output() loadingEvent = new EventEmitter(false); channels$: Observable; // @ts-ignore @@ -26,6 +27,7 @@ export class ChannelsListComponent implements OnInit, OnChanges { defaultStatus = 'open'; status = 'open'; publicKeySize = 25; + isLoading = false; constructor( private lightningApiService: LightningApiService, @@ -56,6 +58,8 @@ export class ChannelsListComponent implements OnInit, OnChanges { ) .pipe( tap((val) => { + this.isLoading = true; + this.loadingEvent.emit(true); if (typeof val === 'string') { this.status = val; this.page = 1; @@ -64,10 +68,12 @@ export class ChannelsListComponent implements OnInit, OnChanges { } }), switchMap(() => { - this.channelsStatusChangedEvent.emit(this.status); - return this.lightningApiService.getChannelsByNodeId$(this.publicKey, (this.page - 1) * this.itemsPerPage, this.status); + this.channelsStatusChangedEvent.emit(this.status); + return this.lightningApiService.getChannelsByNodeId$(this.publicKey, (this.page - 1) * this.itemsPerPage, this.status); }), map((response) => { + this.isLoading = false; + this.loadingEvent.emit(false); return { channels: response.body, totalItems: parseInt(response.headers.get('x-total-count'), 10) diff --git a/frontend/src/app/lightning/channels-statistics/channels-statistics.component.html b/frontend/src/app/lightning/channels-statistics/channels-statistics.component.html index 033438cf3..31261a84b 100644 --- a/frontend/src/app/lightning/channels-statistics/channels-statistics.component.html +++ b/frontend/src/app/lightning/channels-statistics/channels-statistics.component.html @@ -9,44 +9,44 @@
-
+
Avg Capacity
-
+
{{ statistics.latest?.avg_capacity || 0 | number: '1.0-0' }} sats
- +
-
+
Avg Fee Rate
-
+
{{ statistics.latest?.avg_fee_rate || 0 | number: '1.0-0' }} ppm
- +
-
+
Avg Base Fee
-
+
{{ statistics.latest?.avg_base_fee_mtokens || 0 | number: '1.0-0' }} msats
- +
@@ -55,43 +55,45 @@
-
+
Med Capacity
-
+
{{ statistics.latest?.med_capacity || 0 | number: '1.0-0' }} sats
- +
-
+ +
Med Fee Rate
-
+
{{ statistics.latest?.med_fee_rate || 0 | number: '1.0-0' }} ppm
- +
-
+ +
Med Base Fee
-
+
{{ statistics.latest?.med_base_fee_mtokens || 0 | number: '1.0-0' }} msats
- +
diff --git a/frontend/src/app/lightning/channels-statistics/channels-statistics.component.scss b/frontend/src/app/lightning/channels-statistics/channels-statistics.component.scss index 372d9eb78..e97f0d0af 100644 --- a/frontend/src/app/lightning/channels-statistics/channels-statistics.component.scss +++ b/frontend/src/app/lightning/channels-statistics/channels-statistics.component.scss @@ -18,6 +18,10 @@ } } +.fee-estimation-wrapper { + min-height: 77px; +} + .fee-estimation-container { display: flex; justify-content: space-between; @@ -30,7 +34,10 @@ width: -webkit-fill-available; @media (min-width: 376px) { margin: 0 auto 0px; - } + } + &.more-padding { + padding-top: 10px; + } &:first-child{ display: none; @media (min-width: 485px) { @@ -57,6 +64,9 @@ margin: auto; line-height: 1.45; padding: 0px 2px; + &.no-border { + border-bottom: none; + } } .fiat { display: block; diff --git a/frontend/src/app/lightning/node-statistics/node-statistics.component.html b/frontend/src/app/lightning/node-statistics/node-statistics.component.html index 152ee2dbe..ae4ea3dd7 100644 --- a/frontend/src/app/lightning/node-statistics/node-statistics.component.html +++ b/frontend/src/app/lightning/node-statistics/node-statistics.component.html @@ -1,76 +1,64 @@
-
-
Capacity
-
-
+
+
Capacity
+
+
- +
-
-
Nodes
-
-
+
+
Nodes
+
+
{{ statistics.latest?.node_count || 0 | number }}
- +
-
-
Channels
-
-
+
+
Channels
+
+
{{ statistics.latest?.channel_count || 0 | number }}
- +
-
-
Nodes
+
Nodes
-
Channels
+
Channels
-
Average Channel
+
Average Channel
diff --git a/frontend/src/app/lightning/node-statistics/node-statistics.component.scss b/frontend/src/app/lightning/node-statistics/node-statistics.component.scss index acc4578f3..1532f9c4b 100644 --- a/frontend/src/app/lightning/node-statistics/node-statistics.component.scss +++ b/frontend/src/app/lightning/node-statistics/node-statistics.component.scss @@ -18,6 +18,10 @@ } } +.fee-estimation-wrapper { + min-height: 77px; +} + .fee-estimation-container { display: flex; justify-content: space-between; @@ -30,7 +34,10 @@ width: -webkit-fill-available; @media (min-width: 376px) { margin: 0 auto 0px; - } + } + &.more-padding { + padding-top: 10px; + } &:first-child{ display: none; @media (min-width: 485px) { @@ -57,6 +64,9 @@ margin: auto; line-height: 1.45; padding: 0px 2px; + &.no-border { + border-bottom: none; + } } .fiat { display: block; diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index f2a6ce6f4..def6e28e6 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -133,7 +133,7 @@ -
+

Open channels ({{ node.opened_channel_count }}) @@ -142,10 +142,13 @@ Closed channels ({{ node.closed_channel_count }})

+
+ (channelsStatusChangedEvent)="onChannelsListStatusChanged($event)" + (loadingEvent)="onLoadingEvent($event)" + >
diff --git a/frontend/src/app/lightning/node/node.component.scss b/frontend/src/app/lightning/node/node.component.scss index 2b171416f..b2e6f573b 100644 --- a/frontend/src/app/lightning/node/node.component.scss +++ b/frontend/src/app/lightning/node/node.component.scss @@ -56,4 +56,17 @@ app-fiat { display: inline-block; margin-left: 10px; } +} + +.spinner-border { + @media (min-width: 768px) { + margin-top: 6.5px; + width: 1.75rem; + height: 1.75rem; + } + @media (max-width: 768px) { + margin-top: 2.3px; + width: 1.5rem; + height: 1.5rem; + } } \ No newline at end of file diff --git a/frontend/src/app/lightning/node/node.component.ts b/frontend/src/app/lightning/node/node.component.ts index 8ddaacf95..bfee9252d 100644 --- a/frontend/src/app/lightning/node/node.component.ts +++ b/frontend/src/app/lightning/node/node.component.ts @@ -22,7 +22,7 @@ export class NodeComponent implements OnInit { channelsListStatus: string; error: Error; publicKey: string; - + channelListLoading = false; publicKeySize = 99; constructor( @@ -97,4 +97,8 @@ export class NodeComponent implements OnInit { onChannelsListStatusChanged(e) { this.channelsListStatus = e; } + + onLoadingEvent(e) { + this.channelListLoading = e; + } } diff --git a/frontend/src/app/lightning/nodes-channels/node-channels.component.ts b/frontend/src/app/lightning/nodes-channels/node-channels.component.ts index 074315b35..9447c6376 100644 --- a/frontend/src/app/lightning/nodes-channels/node-channels.component.ts +++ b/frontend/src/app/lightning/nodes-channels/node-channels.component.ts @@ -44,13 +44,13 @@ export class NodeChannels implements OnChanges { switchMap((response) => { this.isLoading = true; if ((response.body?.length ?? 0) <= 0) { - return []; + this.isLoading = false; + return ['']; } return [response.body]; }), tap((body: any[]) => { - if (body.length === 0) { - this.isLoading = false; + if (body.length === 0 || body[0].length === 0) { return; } const biggestCapacity = body[0].capacity; diff --git a/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.scss b/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.scss index 760e782ca..4ff4dbf3e 100644 --- a/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.scss +++ b/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.scss @@ -60,7 +60,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html index 25773a06e..ef1c71efd 100644 --- a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html +++ b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html @@ -28,7 +28,7 @@
- Top 100 ISP hosting LN nodes + Top 100 ISPs hosting LN nodes diff --git a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss index c6897cda9..54055de36 100644 --- a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss +++ b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss @@ -40,7 +40,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.scss b/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.scss index c2e00e520..c9b835054 100644 --- a/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.scss +++ b/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.scss @@ -60,7 +60,7 @@ flex-direction: column; @media (min-width: 991px) { position: relative; - top: -65px; + top: -100px; } @media (min-width: 830px) and (max-width: 991px) { position: relative; diff --git a/frontend/src/index.bisq.html b/frontend/src/index.bisq.html index b5c65e8d4..8da1e77e0 100644 --- a/frontend/src/index.bisq.html +++ b/frontend/src/index.bisq.html @@ -5,7 +5,7 @@ mempool - Bisq Markets - + @@ -14,7 +14,7 @@ - + diff --git a/frontend/src/index.liquid.html b/frontend/src/index.liquid.html index 65d397aac..89a6984ba 100644 --- a/frontend/src/index.liquid.html +++ b/frontend/src/index.liquid.html @@ -5,7 +5,7 @@ mempool - Liquid Network - + @@ -14,7 +14,7 @@ - + diff --git a/frontend/src/index.mempool.html b/frontend/src/index.mempool.html index 05c8d8398..1176a3da2 100644 --- a/frontend/src/index.mempool.html +++ b/frontend/src/index.mempool.html @@ -5,7 +5,7 @@ mempool - Bitcoin Explorer - + @@ -14,7 +14,7 @@ - + diff --git a/frontend/src/resources/previews/dashboard.png b/frontend/src/resources/previews/dashboard.png new file mode 100644 index 000000000..f2f20b9d8 Binary files /dev/null and b/frontend/src/resources/previews/dashboard.png differ diff --git a/frontend/src/resources/previews/lightning.png b/frontend/src/resources/previews/lightning.png new file mode 100644 index 000000000..f214dc1f9 Binary files /dev/null and b/frontend/src/resources/previews/lightning.png differ diff --git a/frontend/src/resources/previews/mining.png b/frontend/src/resources/previews/mining.png new file mode 100644 index 000000000..37c3873c1 Binary files /dev/null and b/frontend/src/resources/previews/mining.png differ diff --git a/production/install b/production/install index 71bcb7f6b..4eac1817b 100755 --- a/production/install +++ b/production/install @@ -48,6 +48,9 @@ BITCOIN_MAINNET_ENABLE=ON BITCOIN_MAINNET_MINFEE_ENABLE=ON BITCOIN_TESTNET_ENABLE=ON BITCOIN_SIGNET_ENABLE=ON +LN_BITCOIN_MAINNET_ENABLE=ON +LN_BITCOIN_TESTNET_ENABLE=ON +LN_BITCOIN_SIGNET_ENABLE=ON BISQ_MAINNET_ENABLE=ON ELEMENTS_LIQUID_ENABLE=ON ELEMENTS_LIQUIDTESTNET_ENABLE=ON @@ -227,6 +230,9 @@ MYSQL_GROUP=mysql MEMPOOL_MAINNET_USER='mempool' MEMPOOL_TESTNET_USER='mempool_testnet' MEMPOOL_SIGNET_USER='mempool_signet' +LN_MEMPOOL_MAINNET_USER='mempool_mainnet_lightning' +LN_MEMPOOL_TESTNET_USER='mempool_testnet_lightning' +LN_MEMPOOL_SIGNET_USER='mempool_signet_lightning' MEMPOOL_LIQUID_USER='mempool_liquid' MEMPOOL_LIQUIDTESTNET_USER='mempool_liquidtestnet' MEMPOOL_BISQ_USER='mempool_bisq' @@ -234,6 +240,9 @@ MEMPOOL_BISQ_USER='mempool_bisq' MEMPOOL_MAINNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') MEMPOOL_TESTNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') MEMPOOL_SIGNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') +LN_MEMPOOL_MAINNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') +LN_MEMPOOL_TESTNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') +LN_MEMPOOL_SIGNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') MEMPOOL_LIQUID_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') MEMPOOL_LIQUIDTESTNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') MEMPOOL_BISQ_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') @@ -391,6 +400,10 @@ FREEBSD_PKG+=(boost-libs autoconf automake gmake gcc libevent libtool pkgconf) FREEBSD_PKG+=(nginx rsync py39-certbot-nginx mariadb105-server keybase) FREEBSD_PKG+=(geoipupdate) +FREEBSD_UNFURL_PKG=() +FREEBSD_UNFURL_PKG+=(nvidia-driver-470-470.129.06 chromium xinit xterm twm ja-sourcehansans-otf) +FREEBSD_UNFURL_PKG+=(zh-sourcehansans-sc-otf ko-aleefonts-ttf lohit tlwg-ttf) + ############################# ##### utility functions ##### ############################# @@ -747,6 +760,9 @@ $CUT >$input <<-EOF Tor:Enable Tor v3 HS Onion:ON Mainnet:Enable Bitcoin Mainnet:ON Mainnet-Minfee:Enable Bitcoin Mainnet Minfee:ON +LN-Mainnet:Enable Bitcoin Mainnet Lightning:ON +LN-Testnet:Enable Bitcoin Testnet Lightning:ON +LN-Signet:Enable Bitcoin Signet Lightning:ON Testnet:Enable Bitcoin Testnet:ON Signet:Enable Bitcoin Signet:ON Liquid:Enable Elements Liquid:ON @@ -809,6 +825,24 @@ else BITCOIN_INSTALL=OFF fi +if grep LN-Mainnet $tempfile >/dev/null 2>&1;then + LN_BITCOIN_MAINNET_ENABLE=ON +else + LN_BITCOIN_MAINNET_ENABLE=OFF +fi + +if grep LN-Testnet $tempfile >/dev/null 2>&1;then + LN_BITCOIN_TESTNET_ENABLE=ON +else + LN_BITCOIN_TESTNET_ENABLE=OFF +fi + +if grep LN-Signet $tempfile >/dev/null 2>&1;then + LN_BITCOIN_SIGNET_ENABLE=ON +else + LN_BITCOIN_SIGNET_ENABLE=OFF +fi + if grep Liquid $tempfile >/dev/null 2>&1;then ELEMENTS_LIQUID_ENABLE=ON else @@ -831,6 +865,7 @@ if grep CoreLN $tempfile >/dev/null 2>&1;then CLN_INSTALL=ON else CLN_INSTALL=OFF +fi if [ "${BITCOIN_MAINNET_ENABLE}" = ON -o "${BITCOIN_TESTNET_ENABLE}" = ON -o "${BITCOIN_SIGNET_ENABLE}" = ON ];then BITCOIN_ELECTRS_INSTALL=ON @@ -1279,8 +1314,11 @@ case $OS in echo "[*] Creating Core Lightning user" osGroupCreate "${CLN_GROUP}" osUserCreate "${CLN_USER}" "${CLN_HOME}" "${CLN_GROUP}" + osSudo "${ROOT_USER}" pw usermod ${MEMPOOL_USER} -G "${CLN_GROUP}" osSudo "${ROOT_USER}" chsh -s `which zsh` "${CLN_USER}" echo "export PATH=$PATH:$HOME/.local/bin" >> "${CLN_HOME}/.zshrc" + osSudo "${ROOT_USER}" mkdir -p "${CLN_HOME}/.lightning/{bitcoin,signet,testnet}" + osSudo "${ROOT_USER}" chmod 750 "${CLN_HOME}" "${CLN_HOME}/.lightning" "${CLN_HOME}/.lightning/{bitcoin,signet,testnet}" osSudo "${ROOT_USER}" chown -R "${CLN_USER}:${CLN_GROUP}" "${CLN_HOME}" echo "[*] Installing Core Lightning package" @@ -1397,7 +1435,42 @@ if [ "${UNFURL_INSTALL}" = ON ];then case $OS in FreeBSD) - echo "[*] FIXME: Unfurl must be installed manually on FreeBSD" + + if pciconf -lv|grep -i nvidia >/dev/null 2>&1;then + echo "[*] GPU detected: Installing packages for Unfurl" + osPackageInstall ${FREEBSD_UNFURL_PKG[@]} + + echo 'allowed_users = anybody' >> /usr/local/etc/X11/Xwrapper.config + echo 'kld_list="nvidia"' >> /etc/rc.conf + echo 'nvidia_xorg_enable="YES"' >> /etc/rc.conf + + echo "[*] Installing color emoji" + osSudo "${ROOT_USER}" curl "https://github.com/samuelngs/apple-emoji-linux/releases/download/ios-15.4/AppleColorEmoji.ttf" -o /usr/local/share/fonts/TTF/AppleColorEmoji.ttf + cat >> /usr/local/etc/fonts/conf.d/01-emoji.conf < + + + +sans-serif + +Apple Color Emoji + + + +serif + +Apple Color Emoji + + + +Apple Color Emoji + +Apple Color Emoji + + + +EOF + fi ;; Debian) @@ -1671,7 +1744,16 @@ if [ "${BITCOIN_MAINNET_ENABLE}" = ON -o "${BITCOIN_TESTNET_ENABLE}" = ON -o "${ osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/mainnet" echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Bitcoin Mainnet" - osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/mainnet && git checkout ${MEMPOOL_LATEST_RELEASE}" +fi + +if [ "${LN_BITCOIN_MAINNET_ENABLE}" = ON ];then + echo "[*] Creating Mempool instance for Lightning Network on Bitcoin Mainnet" + osSudo "${MEMPOOL_USER}" git config --global advice.detachedHead false + osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/mainnet-lightning" + + echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Lightning Network on Bitcoin Mainnet" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/mainnet-lightning && git checkout ${MEMPOOL_LATEST_RELEASE}" fi if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then @@ -1680,7 +1762,16 @@ if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/testnet" echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Bitcoin Testnet" - osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/testnet && git checkout ${MEMPOOL_LATEST_RELEASE}" +fi + +if [ "${LN_BITCOIN_TESTNET_ENABLE}" = ON ];then + echo "[*] Creating Mempool instance for Lightning Network on Bitcoin Testnet" + osSudo "${MEMPOOL_USER}" git config --global advice.detachedHead false + osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/testnet-lightning" + + echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Lightning Network on Bitcoin Testnet" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/testnet-lightning && git checkout ${MEMPOOL_LATEST_RELEASE}" fi if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then @@ -1689,7 +1780,16 @@ if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/signet" echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Bitcoin Signet" - osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/signet && git checkout ${MEMPOOL_LATEST_RELEASE}" +fi + +if [ "${LN_BITCOIN_SIGNET_ENABLE}" = ON ];then + echo "[*] Creating Mempool instance for Lightning Network on Bitcoin Signet" + osSudo "${MEMPOOL_USER}" git config --global advice.detachedHead false + osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/signet-lightning" + + echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Lightning Network on Bitcoin Signet" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/signet-lightning && git checkout ${MEMPOOL_LATEST_RELEASE}" fi if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then @@ -1698,7 +1798,7 @@ if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/liquid" echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Liquid" - osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/liquid && git checkout ${MEMPOOL_LATEST_RELEASE}" fi if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then @@ -1707,7 +1807,7 @@ if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/liquidtestnet" echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Liquid Testnet" - osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/liquidtestnet && git checkout ${MEMPOOL_LATEST_RELEASE}" fi if [ "${BISQ_INSTALL}" = ON ];then @@ -1716,7 +1816,7 @@ if [ "${BISQ_INSTALL}" = ON ];then osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/bisq" echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Bisq" - osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/bisq && git checkout ${MEMPOOL_LATEST_RELEASE}" fi ##### mariadb @@ -1742,6 +1842,15 @@ grant all on mempool_testnet.* to '${MEMPOOL_TESTNET_USER}'@'localhost' identifi create database mempool_signet; grant all on mempool_signet.* to '${MEMPOOL_SIGNET_USER}'@'localhost' identified by '${MEMPOOL_SIGNET_PASS}'; +create database mempool_mainnet_lightning; +grant all on mempool_mainnet_lightning.* to '${LN_MEMPOOL_MAINNET_USER}'@'%' identified by '${LN_MEMPOOL_MAINNET_PASS}'; + +create database mempool_testnet_lightning; +grant all on mempool_testnet_lightning.* to '${LN_MEMPOOL_TESTNET_USER}'@'%' identified by '${LN_MEMPOOL_TESTNET_PASS}'; + +create database mempool_signet_lightning; +grant all on mempool_signet_lightning.* to '${LN_MEMPOOL_SIGNET_USER}'@'%' identified by '${LN_MEMPOOL_SIGNET_PASS}'; + create database mempool_liquid; grant all on mempool_liquid.* to '${MEMPOOL_LIQUID_USER}'@'localhost' identified by '${MEMPOOL_LIQUID_PASS}'; @@ -1760,6 +1869,12 @@ declare -x MEMPOOL_TESTNET_USER="${MEMPOOL_TESTNET_USER}" declare -x MEMPOOL_TESTNET_PASS="${MEMPOOL_TESTNET_PASS}" declare -x MEMPOOL_SIGNET_USER="${MEMPOOL_SIGNET_USER}" declare -x MEMPOOL_SIGNET_PASS="${MEMPOOL_SIGNET_PASS}" +declare -x LN_MEMPOOL_MAINNET_USER="${LN_MEMPOOL_MAINNET_USER}" +declare -x LN_MEMPOOL_MAINNET_PASS="${LN_MEMPOOL_MAINNET_PASS}" +declare -x LN_MEMPOOL_TESTNET_USER="${LN_MEMPOOL_TESTNET_USER}" +declare -x LN_MEMPOOL_TESTNET_PASS="${LN_MEMPOOL_TESTNET_PASS}" +declare -x LN_MEMPOOL_SIGNET_USER="${LN_MEMPOOL_SIGNET_USER}" +declare -x LN_MEMPOOL_SIGNET_PASS="${LN_MEMPOOL_SIGNET_PASS}" declare -x MEMPOOL_LIQUID_USER="${MEMPOOL_LIQUID_USER}" declare -x MEMPOOL_LIQUID_PASS="${MEMPOOL_LIQUID_PASS}" declare -x MEMPOOL_LIQUIDTESTNET_USER="${MEMPOOL_LIQUIDTESTNET_USER}" @@ -1770,24 +1885,32 @@ _EOF_ ##### nginx -echo "[*] Adding Nginx configuration" -osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/nginx/nginx.conf" "${NGINX_CONFIGURATION}" -mkdir -p /var/cache/nginx/services /var/cache/nginx/api -chown ${NGINX_USER}: /var/cache/nginx/services /var/cache/nginx/api -ln -s /mempool/mempool /etc/nginx/mempool -osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_USER__!${NGINX_USER}!" "${NGINX_CONFIGURATION}" -osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_ETC_FOLDER__!${NGINX_ETC_FOLDER}!" "${NGINX_CONFIGURATION}" -if [ "${TOR_INSTALL}" = ON ];then -echo "[*] Read tor v3 onion hostnames" - NGINX_MEMPOOL_ONION=$(cat "${TOR_RESOURCES}/mempool/hostname") - NGINX_BISQ_ONION=$(cat "${TOR_RESOURCES}/bisq/hostname") - NGINX_LIQUID_ONION=$(cat "${TOR_RESOURCES}/liquid/hostname") - osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_MEMPOOL_ONION__!${NGINX_MEMPOOL_ONION%.onion}!" "${NGINX_CONFIGURATION}" - osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_BISQ_ONION__!${NGINX_BISQ_ONION%.onion}!" "${NGINX_CONFIGURATION}" - osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_LIQUID_ONION__!${NGINX_LIQUID_ONIONi%.onion}!" "${NGINX_CONFIGURATION}" -fi -echo "[*] Restarting Nginx" -osSudo "${ROOT_USER}" service nginx restart +case $OS in + + FreeBSD) + ;; + +Debian) + echo "[*] Adding Nginx configuration" + osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/nginx/nginx.conf" "${NGINX_CONFIGURATION}" + mkdir -p /var/cache/nginx/services /var/cache/nginx/api + chown ${NGINX_USER}: /var/cache/nginx/services /var/cache/nginx/api + ln -s /mempool/mempool /etc/nginx/mempool + osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_USER__!${NGINX_USER}!" "${NGINX_CONFIGURATION}" + osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_ETC_FOLDER__!${NGINX_ETC_FOLDER}!" "${NGINX_CONFIGURATION}" + if [ "${TOR_INSTALL}" = ON ];then + echo "[*] Read tor v3 onion hostnames" + NGINX_MEMPOOL_ONION=$(cat "${TOR_RESOURCES}/mempool/hostname") + NGINX_BISQ_ONION=$(cat "${TOR_RESOURCES}/bisq/hostname") + NGINX_LIQUID_ONION=$(cat "${TOR_RESOURCES}/liquid/hostname") + osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_MEMPOOL_ONION__!${NGINX_MEMPOOL_ONION%.onion}!" "${NGINX_CONFIGURATION}" + osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_BISQ_ONION__!${NGINX_BISQ_ONION%.onion}!" "${NGINX_CONFIGURATION}" + osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_LIQUID_ONION__!${NGINX_LIQUID_ONIONi%.onion}!" "${NGINX_CONFIGURATION}" + fi + echo "[*] Restarting Nginx" + osSudo "${ROOT_USER}" service nginx restart + ;; +esac ##### OS systemd diff --git a/production/mempool-build-all b/production/mempool-build-all index 6a4e0c028..048aeefdc 100755 --- a/production/mempool-build-all +++ b/production/mempool-build-all @@ -98,6 +98,12 @@ build_backend() -e "s!__MEMPOOL_TESTNET_PASS__!${MEMPOOL_TESTNET_PASS}!" \ -e "s!__MEMPOOL_SIGNET_USER__!${MEMPOOL_SIGNET_USER}!" \ -e "s!__MEMPOOL_SIGNET_PASS__!${MEMPOOL_SIGNET_PASS}!" \ + -e "s!__LN_MEMPOOL_MAINNET_USER__!${LN_MEMPOOL_MAINNET_USER}!" \ + -e "s!__LN_MEMPOOL_MAINNET_PASS__!${LN_MEMPOOL_MAINNET_PASS}!" \ + -e "s!__LN_MEMPOOL_TESTNET_USER__!${LN_MEMPOOL_TESTNET_USER}!" \ + -e "s!__LN_MEMPOOL_TESTNET_PASS__!${LN_MEMPOOL_TESTNET_PASS}!" \ + -e "s!__LN_MEMPOOL_SIGNET_USER__!${LN_MEMPOOL_SIGNET_USER}!" \ + -e "s!__LN_MEMPOOL_SIGNET_PASS__!${LN_MEMPOOL_SIGNET_PASS}!" \ -e "s!__MEMPOOL_LIQUID_USER__!${MEMPOOL_LIQUID_USER}!" \ -e "s!__MEMPOOL_LIQUID_PASS__!${MEMPOOL_LIQUID_PASS}!" \ -e "s!__MEMPOOL_LIQUIDTESTNET_USER__!${LIQUIDTESTNET_USER}!" \ @@ -145,7 +151,7 @@ for repo in $backend_repos;do done # build unfurlers -for repo in mainnet liquid;do +for repo in mainnet liquid bisq;do build_unfurler "${repo}" done diff --git a/production/mempool-start-all b/production/mempool-start-all index 9751481ad..13fd30430 100755 --- a/production/mempool-start-all +++ b/production/mempool-start-all @@ -9,18 +9,20 @@ for site in mainnet mainnet-lightning testnet testnet-lightning signet signet-li screen -dmS "${site}" sh -c 'while true;do npm run start-production;sleep 1;done' done -# only start unfurler if GPU present +# only start xorg if GPU present if pciconf -lv|grep -i nvidia >/dev/null 2>&1;then export DISPLAY=:0 screen -dmS x startx sleep 3 - for site in mainnet liquid;do - cd "$HOME/${site}/unfurler" && \ - echo "starting mempool unfurler: ${site}" && \ - screen -dmS "unfurler-${site}" sh -c 'while true;do npm run unfurler;sleep 2;done' - done fi +# start unfurlers for each frontend +for site in mainnet liquid bisq;do + cd "$HOME/${site}/unfurler" && \ + echo "starting mempool unfurler: ${site}" && \ + screen -dmS "unfurler-${site}" sh -c 'while true;do npm run unfurler;sleep 2;done' +done + # start nginx warm cacher for site in mainnet;do echo "starting mempool cache warmer: ${site}" diff --git a/production/mempool.crontab b/production/mempool.crontab index 08639362f..cc1bcd878 100644 --- a/production/mempool.crontab +++ b/production/mempool.crontab @@ -1,9 +1,6 @@ # start on reboot @reboot sleep 10 ; $HOME/start -# start cache warmer on reboot -@reboot sleep 180 ; /mempool/mempool/production/nginx-cache-warmer >/dev/null 2>&1 & - # daily backup 37 13 * * * sleep 30 ; /mempool/mempool.space/backup >/dev/null 2>&1 & diff --git a/production/unfurler-config.bisq.json b/production/unfurler-config.bisq.json new file mode 100644 index 000000000..9742743f0 --- /dev/null +++ b/production/unfurler-config.bisq.json @@ -0,0 +1,17 @@ +{ + "SERVER": { + "HOST": "https://bisq.fra.mempool.space", + "HTTP_PORT": 8002 + }, + "MEMPOOL": { + "HTTP_HOST": "http://127.0.0.1", + "HTTP_PORT": 82, + "NETWORK": "bisq" + }, + "PUPPETEER": { + "CLUSTER_SIZE": 8, + "EXEC_PATH": "/usr/local/bin/chrome", + "MAX_PAGE_AGE": 86400, + "RENDER_TIMEOUT": 3000 + } +} diff --git a/production/unfurler-config.liquid.json b/production/unfurler-config.liquid.json index 39df7e3fd..243af6ed3 100644 --- a/production/unfurler-config.liquid.json +++ b/production/unfurler-config.liquid.json @@ -1,12 +1,12 @@ { "SERVER": { - "HOST": "https://liquid.network", - "HTTP_PORT": 8002 + "HOST": "https://liquid.fra.mempool.space", + "HTTP_PORT": 8003 }, "MEMPOOL": { - "HTTP_HOST": "https://liquid.network", - "HTTP_PORT": 443, - "NETWORK": "liquid" + "HTTP_HOST": "http://127.0.0.1", + "HTTP_PORT": 83, + "NETWORK": "bitcoin" }, "PUPPETEER": { "CLUSTER_SIZE": 8, diff --git a/production/unfurler-config.mainnet.json b/production/unfurler-config.mainnet.json index 752cd5706..77df23704 100644 --- a/production/unfurler-config.mainnet.json +++ b/production/unfurler-config.mainnet.json @@ -1,11 +1,11 @@ { "SERVER": { - "HOST": "https://mempool.space", + "HOST": "https://mempool.fra.mempool.space", "HTTP_PORT": 8001 }, "MEMPOOL": { - "HTTP_HOST": "https://mempool.space", - "HTTP_PORT": 443, + "HTTP_HOST": "http://127.0.0.1", + "HTTP_PORT": 81, "NETWORK": "bitcoin" }, "PUPPETEER": { diff --git a/unfurler/config.sample.json b/unfurler/config.sample.json index e080ee68a..64f56c1f7 100644 --- a/unfurler/config.sample.json +++ b/unfurler/config.sample.json @@ -9,6 +9,7 @@ "NETWORK": "bitcoin" // "bitcoin" | "liquid" | "bisq" (optional - defaults to "bitcoin") }, "PUPPETEER": { + "DISABLE": false, // optional, boolean, disables puppeteer and /render endpoints "CLUSTER_SIZE": 2, "EXEC_PATH": "/usr/local/bin/chrome", // optional "MAX_PAGE_AGE": 86400, // maximum lifetime of a page session (in seconds) diff --git a/unfurler/package-lock.json b/unfurler/package-lock.json index 3da33c69f..40520d413 100644 --- a/unfurler/package-lock.json +++ b/unfurler/package-lock.json @@ -1,12 +1,12 @@ { "name": "mempool-unfurl", - "version": "0.0.1", + "version": "0.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "mempool-unfurl", - "version": "0.0.1", + "version": "0.1.0", "dependencies": { "@types/node": "^16.11.41", "express": "^4.18.0", diff --git a/unfurler/package.json b/unfurler/package.json index ca60201b3..59d48aa50 100644 --- a/unfurler/package.json +++ b/unfurler/package.json @@ -1,6 +1,6 @@ { "name": "mempool-unfurl", - "version": "0.0.2", + "version": "0.1.0", "description": "Renderer for mempool open graph link preview images", "repository": { "type": "git", diff --git a/unfurler/src/config.ts b/unfurler/src/config.ts index a65d48f6f..3c4a4e422 100644 --- a/unfurler/src/config.ts +++ b/unfurler/src/config.ts @@ -11,6 +11,7 @@ interface IConfig { NETWORK?: string; }; PUPPETEER: { + DISABLE: boolean; CLUSTER_SIZE: number; EXEC_PATH?: string; MAX_PAGE_AGE?: number; @@ -28,6 +29,7 @@ const defaults: IConfig = { 'HTTP_PORT': 4200, }, 'PUPPETEER': { + 'DISABLE': false, 'CLUSTER_SIZE': 1, }, }; diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 08dff3964..eab4723b7 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -1,10 +1,12 @@ import express from "express"; import { Application, Request, Response, NextFunction } from 'express'; import * as http from 'http'; +import * as https from 'https'; import config from './config'; import { Cluster } from 'puppeteer-cluster'; import ReusablePage from './concurrency/ReusablePage'; import { parseLanguageUrl } from './language/lang'; +import { matchRoute } from './routes'; const puppeteerConfig = require('../puppeteer.config.json'); if (config.PUPPETEER.EXEC_PATH) { @@ -17,13 +19,13 @@ class Server { cluster?: Cluster; mempoolHost: string; network: string; - defaultImageUrl: string; + secureHost = true; constructor() { this.app = express(); this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : ''); + this.secureHost = this.mempoolHost.startsWith('https'); this.network = config.MEMPOOL.NETWORK || 'bitcoin'; - this.defaultImageUrl = this.getDefaultImageUrl(); this.startServer(); } @@ -37,12 +39,14 @@ class Server { .use(express.text()) ; - this.cluster = await Cluster.launch({ - concurrency: ReusablePage, - maxConcurrency: config.PUPPETEER.CLUSTER_SIZE, - puppeteerOptions: puppeteerConfig, - }); - await this.cluster?.task(async (args) => { return this.clusterTask(args) }); + if (!config.PUPPETEER.DISABLE) { + this.cluster = await Cluster.launch({ + concurrency: ReusablePage, + maxConcurrency: config.PUPPETEER.CLUSTER_SIZE, + puppeteerOptions: puppeteerConfig, + }); + await this.cluster?.task(async (args) => { return this.clusterTask(args) }); + } this.setUpRoutes(); @@ -64,7 +68,11 @@ class Server { } setUpRoutes() { - this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) }) + if (!config.PUPPETEER.DISABLE) { + this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) }) + } else { + this.app.get('/render*', async (req, res) => { return this.renderDisabled(req, res) }) + } this.app.get('*', (req, res) => { return this.renderHTML(req, res) }) } @@ -111,13 +119,31 @@ class Server { } } + async renderDisabled(req, res) { + res.status(500).send("preview rendering disabled"); + } + async renderPreview(req, res) { try { - const path = req.params[0] - const img = await this.cluster?.execute({ url: this.mempoolHost + path, path: path, action: 'screenshot' }); + const rawPath = req.params[0]; + + let img = null; + + const { lang, path } = parseLanguageUrl(rawPath); + const matchedRoute = matchRoute(this.network, path); + + // don't bother unless the route is definitely renderable + if (rawPath.includes('/preview/') && matchedRoute.render) { + img = await this.cluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'screenshot' }); + } if (!img) { - res.status(500).send('failed to render page preview'); + // proxy fallback image from the frontend + if (this.secureHost) { + https.get(config.SERVER.HOST + matchedRoute.fallbackImg, (got) => got.pipe(res)); + } else { + http.get(config.SERVER.HOST + matchedRoute.fallbackImg, (got) => got.pipe(res)); + } } else { res.contentType('image/png'); res.send(img); @@ -137,50 +163,14 @@ class Server { return; } - let previewSupported = true; - let mode = 'mainnet' - let ogImageUrl = this.defaultImageUrl; - let ogTitle; const { lang, path } = parseLanguageUrl(rawPath); - const parts = path.slice(1).split('/'); + const matchedRoute = matchRoute(this.network, path); + let ogImageUrl = config.SERVER.HOST + (matchedRoute.staticImg || matchedRoute.fallbackImg); + let ogTitle = 'The Mempool Open Source Project™'; - // handle network mode modifiers - if (['testnet', 'signet'].includes(parts[0])) { - mode = parts.shift(); - } - - // handle supported preview routes - switch (parts[0]) { - case 'block': - ogTitle = `Block: ${parts[1]}`; - break; - case 'address': - ogTitle = `Address: ${parts[1]}`; - break; - case 'tx': - ogTitle = `Transaction: ${parts[1]}`; - break; - case 'lightning': - switch (parts[1]) { - case 'node': - ogTitle = `Lightning Node: ${parts[2]}`; - break; - case 'channel': - ogTitle = `Lightning Channel: ${parts[2]}`; - break; - default: - previewSupported = false; - } - break; - default: - previewSupported = false; - } - - if (previewSupported) { + if (matchedRoute.render) { ogImageUrl = `${config.SERVER.HOST}/render/${lang || 'en'}/preview${path}`; - ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${mode !== 'mainnet' ? capitalize(mode) + ' ' : ''}${ogTitle}`; - } else { - ogTitle = 'The Mempool Open Source Project™'; + ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${matchedRoute.networkMode !== 'mainnet' ? capitalize(matchedRoute.networkMode) + ' ' : ''}${matchedRoute.title}`; } res.send(` @@ -189,34 +179,23 @@ class Server { ${ogTitle} - + - - + + - + `); } - - getDefaultImageUrl() { - switch (this.network) { - case 'liquid': - return this.mempoolHost + '/resources/liquid/liquid-network-preview.png'; - case 'bisq': - return this.mempoolHost + '/resources/bisq/bisq-markets-preview.png'; - default: - return this.mempoolHost + '/resources/mempool-space-preview.png'; - } - } } const server = new Server(); diff --git a/unfurler/src/routes.ts b/unfurler/src/routes.ts new file mode 100644 index 000000000..24922c85d --- /dev/null +++ b/unfurler/src/routes.ts @@ -0,0 +1,124 @@ +interface Match { + render: boolean; + title: string; + fallbackImg: string; + staticImg?: string; + networkMode: string; +} + +const routes = { + block: { + render: true, + params: 1, + getTitle(path) { + return `Block: ${path[0]}`; + } + }, + address: { + render: true, + params: 1, + getTitle(path) { + return `Address: ${path[0]}`; + } + }, + tx: { + render: true, + params: 1, + getTitle(path) { + return `Transaction: ${path[0]}`; + } + }, + lightning: { + title: "Lightning", + fallbackImg: '/resources/previews/lightning.png', + routes: { + node: { + render: true, + params: 1, + getTitle(path) { + return `Lightning Node: ${path[0]}`; + } + }, + channel: { + render: true, + params: 1, + getTitle(path) { + return `Lightning Channel: ${path[0]}`; + } + }, + } + }, + mining: { + title: "Mining", + fallbackImg: '/resources/previews/mining.png' + } +}; + +const networks = { + bitcoin: { + fallbackImg: '/resources/mempool-space-preview.png', + staticImg: '/resources/previews/dashboard.png', + routes: { + ...routes // all routes supported + } + }, + liquid: { + fallbackImg: '/resources/liquid/liquid-network-preview.png', + routes: { // only block, address & tx routes supported + block: routes.block, + address: routes.address, + tx: routes.tx + } + }, + bisq: { + fallbackImg: '/resources/bisq/bisq-markets-preview.png', + routes: {} // no routes supported + } +}; + +export function matchRoute(network: string, path: string): Match { + const match: Match = { + render: false, + title: '', + fallbackImg: '', + networkMode: 'mainnet' + } + + const parts = path.slice(1).split('/').filter(p => p.length); + + if (parts[0] === 'preview') { + parts.shift(); + } + if (['testnet', 'signet'].includes(parts[0])) { + match.networkMode = parts.shift() || 'mainnet'; + } + + let route = networks[network] || networks.bitcoin; + match.fallbackImg = route.fallbackImg; + + // traverse the route tree until we run out of route or tree, or hit a renderable match + while (!route.render && route.routes && parts.length && route.routes[parts[0]]) { + route = route.routes[parts[0]]; + parts.shift(); + if (route.fallbackImg) { + match.fallbackImg = route.fallbackImg; + } + } + + // enough route parts left for title & rendering + if (route.render && parts.length >= route.params) { + match.render = true; + } + // only use set a static image for exact matches + if (!parts.length && route.staticImg) { + match.staticImg = route.staticImg; + } + // apply the title function if present + if (route.getTitle && typeof route.getTitle === 'function') { + match.title = route.getTitle(parts); + } else { + match.title = route.title; + } + + return match; +} \ No newline at end of file