diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index 66bcb2569..2c3fd9467 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -510,7 +510,12 @@ class BitcoinRoutes { private getDifficultyChange(req: Request, res: Response) { try { - res.json(difficultyAdjustment.getDifficultyAdjustment()); + const da = difficultyAdjustment.getDifficultyAdjustment(); + if (da) { + res.json(da); + } else { + res.status(503).send(`Service Temporarily Unavailable`); + } } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); } diff --git a/backend/src/api/difficulty-adjustment.ts b/backend/src/api/difficulty-adjustment.ts index 43c6e463f..a1b6ab70e 100644 --- a/backend/src/api/difficulty-adjustment.ts +++ b/backend/src/api/difficulty-adjustment.ts @@ -81,14 +81,15 @@ export function calcDifficultyAdjustment( } class DifficultyAdjustmentApi { - constructor() { } - - public getDifficultyAdjustment(): IDifficultyAdjustment { + public getDifficultyAdjustment(): IDifficultyAdjustment | null { const DATime = blocks.getLastDifficultyAdjustmentTime(); const previousRetarget = blocks.getPreviousDifficultyRetarget(); const blockHeight = blocks.getCurrentBlockHeight(); const blocksCache = blocks.getBlocks(); const latestBlock = blocksCache[blocksCache.length - 1]; + if (!latestBlock) { + return null; + } const nowSeconds = Math.floor(new Date().getTime() / 1000); return calcDifficultyAdjustment( diff --git a/backend/src/api/explorer/statistics.api.ts b/backend/src/api/explorer/statistics.api.ts index 7bf3d9107..558ee86fd 100644 --- a/backend/src/api/explorer/statistics.api.ts +++ b/backend/src/api/explorer/statistics.api.ts @@ -27,7 +27,7 @@ class StatisticsApi { public async $getLatestStatistics(): Promise { try { const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1`); - const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1 OFFSET 7`); + const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats WHERE DATE(added) = DATE(NOW() - INTERVAL 7 DAY)`); return { latest: rows[0], previous: rows2[0], diff --git a/backend/src/tasks/lightning/sync-tasks/node-locations.ts b/backend/src/tasks/lightning/sync-tasks/node-locations.ts index 30a6bfc2a..9069e0fff 100644 --- a/backend/src/tasks/lightning/sync-tasks/node-locations.ts +++ b/backend/src/tasks/lightning/sync-tasks/node-locations.ts @@ -4,6 +4,7 @@ import nodesApi from '../../../api/explorer/nodes.api'; import config from '../../../config'; import DB from '../../../database'; import logger from '../../../logger'; +import * as IPCheck from '../../../utils/ipcheck.js'; export async function $lookupNodeLocation(): Promise { let loggerTimer = new Date().getTime() / 1000; @@ -27,6 +28,26 @@ export async function $lookupNodeLocation(): Promise { const asn = lookupAsn.get(ip); const isp = lookupIsp.get(ip); + let asOverwrite: any | undefined; + if (asn && (IPCheck.match(ip, '170.75.160.0/20') || IPCheck.match(ip, '172.81.176.0/21'))) { + asOverwrite = { + asn: 394745, + name: 'Lunanode', + }; + } + else if (asn && (IPCheck.match(ip, '50.7.0.0/16') || IPCheck.match(ip, '66.90.64.0/18'))) { + asOverwrite = { + asn: 30058, + name: 'FDCservers.net', + }; + } + else if (asn && asn.autonomous_system_number === 174) { + asOverwrite = { + asn: 174, + name: 'Cogent Communications', + }; + } + if (city && (asn || isp)) { const query = ` UPDATE nodes SET @@ -41,7 +62,7 @@ export async function $lookupNodeLocation(): Promise { `; const params = [ - isp?.autonomous_system_number ?? asn?.autonomous_system_number, + asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number, city.city?.geoname_id, city.country?.geoname_id, city.subdivisions ? city.subdivisions[0].geoname_id : null, @@ -91,7 +112,10 @@ export async function $lookupNodeLocation(): Promise { if (isp?.autonomous_system_organization ?? asn?.autonomous_system_organization) { await DB.query( `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'as_organization', ?)`, - [isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]); + [ + asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number, + JSON.stringify(asOverwrite?.name ?? isp?.isp ?? asn?.autonomous_system_organization) + ]); } } diff --git a/backend/src/utils/ipcheck.js b/backend/src/utils/ipcheck.js new file mode 100644 index 000000000..06d4a6f15 --- /dev/null +++ b/backend/src/utils/ipcheck.js @@ -0,0 +1,119 @@ +var net = require('net'); + +var IPCheck = module.exports = function(input) { + var self = this; + + if (!(self instanceof IPCheck)) { + return new IPCheck(input); + } + + self.input = input; + self.parse(); +}; + +IPCheck.prototype.parse = function() { + var self = this; + + if (!self.input || typeof self.input !== 'string') return self.valid = false; + + var ip; + + var pos = self.input.lastIndexOf('/'); + if (pos !== -1) { + ip = self.input.substring(0, pos); + self.mask = +self.input.substring(pos + 1); + } else { + ip = self.input; + self.mask = null; + } + + self.ipv = net.isIP(ip); + self.valid = !!self.ipv && !isNaN(self.mask); + + if (!self.valid) return; + + // default mask = 32 for ipv4 and 128 for ipv6 + if (self.mask === null) self.mask = self.ipv === 4 ? 32 : 128; + + if (self.ipv === 4) { + // difference between ipv4 and ipv6 masks + self.mask += 96; + } + + if (self.mask < 0 || self.mask > 128) { + self.valid = false; + return; + } + + self.address = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]; + + if(self.ipv === 4){ + self.parseIPv4(ip); + }else{ + self.parseIPv6(ip); + } +}; + +IPCheck.prototype.parseIPv4 = function(ip) { + var self = this; + + // ipv4 addresses live under ::ffff:0:0 + self.address[10] = self.address[11] = 0xff; + + var octets = ip.split('.'); + for (var i = 0; i < 4; i++) { + self.address[i + 12] = parseInt(octets[i], 10); + } +}; + + +var V6_TRANSITIONAL = /:(\d+\.\d+\.\d+\.\d+)$/; + +IPCheck.prototype.parseIPv6 = function(ip) { + var self = this; + + var transitionalMatch = V6_TRANSITIONAL.exec(ip); + if(transitionalMatch){ + self.parseIPv4(transitionalMatch[1]); + return; + } + + var bits = ip.split(':'); + if (bits.length < 8) { + ip = ip.replace('::', Array(11 - bits.length).join(':')); + bits = ip.split(':'); + } + + var j = 0; + for (var i = 0; i < bits.length; i += 1) { + var x = bits[i] ? parseInt(bits[i], 16) : 0; + self.address[j++] = x >> 8; + self.address[j++] = x & 0xff; + } +}; + +IPCheck.prototype.match = function(cidr) { + var self = this; + + if (!(cidr instanceof IPCheck)) cidr = new IPCheck(cidr); + if (!self.valid || !cidr.valid) return false; + + var mask = cidr.mask; + var i = 0; + + while (mask >= 8) { + if (self.address[i] !== cidr.address[i]) return false; + + i++; + mask -= 8; + } + + var shift = 8 - mask; + return (self.address[i] >>> shift) === (cidr.address[i] >>> shift); +}; + + +IPCheck.match = function(ip, cidr) { + ip = ip instanceof IPCheck ? ip : new IPCheck(ip); + return ip.match(cidr); +}; diff --git a/backend/tsconfig.json b/backend/tsconfig.json index 6a1970331..0670010e1 100644 --- a/backend/tsconfig.json +++ b/backend/tsconfig.json @@ -13,7 +13,8 @@ "node_modules/@types" ], "allowSyntheticDefaultImports": true, - "esModuleInterop": true + "esModuleInterop": true, + "allowJs": true, }, "include": [ "src/**/*.ts" diff --git a/frontend/src/app/lightning/channels-statistics/channels-statistics.component.html b/frontend/src/app/lightning/channels-statistics/channels-statistics.component.html index d0ecea3a2..60a43216c 100644 --- a/frontend/src/app/lightning/channels-statistics/channels-statistics.component.html +++ b/frontend/src/app/lightning/channels-statistics/channels-statistics.component.html @@ -9,44 +9,44 @@
-
+
Avg Capacity
-
+
{{ statistics.latest?.avg_capacity || 0 | number: '1.0-0' }} sats
- +
-
+
Avg Fee Rate
-
+
{{ statistics.latest?.avg_fee_rate || 0 | number: '1.0-0' }} ppm
- +
-
+
Avg Base Fee
-
+
{{ statistics.latest?.avg_base_fee_mtokens || 0 | number: '1.0-0' }} msats
- +
@@ -55,43 +55,45 @@
-
+
Med Capacity
-
+
{{ statistics.latest?.med_capacity || 0 | number: '1.0-0' }} sats
- +
-
+ +
Med Fee Rate
-
+
{{ statistics.latest?.med_fee_rate || 0 | number: '1.0-0' }} ppm
- +
-
+ +
Med Base Fee
-
+
{{ statistics.latest?.med_base_fee_mtokens || 0 | number: '1.0-0' }} msats
- +
diff --git a/frontend/src/app/lightning/channels-statistics/channels-statistics.component.scss b/frontend/src/app/lightning/channels-statistics/channels-statistics.component.scss index 372d9eb78..e97f0d0af 100644 --- a/frontend/src/app/lightning/channels-statistics/channels-statistics.component.scss +++ b/frontend/src/app/lightning/channels-statistics/channels-statistics.component.scss @@ -18,6 +18,10 @@ } } +.fee-estimation-wrapper { + min-height: 77px; +} + .fee-estimation-container { display: flex; justify-content: space-between; @@ -30,7 +34,10 @@ width: -webkit-fill-available; @media (min-width: 376px) { margin: 0 auto 0px; - } + } + &.more-padding { + padding-top: 10px; + } &:first-child{ display: none; @media (min-width: 485px) { @@ -57,6 +64,9 @@ margin: auto; line-height: 1.45; padding: 0px 2px; + &.no-border { + border-bottom: none; + } } .fiat { display: block; diff --git a/frontend/src/app/lightning/node-statistics/node-statistics.component.html b/frontend/src/app/lightning/node-statistics/node-statistics.component.html index 152ee2dbe..ae4ea3dd7 100644 --- a/frontend/src/app/lightning/node-statistics/node-statistics.component.html +++ b/frontend/src/app/lightning/node-statistics/node-statistics.component.html @@ -1,76 +1,64 @@
-
-
Capacity
-
-
+
+
Capacity
+
+
- +
-
-
Nodes
-
-
+
+
Nodes
+
+
{{ statistics.latest?.node_count || 0 | number }}
- +
-
-
Channels
-
-
+
+
Channels
+
+
{{ statistics.latest?.channel_count || 0 | number }}
- +
-
-
Nodes
+
Nodes
-
Channels
+
Channels
-
Average Channel
+
Average Channel
diff --git a/frontend/src/app/lightning/node-statistics/node-statistics.component.scss b/frontend/src/app/lightning/node-statistics/node-statistics.component.scss index acc4578f3..1532f9c4b 100644 --- a/frontend/src/app/lightning/node-statistics/node-statistics.component.scss +++ b/frontend/src/app/lightning/node-statistics/node-statistics.component.scss @@ -18,6 +18,10 @@ } } +.fee-estimation-wrapper { + min-height: 77px; +} + .fee-estimation-container { display: flex; justify-content: space-between; @@ -30,7 +34,10 @@ width: -webkit-fill-available; @media (min-width: 376px) { margin: 0 auto 0px; - } + } + &.more-padding { + padding-top: 10px; + } &:first-child{ display: none; @media (min-width: 485px) { @@ -57,6 +64,9 @@ margin: auto; line-height: 1.45; padding: 0px 2px; + &.no-border { + border-bottom: none; + } } .fiat { display: block; diff --git a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts index c6a2f8f5c..1f30e96f5 100644 --- a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts +++ b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts @@ -238,7 +238,7 @@ export class NodesChannelsMap implements OnInit { roam: this.style === 'widget' ? false : true, itemStyle: { borderColor: 'black', - color: '#ffffff44' + color: '#272b3f' }, scaleLimit: { min: 1.3, diff --git a/frontend/src/app/lightning/nodes-channels/node-channels.component.ts b/frontend/src/app/lightning/nodes-channels/node-channels.component.ts index 074315b35..f675c81b5 100644 --- a/frontend/src/app/lightning/nodes-channels/node-channels.component.ts +++ b/frontend/src/app/lightning/nodes-channels/node-channels.component.ts @@ -44,13 +44,13 @@ export class NodeChannels implements OnChanges { switchMap((response) => { this.isLoading = true; if ((response.body?.length ?? 0) <= 0) { - return []; + this.isLoading = false; + return ['']; } return [response.body]; }), tap((body: any[]) => { - if (body.length === 0) { - this.isLoading = false; + if (body.length === 0 || body[0].length === 0) { return; } const biggestCapacity = body[0].capacity; @@ -130,10 +130,6 @@ export class NodeChannels implements OnChanges { } onChartInit(ec: ECharts): void { - if (this.chartInstance !== undefined) { - return; - } - this.chartInstance = ec; this.chartInstance.on('click', (e) => { diff --git a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html index 25773a06e..ef1c71efd 100644 --- a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html +++ b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html @@ -28,7 +28,7 @@
- Top 100 ISP hosting LN nodes + Top 100 ISPs hosting LN nodes diff --git a/frontend/src/index.bisq.html b/frontend/src/index.bisq.html index b5c65e8d4..8da1e77e0 100644 --- a/frontend/src/index.bisq.html +++ b/frontend/src/index.bisq.html @@ -5,7 +5,7 @@ mempool - Bisq Markets - + @@ -14,7 +14,7 @@ - + diff --git a/frontend/src/index.liquid.html b/frontend/src/index.liquid.html index 65d397aac..89a6984ba 100644 --- a/frontend/src/index.liquid.html +++ b/frontend/src/index.liquid.html @@ -5,7 +5,7 @@ mempool - Liquid Network - + @@ -14,7 +14,7 @@ - + diff --git a/frontend/src/index.mempool.html b/frontend/src/index.mempool.html index 05c8d8398..1176a3da2 100644 --- a/frontend/src/index.mempool.html +++ b/frontend/src/index.mempool.html @@ -5,7 +5,7 @@ mempool - Bitcoin Explorer - + @@ -14,7 +14,7 @@ - + diff --git a/frontend/src/resources/bisq/bisq-markets-preview.png b/frontend/src/resources/bisq/bisq-markets-preview.png index 2fff8f99e..2b5e1250b 100644 Binary files a/frontend/src/resources/bisq/bisq-markets-preview.png and b/frontend/src/resources/bisq/bisq-markets-preview.png differ diff --git a/frontend/src/resources/liquid/liquid-network-preview.png b/frontend/src/resources/liquid/liquid-network-preview.png index 5a6ed9eb1..72942110c 100644 Binary files a/frontend/src/resources/liquid/liquid-network-preview.png and b/frontend/src/resources/liquid/liquid-network-preview.png differ diff --git a/frontend/src/resources/previews/dashboard.png b/frontend/src/resources/previews/dashboard.png new file mode 100644 index 000000000..e11588994 Binary files /dev/null and b/frontend/src/resources/previews/dashboard.png differ diff --git a/frontend/src/resources/previews/lightning.png b/frontend/src/resources/previews/lightning.png new file mode 100644 index 000000000..4686b0ef0 Binary files /dev/null and b/frontend/src/resources/previews/lightning.png differ diff --git a/frontend/src/resources/previews/mining.png b/frontend/src/resources/previews/mining.png new file mode 100644 index 000000000..6a2aa1b41 Binary files /dev/null and b/frontend/src/resources/previews/mining.png differ diff --git a/production/install b/production/install index 71bcb7f6b..4eac1817b 100755 --- a/production/install +++ b/production/install @@ -48,6 +48,9 @@ BITCOIN_MAINNET_ENABLE=ON BITCOIN_MAINNET_MINFEE_ENABLE=ON BITCOIN_TESTNET_ENABLE=ON BITCOIN_SIGNET_ENABLE=ON +LN_BITCOIN_MAINNET_ENABLE=ON +LN_BITCOIN_TESTNET_ENABLE=ON +LN_BITCOIN_SIGNET_ENABLE=ON BISQ_MAINNET_ENABLE=ON ELEMENTS_LIQUID_ENABLE=ON ELEMENTS_LIQUIDTESTNET_ENABLE=ON @@ -227,6 +230,9 @@ MYSQL_GROUP=mysql MEMPOOL_MAINNET_USER='mempool' MEMPOOL_TESTNET_USER='mempool_testnet' MEMPOOL_SIGNET_USER='mempool_signet' +LN_MEMPOOL_MAINNET_USER='mempool_mainnet_lightning' +LN_MEMPOOL_TESTNET_USER='mempool_testnet_lightning' +LN_MEMPOOL_SIGNET_USER='mempool_signet_lightning' MEMPOOL_LIQUID_USER='mempool_liquid' MEMPOOL_LIQUIDTESTNET_USER='mempool_liquidtestnet' MEMPOOL_BISQ_USER='mempool_bisq' @@ -234,6 +240,9 @@ MEMPOOL_BISQ_USER='mempool_bisq' MEMPOOL_MAINNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') MEMPOOL_TESTNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') MEMPOOL_SIGNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') +LN_MEMPOOL_MAINNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') +LN_MEMPOOL_TESTNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') +LN_MEMPOOL_SIGNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') MEMPOOL_LIQUID_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') MEMPOOL_LIQUIDTESTNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') MEMPOOL_BISQ_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}') @@ -391,6 +400,10 @@ FREEBSD_PKG+=(boost-libs autoconf automake gmake gcc libevent libtool pkgconf) FREEBSD_PKG+=(nginx rsync py39-certbot-nginx mariadb105-server keybase) FREEBSD_PKG+=(geoipupdate) +FREEBSD_UNFURL_PKG=() +FREEBSD_UNFURL_PKG+=(nvidia-driver-470-470.129.06 chromium xinit xterm twm ja-sourcehansans-otf) +FREEBSD_UNFURL_PKG+=(zh-sourcehansans-sc-otf ko-aleefonts-ttf lohit tlwg-ttf) + ############################# ##### utility functions ##### ############################# @@ -747,6 +760,9 @@ $CUT >$input <<-EOF Tor:Enable Tor v3 HS Onion:ON Mainnet:Enable Bitcoin Mainnet:ON Mainnet-Minfee:Enable Bitcoin Mainnet Minfee:ON +LN-Mainnet:Enable Bitcoin Mainnet Lightning:ON +LN-Testnet:Enable Bitcoin Testnet Lightning:ON +LN-Signet:Enable Bitcoin Signet Lightning:ON Testnet:Enable Bitcoin Testnet:ON Signet:Enable Bitcoin Signet:ON Liquid:Enable Elements Liquid:ON @@ -809,6 +825,24 @@ else BITCOIN_INSTALL=OFF fi +if grep LN-Mainnet $tempfile >/dev/null 2>&1;then + LN_BITCOIN_MAINNET_ENABLE=ON +else + LN_BITCOIN_MAINNET_ENABLE=OFF +fi + +if grep LN-Testnet $tempfile >/dev/null 2>&1;then + LN_BITCOIN_TESTNET_ENABLE=ON +else + LN_BITCOIN_TESTNET_ENABLE=OFF +fi + +if grep LN-Signet $tempfile >/dev/null 2>&1;then + LN_BITCOIN_SIGNET_ENABLE=ON +else + LN_BITCOIN_SIGNET_ENABLE=OFF +fi + if grep Liquid $tempfile >/dev/null 2>&1;then ELEMENTS_LIQUID_ENABLE=ON else @@ -831,6 +865,7 @@ if grep CoreLN $tempfile >/dev/null 2>&1;then CLN_INSTALL=ON else CLN_INSTALL=OFF +fi if [ "${BITCOIN_MAINNET_ENABLE}" = ON -o "${BITCOIN_TESTNET_ENABLE}" = ON -o "${BITCOIN_SIGNET_ENABLE}" = ON ];then BITCOIN_ELECTRS_INSTALL=ON @@ -1279,8 +1314,11 @@ case $OS in echo "[*] Creating Core Lightning user" osGroupCreate "${CLN_GROUP}" osUserCreate "${CLN_USER}" "${CLN_HOME}" "${CLN_GROUP}" + osSudo "${ROOT_USER}" pw usermod ${MEMPOOL_USER} -G "${CLN_GROUP}" osSudo "${ROOT_USER}" chsh -s `which zsh` "${CLN_USER}" echo "export PATH=$PATH:$HOME/.local/bin" >> "${CLN_HOME}/.zshrc" + osSudo "${ROOT_USER}" mkdir -p "${CLN_HOME}/.lightning/{bitcoin,signet,testnet}" + osSudo "${ROOT_USER}" chmod 750 "${CLN_HOME}" "${CLN_HOME}/.lightning" "${CLN_HOME}/.lightning/{bitcoin,signet,testnet}" osSudo "${ROOT_USER}" chown -R "${CLN_USER}:${CLN_GROUP}" "${CLN_HOME}" echo "[*] Installing Core Lightning package" @@ -1397,7 +1435,42 @@ if [ "${UNFURL_INSTALL}" = ON ];then case $OS in FreeBSD) - echo "[*] FIXME: Unfurl must be installed manually on FreeBSD" + + if pciconf -lv|grep -i nvidia >/dev/null 2>&1;then + echo "[*] GPU detected: Installing packages for Unfurl" + osPackageInstall ${FREEBSD_UNFURL_PKG[@]} + + echo 'allowed_users = anybody' >> /usr/local/etc/X11/Xwrapper.config + echo 'kld_list="nvidia"' >> /etc/rc.conf + echo 'nvidia_xorg_enable="YES"' >> /etc/rc.conf + + echo "[*] Installing color emoji" + osSudo "${ROOT_USER}" curl "https://github.com/samuelngs/apple-emoji-linux/releases/download/ios-15.4/AppleColorEmoji.ttf" -o /usr/local/share/fonts/TTF/AppleColorEmoji.ttf + cat >> /usr/local/etc/fonts/conf.d/01-emoji.conf < + + + +sans-serif + +Apple Color Emoji + + + +serif + +Apple Color Emoji + + + +Apple Color Emoji + +Apple Color Emoji + + + +EOF + fi ;; Debian) @@ -1671,7 +1744,16 @@ if [ "${BITCOIN_MAINNET_ENABLE}" = ON -o "${BITCOIN_TESTNET_ENABLE}" = ON -o "${ osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/mainnet" echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Bitcoin Mainnet" - osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/mainnet && git checkout ${MEMPOOL_LATEST_RELEASE}" +fi + +if [ "${LN_BITCOIN_MAINNET_ENABLE}" = ON ];then + echo "[*] Creating Mempool instance for Lightning Network on Bitcoin Mainnet" + osSudo "${MEMPOOL_USER}" git config --global advice.detachedHead false + osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/mainnet-lightning" + + echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Lightning Network on Bitcoin Mainnet" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/mainnet-lightning && git checkout ${MEMPOOL_LATEST_RELEASE}" fi if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then @@ -1680,7 +1762,16 @@ if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/testnet" echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Bitcoin Testnet" - osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/testnet && git checkout ${MEMPOOL_LATEST_RELEASE}" +fi + +if [ "${LN_BITCOIN_TESTNET_ENABLE}" = ON ];then + echo "[*] Creating Mempool instance for Lightning Network on Bitcoin Testnet" + osSudo "${MEMPOOL_USER}" git config --global advice.detachedHead false + osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/testnet-lightning" + + echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Lightning Network on Bitcoin Testnet" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/testnet-lightning && git checkout ${MEMPOOL_LATEST_RELEASE}" fi if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then @@ -1689,7 +1780,16 @@ if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/signet" echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Bitcoin Signet" - osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/signet && git checkout ${MEMPOOL_LATEST_RELEASE}" +fi + +if [ "${LN_BITCOIN_SIGNET_ENABLE}" = ON ];then + echo "[*] Creating Mempool instance for Lightning Network on Bitcoin Signet" + osSudo "${MEMPOOL_USER}" git config --global advice.detachedHead false + osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/signet-lightning" + + echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Lightning Network on Bitcoin Signet" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/signet-lightning && git checkout ${MEMPOOL_LATEST_RELEASE}" fi if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then @@ -1698,7 +1798,7 @@ if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/liquid" echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Liquid" - osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/liquid && git checkout ${MEMPOOL_LATEST_RELEASE}" fi if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then @@ -1707,7 +1807,7 @@ if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/liquidtestnet" echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Liquid Testnet" - osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/liquidtestnet && git checkout ${MEMPOOL_LATEST_RELEASE}" fi if [ "${BISQ_INSTALL}" = ON ];then @@ -1716,7 +1816,7 @@ if [ "${BISQ_INSTALL}" = ON ];then osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/bisq" echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Bisq" - osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}" + osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/bisq && git checkout ${MEMPOOL_LATEST_RELEASE}" fi ##### mariadb @@ -1742,6 +1842,15 @@ grant all on mempool_testnet.* to '${MEMPOOL_TESTNET_USER}'@'localhost' identifi create database mempool_signet; grant all on mempool_signet.* to '${MEMPOOL_SIGNET_USER}'@'localhost' identified by '${MEMPOOL_SIGNET_PASS}'; +create database mempool_mainnet_lightning; +grant all on mempool_mainnet_lightning.* to '${LN_MEMPOOL_MAINNET_USER}'@'%' identified by '${LN_MEMPOOL_MAINNET_PASS}'; + +create database mempool_testnet_lightning; +grant all on mempool_testnet_lightning.* to '${LN_MEMPOOL_TESTNET_USER}'@'%' identified by '${LN_MEMPOOL_TESTNET_PASS}'; + +create database mempool_signet_lightning; +grant all on mempool_signet_lightning.* to '${LN_MEMPOOL_SIGNET_USER}'@'%' identified by '${LN_MEMPOOL_SIGNET_PASS}'; + create database mempool_liquid; grant all on mempool_liquid.* to '${MEMPOOL_LIQUID_USER}'@'localhost' identified by '${MEMPOOL_LIQUID_PASS}'; @@ -1760,6 +1869,12 @@ declare -x MEMPOOL_TESTNET_USER="${MEMPOOL_TESTNET_USER}" declare -x MEMPOOL_TESTNET_PASS="${MEMPOOL_TESTNET_PASS}" declare -x MEMPOOL_SIGNET_USER="${MEMPOOL_SIGNET_USER}" declare -x MEMPOOL_SIGNET_PASS="${MEMPOOL_SIGNET_PASS}" +declare -x LN_MEMPOOL_MAINNET_USER="${LN_MEMPOOL_MAINNET_USER}" +declare -x LN_MEMPOOL_MAINNET_PASS="${LN_MEMPOOL_MAINNET_PASS}" +declare -x LN_MEMPOOL_TESTNET_USER="${LN_MEMPOOL_TESTNET_USER}" +declare -x LN_MEMPOOL_TESTNET_PASS="${LN_MEMPOOL_TESTNET_PASS}" +declare -x LN_MEMPOOL_SIGNET_USER="${LN_MEMPOOL_SIGNET_USER}" +declare -x LN_MEMPOOL_SIGNET_PASS="${LN_MEMPOOL_SIGNET_PASS}" declare -x MEMPOOL_LIQUID_USER="${MEMPOOL_LIQUID_USER}" declare -x MEMPOOL_LIQUID_PASS="${MEMPOOL_LIQUID_PASS}" declare -x MEMPOOL_LIQUIDTESTNET_USER="${MEMPOOL_LIQUIDTESTNET_USER}" @@ -1770,24 +1885,32 @@ _EOF_ ##### nginx -echo "[*] Adding Nginx configuration" -osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/nginx/nginx.conf" "${NGINX_CONFIGURATION}" -mkdir -p /var/cache/nginx/services /var/cache/nginx/api -chown ${NGINX_USER}: /var/cache/nginx/services /var/cache/nginx/api -ln -s /mempool/mempool /etc/nginx/mempool -osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_USER__!${NGINX_USER}!" "${NGINX_CONFIGURATION}" -osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_ETC_FOLDER__!${NGINX_ETC_FOLDER}!" "${NGINX_CONFIGURATION}" -if [ "${TOR_INSTALL}" = ON ];then -echo "[*] Read tor v3 onion hostnames" - NGINX_MEMPOOL_ONION=$(cat "${TOR_RESOURCES}/mempool/hostname") - NGINX_BISQ_ONION=$(cat "${TOR_RESOURCES}/bisq/hostname") - NGINX_LIQUID_ONION=$(cat "${TOR_RESOURCES}/liquid/hostname") - osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_MEMPOOL_ONION__!${NGINX_MEMPOOL_ONION%.onion}!" "${NGINX_CONFIGURATION}" - osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_BISQ_ONION__!${NGINX_BISQ_ONION%.onion}!" "${NGINX_CONFIGURATION}" - osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_LIQUID_ONION__!${NGINX_LIQUID_ONIONi%.onion}!" "${NGINX_CONFIGURATION}" -fi -echo "[*] Restarting Nginx" -osSudo "${ROOT_USER}" service nginx restart +case $OS in + + FreeBSD) + ;; + +Debian) + echo "[*] Adding Nginx configuration" + osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/nginx/nginx.conf" "${NGINX_CONFIGURATION}" + mkdir -p /var/cache/nginx/services /var/cache/nginx/api + chown ${NGINX_USER}: /var/cache/nginx/services /var/cache/nginx/api + ln -s /mempool/mempool /etc/nginx/mempool + osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_USER__!${NGINX_USER}!" "${NGINX_CONFIGURATION}" + osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_ETC_FOLDER__!${NGINX_ETC_FOLDER}!" "${NGINX_CONFIGURATION}" + if [ "${TOR_INSTALL}" = ON ];then + echo "[*] Read tor v3 onion hostnames" + NGINX_MEMPOOL_ONION=$(cat "${TOR_RESOURCES}/mempool/hostname") + NGINX_BISQ_ONION=$(cat "${TOR_RESOURCES}/bisq/hostname") + NGINX_LIQUID_ONION=$(cat "${TOR_RESOURCES}/liquid/hostname") + osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_MEMPOOL_ONION__!${NGINX_MEMPOOL_ONION%.onion}!" "${NGINX_CONFIGURATION}" + osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_BISQ_ONION__!${NGINX_BISQ_ONION%.onion}!" "${NGINX_CONFIGURATION}" + osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_LIQUID_ONION__!${NGINX_LIQUID_ONIONi%.onion}!" "${NGINX_CONFIGURATION}" + fi + echo "[*] Restarting Nginx" + osSudo "${ROOT_USER}" service nginx restart + ;; +esac ##### OS systemd diff --git a/production/mempool-build-all b/production/mempool-build-all index 6a4e0c028..048aeefdc 100755 --- a/production/mempool-build-all +++ b/production/mempool-build-all @@ -98,6 +98,12 @@ build_backend() -e "s!__MEMPOOL_TESTNET_PASS__!${MEMPOOL_TESTNET_PASS}!" \ -e "s!__MEMPOOL_SIGNET_USER__!${MEMPOOL_SIGNET_USER}!" \ -e "s!__MEMPOOL_SIGNET_PASS__!${MEMPOOL_SIGNET_PASS}!" \ + -e "s!__LN_MEMPOOL_MAINNET_USER__!${LN_MEMPOOL_MAINNET_USER}!" \ + -e "s!__LN_MEMPOOL_MAINNET_PASS__!${LN_MEMPOOL_MAINNET_PASS}!" \ + -e "s!__LN_MEMPOOL_TESTNET_USER__!${LN_MEMPOOL_TESTNET_USER}!" \ + -e "s!__LN_MEMPOOL_TESTNET_PASS__!${LN_MEMPOOL_TESTNET_PASS}!" \ + -e "s!__LN_MEMPOOL_SIGNET_USER__!${LN_MEMPOOL_SIGNET_USER}!" \ + -e "s!__LN_MEMPOOL_SIGNET_PASS__!${LN_MEMPOOL_SIGNET_PASS}!" \ -e "s!__MEMPOOL_LIQUID_USER__!${MEMPOOL_LIQUID_USER}!" \ -e "s!__MEMPOOL_LIQUID_PASS__!${MEMPOOL_LIQUID_PASS}!" \ -e "s!__MEMPOOL_LIQUIDTESTNET_USER__!${LIQUIDTESTNET_USER}!" \ @@ -145,7 +151,7 @@ for repo in $backend_repos;do done # build unfurlers -for repo in mainnet liquid;do +for repo in mainnet liquid bisq;do build_unfurler "${repo}" done diff --git a/production/mempool-start-all b/production/mempool-start-all index 9751481ad..13fd30430 100755 --- a/production/mempool-start-all +++ b/production/mempool-start-all @@ -9,18 +9,20 @@ for site in mainnet mainnet-lightning testnet testnet-lightning signet signet-li screen -dmS "${site}" sh -c 'while true;do npm run start-production;sleep 1;done' done -# only start unfurler if GPU present +# only start xorg if GPU present if pciconf -lv|grep -i nvidia >/dev/null 2>&1;then export DISPLAY=:0 screen -dmS x startx sleep 3 - for site in mainnet liquid;do - cd "$HOME/${site}/unfurler" && \ - echo "starting mempool unfurler: ${site}" && \ - screen -dmS "unfurler-${site}" sh -c 'while true;do npm run unfurler;sleep 2;done' - done fi +# start unfurlers for each frontend +for site in mainnet liquid bisq;do + cd "$HOME/${site}/unfurler" && \ + echo "starting mempool unfurler: ${site}" && \ + screen -dmS "unfurler-${site}" sh -c 'while true;do npm run unfurler;sleep 2;done' +done + # start nginx warm cacher for site in mainnet;do echo "starting mempool cache warmer: ${site}" diff --git a/production/nginx-cache-warmer b/production/nginx-cache-warmer index 0928990c8..27d1e3a8f 100755 --- a/production/nginx-cache-warmer +++ b/production/nginx-cache-warmer @@ -99,7 +99,7 @@ do for url in / \ '/api/v1/lightning/nodes/isp/39572' `# DataWeb` \ '/api/v1/lightning/nodes/isp/14061' `# Digital Ocean` \ '/api/v1/lightning/nodes/isp/24940,213230' `# Hetzner` \ - '/api/v1/lightning/nodes/isp/174' `# LunaNode` \ + '/api/v1/lightning/nodes/isp/394745' `# LunaNode` \ '/api/v1/lightning/nodes/isp/45102' `# Alibaba` \ '/api/v1/lightning/nodes/isp/3209' `# Vodafone Germany` \ '/api/v1/lightning/nodes/isp/7922' `# Comcast Cable` \ diff --git a/production/nginx/server-common.conf b/production/nginx/server-common.conf index 6ce34c2c3..26e81f7fa 100644 --- a/production/nginx/server-common.conf +++ b/production/nginx/server-common.conf @@ -48,6 +48,9 @@ add_header Vary Cookie; # for exact / requests, redirect based on $lang # cache redirect for 5 minutes location = / { + if ($unfurlbot) { + proxy_pass $mempoolSpaceUnfurler; + } if ($lang != '') { return 302 $scheme://$host/$lang/; } diff --git a/production/unfurler-config.bisq.json b/production/unfurler-config.bisq.json new file mode 100644 index 000000000..9742743f0 --- /dev/null +++ b/production/unfurler-config.bisq.json @@ -0,0 +1,17 @@ +{ + "SERVER": { + "HOST": "https://bisq.fra.mempool.space", + "HTTP_PORT": 8002 + }, + "MEMPOOL": { + "HTTP_HOST": "http://127.0.0.1", + "HTTP_PORT": 82, + "NETWORK": "bisq" + }, + "PUPPETEER": { + "CLUSTER_SIZE": 8, + "EXEC_PATH": "/usr/local/bin/chrome", + "MAX_PAGE_AGE": 86400, + "RENDER_TIMEOUT": 3000 + } +} diff --git a/production/unfurler-config.liquid.json b/production/unfurler-config.liquid.json index 39df7e3fd..243af6ed3 100644 --- a/production/unfurler-config.liquid.json +++ b/production/unfurler-config.liquid.json @@ -1,12 +1,12 @@ { "SERVER": { - "HOST": "https://liquid.network", - "HTTP_PORT": 8002 + "HOST": "https://liquid.fra.mempool.space", + "HTTP_PORT": 8003 }, "MEMPOOL": { - "HTTP_HOST": "https://liquid.network", - "HTTP_PORT": 443, - "NETWORK": "liquid" + "HTTP_HOST": "http://127.0.0.1", + "HTTP_PORT": 83, + "NETWORK": "bitcoin" }, "PUPPETEER": { "CLUSTER_SIZE": 8, diff --git a/production/unfurler-config.mainnet.json b/production/unfurler-config.mainnet.json index 752cd5706..77df23704 100644 --- a/production/unfurler-config.mainnet.json +++ b/production/unfurler-config.mainnet.json @@ -1,11 +1,11 @@ { "SERVER": { - "HOST": "https://mempool.space", + "HOST": "https://mempool.fra.mempool.space", "HTTP_PORT": 8001 }, "MEMPOOL": { - "HTTP_HOST": "https://mempool.space", - "HTTP_PORT": 443, + "HTTP_HOST": "http://127.0.0.1", + "HTTP_PORT": 81, "NETWORK": "bitcoin" }, "PUPPETEER": { diff --git a/unfurler/config.sample.json b/unfurler/config.sample.json index e080ee68a..64f56c1f7 100644 --- a/unfurler/config.sample.json +++ b/unfurler/config.sample.json @@ -9,6 +9,7 @@ "NETWORK": "bitcoin" // "bitcoin" | "liquid" | "bisq" (optional - defaults to "bitcoin") }, "PUPPETEER": { + "DISABLE": false, // optional, boolean, disables puppeteer and /render endpoints "CLUSTER_SIZE": 2, "EXEC_PATH": "/usr/local/bin/chrome", // optional "MAX_PAGE_AGE": 86400, // maximum lifetime of a page session (in seconds) diff --git a/unfurler/package-lock.json b/unfurler/package-lock.json index 3da33c69f..40520d413 100644 --- a/unfurler/package-lock.json +++ b/unfurler/package-lock.json @@ -1,12 +1,12 @@ { "name": "mempool-unfurl", - "version": "0.0.1", + "version": "0.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "mempool-unfurl", - "version": "0.0.1", + "version": "0.1.0", "dependencies": { "@types/node": "^16.11.41", "express": "^4.18.0", diff --git a/unfurler/package.json b/unfurler/package.json index ca60201b3..59d48aa50 100644 --- a/unfurler/package.json +++ b/unfurler/package.json @@ -1,6 +1,6 @@ { "name": "mempool-unfurl", - "version": "0.0.2", + "version": "0.1.0", "description": "Renderer for mempool open graph link preview images", "repository": { "type": "git", diff --git a/unfurler/src/config.ts b/unfurler/src/config.ts index a65d48f6f..3c4a4e422 100644 --- a/unfurler/src/config.ts +++ b/unfurler/src/config.ts @@ -11,6 +11,7 @@ interface IConfig { NETWORK?: string; }; PUPPETEER: { + DISABLE: boolean; CLUSTER_SIZE: number; EXEC_PATH?: string; MAX_PAGE_AGE?: number; @@ -28,6 +29,7 @@ const defaults: IConfig = { 'HTTP_PORT': 4200, }, 'PUPPETEER': { + 'DISABLE': false, 'CLUSTER_SIZE': 1, }, }; diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 08dff3964..eab4723b7 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -1,10 +1,12 @@ import express from "express"; import { Application, Request, Response, NextFunction } from 'express'; import * as http from 'http'; +import * as https from 'https'; import config from './config'; import { Cluster } from 'puppeteer-cluster'; import ReusablePage from './concurrency/ReusablePage'; import { parseLanguageUrl } from './language/lang'; +import { matchRoute } from './routes'; const puppeteerConfig = require('../puppeteer.config.json'); if (config.PUPPETEER.EXEC_PATH) { @@ -17,13 +19,13 @@ class Server { cluster?: Cluster; mempoolHost: string; network: string; - defaultImageUrl: string; + secureHost = true; constructor() { this.app = express(); this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : ''); + this.secureHost = this.mempoolHost.startsWith('https'); this.network = config.MEMPOOL.NETWORK || 'bitcoin'; - this.defaultImageUrl = this.getDefaultImageUrl(); this.startServer(); } @@ -37,12 +39,14 @@ class Server { .use(express.text()) ; - this.cluster = await Cluster.launch({ - concurrency: ReusablePage, - maxConcurrency: config.PUPPETEER.CLUSTER_SIZE, - puppeteerOptions: puppeteerConfig, - }); - await this.cluster?.task(async (args) => { return this.clusterTask(args) }); + if (!config.PUPPETEER.DISABLE) { + this.cluster = await Cluster.launch({ + concurrency: ReusablePage, + maxConcurrency: config.PUPPETEER.CLUSTER_SIZE, + puppeteerOptions: puppeteerConfig, + }); + await this.cluster?.task(async (args) => { return this.clusterTask(args) }); + } this.setUpRoutes(); @@ -64,7 +68,11 @@ class Server { } setUpRoutes() { - this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) }) + if (!config.PUPPETEER.DISABLE) { + this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) }) + } else { + this.app.get('/render*', async (req, res) => { return this.renderDisabled(req, res) }) + } this.app.get('*', (req, res) => { return this.renderHTML(req, res) }) } @@ -111,13 +119,31 @@ class Server { } } + async renderDisabled(req, res) { + res.status(500).send("preview rendering disabled"); + } + async renderPreview(req, res) { try { - const path = req.params[0] - const img = await this.cluster?.execute({ url: this.mempoolHost + path, path: path, action: 'screenshot' }); + const rawPath = req.params[0]; + + let img = null; + + const { lang, path } = parseLanguageUrl(rawPath); + const matchedRoute = matchRoute(this.network, path); + + // don't bother unless the route is definitely renderable + if (rawPath.includes('/preview/') && matchedRoute.render) { + img = await this.cluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'screenshot' }); + } if (!img) { - res.status(500).send('failed to render page preview'); + // proxy fallback image from the frontend + if (this.secureHost) { + https.get(config.SERVER.HOST + matchedRoute.fallbackImg, (got) => got.pipe(res)); + } else { + http.get(config.SERVER.HOST + matchedRoute.fallbackImg, (got) => got.pipe(res)); + } } else { res.contentType('image/png'); res.send(img); @@ -137,50 +163,14 @@ class Server { return; } - let previewSupported = true; - let mode = 'mainnet' - let ogImageUrl = this.defaultImageUrl; - let ogTitle; const { lang, path } = parseLanguageUrl(rawPath); - const parts = path.slice(1).split('/'); + const matchedRoute = matchRoute(this.network, path); + let ogImageUrl = config.SERVER.HOST + (matchedRoute.staticImg || matchedRoute.fallbackImg); + let ogTitle = 'The Mempool Open Source Project™'; - // handle network mode modifiers - if (['testnet', 'signet'].includes(parts[0])) { - mode = parts.shift(); - } - - // handle supported preview routes - switch (parts[0]) { - case 'block': - ogTitle = `Block: ${parts[1]}`; - break; - case 'address': - ogTitle = `Address: ${parts[1]}`; - break; - case 'tx': - ogTitle = `Transaction: ${parts[1]}`; - break; - case 'lightning': - switch (parts[1]) { - case 'node': - ogTitle = `Lightning Node: ${parts[2]}`; - break; - case 'channel': - ogTitle = `Lightning Channel: ${parts[2]}`; - break; - default: - previewSupported = false; - } - break; - default: - previewSupported = false; - } - - if (previewSupported) { + if (matchedRoute.render) { ogImageUrl = `${config.SERVER.HOST}/render/${lang || 'en'}/preview${path}`; - ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${mode !== 'mainnet' ? capitalize(mode) + ' ' : ''}${ogTitle}`; - } else { - ogTitle = 'The Mempool Open Source Project™'; + ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${matchedRoute.networkMode !== 'mainnet' ? capitalize(matchedRoute.networkMode) + ' ' : ''}${matchedRoute.title}`; } res.send(` @@ -189,34 +179,23 @@ class Server { ${ogTitle} - + - - + + - + `); } - - getDefaultImageUrl() { - switch (this.network) { - case 'liquid': - return this.mempoolHost + '/resources/liquid/liquid-network-preview.png'; - case 'bisq': - return this.mempoolHost + '/resources/bisq/bisq-markets-preview.png'; - default: - return this.mempoolHost + '/resources/mempool-space-preview.png'; - } - } } const server = new Server(); diff --git a/unfurler/src/routes.ts b/unfurler/src/routes.ts new file mode 100644 index 000000000..24922c85d --- /dev/null +++ b/unfurler/src/routes.ts @@ -0,0 +1,124 @@ +interface Match { + render: boolean; + title: string; + fallbackImg: string; + staticImg?: string; + networkMode: string; +} + +const routes = { + block: { + render: true, + params: 1, + getTitle(path) { + return `Block: ${path[0]}`; + } + }, + address: { + render: true, + params: 1, + getTitle(path) { + return `Address: ${path[0]}`; + } + }, + tx: { + render: true, + params: 1, + getTitle(path) { + return `Transaction: ${path[0]}`; + } + }, + lightning: { + title: "Lightning", + fallbackImg: '/resources/previews/lightning.png', + routes: { + node: { + render: true, + params: 1, + getTitle(path) { + return `Lightning Node: ${path[0]}`; + } + }, + channel: { + render: true, + params: 1, + getTitle(path) { + return `Lightning Channel: ${path[0]}`; + } + }, + } + }, + mining: { + title: "Mining", + fallbackImg: '/resources/previews/mining.png' + } +}; + +const networks = { + bitcoin: { + fallbackImg: '/resources/mempool-space-preview.png', + staticImg: '/resources/previews/dashboard.png', + routes: { + ...routes // all routes supported + } + }, + liquid: { + fallbackImg: '/resources/liquid/liquid-network-preview.png', + routes: { // only block, address & tx routes supported + block: routes.block, + address: routes.address, + tx: routes.tx + } + }, + bisq: { + fallbackImg: '/resources/bisq/bisq-markets-preview.png', + routes: {} // no routes supported + } +}; + +export function matchRoute(network: string, path: string): Match { + const match: Match = { + render: false, + title: '', + fallbackImg: '', + networkMode: 'mainnet' + } + + const parts = path.slice(1).split('/').filter(p => p.length); + + if (parts[0] === 'preview') { + parts.shift(); + } + if (['testnet', 'signet'].includes(parts[0])) { + match.networkMode = parts.shift() || 'mainnet'; + } + + let route = networks[network] || networks.bitcoin; + match.fallbackImg = route.fallbackImg; + + // traverse the route tree until we run out of route or tree, or hit a renderable match + while (!route.render && route.routes && parts.length && route.routes[parts[0]]) { + route = route.routes[parts[0]]; + parts.shift(); + if (route.fallbackImg) { + match.fallbackImg = route.fallbackImg; + } + } + + // enough route parts left for title & rendering + if (route.render && parts.length >= route.params) { + match.render = true; + } + // only use set a static image for exact matches + if (!parts.length && route.staticImg) { + match.staticImg = route.staticImg; + } + // apply the title function if present + if (route.getTitle && typeof route.getTitle === 'function') { + match.title = route.getTitle(parts); + } else { + match.title = route.title; + } + + return match; +} \ No newline at end of file