From 665d85204b1d56830c60df24c118d4c36b5aee85 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 9 Jul 2022 17:45:34 +0200 Subject: [PATCH 1/5] Backfill node_stats --- .../src/tasks/lightning/node-sync.service.ts | 4 +- .../tasks/lightning/stats-updater.service.ts | 84 ++++++++++++++++--- 2 files changed, 77 insertions(+), 11 deletions(-) diff --git a/backend/src/tasks/lightning/node-sync.service.ts b/backend/src/tasks/lightning/node-sync.service.ts index c5a6c8a9d..b7e23a7fc 100644 --- a/backend/src/tasks/lightning/node-sync.service.ts +++ b/backend/src/tasks/lightning/node-sync.service.ts @@ -44,7 +44,9 @@ class NodeSyncService { await this.$lookUpCreationDateFromChain(); await this.$updateNodeFirstSeen(); await this.$scanForClosedChannels(); - await this.$runClosedChannelsForensics(); + if (config.MEMPOOL.BACKEND === 'esplora') { + await this.$runClosedChannelsForensics(); + } } catch (e) { logger.err('$updateNodes() error: ' + (e instanceof Error ? e.message : e)); diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index 01de7ede1..5e8c0dcbb 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -6,7 +6,7 @@ import channelsApi from '../../api/explorer/channels.api'; import * as net from 'net'; class LightningStatsUpdater { - constructor() {} + hardCodedStartTime = '2018-01-12'; public async $startService() { logger.info('Starting Lightning Stats service'); @@ -47,7 +47,8 @@ class LightningStatsUpdater { } private async $runTasks() { - await this.$populateHistoricalData(); + await this.$populateHistoricalStatistics(); + await this.$populateHistoricalNodeStatistics(); await this.$logLightningStatsDaily(); await this.$logNodeStatsDaily(); } @@ -85,11 +86,10 @@ class LightningStatsUpdater { } // We only run this on first launch - private async $populateHistoricalData() { - const startTime = '2018-01-13'; + private async $populateHistoricalStatistics() { try { const [rows]: any = await DB.query(`SELECT COUNT(*) FROM lightning_stats`); - // Only store once per day + // Only run if table is empty if (rows[0]['COUNT(*)'] > 0) { return; } @@ -97,10 +97,11 @@ class LightningStatsUpdater { const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels ORDER BY created ASC`); - let date: Date = new Date(startTime); + let date: Date = new Date(this.hardCodedStartTime); const currentDate = new Date(); while (date < currentDate) { + date.setUTCDate(date.getUTCDate() + 1); let totalCapacity = 0; let channelsCount = 0; for (const channel of channels) { @@ -140,9 +141,10 @@ class LightningStatsUpdater { } const [nodes]: any = await DB.query(`SELECT first_seen, sockets FROM nodes ORDER BY first_seen ASC`); - date = new Date(startTime); + date = new Date(this.hardCodedStartTime); while (date < currentDate) { + date.setUTCDate(date.getUTCDate() + 1); let nodeCount = 0; let clearnetNodes = 0; let torNodes = 0; @@ -181,9 +183,6 @@ class LightningStatsUpdater { unannouncedNodes, date.getTime() / 1000, ]); - - // Add one day and continue - date.setDate(date.getDate() + 1); } logger.info('Historical stats populated.'); @@ -192,6 +191,71 @@ class LightningStatsUpdater { } } + private async $populateHistoricalNodeStatistics() { + try { + const [rows]: any = await DB.query(`SELECT COUNT(*) FROM node_stats`); + // Only run if table is empty + if (rows[0]['COUNT(*)'] > 0) { + return; + } + logger.info(`Running historical node stats population...`); + + const [nodes]: any = await DB.query(`SELECT public_key, first_seen, alias FROM nodes ORDER BY first_seen ASC`); + + for (const node of nodes) { + const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels WHERE node1_public_key = ? OR node2_public_key = ? ORDER BY created ASC`, [node.public_key, node.public_key]); + + let date: Date = new Date(this.hardCodedStartTime); + const currentDate = new Date(); + + let lastTotalCapacity = 0; + let lastChannelsCount = 0; + + while (date < currentDate) { + date.setUTCDate(date.getUTCDate() + 1); + let totalCapacity = 0; + let channelsCount = 0; + for (const channel of channels) { + if (new Date(channel.created) > date) { + break; + } + if (channel.closing_date !== null && new Date(channel.closing_date) < date) { + continue; + } + totalCapacity += channel.capacity; + channelsCount++; + } + + if (lastTotalCapacity === totalCapacity && lastChannelsCount === channelsCount) { + continue; + } + + lastTotalCapacity = totalCapacity; + lastChannelsCount = channelsCount; + + const query = `INSERT INTO node_stats( + public_key, + added, + capacity, + channels + ) + VALUES (?, FROM_UNIXTIME(?), ?, ?)`; + + await DB.query(query, [ + node.public_key, + date.getTime() / 1000, + totalCapacity, + channelsCount, + ]); + } + logger.debug('Updated node_stats for: ' + node.alias); + } + logger.info('Historical stats populated.'); + } catch (e) { + logger.err('$populateHistoricalNodeData() error: ' + (e instanceof Error ? e.message : e)); + } + } + private async $logLightningStatsDaily() { const currentDate = new Date().toISOString().split('T')[0]; try { From 7320fadec9f6a6b107d3a85b2761bef2830a9fa0 Mon Sep 17 00:00:00 2001 From: wiz Date: Mon, 11 Jul 2022 14:29:56 +0200 Subject: [PATCH 2/5] Add maxmind geoip-db update utility to prod installer --- production/install | 2 ++ 1 file changed, 2 insertions(+) diff --git a/production/install b/production/install index 22d15b5af..fb3aa9281 100755 --- a/production/install +++ b/production/install @@ -334,6 +334,7 @@ DEBIAN_PKG+=(autotools-dev autoconf automake pkg-config bsdmainutils) DEBIAN_PKG+=(libevent-dev libdb-dev libssl-dev libtool autotools-dev) DEBIAN_PKG+=(libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev) DEBIAN_PKG+=(nodejs npm mariadb-server nginx-core python3-certbot-nginx rsync ufw) +DEBIAN_PKG+=(geoipupdate) # packages needed for mempool ecosystem FREEBSD_PKG=() @@ -341,6 +342,7 @@ FREEBSD_PKG+=(zsh sudo git screen curl wget calc neovim) FREEBSD_PKG+=(openssh-portable py39-pip rust llvm90 jq base64 libzmq4) FREEBSD_PKG+=(boost-libs autoconf automake gmake gcc libevent libtool pkgconf) FREEBSD_PKG+=(nginx rsync py39-certbot-nginx mariadb105-server keybase) +FREEBSD_PKG+=(geoipupdate) ############################# ##### utility functions ##### From 1c862730598f46bccb37f9304847dab8f3f535d9 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 10 Jul 2022 20:01:15 +0200 Subject: [PATCH 3/5] Run daily stats at midnight and backfill first launch --- backend/src/api/database-migration.ts | 8 +- .../tasks/lightning/stats-updater.service.ts | 421 +++++++++--------- 2 files changed, 207 insertions(+), 222 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index a4612e0ba..4870c5d03 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 27; + private static currentVersion = 28; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -274,6 +274,12 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"'); } + if (databaseSchemaVersion < 28 && isBitcoin === true) { + await this.$executeQuery(`TRUNCATE lightning_stats`); + await this.$executeQuery(`TRUNCATE node_stats`); + await this.$executeQuery(`ALTER TABLE lightning_stats MODIFY added DATE`); + } + } catch (e) { throw e; } diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index 5e8c0dcbb..1e718188a 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -28,17 +28,26 @@ class LightningStatsUpdater { return; } - const now = new Date(); - const nextHourInterval = new Date(now.getFullYear(), now.getMonth(), now.getDate(), Math.floor(now.getHours() / 1) + 1, 0, 0, 0); - const difference = nextHourInterval.getTime() - now.getTime(); + await this.$populateHistoricalStatistics(); + await this.$populateHistoricalNodeStatistics(); setTimeout(() => { - setInterval(async () => { - await this.$runTasks(); - }, 1000 * 60 * 60); - }, difference); + this.$runTasks(); + }, this.timeUntilMidnight()); + } - await this.$runTasks(); + private timeUntilMidnight(): number { + const date = new Date(); + this.setDateMidnight(date); + date.setUTCHours(24); + return date.getTime() - new Date().getTime(); + } + + private setDateMidnight(date: Date): void { + date.setUTCHours(0); + date.setUTCMinutes(0); + date.setUTCSeconds(0); + date.setUTCMilliseconds(0); } private async $lightningIsSynced(): Promise { @@ -46,225 +55,17 @@ class LightningStatsUpdater { return nodeInfo.is_synced_to_chain && nodeInfo.is_synced_to_graph; } - private async $runTasks() { - await this.$populateHistoricalStatistics(); - await this.$populateHistoricalNodeStatistics(); + private async $runTasks(): Promise { await this.$logLightningStatsDaily(); await this.$logNodeStatsDaily(); - } - private async $logNodeStatsDaily() { - const currentDate = new Date().toISOString().split('T')[0]; - try { - const [state]: any = await DB.query(`SELECT string FROM state WHERE name = 'last_node_stats'`); - // Only store once per day - if (state[0].string === currentDate) { - return; - } - - logger.info(`Running daily node stats update...`); - - const query = `SELECT nodes.public_key, c1.channels_count_left, c2.channels_count_right, c1.channels_capacity_left, c2.channels_capacity_right FROM nodes LEFT JOIN (SELECT node1_public_key, COUNT(id) AS channels_count_left, SUM(capacity) AS channels_capacity_left FROM channels WHERE channels.status < 2 GROUP BY node1_public_key) c1 ON c1.node1_public_key = nodes.public_key LEFT JOIN (SELECT node2_public_key, COUNT(id) AS channels_count_right, SUM(capacity) AS channels_capacity_right FROM channels WHERE channels.status < 2 GROUP BY node2_public_key) c2 ON c2.node2_public_key = nodes.public_key`; - const [nodes]: any = await DB.query(query); - - // First run we won't have any nodes yet - if (nodes.length < 10) { - return; - } - - for (const node of nodes) { - await DB.query( - `INSERT INTO node_stats(public_key, added, capacity, channels) VALUES (?, NOW(), ?, ?)`, - [node.public_key, (parseInt(node.channels_capacity_left || 0, 10)) + (parseInt(node.channels_capacity_right || 0, 10)), - node.channels_count_left + node.channels_count_right]); - } - await DB.query(`UPDATE state SET string = ? WHERE name = 'last_node_stats'`, [currentDate]); - logger.info('Daily node stats has updated.'); - } catch (e) { - logger.err('$logNodeStatsDaily() error: ' + (e instanceof Error ? e.message : e)); - } - } - - // We only run this on first launch - private async $populateHistoricalStatistics() { - try { - const [rows]: any = await DB.query(`SELECT COUNT(*) FROM lightning_stats`); - // Only run if table is empty - if (rows[0]['COUNT(*)'] > 0) { - return; - } - logger.info(`Running historical stats population...`); - - const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels ORDER BY created ASC`); - - let date: Date = new Date(this.hardCodedStartTime); - const currentDate = new Date(); - - while (date < currentDate) { - date.setUTCDate(date.getUTCDate() + 1); - let totalCapacity = 0; - let channelsCount = 0; - for (const channel of channels) { - if (new Date(channel.created) > date) { - break; - } - if (channel.closing_date !== null && new Date(channel.closing_date) < date) { - continue; - } - totalCapacity += channel.capacity; - channelsCount++; - } - - const query = `INSERT INTO lightning_stats( - added, - channel_count, - node_count, - total_capacity, - tor_nodes, - clearnet_nodes, - unannounced_nodes - ) - VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?)`; - - await DB.query(query, [ - date.getTime() / 1000, - channelsCount, - 0, - totalCapacity, - 0, - 0, - 0 - ]); - - // Add one day and continue - date.setDate(date.getDate() + 1); - } - - const [nodes]: any = await DB.query(`SELECT first_seen, sockets FROM nodes ORDER BY first_seen ASC`); - date = new Date(this.hardCodedStartTime); - - while (date < currentDate) { - date.setUTCDate(date.getUTCDate() + 1); - let nodeCount = 0; - let clearnetNodes = 0; - let torNodes = 0; - let unannouncedNodes = 0; - for (const node of nodes) { - if (new Date(node.first_seen) > date) { - break; - } - nodeCount++; - - const sockets = node.sockets.split(','); - let isUnnanounced = true; - for (const socket of sockets) { - const hasOnion = socket.indexOf('.onion') !== -1; - if (hasOnion) { - torNodes++; - isUnnanounced = false; - } - const hasClearnet = [4, 6].includes(net.isIP(socket.split(':')[0])); - if (hasClearnet) { - clearnetNodes++; - isUnnanounced = false; - } - } - if (isUnnanounced) { - unannouncedNodes++; - } - } - - const query = `UPDATE lightning_stats SET node_count = ?, tor_nodes = ?, clearnet_nodes = ?, unannounced_nodes = ? WHERE added = FROM_UNIXTIME(?)`; - - await DB.query(query, [ - nodeCount, - torNodes, - clearnetNodes, - unannouncedNodes, - date.getTime() / 1000, - ]); - } - - logger.info('Historical stats populated.'); - } catch (e) { - logger.err('$populateHistoricalData() error: ' + (e instanceof Error ? e.message : e)); - } - } - - private async $populateHistoricalNodeStatistics() { - try { - const [rows]: any = await DB.query(`SELECT COUNT(*) FROM node_stats`); - // Only run if table is empty - if (rows[0]['COUNT(*)'] > 0) { - return; - } - logger.info(`Running historical node stats population...`); - - const [nodes]: any = await DB.query(`SELECT public_key, first_seen, alias FROM nodes ORDER BY first_seen ASC`); - - for (const node of nodes) { - const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels WHERE node1_public_key = ? OR node2_public_key = ? ORDER BY created ASC`, [node.public_key, node.public_key]); - - let date: Date = new Date(this.hardCodedStartTime); - const currentDate = new Date(); - - let lastTotalCapacity = 0; - let lastChannelsCount = 0; - - while (date < currentDate) { - date.setUTCDate(date.getUTCDate() + 1); - let totalCapacity = 0; - let channelsCount = 0; - for (const channel of channels) { - if (new Date(channel.created) > date) { - break; - } - if (channel.closing_date !== null && new Date(channel.closing_date) < date) { - continue; - } - totalCapacity += channel.capacity; - channelsCount++; - } - - if (lastTotalCapacity === totalCapacity && lastChannelsCount === channelsCount) { - continue; - } - - lastTotalCapacity = totalCapacity; - lastChannelsCount = channelsCount; - - const query = `INSERT INTO node_stats( - public_key, - added, - capacity, - channels - ) - VALUES (?, FROM_UNIXTIME(?), ?, ?)`; - - await DB.query(query, [ - node.public_key, - date.getTime() / 1000, - totalCapacity, - channelsCount, - ]); - } - logger.debug('Updated node_stats for: ' + node.alias); - } - logger.info('Historical stats populated.'); - } catch (e) { - logger.err('$populateHistoricalNodeData() error: ' + (e instanceof Error ? e.message : e)); - } + setTimeout(() => { + this.$runTasks(); + }, this.timeUntilMidnight()); } private async $logLightningStatsDaily() { - const currentDate = new Date().toISOString().split('T')[0]; try { - const [state]: any = await DB.query(`SELECT string FROM state WHERE name = 'last_node_stats'`); - // Only store once per day - if (state[0].string === currentDate) { - return; - } - logger.info(`Running lightning daily stats log...`); const networkGraph = await lightningApi.$getNetworkGraph(); @@ -314,7 +115,7 @@ class LightningStatsUpdater { med_fee_rate, med_base_fee_mtokens ) - VALUES (NOW(), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; + VALUES (NOW() - INTERVAL 1 DAY, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; await DB.query(query, [ networkGraph.channels.length, @@ -335,6 +136,184 @@ class LightningStatsUpdater { logger.err('$logLightningStatsDaily() error: ' + (e instanceof Error ? e.message : e)); } } + + private async $logNodeStatsDaily() { + try { + logger.info(`Running daily node stats update...`); + + const query = `SELECT nodes.public_key, c1.channels_count_left, c2.channels_count_right, c1.channels_capacity_left, c2.channels_capacity_right FROM nodes LEFT JOIN (SELECT node1_public_key, COUNT(id) AS channels_count_left, SUM(capacity) AS channels_capacity_left FROM channels WHERE channels.status < 2 GROUP BY node1_public_key) c1 ON c1.node1_public_key = nodes.public_key LEFT JOIN (SELECT node2_public_key, COUNT(id) AS channels_count_right, SUM(capacity) AS channels_capacity_right FROM channels WHERE channels.status < 2 GROUP BY node2_public_key) c2 ON c2.node2_public_key = nodes.public_key`; + const [nodes]: any = await DB.query(query); + + for (const node of nodes) { + await DB.query( + `INSERT INTO node_stats(public_key, added, capacity, channels) VALUES (?, NOW() - INTERVAL 1 DAY, ?, ?)`, + [node.public_key, (parseInt(node.channels_capacity_left || 0, 10)) + (parseInt(node.channels_capacity_right || 0, 10)), + node.channels_count_left + node.channels_count_right]); + } + logger.info('Daily node stats has updated.'); + } catch (e) { + logger.err('$logNodeStatsDaily() error: ' + (e instanceof Error ? e.message : e)); + } + } + + // We only run this on first launch + private async $populateHistoricalStatistics() { + try { + const [rows]: any = await DB.query(`SELECT COUNT(*) FROM lightning_stats`); + // Only run if table is empty + if (rows[0]['COUNT(*)'] > 0) { + return; + } + logger.info(`Running historical stats population...`); + + const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels ORDER BY created ASC`); + const [nodes]: any = await DB.query(`SELECT first_seen, sockets FROM nodes ORDER BY first_seen ASC`); + + const date: Date = new Date(this.hardCodedStartTime); + const currentDate = new Date(); + this.setDateMidnight(currentDate); + + while (date < currentDate) { + let totalCapacity = 0; + let channelsCount = 0; + + for (const channel of channels) { + if (new Date(channel.created) > date) { + break; + } + if (channel.closing_date === null || new Date(channel.closing_date) > date) { + totalCapacity += channel.capacity; + channelsCount++; + } + } + + let nodeCount = 0; + let clearnetNodes = 0; + let torNodes = 0; + let unannouncedNodes = 0; + + for (const node of nodes) { + if (new Date(node.first_seen) > date) { + break; + } + nodeCount++; + + const sockets = node.sockets.split(','); + let isUnnanounced = true; + for (const socket of sockets) { + const hasOnion = socket.indexOf('.onion') !== -1; + if (hasOnion) { + torNodes++; + isUnnanounced = false; + } + const hasClearnet = [4, 6].includes(net.isIP(socket.split(':')[0])); + if (hasClearnet) { + clearnetNodes++; + isUnnanounced = false; + } + } + if (isUnnanounced) { + unannouncedNodes++; + } + } + + const query = `INSERT INTO lightning_stats( + added, + channel_count, + node_count, + total_capacity, + tor_nodes, + clearnet_nodes, + unannounced_nodes + ) + VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?)`; + + await DB.query(query, [ + date.getTime() / 1000, + channelsCount, + nodeCount, + totalCapacity, + torNodes, + clearnetNodes, + unannouncedNodes, + ]); + + date.setUTCDate(date.getUTCDate() + 1); + } + + logger.info('Historical stats populated.'); + } catch (e) { + logger.err('$populateHistoricalData() error: ' + (e instanceof Error ? e.message : e)); + } + } + + private async $populateHistoricalNodeStatistics() { + try { + const [rows]: any = await DB.query(`SELECT COUNT(*) FROM node_stats`); + // Only run if table is empty + if (rows[0]['COUNT(*)'] > 0) { + return; + } + logger.info(`Running historical node stats population...`); + + const [nodes]: any = await DB.query(`SELECT public_key, first_seen, alias FROM nodes ORDER BY first_seen ASC`); + + for (const node of nodes) { + const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels WHERE node1_public_key = ? OR node2_public_key = ? ORDER BY created ASC`, [node.public_key, node.public_key]); + + const date: Date = new Date(this.hardCodedStartTime); + const currentDate = new Date(); + this.setDateMidnight(currentDate); + + let lastTotalCapacity = 0; + let lastChannelsCount = 0; + + while (date < currentDate) { + let totalCapacity = 0; + let channelsCount = 0; + for (const channel of channels) { + if (new Date(channel.created) > date) { + break; + } + if (channel.closing_date !== null && new Date(channel.closing_date) < date) { + date.setUTCDate(date.getUTCDate() + 1); + continue; + } + totalCapacity += channel.capacity; + channelsCount++; + } + + if (lastTotalCapacity === totalCapacity && lastChannelsCount === channelsCount) { + date.setUTCDate(date.getUTCDate() + 1); + continue; + } + + lastTotalCapacity = totalCapacity; + lastChannelsCount = channelsCount; + + const query = `INSERT INTO node_stats( + public_key, + added, + capacity, + channels + ) + VALUES (?, FROM_UNIXTIME(?), ?, ?)`; + + await DB.query(query, [ + node.public_key, + date.getTime() / 1000, + totalCapacity, + channelsCount, + ]); + date.setUTCDate(date.getUTCDate() + 1); + } + logger.debug('Updated node_stats for: ' + node.alias); + } + logger.info('Historical stats populated.'); + } catch (e) { + logger.err('$populateHistoricalNodeData() error: ' + (e instanceof Error ? e.message : e)); + } + } } export default new LightningStatsUpdater(); From eec82e1bf96a2e895ba7f0ad6d3ba19dbd32e576 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 11 Jul 2022 15:25:42 +0200 Subject: [PATCH 4/5] nginx: Fix gixy test `http_splitting` Fixes test error: ``` >> Problem: [http_splitting] Possible HTTP-Splitting vulnerability. Description: Using variables that can contain "\n" or "\r" may lead to http injection. ``` Summary: `$uri` should never be used in `return` statements. See: https://github.com/yandex/gixy/blob/master/docs/en/plugins/httpsplitting.md In this case, `$uri` always equals `/`, so just replace it. --- production/nginx/server-common.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/nginx/server-common.conf b/production/nginx/server-common.conf index ef63194c0..901a490cb 100644 --- a/production/nginx/server-common.conf +++ b/production/nginx/server-common.conf @@ -49,7 +49,7 @@ add_header Vary Cookie; # cache redirect for 10 minutes location = / { if ($lang != '') { - return 302 $scheme://$host/$lang$uri; + return 302 $scheme://$host/$lang/; } try_files /en-US/index.html =404; expires 10m; From 81bc449043b1e94f472b55b1ca26929fefd73b7e Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 11 Jul 2022 15:31:13 +0200 Subject: [PATCH 5/5] nginx: Fix gixy test `host_spoofing` This patch was generated by replacing: `proxy_set_header Host $http_host` -> `proxy_set_header Host $host` Script: find . -type f -exec sed -i 's|proxy_set_header Host \$http_host|proxy_set_header Host \$host|g' {} \; Fixes test error: ``` >> Problem: [host_spoofing] The proxied Host header may be spoofed. Description: In most cases "$host" variable are more appropriate, just use it. Additional info: https://github.com/yandex/gixy/blob/master/docs/en/plugins/hostspoofing.md ``` `proxy_set_header Host $host` is indeed the recommended default proxy header setting. --- production/nginx/location-api-v1-lightning.conf | 2 +- production/nginx/location-api.conf | 14 +++++++------- production/nginx/location-liquid-api.conf | 14 +++++++------- production/nginx/location-liquidtestnet-api.conf | 14 +++++++------- .../nginx/location-signet-api-v1-lightning.conf | 2 +- production/nginx/location-signet-api.conf | 14 +++++++------- .../nginx/location-testnet-api-v1-lightning.conf | 2 +- production/nginx/location-testnet-api.conf | 14 +++++++------- production/nginx/server-bisq.conf | 4 ++-- 9 files changed, 40 insertions(+), 40 deletions(-) diff --git a/production/nginx/location-api-v1-lightning.conf b/production/nginx/location-api-v1-lightning.conf index 288fe4184..f90fd529a 100644 --- a/production/nginx/location-api-v1-lightning.conf +++ b/production/nginx/location-api-v1-lightning.conf @@ -5,7 +5,7 @@ location /api/v1/lightning { location @mempool-api-v1-lightning { proxy_pass $mempoolMainnetLightning; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; diff --git a/production/nginx/location-api.conf b/production/nginx/location-api.conf index 2b5cb0152..638e1911c 100644 --- a/production/nginx/location-api.conf +++ b/production/nginx/location-api.conf @@ -48,7 +48,7 @@ location @mempool-api-v1-websocket { proxy_pass $mempoolMainnet; proxy_http_version 1.1; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Upgrade $http_upgrade; @@ -59,7 +59,7 @@ location @mempool-api-v1-websocket { location @mempool-api-v1-cache-forever { proxy_pass $mempoolMainnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -76,7 +76,7 @@ location @mempool-api-v1-cache-forever { location @mempool-api-v1-cache-warm { proxy_pass $mempoolMainnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -91,7 +91,7 @@ location @mempool-api-v1-cache-warm { location @mempool-api-v1-cache-normal { proxy_pass $mempoolMainnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -106,7 +106,7 @@ location @mempool-api-v1-cache-normal { location @mempool-api-v1-cache-disabled { proxy_pass $mempoolMainnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -120,7 +120,7 @@ location @mempool-api-v1-cache-disabled { location @esplora-api-cache-disabled { proxy_pass $esploraMainnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -134,7 +134,7 @@ location @esplora-api-cache-disabled { location @esplora-api-cache-forever { proxy_pass $esploraMainnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; diff --git a/production/nginx/location-liquid-api.conf b/production/nginx/location-liquid-api.conf index 26ffffc70..e438d1cdc 100644 --- a/production/nginx/location-liquid-api.conf +++ b/production/nginx/location-liquid-api.conf @@ -49,7 +49,7 @@ location @mempool-liquid-api-v1-websocket { proxy_pass $mempoolMainnet; proxy_http_version 1.1; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Upgrade $http_upgrade; @@ -60,7 +60,7 @@ location @mempool-liquid-api-v1-websocket { location @mempool-liquid-api-v1-cache-forever { proxy_pass $mempoolMainnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -77,7 +77,7 @@ location @mempool-liquid-api-v1-cache-forever { location @mempool-liquid-api-v1-cache-warm { proxy_pass $mempoolMainnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -92,7 +92,7 @@ location @mempool-liquid-api-v1-cache-warm { location @mempool-liquid-api-v1-cache-normal { proxy_pass $mempoolMainnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -107,7 +107,7 @@ location @mempool-liquid-api-v1-cache-normal { location @mempool-liquid-api-v1-cache-disabled { proxy_pass $mempoolMainnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -121,7 +121,7 @@ location @mempool-liquid-api-v1-cache-disabled { location @esplora-liquid-api-cache-disabled { proxy_pass $esploraMainnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -135,7 +135,7 @@ location @esplora-liquid-api-cache-disabled { location @esplora-liquid-api-cache-forever { proxy_pass $esploraMainnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; diff --git a/production/nginx/location-liquidtestnet-api.conf b/production/nginx/location-liquidtestnet-api.conf index 311a6c317..329b7e2e9 100644 --- a/production/nginx/location-liquidtestnet-api.conf +++ b/production/nginx/location-liquidtestnet-api.conf @@ -53,7 +53,7 @@ location @mempool-liquidtestnet-api-v1-websocket { proxy_pass $mempoolTestnet; proxy_http_version 1.1; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Upgrade $http_upgrade; @@ -64,7 +64,7 @@ location @mempool-liquidtestnet-api-v1-websocket { location @mempool-liquidtestnet-api-v1-cache-forever { proxy_pass $mempoolTestnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -81,7 +81,7 @@ location @mempool-liquidtestnet-api-v1-cache-forever { location @mempool-liquidtestnet-api-v1-cache-warm { proxy_pass $mempoolTestnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -96,7 +96,7 @@ location @mempool-liquidtestnet-api-v1-cache-warm { location @mempool-liquidtestnet-api-v1-cache-normal { proxy_pass $mempoolTestnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -111,7 +111,7 @@ location @mempool-liquidtestnet-api-v1-cache-normal { location @mempool-liquidtestnet-api-v1-cache-disabled { proxy_pass $mempoolTestnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -125,7 +125,7 @@ location @mempool-liquidtestnet-api-v1-cache-disabled { location @esplora-liquidtestnet-api-cache-disabled { proxy_pass $esploraTestnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -139,7 +139,7 @@ location @esplora-liquidtestnet-api-cache-disabled { location @esplora-liquidtestnet-api-cache-forever { proxy_pass $esploraTestnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; diff --git a/production/nginx/location-signet-api-v1-lightning.conf b/production/nginx/location-signet-api-v1-lightning.conf index eb25c9820..ab14a170b 100644 --- a/production/nginx/location-signet-api-v1-lightning.conf +++ b/production/nginx/location-signet-api-v1-lightning.conf @@ -6,7 +6,7 @@ location /signet/api/v1/lightning { location @mempool-signet-api-v1-lightning { proxy_pass $mempoolSignetLightning; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; diff --git a/production/nginx/location-signet-api.conf b/production/nginx/location-signet-api.conf index 262dea48e..54bdc3648 100644 --- a/production/nginx/location-signet-api.conf +++ b/production/nginx/location-signet-api.conf @@ -53,7 +53,7 @@ location @mempool-signet-api-v1-websocket { proxy_pass $mempoolSignet; proxy_http_version 1.1; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Upgrade $http_upgrade; @@ -64,7 +64,7 @@ location @mempool-signet-api-v1-websocket { location @mempool-signet-api-v1-cache-forever { proxy_pass $mempoolSignet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -81,7 +81,7 @@ location @mempool-signet-api-v1-cache-forever { location @mempool-signet-api-v1-cache-warm { proxy_pass $mempoolSignet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -96,7 +96,7 @@ location @mempool-signet-api-v1-cache-warm { location @mempool-signet-api-v1-cache-normal { proxy_pass $mempoolSignet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -111,7 +111,7 @@ location @mempool-signet-api-v1-cache-normal { location @mempool-signet-api-v1-cache-disabled { proxy_pass $mempoolSignet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -125,7 +125,7 @@ location @mempool-signet-api-v1-cache-disabled { location @esplora-signet-api-cache-disabled { proxy_pass $esploraSignet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -139,7 +139,7 @@ location @esplora-signet-api-cache-disabled { location @esplora-signet-api-cache-forever { proxy_pass $esploraSignet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; diff --git a/production/nginx/location-testnet-api-v1-lightning.conf b/production/nginx/location-testnet-api-v1-lightning.conf index dbabc9aef..5319004ee 100644 --- a/production/nginx/location-testnet-api-v1-lightning.conf +++ b/production/nginx/location-testnet-api-v1-lightning.conf @@ -6,7 +6,7 @@ location /testnet/api/v1/lightning { location @mempool-testnet-api-v1-lightning { proxy_pass $mempoolSignetLightning; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; diff --git a/production/nginx/location-testnet-api.conf b/production/nginx/location-testnet-api.conf index 1f74aa533..656a705ff 100644 --- a/production/nginx/location-testnet-api.conf +++ b/production/nginx/location-testnet-api.conf @@ -53,7 +53,7 @@ location @mempool-testnet-api-v1-websocket { proxy_pass $mempoolTestnet; proxy_http_version 1.1; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Upgrade $http_upgrade; @@ -64,7 +64,7 @@ location @mempool-testnet-api-v1-websocket { location @mempool-testnet-api-v1-cache-forever { proxy_pass $mempoolTestnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -81,7 +81,7 @@ location @mempool-testnet-api-v1-cache-forever { location @mempool-testnet-api-v1-cache-warm { proxy_pass $mempoolTestnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -96,7 +96,7 @@ location @mempool-testnet-api-v1-cache-warm { location @mempool-testnet-api-v1-cache-normal { proxy_pass $mempoolTestnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -111,7 +111,7 @@ location @mempool-testnet-api-v1-cache-normal { location @mempool-testnet-api-v1-cache-disabled { proxy_pass $mempoolTestnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -125,7 +125,7 @@ location @mempool-testnet-api-v1-cache-disabled { location @esplora-testnet-api-cache-disabled { proxy_pass $esploraTestnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -139,7 +139,7 @@ location @esplora-testnet-api-cache-disabled { location @esplora-testnet-api-cache-forever { proxy_pass $esploraTestnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; diff --git a/production/nginx/server-bisq.conf b/production/nginx/server-bisq.conf index 704ef5a6c..2ea99843b 100644 --- a/production/nginx/server-bisq.conf +++ b/production/nginx/server-bisq.conf @@ -78,7 +78,7 @@ location @mempool-bisq-websocket { location @mempool-bisq { proxy_pass $mempoolBisq; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -89,7 +89,7 @@ location @mempool-bisq { location @esplora-api-cache-disabled { proxy_pass $esploraMainnet; - proxy_set_header Host $http_host; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme;