From 1b2122cd353a45cd5d8eb77077bc01058a859f69 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 18 Aug 2023 23:38:45 +0900 Subject: [PATCH 1/8] Don't overload core with mempool tx requests --- backend/src/api/transaction-utils.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index 02ee7c055..9fb633312 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -74,8 +74,18 @@ class TransactionUtils { public async $getMempoolTransactionsExtended(txids: string[], addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise { if (forceCore || config.MEMPOOL.BACKEND !== 'esplora') { - const results = await Promise.allSettled(txids.map(txid => this.$getTransactionExtended(txid, addPrevouts, lazyPrevouts, forceCore, true))); - return (results.filter(r => r.status === 'fulfilled') as PromiseFulfilledResult[]).map(r => r.value); + const results: MempoolTransactionExtended[] = []; + for (const txid of txids) { + try { + const result = await this.$getMempoolTransactionExtended(txid, addPrevouts, lazyPrevouts, forceCore); + if (result) { + results.push(result); + } + } catch { + // skip failures + } + } + return results; } else { const transactions = await bitcoinApi.$getMempoolTransactions(txids); return transactions.map(transaction => { From e4fcadf39b4bdfc8f0cba15b404f02cb58cb7ce5 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 19 Aug 2023 02:06:57 +0900 Subject: [PATCH 2/8] More verbose comments on $getMempoolTransactionsExtended --- backend/src/api/transaction-utils.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index 9fb633312..9b29784ce 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -82,7 +82,8 @@ class TransactionUtils { results.push(result); } } catch { - // skip failures + // we don't always expect to find a transaction for every txid + // so it's fine to silently skip failures } } return results; From 2a8a403da7078372ed9fcbc177df1cf6dac7ef36 Mon Sep 17 00:00:00 2001 From: junderw Date: Fri, 18 Aug 2023 23:42:13 -0700 Subject: [PATCH 3/8] Use p-limit to limit concurrent requests --- backend/src/api/transaction-utils.ts | 20 ++- backend/src/utils/p-limit.ts | 179 +++++++++++++++++++++++++++ 2 files changed, 186 insertions(+), 13 deletions(-) create mode 100644 backend/src/utils/p-limit.ts diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index 9b29784ce..00d8c9af3 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -5,6 +5,7 @@ import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; import * as bitcoinjs from 'bitcoinjs-lib'; import logger from '../logger'; import config from '../config'; +import pLimit from '../utils/p-limit'; class TransactionUtils { constructor() { } @@ -74,19 +75,12 @@ class TransactionUtils { public async $getMempoolTransactionsExtended(txids: string[], addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise { if (forceCore || config.MEMPOOL.BACKEND !== 'esplora') { - const results: MempoolTransactionExtended[] = []; - for (const txid of txids) { - try { - const result = await this.$getMempoolTransactionExtended(txid, addPrevouts, lazyPrevouts, forceCore); - if (result) { - results.push(result); - } - } catch { - // we don't always expect to find a transaction for every txid - // so it's fine to silently skip failures - } - } - return results; + const limiter = pLimit(32); // Run 32 requests at a time + const results = await Promise.allSettled(txids.map( + txid => limiter(() => this.$getMempoolTransactionExtended(txid, addPrevouts, lazyPrevouts, forceCore)) + )); + return results.filter(reply => reply.status === 'fulfilled') + .map(r => (r as PromiseFulfilledResult).value); } else { const transactions = await bitcoinApi.$getMempoolTransactions(txids); return transactions.map(transaction => { diff --git a/backend/src/utils/p-limit.ts b/backend/src/utils/p-limit.ts new file mode 100644 index 000000000..20cead411 --- /dev/null +++ b/backend/src/utils/p-limit.ts @@ -0,0 +1,179 @@ +/* +MIT License + +Copyright (c) Sindre Sorhus (https://sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, +merge, publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be included in all copies +or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +/* +How it works: +`this._head` is an instance of `Node` which keeps track of its current value and nests +another instance of `Node` that keeps the value that comes after it. When a value is +provided to `.enqueue()`, the code needs to iterate through `this._head`, going deeper +and deeper to find the last value. However, iterating through every single item is slow. +This problem is solved by saving a reference to the last value as `this._tail` so that +it can reference it to add a new value. +*/ + +class Node { + value; + next; + + constructor(value) { + this.value = value; + } +} + +class Queue { + private _head; + private _tail; + private _size; + + constructor() { + this.clear(); + } + + enqueue(value) { + const node = new Node(value); + + if (this._head) { + this._tail.next = node; + this._tail = node; + } else { + this._head = node; + this._tail = node; + } + + this._size++; + } + + dequeue() { + const current = this._head; + if (!current) { + return; + } + + this._head = this._head.next; + this._size--; + return current.value; + } + + clear() { + this._head = undefined; + this._tail = undefined; + this._size = 0; + } + + get size() { + return this._size; + } + + *[Symbol.iterator]() { + let current = this._head; + + while (current) { + yield current.value; + current = current.next; + } + } +} + +interface LimitFunction { + readonly activeCount: number; + readonly pendingCount: number; + clearQueue: () => void; + ( + fn: (...args: Arguments) => PromiseLike | ReturnType, + ...args: Arguments + ): Promise; +} + +export default function pLimit(concurrency: number): LimitFunction { + if ( + !( + (Number.isInteger(concurrency) || + concurrency === Number.POSITIVE_INFINITY) && + concurrency > 0 + ) + ) { + throw new TypeError('Expected `concurrency` to be a number from 1 and up'); + } + + const queue = new Queue(); + let activeCount = 0; + + const next = () => { + activeCount--; + + if (queue.size > 0) { + queue.dequeue()(); + } + }; + + const run = async (fn, resolve, args) => { + activeCount++; + + const result = (async () => fn(...args))(); + + resolve(result); + + try { + await result; + } catch {} + + next(); + }; + + const enqueue = (fn, resolve, args) => { + queue.enqueue(run.bind(undefined, fn, resolve, args)); + + (async () => { + // This function needs to wait until the next microtask before comparing + // `activeCount` to `concurrency`, because `activeCount` is updated asynchronously + // when the run function is dequeued and called. The comparison in the if-statement + // needs to happen asynchronously as well to get an up-to-date value for `activeCount`. + await Promise.resolve(); + + if (activeCount < concurrency && queue.size > 0) { + queue.dequeue()(); + } + })(); + }; + + const generator = (fn, ...args) => + new Promise((resolve) => { + enqueue(fn, resolve, args); + }); + + Object.defineProperties(generator, { + activeCount: { + get: () => activeCount, + }, + pendingCount: { + get: () => queue.size, + }, + clearQueue: { + value: () => { + queue.clear(); + }, + }, + }); + + return generator as any; +} From 2819cea50943c079846cb2fdfd225314444b1d07 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 19 Aug 2023 19:02:30 +0900 Subject: [PATCH 4/8] Reduce core mempool tx sync to 8 concurrent requests --- backend/src/api/transaction-utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index 00d8c9af3..ef4a34012 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -75,7 +75,7 @@ class TransactionUtils { public async $getMempoolTransactionsExtended(txids: string[], addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise { if (forceCore || config.MEMPOOL.BACKEND !== 'esplora') { - const limiter = pLimit(32); // Run 32 requests at a time + const limiter = pLimit(8); // Run 8 requests at a time const results = await Promise.allSettled(txids.map( txid => limiter(() => this.$getMempoolTransactionExtended(txid, addPrevouts, lazyPrevouts, forceCore)) )); From c1b2f1f2c7d575b65ca679aa55577f9e373df7a2 Mon Sep 17 00:00:00 2001 From: wiz Date: Fri, 25 Aug 2023 23:24:51 +0900 Subject: [PATCH 5/8] ops: Disable tor in prod install script --- production/install | 71 +++++++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/production/install b/production/install index 67e2f7d78..13449a904 100755 --- a/production/install +++ b/production/install @@ -1857,22 +1857,22 @@ ln -s "${MEMPOOL_HOME}/mempool" "${NGINX_ETC_FOLDER}/mempool" osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_USER__!${NGINX_USER}!" "${NGINX_CONFIGURATION}" osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_ETC_FOLDER__!${NGINX_ETC_FOLDER}!" "${NGINX_CONFIGURATION}" -if [ "${TOR_INSTALL}" = ON ];then - echo "[*] Read tor v3 onion hostnames" - - NGINX_MEMPOOL_ONION=$(cat "${TOR_RESOURCES}/mempool/hostname") - osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_MEMPOOL_ONION__!${NGINX_MEMPOOL_ONION%.onion}!" "${NGINX_CONFIGURATION}" - - if [ "${ELEMENTS_LIQUID_ENABLE}" = "ON" ];then - NGINX_LIQUID_ONION=$(cat "${TOR_RESOURCES}/liquid/hostname") - osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_LIQUID_ONION__!${NGINX_LIQUID_ONIONi%.onion}!" "${NGINX_CONFIGURATION}" - fi - - if [ "${BISQ_MAINNET_ENABLE}" = "ON" ];then - NGINX_BISQ_ONION=$(cat "${TOR_RESOURCES}/bisq/hostname") - osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_BISQ_ONION__!${NGINX_BISQ_ONION%.onion}!" "${NGINX_CONFIGURATION}" - fi -fi +#if [ "${TOR_INSTALL}" = ON ];then +# echo "[*] Read tor v3 onion hostnames" +# +# NGINX_MEMPOOL_ONION=$(cat "${TOR_RESOURCES}/mempool/hostname") +# osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_MEMPOOL_ONION__!${NGINX_MEMPOOL_ONION%.onion}!" "${NGINX_CONFIGURATION}" +# +# if [ "${ELEMENTS_LIQUID_ENABLE}" = "ON" ];then +# NGINX_LIQUID_ONION=$(cat "${TOR_RESOURCES}/liquid/hostname") +# osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_LIQUID_ONION__!${NGINX_LIQUID_ONIONi%.onion}!" "${NGINX_CONFIGURATION}" +# fi +# +# if [ "${BISQ_MAINNET_ENABLE}" = "ON" ];then +# NGINX_BISQ_ONION=$(cat "${TOR_RESOURCES}/bisq/hostname") +# osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_BISQ_ONION__!${NGINX_BISQ_ONION%.onion}!" "${NGINX_CONFIGURATION}" +# fi +#fi ##### OS systemd @@ -1896,13 +1896,26 @@ echo "[*] Updating system startup configuration" case $OS in FreeBSD) - echo 'nginx_enable="YES"' >> /etc/rc.conf - echo 'bitcoin_enable="YES"' >> /etc/rc.conf - echo 'tor_enable="YES"' >> /etc/rc.conf - echo 'postfix_enable="YES"' >> /etc/rc.conf - echo 'mysql_enable="YES"' >> /etc/rc.conf - echo 'mysql_dbdir="/mysql"' >> /etc/rc.conf - echo 'tor_enable="YES"' >> /etc/rc.conf + cat >> /etc/rc.conf < Date: Fri, 25 Aug 2023 23:29:03 +0900 Subject: [PATCH 6/8] ops: Increase FreeBSD bitcoin node shutdown timeout to 600 --- production/freebsd/rc.d/bitcoin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/freebsd/rc.d/bitcoin b/production/freebsd/rc.d/bitcoin index 76c217589..c22fbbabc 100644 --- a/production/freebsd/rc.d/bitcoin +++ b/production/freebsd/rc.d/bitcoin @@ -48,7 +48,7 @@ load_rc_config ${name} : ${bitcoin_syslog_facility:="local0"} : ${bitcoin_syslog_priority:="info"} : ${bitcoin_syslog_tag:="bitcoin"} -: ${bitcoin_kill_after:="300"} +: ${bitcoin_kill_after:="600"} : ${bitcoinlimits_args:="-e -U ${bitcoin_user}"} # set up dependant variables From 0fde6dd908864b741fa0adb82e6fb007a462a1a1 Mon Sep 17 00:00:00 2001 From: wiz Date: Fri, 25 Aug 2023 23:34:50 +0900 Subject: [PATCH 7/8] ops: Bump prod NodeJS version to v20.5.1 --- production/install | 8 +++----- production/mempool-start-all | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/production/install b/production/install index 13449a904..7ab1e657b 100755 --- a/production/install +++ b/production/install @@ -1045,11 +1045,9 @@ osSudo "${ROOT_USER}" crontab -u "${MEMPOOL_USER}" "${MEMPOOL_HOME}/${MEMPOOL_RE echo "[*] Installing nvm.sh from GitHub" osSudo "${MEMPOOL_USER}" sh -c 'curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | zsh' -echo "[*] Building NodeJS v20.4.0 via nvm.sh" -osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm install v20.4.0 --shared-zlib' -echo "[*] Building NodeJS v18.16.1 via nvm.sh" -osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm install v18.16.1 --shared-zlib' -osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm alias default 18.16.1' +echo "[*] Building NodeJS v20.5.1 via nvm.sh" +osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm install v20.5.1 --shared-zlib' +osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm alias default 20.5.1' #################### # Tor installation # diff --git a/production/mempool-start-all b/production/mempool-start-all index 7efbf7dd6..5b4f85577 100755 --- a/production/mempool-start-all +++ b/production/mempool-start-all @@ -1,7 +1,7 @@ #!/usr/bin/env zsh export NVM_DIR="$HOME/.nvm" source "$NVM_DIR/nvm.sh" -nvm use v20.4.0 +nvm use v20.5.1 # start all mempool backends that exist for site in mainnet mainnet-lightning testnet testnet-lightning signet signet-lightning bisq liquid liquidtestnet;do From dc44f1b618cf364a1eeddbb6c298cf1fca9ed47c Mon Sep 17 00:00:00 2001 From: wiz Date: Sat, 26 Aug 2023 04:40:12 +0900 Subject: [PATCH 8/8] ops: Fix WebGL for unfurler --- unfurler/puppeteer.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unfurler/puppeteer.config.json b/unfurler/puppeteer.config.json index 4d525755e..583ce7148 100644 --- a/unfurler/puppeteer.config.json +++ b/unfurler/puppeteer.config.json @@ -42,6 +42,6 @@ "--use-mock-keychain", "--ignore-gpu-blacklist", "--ignore-gpu-blocklist", - "--use-gl=egl" + "--use-angle=default" ] }