Merge branch 'master' into nymkappa/menu

backend/src/api/transaction-utils.ts

@@ -5,6 +5,7 @@ import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 import * as bitcoinjs from 'bitcoinjs-lib';
 import logger from '../logger';
 import config from '../config';
+import pLimit from '../utils/p-limit';
 
 class TransactionUtils {
   constructor() { }
@@ -74,8 +75,12 @@ class TransactionUtils {
 
   public async $getMempoolTransactionsExtended(txids: string[], addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise<MempoolTransactionExtended[]> {
     if (forceCore || config.MEMPOOL.BACKEND !== 'esplora') {
-      const results = await Promise.allSettled(txids.map(txid => this.$getTransactionExtended(txid, addPrevouts, lazyPrevouts, forceCore, true)));
+      const limiter = pLimit(8); // Run 8 requests at a time
-      return (results.filter(r => r.status === 'fulfilled') as PromiseFulfilledResult<MempoolTransactionExtended>[]).map(r => r.value);
+      const results = await Promise.allSettled(txids.map(
+        txid => limiter(() => this.$getMempoolTransactionExtended(txid, addPrevouts, lazyPrevouts, forceCore))
+      ));
+      return results.filter(reply => reply.status === 'fulfilled')
+        .map(r => (r as PromiseFulfilledResult<MempoolTransactionExtended>).value);
     } else {
       const transactions = await bitcoinApi.$getMempoolTransactions(txids);
       return transactions.map(transaction => {

backend/src/utils/p-limit.ts

@@ -0,0 +1,179 @@
+/*
+MIT License
+
+Copyright (c) Sindre Sorhus <sindresorhus@gmail.com> (https://sindresorhus.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this
+software and associated documentation files (the "Software"), to deal in the Software
+without restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be included in all copies
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
+OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+/*
+How it works:
+`this._head` is an instance of `Node` which keeps track of its current value and nests
+another instance of `Node` that keeps the value that comes after it. When a value is
+provided to `.enqueue()`, the code needs to iterate through `this._head`, going deeper
+and deeper to find the last value. However, iterating through every single item is slow.
+This problem is solved by saving a reference to the last value as `this._tail` so that
+it can reference it to add a new value.
+*/
+
+class Node {
+  value;
+  next;
+
+  constructor(value) {
+    this.value = value;
+  }
+}
+
+class Queue {
+  private _head;
+  private _tail;
+  private _size;
+
+  constructor() {
+    this.clear();
+  }
+
+  enqueue(value) {
+    const node = new Node(value);
+
+    if (this._head) {
+      this._tail.next = node;
+      this._tail = node;
+    } else {
+      this._head = node;
+      this._tail = node;
+    }
+
+    this._size++;
+  }
+
+  dequeue() {
+    const current = this._head;
+    if (!current) {
+      return;
+    }
+
+    this._head = this._head.next;
+    this._size--;
+    return current.value;
+  }
+
+  clear() {
+    this._head = undefined;
+    this._tail = undefined;
+    this._size = 0;
+  }
+
+  get size() {
+    return this._size;
+  }
+
+  *[Symbol.iterator]() {
+    let current = this._head;
+
+    while (current) {
+      yield current.value;
+      current = current.next;
+    }
+  }
+}
+
+interface LimitFunction {
+  readonly activeCount: number;
+  readonly pendingCount: number;
+  clearQueue: () => void;
+  <Arguments extends unknown[], ReturnType>(
+    fn: (...args: Arguments) => PromiseLike<ReturnType> | ReturnType,
+    ...args: Arguments
+  ): Promise<ReturnType>;
+}
+
+export default function pLimit(concurrency: number): LimitFunction {
+  if (
+    !(
+      (Number.isInteger(concurrency) ||
+        concurrency === Number.POSITIVE_INFINITY) &&
+      concurrency > 0
+    )
+  ) {
+    throw new TypeError('Expected `concurrency` to be a number from 1 and up');
+  }
+
+  const queue = new Queue();
+  let activeCount = 0;
+
+  const next = () => {
+    activeCount--;
+
+    if (queue.size > 0) {
+      queue.dequeue()();
+    }
+  };
+
+  const run = async (fn, resolve, args) => {
+    activeCount++;
+
+    const result = (async () => fn(...args))();
+
+    resolve(result);
+
+    try {
+      await result;
+    } catch {}
+
+    next();
+  };
+
+  const enqueue = (fn, resolve, args) => {
+    queue.enqueue(run.bind(undefined, fn, resolve, args));
+
+    (async () => {
+      // This function needs to wait until the next microtask before comparing
+      // `activeCount` to `concurrency`, because `activeCount` is updated asynchronously
+      // when the run function is dequeued and called. The comparison in the if-statement
+      // needs to happen asynchronously as well to get an up-to-date value for `activeCount`.
+      await Promise.resolve();
+
+      if (activeCount < concurrency && queue.size > 0) {
+        queue.dequeue()();
+      }
+    })();
+  };
+
+  const generator = (fn, ...args) =>
+    new Promise((resolve) => {
+      enqueue(fn, resolve, args);
+    });
+
+  Object.defineProperties(generator, {
+    activeCount: {
+      get: () => activeCount,
+    },
+    pendingCount: {
+      get: () => queue.size,
+    },
+    clearQueue: {
+      value: () => {
+        queue.clear();
+      },
+    },
+  });
+
+  return generator as any;
+}

production/freebsd/rc.d/bitcoin

@@ -48,7 +48,7 @@ load_rc_config ${name}
 : ${bitcoin_syslog_facility:="local0"}
 : ${bitcoin_syslog_priority:="info"}
 : ${bitcoin_syslog_tag:="bitcoin"}
-: ${bitcoin_kill_after:="300"}
+: ${bitcoin_kill_after:="600"}
 : ${bitcoinlimits_args:="-e -U ${bitcoin_user}"}
 
 # set up dependant variables

production/install

@@ -1045,11 +1045,9 @@ osSudo "${ROOT_USER}" crontab -u "${MEMPOOL_USER}" "${MEMPOOL_HOME}/${MEMPOOL_RE
 echo "[*] Installing nvm.sh from GitHub"
 osSudo "${MEMPOOL_USER}" sh -c 'curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | zsh'
 
-echo "[*] Building NodeJS v20.4.0 via nvm.sh"
+echo "[*] Building NodeJS v20.5.1 via nvm.sh"
-osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm install v20.4.0 --shared-zlib'
+osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm install v20.5.1 --shared-zlib'
-echo "[*] Building NodeJS v18.16.1 via nvm.sh"
+osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm alias default 20.5.1'
-osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm install v18.16.1 --shared-zlib'
-osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm alias default 18.16.1'
 
 ####################
 # Tor installation #
@@ -1857,22 +1855,22 @@ ln -s "${MEMPOOL_HOME}/mempool" "${NGINX_ETC_FOLDER}/mempool"
 osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_USER__!${NGINX_USER}!" "${NGINX_CONFIGURATION}"
 osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_ETC_FOLDER__!${NGINX_ETC_FOLDER}!" "${NGINX_CONFIGURATION}"
 
-if [ "${TOR_INSTALL}" = ON ];then
+#if [ "${TOR_INSTALL}" = ON ];then
-    echo "[*] Read tor v3 onion hostnames"
+#    echo "[*] Read tor v3 onion hostnames"
-
+#
-    NGINX_MEMPOOL_ONION=$(cat "${TOR_RESOURCES}/mempool/hostname")
+#    NGINX_MEMPOOL_ONION=$(cat "${TOR_RESOURCES}/mempool/hostname")
-    osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_MEMPOOL_ONION__!${NGINX_MEMPOOL_ONION%.onion}!" "${NGINX_CONFIGURATION}"
+#    osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_MEMPOOL_ONION__!${NGINX_MEMPOOL_ONION%.onion}!" "${NGINX_CONFIGURATION}"
-
+#
-    if [ "${ELEMENTS_LIQUID_ENABLE}" = "ON" ];then
+#    if [ "${ELEMENTS_LIQUID_ENABLE}" = "ON" ];then
-        NGINX_LIQUID_ONION=$(cat "${TOR_RESOURCES}/liquid/hostname")
+#        NGINX_LIQUID_ONION=$(cat "${TOR_RESOURCES}/liquid/hostname")
-        osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_LIQUID_ONION__!${NGINX_LIQUID_ONIONi%.onion}!" "${NGINX_CONFIGURATION}"
+#        osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_LIQUID_ONION__!${NGINX_LIQUID_ONIONi%.onion}!" "${NGINX_CONFIGURATION}"
-    fi
+#    fi
-
+#
-    if [ "${BISQ_MAINNET_ENABLE}" = "ON" ];then
+#    if [ "${BISQ_MAINNET_ENABLE}" = "ON" ];then
-        NGINX_BISQ_ONION=$(cat "${TOR_RESOURCES}/bisq/hostname")
+#        NGINX_BISQ_ONION=$(cat "${TOR_RESOURCES}/bisq/hostname")
-        osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_BISQ_ONION__!${NGINX_BISQ_ONION%.onion}!" "${NGINX_CONFIGURATION}"
+#        osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_BISQ_ONION__!${NGINX_BISQ_ONION%.onion}!" "${NGINX_CONFIGURATION}"
-    fi
+#    fi
-fi
+#fi
 
 ##### OS systemd
 
@@ -1896,13 +1894,26 @@ echo "[*] Updating system startup configuration"
 case $OS in
 
     FreeBSD)
-        echo 'nginx_enable="YES"' >> /etc/rc.conf
+        cat >> /etc/rc.conf <<EOF
-        echo 'bitcoin_enable="YES"' >> /etc/rc.conf
+moused_nondefault_enable="NO"
-        echo 'tor_enable="YES"' >> /etc/rc.conf
+
-        echo 'postfix_enable="YES"' >> /etc/rc.conf
+nginx_enable="YES"
-        echo 'mysql_enable="YES"' >> /etc/rc.conf
+nginx_profiles="mempool"
-        echo 'mysql_dbdir="/mysql"' >> /etc/rc.conf
+nginx_mempool_flags="-p /mempool"
-        echo 'tor_enable="YES"' >> /etc/rc.conf
+nginx_mempool_configfile="/mempool/mempool/nginx/nginx.conf"
+
+mysql_enable="YES"
+mysql_dbdir="/mysql"
+mysql_args="--innodb-buffer-pool-size=8G --bind-address 127.0.0.1"
+
+kld_list="nvidia"
+nvidia_xorg_enable="YES"
+
+dbus_enable="YES"
+tor_enable="YES"
+bitcoin_enable="YES"
+postfix_enable="YES"
+EOF
     ;;
 
     Debian)
@@ -2038,12 +2049,12 @@ osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME} && ./upgrade" || true
 
 ##### finish
 
-if [ "${TOR_INSTALL}" = ON ];then
+#if [ "${TOR_INSTALL}" = ON ];then
-    echo "Your auto-generated Tor addresses are:"
+#    echo "Your auto-generated Tor addresses are:"
-    echo "${NGINX_MEMPOOL_ONION}"
+#    echo "${NGINX_MEMPOOL_ONION}"
-    echo "${NGINX_BISQ_ONION}"
+#    echo "${NGINX_BISQ_ONION}"
-    echo "${NGINX_LIQUID_ONION}"
+#    echo "${NGINX_LIQUID_ONION}"
-fi
+#fi
 
 echo
 echo 'Please reboot to start all the services.'

production/mempool-start-all

@@ -1,7 +1,7 @@
 #!/usr/bin/env zsh
 export NVM_DIR="$HOME/.nvm"
 source "$NVM_DIR/nvm.sh"
-nvm use v20.4.0
+nvm use v20.5.1
 
 # start all mempool backends that exist
 for site in mainnet mainnet-lightning testnet testnet-lightning signet signet-lightning bisq liquid liquidtestnet;do

unfurler/puppeteer.config.json

@@ -42,6 +42,6 @@
     "--use-mock-keychain",
     "--ignore-gpu-blacklist",
     "--ignore-gpu-blocklist",
-    "--use-gl=egl"
+    "--use-angle=default"
   ]
 }