Merge pull request #2419 from mempool/wiz/add-unfurl-to-ops-scripts

[ops] Update prod scripts for unfurler and cache warmer

production/mempool-build-all

@@ -63,6 +63,19 @@ build_frontend()
     npm run build || exit 1
 }
 
+build_unfurler()
+{
+    local site="$1"
+    echo "[*] Building unfurler for ${site}"
+    [ -z "${HASH}" ] && exit 1
+    cd "$HOME/${site}/unfurler" || exit 1
+    if [ ! -e "config.json" ];then
+        cp "${HOME}/mempool/production/unfurler-config.${site}.json" "config.json"
+    fi
+    PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true npm install || exit 1
+    npm run build || exit 1
+}
+
 build_backend()
 {
     local site="$1"
@@ -128,6 +141,11 @@ for repo in $backend_repos;do
     update_repo "${repo}"
 done
 
+# build unfurlers
+for repo in mainnet liquid;do
+    build_unfurler "${repo}"
+done
+
 # build backends
 for repo in $backend_repos;do
     build_backend "${repo}"

production/mempool-kill-all

@@ -1,2 +1,8 @@
 #!/usr/bin/env zsh
-killall sh node
+killall sh
+killall node
+killall chrome
+killall xinit
+for pid in `ps uaxww|grep warmer|grep zsh|awk '{print $2}'`;do
+    kill $pid
+done

production/mempool-start-all

@@ -2,7 +2,29 @@
 export NVM_DIR="$HOME/.nvm"
 source "$NVM_DIR/nvm.sh"
 
+# start all mempool backends that exist
 for site in mainnet mainnet-lightning testnet testnet-lightning signet signet-lightning bisq liquid liquidtestnet;do
     cd "${HOME}/${site}/backend/" && \
+    echo "starting mempool backend: ${site}" && \
     screen -dmS "${site}" sh -c 'while true;do npm run start-production;sleep 1;done'
 done
+
+# only start unfurler if GPU present
+if pciconf -lv|grep -i nvidia >/dev/null 2>&1;then
+    export DISPLAY=:0
+    screen -dmS x startx
+    sleep 3
+    for site in mainnet liquid;do
+        cd "$HOME/${site}/unfurler" && \
+        echo "starting mempool unfurler: ${site}" && \
+        screen -dmS "unfurler-${site}" sh -c 'while true;do npm run unfurler;sleep 2;done'
+    done
+fi
+
+# start nginx warm cacher
+for site in mainnet;do
+    echo "starting mempool cache warmer: ${site}"
+    screen -dmS "warmer-${site}" $HOME/mempool/production/nginx-cache-warmer
+done
+
+exit 0

production/nginx-cache-warmer

@@ -2,6 +2,12 @@
 hostname=$(hostname)
 slugs=(`curl -sSL https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json | jq -r '.slugs[]'`)
 
+warm()
+{
+	echo "$1"
+	curl -i -s "$1" | head -1
+}
+
 while true
 do for url in / \
 	'/api/v1/blocks' \
@@ -81,14 +87,14 @@ do for url in / \
 	'/api/v1/lightning/channels-geo?style=graph' \
 
 	do
-		curl -s "https://${hostname}${url}" >/dev/null
+		warm "https://${hostname}${url}"
 	done
 
 	for slug in $slugs
 	do
-		curl -s "https://${hostname}/api/v1/mining/pool/${slug}" >/dev/null
+		warm "https://${hostname}/api/v1/mining/pool/${slug}"
-		curl -s "https://${hostname}/api/v1/mining/pool/${slug}/hashrate" >/dev/null
+		warm "https://${hostname}/api/v1/mining/pool/${slug}/hashrate"
-		curl -s "https://${hostname}/api/v1/mining/pool/${slug}/blocks" >/dev/null
+		warm "https://${hostname}/api/v1/mining/pool/${slug}/blocks"
 	done
 
 	sleep 10

production/unfurl-build

@@ -1,62 +0,0 @@
-#!/usr/bin/env zsh
-PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:$HOME/bin
-HOSTNAME=$(hostname)
-LOCATION=$(hostname|cut -d . -f2)
-LOCKFILE="${HOME}/lock"
-REF=$(echo "${1:=origin/master}"|sed -e 's!:!/!')
-
-if [ -f "${LOCKFILE}" ];then
-    echo "upgrade already running? check lockfile ${LOCKFILE}"
-    exit 1
-fi
-
-# on exit, remove lockfile but preserve exit code
-trap "rv=\$?; rm -f "${LOCKFILE}"; exit \$rv" INT TERM EXIT
-
-# create lockfile
-touch "${LOCKFILE}"
-
-# notify logged in users
-echo "Upgrading unfurler to ${REF}" | wall
-
-update_repo()
-{
-    echo "[*] Upgrading unfurler to ${REF}"
-    cd "$HOME/unfurl/unfurler" || exit 1
-
-    git fetch origin || exit 1
-    for remote in origin;do
-        git remote add "${remote}" "https://github.com/${remote}/mempool" >/dev/null 2>&1
-        git fetch "${remote}" || exit 1
-    done
-
-    if [ $(git tag -l "${REF}") ];then
-        git reset --hard "tags/${REF}" || exit 1
-    elif [ $(git branch -r -l "origin/${REF}") ];then
-        git reset --hard "origin/${REF}" || exit 1
-    else
-        git reset --hard "${REF}" || exit 1
-    fi
-    export HASH=$(git rev-parse HEAD)
-}
-
-build_backend()
-{
-    echo "[*] Building backend for unfurler"
-    [ -z "${HASH}" ] && exit 1
-    cd "$HOME/unfurl/unfurler" || exit 1
-    if [ ! -e "config.json" ];then
-        cp "${HOME}/unfurl/production/mempool-config.unfurl.json" "config.json"
-    fi
-    npm install || exit 1
-    npm run build || exit 1
-}
-
-update_repo
-build_backend
-
-# notify everyone
-echo "${HOSTNAME} unfurl updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general mempool.dev
-echo "${HOSTNAME} unfurl updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general "mempool.ops.${LOCATION}"
-
-exit 0

production/unfurl-kill

@@ -1,2 +0,0 @@
-#!/usr/bin/env zsh
-killall sh node

production/unfurl-start

@@ -1,6 +0,0 @@
-#!/usr/bin/env zsh
-export NVM_DIR="$HOME/.nvm"
-source "$NVM_DIR/nvm.sh"
-
-cd "${HOME}/unfurl/unfurler/" && \
-screen -dmS "unfurl" sh -c 'while true;do npm run start-production;sleep 1;done'

production/unfurler-config.liquid.json

@@ -0,0 +1,17 @@
+{
+  "SERVER": {
+    "HOST": "https://liquid.network",
+    "HTTP_PORT": 8002
+  },
+  "MEMPOOL": {
+    "HTTP_HOST": "https://liquid.network",
+    "HTTP_PORT": 443,
+    "NETWORK": "liquid"
+  },
+  "PUPPETEER": {
+    "CLUSTER_SIZE": 8,
+    "EXEC_PATH": "/usr/local/bin/chrome",
+    "MAX_PAGE_AGE": 86400,
+    "RENDER_TIMEOUT": 3000
+  }
+}

production/unfurler-config.mainnet.json

@@ -0,0 +1,17 @@
+{
+  "SERVER": {
+    "HOST": "https://mempool.space",
+    "HTTP_PORT": 8001
+  },
+  "MEMPOOL": {
+    "HTTP_HOST": "https://mempool.space",
+    "HTTP_PORT": 443,
+    "NETWORK": "bitcoin"
+  },
+  "PUPPETEER": {
+    "CLUSTER_SIZE": 8,
+    "EXEC_PATH": "/usr/local/bin/chrome",
+    "MAX_PAGE_AGE": 86400,
+    "RENDER_TIMEOUT": 3000
+  }
+}

unfurler/package.json

@@ -11,7 +11,7 @@
     "tsc": "./node_modules/typescript/bin/tsc",
     "build": "npm run tsc",
     "start": "node --max-old-space-size=2048 dist/index.js",
-    "start-production": "node --max-old-space-size=4096 dist/index.js",
+    "unfurler": "node --max-old-space-size=4096 dist/index.js",
     "lint": "./node_modules/.bin/eslint . --ext .ts",
     "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
     "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""

unfurler/puppeteer.config.json

@@ -41,6 +41,6 @@
     "--use-mock-keychain",
     "--ignore-gpu-blacklist",
     "--ignore-gpu-blocklist",
-    "--use-gl=swiftshader"
+    "--use-gl=egl"
   ]
 }