Merge branch 'master' into alt-tx-unfurls

backend/src/tasks/lightning/sync-tasks/stats-importer.ts

@@ -8,6 +8,7 @@ import { isIP } from 'net';
 import { Common } from '../../../api/common';
 import channelsApi from '../../../api/explorer/channels.api';
 import nodesApi from '../../../api/explorer/nodes.api';
+import { ResultSetHeader } from 'mysql2';
 
 const fsPromises = promises;
 
@@ -20,6 +21,7 @@ class LightningStatsImporter {
     await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
 
     await this.$importHistoricalLightningStats();
+    await this.$cleanupIncorrectSnapshot();
   }
 
   /**
@@ -367,6 +369,12 @@ class LightningStatsImporter {
           continue;
         }
     
+        if (this.isIncorrectSnapshot(timestamp, graph)) {
+          logger.debug(`Ignoring ${this.topologiesFolder}/${filename}, because we defined it as an incorrect snapshot`);
+          ++totalProcessed;
+          continue;
+        }
+
         if (!logStarted) {
           logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`);
           logStarted = true;
@@ -397,7 +405,7 @@ class LightningStatsImporter {
     }
   }
 
-  async cleanupTopology(graph) {
+  cleanupTopology(graph): ILightningApi.NetworkGraph {
     const newGraph = {
       nodes: <ILightningApi.Node[]>[],
       edges: <ILightningApi.Channel[]>[],
@@ -456,6 +464,69 @@ class LightningStatsImporter {
 
     return newGraph;
   }
+
+  private isIncorrectSnapshot(timestamp, graph): boolean {
+    if (timestamp >= 1549065600 /* 2019-02-02 */ && timestamp <= 1550620800 /* 2019-02-20 */ && graph.nodes.length < 2600) {
+        return true;
+    }
+    if (timestamp >= 1552953600 /* 2019-03-19 */ && timestamp <= 1556323200 /* 2019-05-27 */ && graph.nodes.length < 4000) {
+      return true;
+    }
+    if (timestamp >= 1557446400 /* 2019-05-10 */ && timestamp <= 1560470400 /* 2019-06-14 */ && graph.nodes.length < 4000) {
+      return true;
+    }
+    if (timestamp >= 1561680000 /* 2019-06-28 */ && timestamp <= 1563148800 /* 2019-07-15 */ && graph.nodes.length < 4000) {
+      return true;
+    }
+    if (timestamp >= 1571270400 /* 2019-11-17 */ && timestamp <= 1580601600 /* 2020-02-02 */ && graph.nodes.length < 4500) {
+      return true;
+    }
+    if (timestamp >= 1591142400 /* 2020-06-03 */ && timestamp <= 1592006400 /* 2020-06-13 */ && graph.nodes.length < 5500) {
+      return true;
+    }
+    if (timestamp >= 1632787200 /* 2021-09-28 */ && timestamp <= 1633564800 /* 2021-10-07 */ && graph.nodes.length < 13000) {
+      return true;
+    }
+    if (timestamp >= 1634256000 /* 2021-10-15 */ && timestamp <= 1645401600 /* 2022-02-21 */ && graph.nodes.length < 17000) {
+      return true;
+    }
+    if (timestamp >= 1654992000 /* 2022-06-12 */ && timestamp <= 1661472000 /* 2022-08-26 */ && graph.nodes.length < 14000) {
+      return true;
+    }
+
+    return false;
+  }
+
+  private async $cleanupIncorrectSnapshot(): Promise<void> {
+    // We do not run this one automatically because those stats are not supposed to be inserted in the first
+    // place, but I write them here to remind us we manually run those queries
+
+    // DELETE FROM lightning_stats
+    // WHERE (
+    //   UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 AND node_count < 2600 OR
+    //   UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 AND node_count < 4000 OR
+    //   UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 AND node_count < 4000 OR
+    //   UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 AND node_count < 4000 OR
+    //   UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 AND node_count < 4500 OR
+    //   UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 AND node_count < 5500 OR
+    //   UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 AND node_count < 13000 OR
+    //   UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 AND node_count < 17000 OR
+    //   UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000 AND node_count < 14000
+    // )
+
+    // DELETE FROM node_stats
+    // WHERE (
+    //   UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 OR
+    //   UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 OR
+    //   UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 OR
+    //   UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 OR
+    //   UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 OR
+    //   UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 OR
+    //   UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 OR
+    //   UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 OR
+    //   UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000 
+    // )
+  }
 }
 
 export default new LightningStatsImporter;

frontend/src/app/lightning/channel/channel-preview.component.html

@@ -58,7 +58,7 @@
       </table>
     </div>
     <div class="col-md map-col">
-      <app-nodes-channels-map *ngIf="!error" [style]="'channelpage'" [channel]="channelGeo" [fitContainer]="true" (readyEvent)="onMapReady()"></app-nodes-channels-map>
+      <app-nodes-channels-map *ngIf="!error" [style]="'channelpage'" [channel]="channelGeo" [fitContainer]="true" [placeholder]="true" (readyEvent)="onMapReady()"></app-nodes-channels-map>
     </div>
   </div>
 </div>

frontend/src/app/lightning/node/node-preview.component.html

@@ -52,7 +52,7 @@
       </table>
     </div>
     <div class="col-md map-col">
-      <app-nodes-channels-map *ngIf="!error" [style]="'nodepage'" [publicKey]="node.public_key" [fitContainer]="true" (readyEvent)="onMapReady()"></app-nodes-channels-map>
+      <app-nodes-channels-map *ngIf="!error" [style]="'nodepage'" [publicKey]="node.public_key" [fitContainer]="true" [placeholder]="true" [hasLocation]="!!node.as_number" (readyEvent)="onMapReady()"></app-nodes-channels-map>
     </div>
   </div>
 </div>

frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts

@@ -22,6 +22,7 @@ export class NodesChannelsMap implements OnInit {
   @Input() channel: any[] = [];
   @Input() fitContainer = false;
   @Input() hasLocation = true;
+  @Input() placeholder = false;
   @Output() readyEvent = new EventEmitter();
 
   channelsObservable: Observable<any>; 
@@ -201,11 +202,26 @@ export class NodesChannelsMap implements OnInit {
 
   prepareChartOptions(nodes, channels) {
     let title: object;
-    if (channels.length === 0) {
+    if (channels.length === 0 && !this.placeholder) {
       this.chartOptions = null;
       return;
     }
 
+    // empty map fallback
+    if (channels.length === 0 && this.placeholder) {
+      title = {
+        textStyle: {
+          color: 'white',
+          fontSize: 18
+        },
+        text: $localize`No geolocation data available`,
+        left: 'center',
+        top: 'center'
+      };
+      this.zoom = 1.5;
+      this.center = [0, 20];
+    }
+
     this.chartOptions = {
       silent: this.style === 'widget',
       title: title ?? undefined,

frontend/src/app/services/state.service.ts

@@ -153,7 +153,12 @@ export class StateService {
     if (this.env.BASE_MODULE !== 'mempool' && this.env.BASE_MODULE !== 'liquid') {
       return;
     }
-    const networkMatches = url.match(/^\/(bisq|testnet|liquidtestnet|liquid|signet)/);
+    // horrible network regex breakdown:
+    // /^\/                                         starts with a forward slash...
+    // (?:[a-z]{2}(?:-[A-Z]{2})?\/)?                optional locale prefix (non-capturing)
+    // (?:preview\/)?                               optional "preview" prefix (non-capturing)
+    // (bisq|testnet|liquidtestnet|liquid|signet)/  network string (captured as networkMatches[1])
+    const networkMatches = url.match(/^\/(?:[a-z]{2}(?:-[A-Z]{2})?\/)?(?:preview\/)?(bisq|testnet|liquidtestnet|liquid|signet)/);
     switch (networkMatches && networkMatches[1]) {
       case 'liquid':
         if (this.network !== 'liquid') {

production/install

@@ -1287,9 +1287,9 @@ case $OS in
         osPackageInstall ${CLN_PKG}
 
         echo "[*] Installing Core Lightning mainnet Cronjob"
-        crontab_cln+='@reboot sleep 30 ; screen -dmS main lightningd --alias `hostname` --bitcoin-datadir /bitcoin\n'
+        crontab_cln+='@reboot sleep 60 ; screen -dmS main lightningd --alias `hostname` --bitcoin-datadir /bitcoin\n'
-        crontab_cln+='@reboot sleep 60 ; screen -dmS sig lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network signet\n'
         crontab_cln+='@reboot sleep 90 ; screen -dmS tes lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network testnet\n'
+        crontab_cln+='@reboot sleep 120 ; screen -dmS sig lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network signet\n'
         echo "${crontab_cln}" | crontab -u "${CLN_USER}" -
     ;;
     Debian)

production/mempool-build-all

@@ -63,6 +63,19 @@ build_frontend()
     npm run build || exit 1
 }
 
+build_unfurler()
+{
+    local site="$1"
+    echo "[*] Building unfurler for ${site}"
+    [ -z "${HASH}" ] && exit 1
+    cd "$HOME/${site}/unfurler" || exit 1
+    if [ ! -e "config.json" ];then
+        cp "${HOME}/mempool/production/unfurler-config.${site}.json" "config.json"
+    fi
+    PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true npm install || exit 1
+    npm run build || exit 1
+}
+
 build_backend()
 {
     local site="$1"
@@ -128,6 +141,11 @@ for repo in $backend_repos;do
     update_repo "${repo}"
 done
 
+# build unfurlers
+for repo in mainnet liquid;do
+    build_unfurler "${repo}"
+done
+
 # build backends
 for repo in $backend_repos;do
     build_backend "${repo}"

production/mempool-kill-all

@@ -1,2 +1,8 @@
 #!/usr/bin/env zsh
-killall sh node
+killall sh
+killall node
+killall chrome
+killall xinit
+for pid in `ps uaxww|grep warmer|grep zsh|awk '{print $2}'`;do
+    kill $pid
+done

production/mempool-start-all

@@ -2,7 +2,29 @@
 export NVM_DIR="$HOME/.nvm"
 source "$NVM_DIR/nvm.sh"
 
+# start all mempool backends that exist
 for site in mainnet mainnet-lightning testnet testnet-lightning signet signet-lightning bisq liquid liquidtestnet;do
     cd "${HOME}/${site}/backend/" && \
+    echo "starting mempool backend: ${site}" && \
     screen -dmS "${site}" sh -c 'while true;do npm run start-production;sleep 1;done'
 done
+
+# only start unfurler if GPU present
+if pciconf -lv|grep -i nvidia >/dev/null 2>&1;then
+    export DISPLAY=:0
+    screen -dmS x startx
+    sleep 3
+    for site in mainnet liquid;do
+        cd "$HOME/${site}/unfurler" && \
+        echo "starting mempool unfurler: ${site}" && \
+        screen -dmS "unfurler-${site}" sh -c 'while true;do npm run unfurler;sleep 2;done'
+    done
+fi
+
+# start nginx warm cacher
+for site in mainnet;do
+    echo "starting mempool cache warmer: ${site}"
+    screen -dmS "warmer-${site}" $HOME/mempool/production/nginx-cache-warmer
+done
+
+exit 0

production/nginx-cache-warmer

@@ -2,6 +2,12 @@
 hostname=$(hostname)
 slugs=(`curl -sSL https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json | jq -r '.slugs[]'`)
 
+warm()
+{
+	echo "$1"
+	curl -i -s "$1" | head -1
+}
+
 while true
 do for url in / \
 	'/api/v1/blocks' \
@@ -81,14 +87,14 @@ do for url in / \
 	'/api/v1/lightning/channels-geo?style=graph' \
 
 	do
-		curl -s "https://${hostname}${url}" >/dev/null
+		warm "https://${hostname}${url}"
 	done
 
 	for slug in $slugs
 	do
-		curl -s "https://${hostname}/api/v1/mining/pool/${slug}" >/dev/null
+		warm "https://${hostname}/api/v1/mining/pool/${slug}"
-		curl -s "https://${hostname}/api/v1/mining/pool/${slug}/hashrate" >/dev/null
+		warm "https://${hostname}/api/v1/mining/pool/${slug}/hashrate"
-		curl -s "https://${hostname}/api/v1/mining/pool/${slug}/blocks" >/dev/null
+		warm "https://${hostname}/api/v1/mining/pool/${slug}/blocks"
 	done
 
 	sleep 10

production/unfurl-build

@@ -1,62 +0,0 @@
-#!/usr/bin/env zsh
-PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:$HOME/bin
-HOSTNAME=$(hostname)
-LOCATION=$(hostname|cut -d . -f2)
-LOCKFILE="${HOME}/lock"
-REF=$(echo "${1:=origin/master}"|sed -e 's!:!/!')
-
-if [ -f "${LOCKFILE}" ];then
-    echo "upgrade already running? check lockfile ${LOCKFILE}"
-    exit 1
-fi
-
-# on exit, remove lockfile but preserve exit code
-trap "rv=\$?; rm -f "${LOCKFILE}"; exit \$rv" INT TERM EXIT
-
-# create lockfile
-touch "${LOCKFILE}"
-
-# notify logged in users
-echo "Upgrading unfurler to ${REF}" | wall
-
-update_repo()
-{
-    echo "[*] Upgrading unfurler to ${REF}"
-    cd "$HOME/unfurl/unfurler" || exit 1
-
-    git fetch origin || exit 1
-    for remote in origin;do
-        git remote add "${remote}" "https://github.com/${remote}/mempool" >/dev/null 2>&1
-        git fetch "${remote}" || exit 1
-    done
-
-    if [ $(git tag -l "${REF}") ];then
-        git reset --hard "tags/${REF}" || exit 1
-    elif [ $(git branch -r -l "origin/${REF}") ];then
-        git reset --hard "origin/${REF}" || exit 1
-    else
-        git reset --hard "${REF}" || exit 1
-    fi
-    export HASH=$(git rev-parse HEAD)
-}
-
-build_backend()
-{
-    echo "[*] Building backend for unfurler"
-    [ -z "${HASH}" ] && exit 1
-    cd "$HOME/unfurl/unfurler" || exit 1
-    if [ ! -e "config.json" ];then
-        cp "${HOME}/unfurl/production/mempool-config.unfurl.json" "config.json"
-    fi
-    npm install || exit 1
-    npm run build || exit 1
-}
-
-update_repo
-build_backend
-
-# notify everyone
-echo "${HOSTNAME} unfurl updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general mempool.dev
-echo "${HOSTNAME} unfurl updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general "mempool.ops.${LOCATION}"
-
-exit 0

production/unfurl-kill

@@ -1,2 +0,0 @@
-#!/usr/bin/env zsh
-killall sh node

production/unfurl-start

@@ -1,6 +0,0 @@
-#!/usr/bin/env zsh
-export NVM_DIR="$HOME/.nvm"
-source "$NVM_DIR/nvm.sh"
-
-cd "${HOME}/unfurl/unfurler/" && \
-screen -dmS "unfurl" sh -c 'while true;do npm run start-production;sleep 1;done'

production/unfurler-config.liquid.json

@@ -0,0 +1,17 @@
+{
+  "SERVER": {
+    "HOST": "https://liquid.network",
+    "HTTP_PORT": 8002
+  },
+  "MEMPOOL": {
+    "HTTP_HOST": "https://liquid.network",
+    "HTTP_PORT": 443,
+    "NETWORK": "liquid"
+  },
+  "PUPPETEER": {
+    "CLUSTER_SIZE": 8,
+    "EXEC_PATH": "/usr/local/bin/chrome",
+    "MAX_PAGE_AGE": 86400,
+    "RENDER_TIMEOUT": 3000
+  }
+}

production/unfurler-config.mainnet.json

@@ -0,0 +1,17 @@
+{
+  "SERVER": {
+    "HOST": "https://mempool.space",
+    "HTTP_PORT": 8001
+  },
+  "MEMPOOL": {
+    "HTTP_HOST": "https://mempool.space",
+    "HTTP_PORT": 443,
+    "NETWORK": "bitcoin"
+  },
+  "PUPPETEER": {
+    "CLUSTER_SIZE": 8,
+    "EXEC_PATH": "/usr/local/bin/chrome",
+    "MAX_PAGE_AGE": 86400,
+    "RENDER_TIMEOUT": 3000
+  }
+}

unfurler/package.json

@@ -11,7 +11,7 @@
     "tsc": "./node_modules/typescript/bin/tsc",
     "build": "npm run tsc",
     "start": "node --max-old-space-size=2048 dist/index.js",
-    "start-production": "node --max-old-space-size=4096 dist/index.js",
+    "unfurler": "node --max-old-space-size=4096 dist/index.js",
     "lint": "./node_modules/.bin/eslint . --ext .ts",
     "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
     "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""

unfurler/puppeteer.config.json

@@ -41,6 +41,6 @@
     "--use-mock-keychain",
     "--ignore-gpu-blacklist",
     "--ignore-gpu-blocklist",
-    "--use-gl=swiftshader"
+    "--use-gl=egl"
   ]
 }