Merge pull request #2585 from mempool/simon/search-bar-placeholder-update

Updating search bar placeholder

.github/FUNDING.yml

@@ -1,12 +0,0 @@
-# These are supported funding model platforms
-
-github: ['mempool'] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
-patreon: # Replace with a single Patreon username
-open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-otechie: # Replace with a single Otechie username
-custom: ['https://mempool.space/sponsor'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/workflows/ci.yml

@@ -42,8 +42,10 @@ jobs:
         run: npm run lint
         working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
 
-      #  - name: Test
-      #    run: npm run test
+      - name: Unit Tests
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm run test
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
 
       - name: Build
         run: npm run build

.github/workflows/on-tag.yml

@@ -1,7 +1,7 @@
 name: Docker build on tag
 env:
   DOCKER_CLI_EXPERIMENTAL: enabled
-  TAG_FMT: '^refs/tags/(((.?[0-9]+){3,4}))$'
+  TAG_FMT: "^refs/tags/(((.?[0-9]+){3,4}))$"
   DOCKER_BUILDKIT: 0
   COMPOSE_DOCKER_CLI_BUILD: 0
 
@@ -21,16 +21,46 @@ jobs:
         service:
           - frontend
           - backend
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
     name: Build and push to DockerHub
     steps:
+      # Workaround based on JonasAlfredsson/docker-on-tmpfs@v1.0.1
+      - name: Replace the current swap file
+        shell: bash
+        run: |
+          sudo swapoff /mnt/swapfile
+          sudo rm -v /mnt/swapfile
+          sudo fallocate -l 10G /mnt/swapfile
+          sudo chmod 600 /mnt/swapfile
+          sudo mkswap /mnt/swapfile
+          sudo swapon /mnt/swapfile
+
+      - name: Show current memory and swap status
+        shell: bash
+        run: |
+          sudo free -h
+          echo
+          sudo swapon --show
+
+      - name: Mount a tmpfs over /var/lib/docker
+        shell: bash
+        run: |
+          if [ ! -d "/var/lib/docker" ]; then
+            echo "Directory '/var/lib/docker' not found"
+            exit 1
+          fi
+          sudo mount -t tmpfs -o size=10G tmpfs /var/lib/docker
+          sudo systemctl restart docker
+          sudo df -h | grep docker
+
       - name: Set env variables
         run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
 
       - name: Show set environment variables
         run: |
           printf "      TAG: %s\n"  "$TAG"
-      
+
       - name: Add SHORT_SHA env property with commit short sha
         run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV
 

backend/.eslintrc

@@ -31,6 +31,7 @@
     "prefer-const": 1,
     "prefer-rest-params": 1,
     "quotes": [1, "single", { "allowTemplateLiterals": true }],
-    "semi": 1
+    "semi": 1,
+    "eqeqeq": 1
   }
 }

backend/.gitignore

@@ -1,7 +1,8 @@
 # See http://help.github.com/ignore-files/ for more about ignoring files.
 
 # production config and external assets
-
+!mempool-config.template.json
+!mempool-config.sample.json
 mempool-config.json
 pools.json
 icons.json

backend/README.md

@@ -110,6 +110,11 @@ Run the Mempool backend:
 
 ```
 npm run start
+
+```
+You can also set env var `MEMPOOL_CONFIG_FILE` to specify a custom config file location:
+```
+MEMPOOL_CONFIG_FILE=/path/to/mempool-config.json npm run start
 ```
 
 When it's running, you should see output like this:

backend/jest.config.ts

@@ -0,0 +1,20 @@
+import type { Config } from "@jest/types"
+
+const config: Config.InitialOptions = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  verbose: true,
+  automock: false,
+  collectCoverage: true,
+  collectCoverageFrom: ["./src/**/**.ts"],
+  coverageProvider: "babel",
+  coverageThreshold: {
+    global: {
+      lines: 1
+    }
+  },
+  setupFiles: [
+    "./testSetup.ts",
+  ],
+}
+export default config;

backend/mempool-config.sample.json

@@ -21,7 +21,9 @@
     "EXTERNAL_RETRY_INTERVAL": 0,
     "USER_AGENT": "mempool",
     "STDOUT_LOG_MIN_PRIORITY": "debug",
-    "AUTOMATIC_BLOCK_REINDEXING": false
+    "AUTOMATIC_BLOCK_REINDEXING": false,
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
+    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master"
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",
@@ -75,12 +77,18 @@
   },
   "LIGHTNING": {
     "ENABLED": false,
-    "BACKEND": "lnd"
+    "BACKEND": "lnd",
+    "STATS_REFRESH_INTERVAL": 600,
+    "GRAPH_REFRESH_INTERVAL": 600,
+    "LOGGER_UPDATE_INTERVAL": 30
   },
   "LND": {
     "TLS_CERT_PATH": "tls.cert",
-    "MACAROON_PATH": "admin.macaroon",
-    "SOCKET": "localhost:10009"
+    "MACAROON_PATH": "readonly.macaroon",
+    "REST_API_URL": "https://localhost:8080"
+  },
+  "CLIGHTNING": {
+    "SOCKET": "lightning-rpc"
   },
   "SOCKS5PROXY": {
     "ENABLED": false,

backend/package.json

@@ -16,28 +16,31 @@
     "mempool",
     "blockchain",
     "explorer",
-    "liquid"
+    "liquid",
+    "lightning"
   ],
   "main": "index.ts",
   "scripts": {
-    "tsc": "./node_modules/typescript/bin/tsc",
-    "build": "npm run tsc",
+    "tsc": "./node_modules/typescript/bin/tsc -p tsconfig.build.json",
+    "build": "npm run tsc && npm run create-resources",
+    "create-resources": "cp ./src/tasks/price-feeds/mtgox-weekly.json ./dist/tasks && node dist/api/fetch-version.js",
+    "package": "npm run build && rm -rf package && mv dist package && mv node_modules package && npm run package-rm-build-deps",
+    "package-rm-build-deps": "(cd package/node_modules; rm -r typescript @typescript-eslint)",
     "start": "node --max-old-space-size=2048 dist/index.js",
     "start-production": "node --max-old-space-size=4096 dist/index.js",
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test": "./node_modules/.bin/jest --coverage",
     "lint": "./node_modules/.bin/eslint . --ext .ts",
     "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
     "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
   },
   "dependencies": {
+    "@babel/core": "^7.18.6",
     "@mempool/electrum-client": "^1.1.7",
     "@types/node": "^16.11.41",
     "axios": "~0.27.2",
-    "bitcoinjs-lib": "6.0.1",
-    "bolt07": "^1.8.1",
+    "bitcoinjs-lib": "6.0.2",
     "crypto-js": "^4.0.0",
     "express": "^4.18.0",
-    "lightning": "^5.16.3",
     "maxmind": "^4.3.6",
     "mysql2": "2.3.3",
     "node-worker-threads-pool": "^1.5.1",
@@ -46,14 +49,20 @@
     "ws": "~8.8.0"
   },
   "devDependencies": {
+    "@babel/core": "^7.18.6",
+    "@babel/code-frame": "^7.18.6",
     "@types/compression": "^1.7.2",
     "@types/crypto-js": "^4.1.1",
     "@types/express": "^4.17.13",
+    "@types/jest": "^28.1.4",
     "@types/ws": "~8.5.3",
     "@typescript-eslint/eslint-plugin": "^5.30.5",
     "@typescript-eslint/parser": "^5.30.5",
     "eslint": "^8.19.0",
     "eslint-config-prettier": "^8.5.0",
-    "prettier": "^2.7.1"
+    "jest": "^28.1.2",
+    "prettier": "^2.7.1",
+    "ts-jest": "^28.0.5",
+    "ts-node": "^10.8.2"
   }
 }

backend/src/__fixtures__/mempool-config.template.json

@@ -0,0 +1,108 @@
+{
+  "MEMPOOL": {
+    "NETWORK": "__MEMPOOL_NETWORK__",
+    "BACKEND": "__MEMPOOL_BACKEND__",
+    "BLOCKS_SUMMARIES_INDEXING": true,
+    "HTTP_PORT": 1,
+    "SPAWN_CLUSTER_PROCS": 2,
+    "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
+    "AUTOMATIC_BLOCK_REINDEXING": true,
+    "POLL_RATE_MS": 3,
+    "CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
+    "CLEAR_PROTECTION_MINUTES": 4,
+    "RECOMMENDED_FEE_PERCENTILE": 5,
+    "BLOCK_WEIGHT_UNITS": 6,
+    "INITIAL_BLOCKS_AMOUNT": 7,
+    "MEMPOOL_BLOCKS_AMOUNT": 8,
+    "PRICE_FEED_UPDATE_INTERVAL": 9,
+    "USE_SECOND_NODE_FOR_MINFEE": 10,
+    "EXTERNAL_ASSETS": 11,
+    "EXTERNAL_MAX_RETRY": 12,
+    "EXTERNAL_RETRY_INTERVAL": 13,
+    "USER_AGENT": "__MEMPOOL_USER_AGENT__",
+    "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
+    "INDEXING_BLOCKS_AMOUNT": 14,
+    "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
+    "POOLS_JSON_URL": "__POOLS_JSON_URL__"
+  },
+  "CORE_RPC": {
+    "HOST": "__CORE_RPC_HOST__",
+    "PORT": 15,
+    "USERNAME": "__CORE_RPC_USERNAME__",
+    "PASSWORD": "__CORE_RPC_PASSWORD__"
+  },
+  "ELECTRUM": {
+    "HOST": "__ELECTRUM_HOST__",
+    "PORT": 16,
+    "TLS_ENABLED": true
+  },
+  "ESPLORA": {
+    "REST_API_URL": "__ESPLORA_REST_API_URL__"
+  },
+  "SECOND_CORE_RPC": {
+    "HOST": "__SECOND_CORE_RPC_HOST__",
+    "PORT": 17,
+    "USERNAME": "__SECOND_CORE_RPC_USERNAME__",
+    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__"
+  },
+  "DATABASE": {
+    "ENABLED": false,
+    "HOST": "__DATABASE_HOST__",
+    "SOCKET": "__DATABASE_SOCKET__",
+    "PORT": 18,
+    "DATABASE": "__DATABASE_DATABASE__",
+    "USERNAME": "__DATABASE_USERNAME__",
+    "PASSWORD": "__DATABASE_PASSWORD__"
+  },
+  "SYSLOG": {
+    "ENABLED": false,
+    "HOST": "__SYSLOG_HOST__",
+    "PORT": 19,
+    "MIN_PRIORITY": "__SYSLOG_MIN_PRIORITY__",
+    "FACILITY": "__SYSLOG_FACILITY__"
+  },
+  "STATISTICS": {
+    "ENABLED": false,
+    "TX_PER_SECOND_SAMPLE_PERIOD": 20
+  },
+  "BISQ": {
+    "ENABLED": true,
+    "DATA_PATH": "__BISQ_DATA_PATH__"
+  },
+  "SOCKS5PROXY": {
+    "ENABLED": true,
+    "USE_ONION": true,
+    "HOST": "__SOCKS5PROXY_HOST__",
+    "PORT": "__SOCKS5PROXY_PORT__",
+    "USERNAME": "__SOCKS5PROXY_USERNAME__",
+    "PASSWORD": "__SOCKS5PROXY_PASSWORD__"
+  },
+  "PRICE_DATA_SERVER": {
+    "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
+    "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
+  },
+  "EXTERNAL_DATA_SERVER": {
+    "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
+    "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",
+    "LIQUID_API": "__EXTERNAL_DATA_SERVER_LIQUID_API__",
+    "LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__",
+    "BISQ_URL": "__EXTERNAL_DATA_SERVER_BISQ_URL__",
+    "BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__"
+  },
+  "LIGHTNING": {
+    "ENABLED": "__LIGHTNING_ENABLED__",
+    "BACKEND": "__LIGHTNING_BACKEND__",
+    "TOPOLOGY_FOLDER": "__LIGHTNING_TOPOLOGY_FOLDER__",
+    "STATS_REFRESH_INTERVAL": 600,
+    "GRAPH_REFRESH_INTERVAL": 600,
+    "LOGGER_UPDATE_INTERVAL": 30
+  },
+  "LND": {
+    "TLS_CERT_PATH": "",
+    "MACAROON_PATH": "",
+    "REST_API_URL": "https://localhost:8080"
+  },
+  "CLIGHTNING": {
+    "SOCKET": "__CLIGHTNING_SOCKET__"
+  }
+}

backend/src/__tests__/api/difficulty-adjustment.test.ts

@@ -0,0 +1,62 @@
+import { calcDifficultyAdjustment, DifficultyAdjustment } from '../../api/difficulty-adjustment';
+
+describe('Mempool Difficulty Adjustment', () => {
+  test('should calculate Difficulty Adjustments properly', () => {
+    const dt = (dtString) => {
+      return Math.floor(new Date(dtString).getTime() / 1000);
+    };
+
+    const vectors = [
+      [ // Vector 1
+        [ // Inputs
+          dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds)
+          dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds)
+          750134,                         // Current block height
+          0.6280047707459726,             // Previous retarget % (Passed through)
+          'mainnet',                      // Network (if testnet, next value is non-zero)
+          0,                              // If not testnet, not used
+        ],
+        { // Expected Result
+          progressPercent: 9.027777777777777,
+          difficultyChange: 12.562233927411782,
+          estimatedRetargetDate: 1661895424692,
+          remainingBlocks: 1834,
+          remainingTime: 977591692,
+          previousRetarget: 0.6280047707459726,
+          nextRetargetHeight: 751968,
+          timeAvg: 533038,
+          timeOffset: 0,
+        },
+      ],
+      [ // Vector 2 (testnet)
+        [ // Inputs
+          dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds)
+          dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds)
+          750134,                         // Current block height
+          0.6280047707459726,             // Previous retarget % (Passed through)
+          'testnet',                      // Network
+          dt('2022-08-19T13:52:46.000Z'), // Latest block timestamp in seconds
+        ],
+        { // Expected Result is same other than timeOffset
+          progressPercent: 9.027777777777777,
+          difficultyChange: 12.562233927411782,
+          estimatedRetargetDate: 1661895424692,
+          remainingBlocks: 1834,
+          remainingTime: 977591692,
+          previousRetarget: 0.6280047707459726,
+          nextRetargetHeight: 751968,
+          timeAvg: 533038,
+          timeOffset: -667000, // 11 min 7 seconds since last block (testnet only)
+          // If we add time avg to abs(timeOffset) it makes exactly 1200000 ms, or 20 minutes
+        },
+      ],
+    ] as [[number, number, number, number, string, number], DifficultyAdjustment][];
+
+    for (const vector of vectors) {
+      const result = calcDifficultyAdjustment(...vector[0]);
+      // previousRetarget is passed through untouched
+      expect(result.previousRetarget).toStrictEqual(vector[0][3]);
+      expect(result).toStrictEqual(vector[1]);
+    }
+  });
+});

backend/src/__tests__/config.test.ts

@@ -0,0 +1,139 @@
+import * as fs from 'fs';
+
+describe('Mempool Backend Config', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+    jest.resetModules();
+  });
+
+  test('should return defaults when no file is present', () => {
+    jest.isolateModules(() => {
+      jest.mock('../../mempool-config.json', () => ({}), { virtual: true });
+
+      const config = jest.requireActual('../config').default;
+
+      expect(config.MEMPOOL).toStrictEqual({
+        NETWORK: 'mainnet',
+        BACKEND: 'none',
+        BLOCKS_SUMMARIES_INDEXING: false,
+        HTTP_PORT: 8999,
+        SPAWN_CLUSTER_PROCS: 0,
+        API_URL_PREFIX: '/api/v1/',
+        AUTOMATIC_BLOCK_REINDEXING: false,
+        POLL_RATE_MS: 2000,
+        CACHE_DIR: './cache',
+        CLEAR_PROTECTION_MINUTES: 20,
+        RECOMMENDED_FEE_PERCENTILE: 50,
+        BLOCK_WEIGHT_UNITS: 4000000,
+        INITIAL_BLOCKS_AMOUNT: 8,
+        MEMPOOL_BLOCKS_AMOUNT: 8,
+        INDEXING_BLOCKS_AMOUNT: 11000,
+        PRICE_FEED_UPDATE_INTERVAL: 600,
+        USE_SECOND_NODE_FOR_MINFEE: false,
+        EXTERNAL_ASSETS: [],
+        EXTERNAL_MAX_RETRY: 1,
+        EXTERNAL_RETRY_INTERVAL: 0,
+        USER_AGENT: 'mempool',
+        STDOUT_LOG_MIN_PRIORITY: 'debug',
+        POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
+        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json'
+      });
+
+      expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
+
+      expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000' });
+
+      expect(config.CORE_RPC).toStrictEqual({
+        HOST: '127.0.0.1',
+        PORT: 8332,
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool'
+      });
+
+      expect(config.SECOND_CORE_RPC).toStrictEqual({
+        HOST: '127.0.0.1',
+        PORT: 8332,
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool'
+      });
+
+      expect(config.DATABASE).toStrictEqual({
+        ENABLED: true,
+        HOST: '127.0.0.1',
+        SOCKET: '',
+        PORT: 3306,
+        DATABASE: 'mempool',
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool'
+      });
+
+      expect(config.SYSLOG).toStrictEqual({
+        ENABLED: true,
+        HOST: '127.0.0.1',
+        PORT: 514,
+        MIN_PRIORITY: 'info',
+        FACILITY: 'local7'
+      });
+
+      expect(config.STATISTICS).toStrictEqual({ ENABLED: true, TX_PER_SECOND_SAMPLE_PERIOD: 150 });
+
+      expect(config.BISQ).toStrictEqual({ ENABLED: false, DATA_PATH: '/bisq/statsnode-data/btc_mainnet/db' });
+
+      expect(config.SOCKS5PROXY).toStrictEqual({
+        ENABLED: false,
+        USE_ONION: true,
+        HOST: '127.0.0.1',
+        PORT: 9050,
+        USERNAME: '',
+        PASSWORD: ''
+      });
+
+      expect(config.PRICE_DATA_SERVER).toStrictEqual({
+        TOR_URL: 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
+        CLEARNET_URL: 'https://price.bisq.wiz.biz/getAllMarketPrices'
+      });
+
+      expect(config.EXTERNAL_DATA_SERVER).toStrictEqual({
+        MEMPOOL_API: 'https://mempool.space/api/v1',
+        MEMPOOL_ONION: 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
+        LIQUID_API: 'https://liquid.network/api/v1',
+        LIQUID_ONION: 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1',
+        BISQ_URL: 'https://bisq.markets/api',
+        BISQ_ONION: 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
+      });
+    });
+  });
+
+  test('should override the default values with the passed values', () => {
+    jest.isolateModules(() => {
+      const fixture = JSON.parse(fs.readFileSync(`${__dirname}/../__fixtures__/mempool-config.template.json`, 'utf8'));
+      jest.mock('../../mempool-config.json', () => (fixture), { virtual: true });
+
+      const config = jest.requireActual('../config').default;
+
+      expect(config.MEMPOOL).toStrictEqual(fixture.MEMPOOL);
+
+      expect(config.ELECTRUM).toStrictEqual(fixture.ELECTRUM);
+
+      expect(config.ESPLORA).toStrictEqual(fixture.ESPLORA);
+
+      expect(config.CORE_RPC).toStrictEqual(fixture.CORE_RPC);
+
+      expect(config.SECOND_CORE_RPC).toStrictEqual(fixture.SECOND_CORE_RPC);
+
+      expect(config.DATABASE).toStrictEqual(fixture.DATABASE);
+
+      expect(config.SYSLOG).toStrictEqual(fixture.SYSLOG);
+
+      expect(config.STATISTICS).toStrictEqual(fixture.STATISTICS);
+
+      expect(config.BISQ).toStrictEqual(fixture.BISQ);
+
+      expect(config.SOCKS5PROXY).toStrictEqual(fixture.SOCKS5PROXY);
+
+      expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER);
+
+      expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER);
+    });
+  });
+});

backend/src/api/backend-info.ts

@@ -1,60 +1,37 @@
-import * as fs from 'fs';
-import * as os from 'os';
-import logger from '../logger';
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
 import { IBackendInfo } from '../mempool.interfaces';
-const { spawnSync } = require('child_process');
 
 class BackendInfo {
-  private gitCommitHash = '';
-  private hostname = '';
-  private version = '';
+  private backendInfo: IBackendInfo;
 
   constructor() {
-    this.setLatestCommitHash();
-    this.setVersion();
-    this.hostname = os.hostname();
-  }
-
-  public getBackendInfo(): IBackendInfo {
-    return {
-      hostname: this.hostname,
-      gitCommit: this.gitCommitHash,
-      version: this.version,
+    // This file is created by ./fetch-version.ts during building
+    const versionFile = path.join(__dirname, 'version.json')
+    var versionInfo;
+    if (fs.existsSync(versionFile)) {
+      versionInfo = JSON.parse(fs.readFileSync(versionFile).toString());
+    } else {
+      // Use dummy values if `versionFile` doesn't exist (e.g., during testing)
+      versionInfo = {
+        version: '?',
+        gitCommit: '?'
+      };
+    }
+    this.backendInfo = {
+      hostname: os.hostname(),
+      version: versionInfo.version,
+      gitCommit: versionInfo.gitCommit
     };
   }
 
+  public getBackendInfo(): IBackendInfo {
+    return this.backendInfo;
+  }
+
   public getShortCommitHash() {
-    return this.gitCommitHash.slice(0, 7);
-  }
-
-  private setLatestCommitHash(): void {
-    //TODO: share this logic with `generate-config.js`
-    if (process.env.DOCKER_COMMIT_HASH) {
-      this.gitCommitHash = process.env.DOCKER_COMMIT_HASH;
-    } else {
-      try {
-        const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
-        if (!gitRevParse.error) {
-          const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
-          this.gitCommitHash = output ? output : '?';
-        } else if (gitRevParse.error.code === 'ENOENT') {
-          console.log('git not found, cannot parse git hash');
-          this.gitCommitHash = '?';
-        }
-      } catch (e: any) {
-        console.log('Could not load git commit info: ' + e.message);
-        this.gitCommitHash = '?';
-      }
-    }
-  }
-
-  private setVersion(): void {
-    try {
-      const packageJson = fs.readFileSync('package.json').toString();
-      this.version = JSON.parse(packageJson).version;
-    } catch (e) {
-      throw new Error(e instanceof Error ? e.message : 'Error');
-    }
+    return this.backendInfo.gitCommit.slice(0, 7);
   }
 }
 

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -510,7 +510,12 @@ class BitcoinRoutes {
 
   private getDifficultyChange(req: Request, res: Response) {
     try {
-      res.json(difficultyAdjustment.getDifficultyAdjustment());
+      const da = difficultyAdjustment.getDifficultyAdjustment();
+      if (da) {
+        res.json(da);
+      } else {
+        res.status(503).send(`Service Temporarily Unavailable`);
+      }
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }

backend/src/api/blocks.ts

@@ -22,6 +22,8 @@ import poolsParser from './pools-parser';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
 import mining from './mining/mining';
 import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
+import PricesRepository from '../repositories/PricesRepository';
+import priceUpdater from '../tasks/price-updater';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -457,6 +459,19 @@ class Blocks {
           }
           await blocksRepository.$saveBlockInDatabase(blockExtended);
 
+          const lastestPriceId = await PricesRepository.$getLatestPriceId();
+          if (priceUpdater.historyInserted === true && lastestPriceId !== null) {
+            await blocksRepository.$saveBlockPrices([{
+              height: blockExtended.height,
+              priceId: lastestPriceId,
+            }]);
+          } else {
+            logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`)
+            setTimeout(() => {
+              indexer.runSingleTask('blocksPrices');
+            }, 10000);
+          }
+
           // Save blocks summary for visualization if it's enabled
           if (Common.blocksSummariesIndexingEnabled() === true) {
             await this.$getStrippedBlockTransactions(blockExtended.id, true);

backend/src/api/common.ts

@@ -1,5 +1,7 @@
 import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
 import config from '../config';
+import { NodeSocket } from '../repositories/NodesSocketsRepository';
+import { isIP } from 'net';
 export class Common {
   static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ?
     '144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49'
@@ -184,4 +186,117 @@ export class Common {
       config.MEMPOOL.BLOCKS_SUMMARIES_INDEXING === true
     );
   }
+
+  static setDateMidnight(date: Date): void {
+    date.setUTCHours(0);
+    date.setUTCMinutes(0);
+    date.setUTCSeconds(0);
+    date.setUTCMilliseconds(0);
+  }
+
+  static channelShortIdToIntegerId(channelId: string): string {
+    if (channelId.indexOf('x') === -1) { // Already an integer id
+      return channelId;
+    }
+    if (channelId.indexOf('/') !== -1) { // Topology import
+      channelId = channelId.slice(0, -2);
+    }
+    const s = channelId.split('x').map(part => BigInt(part));
+    return ((s[0] << 40n) | (s[1] << 16n) | s[2]).toString();
+  }
+
+  /** Decodes a channel id returned by lnd as uint64 to a short channel id */
+  static channelIntegerIdToShortId(id: string): string {
+    if (id.indexOf('/') !== -1) {
+      id = id.slice(0, -2);
+    }
+    
+    if (id.indexOf('x') !== -1) { // Already a short id
+      return id;
+    }
+
+    const n = BigInt(id);
+    return [
+      n >> 40n, // nth block
+      (n >> 16n) & 0xffffffn, // nth tx of the block
+      n & 0xffffn // nth output of the tx
+    ].join('x');
+  }
+
+  static utcDateToMysql(date?: number): string {
+    const d = new Date((date || 0) * 1000);
+    return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
+  }
+
+  static findSocketNetwork(addr: string): {network: string | null, url: string} {
+    let network: string | null = null;
+    let url = addr.split('://')[1];
+
+    if (!url) {
+      return {
+        network: null,
+        url: addr,
+      };
+    }
+
+    if (addr.indexOf('onion') !== -1) {
+      if (url.split('.')[0].length >= 56) {
+        network = 'torv3';
+      } else {
+        network = 'torv2';
+      }
+    } else if (addr.indexOf('i2p') !== -1) {
+      network = 'i2p';
+    } else if (addr.indexOf('ipv4') !== -1) {
+      const ipv = isIP(url.split(':')[0]);
+      if (ipv === 4) {
+        network = 'ipv4';
+      } else {
+        return {
+          network: null,
+          url: addr,
+        };
+      }
+    } else if (addr.indexOf('ipv6') !== -1) {
+      url = url.split('[')[1].split(']')[0];
+      const ipv = isIP(url);
+      if (ipv === 6) {
+        const parts = addr.split(':');
+        network = 'ipv6';
+        url = `[${url}]:${parts[parts.length - 1]}`;
+      } else {
+        return {
+          network: null,
+          url: addr,
+        };
+      }
+    } else {
+      return {
+        network: null,
+        url: addr,
+      };
+    }
+
+    return {
+      network: network,
+      url: url,
+    };
+  }
+
+  static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket {
+    if (config.LIGHTNING.BACKEND === 'cln') {
+      return {
+        publicKey: publicKey,
+        network: socket.network,
+        addr: socket.addr,
+      };
+    } else /* if (config.LIGHTNING.BACKEND === 'lnd') */ {
+      const formatted = this.findSocketNetwork(socket.addr);
+      return {
+        publicKey: publicKey,
+        network: formatted.network,
+        addr: formatted.url,
+      };
+    }
+  }
 }

backend/src/api/database-migration.ts

@@ -4,7 +4,7 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 33;
+  private static currentVersion = 40;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -248,7 +248,6 @@ class DatabaseMigration {
     }
 
     if (databaseSchemaVersion < 25 && isBitcoin === true) {
-      await this.$executeQuery(`INSERT INTO state VALUES('last_node_stats', 0, '1970-01-01');`);
       await this.$executeQuery(this.getCreateLightningStatisticsQuery(), await this.$checkIfTableExists('lightning_stats'));
       await this.$executeQuery(this.getCreateNodesQuery(), await this.$checkIfTableExists('nodes'));
       await this.$executeQuery(this.getCreateChannelsQuery(), await this.$checkIfTableExists('channels'));
@@ -311,6 +310,44 @@ class DatabaseMigration {
     if (databaseSchemaVersion < 33 && isBitcoin == true) {
       await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization", "country_iso_code") NOT NULL');
     }
+
+    if (databaseSchemaVersion < 34 && isBitcoin == true) {
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_tor_nodes int(11) NOT NULL DEFAULT "0"');
+    }
+
+    if (databaseSchemaVersion < 35 && isBitcoin == true) {
+      await this.$executeQuery('DELETE from `lightning_stats` WHERE added > "2021-09-19"');
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD CONSTRAINT added_unique UNIQUE (added);');
+    }
+
+    if (databaseSchemaVersion < 36 && isBitcoin == true) {
+      await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"');
+    }
+
+    if (databaseSchemaVersion < 37 && isBitcoin == true) {
+      await this.$executeQuery(this.getCreateLNNodesSocketsTableQuery(), await this.$checkIfTableExists('nodes_sockets'));
+    }
+
+    if (databaseSchemaVersion < 38 && isBitcoin == true) {
+      if (config.LIGHTNING.ENABLED) {
+        this.uniqueLog(logger.notice, `'lightning_stats' and 'node_stats' tables have been truncated.`);
+      }
+      await this.$executeQuery(`TRUNCATE lightning_stats`);
+      await this.$executeQuery(`TRUNCATE node_stats`);
+      await this.$executeQuery('ALTER TABLE `lightning_stats` CHANGE `added` `added` timestamp NULL');
+      await this.$executeQuery('ALTER TABLE `node_stats` CHANGE `added` `added` timestamp NULL');
+    }
+
+    if (databaseSchemaVersion < 39 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `nodes` ADD alias_search TEXT NULL DEFAULT NULL AFTER `alias`');
+      await this.$executeQuery('ALTER TABLE nodes ADD FULLTEXT(alias_search)');
+    }
+
+    if (databaseSchemaVersion < 40 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `nodes` ADD capacity bigint(20) unsigned DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `nodes` ADD channels int(11) unsigned DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `nodes` ADD INDEX `capacity` (`capacity`);');
+    }
   }
 
   /**
@@ -724,7 +761,7 @@ class DatabaseMigration {
       names text DEFAULT NULL,
       UNIQUE KEY id (id,type),
       KEY id_2 (id)
-    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
   private getCreateBlocksPricesTableQuery(): string {
@@ -736,6 +773,16 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateLNNodesSocketsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS nodes_sockets (
+      public_key varchar(66) NOT NULL,
+      socket varchar(100) NOT NULL,
+      type enum('ipv4', 'ipv6', 'torv2', 'torv3', 'i2p', 'dns', 'websocket') NULL,
+      UNIQUE KEY public_key_socket (public_key, socket),
+      INDEX (public_key)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
   public async $truncateIndexedData(tables: string[]) {
     const allowedTables = ['blocks', 'hashrates', 'prices'];
 

backend/src/api/difficulty-adjustment.ts

@@ -2,65 +2,100 @@ import config from '../config';
 import { IDifficultyAdjustment } from '../mempool.interfaces';
 import blocks from './blocks';
 
-class DifficultyAdjustmentApi {
-  constructor() { }
+export interface DifficultyAdjustment {
+  progressPercent: number;       // Percent: 0 to 100
+  difficultyChange: number;      // Percent: -75 to 300
+  estimatedRetargetDate: number; // Unix time in ms
+  remainingBlocks: number;       // Block count
+  remainingTime: number;         // Duration of time in ms
+  previousRetarget: number;      // Percent: -75 to 300
+  nextRetargetHeight: number;    // Block Height
+  timeAvg: number;               // Duration of time in ms
+  timeOffset: number;            // (Testnet) Time since last block (cap @ 20min) in ms
+}
 
-  public getDifficultyAdjustment(): IDifficultyAdjustment {
+export function calcDifficultyAdjustment(
+  DATime: number,
+  nowSeconds: number,
+  blockHeight: number,
+  previousRetarget: number,
+  network: string,
+  latestBlockTimestamp: number,
+): DifficultyAdjustment {
+  const ESTIMATE_LAG_BLOCKS = 146; // For first 7.2% of epoch, don't estimate.
+  const EPOCH_BLOCK_LENGTH = 2016; // Bitcoin mainnet
+  const BLOCK_SECONDS_TARGET = 600; // Bitcoin mainnet
+  const TESTNET_MAX_BLOCK_SECONDS = 1200; // Bitcoin testnet
+
+  const diffSeconds = nowSeconds - DATime;
+  const blocksInEpoch = (blockHeight >= 0) ? blockHeight % EPOCH_BLOCK_LENGTH : 0;
+  const progressPercent = (blockHeight >= 0) ? blocksInEpoch / EPOCH_BLOCK_LENGTH * 100 : 100;
+  const remainingBlocks = EPOCH_BLOCK_LENGTH - blocksInEpoch;
+  const nextRetargetHeight = (blockHeight >= 0) ? blockHeight + remainingBlocks : 0;
+
+  let difficultyChange = 0;
+  let timeAvgSecs = BLOCK_SECONDS_TARGET;
+  // Only calculate the estimate once we have 7.2% of blocks in current epoch
+  if (blocksInEpoch >= ESTIMATE_LAG_BLOCKS) {
+    timeAvgSecs = diffSeconds / blocksInEpoch;
+    difficultyChange = (BLOCK_SECONDS_TARGET / timeAvgSecs - 1) * 100;
+    // Max increase is x4 (+300%)
+    if (difficultyChange > 300) {
+      difficultyChange = 300;
+    }
+    // Max decrease is /4 (-75%)
+    if (difficultyChange < -75) {
+      difficultyChange = -75;
+    }
+  }
+
+  // Testnet difficulty is set to 1 after 20 minutes of no blocks,
+  // therefore the time between blocks will always be below 20 minutes (1200s).
+  let timeOffset = 0;
+  if (network === 'testnet') {
+    if (timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) {
+      timeAvgSecs = TESTNET_MAX_BLOCK_SECONDS;
+    }
+
+    const secondsSinceLastBlock = nowSeconds - latestBlockTimestamp;
+    if (secondsSinceLastBlock + timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) {
+      timeOffset = -Math.min(secondsSinceLastBlock, TESTNET_MAX_BLOCK_SECONDS) * 1000;
+    }
+  }
+
+  const timeAvg = Math.floor(timeAvgSecs * 1000);
+  const remainingTime = remainingBlocks * timeAvg;
+  const estimatedRetargetDate = remainingTime + nowSeconds * 1000;
+
+  return {
+    progressPercent,
+    difficultyChange,
+    estimatedRetargetDate,
+    remainingBlocks,
+    remainingTime,
+    previousRetarget,
+    nextRetargetHeight,
+    timeAvg,
+    timeOffset,
+  };
+}
+
+class DifficultyAdjustmentApi {
+  public getDifficultyAdjustment(): IDifficultyAdjustment | null {
     const DATime = blocks.getLastDifficultyAdjustmentTime();
     const previousRetarget = blocks.getPreviousDifficultyRetarget();
     const blockHeight = blocks.getCurrentBlockHeight();
     const blocksCache = blocks.getBlocks();
     const latestBlock = blocksCache[blocksCache.length - 1];
-
-    const now = new Date().getTime() / 1000;
-    const diff = now - DATime;
-    const blocksInEpoch = blockHeight % 2016;
-    const progressPercent = (blocksInEpoch >= 0) ? blocksInEpoch / 2016 * 100 : 100;
-    const remainingBlocks = 2016 - blocksInEpoch;
-    const nextRetargetHeight = blockHeight + remainingBlocks;
-
-    let difficultyChange = 0;
-    if (remainingBlocks < 1870) {
-      if (blocksInEpoch > 0) {
-        difficultyChange = (600 / (diff / blocksInEpoch) - 1) * 100;
-      }
-      if (difficultyChange > 300) {
-        difficultyChange = 300;
-      }
-      if (difficultyChange < -75) {
-        difficultyChange = -75;
-      }
+    if (!latestBlock) {
+      return null;
     }
+    const nowSeconds = Math.floor(new Date().getTime() / 1000);
 
-    let timeAvgMins = blocksInEpoch && blocksInEpoch > 146 ? diff / blocksInEpoch / 60 : 10;
-
-    // Testnet difficulty is set to 1 after 20 minutes of no blocks,
-    // therefore the time between blocks will always be below 20 minutes (1200s).
-    let timeOffset = 0;
-    if (config.MEMPOOL.NETWORK === 'testnet') {
-      if (timeAvgMins > 20) {
-        timeAvgMins = 20;
-      }
-      if (now - latestBlock.timestamp + timeAvgMins * 60 > 1200) {
-        timeOffset = -Math.min(now - latestBlock.timestamp, 1200) * 1000;
-      }
-    }
-
-    const timeAvg = timeAvgMins * 60 * 1000 ;
-    const remainingTime = (remainingBlocks * timeAvg) + (now * 1000);
-    const estimatedRetargetDate = remainingTime + now;
-
-    return {
-      progressPercent,
-      difficultyChange,
-      estimatedRetargetDate,
-      remainingBlocks,
-      remainingTime,
-      previousRetarget,
-      nextRetargetHeight,
-      timeAvg,
-      timeOffset,
-    };
+    return calcDifficultyAdjustment(
+      DATime, nowSeconds, blockHeight, previousRetarget,
+      config.MEMPOOL.NETWORK, latestBlock.timestamp
+    );
   }
 }
 

backend/src/api/explorer/channels.api.ts

@@ -1,5 +1,9 @@
 import logger from '../../logger';
 import DB from '../../database';
+import nodesApi from './nodes.api';
+import { ResultSetHeader } from 'mysql2';
+import { ILightningApi } from '../lightning/lightning-api.interface';
+import { Common } from '../common';
 
 class ChannelsApi {
   public async $getAllChannels(): Promise<any[]> {
@@ -13,32 +17,61 @@ class ChannelsApi {
     }
   }
 
-  public async $getAllChannelsGeo(publicKey?: string): Promise<any[]> {
+  public async $getAllChannelsGeo(publicKey?: string, style?: string): Promise<any[]> {
     try {
+      let select: string;
+      if (style === 'widget') {
+        select = `
+          nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
+          nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude
+        `;
+      } else {
+        select = `
+          nodes_1.public_key as node1_public_key, nodes_1.alias AS node1_alias,
+          nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
+          nodes_2.public_key as node2_public_key, nodes_2.alias AS node2_alias,
+          nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude
+        `;
+      }
+
       const params: string[] = [];
-      let query = `SELECT nodes_1.public_key as node1_public_key, nodes_1.alias AS node1_alias,
-        nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
-        nodes_2.public_key as node2_public_key, nodes_2.alias AS node2_alias,
-        nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude,
-        channels.capacity
-      FROM channels
-      JOIN nodes AS nodes_1 on nodes_1.public_key = channels.node1_public_key
-      JOIN nodes AS nodes_2 on nodes_2.public_key = channels.node2_public_key
-      WHERE nodes_1.latitude IS NOT NULL AND nodes_1.longitude IS NOT NULL
-        AND nodes_2.latitude IS NOT NULL AND nodes_2.longitude IS NOT NULL
+      let query = `SELECT ${select}
+        FROM channels
+        JOIN nodes AS nodes_1 on nodes_1.public_key = channels.node1_public_key
+        JOIN nodes AS nodes_2 on nodes_2.public_key = channels.node2_public_key
+        WHERE channels.status = 1
+          AND nodes_1.latitude IS NOT NULL AND nodes_1.longitude IS NOT NULL
+          AND nodes_2.latitude IS NOT NULL AND nodes_2.longitude IS NOT NULL
       `;
 
       if (publicKey !== undefined) {
         query += ' AND (nodes_1.public_key = ? OR nodes_2.public_key = ?)';
         params.push(publicKey);
         params.push(publicKey);
+      } else {
+        query += ` AND channels.capacity > 1000000
+          GROUP BY nodes_1.public_key, nodes_2.public_key
+          ORDER BY channels.capacity DESC
+          LIMIT 10000
+        `;        
       }
 
       const [rows]: any = await DB.query(query, params);
-      return rows.map((row) => [
-        row.node1_public_key, row.node1_alias, row.node1_longitude, row.node1_latitude,
-        row.node2_public_key, row.node2_alias, row.node2_longitude, row.node2_latitude,
-        row.capacity]);
+      return rows.map((row) => {
+        if (style === 'widget') {
+          return [
+            row.node1_longitude, row.node1_latitude,
+            row.node2_longitude, row.node2_latitude,
+          ];
+        } else {
+          return [
+            row.node1_public_key, row.node1_alias,
+            row.node1_longitude, row.node1_latitude,
+            row.node2_public_key, row.node2_alias,
+            row.node2_longitude, row.node2_latitude,
+          ];
+        }
+      });
     } catch (e) {
       logger.err('$getAllChannelsGeo error: ' + (e instanceof Error ? e.message : e));
       throw e;
@@ -48,7 +81,7 @@ class ChannelsApi {
   public async $searchChannelsById(search: string): Promise<any[]> {
     try {
       const searchStripped = search.replace('%', '') + '%';
-      const query = `SELECT id, short_id, capacity FROM channels WHERE id LIKE ? OR short_id LIKE ? LIMIT 10`;
+      const query = `SELECT id, short_id, capacity, status FROM channels WHERE id LIKE ? OR short_id LIKE ? LIMIT 10`;
       const [rows]: any = await DB.query(query, [searchStripped, searchStripped]);
       return rows;
     } catch (e) {
@@ -57,9 +90,14 @@ class ChannelsApi {
     }
   }
 
-  public async $getChannelsByStatus(status: number): Promise<any[]> {
+  public async $getChannelsByStatus(status: number | number[]): Promise<any[]> {
     try {
-      const query = `SELECT * FROM channels WHERE status = ?`;
+      let query: string;
+      if (Array.isArray(status)) {
+        query = `SELECT * FROM channels WHERE status IN (${status.join(',')})`;
+      } else {
+        query = `SELECT * FROM channels WHERE status = ?`;
+      }
       const [rows]: any = await DB.query(query, [status]);
       return rows;
     } catch (e) {
@@ -92,7 +130,31 @@ class ChannelsApi {
 
   public async $getChannel(id: string): Promise<any> {
     try {
-      const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*, ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key WHERE (ns1.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node1_public_key) AND ns2.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node2_public_key)) AND channels.id = ?`;
+      const query = `
+        SELECT n1.alias AS alias_left, n1.longitude as node1_longitude, n1.latitude as node1_latitude,
+          n2.alias AS alias_right, n2.longitude as node2_longitude, n2.latitude as node2_latitude,
+          channels.*,
+          ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right
+        FROM channels
+        LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
+        LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
+        LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key
+        LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key
+        WHERE (
+          ns1.id = (
+            SELECT MAX(id)
+            FROM node_stats
+            WHERE public_key = channels.node1_public_key
+          )
+          AND ns2.id = (
+            SELECT MAX(id)
+            FROM node_stats
+            WHERE public_key = channels.node2_public_key
+          )
+        )
+        AND channels.id = ?
+      `;
+
       const [rows]: any = await DB.query(query, [id]);
       if (rows[0]) {
         return this.convertChannel(rows[0]);
@@ -168,9 +230,14 @@ class ChannelsApi {
 
   public async $getChannelsByTransactionId(transactionIds: string[]): Promise<any[]> {
     try {
-      transactionIds = transactionIds.map((id) => '\'' + id + '\'');
-      const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.* FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key WHERE channels.transaction_id IN (${transactionIds.join(', ')}) OR channels.closing_transaction_id IN (${transactionIds.join(', ')})`;
-      const [rows]: any = await DB.query(query);
+      const query = `
+        SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*
+        FROM channels
+        LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
+        LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
+        WHERE channels.transaction_id IN ? OR channels.closing_transaction_id IN ?
+      `;
+      const [rows]: any = await DB.query(query, [[transactionIds], [transactionIds]]);
       const channels = rows.map((row) => this.convertChannel(row));
       return channels;
     } catch (e) {
@@ -181,15 +248,95 @@ class ChannelsApi {
 
   public async $getChannelsForNode(public_key: string, index: number, length: number, status: string): Promise<any[]> {
     try {
-      // Default active and inactive channels
-      let statusQuery = '< 2';
-      // Closed channels only
-      if (status === 'closed') {
-        statusQuery = '= 2';
+      let channelStatusFilter;
+      if (status === 'open') {
+        channelStatusFilter = '< 2';
+      } else if (status === 'active') {
+        channelStatusFilter = '= 1';
+      } else if (status === 'closed') {
+        channelStatusFilter = '= 2';
+      } else {
+        throw new Error('getChannelsForNode: Invalid status requested');
       }
-      const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*, ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key WHERE (ns1.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node1_public_key) AND ns2.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node2_public_key)) AND (node1_public_key = ? OR node2_public_key = ?) AND status ${statusQuery} ORDER BY channels.capacity DESC LIMIT ?, ?`;
-      const [rows]: any = await DB.query(query, [public_key, public_key, index, length]);
-      const channels = rows.map((row) => this.convertChannel(row));
+
+      // Channels originating from node
+      let query = `
+        SELECT COALESCE(node2.alias, SUBSTRING(node2_public_key, 0, 20)) AS alias, COALESCE(node2.public_key, node2_public_key) AS public_key,
+          channels.status, channels.node1_fee_rate,
+          channels.capacity, channels.short_id, channels.id, channels.closing_reason,
+          UNIX_TIMESTAMP(closing_date) as closing_date, UNIX_TIMESTAMP(channels.updated_at) as updated_at
+        FROM channels
+        LEFT JOIN nodes AS node2 ON node2.public_key = channels.node2_public_key
+        WHERE node1_public_key = ? AND channels.status ${channelStatusFilter}
+      `;
+      const [channelsFromNode]: any = await DB.query(query, [public_key]);
+
+      // Channels incoming to node
+      query = `
+        SELECT COALESCE(node1.alias, SUBSTRING(node1_public_key, 0, 20)) AS alias, COALESCE(node1.public_key, node1_public_key) AS public_key,
+          channels.status, channels.node2_fee_rate,
+          channels.capacity, channels.short_id, channels.id, channels.closing_reason,
+          UNIX_TIMESTAMP(closing_date) as closing_date, UNIX_TIMESTAMP(channels.updated_at) as updated_at
+        FROM channels
+        LEFT JOIN nodes AS node1 ON node1.public_key = channels.node1_public_key
+        WHERE node2_public_key = ? AND channels.status ${channelStatusFilter}
+      `;
+      const [channelsToNode]: any = await DB.query(query, [public_key]);
+
+      let allChannels = channelsFromNode.concat(channelsToNode);
+      allChannels.sort((a, b) => {
+        if (status === 'closed') {
+          if (!b.closing_date && !a.closing_date) {
+            return (b.updated_at ?? 0) - (a.updated_at ?? 0);
+          } else {
+            return (b.closing_date ?? 0) - (a.closing_date ?? 0);
+          }
+        } else {
+          return b.capacity - a.capacity;
+        }
+      });
+
+      if (index >= 0) {
+        allChannels = allChannels.slice(index, index + length);
+      } else if (index === -1) { // Node channels tree chart
+        allChannels = allChannels.slice(0, 1000);
+      }
+
+      const channels: any[] = []
+      for (const row of allChannels) {
+        let channel;
+        if (index >= 0) {
+          const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key);
+          channel = {
+            status: row.status,
+            closing_reason: row.closing_reason,
+            closing_date: row.closing_date,
+            capacity: row.capacity ?? 0,
+            short_id: row.short_id,
+            id: row.id,
+            fee_rate: row.node1_fee_rate ?? row.node2_fee_rate ?? 0,
+            node: {
+              alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20),
+              public_key: row.public_key,
+              channels: activeChannelsStats.active_channel_count ?? 0,
+              capacity: activeChannelsStats.capacity ?? 0,
+            }
+          };
+        } else if (index === -1) {
+          channel = {
+            capacity: row.capacity ?? 0,
+            short_id: row.short_id,
+            id: row.id,
+            node: {
+              alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20),
+              public_key: row.public_key,
+            }
+          };
+        }
+
+        channels.push(channel);
+      }
+
       return channels;
     } catch (e) {
       logger.err('$getChannelsForNode error: ' + (e instanceof Error ? e.message : e));
@@ -205,7 +352,12 @@ class ChannelsApi {
       if (status === 'closed') {
         statusQuery = '= 2';
       }
-      const query = `SELECT COUNT(*) AS count FROM channels WHERE (node1_public_key = ? OR node2_public_key = ?) AND status ${statusQuery}`;
+      const query = `
+        SELECT COUNT(*) AS count
+        FROM channels
+        WHERE (node1_public_key = ? OR node2_public_key = ?)
+        AND status ${statusQuery}
+      `;
       const [rows]: any = await DB.query(query, [public_key, public_key]);
       return rows[0]['count'];
     } catch (e) {
@@ -223,6 +375,7 @@ class ChannelsApi {
       'transaction_vout': channel.transaction_vout,
       'closing_transaction_id': channel.closing_transaction_id,
       'closing_reason': channel.closing_reason,
+      'closing_date': channel.closing_date,
       'updated_at': channel.updated_at,
       'created': channel.created,
       'status': channel.status,
@@ -238,6 +391,8 @@ class ChannelsApi {
         'max_htlc_mtokens': channel.node1_max_htlc_mtokens,
         'min_htlc_mtokens': channel.node1_min_htlc_mtokens,
         'updated_at': channel.node1_updated_at,
+        'longitude': channel.node1_longitude,
+        'latitude': channel.node1_latitude,
       },
       'node_right': {
         'alias': channel.alias_right,
@@ -251,9 +406,157 @@ class ChannelsApi {
         'max_htlc_mtokens': channel.node2_max_htlc_mtokens,
         'min_htlc_mtokens': channel.node2_min_htlc_mtokens,
         'updated_at': channel.node2_updated_at,
+        'longitude': channel.node2_longitude,
+        'latitude': channel.node2_latitude,
       },
     };
   }
+
+  /**
+   * Save or update a channel present in the graph
+   */
+  public async $saveChannel(channel: ILightningApi.Channel, status = 1): Promise<void> {
+    const [ txid, vout ] = channel.chan_point.split(':');
+
+    const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
+    const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
+
+    const query = `INSERT INTO channels
+      (
+        id,
+        short_id,
+        capacity,
+        transaction_id,
+        transaction_vout,
+        updated_at,
+        status,
+        node1_public_key,
+        node1_base_fee_mtokens,
+        node1_cltv_delta,
+        node1_fee_rate,
+        node1_is_disabled,
+        node1_max_htlc_mtokens,
+        node1_min_htlc_mtokens,
+        node1_updated_at,
+        node2_public_key,
+        node2_base_fee_mtokens,
+        node2_cltv_delta,
+        node2_fee_rate,
+        node2_is_disabled,
+        node2_max_htlc_mtokens,
+        node2_min_htlc_mtokens,
+        node2_updated_at
+      )
+      VALUES (?, ?, ?, ?, ?, ?, ${status}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON DUPLICATE KEY UPDATE
+        capacity = ?,
+        updated_at = ?,
+        status = ${status},
+        node1_public_key = ?,
+        node1_base_fee_mtokens = ?,
+        node1_cltv_delta = ?,
+        node1_fee_rate = ?,
+        node1_is_disabled = ?,
+        node1_max_htlc_mtokens = ?,
+        node1_min_htlc_mtokens = ?,
+        node1_updated_at = ?,
+        node2_public_key = ?,
+        node2_base_fee_mtokens = ?,
+        node2_cltv_delta = ?,
+        node2_fee_rate = ?,
+        node2_is_disabled = ?,
+        node2_max_htlc_mtokens = ?,
+        node2_min_htlc_mtokens = ?,
+        node2_updated_at = ?
+      ;`;
+
+    await DB.query(query, [
+      Common.channelShortIdToIntegerId(channel.channel_id),
+      Common.channelIntegerIdToShortId(channel.channel_id),
+      channel.capacity,
+      txid,
+      vout,
+      Common.utcDateToMysql(channel.last_update),
+      channel.node1_pub,
+      policy1.fee_base_msat,
+      policy1.time_lock_delta,
+      policy1.fee_rate_milli_msat,
+      policy1.disabled,
+      policy1.max_htlc_msat,
+      policy1.min_htlc,
+      Common.utcDateToMysql(policy1.last_update),
+      channel.node2_pub,
+      policy2.fee_base_msat,
+      policy2.time_lock_delta,
+      policy2.fee_rate_milli_msat,
+      policy2.disabled,
+      policy2.max_htlc_msat,
+      policy2.min_htlc,
+      Common.utcDateToMysql(policy2.last_update),
+      channel.capacity,
+      Common.utcDateToMysql(channel.last_update),
+      channel.node1_pub,
+      policy1.fee_base_msat,
+      policy1.time_lock_delta,
+      policy1.fee_rate_milli_msat,
+      policy1.disabled,
+      policy1.max_htlc_msat,
+      policy1.min_htlc,
+      Common.utcDateToMysql(policy1.last_update),
+      channel.node2_pub,
+      policy2.fee_base_msat,
+      policy2.time_lock_delta,
+      policy2.fee_rate_milli_msat,
+      policy2.disabled,
+      policy2.max_htlc_msat,
+      policy2.min_htlc,
+      Common.utcDateToMysql(policy2.last_update)
+    ]);
+  }
+
+  /**
+   * Set all channels not in `graphChannelsIds` as inactive (status = 0)
+   */
+  public async $setChannelsInactive(graphChannelsIds: string[]): Promise<void> {
+    if (graphChannelsIds.length === 0) {
+      return;
+    }
+
+    try {
+      const result = await DB.query<ResultSetHeader>(`
+        UPDATE channels
+        SET status = 0
+        WHERE id NOT IN (
+          ${graphChannelsIds.map(id => `"${id}"`).join(',')}
+        )
+        AND status != 2
+      `);
+      if (result[0].changedRows ?? 0 > 0) {
+        logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`);
+      } else {
+        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`);
+      }
+    } catch (e) {
+      logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  public async $getLatestChannelUpdateForNode(publicKey: string): Promise<number> {
+    try {
+      const query = `
+        SELECT MAX(UNIX_TIMESTAMP(updated_at)) as updated_at
+        FROM channels
+        WHERE node1_public_key = ?
+      `;
+      const [rows]: any[] = await DB.query(query, [publicKey]);
+      if (rows.length > 0) {
+        return rows[0].updated_at;
+      }
+    } catch (e) {
+      logger.err(`Can't getLatestChannelUpdateForNode for ${publicKey}. Reason ${e instanceof Error ? e.message : e}`);
+    }
+    return 0;
+  }
 }
 
 export default new ChannelsApi();

backend/src/api/explorer/channels.routes.ts

@@ -32,6 +32,9 @@ class ChannelsRoutes {
         res.status(404).send('Channel not found');
         return;
       }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(channel);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
@@ -44,11 +47,22 @@ class ChannelsRoutes {
         res.status(400).send('Missing parameter: public_key');
         return;
       }
+
       const index = parseInt(typeof req.query.index === 'string' ? req.query.index : '0', 10) || 0;
       const status: string = typeof req.query.status === 'string' ? req.query.status : '';
-      const length = 25;
-      const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, length, status);
+
+      if (index < -1) {
+        res.status(400).send('Invalid index');
+      }
+      if (['open', 'active', 'closed'].includes(status) === false) {
+        res.status(400).send('Invalid status');
+      }
+
+      const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, 10, status);
       const channelsCount = await channelsApi.$getChannelsCountForNode(req.query.public_key, status);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.header('X-Total-Count', channelsCount.toString());
       res.json(channels);
     } catch (e) {
@@ -56,7 +70,7 @@ class ChannelsRoutes {
     }
   }
 
-  private async $getChannelsByTransactionIds(req: Request, res: Response) {
+  private async $getChannelsByTransactionIds(req: Request, res: Response): Promise<void> {
     try {
       if (!Array.isArray(req.query.txId)) {
         res.status(400).send('Not an array');
@@ -69,27 +83,26 @@ class ChannelsRoutes {
         }
       }
       const channels = await channelsApi.$getChannelsByTransactionId(txIds);
-      const inputs: any[] = [];
-      const outputs: any[] = [];
+      const result: any[] = [];
       for (const txid of txIds) {
-        const foundChannelInputs = channels.find((channel) => channel.closing_transaction_id === txid);
-        if (foundChannelInputs) {
-          inputs.push(foundChannelInputs);
-        } else {
-          inputs.push(null);
+        const inputs: any = {};
+        const outputs: any = {};
+        // Assuming that we only have one lightning close input in each transaction. This may not be true in the future
+        const foundChannelsFromInput = channels.find((channel) => channel.closing_transaction_id === txid);
+        if (foundChannelsFromInput) {
+          inputs[0] = foundChannelsFromInput;
         }
-        const foundChannelOutputs = channels.find((channel) => channel.transaction_id === txid);
-        if (foundChannelOutputs) {
-          outputs.push(foundChannelOutputs);
-        } else {
-          outputs.push(null);
+        const foundChannelsFromOutputs = channels.filter((channel) => channel.transaction_id === txid);
+        for (const output of foundChannelsFromOutputs) {
+          outputs[output.transaction_vout] = output;
         }
+        result.push({
+          inputs,
+          outputs,
+        });
       }
 
-      res.json({
-        inputs: inputs,
-        outputs: outputs,
-      });
+      res.json(result);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }
@@ -97,7 +110,11 @@ class ChannelsRoutes {
 
   private async $getAllChannelsGeo(req: Request, res: Response) {
     try {
-      const channels = await channelsApi.$getAllChannelsGeo(req.params?.publicKey);
+      const style: string = typeof req.query.style === 'string' ? req.query.style : '';
+      const channels = await channelsApi.$getAllChannelsGeo(req.params?.publicKey, style);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(channels);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);

backend/src/api/explorer/nodes.api.ts

@@ -1,24 +1,62 @@
 import logger from '../../logger';
 import DB from '../../database';
+import { ResultSetHeader } from 'mysql2';
+import { ILightningApi } from '../lightning/lightning-api.interface';
+import { ITopNodesPerCapacity, ITopNodesPerChannels } from '../../mempool.interfaces';
 
 class NodesApi {
+  public async $getWorldNodes(): Promise<any> {
+    try {
+      let query = `
+        SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+        CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
+        CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+        nodes.longitude, nodes.latitude,
+        geo_names_country.names as country, geo_names_iso.names as isoCode
+        FROM nodes
+        LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        WHERE status = 1 AND nodes.as_number IS NOT NULL
+        ORDER BY capacity
+      `;
+
+      const [nodes]: any[] = await DB.query(query);
+
+      for (let i = 0; i < nodes.length; ++i) {
+        nodes[i].country = JSON.parse(nodes[i].country);
+      }
+
+      query = `
+        SELECT MAX(nodes.capacity) as maxLiquidity, MAX(nodes.channels) as maxChannels
+        FROM nodes
+        WHERE status = 1 AND nodes.as_number IS NOT NULL
+      `;
+
+      const [maximums]: any[] = await DB.query(query);
+      
+      return {
+        maxLiquidity: maximums[0].maxLiquidity,
+        maxChannels: maximums[0].maxChannels,
+        nodes: nodes.map(node => [
+          node.longitude, node.latitude,
+          node.publicKey, node.alias, node.capacity, node.channels,
+          node.country, node.isoCode
+        ])
+      };
+    } catch (e) {
+      logger.err(`Can't get world nodes list. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
   public async $getNode(public_key: string): Promise<any> {
     try {
-      const query = `
-        SELECT nodes.*, geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city,
-        geo_names_country.names as country, geo_names_subdivision.names as subdivision,
-          (SELECT Count(*)
-          FROM channels
-          WHERE channels.status = 2 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS channel_closed_count,
-          (SELECT Count(*)
-          FROM channels
-          WHERE channels.status = 1 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS channel_active_count,
-          (SELECT Sum(capacity)
-          FROM channels
-          WHERE channels.status = 1 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS capacity,
-          (SELECT Avg(capacity)
-          FROM channels
-          WHERE status = 1 AND ( node1_public_key = ? OR node2_public_key = ? )) AS channels_capacity_avg
+      // General info
+      let query = `
+        SELECT public_key, alias, UNIX_TIMESTAMP(first_seen) AS first_seen,
+          UNIX_TIMESTAMP(updated_at) AS updated_at, color, sockets as sockets,
+          as_number, city_id, country_id, subdivision_id, longitude, latitude,
+          geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city,
+          geo_names_country.names as country, geo_names_subdivision.names as subdivision
         FROM nodes
         LEFT JOIN geo_names geo_names_as on geo_names_as.id = as_number
         LEFT JOIN geo_names geo_names_city on geo_names_city.id = city_id
@@ -27,21 +65,70 @@ class NodesApi {
         LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
         WHERE public_key = ?
       `;
-      const [rows]: any = await DB.query(query, [public_key, public_key, public_key, public_key, public_key, public_key, public_key, public_key, public_key]);
-      if (rows.length > 0) {
-        rows[0].as_organization = JSON.parse(rows[0].as_organization);
-        rows[0].subdivision = JSON.parse(rows[0].subdivision);
-        rows[0].city = JSON.parse(rows[0].city);
-        rows[0].country = JSON.parse(rows[0].country);
-        return rows[0];
+      let [rows]: any[] = await DB.query(query, [public_key]);
+      if (rows.length === 0) {
+        throw new Error(`This node does not exist, or our node is not seeing it yet`);
       }
-      return null;
+
+      const node = rows[0];
+      node.as_organization = JSON.parse(node.as_organization);
+      node.subdivision = JSON.parse(node.subdivision);
+      node.city = JSON.parse(node.city);
+      node.country = JSON.parse(node.country);
+
+      // Active channels and capacity
+      const activeChannelsStats: any = await this.$getActiveChannelsStats(public_key);
+      node.active_channel_count = activeChannelsStats.active_channel_count ?? 0;
+      node.capacity = activeChannelsStats.capacity ?? 0;
+
+      // Opened channels count
+      query = `
+        SELECT count(short_id) as opened_channel_count
+        FROM channels
+        WHERE status != 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+      `;
+      [rows] = await DB.query(query, [public_key, public_key]);
+      node.opened_channel_count = 0;
+      if (rows.length > 0) {
+        node.opened_channel_count = rows[0].opened_channel_count;
+      }
+
+      // Closed channels count
+      query = `
+        SELECT count(short_id) as closed_channel_count
+        FROM channels
+        WHERE status = 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+      `;
+      [rows] = await DB.query(query, [public_key, public_key]);
+      node.closed_channel_count = 0;
+      if (rows.length > 0) {
+        node.closed_channel_count = rows[0].closed_channel_count;
+      }
+
+      return node;
     } catch (e) {
-      logger.err('$getNode error: ' + (e instanceof Error ? e.message : e));
+      logger.err(`Cannot get node information for ${public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);
       throw e;
     }
   }
 
+  public async $getActiveChannelsStats(node_public_key: string): Promise<unknown> {
+    const query = `
+      SELECT count(short_id) as active_channel_count, sum(capacity) as capacity
+      FROM channels
+      WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+    `;
+    const [rows]: any[] = await DB.query(query, [node_public_key, node_public_key]);
+    if (rows.length > 0) {
+      return {
+        active_channel_count: rows[0].active_channel_count,
+        capacity: rows[0].capacity
+      };
+    } else {
+      return null;
+    }
+  }
+
   public async $getAllNodes(): Promise<any> {
     try {
       const query = `SELECT * FROM nodes`;
@@ -55,7 +142,12 @@ class NodesApi {
 
   public async $getNodeStats(public_key: string): Promise<any> {
     try {
-      const query = `SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels FROM node_stats WHERE public_key = ? ORDER BY added DESC`;
+      const query = `
+        SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels
+        FROM node_stats
+        WHERE public_key = ?
+        ORDER BY added DESC
+      `;
       const [rows]: any = await DB.query(query, [public_key]);
       return rows;
     } catch (e) {
@@ -64,10 +156,44 @@ class NodesApi {
     }
   }
 
-  public async $getTopCapacityNodes(): Promise<any> {
+  public async $getTopCapacityNodes(full: boolean): Promise<ITopNodesPerCapacity[]> {
     try {
-      const query = `SELECT nodes.*, node_stats.capacity, node_stats.channels FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key ORDER BY node_stats.added DESC, node_stats.capacity DESC LIMIT 10`;
-      const [rows]: any = await DB.query(query);
+      let rows: any;
+      let query: string;
+      if (full === false) {
+        query = `
+          SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            nodes.capacity
+          FROM nodes
+          ORDER BY capacity DESC
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+      } else {
+        query = `
+          SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
+            CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+            UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM nodes
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          ORDER BY capacity DESC
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      }
+
       return rows;
     } catch (e) {
       logger.err('$getTopCapacityNodes error: ' + (e instanceof Error ? e.message : e));
@@ -75,10 +201,95 @@ class NodesApi {
     }
   }
 
-  public async $getTopChannelsNodes(): Promise<any> {
+  public async $getTopChannelsNodes(full: boolean): Promise<ITopNodesPerChannels[]> {
     try {
-      const query = `SELECT nodes.*, node_stats.capacity, node_stats.channels FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key ORDER BY node_stats.added DESC, node_stats.channels DESC LIMIT 10`;
-      const [rows]: any = await DB.query(query);
+      let rows: any;
+      let query: string;
+      if (full === false) {
+        query = `
+          SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            nodes.channels
+          FROM nodes
+          ORDER BY channels DESC
+          LIMIT 100;
+        `;
+
+        [rows] = await DB.query(query);
+      } else {
+        query = `
+          SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+            CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
+            UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM nodes
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          ORDER BY channels DESC
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      }
+
+      return rows;
+    } catch (e) {
+      logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getOldestNodes(full: boolean): Promise<ITopNodesPerChannels[]> {
+    try {
+      let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats');
+      const latestDate = rows[0].maxAdded;
+
+      let query: string;
+      if (full === false) {
+        query = `
+          SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            node_stats.channels
+          FROM node_stats
+          JOIN nodes ON nodes.public_key = node_stats.public_key
+          WHERE added = FROM_UNIXTIME(${latestDate})
+          ORDER BY first_seen
+          LIMIT 100;
+        `;
+
+        [rows] = await DB.query(query);
+      } else {
+        query = `
+          SELECT node_stats.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(node_stats.public_key, 1, 20), alias) as alias,
+            CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
+            CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity,
+            UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM node_stats
+          RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          WHERE added = FROM_UNIXTIME(${latestDate})
+          ORDER BY first_seen
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      }
+
       return rows;
     } catch (e) {
       logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e));
@@ -88,9 +299,10 @@ class NodesApi {
 
   public async $searchNodeByPublicKeyOrAlias(search: string) {
     try {
-      const searchStripped = search.replace('%', '') + '%';
-      const query = `SELECT nodes.public_key, nodes.alias, node_stats.capacity FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key WHERE nodes.public_key LIKE ? OR nodes.alias LIKE ? GROUP BY nodes.public_key ORDER BY node_stats.capacity DESC LIMIT 10`;
-      const [rows]: any = await DB.query(query, [searchStripped, searchStripped]);
+      const publicKeySearch = search.replace('%', '') + '%';
+      const aliasSearch = search.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '').split(' ').map((search) => '+' + search + '*').join(' ');
+      const query = `SELECT public_key, alias, capacity, channels, status FROM nodes WHERE public_key LIKE ? OR MATCH alias_search AGAINST (? IN BOOLEAN MODE) ORDER BY capacity DESC LIMIT 10`;
+      const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]);
       return rows;
     } catch (e) {
       logger.err('$searchNodeByPublicKeyOrAlias error: ' + (e instanceof Error ? e.message : e));
@@ -98,64 +310,120 @@ class NodesApi {
     }
   }
 
-  public async $getNodesISP(groupBy: string, showTor: boolean) {
+  public async $getNodesISPRanking() {
     try {
-      const orderBy = groupBy === 'capacity' ? `CAST(SUM(capacity) as INT)` : `COUNT(DISTINCT nodes.public_key)`;
-      
-      // Clearnet
-      let query = `SELECT GROUP_CONCAT(DISTINCT(nodes.as_number)) as ispId, geo_names.names as names,
-          COUNT(DISTINCT nodes.public_key) as nodesCount, CAST(SUM(capacity) as INT) as capacity
-        FROM nodes
-        JOIN geo_names ON geo_names.id = nodes.as_number
-        JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key
-        GROUP BY geo_names.names
-        ORDER BY ${orderBy} DESC
-      `;      
-      const [nodesCountPerAS]: any = await DB.query(query);
+      let query = '';
 
-      let total = 0;
-      const nodesPerAs: any[] = [];
+      // List all channels and the two linked ISP
+      query = `
+        SELECT short_id, channels.capacity,
+          channels.node1_public_key AS node1PublicKey, isp1.names AS isp1, isp1.id as isp1ID,
+          channels.node2_public_key AS node2PublicKey, isp2.names AS isp2, isp2.id as isp2ID
+        FROM channels
+        JOIN nodes node1 ON node1.public_key = channels.node1_public_key
+        JOIN nodes node2 ON node2.public_key = channels.node2_public_key
+        JOIN geo_names isp1 ON isp1.id = node1.as_number
+        JOIN geo_names isp2 ON isp2.id = node2.as_number
+        WHERE channels.status = 1
+        ORDER BY short_id DESC
+      `;
+      const [channelsIsp]: any = await DB.query(query);
 
-      for (const asGroup of nodesCountPerAS) {
-        if (groupBy === 'capacity') {
-          total += asGroup.capacity;
-        } else {
-          total += asGroup.nodesCount;
+      // Sum channels capacity and node count per ISP
+      const ispList = {};
+      for (const channel of channelsIsp) {
+        const isp1 = JSON.parse(channel.isp1);
+        const isp2 = JSON.parse(channel.isp2);
+
+        if (!ispList[isp1]) {
+          ispList[isp1] = {
+            id: channel.isp1ID.toString(),
+            capacity: 0,
+            channels: 0,
+            nodes: {},
+          };
+        } else if (ispList[isp1].id.indexOf(channel.isp1ID) === -1) {
+          ispList[isp1].id += ',' + channel.isp1ID.toString();
         }
+
+        if (!ispList[isp2]) {
+          ispList[isp2] = {
+            id: channel.isp2ID.toString(),
+            capacity: 0,
+            channels: 0,
+            nodes: {},
+          };
+        } else if (ispList[isp2].id.indexOf(channel.isp2ID) === -1) {
+          ispList[isp2].id += ',' + channel.isp2ID.toString();
+        }
+        
+        ispList[isp1].capacity += channel.capacity;
+        ispList[isp1].channels += 1;
+        ispList[isp1].nodes[channel.node1PublicKey] = true;
+        ispList[isp2].capacity += channel.capacity;
+        ispList[isp2].channels += 1;
+        ispList[isp2].nodes[channel.node2PublicKey] = true;
       }
 
-      // Tor
-      if (showTor) {
-        query = `SELECT COUNT(DISTINCT nodes.public_key) as nodesCount, CAST(SUM(capacity) as INT) as capacity
-          FROM nodes
-          JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key
-          ORDER BY ${orderBy} DESC
-        `;      
-        const [nodesCountTor]: any = await DB.query(query);
-
-        total += groupBy === 'capacity' ? nodesCountTor[0].capacity : nodesCountTor[0].nodesCount;
-        nodesPerAs.push({
-          ispId: null,
-          name: 'Tor',
-          count: nodesCountTor[0].nodesCount,
-          share: Math.floor((groupBy === 'capacity' ? nodesCountTor[0].capacity : nodesCountTor[0].nodesCount) / total * 10000) / 100,
-          capacity: nodesCountTor[0].capacity,
-        });
+      const ispRanking: any[] = [];
+      for (const isp of Object.keys(ispList)) {
+        ispRanking.push([
+          ispList[isp].id,
+          isp,
+          ispList[isp].capacity,
+          ispList[isp].channels,
+          Object.keys(ispList[isp].nodes).length,
+        ]);
       }
 
-      for (const as of nodesCountPerAS) {
-        nodesPerAs.push({
-          ispId: as.ispId,
-          name: JSON.parse(as.names),
-          count: as.nodesCount,
-          share: Math.floor((groupBy === 'capacity' ? as.capacity : as.nodesCount) / total * 10000) / 100,
-          capacity: as.capacity,
-        });
-      }
+      // Total active channels capacity
+      query = `SELECT SUM(capacity) AS capacity FROM channels WHERE status = 1`;
+      const [totalCapacity]: any = await DB.query(query);
 
-      return nodesPerAs;
+      // Get the total capacity of all channels which have at least one node on clearnet
+      query = `
+        SELECT SUM(capacity) as capacity
+        FROM (
+          SELECT capacity, GROUP_CONCAT(socket1.type, socket2.type) as networks
+          FROM channels
+          JOIN nodes_sockets socket1 ON node1_public_key = socket1.public_key
+          JOIN nodes_sockets socket2 ON node2_public_key = socket2.public_key
+          AND channels.status = 1
+          GROUP BY short_id
+        ) channels_tmp
+        WHERE channels_tmp.networks LIKE '%ipv%'
+      `;
+      const [clearnetCapacity]: any = await DB.query(query);
+
+      // Get the total capacity of all channels which have both nodes on Tor 
+      query = `
+        SELECT SUM(capacity) as capacity
+        FROM (
+          SELECT capacity, GROUP_CONCAT(socket1.type, socket2.type) as networks
+          FROM channels
+          JOIN nodes_sockets socket1 ON node1_public_key = socket1.public_key
+          JOIN nodes_sockets socket2 ON node2_public_key = socket2.public_key
+          AND channels.status = 1
+          GROUP BY short_id
+        ) channels_tmp
+        WHERE channels_tmp.networks NOT LIKE '%ipv%' AND
+          channels_tmp.networks NOT LIKE '%dns%' AND
+          channels_tmp.networks NOT LIKE '%websocket%'
+      `;
+      const [torCapacity]: any = await DB.query(query);
+
+      const clearnetCapacityValue = parseInt(clearnetCapacity[0].capacity, 10);
+      const torCapacityValue = parseInt(torCapacity[0].capacity, 10);
+      const unknownCapacityValue = parseInt(totalCapacity[0].capacity) - clearnetCapacityValue - torCapacityValue;
+
+      return {
+        clearnetCapacity: clearnetCapacityValue,
+        torCapacity: torCapacityValue,
+        unknownCapacity: unknownCapacityValue,
+        ispRanking: ispRanking,
+      };
     } catch (e) {
-      logger.err(`Cannot get nodes grouped by AS. Reason: ${e instanceof Error ? e.message : e}`);
+      logger.err(`Cannot get LN ISP ranking. Reason: ${e instanceof Error ? e.message : e}`);
       throw e;
     }
   }
@@ -163,25 +431,27 @@ class NodesApi {
   public async $getNodesPerCountry(countryId: string) {
     try {
       const query = `
-        SELECT node_stats.public_key, node_stats.capacity, node_stats.channels, nodes.alias,
-          UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
-          geo_names_city.names as city
-        FROM node_stats
-        JOIN (
-          SELECT public_key, MAX(added) as last_added
-          FROM node_stats
-          GROUP BY public_key
-        ) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added
-        JOIN nodes ON nodes.public_key = node_stats.public_key
-        JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+        SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+          nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
+          geo_names_city.names as city, geo_names_country.names as country,
+          geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision,
+          nodes.longitude, nodes.latitude, nodes.as_number, geo_names_isp.names as isp
+        FROM nodes
+        LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
         LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+        LEFT JOIN geo_names geo_names_isp on geo_names_isp.id = nodes.as_number AND geo_names_isp.type = 'as_organization'
         WHERE geo_names_country.id = ?
         ORDER BY capacity DESC
       `;
 
       const [rows]: any = await DB.query(query, [countryId]);
       for (let i = 0; i < rows.length; ++i) {
+        rows[i].country = JSON.parse(rows[i].country);
         rows[i].city = JSON.parse(rows[i].city);
+        rows[i].subdivision = JSON.parse(rows[i].subdivision);
+        rows[i].isp = JSON.parse(rows[i].isp);
       }
       return rows;
     } catch (e) {
@@ -193,18 +463,16 @@ class NodesApi {
   public async $getNodesPerISP(ISPId: string) {
     try {
       const query = `
-        SELECT node_stats.public_key, node_stats.capacity, node_stats.channels, nodes.alias,
-          UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
-          geo_names_city.names as city, geo_names_country.names as country
-        FROM node_stats
-        JOIN (
-          SELECT public_key, MAX(added) as last_added
-          FROM node_stats
-          GROUP BY public_key
-        ) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added
-        RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
-        JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+        SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+          nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
+          geo_names_city.names as city, geo_names_country.names as country,
+          geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision,
+          nodes.longitude, nodes.latitude
+        FROM nodes
+        LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
         LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
         WHERE nodes.as_number IN (?)
         ORDER BY capacity DESC
       `;
@@ -213,6 +481,7 @@ class NodesApi {
       for (let i = 0; i < rows.length; ++i) {
         rows[i].country = JSON.parse(rows[i].country);
         rows[i].city = JSON.parse(rows[i].city);
+        rows[i].subdivision = JSON.parse(rows[i].subdivision);
       }
       return rows;
     } catch (e) {
@@ -227,7 +496,6 @@ class NodesApi {
         FROM nodes
         JOIN geo_names ON geo_names.id = nodes.country_id AND geo_names.type = 'country'
         JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
-        JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key
         GROUP BY country_id
         ORDER BY COUNT(DISTINCT nodes.public_key) DESC
       `;
@@ -253,6 +521,85 @@ class NodesApi {
       throw e;
     }
   }
+
+  /**
+   * Save or update a node present in the graph
+   */
+  public async $saveNode(node: ILightningApi.Node): Promise<void> {
+    try {
+      const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
+      const query = `INSERT INTO nodes(
+          public_key,
+          first_seen,
+          updated_at,
+          alias,
+          alias_search,
+          color,
+          sockets,
+          status
+        )
+        VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, ?, 1)
+        ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, alias_search = ?, color = ?, sockets = ?, status = 1`;
+
+      await DB.query(query, [
+        node.pub_key,
+        node.last_update,
+        node.alias,
+        this.aliasToSearchText(node.alias),
+        node.color,
+        sockets,
+        node.last_update,
+        node.alias,
+        this.aliasToSearchText(node.alias),
+        node.color,
+        sockets,
+      ]);
+    } catch (e) {
+      logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Update node sockets
+   */
+  public async $updateNodeSockets(publicKey: string, sockets: {network: string; addr: string}[]): Promise<void> {
+    const formattedSockets = (sockets.map(a => a.addr).join(',')) ?? '';
+    try {
+      await DB.query(`UPDATE nodes SET sockets = ? WHERE public_key = ?`, [formattedSockets, publicKey]);
+    } catch (e) {
+      logger.err(`Cannot update node sockets for ${publicKey}. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  /**
+   * Set all nodes not in `nodesPubkeys` as inactive (status = 0)
+   */
+   public async $setNodesInactive(graphNodesPubkeys: string[]): Promise<void> {
+    if (graphNodesPubkeys.length === 0) {
+      return;
+    }
+
+    try {
+      const result = await DB.query<ResultSetHeader>(`
+        UPDATE nodes
+        SET status = 0
+        WHERE public_key NOT IN (
+          ${graphNodesPubkeys.map(pubkey => `"${pubkey}"`).join(',')}
+        )
+      `);
+      if (result[0].changedRows ?? 0 > 0) {
+        logger.info(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`);
+      } else {
+        logger.debug(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`);
+      }
+    } catch (e) {
+      logger.err('$setNodesInactive() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private aliasToSearchText(str: string): string {
+    return str.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '');
+  }
 }
 
 export default new NodesApi();

backend/src/api/explorer/nodes.routes.ts

@@ -2,20 +2,26 @@ import config from '../../config';
 import { Application, Request, Response } from 'express';
 import nodesApi from './nodes.api';
 import DB from '../../database';
+import { INodesRanking } from '../../mempool.interfaces';
 
 class NodesRoutes {
   constructor() { }
 
   public initRoutes(app: Application) {
     app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/world', this.$getWorldNodes)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/country/:country', this.$getNodesPerCountry)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/search/:search', this.$searchNode)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/top', this.$getTopNodes)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp-ranking', this.$getISPRanking)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp/:isp', this.$getNodesPerISP)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/countries', this.$getNodesCountries)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings', this.$getNodesRanking)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/liquidity', this.$getTopNodesByCapacity)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/connectivity', this.$getTopNodesByChannels)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/age', this.$getOldestNodes)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/statistics', this.$getHistoricalNodeStats)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key', this.$getNode)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/group/:name', this.$getNodeGroup)
     ;
   }
 
@@ -28,6 +34,39 @@ class NodesRoutes {
     }
   }
 
+  private async $getNodeGroup(req: Request, res: Response) {
+    try {
+      let nodesList;
+      let nodes: any[] = [];
+      switch (config.MEMPOOL.NETWORK) {
+        case 'testnet':
+          nodesList = ['032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b', '025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7', '0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55', '032ab2028c0b614c6d87824e2373529652fd7e4221b4c70cc4da7c7005c49afcf0', '029001b22fe70b48bee12d014df91982eb85ff1bd404ec772d5c83c4ee3e88d2c3', '0212e2848d79f928411da5f2ff0a8c95ec6ccb5a09d2031b6f71e91309dcde63af', '03e871a2229523d34f76e6311ff197cfe7f26c2fbec13554b93a46f4e710c47dab', '032202ec98d976b0e928bd1d91924e8bd3eab07231fc39feb3737b010071073df8', '02fa7c5a948d03d563a9f36940c2205a814e594d17c0042ced242c71a857d72605', '039c14fdec2d958e3d14cebf657451bbd9e039196615785e82c917f274e3fb2205', '033589bbcb233ffc416cefd5437c7f37e9d7cb7942d405e39e72c4c846d9b37f18', '029293110441c6e2eacb57e1255bf6ef05c41a6a676fe474922d33c19f98a7d584'];
+          break;
+        case 'signet':
+          nodesList = ['03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956', '033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de', '02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781', '025196512905b8a3f1597428b867bec63ec9a95e5089eb7dc7e63e2d2691669029', '027c625aa1fbe3768db68ebcb05b53b6dc0ce68b7b54b8900d326d167363e684fe', '03f1629af3101fcc56b7aac2667016be84e3defbf3d0c8719f836c9b41c9a57a43', '02dfb81e2f7a3c4c9e8a51b70ef82b4a24549cc2fab1f5b2fd636501774a918991', '02d01ccf832944c68f10d39006093769c5b8bda886d561b128534e313d729fdb34', '02499ed23027d4698a6904ff4ec1b6085a61f10b9a6937f90438f9947e38e8ea86', '038310e3a786340f2bd7770704c7ccfe560fd163d9a1c99d67894597419d12cbf7', '03e5e9d879b72c7d67ecd483bae023bd33e695bb32b981a4021260f7b9d62bc761', '028d16e1a0ace4c0c0a421536d8d32ce484dfe6e2f726b7b0e7c30f12a195f8cc7'];
+          break;
+        default:
+          nodesList = ['03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61', '03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437', '03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144', '0238bd27f02d67d6c51e269692bc8c9a32357a00e7777cba7f4f1f18a2a700b108', '03f983dcabed6baa1eab5b56c8b2e8fdc846ab3fd931155377897335e85a9fa57c', '03e399589533581e48796e29a825839a010036a61b20744fda929d6709fcbffcc5', '021f5288b5f72c42cd0d8801086af7ce09a816d8ee9a4c47a4b436399b26cb601a', '032b01b7585f781420cd4148841a82831ba37fa952342052cec16750852d4f2dd9', '02848036488d4b8fb1f1c4064261ec36151f43b085f0b51bd239ade3ddfc940c34', '02b6b1640fe029e304c216951af9fbefdb23b0bdc9baaf327540d31b6107841fdf', '03694289827203a5b3156d753071ddd5bf92e371f5a462943f9555eef6d2d6606c', '0283d850db7c3e8ea7cc9c4abc7afaab12bbdf72b677dcba1d608350d2537d7d43'];
+      }
+
+      for (let pubKey of nodesList) {
+        try {
+          const node = await nodesApi.$getNode(pubKey);
+          if (node) {
+            nodes.push(node);
+          }
+        } catch (e) {}
+      }
+      
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(nodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async $getNode(req: Request, res: Response) {
     try {
       const node = await nodesApi.$getNode(req.params.public_key);
@@ -35,6 +74,9 @@ class NodesRoutes {
         res.status(404).send('Node not found');
         return;
       }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(node);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
@@ -44,17 +86,23 @@ class NodesRoutes {
   private async $getHistoricalNodeStats(req: Request, res: Response) {
     try {
       const statistics = await nodesApi.$getNodeStats(req.params.public_key);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(statistics);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }
   }
 
-  private async $getTopNodes(req: Request, res: Response) {
+  private async $getNodesRanking(req: Request, res: Response): Promise<void> {
     try {
-      const topCapacityNodes = await nodesApi.$getTopCapacityNodes();
-      const topChannelsNodes = await nodesApi.$getTopChannelsNodes();
-      res.json({
+      const topCapacityNodes = await nodesApi.$getTopCapacityNodes(false);
+      const topChannelsNodes = await nodesApi.$getTopChannelsNodes(false);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(<INodesRanking>{
         topByCapacity: topCapacityNodes,
         topByChannels: topChannelsNodes,
       });
@@ -63,18 +111,45 @@ class NodesRoutes {
     }
   }
 
+  private async $getTopNodesByCapacity(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getTopCapacityNodes(true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(topCapacityNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getTopNodesByChannels(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getTopChannelsNodes(true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(topCapacityNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getOldestNodes(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getOldestNodes(true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(topCapacityNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async $getISPRanking(req: Request, res: Response): Promise<void> {
     try {
-      const groupBy = req.query.groupBy as string;
-      const showTor = req.query.showTor as string === 'true' ? true : false;
-
-      if (!['capacity', 'node-count'].includes(groupBy)) {
-        res.status(400).send(`groupBy must be one of 'capacity' or 'node-count'`);
-        return;
-      }
-
-      const nodesPerAs = await nodesApi.$getNodesISP(groupBy, showTor);
-
+      const nodesPerAs = await nodesApi.$getNodesISPRanking();
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
@@ -84,6 +159,18 @@ class NodesRoutes {
     }
   }
 
+  private async $getWorldNodes(req: Request, res: Response) {
+    try {
+      const worldNodes = await nodesApi.$getWorldNodes();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
+      res.json(worldNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async $getNodesPerCountry(req: Request, res: Response) {
     try {
       const [country]: any[] = await DB.query(

backend/src/api/explorer/statistics.api.ts

@@ -6,7 +6,8 @@ class StatisticsApi {
   public async $getStatistics(interval: string | null = null): Promise<any> {
     interval = Common.getSqlInterval(interval);
 
-    let query = `SELECT UNIX_TIMESTAMP(added) AS added, channel_count, total_capacity, tor_nodes, clearnet_nodes, unannounced_nodes
+    let query = `SELECT UNIX_TIMESTAMP(added) AS added, channel_count, total_capacity,
+      tor_nodes, clearnet_nodes, unannounced_nodes, clearnet_tor_nodes
       FROM lightning_stats`;
 
     if (interval) {
@@ -27,7 +28,7 @@ class StatisticsApi {
   public async $getLatestStatistics(): Promise<any> {
     try {
       const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1`);
-      const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1 OFFSET 7`);
+      const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats WHERE DATE(added) = DATE(NOW() - INTERVAL 7 DAY)`);
       return {
         latest: rows[0],
         previous: rows2[0],

backend/src/api/fetch-version.ts

@@ -0,0 +1,37 @@
+import fs from 'fs';
+import path from "path";
+const { spawnSync } = require('child_process');
+
+function getVersion(): string {
+  const packageJson = fs.readFileSync('package.json').toString();
+  return JSON.parse(packageJson).version;
+}
+
+function getGitCommit(): string {
+  if (process.env.MEMPOOL_COMMIT_HASH) {
+    return process.env.MEMPOOL_COMMIT_HASH;
+  } else {
+    const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
+    if (!gitRevParse.error) {
+      const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
+      if (output) {
+        return output;
+      } else {
+        console.log('Could not fetch git commit: No repo available');
+      }
+    } else if (gitRevParse.error.code === 'ENOENT') {
+      console.log('Could not fetch git commit: Command `git` is unavailable');
+    }
+  }
+  return '?';
+}
+
+const versionInfo = {
+  version: getVersion(),
+  gitCommit: getGitCommit()
+}
+
+fs.writeFileSync(
+  path.join(__dirname, 'version.json'),
+  JSON.stringify(versionInfo, null, 2) + "\n"
+);

backend/src/api/lightning/clightning/clightning-client.ts

@@ -0,0 +1,272 @@
+// Imported from https://github.com/shesek/lightning-client-js
+
+'use strict';
+
+const methods = [
+  'addgossip',
+  'autocleaninvoice',
+  'check',
+  'checkmessage',
+  'close',
+  'connect',
+  'createinvoice',
+  'createinvoicerequest',
+  'createoffer',
+  'createonion',
+  'decode',
+  'decodepay',
+  'delexpiredinvoice',
+  'delinvoice',
+  'delpay',
+  'dev-listaddrs',
+  'dev-rescan-outputs',
+  'disableoffer',
+  'disconnect',
+  'estimatefees',
+  'feerates',
+  'fetchinvoice',
+  'fundchannel',
+  'fundchannel_cancel',
+  'fundchannel_complete',
+  'fundchannel_start',
+  'fundpsbt',
+  'getchaininfo',
+  'getinfo',
+  'getlog',
+  'getrawblockbyheight',
+  'getroute',
+  'getsharedsecret',
+  'getutxout',
+  'help',
+  'invoice',
+  'keysend',
+  'legacypay',
+  'listchannels',
+  'listconfigs',
+  'listforwards',
+  'listfunds',
+  'listinvoices',
+  'listnodes',
+  'listoffers',
+  'listpays',
+  'listpeers',
+  'listsendpays',
+  'listtransactions',
+  'multifundchannel',
+  'multiwithdraw',
+  'newaddr',
+  'notifications',
+  'offer',
+  'offerout',
+  'openchannel_abort',
+  'openchannel_bump',
+  'openchannel_init',
+  'openchannel_signed',
+  'openchannel_update',
+  'pay',
+  'payersign',
+  'paystatus',
+  'ping',
+  'plugin',
+  'reserveinputs',
+  'sendinvoice',
+  'sendonion',
+  'sendonionmessage',
+  'sendpay',
+  'sendpsbt',
+  'sendrawtransaction',
+  'setchannelfee',
+  'signmessage',
+  'signpsbt',
+  'stop',
+  'txdiscard',
+  'txprepare',
+  'txsend',
+  'unreserveinputs',
+  'utxopsbt',
+  'waitanyinvoice',
+  'waitblockheight',
+  'waitinvoice',
+  'waitsendpay',
+  'withdraw'
+];
+
+
+import EventEmitter from 'events';
+import { existsSync, statSync } from 'fs';
+import { createConnection, Socket } from 'net';
+import { homedir } from 'os';
+import path from 'path';
+import { createInterface, Interface } from 'readline';
+import logger from '../../../logger';
+import { AbstractLightningApi } from '../lightning-api-abstract-factory';
+import { ILightningApi } from '../lightning-api.interface';
+import { convertAndmergeBidirectionalChannels, convertNode } from './clightning-convert';
+
+class LightningError extends Error {
+  type: string = 'lightning';
+  message: string = 'lightning-client error';
+
+  constructor(error) {
+    super();
+    this.type = error.type;
+    this.message = error.message;
+  }
+}
+
+const defaultRpcPath = path.join(homedir(), '.lightning')
+  , fStat = (...p) => statSync(path.join(...p))
+  , fExists = (...p) => existsSync(path.join(...p))
+
+export default class CLightningClient extends EventEmitter implements AbstractLightningApi {
+  private rpcPath: string;
+  private reconnectWait: number;
+  private reconnectTimeout;
+  private reqcount: number;
+  private client: Socket;
+  private rl: Interface;
+  private clientConnectionPromise: Promise<unknown>;
+
+  constructor(rpcPath = defaultRpcPath) {
+    if (!path.isAbsolute(rpcPath)) {
+      throw new Error('The rpcPath must be an absolute path');
+    }
+
+    if (!fExists(rpcPath) || !fStat(rpcPath).isSocket()) {
+      // network directory provided, use the lightning-rpc within in
+      if (fExists(rpcPath, 'lightning-rpc')) {
+        rpcPath = path.join(rpcPath, 'lightning-rpc');
+      }
+
+      // main data directory provided, default to using the bitcoin mainnet subdirectory
+      // to be removed in v0.2.0
+      else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) {
+        logger.warn(`[CLightningClient] ${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`)
+        logger.warn(`[CLightningClient] specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`)
+        rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc')
+      }
+    }
+
+    logger.debug(`[CLightningClient] Connecting to ${rpcPath}`);
+
+    super();
+    this.rpcPath = rpcPath;
+    this.reconnectWait = 0.5;
+    this.reconnectTimeout = null;
+    this.reqcount = 0;
+
+    const _self = this;
+
+    this.client = createConnection(rpcPath).on(
+      'error', () => {
+        _self.increaseWaitTime();
+        _self.reconnect();
+      }
+    );
+    this.rl = createInterface({ input: this.client }).on(
+      'error', () => {
+        _self.increaseWaitTime();
+        _self.reconnect();
+      }
+    );
+
+    this.clientConnectionPromise = new Promise<void>(resolve => {
+      _self.client.on('connect', () => {
+        logger.info(`[CLightningClient] Lightning client connected`);
+        _self.reconnectWait = 1;
+        resolve();
+      });
+
+      _self.client.on('end', () => {
+        logger.err('[CLightningClient] Lightning client connection closed, reconnecting');
+        _self.increaseWaitTime();
+        _self.reconnect();
+      });
+
+      _self.client.on('error', error => {
+        logger.err(`[CLightningClient] Lightning client connection error: ${error}`);
+        _self.increaseWaitTime();
+        _self.reconnect();
+      });
+    });
+
+    this.rl.on('line', line => {
+      line = line.trim();
+      if (!line) {
+        return;
+      }
+      const data = JSON.parse(line);
+      // logger.debug(`[CLightningClient] #${data.id} <-- ${JSON.stringify(data.error || data.result)}`);
+      _self.emit('res:' + data.id, data);
+    });
+  }
+
+  increaseWaitTime(): void {
+    if (this.reconnectWait >= 16) {
+      this.reconnectWait = 16;
+    } else {
+      this.reconnectWait *= 2;
+    }
+  }
+
+  reconnect(): void {
+    const _self = this;
+
+    if (this.reconnectTimeout) {
+      return;
+    }
+
+    this.reconnectTimeout = setTimeout(() => {
+      logger.debug('[CLightningClient] Trying to reconnect...');
+
+      _self.client.connect(_self.rpcPath);
+      _self.reconnectTimeout = null;
+    }, this.reconnectWait * 1000);
+  }
+
+  call(method, args = []): Promise<any> {
+    const _self = this;
+
+    const callInt = ++this.reqcount;
+    const sendObj = {
+      jsonrpc: '2.0',
+      method,
+      params: args,
+      id: '' + callInt
+    };
+
+    // logger.debug(`[CLightningClient] #${callInt} --> ${method} ${args}`);
+
+    // Wait for the client to connect
+    return this.clientConnectionPromise
+      .then(() => new Promise((resolve, reject) => {
+        // Wait for a response
+        this.once('res:' + callInt, res => res.error == null
+          ? resolve(res.result)
+          : reject(new LightningError(res.error))
+        );
+
+        // Send the command
+        _self.client.write(JSON.stringify(sendObj));
+      }));
+  }
+
+  async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
+    const listnodes: any[] = await this.call('listnodes');
+    const listchannels: any[] = await this.call('listchannels');
+    const channelsList = await convertAndmergeBidirectionalChannels(listchannels['channels']);
+
+    return {
+      nodes: listnodes['nodes'].map(node => convertNode(node)),
+      edges: channelsList,
+    };
+  }
+}
+
+const protify = s => s.replace(/-([a-z])/g, m => m[1].toUpperCase());
+
+methods.forEach(k => {
+  CLightningClient.prototype[protify(k)] = function (...args: any) {
+    return this.call(k, args);
+  };
+});

backend/src/api/lightning/clightning/clightning-convert.ts

@@ -0,0 +1,135 @@
+import { ILightningApi } from '../lightning-api.interface';
+import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher';
+import logger from '../../../logger';
+import { Common } from '../../common';
+
+/**
+ * Convert a clightning "listnode" entry to a lnd node entry
+ */
+export function convertNode(clNode: any): ILightningApi.Node {
+  return {
+    alias: clNode.alias ?? '',
+    color: `#${clNode.color ?? ''}`,
+    features: [], // TODO parse and return clNode.feature
+    pub_key: clNode.nodeid,
+    addresses: clNode.addresses?.map((addr) => {
+      let address = addr.address;
+      if (addr.type === 'ipv6') {
+        address = `[${address}]`;
+      }
+      return {
+        network: addr.type,
+        addr: `${address}:${addr.port}`
+      };
+    }) ?? [],
+    last_update: clNode?.last_timestamp ?? 0,
+  };
+}
+
+/**
+ * Convert clightning "listchannels" response to lnd "describegraph.edges" format
+ */
+export async function convertAndmergeBidirectionalChannels(clChannels: any[]): Promise<ILightningApi.Channel[]> {
+  logger.info('Converting clightning nodes and channels to lnd graph format');
+
+  let loggerTimer = new Date().getTime() / 1000;
+  let channelProcessed = 0;
+
+  const consolidatedChannelList: ILightningApi.Channel[] = [];
+  const clChannelsDict = {};
+  const clChannelsDictCount = {};
+
+  for (const clChannel of clChannels) {
+    if (!clChannelsDict[clChannel.short_channel_id]) {
+      clChannelsDict[clChannel.short_channel_id] = clChannel;
+      clChannelsDictCount[clChannel.short_channel_id] = 1;
+    } else {
+      consolidatedChannelList.push(
+        await buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id])
+      );
+      delete clChannelsDict[clChannel.short_channel_id];
+      clChannelsDictCount[clChannel.short_channel_id]++;
+    }
+
+    const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+    if (elapsedSeconds > 10) {
+      logger.info(`Building complete channels from clightning output. Channels processed: ${channelProcessed + 1} of ${clChannels.length}`);
+      loggerTimer = new Date().getTime() / 1000;
+    }
+
+    ++channelProcessed;
+  }
+
+  channelProcessed = 0;
+  const keys = Object.keys(clChannelsDict);
+  for (const short_channel_id of keys) {
+    consolidatedChannelList.push(await buildIncompleteChannel(clChannelsDict[short_channel_id]));
+
+    const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+    if (elapsedSeconds > 10) {
+      logger.info(`Building partial channels from clightning output. Channels processed: ${channelProcessed + 1} of ${keys.length}`);
+      loggerTimer = new Date().getTime() / 1000;
+    }
+  }
+
+  return consolidatedChannelList;
+}
+
+/**
+ * Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format
+ * In this case, clightning knows the channel policy for both nodes
+ */
+async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILightningApi.Channel> {
+  const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0);
+
+  const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannelA.short_channel_id);
+  const parts = clChannelA.short_channel_id.split('x');
+  const outputIdx = parts[2];
+
+  return {
+    channel_id: Common.channelShortIdToIntegerId(clChannelA.short_channel_id),
+    capacity: clChannelA.satoshis,
+    last_update: lastUpdate,
+    node1_policy: convertPolicy(clChannelA),
+    node2_policy: convertPolicy(clChannelB),
+    chan_point: `${tx.txid}:${outputIdx}`,
+    node1_pub: clChannelA.source,
+    node2_pub: clChannelB.source,
+  };
+}
+
+/**
+ * Convert one clightning "getchannels" entry into a full a lnd "describegraph.edges" format
+ * In this case, clightning knows the channel policy of only one node
+ */
+async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Channel> {
+  const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannel.short_channel_id);
+  const parts = clChannel.short_channel_id.split('x');
+  const outputIdx = parts[2];
+
+  return {
+    channel_id: Common.channelShortIdToIntegerId(clChannel.short_channel_id),
+    capacity: clChannel.satoshis,
+    last_update: clChannel.last_update ?? 0,
+    node1_policy: convertPolicy(clChannel),
+    node2_policy: null,
+    chan_point: `${tx.txid}:${outputIdx}`,
+    node1_pub: clChannel.source,
+    node2_pub: clChannel.destination,
+  };
+}
+
+/**
+ * Convert a clightning "listnode" response to a lnd channel policy format
+ */
+function convertPolicy(clChannel: any): ILightningApi.RoutingPolicy {
+  return {
+    time_lock_delta: clChannel.delay,
+    min_htlc: clChannel.htlc_minimum_msat.slice(0, -4),
+    max_htlc_msat: clChannel.htlc_maximum_msat.slice(0, -4),
+    fee_base_msat: clChannel.base_fee_millisatoshi,
+    fee_rate_milli_msat: clChannel.fee_per_millionth,
+    disabled: !clChannel.active,
+    last_update: clChannel.last_update ?? 0,
+  };
+}

backend/src/api/lightning/lightning-api-abstract-factory.ts

@@ -1,7 +1,5 @@
 import { ILightningApi } from './lightning-api.interface';
 
 export interface AbstractLightningApi {
-  $getNetworkInfo(): Promise<ILightningApi.NetworkInfo>;
   $getNetworkGraph(): Promise<ILightningApi.NetworkGraph>;
-  $getInfo(): Promise<ILightningApi.Info>;
 }

backend/src/api/lightning/lightning-api-factory.ts

@@ -1,9 +1,12 @@
 import config from '../../config';
+import CLightningClient from './clightning/clightning-client';
 import { AbstractLightningApi } from './lightning-api-abstract-factory';
 import LndApi from './lnd/lnd-api';
 
 function lightningApiFactory(): AbstractLightningApi {
-  switch (config.LIGHTNING.BACKEND) {
+  switch (config.LIGHTNING.ENABLED === true && config.LIGHTNING.BACKEND) {
+    case 'cln':
+      return new CLightningClient(config.CLIGHTNING.SOCKET);
     case 'lnd':
     default:
       return new LndApi();

backend/src/api/lightning/lightning-api.interface.ts

@@ -1,71 +1,85 @@
 export namespace ILightningApi {
   export interface NetworkInfo {
-    average_channel_size: number;
-    channel_count: number;
-    max_channel_size: number;
-    median_channel_size: number;
-    min_channel_size: number;
-    node_count: number;
-    not_recently_updated_policy_count: number;
-    total_capacity: number;
+    graph_diameter: number;
+    avg_out_degree: number;
+    max_out_degree: number;
+    num_nodes: number;
+    num_channels: number;
+    total_network_capacity: string;
+    avg_channel_size: number;
+    min_channel_size: string;
+    max_channel_size: string;
+    median_channel_size_sat: string;
+    num_zombie_chans: string;
   }
 
   export interface NetworkGraph {
-    channels: Channel[];
     nodes: Node[];
+    edges: Channel[];
   }
 
   export interface Channel {
-    id: string;
-    capacity: number;
-    policies: Policy[];
-    transaction_id: string;
-    transaction_vout: number;
-    updated_at?: string;
+    channel_id: string;
+    chan_point: string;
+    last_update: number;
+    node1_pub: string;
+    node2_pub: string;
+    capacity: string;
+    node1_policy: RoutingPolicy | null;
+    node2_policy: RoutingPolicy | null;
   }
 
-  interface Policy {
-    public_key: string;
-    base_fee_mtokens?: string;
-    cltv_delta?: number;
-    fee_rate?: number;
-    is_disabled?: boolean;
-    max_htlc_mtokens?: string;
-    min_htlc_mtokens?: string;
-    updated_at?: string;
+  export interface RoutingPolicy {
+    time_lock_delta: number;
+    min_htlc: string;
+    fee_base_msat: string;
+    fee_rate_milli_msat: string;
+    disabled: boolean;
+    max_htlc_msat: string;
+    last_update: number;
   }
 
   export interface Node {
+    last_update: number;
+    pub_key: string;
     alias: string;
+    addresses: {
+      network: string;
+      addr: string;
+    }[];
     color: string;
-    features: Feature[];
-    public_key: string;
-    sockets: string[];
-    updated_at?: string;
+    features: { [key: number]: Feature };
   }
 
   export interface Info {
-    chains: string[];
-    color: string;
-    active_channels_count: number;
+    identity_pubkey: string;
     alias: string;
-    current_block_hash: string;
-    current_block_height: number;
-    features: Feature[];
-    is_synced_to_chain: boolean;
-    is_synced_to_graph: boolean;
-    latest_block_at: string;
-    peers_count: number;
-    pending_channels_count: number;
-    public_key: string;
-    uris: any[];
+    num_pending_channels: number;
+    num_active_channels: number;
+    num_peers: number;
+    block_height: number;
+    block_hash: string;
+    synced_to_chain: boolean;
+    testnet: boolean;
+    uris: string[];
+    best_header_timestamp: string;
     version: string;
+    num_inactive_channels: number;
+    chains: {
+      chain: string;
+      network: string;
+    }[];
+    color: string;
+    synced_to_graph: boolean;
+    features: { [key: number]: Feature };
+    commit_hash: string;
+    /** Available on LND since v0.15.0-beta */
+    require_htlc_interceptor?: boolean;
   }
-  
+
   export interface Feature {
-    bit: number;
-    is_known: boolean;
+    name: string;
     is_required: boolean;
-    type?: string;
+    is_known: boolean;
   }
-}
+}

backend/src/api/lightning/lnd/lnd-api.ts

@@ -1,44 +1,40 @@
+import axios, { AxiosRequestConfig } from 'axios';
+import { Agent } from 'https';
+import * as fs from 'fs';
 import { AbstractLightningApi } from '../lightning-api-abstract-factory';
 import { ILightningApi } from '../lightning-api.interface';
-import * as fs from 'fs';
-import { authenticatedLndGrpc, getWalletInfo, getNetworkGraph, getNetworkInfo } from 'lightning';
 import config from '../../../config';
-import logger from '../../../logger';
 
 class LndApi implements AbstractLightningApi {
-  private lnd: any;
+  axiosConfig: AxiosRequestConfig = {};
+
   constructor() {
-    if (!config.LIGHTNING.ENABLED) {
-      return;
-    }
-    try {
-      const tls = fs.readFileSync(config.LND.TLS_CERT_PATH).toString('base64');
-      const macaroon = fs.readFileSync(config.LND.MACAROON_PATH).toString('base64');
-
-      const { lnd } = authenticatedLndGrpc({
-        cert: tls,
-        macaroon: macaroon,
-        socket: config.LND.SOCKET,
-      });
-
-      this.lnd = lnd;
-    } catch (e) {
-      logger.err('Could not initiate the LND service handler: ' + (e instanceof Error ? e.message : e));
-      process.exit(1);
+    if (config.LIGHTNING.ENABLED) {
+      this.axiosConfig = {
+        headers: {
+          'Grpc-Metadata-macaroon': fs.readFileSync(config.LND.MACAROON_PATH).toString('hex')
+        },
+        httpsAgent: new Agent({
+          ca: fs.readFileSync(config.LND.TLS_CERT_PATH)
+        }),
+        timeout: 10000
+      };
     }
   }
 
   async $getNetworkInfo(): Promise<ILightningApi.NetworkInfo> {
-    return await getNetworkInfo({ lnd: this.lnd });
+    return axios.get<ILightningApi.NetworkInfo>(config.LND.REST_API_URL + '/v1/graph/info', this.axiosConfig)
+      .then((response) => response.data);
   }
 
   async $getInfo(): Promise<ILightningApi.Info> {
-    // @ts-ignore
-    return await getWalletInfo({ lnd: this.lnd });
+    return axios.get<ILightningApi.Info>(config.LND.REST_API_URL + '/v1/getinfo', this.axiosConfig)
+      .then((response) => response.data);
   }
 
   async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
-    return await getNetworkGraph({ lnd: this.lnd });
+    return axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
+      .then((response) => response.data);
   }
 }
 

backend/src/api/mining/mining.ts

@@ -473,7 +473,7 @@ class Mining {
 
       for (const block of blocksWithoutPrices) {
         // Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks
-        if (block.height < 68951) {
+        if (['mainnet', 'testnet'].includes(config.MEMPOOL.NETWORK) && block.height < 68951) {
           blocksPrices.push({
             height: block.height,
             priceId: prices[0].id,
@@ -492,11 +492,11 @@ class Mining {
 
         if (blocksPrices.length >= 100000) {
           totalInserted += blocksPrices.length;
+          let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
           if (blocksWithoutPrices.length > 200000) {
-            logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`);
-          } else {
-            logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
+            logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
           }
+          logger.debug(logStr);
           await BlocksRepository.$saveBlockPrices(blocksPrices);
           blocksPrices.length = 0;
         }
@@ -504,11 +504,11 @@ class Mining {
 
       if (blocksPrices.length > 0) {
         totalInserted += blocksPrices.length;
+        let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
         if (blocksWithoutPrices.length > 200000) {
-          logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`);
-        } else {
-          logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
+          logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
         }
+        logger.debug(logStr);
         await BlocksRepository.$saveBlockPrices(blocksPrices);
       }
     } catch (e) {

backend/src/config.ts

@@ -1,4 +1,6 @@
-const configFile = require('../mempool-config.json');
+const configFromFile = require(
+    process.env.MEMPOOL_CONFIG_FILE ? process.env.MEMPOOL_CONFIG_FILE : '../mempool-config.json'
+);
 
 interface IConfig {
   MEMPOOL: {
@@ -24,6 +26,8 @@ interface IConfig {
     USER_AGENT: string;
     STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
     AUTOMATIC_BLOCK_REINDEXING: boolean;
+    POOLS_JSON_URL: string,
+    POOLS_JSON_TREE_URL: string,
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -31,10 +35,17 @@ interface IConfig {
   LIGHTNING: {
     ENABLED: boolean;
     BACKEND: 'lnd' | 'cln' | 'ldk';
+    TOPOLOGY_FOLDER: string;
+    STATS_REFRESH_INTERVAL: number;
+    GRAPH_REFRESH_INTERVAL: number;
+    LOGGER_UPDATE_INTERVAL: number;
   };
   LND: {
     TLS_CERT_PATH: string;
     MACAROON_PATH: string;
+    REST_API_URL: string;
+  };
+  CLIGHTNING: {
     SOCKET: string;
   };
   ELECTRUM: {
@@ -130,6 +141,8 @@ const defaults: IConfig = {
     'USER_AGENT': 'mempool',
     'STDOUT_LOG_MIN_PRIORITY': 'debug',
     'AUTOMATIC_BLOCK_REINDEXING': false,
+    'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
+    'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
@@ -177,12 +190,19 @@ const defaults: IConfig = {
   },
   'LIGHTNING': {
     'ENABLED': false,
-    'BACKEND': 'lnd'
+    'BACKEND': 'lnd',
+    'TOPOLOGY_FOLDER': '',
+    'STATS_REFRESH_INTERVAL': 600,
+    'GRAPH_REFRESH_INTERVAL': 600,
+    'LOGGER_UPDATE_INTERVAL': 30,
   },
   'LND': {
     'TLS_CERT_PATH': '',
     'MACAROON_PATH': '',
-    'SOCKET': 'localhost:10009',
+    'REST_API_URL': 'https://localhost:8080',
+  },
+  'CLIGHTNING': {
+    'SOCKET': '',
   },
   'SOCKS5PROXY': {
     'ENABLED': false,
@@ -224,13 +244,14 @@ class Config implements IConfig {
   BISQ: IConfig['BISQ'];
   LIGHTNING: IConfig['LIGHTNING'];
   LND: IConfig['LND'];
+  CLIGHTNING: IConfig['CLIGHTNING'];
   SOCKS5PROXY: IConfig['SOCKS5PROXY'];
   PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
   EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
   MAXMIND: IConfig['MAXMIND'];
 
   constructor() {
-    const configs = this.merge(configFile, defaults);
+    const configs = this.merge(configFromFile, defaults);
     this.MEMPOOL = configs.MEMPOOL;
     this.ESPLORA = configs.ESPLORA;
     this.ELECTRUM = configs.ELECTRUM;
@@ -242,6 +263,7 @@ class Config implements IConfig {
     this.BISQ = configs.BISQ;
     this.LIGHTNING = configs.LIGHTNING;
     this.LND = configs.LND;
+    this.CLIGHTNING = configs.CLIGHTNING;
     this.SOCKS5PROXY = configs.SOCKS5PROXY;
     this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
     this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;

backend/src/database.ts

@@ -1,7 +1,7 @@
 import config from './config';
 import { createPool, Pool, PoolConnection } from 'mysql2/promise';
 import logger from './logger';
-import { PoolOptions } from 'mysql2/typings/mysql';
+import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } from 'mysql2/typings/mysql';
 
  class DB {
   constructor() {
@@ -28,7 +28,9 @@ import { PoolOptions } from 'mysql2/typings/mysql';
     }
   }
 
-  public async query(query, params?) {
+  public async query<T extends RowDataPacket[][] | RowDataPacket[] | OkPacket |
+    OkPacket[] | ResultSetHeader>(query, params?): Promise<[T, FieldPacket[]]>
+  {
     this.checkDBFlag();
     const pool = await this.getPool();
     return pool.query(query, params);

backend/src/index.ts

@@ -28,12 +28,13 @@ import nodesRoutes from './api/explorer/nodes.routes';
 import channelsRoutes from './api/explorer/channels.routes';
 import generalLightningRoutes from './api/explorer/general.routes';
 import lightningStatsUpdater from './tasks/lightning/stats-updater.service';
-import nodeSyncService from './tasks/lightning/node-sync.service';
-import statisticsRoutes from "./api/statistics/statistics.routes";
-import miningRoutes from "./api/mining/mining-routes";
-import bisqRoutes from "./api/bisq/bisq.routes";
-import liquidRoutes from "./api/liquid/liquid.routes";
-import bitcoinRoutes from "./api/bitcoin/bitcoin.routes";
+import networkSyncService from './tasks/lightning/network-sync.service';
+import statisticsRoutes from './api/statistics/statistics.routes';
+import miningRoutes from './api/mining/mining-routes';
+import bisqRoutes from './api/bisq/bisq.routes';
+import liquidRoutes from './api/liquid/liquid.routes';
+import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
+import fundingTxFetcher from "./tasks/lightning/sync-tasks/funding-tx-fetcher";
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -136,8 +137,7 @@ class Server {
     }
 
     if (config.LIGHTNING.ENABLED) {
-      nodeSyncService.$startService()
-        .then(() => lightningStatsUpdater.$startService());
+      this.$runLightningBackend();
     }
 
     this.server.listen(config.MEMPOOL.HTTP_PORT, () => {
@@ -183,6 +183,18 @@ class Server {
     }
   }
 
+  async $runLightningBackend() {
+    try {
+      await fundingTxFetcher.$init();
+      await networkSyncService.$startService();
+      await lightningStatsUpdater.$startService();
+    } catch(e) {
+      logger.err(`Nodejs lightning backend crashed. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
+      await Common.sleep$(1000 * 60);
+      this.$runLightningBackend();
+    };
+}
+
   setUpWebsocketHandling() {
     if (this.wss) {
       websocketHandler.setWebsocketServer(this.wss);

backend/src/indexer.ts

@@ -6,13 +6,12 @@ import logger from './logger';
 import HashratesRepository from './repositories/HashratesRepository';
 import bitcoinClient from './api/bitcoin/bitcoin-client';
 import priceUpdater from './tasks/price-updater';
+import PricesRepository from './repositories/PricesRepository';
 
 class Indexer {
   runIndexer = true;
   indexerRunning = false;
-
-  constructor() {
-  }
+  tasksRunning: string[] = [];
 
   public reindex() {
     if (Common.indexingEnabled()) {
@@ -20,6 +19,28 @@ class Indexer {
     }
   }
 
+  public async runSingleTask(task: 'blocksPrices') {
+    if (!Common.indexingEnabled()) {
+      return;
+    }
+
+    if (task === 'blocksPrices' && !this.tasksRunning.includes(task)) {
+      this.tasksRunning.push(task);
+      const lastestPriceId = await PricesRepository.$getLatestPriceId();
+      if (priceUpdater.historyInserted === false || lastestPriceId === null) {
+        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`)
+        setTimeout(() => {
+          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+          this.runSingleTask('blocksPrices');
+        }, 10000);
+      } else {
+        logger.debug(`Blocks prices indexer will run now`)
+        await mining.$indexBlockPrices();
+        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+      }
+    }
+  }
+
   public async $run() {
     if (!Common.indexingEnabled() || this.runIndexer === false ||
       this.indexerRunning === true || mempool.hasPriority()
@@ -50,7 +71,7 @@ class Indexer {
         return;
       }
 
-      await mining.$indexBlockPrices();
+      this.runSingleTask('blocksPrices');
       await mining.$indexDifficultyAdjustments();
       await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient
       await mining.$generateNetworkHashrateHistory();

backend/src/logger.ts

@@ -74,7 +74,7 @@ class Logger {
 
   private getNetwork(): string {
     if (config.LIGHTNING.ENABLED) {
-      return 'lightning';
+      return config.MEMPOOL.NETWORK === 'mainnet' ? 'lightning' : `${config.MEMPOOL.NETWORK}-lightning`; 
     }
     if (config.BISQ.ENABLED) {
       return 'bisq';

backend/src/mempool.interfaces.ts

@@ -251,3 +251,41 @@ export interface RewardStats {
   totalFee: number;
   totalTx: number;
 }
+
+export interface ITopNodesPerChannels {
+  publicKey: string,
+  alias: string,
+  channels?: number,
+  capacity: number,
+  firstSeen?: number,
+  updatedAt?: number,
+  city?: any,
+  country?: any,
+}
+
+export interface ITopNodesPerCapacity {
+  publicKey: string,
+  alias: string,
+  capacity: number,
+  channels?: number,
+  firstSeen?: number,
+  updatedAt?: number,
+  city?: any,
+  country?: any,
+}
+
+export interface INodesRanking {
+  topByCapacity: ITopNodesPerCapacity[];
+  topByChannels: ITopNodesPerChannels[];
+}
+
+export interface IOldestNodes {
+  publicKey: string,
+  alias: string,
+  firstSeen: number,
+  channels?: number,
+  capacity: number,
+  updatedAt?: number,
+  city?: any,
+  country?: any,
+}

backend/src/repositories/BlocksAuditsRepository.ts

@@ -1,4 +1,3 @@
-import transactionUtils from '../api/transaction-utils';
 import DB from '../database';
 import logger from '../logger';
 import { BlockAudit } from '../mempool.interfaces';

backend/src/repositories/NodesSocketsRepository.ts

@@ -0,0 +1,45 @@
+import { ResultSetHeader } from 'mysql2';
+import DB from '../database';
+import logger from '../logger';
+
+export interface NodeSocket {
+  publicKey: string;
+  network: string | null;
+  addr: string;
+}
+
+class NodesSocketsRepository {
+  public async $saveSocket(socket: NodeSocket): Promise<void> {
+    try {
+      await DB.query(`
+        INSERT INTO nodes_sockets(public_key, socket, type)
+        VALUE (?, ?, ?)
+      `, [socket.publicKey, socket.addr, socket.network]);
+    } catch (e: any) {
+      if (e.errno !== 1062) { // ER_DUP_ENTRY - Not an issue, just ignore this
+        logger.err(`Cannot save node socket (${[socket.publicKey, socket.addr, socket.network]}) into db. Reason: ` + (e instanceof Error ? e.message : e));
+        // We don't throw, not a critical issue if we miss some nodes sockets
+      }
+    }
+   }
+
+   public async $deleteUnusedSockets(publicKey: string, addresses: string[]): Promise<number> {
+    if (addresses.length === 0) {
+      return 0;
+    }
+    try {
+      const query = `
+        DELETE FROM nodes_sockets
+        WHERE public_key = ?
+        AND socket NOT IN (${addresses.map(id => `"${id}"`).join(',')})
+      `;
+      const [result] = await DB.query<ResultSetHeader>(query, [publicKey]);
+      return result.affectedRows;
+    } catch (e) {
+      logger.err(`Cannot delete unused sockets for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return 0;
+    }
+   }
+}
+
+export default new NodesSocketsRepository();

backend/src/repositories/PricesRepository.ts

@@ -27,6 +27,11 @@ class PricesRepository {
     return oldestRow[0] ? oldestRow[0].time : 0;
   }
 
+  public async $getLatestPriceId(): Promise<number | null> {
+    const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
+    return oldestRow[0] ? oldestRow[0].id : null;
+  }
+
   public async $getLatestPriceTime(): Promise<number> {
     const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
     return oldestRow[0] ? oldestRow[0].time : 0;

backend/src/tasks/lightning/network-sync.service.ts

@@ -0,0 +1,431 @@
+import DB from '../../database';
+import logger from '../../logger';
+import channelsApi from '../../api/explorer/channels.api';
+import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
+import config from '../../config';
+import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
+import { ILightningApi } from '../../api/lightning/lightning-api.interface';
+import { $lookupNodeLocation } from './sync-tasks/node-locations';
+import lightningApi from '../../api/lightning/lightning-api-factory';
+import nodesApi from '../../api/explorer/nodes.api';
+import { ResultSetHeader } from 'mysql2';
+import fundingTxFetcher from './sync-tasks/funding-tx-fetcher';
+import NodesSocketsRepository from '../../repositories/NodesSocketsRepository';
+import { Common } from '../../api/common';
+import blocks from '../../api/blocks';
+
+class NetworkSyncService {
+  loggerTimer = 0;
+  closedChannelsScanBlock = 0;
+
+  constructor() {}
+
+  public async $startService(): Promise<void> {
+    logger.info('Starting lightning network sync service');
+
+    this.loggerTimer = new Date().getTime() / 1000;
+
+    await this.$runTasks();
+  }
+
+  private async $runTasks(): Promise<void> {
+    try {
+      logger.info(`Updating nodes and channels`);
+
+      const networkGraph = await lightningApi.$getNetworkGraph();
+      if (networkGraph.nodes.length === 0 || networkGraph.edges.length === 0) {
+        logger.info(`LN Network graph is empty, retrying in 10 seconds`);
+        setTimeout(() => { this.$runTasks(); }, 10000);
+        return;
+      }
+
+      await this.$updateNodesList(networkGraph.nodes);
+      await this.$updateChannelsList(networkGraph.edges);
+      await this.$deactivateChannelsWithoutActiveNodes();
+      await this.$lookUpCreationDateFromChain();
+      await this.$updateNodeFirstSeen();
+      await this.$scanForClosedChannels();
+      if (config.MEMPOOL.BACKEND === 'esplora') {
+        await this.$runClosedChannelsForensics();
+      }
+
+    } catch (e) {
+      logger.err('$runTasks() error: ' + (e instanceof Error ? e.message : e));
+    }
+
+    setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL);
+  }
+
+  /**
+   * Update the `nodes` table to reflect the current network graph state
+   */
+  private async $updateNodesList(nodes: ILightningApi.Node[]): Promise<void> {
+    let progress = 0;
+
+    let deletedSockets = 0;
+    const graphNodesPubkeys: string[] = [];
+    for (const node of nodes) {
+      const latestUpdated = await channelsApi.$getLatestChannelUpdateForNode(node.pub_key);
+      node.last_update = Math.max(node.last_update, latestUpdated);
+
+      await nodesApi.$saveNode(node);
+      graphNodesPubkeys.push(node.pub_key);
+      ++progress;
+
+      const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+      if (elapsedSeconds > 10) {
+        logger.info(`Updating node ${progress}/${nodes.length}`);
+        this.loggerTimer = new Date().getTime() / 1000;
+      }
+
+      const addresses: string[] = [];
+      for (const socket of node.addresses) {
+        await NodesSocketsRepository.$saveSocket(Common.formatSocket(node.pub_key, socket));
+        addresses.push(socket.addr);
+      }
+      deletedSockets += await NodesSocketsRepository.$deleteUnusedSockets(node.pub_key, addresses);
+    }
+    logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted`);
+
+    // If a channel if not present in the graph, mark it as inactive
+    await nodesApi.$setNodesInactive(graphNodesPubkeys);
+
+    if (config.MAXMIND.ENABLED) {
+      $lookupNodeLocation();
+    }
+  }
+
+  /**
+   * Update the `channels` table to reflect the current network graph state
+   */
+  private async $updateChannelsList(channels: ILightningApi.Channel[]): Promise<void> {
+    try {
+      const [closedChannelsRaw]: any[] = await DB.query(`SELECT id FROM channels WHERE status = 2`);
+      const closedChannels = {};
+      for (const closedChannel of closedChannelsRaw) {
+        closedChannels[closedChannel.id] = true;
+      }
+
+      let progress = 0;
+
+      const graphChannelsIds: string[] = [];
+      for (const channel of channels) {
+        if (!closedChannels[channel.channel_id]) {
+          await channelsApi.$saveChannel(channel);
+        }
+        graphChannelsIds.push(channel.channel_id);
+        ++progress;
+
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating channel ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+
+      logger.info(`${progress} channels updated`);
+
+      // If a channel if not present in the graph, mark it as inactive
+      await channelsApi.$setChannelsInactive(graphChannelsIds);
+    } catch (e) {
+      logger.err(`Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`);
+    }
+  }
+
+  // This method look up the creation date of the earliest channel of the node
+  // and update the node to that date in order to get the earliest first seen date
+  private async $updateNodeFirstSeen(): Promise<void> {
+    let progress = 0;
+    let updated = 0;
+
+    try {
+      const [nodes]: any[] = await DB.query(`
+        SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen,
+        (
+          SELECT MIN(UNIX_TIMESTAMP(created))
+          FROM channels
+          WHERE channels.node1_public_key = nodes.public_key
+        ) AS created1,
+        (
+          SELECT MIN(UNIX_TIMESTAMP(created))
+          FROM channels
+          WHERE channels.node2_public_key = nodes.public_key
+        ) AS created2
+        FROM nodes
+      `);
+
+      for (const node of nodes) {
+        const lowest = Math.min(
+          node.created1 ?? Number.MAX_SAFE_INTEGER,
+          node.created2 ?? Number.MAX_SAFE_INTEGER,
+          node.first_seen ?? Number.MAX_SAFE_INTEGER
+        );
+        if (lowest < node.first_seen) {
+          const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`;
+          const params = [lowest, node.public_key];
+          await DB.query(query, params);
+        }
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating node first seen date ${progress}/${nodes.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+          ++updated;
+        }
+      }
+      logger.info(`Updated ${updated} node first seen dates`);
+    } catch (e) {
+      logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private async $lookUpCreationDateFromChain(): Promise<void> {
+    let progress = 0;
+
+    logger.info(`Running channel creation date lookup`);
+    try {
+      const channels = await channelsApi.$getChannelsWithoutCreatedDate();
+      for (const channel of channels) {
+        const transaction = await fundingTxFetcher.$fetchChannelOpenTx(channel.short_id);
+        await DB.query(`
+          UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`,
+          [transaction.timestamp, channel.id]
+        );
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating channel creation date ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+      logger.info(`Updated ${channels.length} channels' creation date`);
+    } catch (e) {
+      logger.err('$lookUpCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * If a channel does not have any active node linked to it, then also
+   * mark that channel as inactive
+   */
+  private async $deactivateChannelsWithoutActiveNodes(): Promise<void> {
+    logger.info(`Find channels which nodes are offline`);
+
+    try {
+      const result = await DB.query<ResultSetHeader>(`
+        UPDATE channels
+        SET status = 0
+        WHERE channels.status = 1
+        AND (
+          (
+            SELECT COUNT(*)
+            FROM nodes
+            WHERE nodes.public_key = channels.node1_public_key
+            AND nodes.status = 1
+          ) = 0
+        OR (
+            SELECT COUNT(*)
+            FROM nodes
+            WHERE nodes.public_key = channels.node2_public_key
+            AND nodes.status = 1
+          ) = 0)
+        `);
+
+      if (result[0].changedRows ?? 0 > 0) {
+        logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`);
+      } else {
+        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`);
+      }
+    } catch (e) {
+      logger.err('$deactivateChannelsWithoutActiveNodes() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private async $scanForClosedChannels(): Promise<void> {
+    if (this.closedChannelsScanBlock === blocks.getCurrentBlockHeight()) {
+      logger.debug(`We've already scan closed channels for this block, skipping.`);
+      return;
+    }
+
+    let progress = 0;
+
+    try {
+      let log = `Starting closed channels scan`;
+      if (this.closedChannelsScanBlock > 0) {
+        log += `. Last scan was at block ${this.closedChannelsScanBlock}`;
+      } else {
+        log += ` for the first time`;
+      }
+      logger.info(log);
+
+      const channels = await channelsApi.$getChannelsByStatus([0, 1]);
+      for (const channel of channels) {
+        const spendingTx = await bitcoinApi.$getOutspend(channel.transaction_id, channel.transaction_vout);
+        if (spendingTx.spent === true && spendingTx.status?.confirmed === true) {
+          logger.debug('Marking channel: ' + channel.id + ' as closed.');
+          await DB.query(`UPDATE channels SET status = 2, closing_date = FROM_UNIXTIME(?) WHERE id = ?`,
+            [spendingTx.status.block_time, channel.id]);
+          if (spendingTx.txid && !channel.closing_transaction_id) {
+            await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]);
+          }
+        }
+
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Checking if channel has been closed ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+
+      this.closedChannelsScanBlock = blocks.getCurrentBlockHeight();
+      logger.info(`Closed channels scan completed at block ${this.closedChannelsScanBlock}`);
+    } catch (e) {
+      logger.err('$scanForClosedChannels() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /*
+    1. Mutually closed
+    2. Forced closed
+    3. Forced closed with penalty
+  */
+
+  private async $runClosedChannelsForensics(): Promise<void> {
+    if (!config.ESPLORA.REST_API_URL) {
+      return;
+    }
+
+    let progress = 0;
+
+    try {
+      logger.info(`Started running closed channel forensics...`);
+      const channels = await channelsApi.$getClosedChannelsWithoutReason();
+      for (const channel of channels) {
+        let reason = 0;
+        // Only Esplora backend can retrieve spent transaction outputs
+        try {
+          let outspends: IEsploraApi.Outspend[] | undefined;
+          try {
+            outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id);
+          } catch (e) {
+            logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
+            continue;
+          }
+          const lightningScriptReasons: number[] = [];
+          for (const outspend of outspends) {
+            if (outspend.spent && outspend.txid) {
+              let spendingTx: IEsploraApi.Transaction | undefined;
+              try {
+                spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid);
+              } catch (e) {
+                logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`);
+                continue;
+              }
+              const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
+              lightningScriptReasons.push(lightningScript);
+            }
+          }
+          if (lightningScriptReasons.length === outspends.length
+            && lightningScriptReasons.filter((r) => r === 1).length === outspends.length) {
+            reason = 1;
+          } else {
+            const filteredReasons = lightningScriptReasons.filter((r) => r !== 1);
+            if (filteredReasons.length) {
+              if (filteredReasons.some((r) => r === 2 || r === 4)) {
+                reason = 3;
+              } else {
+                reason = 2;
+              }
+            } else {
+              /*
+                We can detect a commitment transaction (force close) by reading Sequence and Locktime
+                https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction
+              */
+              let closingTx: IEsploraApi.Transaction | undefined;
+              try {
+                closingTx = await bitcoinApi.$getRawTransaction(channel.closing_transaction_id);
+              } catch (e) {
+                logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id}. Reason ${e instanceof Error ? e.message : e}`);
+                continue;
+              }
+              const sequenceHex: string = closingTx.vin[0].sequence.toString(16);
+              const locktimeHex: string = closingTx.locktime.toString(16);
+              if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') {
+                reason = 2; // Here we can't be sure if it's a penalty or not
+              } else {
+                reason = 1;
+              }
+            }
+          }
+          if (reason) {
+            logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
+            await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
+          }
+        } catch (e) {
+          logger.err(`$runClosedChannelsForensics() failed for channel ${channel.short_id}. Reason: ${e instanceof Error ? e.message : e}`);
+        }
+
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating channel closed channel forensics ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+      logger.info(`Closed channels forensics scan complete.`);
+    } catch (e) {
+      logger.err('$runClosedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private findLightningScript(vin: IEsploraApi.Vin): number {
+    const topElement = vin.witness[vin.witness.length - 2];
+      if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) {
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs
+        if (topElement === '01') {
+          // top element is '01' to get in the revocation path
+          // 'Revoked Lightning Force Close';
+          // Penalty force closed
+          return 2;
+        } else {
+          // top element is '', this is a delayed to_local output
+          // 'Lightning Force Close';
+          return 3;
+        }
+      } else if (
+        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_NOTIF OP_DROP OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) ||
+        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_IF OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_DROP OP_PUSHBYTES_3 \w{6} OP_CLTV OP_DROP OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm)
+      ) {
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs
+        if (topElement.length === 66) {
+          // top element is a public key
+          // 'Revoked Lightning HTLC'; Penalty force closed
+          return 4;
+        } else if (topElement) {
+          // top element is a preimage
+          // 'Lightning HTLC';
+          return 5;
+        } else {
+          // top element is '' to get in the expiry of the script
+          // 'Expired Lightning HTLC';
+          return 6;
+        }
+      } else if (/^OP_PUSHBYTES_33 \w{66} OP_CHECKSIG OP_IFDUP OP_NOTIF OP_PUSHNUM_16 OP_CSV OP_ENDIF$/.test(vin.inner_witnessscript_asm)) {
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#to_local_anchor-and-to_remote_anchor-output-option_anchors
+        if (topElement) {
+          // top element is a signature
+          // 'Lightning Anchor';
+          return 7;
+        } else {
+          // top element is '', it has been swept after 16 blocks
+          // 'Swept Lightning Anchor';
+          return 8;
+        }
+      }
+      return 1;
+  }
+}
+
+export default new NetworkSyncService();

backend/src/tasks/lightning/node-sync.service.ts

@@ -1,427 +0,0 @@
-import { chanNumber } from 'bolt07';
-import DB from '../../database';
-import logger from '../../logger';
-import channelsApi from '../../api/explorer/channels.api';
-import bitcoinClient from '../../api/bitcoin/bitcoin-client';
-import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
-import config from '../../config';
-import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
-import lightningApi from '../../api/lightning/lightning-api-factory';
-import { ILightningApi } from '../../api/lightning/lightning-api.interface';
-import { $lookupNodeLocation } from './sync-tasks/node-locations';
-
-class NodeSyncService {
-  constructor() {}
-
-  public async $startService() {
-    logger.info('Starting node sync service');
-
-    await this.$runUpdater();
-
-    setInterval(async () => {
-      await this.$runUpdater();
-    }, 1000 * 60 * 60);
-  }
-
-  private async $runUpdater() {
-    try {
-      logger.info(`Updating nodes and channels...`);
-
-      const networkGraph = await lightningApi.$getNetworkGraph();
-
-      for (const node of networkGraph.nodes) {
-        await this.$saveNode(node);
-      }
-      logger.info(`Nodes updated.`);
-
-      if (config.MAXMIND.ENABLED) {
-        await $lookupNodeLocation();
-      }
-
-      const graphChannelsIds: string[] = [];
-      for (const channel of networkGraph.channels) {
-        await this.$saveChannel(channel);
-        graphChannelsIds.push(channel.id);
-      }
-      await this.$setChannelsInactive(graphChannelsIds);
-
-      logger.info(`Channels updated.`);
-
-      await this.$findInactiveNodesAndChannels();
-      await this.$lookUpCreationDateFromChain();
-      await this.$updateNodeFirstSeen();
-      await this.$scanForClosedChannels();
-      if (config.MEMPOOL.BACKEND === 'esplora') {
-        await this.$runClosedChannelsForensics();
-      }
-
-    } catch (e) {
-      logger.err('$updateNodes() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  // This method look up the creation date of the earliest channel of the node
-  // and update the node to that date in order to get the earliest first seen date
-  private async $updateNodeFirstSeen() {
-    try {
-      const [nodes]: any[] = await DB.query(`SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node1_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created1, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node2_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created2 FROM nodes`);
-      for (const node of nodes) {
-        let lowest = 0;
-        if (node.created1) {
-          if (node.created2 && node.created2 < node.created1) {
-            lowest = node.created2;
-          } else {
-            lowest = node.created1;
-          }
-        } else if (node.created2) {
-          lowest = node.created2;
-        }
-        if (lowest && lowest < node.first_seen) {
-          const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`;
-          const params = [lowest, node.public_key];
-          await DB.query(query, params);
-        }
-      }
-      logger.info(`Node first seen dates scan complete.`);
-    } catch (e) {
-      logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $lookUpCreationDateFromChain() {
-    logger.info(`Running channel creation date lookup...`);
-    try {
-      const channels = await channelsApi.$getChannelsWithoutCreatedDate();
-      for (const channel of channels) {
-        const transaction = await bitcoinClient.getRawTransaction(channel.transaction_id, 1);
-        await DB.query(`UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`, [transaction.blocktime, channel.id]);
-      }
-      logger.info(`Channel creation dates scan complete.`);
-    } catch (e) {
-      logger.err('$setCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  // Looking for channels whos nodes are inactive
-  private async $findInactiveNodesAndChannels(): Promise<void> {
-    logger.info(`Running inactive channels scan...`);
-
-    try {
-      // @ts-ignore
-      const [channels]: [ILightningApi.Channel[]] = await DB.query(`
-        SELECT channels.id
-        FROM channels
-        WHERE channels.status = 1
-        AND (
-          (
-            SELECT COUNT(*)
-            FROM nodes
-            WHERE nodes.public_key = channels.node1_public_key
-          ) = 0
-        OR (
-            SELECT COUNT(*)
-            FROM nodes
-            WHERE nodes.public_key = channels.node2_public_key
-          ) = 0)
-        `);
-
-      for (const channel of channels) {
-        await this.$updateChannelStatus(channel.id, 0);
-      }
-      logger.info(`Inactive channels scan complete.`);
-    } catch (e) {
-      logger.err('$findInactiveNodesAndChannels() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $scanForClosedChannels(): Promise<void> {
-    try {
-      logger.info(`Starting closed channels scan...`);
-      const channels = await channelsApi.$getChannelsByStatus(0);
-      for (const channel of channels) {
-        const spendingTx = await bitcoinApi.$getOutspend(channel.transaction_id, channel.transaction_vout);
-        if (spendingTx.spent === true && spendingTx.status?.confirmed === true) {
-          logger.debug('Marking channel: ' + channel.id + ' as closed.');
-          await DB.query(`UPDATE channels SET status = 2, closing_date = FROM_UNIXTIME(?) WHERE id = ?`,
-            [spendingTx.status.block_time, channel.id]);
-          if (spendingTx.txid && !channel.closing_transaction_id) {
-            await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]);
-          }
-        }
-      }
-      logger.info(`Closed channels scan complete.`);
-    } catch (e) {
-      logger.err('$scanForClosedChannels() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  /*
-    1. Mutually closed
-    2. Forced closed
-    3. Forced closed with penalty
-  */
-
-  private async $runClosedChannelsForensics(): Promise<void> {
-    if (!config.ESPLORA.REST_API_URL) {
-      return;
-    }
-    try {
-      logger.info(`Started running closed channel forensics...`);
-      const channels = await channelsApi.$getClosedChannelsWithoutReason();
-      for (const channel of channels) {
-        let reason = 0;
-        // Only Esplora backend can retrieve spent transaction outputs
-        const outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id);
-        const lightningScriptReasons: number[] = [];
-        for (const outspend of outspends) {
-          if (outspend.spent && outspend.txid) {
-            const spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid);
-            const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
-            lightningScriptReasons.push(lightningScript);
-          }
-        }
-        if (lightningScriptReasons.length === outspends.length
-          && lightningScriptReasons.filter((r) => r === 1).length === outspends.length) {
-          reason = 1;
-        } else {
-          const filteredReasons = lightningScriptReasons.filter((r) => r !== 1);
-          if (filteredReasons.length) {
-            if (filteredReasons.some((r) => r === 2 || r === 4)) {
-              reason = 3;
-            } else {
-              reason = 2;
-            }
-          } else {
-            /*
-              We can detect a commitment transaction (force close) by reading Sequence and Locktime
-              https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction
-            */
-            const closingTx = await bitcoinApi.$getRawTransaction(channel.closing_transaction_id);
-            const sequenceHex: string = closingTx.vin[0].sequence.toString(16);
-            const locktimeHex: string = closingTx.locktime.toString(16);
-            if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') {
-              reason = 2; // Here we can't be sure if it's a penalty or not
-            } else {
-              reason = 1;
-            }
-          }
-        }
-        if (reason) {
-          logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
-          await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
-        }
-      }
-      logger.info(`Closed channels forensics scan complete.`);
-    } catch (e) {
-      logger.err('$runClosedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private findLightningScript(vin: IEsploraApi.Vin): number {
-    const topElement = vin.witness[vin.witness.length - 2];
-      if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) {
-        // https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs
-        if (topElement === '01') {
-          // top element is '01' to get in the revocation path
-          // 'Revoked Lightning Force Close';
-          // Penalty force closed
-          return 2;
-        } else {
-          // top element is '', this is a delayed to_local output
-          // 'Lightning Force Close';
-          return 3;
-        }
-      } else if (
-        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_NOTIF OP_DROP OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) ||
-        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_IF OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_DROP OP_PUSHBYTES_3 \w{6} OP_CLTV OP_DROP OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm)
-      ) {
-        // https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs
-        // https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs
-        if (topElement.length === 66) {
-          // top element is a public key
-          // 'Revoked Lightning HTLC'; Penalty force closed
-          return 4;
-        } else if (topElement) {
-          // top element is a preimage
-          // 'Lightning HTLC';
-          return 5;
-        } else {
-          // top element is '' to get in the expiry of the script
-          // 'Expired Lightning HTLC';
-          return 6;
-        }
-      } else if (/^OP_PUSHBYTES_33 \w{66} OP_CHECKSIG OP_IFDUP OP_NOTIF OP_PUSHNUM_16 OP_CSV OP_ENDIF$/.test(vin.inner_witnessscript_asm)) {
-        // https://github.com/lightning/bolts/blob/master/03-transactions.md#to_local_anchor-and-to_remote_anchor-output-option_anchors
-        if (topElement) {
-          // top element is a signature
-          // 'Lightning Anchor';
-          return 7;
-        } else {
-          // top element is '', it has been swept after 16 blocks
-          // 'Swept Lightning Anchor';
-          return 8;
-        }
-      }
-      return 1;
-  }
-
-  private async $saveChannel(channel: ILightningApi.Channel): Promise<void> {
-    const fromChannel = chanNumber({ channel: channel.id }).number;
-
-    try {
-      const query = `INSERT INTO channels
-        (
-          id,
-          short_id,
-          capacity,
-          transaction_id,
-          transaction_vout,
-          updated_at,
-          status,
-          node1_public_key,
-          node1_base_fee_mtokens,
-          node1_cltv_delta,
-          node1_fee_rate,
-          node1_is_disabled,
-          node1_max_htlc_mtokens,
-          node1_min_htlc_mtokens,
-          node1_updated_at,
-          node2_public_key,
-          node2_base_fee_mtokens,
-          node2_cltv_delta,
-          node2_fee_rate,
-          node2_is_disabled,
-          node2_max_htlc_mtokens,
-          node2_min_htlc_mtokens,
-          node2_updated_at
-        )
-        VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-        ON DUPLICATE KEY UPDATE
-          capacity = ?,
-          updated_at = ?,
-          status = 1,
-          node1_public_key = ?,
-          node1_base_fee_mtokens = ?,
-          node1_cltv_delta = ?,
-          node1_fee_rate = ?,
-          node1_is_disabled = ?,
-          node1_max_htlc_mtokens = ?,
-          node1_min_htlc_mtokens = ?,
-          node1_updated_at = ?,
-          node2_public_key = ?,
-          node2_base_fee_mtokens = ?,
-          node2_cltv_delta = ?,
-          node2_fee_rate = ?,
-          node2_is_disabled = ?,
-          node2_max_htlc_mtokens = ?,
-          node2_min_htlc_mtokens = ?,
-          node2_updated_at = ?
-        ;`;
-
-      await DB.query(query, [
-        fromChannel,
-        channel.id,
-        channel.capacity,
-        channel.transaction_id,
-        channel.transaction_vout,
-        channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0,
-        channel.policies[0].public_key,
-        channel.policies[0].base_fee_mtokens,
-        channel.policies[0].cltv_delta,
-        channel.policies[0].fee_rate,
-        channel.policies[0].is_disabled,
-        channel.policies[0].max_htlc_mtokens,
-        channel.policies[0].min_htlc_mtokens,
-        channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0,
-        channel.policies[1].public_key,
-        channel.policies[1].base_fee_mtokens,
-        channel.policies[1].cltv_delta,
-        channel.policies[1].fee_rate,
-        channel.policies[1].is_disabled,
-        channel.policies[1].max_htlc_mtokens,
-        channel.policies[1].min_htlc_mtokens,
-        channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0,
-        channel.capacity,
-        channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0,
-        channel.policies[0].public_key,
-        channel.policies[0].base_fee_mtokens,
-        channel.policies[0].cltv_delta,
-        channel.policies[0].fee_rate,
-        channel.policies[0].is_disabled,
-        channel.policies[0].max_htlc_mtokens,
-        channel.policies[0].min_htlc_mtokens,
-        channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0,
-        channel.policies[1].public_key,
-        channel.policies[1].base_fee_mtokens,
-        channel.policies[1].cltv_delta,
-        channel.policies[1].fee_rate,
-        channel.policies[1].is_disabled,
-        channel.policies[1].max_htlc_mtokens,
-        channel.policies[1].min_htlc_mtokens,
-        channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0,
-      ]);
-    } catch (e) {
-      logger.err('$saveChannel() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $updateChannelStatus(channelShortId: string, status: number): Promise<void> {
-    try {
-      await DB.query(`UPDATE channels SET status = ? WHERE id = ?`, [status, channelShortId]);
-    } catch (e) {
-      logger.err('$updateChannelStatus() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $setChannelsInactive(graphChannelsIds: string[]): Promise<void> {
-    try {
-      await DB.query(`
-        UPDATE channels
-        SET status = 0
-        WHERE short_id NOT IN (
-          ${graphChannelsIds.map(id => `"${id}"`).join(',')}
-        )
-        AND status != 2
-      `);
-    } catch (e) {
-      logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $saveNode(node: ILightningApi.Node): Promise<void> {
-    try {
-      const updatedAt = node.updated_at ? this.utcDateToMysql(node.updated_at) : '0000-00-00 00:00:00';
-      const sockets = node.sockets.join(',');
-      const query = `INSERT INTO nodes(
-          public_key,
-          first_seen,
-          updated_at,
-          alias,
-          color,
-          sockets
-        )
-        VALUES (?, NOW(), ?, ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?, sockets = ?;`;
-
-      await DB.query(query, [
-        node.public_key,
-        updatedAt,
-        node.alias,
-        node.color,
-        sockets,
-        updatedAt,
-        node.alias,
-        node.color,
-        sockets,
-      ]);
-    } catch (e) {
-      logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private utcDateToMysql(dateString: string): string {
-    const d = new Date(Date.parse(dateString));
-    return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
-  }
-}
-
-export default new NodeSyncService();

backend/src/tasks/lightning/stats-updater.service.ts

@@ -1,350 +1,33 @@
-
-import DB from '../../database';
 import logger from '../../logger';
 import lightningApi from '../../api/lightning/lightning-api-factory';
-import channelsApi from '../../api/explorer/channels.api';
-import * as net from 'net';
+import LightningStatsImporter from './sync-tasks/stats-importer';
+import config from '../../config';
+import { Common } from '../../api/common';
 
 class LightningStatsUpdater {
-  hardCodedStartTime = '2018-01-12';
-
-  public async $startService() {
+  public async $startService(): Promise<void> {
     logger.info('Starting Lightning Stats service');
-    let isInSync = false;
-    let error: any;
-    try {
-      error = null;
-      isInSync = await this.$lightningIsSynced();
-    } catch (e) {
-      error = e;
-    }
-    if (!isInSync) {
-      if (error) {
-        logger.warn('Was not able to fetch Lightning Node status: ' + (error instanceof Error ? error.message : error) + '. Retrying in 1 minute...');
-      } else {
-        logger.notice('The Lightning graph is not yet in sync. Retrying in 1 minute...');
-      }
-      setTimeout(() => this.$startService(), 60 * 1000);
-      return;
-    }
 
-    await this.$populateHistoricalStatistics();
-    await this.$populateHistoricalNodeStatistics();
-
-    setTimeout(() => {
-      this.$runTasks();
-    }, this.timeUntilMidnight());
-  }
-
-  private timeUntilMidnight(): number {
-    const date = new Date();
-    this.setDateMidnight(date);
-    date.setUTCHours(24);
-    return date.getTime() - new Date().getTime();
-  }
-
-  private setDateMidnight(date: Date): void {
-    date.setUTCHours(0);
-    date.setUTCMinutes(0);
-    date.setUTCSeconds(0);
-    date.setUTCMilliseconds(0);
-  }
-
-  private async $lightningIsSynced(): Promise<boolean> {
-    const nodeInfo = await lightningApi.$getInfo();
-    return nodeInfo.is_synced_to_chain && nodeInfo.is_synced_to_graph;
+    await this.$runTasks();
+    LightningStatsImporter.$run();
   }
 
   private async $runTasks(): Promise<void> {
-    await this.$logLightningStatsDaily();
-    await this.$logNodeStatsDaily();
+    await this.$logStatsDaily();
 
-    setTimeout(() => {
-      this.$runTasks();
-    }, this.timeUntilMidnight());
+    setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL);
   }
 
-  private async $logLightningStatsDaily() {
-    try {
-      logger.info(`Running lightning daily stats log...`);  
-
-      const networkGraph = await lightningApi.$getNetworkGraph();
-      let total_capacity = 0;
-      for (const channel of networkGraph.channels) {
-        if (channel.capacity) {
-          total_capacity += channel.capacity;
-        }
-      }
-
-      let clearnetNodes = 0;
-      let torNodes = 0;
-      let unannouncedNodes = 0;
-      for (const node of networkGraph.nodes) {
-        let isUnnanounced = true;
-        for (const socket of node.sockets) {
-          const hasOnion = socket.indexOf('.onion') !== -1;
-          if (hasOnion) {
-            torNodes++;
-            isUnnanounced = false;
-          }
-          const hasClearnet = [4, 6].includes(net.isIP(socket.split(':')[0]));
-          if (hasClearnet) {
-            clearnetNodes++;
-            isUnnanounced = false;
-          }
-        }
-        if (isUnnanounced) {
-          unannouncedNodes++;
-        }
-      }
-
-      const channelStats = await channelsApi.$getChannelsStats();
-
-      const query = `INSERT INTO lightning_stats(
-          added,
-          channel_count,
-          node_count,
-          total_capacity,
-          tor_nodes,
-          clearnet_nodes,
-          unannounced_nodes,
-          avg_capacity,
-          avg_fee_rate,
-          avg_base_fee_mtokens,
-          med_capacity,
-          med_fee_rate,
-          med_base_fee_mtokens
-        )
-        VALUES (NOW() - INTERVAL 1 DAY, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
-
-      await DB.query(query, [
-        networkGraph.channels.length,
-        networkGraph.nodes.length,
-        total_capacity,
-        torNodes,
-        clearnetNodes,
-        unannouncedNodes,
-        channelStats.avgCapacity,
-        channelStats.avgFeeRate,
-        channelStats.avgBaseFee,
-        channelStats.medianCapacity,
-        channelStats.medianFeeRate,
-        channelStats.medianBaseFee,
-      ]);
-      logger.info(`Lightning daily stats done.`);
-    } catch (e) {
-      logger.err('$logLightningStatsDaily() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $logNodeStatsDaily() {
-    try {
-      logger.info(`Running daily node stats update...`);
-
-      const query = `
-        SELECT nodes.public_key, c1.channels_count_left, c2.channels_count_right, c1.channels_capacity_left,
-          c2.channels_capacity_right
-        FROM nodes
-        LEFT JOIN (
-          SELECT node1_public_key, COUNT(id) AS channels_count_left, SUM(capacity) AS channels_capacity_left
-          FROM channels
-          WHERE channels.status = 1
-          GROUP BY node1_public_key
-        ) c1 ON c1.node1_public_key = nodes.public_key
-        LEFT JOIN (
-          SELECT node2_public_key, COUNT(id) AS channels_count_right, SUM(capacity) AS channels_capacity_right
-          FROM channels WHERE channels.status = 1 GROUP BY node2_public_key
-        ) c2 ON c2.node2_public_key = nodes.public_key
-      `;
-      
-      const [nodes]: any = await DB.query(query);
-
-      for (const node of nodes) {
-        await DB.query(
-          `INSERT INTO node_stats(public_key, added, capacity, channels) VALUES (?, NOW() - INTERVAL 1 DAY, ?, ?)`,
-          [node.public_key, (parseInt(node.channels_capacity_left || 0, 10)) + (parseInt(node.channels_capacity_right || 0, 10)),
-            node.channels_count_left + node.channels_count_right]);
-      }
-      logger.info('Daily node stats has updated.');
-    } catch (e) {
-      logger.err('$logNodeStatsDaily() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  // We only run this on first launch
-  private async $populateHistoricalStatistics() {
-    try {
-      const [rows]: any = await DB.query(`SELECT COUNT(*) FROM lightning_stats`);
-      // Only run if table is empty
-      if (rows[0]['COUNT(*)'] > 0) {
-        return;
-      }
-      logger.info(`Running historical stats population...`);
-
-      const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels ORDER BY created ASC`);
-      const [nodes]: any = await DB.query(`SELECT first_seen, sockets FROM nodes ORDER BY first_seen ASC`);
-
-      const date: Date = new Date(this.hardCodedStartTime);
-      const currentDate = new Date();
-      this.setDateMidnight(currentDate);
-
-      while (date < currentDate) {
-        let totalCapacity = 0;
-        let channelsCount = 0;
-
-        for (const channel of channels) {
-          if (new Date(channel.created) > date) {
-            break;
-          }
-          if (channel.closing_date === null || new Date(channel.closing_date) > date) {
-            totalCapacity += channel.capacity;
-            channelsCount++;
-          }
-        }
-
-        let nodeCount = 0;
-        let clearnetNodes = 0;
-        let torNodes = 0;
-        let unannouncedNodes = 0;
-
-        for (const node of nodes) {
-          if (new Date(node.first_seen) > date) {
-            break;
-          }
-          nodeCount++;
-
-          const sockets = node.sockets.split(',');
-          let isUnnanounced = true;
-          for (const socket of sockets) {
-            const hasOnion = socket.indexOf('.onion') !== -1;
-            if (hasOnion) {
-              torNodes++;
-              isUnnanounced = false;
-            }
-            const hasClearnet = [4, 6].includes(net.isIP(socket.substring(0, socket.lastIndexOf(':'))));
-            if (hasClearnet) {
-              clearnetNodes++;
-              isUnnanounced = false;
-            }
-          }
-          if (isUnnanounced) {
-            unannouncedNodes++;
-          }
-        }
-
-        const query = `INSERT INTO lightning_stats(
-          added,
-          channel_count,
-          node_count,
-          total_capacity,
-          tor_nodes,
-          clearnet_nodes,
-          unannounced_nodes,
-          avg_capacity,
-          avg_fee_rate,
-          avg_base_fee_mtokens,
-          med_capacity,
-          med_fee_rate,
-          med_base_fee_mtokens
-        )
-        VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
-
-        const rowTimestamp = date.getTime() / 1000; // Save timestamp for the row insertion down below
-
-        date.setUTCDate(date.getUTCDate() + 1);
-
-        // Last iteration, save channels stats
-        const channelStats = (date >= currentDate ? await channelsApi.$getChannelsStats() : undefined);
-
-        await DB.query(query, [
-          rowTimestamp,
-          channelsCount,
-          nodeCount,
-          totalCapacity,
-          torNodes,
-          clearnetNodes,
-          unannouncedNodes,
-          channelStats?.avgCapacity ?? 0,
-          channelStats?.avgFeeRate ?? 0,
-          channelStats?.avgBaseFee ?? 0,
-          channelStats?.medianCapacity ?? 0,
-          channelStats?.medianFeeRate ?? 0,
-          channelStats?.medianBaseFee ?? 0,
-          ]);
-      }
-
-      logger.info('Historical stats populated.');
-    } catch (e) {
-      logger.err('$populateHistoricalData() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $populateHistoricalNodeStatistics() {
-    try {
-      const [rows]: any = await DB.query(`SELECT COUNT(*) FROM node_stats`);
-      // Only run if table is empty
-      if (rows[0]['COUNT(*)'] > 0) {
-        return;
-      }
-      logger.info(`Running historical node stats population...`);
-
-      const [nodes]: any = await DB.query(`SELECT public_key, first_seen, alias FROM nodes ORDER BY first_seen ASC`);
-
-      for (const node of nodes) {
-        const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels WHERE node1_public_key = ? OR node2_public_key = ? ORDER BY created ASC`, [node.public_key, node.public_key]);
-        
-        const date: Date = new Date(this.hardCodedStartTime);
-        const currentDate = new Date();
-        this.setDateMidnight(currentDate);
-
-        let lastTotalCapacity = 0;
-        let lastChannelsCount = 0;
-
-        while (date < currentDate) {
-          let totalCapacity = 0;
-          let channelsCount = 0;
-          for (const channel of channels) {
-            if (new Date(channel.created) > date) {
-              break;
-            }
-            if (channel.closing_date !== null && new Date(channel.closing_date) < date) {
-              date.setUTCDate(date.getUTCDate() + 1);
-              continue;
-            }
-            totalCapacity += channel.capacity;
-            channelsCount++;
-          }
-
-          if (lastTotalCapacity === totalCapacity && lastChannelsCount === channelsCount) {
-            date.setUTCDate(date.getUTCDate() + 1);
-            continue;
-          }
-
-          lastTotalCapacity = totalCapacity;
-          lastChannelsCount = channelsCount;
-  
-          const query = `INSERT INTO node_stats(
-            public_key,
-            added,
-            capacity,
-            channels
-          )
-          VALUES (?, FROM_UNIXTIME(?), ?, ?)`;
-
-          await DB.query(query, [
-            node.public_key,
-            date.getTime() / 1000,
-            totalCapacity,
-            channelsCount,
-          ]);
-          date.setUTCDate(date.getUTCDate() + 1);
-        }
-        logger.debug('Updated node_stats for: ' + node.alias);
-      }
-      logger.info('Historical stats populated.');
-    } catch (e) {
-      logger.err('$populateHistoricalNodeData() error: ' + (e instanceof Error ? e.message : e));
-    }
+  /**
+   * Update the latest entry for each node every config.LIGHTNING.STATS_REFRESH_INTERVAL seconds
+   */
+  private async $logStatsDaily(): Promise<void> {
+    const date = new Date();
+    Common.setDateMidnight(date);
+    const networkGraph = await lightningApi.$getNetworkGraph();
+    await LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph);
+    
+    logger.info(`Updated latest network stats`);
   }
 }
 

backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts

@@ -0,0 +1,116 @@
+import { existsSync, promises } from 'fs';
+import bitcoinClient from '../../../api/bitcoin/bitcoin-client';
+import { Common } from '../../../api/common';
+import config from '../../../config';
+import logger from '../../../logger';
+
+const fsPromises = promises;
+
+const BLOCKS_CACHE_MAX_SIZE = 100;  
+const CACHE_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/ln-funding-txs-cache.json';
+
+class FundingTxFetcher {
+  private running = false;
+  private blocksCache = {};
+  private channelNewlyProcessed = 0;
+  public fundingTxCache = {};
+
+  async $init(): Promise<void> {
+    // Load funding tx disk cache
+    if (Object.keys(this.fundingTxCache).length === 0 && existsSync(CACHE_FILE_NAME)) {
+      try {
+        this.fundingTxCache = JSON.parse(await fsPromises.readFile(CACHE_FILE_NAME, 'utf-8'));
+      } catch (e) {
+        logger.err(`Unable to parse channels funding txs disk cache. Starting from scratch`);
+        this.fundingTxCache = {};
+      }
+      logger.debug(`Imported ${Object.keys(this.fundingTxCache).length} funding tx amount from the disk cache`);
+    }
+  }
+
+  async $fetchChannelsFundingTxs(channelIds: string[]): Promise<void> {
+    if (this.running) {
+      return;
+    }
+    this.running = true;
+    
+    const globalTimer = new Date().getTime() / 1000;
+    let cacheTimer = new Date().getTime() / 1000;
+    let loggerTimer = new Date().getTime() / 1000;
+    let channelProcessed = 0;
+    this.channelNewlyProcessed = 0;
+    for (const channelId of channelIds) {
+      await this.$fetchChannelOpenTx(channelId);
+      ++channelProcessed;
+
+      let elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+      if (elapsedSeconds > 10) {
+        elapsedSeconds = Math.round((new Date().getTime() / 1000) - globalTimer);
+        logger.info(`Indexing channels funding tx ${channelProcessed + 1} of ${channelIds.length} ` +
+          `(${Math.floor(channelProcessed / channelIds.length * 10000) / 100}%) | ` +
+          `elapsed: ${elapsedSeconds} seconds`
+        );
+        loggerTimer = new Date().getTime() / 1000;
+      }
+
+      elapsedSeconds = Math.round((new Date().getTime() / 1000) - cacheTimer);
+      if (elapsedSeconds > 60) {
+        logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`);
+        fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache));
+        cacheTimer = new Date().getTime() / 1000;
+      }
+    }
+
+    if (this.channelNewlyProcessed > 0) {
+      logger.info(`Indexed ${this.channelNewlyProcessed} additional channels funding tx`);
+      logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`);
+      fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache));
+    }
+
+    this.running = false;
+  }
+  
+  public async $fetchChannelOpenTx(channelId: string): Promise<{timestamp: number, txid: string, value: number}> {
+    channelId = Common.channelIntegerIdToShortId(channelId);
+
+    if (this.fundingTxCache[channelId]) {
+      return this.fundingTxCache[channelId];
+    }
+
+    const parts = channelId.split('x');
+    const blockHeight = parts[0];
+    const txIdx = parts[1];
+    const outputIdx = parts[2];
+
+    let block = this.blocksCache[blockHeight];
+    // Fetch it from core
+    if (!block) {
+      const blockHash = await bitcoinClient.getBlockHash(parseInt(blockHeight, 10));
+      block = await bitcoinClient.getBlock(blockHash, 1);
+    }
+    this.blocksCache[block.height] = block;
+
+    const blocksCacheHashes = Object.keys(this.blocksCache).sort((a, b) => parseInt(b) - parseInt(a)).reverse();
+    if (blocksCacheHashes.length > BLOCKS_CACHE_MAX_SIZE) {
+      for (let i = 0; i < 10; ++i) {
+        delete this.blocksCache[blocksCacheHashes[i]];
+      }
+    }
+
+    const txid = block.tx[txIdx];
+    const rawTx = await bitcoinClient.getRawTransaction(txid);
+    const tx = await bitcoinClient.decodeRawTransaction(rawTx);
+
+    this.fundingTxCache[channelId] = {
+      timestamp: block.time,
+      txid: txid,
+      value: tx.vout[outputIdx].value,
+    };
+
+    ++this.channelNewlyProcessed;
+
+    return this.fundingTxCache[channelId];
+  }
+}
+
+export default new FundingTxFetcher;

backend/src/tasks/lightning/sync-tasks/node-locations.ts

@@ -4,9 +4,16 @@ import nodesApi from '../../../api/explorer/nodes.api';
 import config from '../../../config';
 import DB from '../../../database';
 import logger from '../../../logger';
+import { ResultSetHeader } from 'mysql2';
+import * as IPCheck from '../../../utils/ipcheck.js';
 
 export async function $lookupNodeLocation(): Promise<void> {
-  logger.info(`Running node location updater using Maxmind...`);
+  let loggerTimer = new Date().getTime() / 1000;
+  let progress = 0;
+  let nodesUpdated = 0;
+  let geoNamesInserted = 0;
+
+  logger.info(`Running node location updater using Maxmind`);
   try {
     const nodes = await nodesApi.$getAllNodes();
     const lookupCity = await maxmind.open<CityResponse>(config.MAXMIND.GEOLITE2_CITY);
@@ -18,24 +25,47 @@ export async function $lookupNodeLocation(): Promise<void> {
       for (const socket of sockets) {
         const ip = socket.substring(0, socket.lastIndexOf(':')).replace('[', '').replace(']', '');
         const hasClearnet = [4, 6].includes(net.isIP(ip));
+
         if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') {
           const city = lookupCity.get(ip);
           const asn = lookupAsn.get(ip);
           const isp = lookupIsp.get(ip);
 
+          let asOverwrite: any | undefined;
+          if (asn && (IPCheck.match(ip, '170.75.160.0/20') || IPCheck.match(ip, '172.81.176.0/21'))) {
+            asOverwrite = {
+              asn: 394745,
+              name: 'Lunanode',
+            };
+          }
+          else if (asn && (IPCheck.match(ip, '50.7.0.0/16') || IPCheck.match(ip, '66.90.64.0/18'))) {
+            asOverwrite = {
+              asn: 30058,
+              name: 'FDCservers.net',
+            };
+          }
+          else if (asn && asn.autonomous_system_number === 174) {
+            asOverwrite = {
+              asn: 174,
+              name: 'Cogent Communications',
+            };
+          }
+
           if (city && (asn || isp)) {
-            const query = `UPDATE nodes SET 
-              as_number = ?, 
-              city_id = ?, 
-              country_id = ?, 
-              subdivision_id = ?, 
-              longitude = ?, 
-              latitude = ?, 
-              accuracy_radius = ?
-            WHERE public_key = ?`;
+            const query = `
+              UPDATE nodes SET 
+                as_number = ?, 
+                city_id = ?, 
+                country_id = ?, 
+                subdivision_id = ?, 
+                longitude = ?, 
+                latitude = ?, 
+                accuracy_radius = ?
+              WHERE public_key = ?
+            `;
 
             const params = [
-              isp?.autonomous_system_number ?? asn?.autonomous_system_number,
+              asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number,
               city.city?.geoname_id,
               city.country?.geoname_id,
               city.subdivisions ? city.subdivisions[0].geoname_id : null,
@@ -44,54 +74,90 @@ export async function $lookupNodeLocation(): Promise<void> {
               city.location?.accuracy_radius,
               node.public_key
             ];
-            await DB.query(query, params);
+            let result = await DB.query<ResultSetHeader>(query, params);
+            if (result[0].changedRows ?? 0 > 0) {
+              ++nodesUpdated;
+            }
 
-             // Store Continent
-             if (city.continent?.geoname_id) {
-               await DB.query(
+            // Store Continent
+            if (city.continent?.geoname_id) {
+              result = await DB.query<ResultSetHeader>(
                 `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'continent', ?)`,
                 [city.continent?.geoname_id, JSON.stringify(city.continent?.names)]);
-             }
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
+            }
 
-             // Store Country
-             if (city.country?.geoname_id) {
-               await DB.query(
+            // Store Country
+            if (city.country?.geoname_id) {
+              result = await DB.query<ResultSetHeader>(
                 `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country', ?)`,
                 [city.country?.geoname_id, JSON.stringify(city.country?.names)]);
-             }
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
+            }
 
             // Store Country ISO code
             if (city.country?.iso_code) {
-              await DB.query(
-               `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`,
-               [city.country?.geoname_id, city.country?.iso_code]);
+              result = await DB.query<ResultSetHeader>(
+                `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`,
+                [city.country?.geoname_id, city.country?.iso_code]);
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
             }
 
             // Store Division
             if (city.subdivisions && city.subdivisions[0]) {
-              await DB.query(
+              result = await DB.query<ResultSetHeader>(
                 `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'division', ?)`,
                 [city.subdivisions[0].geoname_id, JSON.stringify(city.subdivisions[0]?.names)]);
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
             }
 
             // Store City
             if (city.city?.geoname_id) {
-              await DB.query(
+              result = await DB.query<ResultSetHeader>(
                 `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'city', ?)`,
                 [city.city?.geoname_id, JSON.stringify(city.city?.names)]);
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
             }
 
             // Store AS name
             if (isp?.autonomous_system_organization ?? asn?.autonomous_system_organization) {
-              await DB.query(
+              result = await DB.query<ResultSetHeader>(
                 `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'as_organization', ?)`,
-                [isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]);
+                [
+                  asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number,
+                  JSON.stringify(asOverwrite?.name ?? isp?.isp ?? asn?.autonomous_system_organization)
+                ]);
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
             }
           }
+
+          ++progress;
+          const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+          if (elapsedSeconds > 10) {
+            logger.info(`Updating node location data ${progress}/${nodes.length}`);
+            loggerTimer = new Date().getTime() / 1000;
+          }
         }
       }
     }
-    logger.info(`Node location data updated.`);
+
+    if (nodesUpdated > 0) {
+      logger.info(`${nodesUpdated} nodes maxmind data updated, ${geoNamesInserted} geo names inserted`);
+    } else {
+      logger.debug(`${nodesUpdated} nodes maxmind data updated, ${geoNamesInserted} geo names inserted`);
+    }
   } catch (e) {
     logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e));
   }

backend/src/tasks/lightning/sync-tasks/stats-importer.ts

@@ -0,0 +1,542 @@
+import DB from '../../../database';
+import { promises } from 'fs';
+import logger from '../../../logger';
+import fundingTxFetcher from './funding-tx-fetcher';
+import config from '../../../config';
+import { ILightningApi } from '../../../api/lightning/lightning-api.interface';
+import { isIP } from 'net';
+import { Common } from '../../../api/common';
+import channelsApi from '../../../api/explorer/channels.api';
+import nodesApi from '../../../api/explorer/nodes.api';
+import { ResultSetHeader } from 'mysql2';
+
+const fsPromises = promises;
+
+class LightningStatsImporter {
+  topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER;
+
+  async $run(): Promise<void> {
+    const [channels]: any[] = await DB.query('SELECT short_id from channels;');
+    logger.info('Caching funding txs for currently existing channels');
+    await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
+
+    if (config.MEMPOOL.NETWORK !== 'mainnet' || config.DATABASE.ENABLED === false) {
+      return;
+    }
+
+    await this.$importHistoricalLightningStats();
+    await this.$cleanupIncorrectSnapshot();
+  }
+
+  /**
+   * Generate LN network stats for one day
+   */
+  public async computeNetworkStats(timestamp: number,
+    networkGraph: ILightningApi.NetworkGraph, isHistorical: boolean = false): Promise<unknown> {
+    // Node counts and network shares
+    let clearnetNodes = 0;
+    let torNodes = 0;
+    let clearnetTorNodes = 0;
+    let unannouncedNodes = 0;
+
+    const [nodesInDbRaw]: any[] = await DB.query(`SELECT public_key FROM nodes`);
+    const nodesInDb = {};
+    for (const node of nodesInDbRaw) {
+      nodesInDb[node.public_key] = node;
+    }
+
+    for (const node of networkGraph.nodes) {
+      // If we don't know about this node, insert it in db
+      if (isHistorical === true && !nodesInDb[node.pub_key]) {
+        await nodesApi.$saveNode({
+          last_update: node.last_update,
+          pub_key: node.pub_key,
+          alias: node.alias,
+          addresses: node.addresses,
+          color: node.color,
+          features: node.features,
+        });
+        nodesInDb[node.pub_key] = node;
+      } else {
+        await nodesApi.$updateNodeSockets(node.pub_key, node.addresses);
+      }
+
+      let hasOnion = false;
+      let hasClearnet = false;
+      let isUnnanounced = true;
+
+      for (const socket of (node.addresses ?? [])) {
+        if (!socket.network?.length && !socket.addr?.length) {
+          continue;
+        }
+        hasOnion = hasOnion || ['torv2', 'torv3'].includes(socket.network) || socket.addr.indexOf('onion') !== -1 || socket.addr.indexOf('torv2') !== -1 || socket.addr.indexOf('torv3') !== -1;
+        hasClearnet = hasClearnet || ['ipv4', 'ipv6'].includes(socket.network) || [4, 6].includes(isIP(socket.addr.split(':')[0])) || socket.addr.indexOf('ipv4') !== -1 || socket.addr.indexOf('ipv6') !== -1;;
+      }
+      if (hasOnion && hasClearnet) {
+        clearnetTorNodes++;
+        isUnnanounced = false;
+      } else if (hasOnion) {
+        torNodes++;
+        isUnnanounced = false;
+      } else if (hasClearnet) {
+        clearnetNodes++;
+        isUnnanounced = false;
+      }
+      if (isUnnanounced) {
+        unannouncedNodes++;
+      }
+    }
+
+    // Channels and node historical stats
+    const nodeStats = {};
+    let capacity = 0;
+    let avgFeeRate = 0;
+    let avgBaseFee = 0;
+    const capacities: number[] = [];
+    const feeRates: number[] = [];
+    const baseFees: number[] = [];
+    const alreadyCountedChannels = {};
+    
+    const [channelsInDbRaw]: any[] = await DB.query(`SELECT short_id FROM channels`);
+    const channelsInDb = {};
+    for (const channel of channelsInDbRaw) {
+      channelsInDb[channel.short_id] = channel;
+    }
+
+    for (const channel of networkGraph.edges) {
+      const short_id = Common.channelIntegerIdToShortId(channel.channel_id);
+
+      const tx = await fundingTxFetcher.$fetchChannelOpenTx(short_id);
+      if (!tx) {
+        logger.err(`Unable to fetch funding tx for channel ${short_id}. Capacity and creation date is unknown. Skipping channel.`);
+        continue;
+      }
+
+      // If we don't know about this channel, insert it in db
+      if (isHistorical === true && !channelsInDb[short_id]) {
+        await channelsApi.$saveChannel({
+          channel_id: short_id,
+          chan_point: `${tx.txid}:${short_id.split('x')[2]}`,
+          last_update: channel.last_update,
+          node1_pub: channel.node1_pub,
+          node2_pub: channel.node2_pub,
+          capacity: (tx.value * 100000000).toString(),
+          node1_policy: null,
+          node2_policy: null,
+        }, 0);
+        channelsInDb[channel.channel_id] = channel;
+      }
+
+      if (!nodeStats[channel.node1_pub]) {
+        nodeStats[channel.node1_pub] = {
+          capacity: 0,
+          channels: 0,
+        };
+      }
+      if (!nodeStats[channel.node2_pub]) {
+        nodeStats[channel.node2_pub] = {
+          capacity: 0,
+          channels: 0,
+        };
+      }
+      
+      if (!alreadyCountedChannels[short_id]) {
+        capacity += Math.round(tx.value * 100000000);
+        capacities.push(Math.round(tx.value * 100000000));
+        alreadyCountedChannels[short_id] = true;
+
+        nodeStats[channel.node1_pub].capacity += Math.round(tx.value * 100000000);
+        nodeStats[channel.node1_pub].channels++;
+        nodeStats[channel.node2_pub].capacity += Math.round(tx.value * 100000000);
+        nodeStats[channel.node2_pub].channels++;
+      }
+
+      if (isHistorical === false) { // Coming from the node
+        for (const policy of [channel.node1_policy, channel.node2_policy]) {
+          if (policy && parseInt(policy.fee_rate_milli_msat, 10) < 5000) {
+            avgFeeRate += parseInt(policy.fee_rate_milli_msat, 10);
+            feeRates.push(parseInt(policy.fee_rate_milli_msat, 10));
+          }  
+          if (policy && parseInt(policy.fee_base_msat, 10) < 5000) {
+            avgBaseFee += parseInt(policy.fee_base_msat, 10);
+            baseFees.push(parseInt(policy.fee_base_msat, 10));
+          }
+        }
+      } else {
+        // @ts-ignore
+        if (channel.node1_policy.fee_rate_milli_msat < 5000) {
+          // @ts-ignore
+          avgFeeRate += parseInt(channel.node1_policy.fee_rate_milli_msat, 10);
+          // @ts-ignore
+          feeRates.push(parseInt(channel.node1_policy.fee_rate_milli_msat), 10);
+        }
+        // @ts-ignore
+        if (channel.node1_policy.fee_base_msat < 5000) {
+          // @ts-ignore
+          avgBaseFee += parseInt(channel.node1_policy.fee_base_msat, 10);
+          // @ts-ignore
+          baseFees.push(parseInt(channel.node1_policy.fee_base_msat), 10);
+        }
+      }
+    }
+
+    let medCapacity = 0;
+    let medFeeRate = 0;
+    let medBaseFee = 0;
+    let avgCapacity = 0;
+
+    avgFeeRate /= Math.max(networkGraph.edges.length, 1);
+    avgBaseFee /= Math.max(networkGraph.edges.length, 1);
+
+    if (capacities.length > 0) {
+      medCapacity = capacities.sort((a, b) => b - a)[Math.round(capacities.length / 2 - 1)];
+      avgCapacity = Math.round(capacity / Math.max(capacities.length, 1));
+    }
+    if (feeRates.length > 0) {
+      medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)];
+    }
+    if (baseFees.length > 0) {
+      medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)];
+    }
+
+    let query = `INSERT INTO lightning_stats(
+        added,
+        channel_count,
+        node_count,
+        total_capacity,
+        tor_nodes,
+        clearnet_nodes,
+        unannounced_nodes,
+        clearnet_tor_nodes,
+        avg_capacity,
+        avg_fee_rate,
+        avg_base_fee_mtokens,
+        med_capacity,
+        med_fee_rate,
+        med_base_fee_mtokens
+      )
+      VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON DUPLICATE KEY UPDATE
+      added = FROM_UNIXTIME(?),
+      channel_count = ?,
+      node_count = ?,
+      total_capacity = ?,
+      tor_nodes = ?,
+      clearnet_nodes = ?,
+      unannounced_nodes = ?,
+      clearnet_tor_nodes = ?,
+      avg_capacity = ?,
+      avg_fee_rate = ?,
+      avg_base_fee_mtokens = ?,
+      med_capacity = ?,
+      med_fee_rate = ?,
+      med_base_fee_mtokens = ?
+    `;
+
+    await DB.query(query, [
+      timestamp,
+      capacities.length,
+      networkGraph.nodes.length,
+      capacity,
+      torNodes,
+      clearnetNodes,
+      unannouncedNodes,
+      clearnetTorNodes,
+      avgCapacity,
+      avgFeeRate,
+      avgBaseFee,
+      medCapacity,
+      medFeeRate,
+      medBaseFee,
+      timestamp,
+      capacities.length,
+      networkGraph.nodes.length,
+      capacity,
+      torNodes,
+      clearnetNodes,
+      unannouncedNodes,
+      clearnetTorNodes,
+      avgCapacity,
+      avgFeeRate,
+      avgBaseFee,
+      medCapacity,
+      medFeeRate,
+      medBaseFee,
+    ]);
+
+    for (const public_key of Object.keys(nodeStats)) {
+      query = `INSERT INTO node_stats(
+          public_key,
+          added,
+          capacity,
+          channels
+        )
+        VALUES (?, FROM_UNIXTIME(?), ?, ?)
+        ON DUPLICATE KEY UPDATE
+        added = FROM_UNIXTIME(?),
+        capacity = ?,
+        channels = ?
+      `;
+
+      await DB.query(query, [
+        public_key,
+        timestamp,
+        nodeStats[public_key].capacity,
+        nodeStats[public_key].channels,
+        timestamp,
+        nodeStats[public_key].capacity,
+        nodeStats[public_key].channels,
+      ]);
+
+      if (!isHistorical) {
+        await DB.query(
+          `UPDATE nodes SET capacity = ?, channels = ? WHERE public_key = ?`,
+          [
+            nodeStats[public_key].capacity,
+            nodeStats[public_key].channels,
+            public_key,
+          ]
+        );
+      }
+    }
+
+    return {
+      added: timestamp,
+      node_count: networkGraph.nodes.length
+    };
+  }
+
+  /**
+   * Import topology files LN historical data into the database
+   */
+  async $importHistoricalLightningStats(): Promise<void> {
+    logger.debug('Run the historical importer');
+    try {
+      let fileList: string[] = [];
+      try {
+        fileList = await fsPromises.readdir(this.topologiesFolder);
+      } catch (e) {
+        logger.err(`Unable to open topology folder at ${this.topologiesFolder}`);
+        throw e;
+      }
+      // Insert history from the most recent to the oldest
+      // This also put the .json cached files first
+      fileList.sort().reverse();
+
+      const [rows]: any[] = await DB.query(`
+        SELECT UNIX_TIMESTAMP(added) AS added
+        FROM lightning_stats
+        ORDER BY added DESC
+      `);
+      const existingStatsTimestamps = {};
+      for (const row of rows) {
+        existingStatsTimestamps[row.added] = row;
+      }
+
+      // For logging purpose
+      let processed = 10;
+      let totalProcessed = 0;
+      let logStarted = false;
+
+      for (const filename of fileList) {
+        processed++;
+
+        const timestamp = parseInt(filename.split('_')[1], 10);
+
+        // Stats exist already, don't calculate/insert them
+        if (existingStatsTimestamps[timestamp] !== undefined) {
+          totalProcessed++;
+          continue;
+        }
+
+        if (filename.indexOf('topology_') === -1) {
+          totalProcessed++;
+          continue;
+        }
+
+        logger.debug(`Reading ${this.topologiesFolder}/${filename}`);
+        let fileContent = '';
+        try {
+          fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8');
+        } catch (e: any) {
+          if (e.errno == -1) { // EISDIR - Ignore directorie
+            totalProcessed++;
+            continue;
+          }
+          logger.err(`Unable to open ${this.topologiesFolder}/${filename}`);
+          totalProcessed++;
+          continue;
+        }
+
+        let graph;
+        try {
+          graph = JSON.parse(fileContent);
+          graph = await this.cleanupTopology(graph);
+        } catch (e) {
+          logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content. Reason: ${e instanceof Error ? e.message : e}`);
+          totalProcessed++;
+          continue;
+        }
+    
+        if (this.isIncorrectSnapshot(timestamp, graph)) {
+          logger.debug(`Ignoring ${this.topologiesFolder}/${filename}, because we defined it as an incorrect snapshot`);
+          ++totalProcessed;
+          continue;
+        }
+
+        if (!logStarted) {
+          logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`);
+          logStarted = true;
+        }
+        
+        const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`;
+        logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`);
+
+        totalProcessed++;
+
+        if (processed > 10) {
+          logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`);
+          processed = 0;
+        } else {
+          logger.debug(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`);
+        }
+        await fundingTxFetcher.$fetchChannelsFundingTxs(graph.edges.map(channel => channel.channel_id.slice(0, -2)));
+        const stat = await this.computeNetworkStats(timestamp, graph, true);
+
+        existingStatsTimestamps[timestamp] = stat;
+      }
+
+      if (totalProcessed > 0) {
+        logger.info(`Lightning network stats historical import completed`);
+      }
+    } catch (e) {
+      logger.err(`Lightning network stats historical failed. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  cleanupTopology(graph): ILightningApi.NetworkGraph {
+    const newGraph = {
+      nodes: <ILightningApi.Node[]>[],
+      edges: <ILightningApi.Channel[]>[],
+    };
+
+    for (const node of graph.nodes) {
+      const addressesParts = (node.addresses ?? '').split(',');
+      const addresses: any[] = [];
+      for (const address of addressesParts) {
+        const formatted = Common.findSocketNetwork(address);
+        addresses.push({
+          network: formatted.network,
+          addr: formatted.url
+        });
+      }
+
+      let rgb = node.rgb_color ?? '#000000';
+      if (rgb.indexOf('#') === -1) {
+        rgb = `#${rgb}`;
+      }
+      newGraph.nodes.push({
+        last_update: node.timestamp ?? 0,
+        pub_key: node.id ?? null,
+        alias: node.alias ?? node.id.slice(0, 20),
+        addresses: addresses,
+        color: rgb,
+        features: {},
+      });
+    }
+
+    for (const adjacency of graph.adjacency) {
+      if (adjacency.length === 0) {
+        continue;
+      } else {
+        for (const edge of adjacency) {
+          newGraph.edges.push({
+            channel_id: edge.scid,
+            chan_point: '',
+            last_update: edge.timestamp,
+            node1_pub: edge.source ?? null,
+            node2_pub: edge.destination ?? null,
+            capacity: '0', // Will be fetch later
+            node1_policy: {
+              time_lock_delta: edge.cltv_expiry_delta,
+              min_htlc: edge.htlc_minimim_msat,
+              fee_base_msat: edge.fee_base_msat,
+              fee_rate_milli_msat: edge.fee_proportional_millionths,
+              max_htlc_msat: edge.htlc_maximum_msat,
+              last_update: edge.timestamp,
+              disabled: false,          
+            },
+            node2_policy: null,
+          });
+        }
+      }
+    }
+
+    return newGraph;
+  }
+
+  private isIncorrectSnapshot(timestamp, graph): boolean {
+    if (timestamp >= 1549065600 /* 2019-02-02 */ && timestamp <= 1550620800 /* 2019-02-20 */ && graph.nodes.length < 2600) {
+        return true;
+    }
+    if (timestamp >= 1552953600 /* 2019-03-19 */ && timestamp <= 1556323200 /* 2019-05-27 */ && graph.nodes.length < 4000) {
+      return true;
+    }
+    if (timestamp >= 1557446400 /* 2019-05-10 */ && timestamp <= 1560470400 /* 2019-06-14 */ && graph.nodes.length < 4000) {
+      return true;
+    }
+    if (timestamp >= 1561680000 /* 2019-06-28 */ && timestamp <= 1563148800 /* 2019-07-15 */ && graph.nodes.length < 4000) {
+      return true;
+    }
+    if (timestamp >= 1571270400 /* 2019-11-17 */ && timestamp <= 1580601600 /* 2020-02-02 */ && graph.nodes.length < 4500) {
+      return true;
+    }
+    if (timestamp >= 1591142400 /* 2020-06-03 */ && timestamp <= 1592006400 /* 2020-06-13 */ && graph.nodes.length < 5500) {
+      return true;
+    }
+    if (timestamp >= 1632787200 /* 2021-09-28 */ && timestamp <= 1633564800 /* 2021-10-07 */ && graph.nodes.length < 13000) {
+      return true;
+    }
+    if (timestamp >= 1634256000 /* 2021-10-15 */ && timestamp <= 1645401600 /* 2022-02-21 */ && graph.nodes.length < 17000) {
+      return true;
+    }
+    if (timestamp >= 1654992000 /* 2022-06-12 */ && timestamp <= 1661472000 /* 2022-08-26 */ && graph.nodes.length < 14000) {
+      return true;
+    }
+
+    return false;
+  }
+
+  private async $cleanupIncorrectSnapshot(): Promise<void> {
+    // We do not run this one automatically because those stats are not supposed to be inserted in the first
+    // place, but I write them here to remind us we manually run those queries
+
+    // DELETE FROM lightning_stats
+    // WHERE (
+    //   UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 AND node_count < 2600 OR
+    //   UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 AND node_count < 4000 OR
+    //   UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 AND node_count < 4000 OR
+    //   UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 AND node_count < 4000 OR
+    //   UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 AND node_count < 4500 OR
+    //   UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 AND node_count < 5500 OR
+    //   UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 AND node_count < 13000 OR
+    //   UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 AND node_count < 17000 OR
+    //   UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000 AND node_count < 14000
+    // )
+
+    // DELETE FROM node_stats
+    // WHERE (
+    //   UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 OR
+    //   UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 OR
+    //   UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 OR
+    //   UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 OR
+    //   UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 OR
+    //   UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 OR
+    //   UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 OR
+    //   UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 OR
+    //   UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000 
+    // )
+  }
+}
+
+export default new LightningStatsImporter;

backend/src/tasks/pools-updater.ts

@@ -12,14 +12,11 @@ import * as https from 'https';
  */
 class PoolsUpdater {
   lastRun: number = 0;
-  currentSha: any = undefined;
-  poolsUrl: string = 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json';
-  treeUrl: string = 'https://api.github.com/repos/mempool/mining-pools/git/trees/master';
+  currentSha: string | undefined = undefined;
+  poolsUrl: string = config.MEMPOOL.POOLS_JSON_URL;
+  treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL;
 
-  constructor() {
-  }
-
-  public async updatePoolsJson() {
+  public async updatePoolsJson(): Promise<void> {
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
       return;
     }
@@ -77,7 +74,7 @@ class PoolsUpdater {
   /**
    * Fetch our latest pools.json sha from the db
    */
-  private async updateDBSha(githubSha: string) {
+  private async updateDBSha(githubSha: string): Promise<void> {
     this.currentSha = githubSha;
     if (config.DATABASE.ENABLED === true) {
       try {

backend/src/tasks/price-updater.ts

@@ -1,4 +1,6 @@
 import * as fs from 'fs';
+import path from "path";
+import { Common } from '../api/common';
 import config from '../config';
 import logger from '../logger';
 import PricesRepository from '../repositories/PricesRepository';
@@ -34,10 +36,10 @@ export interface Prices {
 }
 
 class PriceUpdater {
-  historyInserted: boolean = false;
-  lastRun: number = 0;
-  lastHistoricalRun: number = 0;
-  running: boolean = false;
+  public historyInserted = false;
+  lastRun = 0;
+  lastHistoricalRun = 0;
+  running = false;
   feeds: PriceFeed[] = [];
   currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
   latestPrices: Prices;
@@ -158,7 +160,7 @@ class PriceUpdater {
     const existingPriceTimes = await PricesRepository.$getPricesTimes();
 
     // Insert MtGox weekly prices
-    const pricesJson: any[] = JSON.parse(fs.readFileSync('./src/tasks/price-feeds/mtgox-weekly.json').toString());
+    const pricesJson: any[] = JSON.parse(fs.readFileSync(path.join(__dirname, 'mtgox-weekly.json')).toString());
     const prices = this.getEmptyPricesObj();
     let insertedCount: number = 0;
     for (const price of pricesJson) {

backend/src/utils/ipcheck.js

@@ -0,0 +1,119 @@
+var net = require('net');
+
+var IPCheck = module.exports = function(input) {
+  var self = this;
+
+  if (!(self instanceof IPCheck)) {
+    return new IPCheck(input);
+  }
+
+  self.input = input;
+  self.parse();
+};
+
+IPCheck.prototype.parse = function() {
+  var self = this;
+
+  if (!self.input || typeof self.input !== 'string') return self.valid = false;
+
+  var ip;
+
+  var pos = self.input.lastIndexOf('/');
+  if (pos !== -1) {
+    ip = self.input.substring(0, pos);
+    self.mask = +self.input.substring(pos + 1);
+  } else {
+    ip = self.input;
+    self.mask = null;
+  }
+
+  self.ipv = net.isIP(ip);
+  self.valid = !!self.ipv && !isNaN(self.mask);
+
+  if (!self.valid) return;
+
+  // default mask = 32 for ipv4 and 128 for ipv6
+  if (self.mask === null) self.mask = self.ipv === 4 ? 32 : 128;
+
+  if (self.ipv === 4) {
+    // difference between ipv4 and ipv6 masks
+    self.mask += 96;
+  }
+
+  if (self.mask < 0 || self.mask > 128) {
+    self.valid = false;
+    return;
+  }
+
+  self.address = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ];
+
+  if(self.ipv === 4){
+    self.parseIPv4(ip);
+  }else{
+    self.parseIPv6(ip);
+  }
+};
+
+IPCheck.prototype.parseIPv4 = function(ip) {
+  var self = this;
+
+  // ipv4 addresses live under ::ffff:0:0
+  self.address[10] = self.address[11] = 0xff;
+
+  var octets = ip.split('.');
+  for (var i = 0; i < 4; i++) {
+    self.address[i + 12] = parseInt(octets[i], 10);
+  }
+};
+
+
+var V6_TRANSITIONAL = /:(\d+\.\d+\.\d+\.\d+)$/;
+
+IPCheck.prototype.parseIPv6 = function(ip) {
+  var self = this;
+
+  var transitionalMatch = V6_TRANSITIONAL.exec(ip);
+  if(transitionalMatch){
+    self.parseIPv4(transitionalMatch[1]);
+    return;
+  }
+
+  var bits = ip.split(':');
+  if (bits.length < 8) {
+    ip = ip.replace('::', Array(11 - bits.length).join(':'));
+    bits = ip.split(':');
+  }
+
+  var j = 0;
+  for (var i = 0; i < bits.length; i += 1) {
+    var x = bits[i] ? parseInt(bits[i], 16) : 0;
+    self.address[j++] = x >> 8;
+    self.address[j++] = x & 0xff;
+  }
+};
+
+IPCheck.prototype.match = function(cidr) {
+  var self = this;
+
+  if (!(cidr instanceof IPCheck)) cidr = new IPCheck(cidr);
+  if (!self.valid || !cidr.valid) return false;
+
+  var mask = cidr.mask;
+  var i = 0;
+
+  while (mask >= 8) {
+    if (self.address[i] !== cidr.address[i]) return false;
+
+    i++;
+    mask -= 8;
+  }
+
+  var shift = 8 - mask;
+  return (self.address[i] >>> shift) === (cidr.address[i] >>> shift);
+};
+
+
+IPCheck.match = function(ip, cidr) {
+  ip = ip instanceof IPCheck ? ip : new IPCheck(ip);
+  return ip.match(cidr);
+};

backend/testSetup.ts

@@ -0,0 +1,5 @@
+jest.mock('./mempool-config.json', () => ({}), { virtual: true });
+jest.mock('./src/logger.ts', () => ({}), { virtual: true });
+jest.mock('./src/api/rbf-cache.ts', () => ({}), { virtual: true });
+jest.mock('./src/api/mempool.ts', () => ({}), { virtual: true });
+jest.mock('./src/api/memory-cache.ts', () => ({}), { virtual: true });

backend/tsconfig.build.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig",
+  "exclude": ["**/*.test.*", "**/__mocks__/*", "**/__tests__/*"],
+  "compilerOptions": {
+    "types": ["node"]
+  },
+
+}

backend/tsconfig.json

@@ -1,8 +1,8 @@
 {
   "compilerOptions": {
-    "types": ["node"],
     "module": "commonjs",
     "target": "esnext",
+    "types": ["node", "jest"],
     "lib": ["es2019", "dom"],
     "strict": true,
     "noImplicitAny": false,
@@ -13,7 +13,8 @@
       "node_modules/@types"
     ],
     "allowSyntheticDefaultImports": true,
-    "esModuleInterop": true
+    "esModuleInterop": true,
+    "allowJs": true,
   },
   "include": [
     "src/**/*.ts"
@@ -21,4 +22,4 @@
   "exclude": [
     "dist/**"
   ]
-}
+}

contributors/WesVleuten.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of September 1, 2022.
+
+Signed: WesVleuten

contributors/junderw.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of August 19, 2022.
+
+Signed: junderw

docker/README.md

@@ -102,7 +102,9 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "PRICE_FEED_UPDATE_INTERVAL": 600,
     "USE_SECOND_NODE_FOR_MINFEE": false,
     "EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"],
-    "STDOUT_LOG_MIN_PRIORITY": "info"
+    "STDOUT_LOG_MIN_PRIORITY": "info",
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
+    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master"
   },
 ```
 
@@ -126,6 +128,8 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_USE_SECOND_NODE_FOR_MINFEE: ""
       MEMPOOL_EXTERNAL_ASSETS: ""
       MEMPOOL_STDOUT_LOG_MIN_PRIORITY: ""
+      MEMPOOL_POOLS_JSON_URL: ""
+      MEMPOOL_POOLS_JSON_TREE_URL: ""
       ...
 ```
 
@@ -346,3 +350,68 @@ Corresponding `docker-compose.yml` overrides:
       PRICE_DATA_SERVER_CLEARNET_URL: ""
       ...
 ```
+
+<br/>
+
+`mempool-config.json`:
+```
+  "LIGHTNING": {
+    "ENABLED": false
+    "BACKEND": "lnd"
+    "TOPOLOGY_FOLDER": ""
+    "STATS_REFRESH_INTERVAL": 600
+    "GRAPH_REFRESH_INTERVAL": 600
+    "LOGGER_UPDATE_INTERVAL": 30
+  }
+```
+
+Corresponding `docker-compose.yml` overrides:
+```
+  api:
+    environment:
+      LIGHTNING_ENABLED: false
+      LIGHTNING_BACKEND: "lnd"
+      LIGHTNING_TOPOLOGY_FOLDER: ""
+      LIGHTNING_STATS_REFRESH_INTERVAL: 600
+      LIGHTNING_GRAPH_REFRESH_INTERVAL: 600
+      LIGHTNING_LOGGER_UPDATE_INTERVAL: 30
+      ...
+```
+
+<br/>
+
+`mempool-config.json`:
+```
+  "LND": {
+    "TLS_CERT_PATH": ""
+    "MACAROON_PATH": ""
+    "REST_API_URL": "https://localhost:8080"
+  }
+```
+
+Corresponding `docker-compose.yml` overrides:
+```
+  api:
+    environment:
+      LND_TLS_CERT_PATH: ""
+      LND_MACAROON_PATH: ""
+      LND_REST_API_URL: "https://localhost:8080"
+      ...
+```
+
+<br/>
+
+`mempool-config.json`:
+```
+  "CLIGHTNING": {
+    "SOCKET": ""
+  }
+```
+
+Corresponding `docker-compose.yml` overrides:
+```
+  api:
+    environment:
+      CLIGHTNING_SOCKET: ""
+      ...
+```

docker/backend/Dockerfile

@@ -1,7 +1,7 @@
 FROM node:16.16.0-buster-slim AS builder
 
 ARG commitHash
-ENV DOCKER_COMMIT_HASH=${commitHash}
+ENV MEMPOOL_COMMIT_HASH=${commitHash}
 
 WORKDIR /build
 COPY . .
@@ -9,18 +9,15 @@ COPY . .
 RUN apt-get update
 RUN apt-get install -y build-essential python3 pkg-config
 RUN npm install --omit=dev --omit=optional
-RUN npm run build
+RUN npm run package
 
 FROM node:16.16.0-buster-slim
 
 WORKDIR /backend
 
-COPY --from=builder /build/ .
-
-RUN chmod +x /backend/start.sh
-RUN chmod +x /backend/wait-for-it.sh
-
-RUN chown -R 1000:1000 /backend && chmod -R 755 /backend
+RUN chown 1000:1000 ./
+COPY --from=builder --chown=1000:1000 /build/package ./package/
+COPY --from=builder --chown=1000:1000 /build/mempool-config.json /build/start.sh /build/wait-for-it.sh ./
 
 USER 1000
 

docker/backend/mempool-config.json

@@ -67,6 +67,22 @@
     "ENABLED": __BISQ_ENABLED__,
     "DATA_PATH": "__BISQ_DATA_PATH__"
   },
+  "LIGHTNING": {
+    "ENABLED": __LIGHTNING_ENABLED__,
+    "BACKEND": "__LIGHTNING_BACKEND__",
+    "STATS_REFRESH_INTERVAL": __LIGHTNING_STATS_REFRESH_INTERVAL__,
+    "GRAPH_REFRESH_INTERVAL": __LIGHTNING_GRAPH_REFRESH_INTERVAL__,
+    "LOGGER_UPDATE_INTERVAL": __LIGHTNING_LOGGER_UPDATE_INTERVAL__,
+    "TOPOLOGY_FOLDER": "__LIGHTNING_TOPOLOGY_FOLDER__"
+  },
+  "LND": {
+    "TLS_CERT_PATH": "__LND_TLS_CERT_PATH__",
+    "MACAROON_PATH": "__LND_MACAROON_PATH__",
+    "REST_API_URL": "__LND_REST_API_URL__"
+  },
+  "CLIGHTNING": {
+    "SOCKET": "__CLIGHTNING_SOCKET__"
+  },
   "SOCKS5PROXY": {
     "ENABLED": __SOCKS5PROXY_ENABLED__,
     "USE_ONION": __SOCKS5PROXY_USE_ONION__,

docker/backend/start.sh

@@ -24,6 +24,8 @@ __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool}
 __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
 __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false}
 __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false}
+__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json}
+__MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@@ -89,6 +91,22 @@ __EXTERNAL_DATA_SERVER_LIQUID_ONION__=${EXTERNAL_DATA_SERVER_LIQUID_ONION:=http:
 __EXTERNAL_DATA_SERVER_BISQ_URL__=${EXTERNAL_DATA_SERVER_BISQ_URL:=https://bisq.markets/api}
 __EXTERNAL_DATA_SERVER_BISQ_ONION__=${EXTERNAL_DATA_SERVER_BISQ_ONION:=http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api}
 
+# LIGHTNING
+__LIGHTNING_ENABLED__=${LIGHTNING_ENABLED:=false}
+__LIGHTNING_BACKEND__=${LIGHTNING_BACKEND:="lnd"}
+__LIGHTNING_TOPOLOGY_FOLDER__=${LIGHTNING_TOPOLOGY_FOLDER:=""}
+__LIGHTNING_STATS_REFRESH_INTERVAL__=${LIGHTNING_STATS_REFRESH_INTERVAL:=600}
+__LIGHTNING_GRAPH_REFRESH_INTERVAL__=${LIGHTNING_GRAPH_REFRESH_INTERVAL:=600}
+__LIGHTNING_LOGGER_UPDATE_INTERVAL__=${LIGHTNING_LOGGER_UPDATE_INTERVAL:=30}
+
+# LND
+__LND_TLS_CERT_PATH__=${LND_TLS_CERT_PATH:=""}
+__LND_MACAROON_PATH__=${LND_MACAROON_PATH:=""}
+__LND_REST_API_URL__=${LND_REST_API_URL:="https://localhost:8080"}
+
+# CLN
+__CLIGHTNING_SOCKET__=${CLIGHTNING_SOCKET:=""}
+
 mkdir -p "${__MEMPOOL_CACHE_DIR__}"
 
 sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json
@@ -114,6 +132,8 @@ sed -i "s!__MEMPOOL_USER_AGENT__!${__MEMPOOL_USER_AGENT__}!g" mempool-config.jso
 sed -i "s/__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__/${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json
+sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json
 
 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json
@@ -169,4 +189,20 @@ sed -i "s!__EXTERNAL_DATA_SERVER_LIQUID_ONION__!${__EXTERNAL_DATA_SERVER_LIQUID_
 sed -i "s!__EXTERNAL_DATA_SERVER_BISQ_URL__!${__EXTERNAL_DATA_SERVER_BISQ_URL__}!g" mempool-config.json
 sed -i "s!__EXTERNAL_DATA_SERVER_BISQ_ONION__!${__EXTERNAL_DATA_SERVER_BISQ_ONION__}!g" mempool-config.json
 
-node /backend/dist/index.js
+# LIGHTNING
+sed -i "s!__LIGHTNING_ENABLED__!${__LIGHTNING_ENABLED__}!g" mempool-config.json
+sed -i "s!__LIGHTNING_BACKEND__!${__LIGHTNING_BACKEND__}!g" mempool-config.json
+sed -i "s!__LIGHTNING_TOPOLOGY_FOLDER__!${__LIGHTNING_TOPOLOGY_FOLDER__}!g" mempool-config.json
+sed -i "s!__LIGHTNING_STATS_REFRESH_INTERVAL__!${__LIGHTNING_STATS_REFRESH_INTERVAL__}!g" mempool-config.json
+sed -i "s!__LIGHTNING_GRAPH_REFRESH_INTERVAL__!${__LIGHTNING_GRAPH_REFRESH_INTERVAL__}!g" mempool-config.json
+sed -i "s!__LIGHTNING_LOGGER_UPDATE_INTERVAL__!${__LIGHTNING_LOGGER_UPDATE_INTERVAL__}!g" mempool-config.json
+
+# LND
+sed -i "s!__LND_TLS_CERT_PATH__!${__LND_TLS_CERT_PATH__}!g" mempool-config.json
+sed -i "s!__LND_MACAROON_PATH__!${__LND_MACAROON_PATH__}!g" mempool-config.json
+sed -i "s!__LND_REST_API_URL__!${__LND_REST_API_URL__}!g" mempool-config.json
+
+# CLN
+sed -i "s!__CLIGHTNING_SOCKET__!${__CLIGHTNING_SOCKET__}!g" mempool-config.json
+
+node /backend/package/index.js

docker/init.sh

@@ -1,10 +1,7 @@
 #!/bin/sh
 
 #backend
-gitMaster="\.\.\/\.git\/refs\/heads\/master"
-git ls-remote https://github.com/mempool/mempool.git "$1^{}" | awk '{ print $1}' > ./backend/master
 cp ./docker/backend/* ./backend/
-sed -i "s/${gitMaster}/master/g" ./backend/src/api/backend-info.ts
 
 #frontend
 localhostIP="127.0.0.1"

frontend/.eslintrc

@@ -32,6 +32,7 @@
     "prefer-const": 1,
     "prefer-rest-params": 1,
     "quotes": [1, "single", { "allowTemplateLiterals": true }],
-    "semi": 1
+    "semi": 1,
+    "eqeqeq": 1
   }
 }

frontend/angular.json

@@ -170,6 +170,10 @@
           },
           "configurations": {
             "production": {
+              "assets": [
+                "src/favicon.ico",
+                "src/robots.txt"
+              ],
               "fileReplacements": [
                 {
                   "replace": "src/environments/environment.ts",

frontend/package.json

@@ -34,8 +34,8 @@
     "start:local-staging": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-staging",
     "start:mixed": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c mixed",
     "build": "npm run generate-config && npm run ng -- build --configuration production --localize && npm run sync-assets && npm run build-mempool.js",
-    "sync-assets": "node sync-assets.js && rsync -av ./dist/mempool/browser/en-US/resources ./dist/mempool/browser/resources",
-    "sync-assets-dev": "node sync-assets.js dev",
+    "sync-assets": "rsync -av ./src/resources ./dist/mempool/browser && node sync-assets.js 'dist/mempool/browser/resources/'",
+    "sync-assets-dev": "node sync-assets.js 'src/resources/'",
     "generate-config": "node generate-config.js",
     "build-mempool.js": "npm run build-mempool-js && npm run build-mempool-liquid-js && npm run build-mempool-bisq-js",
     "build-mempool-js": "browserify -p tinyify ./node_modules/@mempool/mempool.js/lib/index.js --standalone mempoolJS > ./dist/mempool/browser/en-US/mempool.js",

frontend/src/app/app-routing.module.ts

@@ -1,21 +1,18 @@
 import { NgModule } from '@angular/core';
-import { Routes, RouterModule, PreloadAllModules } from '@angular/router';
+import { Routes, RouterModule } from '@angular/router';
+import { AppPreloadingStrategy } from './app.preloading-strategy'
 import { StartComponent } from './components/start/start.component';
 import { TransactionComponent } from './components/transaction/transaction.component';
 import { BlockComponent } from './components/block/block.component';
 import { BlockAuditComponent } from './components/block-audit/block-audit.component';
-import { BlockPreviewComponent } from './components/block/block-preview.component';
 import { AddressComponent } from './components/address/address.component';
-import { AddressPreviewComponent } from './components/address/address-preview.component';
 import { MasterPageComponent } from './components/master-page/master-page.component';
-import { MasterPagePreviewComponent } from './components/master-page-preview/master-page-preview.component';
 import { AboutComponent } from './components/about/about.component';
 import { StatusViewComponent } from './components/status-view/status-view.component';
 import { TermsOfServiceComponent } from './components/terms-of-service/terms-of-service.component';
 import { PrivacyPolicyComponent } from './components/privacy-policy/privacy-policy.component';
 import { TrademarkPolicyComponent } from './components/trademark-policy/trademark-policy.component';
 import { BisqMasterPageComponent } from './components/bisq-master-page/bisq-master-page.component';
-import { SponsorComponent } from './components/sponsor/sponsor.component';
 import { PushTransactionComponent } from './components/push-transaction/push-transaction.component';
 import { BlocksList } from './components/blocks-list/blocks-list.component';
 import { LiquidMasterPageComponent } from './components/liquid-master-page/liquid-master-page.component';
@@ -25,6 +22,10 @@ import { AssetsComponent } from './components/assets/assets.component';
 import { AssetComponent } from './components/asset/asset.component';
 import { AssetsNavComponent } from './components/assets/assets-nav/assets-nav.component';
 
+const browserWindow = window || {};
+// @ts-ignore
+const browserWindowEnv = browserWindow.__env || {};
+
 let routes: Routes = [
   {
     path: 'testnet',
@@ -32,7 +33,8 @@ let routes: Routes = [
       {
         path: '',
         pathMatch: 'full',
-        loadChildren: () => import('./graphs/graphs.module').then(m => m.GraphsModule)
+        loadChildren: () => import('./graphs/graphs.module').then(m => m.GraphsModule),
+        data: { preload: true },
       },
       {
         path: '',
@@ -109,7 +111,8 @@ let routes: Routes = [
           },
           {
             path: 'docs',
-            loadChildren: () => import('./docs/docs.module').then(m => m.DocsModule)
+            loadChildren: () => import('./docs/docs.module').then(m => m.DocsModule),
+            data: { preload: true },
           },
           {
             path: 'api',
@@ -117,7 +120,8 @@ let routes: Routes = [
           },
           {
             path: 'lightning',
-            loadChildren: () => import('./lightning/lightning.module').then(m => m.LightningModule)
+            loadChildren: () => import('./lightning/lightning.module').then(m => m.LightningModule),
+            data: { preload: browserWindowEnv && browserWindowEnv.LIGHTNING === true },
           },
         ],
       },
@@ -338,34 +342,18 @@ let routes: Routes = [
   },
   {
     path: 'preview',
-    component: MasterPagePreviewComponent,
     children: [
       {
-        path: 'block/:id',
-        component: BlockPreviewComponent
+        path: '',
+        loadChildren: () => import('./previews.module').then(m => m.PreviewsModule)
       },
       {
-        path: 'testnet/block/:id',
-        component: BlockPreviewComponent
+        path: 'testnet',
+        loadChildren: () => import('./previews.module').then(m => m.PreviewsModule)
       },
       {
-        path: 'signet/block/:id',
-        component: BlockPreviewComponent
-      },
-      {
-        path: 'address/:id',
-        children: [],
-        component: AddressPreviewComponent
-      },
-      {
-        path: 'testnet/address/:id',
-        children: [],
-        component: AddressPreviewComponent
-      },
-      {
-        path: 'signet/address/:id',
-        children: [],
-        component: AddressPreviewComponent
+        path: 'signet',
+        loadChildren: () => import('./previews.module').then(m => m.PreviewsModule)
       },
     ],
   },
@@ -373,10 +361,6 @@ let routes: Routes = [
     path: 'status',
     component: StatusViewComponent
   },
-  {
-    path: 'sponsor',
-    component: SponsorComponent,
-  },
   {
     path: '',
     loadChildren: () => import('./graphs/graphs.module').then(m => m.GraphsModule)
@@ -387,10 +371,6 @@ let routes: Routes = [
   },
 ];
 
-const browserWindow = window || {};
-// @ts-ignore
-const browserWindowEnv = browserWindow.__env || {};
-
 if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'bisq') {
   routes = [{
     path: '',
@@ -616,25 +596,14 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
     },
     {
       path: 'preview',
-      component: MasterPagePreviewComponent,
       children: [
         {
-          path: 'block/:id',
-          component: BlockPreviewComponent
+          path: '',
+          loadChildren: () => import('./previews.module').then(m => m.PreviewsModule)
         },
         {
-          path: 'testnet/block/:id',
-          component: BlockPreviewComponent
-        },
-        {
-          path: 'address/:id',
-          children: [],
-          component: AddressPreviewComponent
-        },
-        {
-          path: 'testnet/address/:id',
-          children: [],
-          component: AddressPreviewComponent
+          path: 'testnet',
+          loadChildren: () => import('./previews.module').then(m => m.PreviewsModule)
         },
       ],
     },
@@ -642,10 +611,6 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
       path: 'status',
       component: StatusViewComponent
     },
-    {
-      path: 'sponsor',
-      component: SponsorComponent,
-    },
     {
       path: '',
       loadChildren: () => import('./graphs/graphs.module').then(m => m.GraphsModule)
@@ -662,7 +627,7 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
     initialNavigation: 'enabled',
     scrollPositionRestoration: 'enabled',
     anchorScrolling: 'enabled',
-    preloadingStrategy: PreloadAllModules
+    preloadingStrategy: AppPreloadingStrategy
   })],
 })
 export class AppRoutingModule { }

frontend/src/app/app.module.ts

@@ -1,5 +1,5 @@
 import { BrowserModule, BrowserTransferStateModule } from '@angular/platform-browser';
-import { NgModule } from '@angular/core';
+import { ModuleWithProviders, NgModule } from '@angular/core';
 import { HttpClientModule, HTTP_INTERCEPTORS } from '@angular/common/http';
 import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
 import { AppRoutingModule } from './app-routing.module';
@@ -18,6 +18,24 @@ import { LanguageService } from './services/language.service';
 import { FiatShortenerPipe } from './shared/pipes/fiat-shortener.pipe';
 import { ShortenStringPipe } from './shared/pipes/shorten-string-pipe/shorten-string.pipe';
 import { CapAddressPipe } from './shared/pipes/cap-address-pipe/cap-address-pipe';
+import { AppPreloadingStrategy } from './app.preloading-strategy';
+
+const providers = [
+  ElectrsApiService,
+  StateService,
+  WebsocketService,
+  AudioService,
+  SeoService,
+  OpenGraphService,
+  StorageService,
+  EnterpriseService,
+  LanguageService,
+  ShortenStringPipe,
+  FiatShortenerPipe,
+  CapAddressPipe,
+  AppPreloadingStrategy,
+  { provide: HTTP_INTERCEPTORS, useClass: HttpCacheInterceptor, multi: true }
+];
 
 @NgModule({
   declarations: [
@@ -31,21 +49,17 @@ import { CapAddressPipe } from './shared/pipes/cap-address-pipe/cap-address-pipe
     BrowserAnimationsModule,
     SharedModule,
   ],
-  providers: [
-    ElectrsApiService,
-    StateService,
-    WebsocketService,
-    AudioService,
-    SeoService,
-    OpenGraphService,
-    StorageService,
-    EnterpriseService,
-    LanguageService,
-    ShortenStringPipe,
-    FiatShortenerPipe,
-    CapAddressPipe,
-    { provide: HTTP_INTERCEPTORS, useClass: HttpCacheInterceptor, multi: true }
-  ],
+  providers: providers,
   bootstrap: [AppComponent]
 })
 export class AppModule { }
+
+@NgModule({})
+export class MempoolSharedModule{
+  static forRoot(): ModuleWithProviders<MempoolSharedModule> {
+    return {
+      ngModule: AppModule,
+      providers: providers
+    };
+  }
+}

frontend/src/app/app.preloading-strategy.ts

@@ -0,0 +1,10 @@
+import { PreloadingStrategy, Route } from '@angular/router';
+import { Observable, timer, mergeMap, of } from 'rxjs';
+
+export class AppPreloadingStrategy implements PreloadingStrategy {
+  preload(route: Route, load: Function): Observable<any> {
+      return route.data && route.data.preload 
+          ? timer(1500).pipe(mergeMap(() => load()))
+          : of(null);
+    }
+}

frontend/src/app/bisq/bisq-address/bisq-address.component.ts

@@ -1,11 +1,11 @@
 import { Component, OnInit, OnDestroy } from '@angular/core';
-import { SeoService } from 'src/app/services/seo.service';
+import { SeoService } from '../../services/seo.service';
 import { switchMap, filter, catchError } from 'rxjs/operators';
 import { ParamMap, ActivatedRoute } from '@angular/router';
 import { Subscription, of } from 'rxjs';
 import { BisqTransaction } from '../bisq.interfaces';
 import { BisqApiService } from '../bisq-api.service';
-import { WebsocketService } from 'src/app/services/websocket.service';
+import { WebsocketService } from '../../services/websocket.service';
 
 @Component({
   selector: 'app-bisq-address',

frontend/src/app/bisq/bisq-block/bisq-block.component.ts

@@ -1,14 +1,14 @@
 import { Component, OnInit, OnDestroy } from '@angular/core';
-import { BisqBlock } from 'src/app/bisq/bisq.interfaces';
+import { BisqBlock } from '../../bisq/bisq.interfaces';
 import { Location } from '@angular/common';
 import { BisqApiService } from '../bisq-api.service';
 import { ActivatedRoute, ParamMap, Router } from '@angular/router';
 import { Subscription, of } from 'rxjs';
 import { switchMap, catchError } from 'rxjs/operators';
-import { SeoService } from 'src/app/services/seo.service';
-import { ElectrsApiService } from 'src/app/services/electrs-api.service';
+import { SeoService } from '../../services/seo.service';
+import { ElectrsApiService } from '../../services/electrs-api.service';
 import { HttpErrorResponse } from '@angular/common/http';
-import { WebsocketService } from 'src/app/services/websocket.service';
+import { WebsocketService } from '../../services/websocket.service';
 
 @Component({
   selector: 'app-bisq-block',

frontend/src/app/bisq/bisq-blocks/bisq-blocks.component.ts

@@ -3,9 +3,9 @@ import { BisqApiService } from '../bisq-api.service';
 import { switchMap, map, take, mergeMap, tap } from 'rxjs/operators';
 import { Observable } from 'rxjs';
 import { BisqBlock, BisqOutput, BisqTransaction } from '../bisq.interfaces';
-import { SeoService } from 'src/app/services/seo.service';
+import { SeoService } from '../../services/seo.service';
 import { ActivatedRoute, Router } from '@angular/router';
-import { WebsocketService } from 'src/app/services/websocket.service';
+import { WebsocketService } from '../../services/websocket.service';
 
 @Component({
   selector: 'app-bisq-blocks',

frontend/src/app/bisq/bisq-dashboard/bisq-dashboard.component.ts

@@ -1,9 +1,9 @@
 import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core';
 import { Observable, combineLatest, BehaviorSubject, of } from 'rxjs';
 import { map, share, switchMap } from 'rxjs/operators';
-import { SeoService } from 'src/app/services/seo.service';
-import { StateService } from 'src/app/services/state.service';
-import { WebsocketService } from 'src/app/services/websocket.service';
+import { SeoService } from '../../services/seo.service';
+import { StateService } from '../../services/state.service';
+import { WebsocketService } from '../../services/websocket.service';
 import { BisqApiService } from '../bisq-api.service';
 import { Trade } from '../bisq.interfaces';
 

frontend/src/app/bisq/bisq-main-dashboard/bisq-main-dashboard.component.ts

@@ -1,9 +1,9 @@
 import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core';
 import { Observable, combineLatest, BehaviorSubject, of } from 'rxjs';
 import { map, share, switchMap } from 'rxjs/operators';
-import { SeoService } from 'src/app/services/seo.service';
-import { StateService } from 'src/app/services/state.service';
-import { WebsocketService } from 'src/app/services/websocket.service';
+import { SeoService } from '../../services/seo.service';
+import { StateService } from '../../services/state.service';
+import { WebsocketService } from '../../services/websocket.service';
 import { BisqApiService } from '../bisq-api.service';
 import { Trade } from '../bisq.interfaces';
 

frontend/src/app/bisq/bisq-market/bisq-market.component.ts

@@ -3,8 +3,8 @@ import { FormBuilder, FormGroup } from '@angular/forms';
 import { ActivatedRoute, Router } from '@angular/router';
 import { combineLatest, merge, Observable, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
-import { SeoService } from 'src/app/services/seo.service';
-import { WebsocketService } from 'src/app/services/websocket.service';
+import { SeoService } from '../../services/seo.service';
+import { WebsocketService } from '../../services/websocket.service';
 import { BisqApiService } from '../bisq-api.service';
 import { OffersMarket, Trade } from '../bisq.interfaces';
 

frontend/src/app/bisq/bisq-stats/bisq-stats.component.ts

@@ -1,9 +1,9 @@
 import { Component, OnInit } from '@angular/core';
 import { BisqApiService } from '../bisq-api.service';
 import { BisqStats } from '../bisq.interfaces';
-import { SeoService } from 'src/app/services/seo.service';
-import { StateService } from 'src/app/services/state.service';
-import { WebsocketService } from 'src/app/services/websocket.service';
+import { SeoService } from '../../services/seo.service';
+import { StateService } from '../../services/state.service';
+import { WebsocketService } from '../../services/websocket.service';
 
 @Component({
   selector: 'app-bisq-stats',

frontend/src/app/bisq/bisq-transaction-details/bisq-transaction-details.component.ts

@@ -1,5 +1,5 @@
 import { Component, ChangeDetectionStrategy, Input, OnChanges } from '@angular/core';
-import { BisqTransaction } from 'src/app/bisq/bisq.interfaces';
+import { BisqTransaction } from '../../bisq/bisq.interfaces';
 
 @Component({
   selector: 'app-bisq-transaction-details',

frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.ts

@@ -1,15 +1,15 @@
 import { Component, OnInit, OnDestroy } from '@angular/core';
 import { ActivatedRoute, ParamMap, Router } from '@angular/router';
-import { BisqTransaction } from 'src/app/bisq/bisq.interfaces';
+import { BisqTransaction } from '../../bisq/bisq.interfaces';
 import { switchMap, map, catchError } from 'rxjs/operators';
 import { of, Observable, Subscription } from 'rxjs';
-import { StateService } from 'src/app/services/state.service';
-import { Block, Transaction } from 'src/app/interfaces/electrs.interface';
+import { StateService } from '../../services/state.service';
+import { Block, Transaction } from '../../interfaces/electrs.interface';
 import { BisqApiService } from '../bisq-api.service';
-import { SeoService } from 'src/app/services/seo.service';
-import { ElectrsApiService } from 'src/app/services/electrs-api.service';
+import { SeoService } from '../../services/seo.service';
+import { ElectrsApiService } from '../../services/electrs-api.service';
 import { HttpErrorResponse } from '@angular/common/http';
-import { WebsocketService } from 'src/app/services/websocket.service';
+import { WebsocketService } from '../../services/websocket.service';
 
 @Component({
   selector: 'app-bisq-transaction',

frontend/src/app/bisq/bisq-transactions/bisq-transactions.component.ts

@@ -4,11 +4,11 @@ import { BisqTransaction, BisqOutput } from '../bisq.interfaces';
 import { Observable, Subscription } from 'rxjs';
 import { switchMap, map, tap } from 'rxjs/operators';
 import { BisqApiService } from '../bisq-api.service';
-import { SeoService } from 'src/app/services/seo.service';
+import { SeoService } from '../../services/seo.service';
 import { FormGroup, FormBuilder } from '@angular/forms';
 import { Router, ActivatedRoute } from '@angular/router';
-import { IMultiSelectOption, IMultiSelectSettings, IMultiSelectTexts } from 'src/app/components/ngx-bootstrap-multiselect/types'
-import { WebsocketService } from 'src/app/services/websocket.service';
+import { IMultiSelectOption, IMultiSelectSettings, IMultiSelectTexts } from '../../components/ngx-bootstrap-multiselect/types'
+import { WebsocketService } from '../../services/websocket.service';
 
 @Component({
   selector: 'app-bisq-transactions',

frontend/src/app/bisq/bisq-transfers/bisq-transfers.component.ts

@@ -1,9 +1,9 @@
 import { Component, OnInit, ChangeDetectionStrategy, Input, OnChanges } from '@angular/core';
-import { BisqTransaction } from 'src/app/bisq/bisq.interfaces';
-import { StateService } from 'src/app/services/state.service';
+import { BisqTransaction } from '../../bisq/bisq.interfaces';
+import { StateService } from '../../services/state.service';
 import { map } from 'rxjs/operators';
 import { Observable } from 'rxjs';
-import { Block } from 'src/app/interfaces/electrs.interface';
+import { Block } from '../../interfaces/electrs.interface';
 
 @Component({
   selector: 'app-bisq-transfers',

frontend/src/app/bisq/bsq-amount/bsq-amount.component.ts

@@ -1,5 +1,5 @@
 import { Component, OnInit, ChangeDetectionStrategy, Input } from '@angular/core';
-import { StateService } from 'src/app/services/state.service';
+import { StateService } from '../../services/state.service';
 import { Observable } from 'rxjs';
 
 @Component({

frontend/src/app/components/about/about.component.html

@@ -25,12 +25,6 @@
     </a>
   </div>
 
-  <br><br>
-  <div class="sponsor-button">
-    <button [hidden]="showNavigateToSponsor" type="button" class="btn btn-primary" (click)="sponsor()" i18n="about.become-a-sponsor">Become a sponsor ❤️</button>
-    <ng-container *ngIf="showNavigateToSponsor" i18n="about.navigate-to-sponsor">Navigate to <a href="https://mempool.space/sponsor" target="_blank">https://mempool.space/sponsor</a> to sponsor</ng-container>
-  </div>
-
   <div class="enterprise-sponsor">
     <h3 i18n="about.sponsors.enterprise.withRocket">Enterprise Sponsors 🚀</h3>
     <div class="wrapper">
@@ -193,8 +187,8 @@
     </div>
   </div>
 
-  <div class="selfhosted-integrations-sponsor">
-    <h3 i18n="about.self-hosted-integrations">Self-Hosted Integrations</h3>
+  <div class="community-integrations-sponsor">
+    <h3 i18n="about.community-integrations">Community Integrations</h3>
     <div class="wrapper">
       <a href="https://github.com/getumbrel/umbrel" target="_blank" title="Umbrel">
         <img class="image" src="/resources/profile/umbrel.png" />
@@ -224,18 +218,24 @@
         <img class="image" src="/resources/profile/start9.png" />
         <span>EmbassyOS</span>
       </a>
-    </div>
-  </div>
-
-  <div class="community-integrations-sponsor">
-    <h3 i18n="about.wallet-integrations">Wallet Integrations</h3>
-    <div class="wrapper">
+      <a href="https://github.com/btcpayserver/btcpayserver" target="_blank" title="BTCPay Server">
+        <img class="image" src="/resources/profile/btcpayserver.svg" />
+        <span>BTCPay</span>
+      </a>
       <a href="https://github.com/bisq-network/bisq" target="_blank" title="Bisq">
         <img class="image" src="/resources/profile/bisq_network.png" />
         <span>Bisq</span>
       </a>
+      <a href="https://github.com/BlueWallet/BlueWallet" target="_blank" title="BlueWallet">
+        <img class="image" src="/resources/profile/bluewallet.png" />
+        <span>BlueWallet</span>
+      </a>
+      <a href="https://github.com/muun/apollo" target="_blank" title="Muun Wallet">
+        <img class="image" src="/resources/profile/muun.png" />
+        <span>Muun</span>
+      </a>
       <a href="https://github.com/spesmilo/electrum" target="_blank" title="Electrum Wallet">
-        <img class="image" src="/resources/profile/electrum.jpg" />
+        <img class="image" src="/resources/profile/electrum.png" />
         <span>Electrum</span>
       </a>
       <a href="https://github.com/cryptoadvance/specter-desktop" target="_blank" title="Specter Wallet">
@@ -250,18 +250,14 @@
         <img class="image" src="/resources/profile/phoenix.jpg" />
         <span>Phoenix</span>
       </a>
+      <a href="https://github.com/lnbits/lnbits-legend" target="_blank" title="LNbits">
+        <img class="image" src="/resources/profile/lnbits.svg" />
+        <span>LNBits</span>
+      </a>
       <a href="https://github.com/layer2tech/mercury-wallet" target="_blank" title="Mercury Wallet">
         <img class="image" src="/resources/profile/mercury.svg" />
         <span>Mercury</span>
       </a>
-      <a href="https://github.com/muun/apollo" target="_blank" title="Muun Wallet">
-        <img class="image" src="/resources/profile/muun.png" />
-        <span>Muun</span>
-      </a>
-      <a href="https://github.com/BlueWallet/BlueWallet" target="_blank" title="BlueWallet">
-        <img class="image" src="/resources/profile/bluewallet.png" />
-        <span>BlueWallet</span>
-      </a>
       <a href="https://github.com/hsjoberg/blixt-wallet" target="_blank" title="Blixt Wallet">
         <img class="image" src="/resources/profile/blixt.png" />
         <span>Blixt</span>
@@ -297,7 +293,7 @@
   </div>
 
   <ng-container *ngIf="translators$ | async | keyvalue as translators else loadingSponsors">
-    <div class="community-sponsor">
+    <div class="project-translators">
       <h3 i18n="about.translators">Project Translators</h3>
       <div class="wrapper">
         <ng-template ngFor let-translator [ngForOf]="translators">

frontend/src/app/components/about/about.component.scss

@@ -43,7 +43,6 @@
   .alliances,
   .enterprise-sponsor,
   .community-integrations-sponsor,
-  .selfhosted-integrations-sponsor,
   .maintainers {
     margin-top: 68px;
     margin-bottom: 68px;
@@ -53,7 +52,7 @@
     margin-bottom: 50px;
   }
 
-  .community-sponsor {
+  .community-sponsor, .project-translators {
     display: flex;
     flex-direction: column;
     .wrapper {
@@ -67,6 +66,13 @@
     }
   }
 
+  .community-sponsor {
+    img {
+      width: 67px;
+      height: 67px;
+    }
+  }
+
   .alliances {
     margin-bottom: 100px;
     a {
@@ -108,8 +114,8 @@
   .enterprise-sponsor,
   .contributors,
   .community-sponsor,
+  .project-translators,
   .community-integrations-sponsor,
-  .selfhosted-integrations-sponsor,
   .maintainers {
     .wrapper {
       display: inline-block;
@@ -132,7 +138,7 @@
     }
   }
 
-  .community-sponsor .wrapper {
+  .community-sponsor .wrapper, .project-translators .wrapper {
     margin: 10px auto 20px;
     a img {
       margin: 6px;
@@ -185,6 +191,6 @@
 }
 
 .community-integrations-sponsor {
-  max-width: 830px;
+  max-width: 970px;
   margin: auto;
 }

frontend/src/app/components/about/about.component.ts

@@ -1,13 +1,13 @@
 import { ChangeDetectionStrategy, Component, Inject, LOCALE_ID, OnInit } from '@angular/core';
 import { WebsocketService } from '../../services/websocket.service';
-import { SeoService } from 'src/app/services/seo.service';
-import { StateService } from 'src/app/services/state.service';
+import { SeoService } from '../../services/seo.service';
+import { StateService } from '../../services/state.service';
 import { Observable } from 'rxjs';
-import { ApiService } from 'src/app/services/api.service';
-import { IBackendInfo } from 'src/app/interfaces/websocket.interface';
+import { ApiService } from '../../services/api.service';
+import { IBackendInfo } from '../../interfaces/websocket.interface';
 import { Router } from '@angular/router';
 import { map } from 'rxjs/operators';
-import { ITranslators } from 'src/app/interfaces/node-api.interface';
+import { ITranslators } from '../../interfaces/node-api.interface';
 
 @Component({
   selector: 'app-about',
@@ -61,9 +61,9 @@ export class AboutComponent implements OnInit {
     );
   }
 
-  sponsor() {
+  sponsor(): void {
     if (this.officialMempoolSpace && this.stateService.env.BASE_MODULE === 'mempool') {
-      this.router.navigateByUrl('/sponsor');
+      this.router.navigateByUrl('/enterprise');
     } else {
       this.showNavigateToSponsor = true;
     }

frontend/src/app/components/address-labels/address-labels.component.html

@@ -1,9 +1,16 @@
-<a *ngIf="channel; else default" [routerLink]="['/lightning/channel' | relativeUrl, channel.id]">
-  <span
-    *ngIf="label"
-    class="badge badge-pill badge-warning"
-  >{{ label }}</span>
-</a>
+<ng-template [ngIf]="channel" [ngIfElse]="default">
+  <div>
+    <div class="badge-positioner">
+      <a [routerLink]="['/lightning/channel' | relativeUrl, channel.id]">
+        <span 
+          *ngIf="label"
+          class="badge badge-pill badge-warning"
+        >{{ label }}</span>
+      </a>
+    </div>
+    &nbsp;
+  </div>
+</ng-template>
 
 <ng-template #default>
   <span

frontend/src/app/components/address-labels/address-labels.component.scss

@@ -1,3 +1,7 @@
 .badge {
   margin-right: 2px;
 }
+
+.badge-positioner {
+  position: absolute;
+}

frontend/src/app/components/address-labels/address-labels.component.ts

@@ -1,7 +1,7 @@
 import { Component, ChangeDetectionStrategy, Input, OnChanges } from '@angular/core';
 import { Vin, Vout } from '../../interfaces/electrs.interface';
-import { StateService } from 'src/app/services/state.service';
-import { parseMultisigScript } from 'src/app/bitcoin.utils';
+import { StateService } from '../../services/state.service';
+import { parseMultisigScript } from '../../bitcoin.utils';
 
 @Component({
   selector: 'app-address-labels',

frontend/src/app/components/address/address-preview.component.html

@@ -1,12 +1,14 @@
 <div class="box preview-box" *ngIf="address && !error">
+  <app-preview-title>
+    <span i18n="shared.address">Address</span>
+  </app-preview-title>
   <div class="row">
     <div class="col-md">
-      <div class="title-address">
-        <h1 i18n="shared.address">Address</h1>
+      <div class="row d-flex justify-content-between">
+        <div class="title-wrapper">
+          <h1 class="title truncated"><span class="first">{{addressString.slice(0,-4)}}</span><span class="last-four">{{addressString.slice(-4)}}</span></h1>
+        </div>
       </div>
-      <a [routerLink]="['/address/' | relativeUrl, addressString]" class="address-link" >
-        <span class="truncated-address">{{addressString.slice(0,-4)}}</span><span class="last-four">{{addressString.slice(-4)}}</span>
-      </a>
       <table class="table table-borderless table-striped">
         <tbody>
           <tr *ngIf="addressInfo && addressInfo.unconfidential">
@@ -44,7 +46,7 @@
     <div class="w-100 d-block d-md-none"></div>
     <div class="col-md qrcode-col">
       <div class="qr-wrapper">
-        <app-qrcode [data]="address.address" [size]="370"></app-qrcode>
+        <app-qrcode [data]="address.address" [size]="448"></app-qrcode>
       </div>
     </div>
   </div>

frontend/src/app/components/address/address-preview.component.scss

@@ -1,6 +1,5 @@
-h1 {
-  font-size: 42px;
-  margin: 0;
+.title-wrapper {
+  padding: 0 15px;
 }
 
 .qr-wrapper {
@@ -11,36 +10,21 @@ h1 {
 }
 
 .qrcode-col {
-  width: 420px;
-  min-width: 420px;
+  width: 468px;
+  min-width: 468px;
   flex-grow: 0;
   flex-shrink: 0;
   text-align: center;
+  padding: 0;
+  margin-left: 2px;
+  margin-right: 15px;
 }
 
 .table {
-  font-size: 24px;
+  font-size: 32px;
+  margin-top: 48px;
 
   ::ng-deep .symbol {
-    font-size: 18px;
+    font-size: 24px;
   }
-}
-
-.address-link {
-  font-size: 20px;
-  margin-bottom: 0.5em;
-  display: flex;
-  flex-direction: row;
-  align-items: baseline;
-  .truncated-address {
-    text-overflow: ellipsis;
-    overflow: hidden;
-    max-width: calc(505px - 4em);
-    display: inline-block;
-    white-space: nowrap;
-  }
-  .last-four {
-    display: inline-block;
-    white-space: nowrap;
-  }
-}
+}

frontend/src/app/components/address/address-preview.component.ts

@@ -3,13 +3,13 @@ import { ActivatedRoute, ParamMap } from '@angular/router';
 import { ElectrsApiService } from '../../services/electrs-api.service';
 import { switchMap, filter, catchError, map, tap } from 'rxjs/operators';
 import { Address, Transaction } from '../../interfaces/electrs.interface';
-import { StateService } from 'src/app/services/state.service';
-import { OpenGraphService } from 'src/app/services/opengraph.service';
-import { AudioService } from 'src/app/services/audio.service';
-import { ApiService } from 'src/app/services/api.service';
+import { StateService } from '../../services/state.service';
+import { OpenGraphService } from '../../services/opengraph.service';
+import { AudioService } from '../../services/audio.service';
+import { ApiService } from '../../services/api.service';
 import { of, merge, Subscription, Observable } from 'rxjs';
-import { SeoService } from 'src/app/services/seo.service';
-import { AddressInformation } from 'src/app/interfaces/node-api.interface';
+import { SeoService } from '../../services/seo.service';
+import { AddressInformation } from '../../interfaces/node-api.interface';
 
 @Component({
   selector: 'app-address-preview',
@@ -19,6 +19,7 @@ import { AddressInformation } from 'src/app/interfaces/node-api.interface';
 export class AddressPreviewComponent implements OnInit, OnDestroy {
   network = '';
 
+  rawAddress: string;
   address: Address;
   addressString: string;
   isLoadingAddress = true;
@@ -44,7 +45,6 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
   ) { }
 
   ngOnInit() {
-    this.openGraphService.setPreviewLoading();
     this.stateService.networkChanged$.subscribe((network) => this.network = network);
 
     this.addressLoadingStatus$ = this.route.paramMap
@@ -56,6 +56,8 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
     this.mainSubscription = this.route.paramMap
       .pipe(
         switchMap((params: ParamMap) => {
+          this.rawAddress = params.get('id') || '';
+          this.openGraphService.waitFor('address-data-' + this.rawAddress);
           this.error = undefined;
           this.isLoadingAddress = true;
           this.loadedConfirmedTxCount = 0;
@@ -73,6 +75,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
                 this.isLoadingAddress = false;
                 this.error = err;
                 console.log(err);
+                this.openGraphService.fail('address-data-' + this.rawAddress);
                 return of(null);
               })
             );
@@ -90,7 +93,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
           this.address = address;
           this.updateChainStats();
           this.isLoadingAddress = false;
-          this.openGraphService.setPreviewReady();
+          this.openGraphService.waitOver('address-data-' + this.rawAddress);
         })
       )
       .subscribe(() => {},
@@ -98,6 +101,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
           console.log(error);
           this.error = error;
           this.isLoadingAddress = false;
+          this.openGraphService.fail('address-data-' + this.rawAddress);
         }
       );
   }

frontend/src/app/components/address/address.component.ts

@@ -3,13 +3,13 @@ import { ActivatedRoute, ParamMap } from '@angular/router';
 import { ElectrsApiService } from '../../services/electrs-api.service';
 import { switchMap, filter, catchError, map, tap } from 'rxjs/operators';
 import { Address, Transaction } from '../../interfaces/electrs.interface';
-import { WebsocketService } from 'src/app/services/websocket.service';
-import { StateService } from 'src/app/services/state.service';
-import { AudioService } from 'src/app/services/audio.service';
-import { ApiService } from 'src/app/services/api.service';
+import { WebsocketService } from '../../services/websocket.service';
+import { StateService } from '../../services/state.service';
+import { AudioService } from '../../services/audio.service';
+import { ApiService } from '../../services/api.service';
 import { of, merge, Subscription, Observable } from 'rxjs';
-import { SeoService } from 'src/app/services/seo.service';
-import { AddressInformation } from 'src/app/interfaces/node-api.interface';
+import { SeoService } from '../../services/seo.service';
+import { AddressInformation } from '../../interfaces/node-api.interface';
 
 @Component({
   selector: 'app-address',

frontend/src/app/components/app/app.component.ts

@@ -1,8 +1,8 @@
 import { Location } from '@angular/common';
 import { Component, HostListener, OnInit, Inject, LOCALE_ID, HostBinding } from '@angular/core';
 import { Router, NavigationEnd } from '@angular/router';
-import { StateService } from 'src/app/services/state.service';
-import { OpenGraphService } from 'src/app/services/opengraph.service';
+import { StateService } from '../../services/state.service';
+import { OpenGraphService } from '../../services/opengraph.service';
 import { NgbTooltipConfig } from '@ng-bootstrap/ng-bootstrap';
 
 @Component({

frontend/src/app/components/asset-circulation/asset-circulation.component.ts

@@ -1,10 +1,9 @@
 import { ChangeDetectionStrategy, Component, Inject, Input, LOCALE_ID, OnInit } from '@angular/core';
 import { combineLatest, Observable } from 'rxjs';
 import { map } from 'rxjs/operators';
-import { moveDec } from 'src/app/bitcoin.utils';
-import { AssetsService } from 'src/app/services/assets.service';
-import { ElectrsApiService } from 'src/app/services/electrs-api.service';
-import { formatNumber } from '@angular/common';
+import { moveDec } from '../../bitcoin.utils';
+import { AssetsService } from '../../services/assets.service';
+import { ElectrsApiService } from '../../services/electrs-api.service';
 import { environment } from 'src/environments/environment';
 
 @Component({

frontend/src/app/components/asset/asset.component.ts

@@ -3,15 +3,15 @@ import { ActivatedRoute, ParamMap } from '@angular/router';
 import { ElectrsApiService } from '../../services/electrs-api.service';
 import { switchMap, filter, catchError, take } from 'rxjs/operators';
 import { Asset, Transaction } from '../../interfaces/electrs.interface';
-import { WebsocketService } from 'src/app/services/websocket.service';
-import { StateService } from 'src/app/services/state.service';
-import { AudioService } from 'src/app/services/audio.service';
-import { ApiService } from 'src/app/services/api.service';
+import { WebsocketService } from '../../services/websocket.service';
+import { StateService } from '../../services/state.service';
+import { AudioService } from '../../services/audio.service';
+import { ApiService } from '../../services/api.service';
 import { of, merge, Subscription, combineLatest } from 'rxjs';
-import { SeoService } from 'src/app/services/seo.service';
+import { SeoService } from '../../services/seo.service';
 import { environment } from 'src/environments/environment';
-import { AssetsService } from 'src/app/services/assets.service';
-import { moveDec } from 'src/app/bitcoin.utils';
+import { AssetsService } from '../../services/assets.service';
+import { moveDec } from '../../bitcoin.utils';
 
 @Component({
   selector: 'app-asset',

frontend/src/app/components/assets/asset-group/asset-group.component.ts

@@ -2,8 +2,8 @@ import { Component, OnInit } from '@angular/core';
 import { ActivatedRoute, ParamMap } from '@angular/router';
 import { combineLatest, Observable } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
-import { ApiService } from 'src/app/services/api.service';
-import { AssetsService } from 'src/app/services/assets.service';
+import { ApiService } from '../../../services/api.service';
+import { AssetsService } from '../../../services/assets.service';
 
 @Component({
   selector: 'app-asset-group',

frontend/src/app/components/assets/assets-featured/assets-featured.component.ts

@@ -1,6 +1,6 @@
 import { Component, OnInit } from '@angular/core';
 import { Observable } from 'rxjs';
-import { ApiService } from 'src/app/services/api.service';
+import { ApiService } from '../../../services/api.service';
 
 @Component({
   selector: 'app-assets-featured',

frontend/src/app/components/assets/assets-nav/assets-nav.component.ts

@@ -4,11 +4,11 @@ import { Router } from '@angular/router';
 import { NgbTypeahead } from '@ng-bootstrap/ng-bootstrap';
 import { merge, Observable, of, Subject } from 'rxjs';
 import { distinctUntilChanged, filter, map, switchMap } from 'rxjs/operators';
-import { AssetExtended } from 'src/app/interfaces/electrs.interface';
-import { AssetsService } from 'src/app/services/assets.service';
-import { SeoService } from 'src/app/services/seo.service';
-import { StateService } from 'src/app/services/state.service';
-import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.pipe';
+import { AssetExtended } from '../../../interfaces/electrs.interface';
+import { AssetsService } from '../../../services/assets.service';
+import { SeoService } from '../../../services/seo.service';
+import { StateService } from '../../../services/state.service';
+import { RelativeUrlPipe } from '../../../shared/pipes/relative-url/relative-url.pipe';
 import { environment } from 'src/environments/environment';
 
 @Component({

frontend/src/app/components/assets/assets.component.ts

@@ -1,13 +1,13 @@
 import { Component, OnInit, ChangeDetectionStrategy } from '@angular/core';
-import { AssetsService } from 'src/app/services/assets.service';
+import { AssetsService } from '../../services/assets.service';
 import { environment } from 'src/environments/environment';
 import { FormGroup } from '@angular/forms';
 import { filter, map, switchMap, take } from 'rxjs/operators';
 import { ActivatedRoute, Router } from '@angular/router';
 import { combineLatest, Observable } from 'rxjs';
-import { AssetExtended } from 'src/app/interfaces/electrs.interface';
-import { SeoService } from 'src/app/services/seo.service';
-import { StateService } from 'src/app/services/state.service';
+import { AssetExtended } from '../../interfaces/electrs.interface';
+import { SeoService } from '../../services/seo.service';
+import { StateService } from '../../services/state.service';
 
 @Component({
   selector: 'app-assets',

frontend/src/app/components/bisq-master-page/bisq-master-page.component.html

@@ -2,7 +2,36 @@
   <nav class="navbar navbar-expand-md navbar-dark bg-dark">
   <a class="navbar-brand" [routerLink]="['/' | relativeUrl]" style="position: relative;">
     <ng-container *ngIf="{ val: connectionState$ | async } as connectionState">
-      <img src="/resources/bisq/bisq-markets-logo.png" height="35" width="140" class="logo" [ngStyle]="{'opacity': connectionState.val === 2 ? 1 : 0.5 }">
+    <div height="35" width="140" class="logo" [ngStyle]="{'opacity': connectionState.val === 2 ? 1 : 0.5 }">
+      <svg width="140" viewBox="0 0 280 71" fill="none" xmlns="http://www.w3.org/2000/svg">
+        <g clip-path="url(#clip0)">
+        <path d="M68.137 62.1803C68.137 66.8789 64.3484 70.6717 59.6552 70.6717H8.48178C3.78853 70.6717 0 66.8789 0 62.1803V10.9485C0 6.24988 3.8168 2.45703 8.48178 2.45703H59.6552C64.3484 2.45703 68.137 6.24988 68.137 10.9485V62.1803Z" fill="#2E3349"/>
+        <path d="M0 36.6504V62.1814C0 66.88 3.8168 70.6728 8.51005 70.6728H59.6552C64.3484 70.6728 68.1652 66.88 68.1652 62.1814V36.6504H0Z" fill="url(#paint0_linear)"/>
+        <path opacity="0.3" d="M60.054 61.5586C60.054 62.6059 59.3472 63.455 58.4707 63.455H49.6497C48.7732 63.455 48.0664 62.6059 48.0664 61.5586V11.5722C48.0664 10.5249 48.7732 9.67578 49.6497 9.67578H58.4707C59.3472 9.67578 60.054 10.5249 60.054 11.5722V61.5586Z" fill="white"/>
+        <path d="M85.4102 33.8734H89.6242V30.6894H89.7179C91.3567 33.0774 94.447 34.4352 97.4437 34.4352C104.327 34.4352 108.728 29.3315 108.728 22.7763C108.728 16.1274 104.28 11.1173 97.4437 11.1173C94.2597 11.1173 91.2162 12.5688 89.7179 14.8632H89.6242V3.84819H85.4102V33.8734ZM96.9287 30.5021C92.4336 30.5021 89.6242 27.2713 89.6242 22.7763C89.6242 18.2812 92.4336 15.0504 96.9287 15.0504C101.424 15.0504 104.233 18.2812 104.233 22.7763C104.233 27.2713 101.424 30.5021 96.9287 30.5021Z" fill="white"/>
+        <path d="M112.059 33.8734H116.274V11.6792H112.059V33.8734ZM111.076 3.71923C111.076 5.40487 112.481 6.80957 114.167 6.80957C115.852 6.80957 117.257 5.40487 117.257 3.71923C117.257 2.0336 115.852 0.628906 114.167 0.628906C112.481 0.628906 111.076 2.0336 111.076 3.71923Z" fill="white"/>
+        <path d="M136.522 14.7695C134.93 12.1474 131.887 11.1173 128.937 11.1173C124.769 11.1173 120.509 13.318 120.509 17.9535C120.509 22.2144 123.693 23.385 127.298 24.2746C129.124 24.696 132.636 25.1643 132.636 27.6927C132.636 29.6125 130.295 30.5021 128.141 30.5021C125.706 30.5021 124.114 29.2379 122.756 27.88L119.572 30.5021C121.773 33.4988 124.489 34.4352 128.141 34.4352C132.542 34.4352 137.131 32.4687 137.131 27.4586C137.131 23.2913 134.321 21.8866 130.669 20.997C128.796 20.5756 125.004 20.201 125.004 17.5321C125.004 15.9401 126.736 15.0504 128.703 15.0504C130.81 15.0504 132.261 16.0337 133.244 17.2511L136.522 14.7695Z" fill="white"/>
+        <path d="M162.956 11.6792H158.742V14.8632H158.648C157.01 12.4752 153.919 11.1173 150.923 11.1173C144.04 11.1173 139.638 16.221 139.638 22.7763C139.638 29.4252 144.086 34.4352 150.923 34.4352C154.107 34.4352 157.15 32.9837 158.648 30.6894H158.742V39.9435H162.956V11.6792ZM151.438 15.0504C155.933 15.0504 158.742 18.2812 158.742 22.7763C158.742 27.2713 155.933 30.5021 151.438 30.5021C146.943 30.5021 144.133 27.2713 144.133 22.7763C144.133 18.2812 146.943 15.0504 151.438 15.0504Z" fill="white"/>
+        <path d="M84.8989 66.394C86.5846 66.394 87.9893 64.9893 87.9893 63.3037C87.9893 61.6181 86.5846 60.2134 84.8989 60.2134C83.2133 60.2134 81.8086 61.6181 81.8086 63.3037C81.8086 64.9893 83.2133 66.394 84.8989 66.394Z" fill="#25B135"/>
+        <path d="M94.6063 66.1131H98.8204V54.5946C98.8204 49.5845 101.536 47.2902 104.58 47.2902C108.653 47.2902 109.262 50.2869 109.262 54.5009V66.1131H113.476V53.9859C113.476 50.0527 115.068 47.2902 119.142 47.2902C123.215 47.2902 123.918 50.3805 123.918 53.7518V66.1131H128.132V53.1899C128.132 48.2266 126.54 43.357 119.704 43.357C117.035 43.357 114.132 44.7617 112.68 47.4775C111.275 44.7617 109.028 43.357 105.75 43.357C101.77 43.357 99.0545 46.0728 98.6331 47.3838H98.5395V43.9189H94.6063V66.1131Z" fill="#25B135"/>
+        <path d="M135.566 49.2567C137.111 48.0862 138.656 46.7283 141.887 46.7283C145.493 46.7283 147.178 49.1163 147.178 51.4106V52.3471H144.088C137.345 52.3471 131.867 54.3137 131.867 60.0261C131.867 64.3338 135.426 66.675 139.546 66.675C142.917 66.675 145.446 65.598 147.319 62.7418H147.412C147.412 63.8656 147.459 64.9893 147.553 66.1131H151.299C151.158 64.9425 151.111 63.6315 151.111 62.0863V50.7551C151.111 46.9156 148.396 43.357 141.84 43.357C138.75 43.357 135.379 44.434 133.038 46.6346L135.566 49.2567ZM147.178 55.4374V56.8421C147.178 59.8388 145.539 63.3037 140.857 63.3037C137.954 63.3037 136.081 62.2268 136.081 59.6983C136.081 56.1398 140.951 55.4374 144.931 55.4374H147.178Z" fill="#25B135"/>
+        <path d="M155.689 66.1131H159.903V54.9692C159.903 50.0996 162.151 47.8521 166.271 47.8521C166.927 47.8521 167.629 47.9925 168.331 48.1798L168.519 43.638C167.957 43.4507 167.301 43.357 166.646 43.357C163.883 43.357 161.074 44.9958 159.997 47.337H159.903V43.9189H155.689V66.1131Z" fill="#25B135"/>
+        <path d="M171.484 66.1131H175.698V54.5946L185.999 66.1131H191.993L180.755 54.0327L191.103 43.9657H185.25L175.698 53.5645V34.5527H171.484V66.1131Z" fill="#25B135"/>
+        <path d="M215.206 56.5612V55.0628C215.206 49.3504 212.209 43.357 204.39 43.357C197.741 43.357 192.918 48.3671 192.918 55.016C192.918 61.6181 197.32 66.675 204.343 66.675C208.604 66.675 211.835 65.1766 214.176 62.1331L210.992 59.6983C209.353 61.7117 207.48 63.0228 204.905 63.0228C201.019 63.0228 197.413 60.4475 197.413 56.5612H215.206ZM197.413 53.1899C197.413 50.24 200.129 46.7283 204.296 46.7283C208.557 46.7283 210.617 49.4909 210.711 53.1899H197.413Z" fill="#25B135"/>
+        <path d="M232.14 43.9189H226.1V37.6914H221.886V43.9189H217.016V47.5711H221.886V59.1364C221.886 62.695 221.98 66.675 228.488 66.675C229.331 66.675 231.297 66.4877 232.281 65.9258V62.0863C231.438 62.6014 230.267 62.7418 229.284 62.7418C226.1 62.7418 226.1 60.1197 226.1 57.6381V47.5711H232.14V43.9189Z" fill="#25B135"/>
+        <path d="M252.654 47.0092C251.062 44.3871 248.019 43.357 245.069 43.357C240.902 43.357 236.641 45.5577 236.641 50.1932C236.641 54.4541 239.825 55.6247 243.43 56.5143C245.256 56.9358 248.768 57.404 248.768 59.9324C248.768 61.8522 246.427 62.7418 244.273 62.7418C241.838 62.7418 240.246 61.4776 238.888 60.1197L235.704 62.7418C237.905 65.7385 240.621 66.675 244.273 66.675C248.674 66.675 253.263 64.7084 253.263 59.6983C253.263 55.5311 250.454 54.1264 246.801 53.2367C244.929 52.8153 241.136 52.4407 241.136 49.7718C241.136 48.1798 242.868 47.2902 244.835 47.2902C246.942 47.2902 248.393 48.2735 249.377 49.4909L252.654 47.0092Z" fill="#25B135"/>
+        </g>
+        <defs>
+        <linearGradient id="paint0_linear" x1="34.0826" y1="36.6504" x2="34.0826" y2="77.1139" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#25B135"/>
+        <stop offset="1" stop-color="#005209"/>
+        </linearGradient>
+        <clipPath id="clip0">
+        <rect width="280" height="70" fill="white" transform="translate(0 0.671875)"/>
+        </clipPath>
+        </defs>
+        </svg>
+      </div>
       <div class="connection-badge">
         <div class="badge badge-warning" *ngIf="connectionState.val === 0" i18n="master-page.offline">Offline</div>
         <div class="badge badge-warning" *ngIf="connectionState.val === 1" i18n="master-page.reconnecting">Reconnecting...</div>
@@ -12,16 +41,16 @@
 
   <div ngbDropdown (window:resize)="onResize($event)" class="dropdown-container" *ngIf="env.TESTNET_ENABLED || env.SIGNET_ENABLED || env.LIQUID_ENABLED || env.BISQ_ENABLED || env.LIQUID_TESTNET_ENABLED">
     <button ngbDropdownToggle type="button" class="btn btn-secondary dropdown-toggle-split" aria-haspopup="true">
-      <img src="/resources/bisq-logo.png" style="width: 25px; height: 25px;" class="mr-1">
+      <app-svg-images name="bisq" width="20" height="20" viewBox="0 0 75 75" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images>
     </button>
     <div ngbDropdownMenu [ngClass]="{'dropdown-menu-right' : isMobile}">
-      <a [href]="env.MEMPOOL_WEBSITE_URL + urlLanguage" ngbDropdownItem class="mainnet"><img src="/resources/bitcoin-logo.png" style="width: 30px;" class="mr-1"> Mainnet</a>
-      <a [href]="env.MEMPOOL_WEBSITE_URL + urlLanguage + '/signet'" ngbDropdownItem *ngIf="env.SIGNET_ENABLED" class="signet"><img src="/resources/signet-logo.png" style="width: 30px;" class="mr-1"> Signet</a>
-      <a [href]="env.MEMPOOL_WEBSITE_URL + urlLanguage + '/testnet'" ngbDropdownItem *ngIf="env.TESTNET_ENABLED" class="testnet"><img src="/resources/testnet-logo.png" style="width: 30px;" class="mr-1"> Testnet</a>
+      <a [href]="env.MEMPOOL_WEBSITE_URL + urlLanguage" ngbDropdownItem class="mainnet"><app-svg-images name="bitcoin" width="22" height="22" viewBox="0 0 65 65" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Mainnet</a>
+      <a [href]="env.MEMPOOL_WEBSITE_URL + urlLanguage + '/signet'" ngbDropdownItem *ngIf="env.SIGNET_ENABLED" class="signet"><app-svg-images name="signet" width="22" height="22" viewBox="0 0 65 65" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Signet</a>
+      <a [href]="env.MEMPOOL_WEBSITE_URL + urlLanguage + '/testnet'" ngbDropdownItem *ngIf="env.TESTNET_ENABLED" class="testnet"><app-svg-images name="testnet" width="22" height="22" viewBox="0 0 65 65" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Testnet</a>
       <h6 class="dropdown-header" i18n="master-page.layer2-networks-header">Layer 2 Networks</h6>
-      <a ngbDropdownItem class="mainnet active" routerLink="/"><img src="/resources/bisq-logo.png" style="width: 30px;" class="mr-1"> Bisq</a>
-      <a [href]="env.LIQUID_WEBSITE_URL + urlLanguage" ngbDropdownItem *ngIf="env.LIQUID_ENABLED" class="liquid"><img src="/resources/liquid-logo.png" style="width: 30px;" class="mr-1"> Liquid</a>
-      <a [href]="env.LIQUID_WEBSITE_URL + urlLanguage + '/testnet'" ngbDropdownItem *ngIf="env.LIQUID_TESTNET_ENABLED" class="liquidtestnet"><img src="/resources/liquidtestnet-logo.png" style="width: 30px;" class="mr-1"> Liquid Testnet</a>
+      <a ngbDropdownItem class="mainnet active" routerLink="/"><app-svg-images name="bisq" width="20" height="20" viewBox="0 0 75 75" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Bisq</a>
+      <a [href]="env.LIQUID_WEBSITE_URL + urlLanguage" ngbDropdownItem *ngIf="env.LIQUID_ENABLED" class="liquid"><app-svg-images name="liquid" width="22" height="22" viewBox="0 0 125 125" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Liquid</a>
+      <a [href]="env.LIQUID_WEBSITE_URL + urlLanguage + '/testnet'" ngbDropdownItem *ngIf="env.LIQUID_TESTNET_ENABLED" class="liquidtestnet"><app-svg-images name="liquidtestnet" width="22" height="22" viewBox="0 0 125 125" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Liquid Testnet</a>
     </div>
   </div>
 

frontend/src/app/components/bisq-master-page/bisq-master-page.component.ts

@@ -1,7 +1,8 @@
 import { Component, OnInit } from '@angular/core';
 import { Env, StateService } from '../../services/state.service';
 import { Observable } from 'rxjs';
-import { LanguageService } from 'src/app/services/language.service';
+import { LanguageService } from '../../services/language.service';
+import { EnterpriseService } from '../../services/enterprise.service';
 
 @Component({
   selector: 'app-bisq-master-page',
@@ -18,6 +19,7 @@ export class BisqMasterPageComponent implements OnInit {
   constructor(
     private stateService: StateService,
     private languageService: LanguageService,
+    private enterpriseService: EnterpriseService,
   ) { }
 
   ngOnInit() {

frontend/src/app/components/block-audit/block-audit.component.ts

@@ -2,11 +2,11 @@ import { Component, OnDestroy, OnInit, ViewChild } from '@angular/core';
 import { ActivatedRoute, ParamMap, Router } from '@angular/router';
 import { Observable } from 'rxjs';
 import { map, share, switchMap, tap } from 'rxjs/operators';
-import { BlockAudit, TransactionStripped } from 'src/app/interfaces/node-api.interface';
-import { ApiService } from 'src/app/services/api.service';
-import { StateService } from 'src/app/services/state.service';
-import { detectWebGL } from 'src/app/shared/graphs.utils';
-import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.pipe';
+import { BlockAudit, TransactionStripped } from '../../interfaces/node-api.interface';
+import { ApiService } from '../../services/api.service';
+import { StateService } from '../../services/state.service';
+import { detectWebGL } from '../../shared/graphs.utils';
+import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe';
 import { BlockOverviewGraphComponent } from '../block-overview-graph/block-overview-graph.component';
 
 @Component({

frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss

@@ -61,7 +61,7 @@
   flex-direction: column;
   @media (min-width: 991px) {
     position: relative;
-    top: -65px;
+    top: -100px;
   }
   @media (min-width: 830px) and (max-width: 991px) {
     position: relative;