Merge branch 'master' into nymkappa/feature/channel-page-map

backend/mempool-config.sample.json

@@ -21,7 +21,9 @@
     "EXTERNAL_RETRY_INTERVAL": 0,
     "USER_AGENT": "mempool",
     "STDOUT_LOG_MIN_PRIORITY": "debug",
-    "AUTOMATIC_BLOCK_REINDEXING": false
+    "AUTOMATIC_BLOCK_REINDEXING": false,
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
+    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master"
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",

backend/src/api/blocks.ts

@@ -22,6 +22,8 @@ import poolsParser from './pools-parser';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
 import mining from './mining/mining';
 import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
+import PricesRepository from '../repositories/PricesRepository';
+import priceUpdater from '../tasks/price-updater';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -457,6 +459,19 @@ class Blocks {
           }
           await blocksRepository.$saveBlockInDatabase(blockExtended);
 
+          const lastestPriceId = await PricesRepository.$getLatestPriceId();
+          if (priceUpdater.historyInserted === true && lastestPriceId !== null) {
+            await blocksRepository.$saveBlockPrices([{
+              height: blockExtended.height,
+              priceId: lastestPriceId,
+            }]);
+          } else {
+            logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`)
+            setTimeout(() => {
+              indexer.runSingleTask('blocksPrices');
+            }, 10000);
+          }
+
           // Save blocks summary for visualization if it's enabled
           if (Common.blocksSummariesIndexingEnabled() === true) {
             await this.$getStrippedBlockTransactions(blockExtended.id, true);

backend/src/api/mining/mining.ts

@@ -473,7 +473,7 @@ class Mining {
 
       for (const block of blocksWithoutPrices) {
         // Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks
-        if (block.height < 68951) {
+        if (['mainnet', 'testnet'].includes(config.MEMPOOL.NETWORK) && block.height < 68951) {
           blocksPrices.push({
             height: block.height,
             priceId: prices[0].id,
@@ -492,11 +492,11 @@ class Mining {
 
         if (blocksPrices.length >= 100000) {
           totalInserted += blocksPrices.length;
+          let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
           if (blocksWithoutPrices.length > 200000) {
-            logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`);
-          } else {
-            logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
+            logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
           }
+          logger.debug(logStr);
           await BlocksRepository.$saveBlockPrices(blocksPrices);
           blocksPrices.length = 0;
         }
@@ -504,11 +504,11 @@ class Mining {
 
       if (blocksPrices.length > 0) {
         totalInserted += blocksPrices.length;
+        let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
         if (blocksWithoutPrices.length > 200000) {
-          logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`);
-        } else {
-          logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
+          logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
         }
+        logger.debug(logStr);
         await BlocksRepository.$saveBlockPrices(blocksPrices);
       }
     } catch (e) {

backend/src/config.ts

@@ -24,6 +24,8 @@ interface IConfig {
     USER_AGENT: string;
     STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
     AUTOMATIC_BLOCK_REINDEXING: boolean;
+    POOLS_JSON_URL: string,
+    POOLS_JSON_TREE_URL: string,
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -136,6 +138,8 @@ const defaults: IConfig = {
     'USER_AGENT': 'mempool',
     'STDOUT_LOG_MIN_PRIORITY': 'debug',
     'AUTOMATIC_BLOCK_REINDEXING': false,
+    'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
+    'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',

backend/src/indexer.ts

@@ -6,13 +6,12 @@ import logger from './logger';
 import HashratesRepository from './repositories/HashratesRepository';
 import bitcoinClient from './api/bitcoin/bitcoin-client';
 import priceUpdater from './tasks/price-updater';
+import PricesRepository from './repositories/PricesRepository';
 
 class Indexer {
   runIndexer = true;
   indexerRunning = false;
-
-  constructor() {
-  }
+  tasksRunning: string[] = [];
 
   public reindex() {
     if (Common.indexingEnabled()) {
@@ -20,6 +19,28 @@ class Indexer {
     }
   }
 
+  public async runSingleTask(task: 'blocksPrices') {
+    if (!Common.indexingEnabled()) {
+      return;
+    }
+
+    if (task === 'blocksPrices' && !this.tasksRunning.includes(task)) {
+      this.tasksRunning.push(task);
+      const lastestPriceId = await PricesRepository.$getLatestPriceId();
+      if (priceUpdater.historyInserted === false || lastestPriceId === null) {
+        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`)
+        setTimeout(() => {
+          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+          this.runSingleTask('blocksPrices');
+        }, 10000);
+      } else {
+        logger.debug(`Blocks prices indexer will run now`)
+        await mining.$indexBlockPrices();
+        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+      }
+    }
+  }
+
   public async $run() {
     if (!Common.indexingEnabled() || this.runIndexer === false ||
       this.indexerRunning === true || mempool.hasPriority()
@@ -50,7 +71,7 @@ class Indexer {
         return;
       }
 
-      await mining.$indexBlockPrices();
+      this.runSingleTask('blocksPrices');
       await mining.$indexDifficultyAdjustments();
       await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient
       await mining.$generateNetworkHashrateHistory();

backend/src/repositories/PricesRepository.ts

@@ -27,6 +27,11 @@ class PricesRepository {
     return oldestRow[0] ? oldestRow[0].time : 0;
   }
 
+  public async $getLatestPriceId(): Promise<number | null> {
+    const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
+    return oldestRow[0] ? oldestRow[0].id : null;
+  }
+
   public async $getLatestPriceTime(): Promise<number> {
     const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
     return oldestRow[0] ? oldestRow[0].time : 0;

backend/src/tasks/pools-updater.ts

@@ -12,14 +12,11 @@ import * as https from 'https';
  */
 class PoolsUpdater {
   lastRun: number = 0;
-  currentSha: any = undefined;
-  poolsUrl: string = 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json';
-  treeUrl: string = 'https://api.github.com/repos/mempool/mining-pools/git/trees/master';
+  currentSha: string | undefined = undefined;
+  poolsUrl: string = config.MEMPOOL.POOLS_JSON_URL;
+  treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL;
 
-  constructor() {
-  }
-
-  public async updatePoolsJson() {
+  public async updatePoolsJson(): Promise<void> {
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
       return;
     }
@@ -77,7 +74,7 @@ class PoolsUpdater {
   /**
    * Fetch our latest pools.json sha from the db
    */
-  private async updateDBSha(githubSha: string) {
+  private async updateDBSha(githubSha: string): Promise<void> {
     this.currentSha = githubSha;
     if (config.DATABASE.ENABLED === true) {
       try {

backend/src/tasks/price-updater.ts

@@ -1,4 +1,5 @@
 import * as fs from 'fs';
+import { Common } from '../api/common';
 import config from '../config';
 import logger from '../logger';
 import PricesRepository from '../repositories/PricesRepository';
@@ -34,10 +35,10 @@ export interface Prices {
 }
 
 class PriceUpdater {
-  historyInserted: boolean = false;
-  lastRun: number = 0;
-  lastHistoricalRun: number = 0;
-  running: boolean = false;
+  public historyInserted = false;
+  lastRun = 0;
+  lastHistoricalRun = 0;
+  running = false;
   feeds: PriceFeed[] = [];
   currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
   latestPrices: Prices;

docker/README.md

@@ -102,7 +102,9 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "PRICE_FEED_UPDATE_INTERVAL": 600,
     "USE_SECOND_NODE_FOR_MINFEE": false,
     "EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"],
-    "STDOUT_LOG_MIN_PRIORITY": "info"
+    "STDOUT_LOG_MIN_PRIORITY": "info",
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
+    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master"
   },
 ```
 
@@ -126,6 +128,8 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_USE_SECOND_NODE_FOR_MINFEE: ""
       MEMPOOL_EXTERNAL_ASSETS: ""
       MEMPOOL_STDOUT_LOG_MIN_PRIORITY: ""
+      MEMPOOL_POOLS_JSON_URL: ""
+      MEMPOOL_POOLS_JSON_TREE_URL: ""
       ...
 ```
 

docker/backend/start.sh

@@ -24,6 +24,8 @@ __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool}
 __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
 __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false}
 __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false}
+__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=false}
+__MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=false}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@@ -114,6 +116,8 @@ sed -i "s!__MEMPOOL_USER_AGENT__!${__MEMPOOL_USER_AGENT__}!g" mempool-config.jso
 sed -i "s/__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__/${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json
+sed -i "s/__MEMPOOL_POOLS_JSON_URL__/${__MEMPOOL_POOLS_JSON_URL__}/g" mempool-config.json
+sed -i "s/__MEMPOOL_POOLS_JSON_TREE_URL__/${__MEMPOOL_POOLS_JSON_TREE_URL__}/g" mempool-config.json
 
 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json

production/install

@@ -34,10 +34,11 @@ esac
 TOR_INSTALL=ON
 CERTBOT_INSTALL=ON
 
-# install 3 network daemons
+# install 4 network daemons
 BITCOIN_INSTALL=ON
 BISQ_INSTALL=ON
 ELEMENTS_INSTALL=ON
+CLN_INSTALL=ON
 
 # install UNFURL
 UNFURL_INSTALL=ON
@@ -191,6 +192,7 @@ case $OS in
         NGINX_ETC_FOLDER=/usr/local/etc/nginx
         NGINX_CONFIGURATION=/usr/local/etc/nginx/nginx.conf
         CERTBOT_PKG=py39-certbot
+        CLN_PKG=c-lightning
     ;;
 
     Debian)
@@ -275,6 +277,12 @@ ELECTRS_LIQUID_DATA=${ELECTRS_DATA_ROOT}/liquid
 ELECTRS_LIQUIDTESTNET_ZPOOL=${ZPOOL}
 ELECTRS_LIQUIDTESTNET_DATA=${ELECTRS_DATA_ROOT}/liquidtestnet
 
+# Core Lightning user/group
+CLN_USER=cln
+CLN_GROUP=cln
+# Core Lightning home folder
+CLN_HOME=/cln
+
 # bisq user/group
 BISQ_USER=bisq
 BISQ_GROUP=bisq
@@ -596,6 +604,10 @@ zfsCreateFilesystems()
         done
     fi
 
+    if [ "${CLN_INSTALL}" = ON ];then
+        zfs create -o "mountpoint=${CLN_HOME}" "${ZPOOL}/cln"
+    fi
+
     if [ "${BISQ_INSTALL}" = ON ];then
         zfs create -o "mountpoint=${BISQ_HOME}" "${ZPOOL}/bisq"
     fi
@@ -675,6 +687,10 @@ ext4CreateDir()
         done
     fi
 
+    if [ "${CLN_INSTALL}" = ON ];then
+        mkdir -p "${CLN_HOME}"
+    fi
+
     if [ "${BISQ_INSTALL}" = ON ];then
         mkdir -p "${BISQ_HOME}"
     fi
@@ -735,6 +751,7 @@ Testnet:Enable Bitcoin Testnet:ON
 Signet:Enable Bitcoin Signet:ON
 Liquid:Enable Elements Liquid:ON
 Liquidtestnet:Enable Elements Liquidtestnet:ON
+CoreLN:Enable Core Lightning:ON
 Bisq:Enable Bisq:ON
 Unfurl:Enable Unfurl:ON
 EOF
@@ -810,6 +827,11 @@ else
     ELEMENTS_INSTALL=OFF
 fi
 
+if grep CoreLN $tempfile >/dev/null 2>&1;then
+    CLN_INSTALL=ON
+else
+    CLN_INSTALL=OFF
+
 if [ "${BITCOIN_MAINNET_ENABLE}" = ON -o "${BITCOIN_TESTNET_ENABLE}" = ON -o "${BITCOIN_SIGNET_ENABLE}" = ON ];then
     BITCOIN_ELECTRS_INSTALL=ON
 else
@@ -1234,6 +1256,33 @@ if [ "${ELEMENTS_ELECTRS_INSTALL}" = ON ];then
     osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_ELECTRS_HOME} && cargo run --release --features liquid --bin electrs -- --network liquid --version" || true
 fi
     
+#####################################
+# Core Lightning for Bitcoin Mainnet #
+#####################################
+
+echo "[*] Installing Core Lightning"
+case $OS in
+    FreeBSD)
+        echo "[*] Creating Core Lightning user"
+        osGroupCreate "${CLN_GROUP}"
+        osUserCreate "${CLN_USER}" "${CLN_HOME}" "${CLN_GROUP}"
+        osSudo "${ROOT_USER}" chsh -s `which zsh` "${CLN_USER}"
+        osSudo "${CLN_USER}" touch "${CLN_HOME}/.zshrc"
+        osSudo "${ROOT_USER}" chown -R "${CLN_USER}:${CLN_GROUP}" "${CLN_HOME}"
+
+        echo "[*] Installing Core Lightning package"
+        osPackageInstall ${CLN_PKG}
+
+        echo "[*] Installing Core Lightning mainnet Cronjob"
+        crontab_cln+='@reboot sleep 30 ; screen -dmS main lightningd --alias `hostname` --bitcoin-datadir /bitcoin\n'
+        crontab_cln+='@reboot sleep 60 ; screen -dmS sig lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network signet\n'
+        crontab_cln+='@reboot sleep 90 ; screen -dmS tes lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network testnet\n'
+        echo "${crontab_cln}" | crontab -u "${CLN_USER}" -
+    ;;
+    Debian)
+    ;;
+esac
+
 #####################
 # Bisq installation #
 #####################