Merge branch 'master' into fix_electrum_api

backend/src/api/database-migration.ts

@@ -4,7 +4,7 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 22;
+  private static currentVersion = 23;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -231,6 +231,18 @@ class DatabaseMigration {
         await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`');
         await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments'));
       }
+
+      if (databaseSchemaVersion < 23) {
+        await this.$executeQuery('TRUNCATE `prices`');
+        await this.$executeQuery('ALTER TABLE `prices` DROP `avg_prices`');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `USD` float DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `EUR` float DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `GBP` float DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `CAD` float DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `CHF` float DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `AUD` float DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `JPY` float DEFAULT "0"');
+      }
     } catch (e) {
       throw e;
     }

backend/src/api/mining.ts

@@ -9,6 +9,7 @@ import loadingIndicators from './loading-indicators';
 import { escape } from 'mysql2';
 import indexer from '../indexer';
 import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
+import config from '../config';
 
 class Mining {
   constructor() {
@@ -304,7 +305,7 @@ class Mining {
       while (toTimestamp > genesisTimestamp) {
         const fromTimestamp = toTimestamp - 86400000;
 
-        // Skip already indexed weeks
+        // Skip already indexed days
         if (indexedTimestamp.includes(toTimestamp / 1000)) {
           toTimestamp -= 86400000;
           ++totalIndexed;
@@ -315,7 +316,7 @@ class Mining {
         // we are currently indexing has complete data)
         const blockStatsPreviousDay: any = await BlocksRepository.$blockCountBetweenTimestamp(
           null, (fromTimestamp - 86400000) / 1000, (toTimestamp - 86400000) / 1000);
-        if (blockStatsPreviousDay.blockCount === 0) { // We are done indexing
+        if (blockStatsPreviousDay.blockCount === 0 && config.MEMPOOL.NETWORK === 'mainnet') { // We are done indexing
           break;
         }
 
@@ -359,9 +360,10 @@ class Mining {
       // Add genesis block manually
       if (toTimestamp <= genesisTimestamp && !indexedTimestamp.includes(genesisTimestamp)) {
         hashrates.push({
-          hashrateTimestamp: genesisTimestamp,
+          hashrateTimestamp: genesisTimestamp / 1000,
           avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1),
-          poolId: null,
+          poolId: 0,
+          share: 1,
           type: 'daily',
         });
       }
@@ -396,6 +398,15 @@ class Mining {
     let currentDifficulty = 0;
     let totalIndexed = 0;
 
+    if (indexedHeights[0] === false) {
+      await DifficultyAdjustmentsRepository.$saveAdjustments({
+        time: 1231006505,
+        height: 0,
+        difficulty: 1.0,
+        adjustment: 0.0,
+      });
+    }
+
     for (const block of blocks) {
       if (block.difficulty !== currentDifficulty) {
         if (block.height === 0 || indexedHeights[block.height] === true) { // Already indexed

backend/src/index.ts

@@ -285,6 +285,7 @@ class Server {
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug', routes.$getPool)
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/pools/:interval', routes.$getPoolsHistoricalHashrate)
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/:interval', routes.$getHistoricalHashrate)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments', routes.$getDifficultyAdjustments)
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/reward-stats/:blockCount', routes.$getRewardStats)
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fees/:interval', routes.$getHistoricalBlockFees)
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', routes.$getHistoricalBlockRewards)

backend/src/repositories/BlocksRepository.ts

@@ -436,7 +436,7 @@ class BlocksRepository {
         }
 
         if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) {
-          logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}, re-indexing newer blocks and hashrates`);
+          logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}`);
           await this.$deleteBlocksFrom(blocks[idx - 1].height);
           await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height);
           await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);

backend/src/repositories/DifficultyAdjustmentsRepository.ts

@@ -1,4 +1,5 @@
 import { Common } from '../api/common';
+import config from '../config';
 import DB from '../database';
 import logger from '../logger';
 import { IndexedDifficultyAdjustment } from '../mempool.interfaces';
@@ -31,13 +32,19 @@ class DifficultyAdjustmentsRepository {
   public async $getAdjustments(interval: string | null, descOrder: boolean = false): Promise<IndexedDifficultyAdjustment[]> {
     interval = Common.getSqlInterval(interval);
 
-    let query = `SELECT UNIX_TIMESTAMP(time) as time, height, difficulty, adjustment
+    let query = `SELECT 
+      CAST(AVG(UNIX_TIMESTAMP(time)) as INT) as time,
+      CAST(AVG(height) AS INT) as height,
+      CAST(AVG(difficulty) as DOUBLE) as difficulty,
+      CAST(AVG(adjustment) as DOUBLE) as adjustment
       FROM difficulty_adjustments`;
 
     if (interval) {
       query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
     }
 
+    query += ` GROUP BY UNIX_TIMESTAMP(time) DIV ${86400}`;
+
     if (descOrder === true) {
       query += ` ORDER BY time DESC`;
     } else {

backend/src/repositories/HashratesRepository.ts

@@ -1,5 +1,6 @@
 import { escape } from 'mysql2';
 import { Common } from '../api/common';
+import config from '../config';
 import DB from '../database';
 import logger from '../logger';
 import PoolsRepository from './PoolsRepository';
@@ -32,7 +33,9 @@ class HashratesRepository {
   public async $getNetworkDailyHashrate(interval: string | null): Promise<any[]> {
     interval = Common.getSqlInterval(interval);
 
-    let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate
+    let query = `SELECT
+      CAST(AVG(UNIX_TIMESTAMP(hashrate_timestamp)) as INT) as timestamp,
+      CAST(AVG(avg_hashrate) as DOUBLE) as avgHashrate
       FROM hashrates`;
 
     if (interval) {
@@ -42,6 +45,7 @@ class HashratesRepository {
       query += ` WHERE hashrates.type = 'daily'`;
     }
 
+    query += ` GROUP BY UNIX_TIMESTAMP(hashrate_timestamp) DIV ${86400}`;
     query += ` ORDER by hashrate_timestamp`;
 
     try {
@@ -75,6 +79,9 @@ class HashratesRepository {
     interval = Common.getSqlInterval(interval);
 
     const topPoolsId = (await PoolsRepository.$getPoolsInfo('1w')).map((pool) => pool.poolId);
+    if (topPoolsId.length === 0) {
+      return [];
+    }
 
     let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate, share, pools.name as poolName
       FROM hashrates

backend/src/repositories/PricesRepository.ts

@@ -5,7 +5,11 @@ import { Prices } from '../tasks/price-updater';
 class PricesRepository {
   public async $savePrices(time: number, prices: Prices): Promise<void> {
     try {
-      await DB.query(`INSERT INTO prices(time, avg_prices) VALUE (FROM_UNIXTIME(?), ?)`, [time, JSON.stringify(prices)]);
+      await DB.query(`
+        INSERT INTO prices(time,             USD, EUR, GBP, CAD, CHF, AUD, JPY)
+        VALUE             (FROM_UNIXTIME(?), ?,   ?,   ?,   ?,   ?,   ?,   ?  )`,
+        [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY]
+      );
     } catch (e: any) {
       logger.err(`Cannot save exchange rate into db. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;

backend/src/tasks/price-feeds/kraken-api.ts

@@ -87,7 +87,7 @@ class KrakenApi implements PriceFeed {
     }
 
     if (Object.keys(priceHistory).length > 0) {
-      logger.info(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`);
+      logger.notice(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`);
     }
   }
 }

backend/src/tasks/price-updater.ts

@@ -176,7 +176,7 @@ class PriceUpdater {
       ++insertedCount;
     }
     if (insertedCount > 0) {
-      logger.info(`Inserted ${insertedCount} MtGox USD weekly price history into db`);
+      logger.notice(`Inserted ${insertedCount} MtGox USD weekly price history into db`);
     }
 
     // Insert Kraken weekly prices
@@ -205,23 +205,23 @@ class PriceUpdater {
       try {
         historicalPrices.push(await feed.$fetchRecentHourlyPrice(this.currencies));
       } catch (e) {
-        logger.info(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`);
+        logger.err(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`);
       }
     }
 
     // Group them by timestamp and currency, for example
     // grouped[123456789]['USD'] = [1, 2, 3, 4];
-    let grouped: Object = {};
+    const grouped: Object = {};
     for (const historicalEntry of historicalPrices) {
       for (const time in historicalEntry) {
         if (existingPriceTimes.includes(parseInt(time, 10))) {
           continue;
         }
 
-        if (grouped[time] == undefined) {
+        if (grouped[time] === undefined) {
           grouped[time] = {
             USD: [], EUR: [], GBP: [], CAD: [], CHF: [], AUD: [], JPY: []
-          }
+          };
         }
 
         for (const currency of this.currencies) {
@@ -238,13 +238,20 @@ class PriceUpdater {
     for (const time in grouped) {
       const prices: Prices = this.getEmptyPricesObj();
       for (const currency in grouped[time]) {
-        prices[currency] = Math.round((grouped[time][currency].reduce((partialSum, a) => partialSum + a, 0)) / grouped[time][currency].length);
+        if (grouped[time][currency].length === 0) {
+          continue;
+        }
+        prices[currency] = Math.round((grouped[time][currency].reduce(
+          (partialSum, a) => partialSum + a, 0)
+        ) / grouped[time][currency].length);
       }
       await PricesRepository.$savePrices(parseInt(time, 10), prices);
       ++totalInserted;
     }
 
-    logger.info(`Inserted ${totalInserted} hourly historical prices into the db`);
+    if (totalInserted > 0) {
+      logger.notice(`Inserted ${totalInserted} hourly historical prices into the db`);
+    }
   }
 }
 

backend/src/utils/axios-query.ts

@@ -50,10 +50,14 @@ export async function query(path): Promise<object | undefined> {
      }
      return data.data;
    } catch (e) {
-     logger.err(`Could not connect to ${path}. Reason: ` + (e instanceof Error ? e.message : e));
+     logger.warn(`Could not connect to ${path} (Attempt ${retry + 1}/${config.MEMPOOL.EXTERNAL_MAX_RETRY}). Reason: ` + (e instanceof Error ? e.message : e));
      retry++;
    }
-   await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
+   if (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+     await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
+   }
  }
+
+ logger.err(`Could not connect to ${path}. All ${config.MEMPOOL.EXTERNAL_MAX_RETRY} attempts failed`);
  return undefined;
 }

frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts

@@ -11,6 +11,7 @@ import { StorageService } from 'src/app/services/storage.service';
 import { MiningService } from 'src/app/services/mining.service';
 import { download } from 'src/app/shared/graphs.utils';
 import { ActivatedRoute } from '@angular/router';
+import { StateService } from 'src/app/services/state.service';
 
 @Component({
   selector: 'app-hashrate-chart',
@@ -47,7 +48,7 @@ export class HashrateChartComponent implements OnInit {
   formatNumber = formatNumber;
   timespan = '';
   chartInstance: any = undefined;
-  maResolution: number =  30;
+  network = '';
 
   constructor(
     @Inject(LOCALE_ID) public locale: string,
@@ -57,10 +58,13 @@ export class HashrateChartComponent implements OnInit {
     private storageService: StorageService,
     private miningService: MiningService,
     private route: ActivatedRoute,
+    private stateService: StateService
   ) {
   }
 
   ngOnInit(): void {
+    this.stateService.networkChanged$.subscribe((network) => this.network = network);
+
     let firstRun = true;
 
     if (this.widget) {
@@ -124,17 +128,14 @@ export class HashrateChartComponent implements OnInit {
                   ++diffIndex;
                 }
 
-                this.maResolution = 30;
+                let maResolution = 15;
-                if (["3m", "6m"].includes(this.timespan)) {
-                  this.maResolution = 7;
-                }
                 const hashrateMa = [];
-                for (let i = this.maResolution - 1; i < data.hashrates.length; ++i) {
+                for (let i = maResolution - 1; i < data.hashrates.length; ++i) {
                   let avg = 0;
-                  for (let y = this.maResolution - 1; y >= 0; --y) {
+                  for (let y = maResolution - 1; y >= 0; --y) {
                     avg += data.hashrates[i - y].avgHashrate;
                   }
-                  avg /= this.maResolution;
+                  avg /= maResolution;
                   hashrateMa.push([data.hashrates[i].timestamp * 1000, avg]);
                 }
 
@@ -276,17 +277,17 @@ export class HashrateChartComponent implements OnInit {
             },
           },
           {
-            name: $localize`::Difficulty`,
+            name: $localize`:@@25148835d92465353fc5fe8897c27d5369978e5a:Difficulty`,
             inactiveColor: 'rgb(110, 112, 121)',
-            textStyle: {  
+            textStyle: {
               color: 'white',
             },
             icon: 'roundRect',
           },
           {
-            name: $localize`Hashrate` + ` (MA${this.maResolution})`,
+            name: $localize`Hashrate (MA)`,
             inactiveColor: 'rgb(110, 112, 121)',
-            textStyle: {  
+            textStyle: {
               color: 'white',
             },
             icon: 'roundRect',
@@ -295,11 +296,18 @@ export class HashrateChartComponent implements OnInit {
             },
           },
         ],
+        selected: JSON.parse(this.storageService.getValue('hashrate_difficulty_legend')) ?? {
+          '$localize`:@@79a9dc5b1caca3cbeb1733a19515edacc5fc7920:Hashrate`': true,
+          '$localize`::Difficulty`': this.network === '',
+          '$localize`Hashrate (MA)`': true,
+        },
       },
       yAxis: data.hashrates.length === 0 ? undefined : [
         {
           min: (value) => {
-            return value.min * 0.9;
+            const selectedPowerOfTen: any = selectPowerOfTen(value.min);
+            const newMin = Math.floor(value.min / selectedPowerOfTen.divider / 10);
+            return newMin * selectedPowerOfTen.divider * 10;
           },
           type: 'value',
           axisLabel: {
@@ -363,7 +371,7 @@ export class HashrateChartComponent implements OnInit {
         },
         {
           zlevel: 2,
-          name: $localize`Hashrate` + ` (MA${this.maResolution})`,
+          name: $localize`Hashrate (MA)`,
           showSymbol: false,
           symbol: 'none',
           data: data.hashrateMa,
@@ -404,6 +412,10 @@ export class HashrateChartComponent implements OnInit {
 
   onChartInit(ec) {
     this.chartInstance = ec;
+
+    this.chartInstance.on('legendselectchanged', (e) => {
+      this.storageService.setValue('hashrate_difficulty_legend', JSON.stringify(e.selected));
+    });
   }
 
   isMobile() {

frontend/src/app/components/mining-dashboard/mining-dashboard.component.html

@@ -40,7 +40,7 @@
       <div class="card">
         <div class="card-body pl-lg-3 pr-lg-3 pl-2 pr-2">
           <app-hashrate-chart [widget]="true"></app-hashrate-chart>
-          <div class="mt-1"><a [routerLink]="['/graphs/mining/hashrate-difficulty' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div>
+          <div class="mt-1"><a [routerLink]="['/graphs/mining/hashrate-difficulty' | relativeUrl]" fragment="1y" i18n="dashboard.view-more">View more &raquo;</a></div>
         </div>
       </div>
     </div>

frontend/src/app/docs/api-docs/api-docs-data.ts

@@ -6017,6 +6017,20 @@ export const faqData = [
     fragment: "what-are-mining-pools",
     title: "What are mining pools?",
   },
+  {
+    type: "endpoint",
+    category: "basics",
+    showConditions: bitcoinNetworks,
+    fragment: "what-are-vb-wu",
+    title: "What are virtual bytes (vB) and weight units (WU)?",
+  },
+  {
+    type: "endpoint",
+    category: "basics",
+    showConditions: bitcoinNetworks,
+    fragment: "what-is-svb",
+    title: "What is sat/vB?",
+  },
   {
     type: "endpoint",
     category: "basics",

frontend/src/app/docs/api-docs/api-docs.component.html

@@ -134,6 +134,19 @@
   Mining pools are groups of miners that combine their computational power in order to increase the probability of finding new blocks.
 </ng-template>
 
+<ng-template type="what-are-vb-wu">
+  <p>Virtual bytes (vB) and weight units (WU) are used to measure the size of transactions and blocks on the Bitcoin network.</p>
+  <p>A Bitcoin transaction's size in the blockchain is <i>not</i> determined how much bitcoin it transfers—instead, a transaction's size is determined by technical factors such as how many inputs and outputs it has, how many signatures it has, and the format it uses (legacy, SegWit, etc). Since space in the Bitcoin blockchain is limited, bigger transactions pay more in mining fees than smaller transactions.</p>
+  <p>Block sizes are limited to 4,000,000 WU (or 1,000,000 vB since 1 vB = 4 WU).</p>
+  <p>Transaction sizes and block sizes used to be measured in plain bytes, but virtual bytes and weight units were devised to maintain backward compatibility after the SegWit upgrade in 2017. See <a href="https://programmingbitcoin.com/understanding-segwit-block-size" target="_blank">this post</a> for more details.</p>
+</ng-template>
+
+<ng-template type="what-is-svb">
+  <p>The priority of a pending Bitcoin transaction is determined by its feerate. Feerates are measured in sat/vB.</p>
+  <p>Using a higher sat/vB feerate for a Bitcoin transaction will generally result in quicker confirmation than using a lower feerate. But feerates change all the time, so it's important to check suggested feerates right before making a transaction to <a [routerLink]="['/docs/faq' | relativeUrl]" fragment="why-is-transaction-stuck-in-mempool">avoid it from getting stuck</a>.</p>
+  <p>There are feerate estimates on the top of <a [routerLink]="['/' | relativeUrl]">the main dashboard</a> you can use as a guide. See <a [routerLink]="['/docs/faq' | relativeUrl]" fragment="looking-up-fee-estimates">this FAQ</a> for more on picking the right feerate.</p>
+</ng-template>
+
 <ng-template type="what-is-full-mempool">
   <p>When a Bitcoin transaction is made, it is stored in a Bitcoin node's mempool before it is confirmed into a block. When the rate of incoming transactions exceeds the rate transactions are confirmed, the mempool grows in size.</p><p>The default maximum size of a Bitcoin node's mempool is 300MB, so when there are 300MB of transactions in the mempool, we say it's "full".</p>
 </ng-template>