machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into simon/add-4y

Browse Source
This commit is contained in:
wiz
2023-03-07 19:00:27 +09:00
committed by GitHub
parent abd74dc87e 6f376f8687
commit 761cd12978
80 changed files with 3930 additions and 2945 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
backend/src/api/bitcoin/bitcoin.routes.ts
View File

@@ -220,18 +220,17 @@ class BitcoinRoutes {
let cpfpInfo;
if (config.DATABASE.ENABLED) {
cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
}
if (cpfpInfo) {
res.json(cpfpInfo);
return;
} else {
res.json({
ancestors: []
});
return;
}
if (cpfpInfo) {
res.json(cpfpInfo);
return;
}
}
res.status(404).send(`Transaction has no CPFP info available.`);
}
private getBackendInfo(req: Request, res: Response) {
@@ -652,7 +651,7 @@ class BitcoinRoutes {
if (result) {
res.json(result);
} else {
res.status(404).send('not found');
res.status(204).send();
}
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);

2
backend/src/api/database-migration.ts
View File

@@ -1037,7 +1037,7 @@ class DatabaseMigration {
await this.$executeQuery('DELETE FROM `pools`');
await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
}
}
private async $convertCompactCpfpTables(): Promise<void> {
try {

2
backend/src/api/disk-cache.ts
View File

@@ -62,7 +62,7 @@ class DiskCache {
}
wipeCache() {
logger.notice(`Wipping nodejs backend cache/cache*.json files`);
logger.notice(`Wiping nodejs backend cache/cache*.json files`);
try {
fs.unlinkSync(DiskCache.FILE_NAME);
} catch (e: any) {

2
backend/src/api/mining/mining-routes.ts
View File

@@ -263,7 +263,7 @@ class MiningRoutes {
const audit = await BlocksAuditsRepository.$getBlockAudit(req.params.hash);
if (!audit) {
res.status(404).send(`This block has not been audited.`);
res.status(204).send(`This block has not been audited.`);
return;
}

32
backend/src/api/mining/mining.ts
View File

@@ -117,7 +117,7 @@ class Mining {
poolsStatistics['lastEstimatedHashrate'] = await bitcoinClient.getNetworkHashPs(totalBlock24h);
} catch (e) {
poolsStatistics['lastEstimatedHashrate'] = 0;
logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate', logger.tags.mining);
}
return poolsStatistics;
@@ -145,7 +145,7 @@ class Mining {
try {
currentEstimatedHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
} catch (e) {
logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate', logger.tags.mining);
}
return {
@@ -208,7 +208,7 @@ class Mining {
const startedAt = new Date().getTime() / 1000;
let timer = new Date().getTime() / 1000;
logger.debug(`Indexing weekly mining pool hashrate`);
logger.debug(`Indexing weekly mining pool hashrate`, logger.tags.mining);
loadingIndicators.setProgress('weekly-hashrate-indexing', 0);
while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) {
@@ -245,7 +245,7 @@ class Mining {
});
}
newlyIndexed += hashrates.length;
newlyIndexed += hashrates.length / Math.max(1, pools.length);
await HashratesRepository.$saveHashrates(hashrates);
hashrates.length = 0;
}
@@ -256,7 +256,7 @@ class Mining {
const weeksPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
const progress = Math.round(totalIndexed / totalWeekIndexed * 10000) / 100;
const formattedDate = new Date(fromTimestamp).toUTCString();
logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`);
logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
timer = new Date().getTime() / 1000;
indexedThisRun = 0;
loadingIndicators.setProgress('weekly-hashrate-indexing', progress, false);
@@ -268,14 +268,14 @@ class Mining {
}
this.lastWeeklyHashrateIndexingDate = new Date().getUTCDate();
if (newlyIndexed > 0) {
logger.notice(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`, logger.tags.mining);
logger.info(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed} weeks`, logger.tags.mining);
} else {
logger.debug(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`, logger.tags.mining);
logger.debug(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed} weeks`, logger.tags.mining);
}
loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
} catch (e) {
loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
logger.err(`Weekly mining pools hashrates indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
logger.err(`Weekly mining pools hashrates indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
throw e;
}
}
@@ -308,7 +308,7 @@ class Mining {
const startedAt = new Date().getTime() / 1000;
let timer = new Date().getTime() / 1000;
logger.debug(`Indexing daily network hashrate`);
logger.debug(`Indexing daily network hashrate`, logger.tags.mining);
loadingIndicators.setProgress('daily-hashrate-indexing', 0);
while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) {
@@ -346,7 +346,7 @@ class Mining {
const daysPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
const progress = Math.round(totalIndexed / totalDayIndexed * 10000) / 100;
const formattedDate = new Date(fromTimestamp).toUTCString();
logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`);
logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
timer = new Date().getTime() / 1000;
indexedThisRun = 0;
loadingIndicators.setProgress('daily-hashrate-indexing', progress);
@@ -373,14 +373,14 @@ class Mining {
this.lastHashrateIndexingDate = new Date().getUTCDate();
if (newlyIndexed > 0) {
logger.notice(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`, logger.tags.mining);
logger.info(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`, logger.tags.mining);
} else {
logger.debug(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`, logger.tags.mining);
}
loadingIndicators.setProgress('daily-hashrate-indexing', 100);
} catch (e) {
loadingIndicators.setProgress('daily-hashrate-indexing', 100);
logger.err(`Daily network hashrate indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
logger.err(`Daily network hashrate indexing failed. Trying again later. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
throw e;
}
}
@@ -446,13 +446,13 @@ class Mining {
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
if (elapsedSeconds > 5) {
const progress = Math.round(totalBlockChecked / blocks.length * 100);
logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`);
logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`, logger.tags.mining);
timer = new Date().getTime() / 1000;
}
}
if (totalIndexed > 0) {
logger.notice(`Indexed ${totalIndexed} difficulty adjustments`, logger.tags.mining);
logger.info(`Indexed ${totalIndexed} difficulty adjustments`, logger.tags.mining);
} else {
logger.debug(`Indexed ${totalIndexed} difficulty adjustments`, logger.tags.mining);
}
@@ -499,7 +499,7 @@ class Mining {
if (blocksWithoutPrices.length > 200000) {
logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
}
logger.debug(logStr);
logger.debug(logStr, logger.tags.mining);
await BlocksRepository.$saveBlockPrices(blocksPrices);
blocksPrices.length = 0;
}
@@ -511,7 +511,7 @@ class Mining {
if (blocksWithoutPrices.length > 200000) {
logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
}
logger.debug(logStr);
logger.debug(logStr, logger.tags.mining);
await BlocksRepository.$saveBlockPrices(blocksPrices);
}
} catch (e) {

9
backend/src/api/websocket-handler.ts
View File

@@ -1,8 +1,8 @@
import logger from '../logger';
import * as WebSocket from 'ws';
import {
BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, MempoolBlockDelta,
OptimizedStatistic, ILoadingIndicators, IConversionRates
BlockExtended, TransactionExtended, WebsocketResponse,
OptimizedStatistic, ILoadingIndicators
} from '../mempool.interfaces';
import blocks from './blocks';
import memPool from './mempool';
@@ -20,6 +20,7 @@ import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository
import Audit from './audit';
import { deepClone } from '../utils/clone';
import priceUpdater from '../tasks/price-updater';
import { ApiPrice } from '../repositories/PricesRepository';
class WebsocketHandler {
private wss: WebSocket.Server | undefined;
@@ -193,7 +194,7 @@ class WebsocketHandler {
});
}
handleNewConversionRates(conversionRates: IConversionRates) {
handleNewConversionRates(conversionRates: ApiPrice) {
if (!this.wss) {
throw new Error('WebSocket.Server is not set');
}
@@ -214,7 +215,7 @@ class WebsocketHandler {
'mempoolInfo': memPool.getMempoolInfo(),
'vBytesPerSecond': memPool.getVBytesPerSecond(),
'blocks': _blocks,
'conversions': priceUpdater.latestPrices,
'conversions': priceUpdater.getLatestPrices(),
'mempool-blocks': mempoolBlocks.getMempoolBlocks(),
'transactions': memPool.getLatestTransactions(),
'backendInfo': backendInfo.getBackendInfo(),

29
backend/src/index.ts
View File

@@ -38,6 +38,8 @@ import forensicsService from './tasks/lightning/forensics.service';
import priceUpdater from './tasks/price-updater';
import chainTips from './api/chain-tips';
import { AxiosError } from 'axios';
import v8 from 'v8';
import { formatBytes, getBytesUnit } from './utils/format';
class Server {
private wss: WebSocket.Server | undefined;
@@ -45,6 +47,11 @@ class Server {
private app: Application;
private currentBackendRetryInterval = 5;
private maxHeapSize: number = 0;
private heapLogInterval: number = 60;
private warnedHeapCritical: boolean = false;
private lastHeapLogTime: number | null = null;
constructor() {
this.app = express();
@@ -137,6 +144,8 @@ class Server {
this.runMainUpdateLoop();
}
setInterval(() => { this.healthCheck(); }, 2500);
if (config.BISQ.ENABLED) {
bisq.startBisqService();
bisq.setPriceCallbackFunction((price) => websocketHandler.setExtraInitProperties('bsq-price', price));
@@ -255,6 +264,26 @@ class Server {
channelsRoutes.initRoutes(this.app);
}
}
healthCheck(): void {
const now = Date.now();
const stats = v8.getHeapStatistics();
this.maxHeapSize = Math.max(stats.used_heap_size, this.maxHeapSize);
const warnThreshold = 0.8 * stats.heap_size_limit;
const byteUnits = getBytesUnit(Math.max(this.maxHeapSize, stats.heap_size_limit));
if (!this.warnedHeapCritical && this.maxHeapSize > warnThreshold) {
this.warnedHeapCritical = true;
logger.warn(`Used ${(this.maxHeapSize / stats.heap_size_limit).toFixed(2)}% of heap limit (${formatBytes(this.maxHeapSize, byteUnits, true)} / ${formatBytes(stats.heap_size_limit, byteUnits)})!`);
}
if (this.lastHeapLogTime === null || (now - this.lastHeapLogTime) > (this.heapLogInterval * 1000)) {
logger.debug(`Memory usage: ${formatBytes(this.maxHeapSize, byteUnits)} / ${formatBytes(stats.heap_size_limit, byteUnits)}`);
this.warnedHeapCritical = false;
this.maxHeapSize = 0;
this.lastHeapLogTime = now;
}
}
}
((): Server => new Server())();

8
backend/src/indexer.ts
View File

@@ -76,13 +76,13 @@ class Indexer {
this.tasksRunning.push(task);
const lastestPriceId = await PricesRepository.$getLatestPriceId();
if (priceUpdater.historyInserted === false || lastestPriceId === null) {
logger.debug(`Blocks prices indexer is waiting for the price updater to complete`);
logger.debug(`Blocks prices indexer is waiting for the price updater to complete`, logger.tags.mining);
setTimeout(() => {
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
this.runSingleTask('blocksPrices');
}, 10000);
} else {
logger.debug(`Blocks prices indexer will run now`);
logger.debug(`Blocks prices indexer will run now`, logger.tags.mining);
await mining.$indexBlockPrices();
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
}
@@ -112,7 +112,7 @@ class Indexer {
this.runIndexer = false;
this.indexerRunning = true;
logger.info(`Running mining indexer`);
logger.debug(`Running mining indexer`);
await this.checkAvailableCoreIndexes();
@@ -122,7 +122,7 @@ class Indexer {
const chainValid = await blocks.$generateBlockDatabase();
if (chainValid === false) {
// Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration
logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`);
logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`, logger.tags.mining);
setTimeout(() => this.reindex(), 10000);
this.indexerRunning = false;
return;

1
backend/src/mempool.interfaces.ts
View File

@@ -293,7 +293,6 @@ interface RequiredParams {
}
export interface ILoadingIndicators { [name: string]: number; }
export interface IConversionRates { [currency: string]: number; }
export interface IBackendInfo {
hostname: string;

18
backend/src/repositories/DifficultyAdjustmentsRepository.ts
View File

@@ -20,9 +20,9 @@ class DifficultyAdjustmentsRepository {
await DB.query(query, params);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Cannot save difficulty adjustment at block ${adjustment.height}, already indexed, ignoring`);
logger.debug(`Cannot save difficulty adjustment at block ${adjustment.height}, already indexed, ignoring`, logger.tags.mining);
} else {
logger.err(`Cannot save difficulty adjustment at block ${adjustment.height}. Reason: ${e instanceof Error ? e.message : e}`);
logger.err(`Cannot save difficulty adjustment at block ${adjustment.height}. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
throw e;
}
}
@@ -54,7 +54,7 @@ class DifficultyAdjustmentsRepository {
const [rows] = await DB.query(query);
return rows as IndexedDifficultyAdjustment[];
} catch (e) {
logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e));
logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e), logger.tags.mining);
throw e;
}
}
@@ -83,7 +83,7 @@ class DifficultyAdjustmentsRepository {
const [rows] = await DB.query(query);
return rows as IndexedDifficultyAdjustment[];
} catch (e) {
logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e));
logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e), logger.tags.mining);
throw e;
}
}
@@ -93,27 +93,27 @@ class DifficultyAdjustmentsRepository {
const [rows]: any[] = await DB.query(`SELECT height FROM difficulty_adjustments`);
return rows.map(block => block.height);
} catch (e: any) {
logger.err(`Cannot get difficulty adjustment block heights. Reason: ${e instanceof Error ? e.message : e}`);
logger.err(`Cannot get difficulty adjustment block heights. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
throw e;
}
}
public async $deleteAdjustementsFromHeight(height: number): Promise<void> {
try {
logger.info(`Delete newer difficulty adjustments from height ${height} from the database`);
logger.info(`Delete newer difficulty adjustments from height ${height} from the database`, logger.tags.mining);
await DB.query(`DELETE FROM difficulty_adjustments WHERE height >= ?`, [height]);
} catch (e: any) {
logger.err(`Cannot delete difficulty adjustments from the database. Reason: ${e instanceof Error ? e.message : e}`);
logger.err(`Cannot delete difficulty adjustments from the database. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
throw e;
}
}
public async $deleteLastAdjustment(): Promise<void> {
try {
logger.info(`Delete last difficulty adjustment from the database`);
logger.info(`Delete last difficulty adjustment from the database`, logger.tags.mining);
await DB.query(`DELETE FROM difficulty_adjustments ORDER BY time LIMIT 1`);
} catch (e: any) {
logger.err(`Cannot delete last difficulty adjustment from the database. Reason: ${e instanceof Error ? e.message : e}`);
logger.err(`Cannot delete last difficulty adjustment from the database. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
throw e;
}
}

22
backend/src/repositories/HashratesRepository.ts
View File

@@ -25,7 +25,7 @@ class HashratesRepository {
try {
await DB.query(query);
} catch (e: any) {
logger.err('Cannot save indexed hashrate into db. Reason: ' + (e instanceof Error ? e.message : e));
logger.err('Cannot save indexed hashrate into db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
throw e;
}
}
@@ -51,7 +51,7 @@ class HashratesRepository {
const [rows]: any[] = await DB.query(query);
return rows;
} catch (e) {
logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
throw e;
}
}
@@ -78,7 +78,7 @@ class HashratesRepository {
const [rows]: any[] = await DB.query(query);
return rows;
} catch (e) {
logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
throw e;
}
}
@@ -93,7 +93,7 @@ class HashratesRepository {
const [rows]: any[] = await DB.query(query);
return rows.map(row => row.timestamp);
} catch (e) {
logger.err('Cannot retreive indexed weekly hashrate timestamps. Reason: ' + (e instanceof Error ? e.message : e));
logger.err('Cannot retreive indexed weekly hashrate timestamps. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
throw e;
}
}
@@ -128,7 +128,7 @@ class HashratesRepository {
const [rows]: any[] = await DB.query(query);
return rows;
} catch (e) {
logger.err('Cannot fetch weekly pools hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
logger.err('Cannot fetch weekly pools hashrate history. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
throw e;
}
}
@@ -158,7 +158,7 @@ class HashratesRepository {
const [rows]: any[] = await DB.query(query, [pool.id]);
boundaries = rows[0];
} catch (e) {
logger.err('Cannot fetch hashrate start/end timestamps for this pool. Reason: ' + (e instanceof Error ? e.message : e));
logger.err('Cannot fetch hashrate start/end timestamps for this pool. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
}
// Get hashrates entries between boundaries
@@ -173,7 +173,7 @@ class HashratesRepository {
const [rows]: any[] = await DB.query(query, [boundaries.firstTimestamp, boundaries.lastTimestamp, pool.id]);
return rows;
} catch (e) {
logger.err('Cannot fetch pool hashrate history for this pool. Reason: ' + (e instanceof Error ? e.message : e));
logger.err('Cannot fetch pool hashrate history for this pool. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
throw e;
}
}
@@ -192,7 +192,7 @@ class HashratesRepository {
}
return rows[0]['number'];
} catch (e) {
logger.err(`Cannot retrieve last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e));
logger.err(`Cannot retrieve last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e), logger.tags.mining);
throw e;
}
}
@@ -201,7 +201,7 @@ class HashratesRepository {
* Delete most recent data points for re-indexing
*/
public async $deleteLastEntries() {
logger.info(`Delete latest hashrates data points from the database`);
logger.info(`Delete latest hashrates data points from the database`, logger.tags.mining);
try {
const [rows]: any[] = await DB.query(`SELECT MAX(hashrate_timestamp) as timestamp FROM hashrates GROUP BY type`);
@@ -212,7 +212,7 @@ class HashratesRepository {
mining.lastHashrateIndexingDate = null;
mining.lastWeeklyHashrateIndexingDate = null;
} catch (e) {
logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e));
logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
}
}
@@ -228,7 +228,7 @@ class HashratesRepository {
mining.lastHashrateIndexingDate = null;
mining.lastWeeklyHashrateIndexingDate = null;
} catch (e) {
logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e));
logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
}
}
}

154
backend/src/repositories/PricesRepository.ts
View File

@@ -1,6 +1,5 @@
import DB from '../database';
import logger from '../logger';
import { IConversionRates } from '../mempool.interfaces';
import priceUpdater from '../tasks/price-updater';
export interface ApiPrice {
@@ -13,6 +12,16 @@ export interface ApiPrice {
AUD: number,
JPY: number,
}
const ApiPriceFields = `
UNIX_TIMESTAMP(time) as time,
USD,
EUR,
GBP,
CAD,
CHF,
AUD,
JPY
`;
export interface ExchangeRates {
USDEUR: number,
@@ -39,7 +48,7 @@ export const MAX_PRICES = {
};
class PricesRepository {
public async $savePrices(time: number, prices: IConversionRates): Promise<void> {
public async $savePrices(time: number, prices: ApiPrice): Promise<void> {
if (prices.USD === -1) {
// Some historical price entries have no USD prices, so we just ignore them to avoid future UX issues
// As of today there are only 4 (on 2013-09-05, 2013-0909, 2013-09-12 and 2013-09-26) so that's fine
@@ -60,77 +69,115 @@ class PricesRepository {
VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ? )`,
[time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY]
);
} catch (e: any) {
} catch (e) {
logger.err(`Cannot save exchange rate into db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getOldestPriceTime(): Promise<number> {
const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != 0 ORDER BY time LIMIT 1`);
const [oldestRow] = await DB.query(`
SELECT UNIX_TIMESTAMP(time) AS time
FROM prices
ORDER BY time
LIMIT 1
`);
return oldestRow[0] ? oldestRow[0].time : 0;
}
public async $getLatestPriceId(): Promise<number | null> {
const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != 0 ORDER BY time DESC LIMIT 1`);
return oldestRow[0] ? oldestRow[0].id : null;
}
public async $getLatestPriceTime(): Promise<number> {
const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != 0 ORDER BY time DESC LIMIT 1`);
return oldestRow[0] ? oldestRow[0].time : 0;
}
public async $getPricesTimes(): Promise<number[]> {
const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != 0 ORDER BY time`);
return times.map(time => time.time);
}
public async $getPricesTimesAndId(): Promise<number[]> {
const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time, id, USD from prices ORDER BY time`);
return times;
}
public async $getLatestConversionRates(): Promise<any> {
const [rates]: any[] = await DB.query(`
SELECT USD, EUR, GBP, CAD, CHF, AUD, JPY
const [oldestRow] = await DB.query(`
SELECT id
FROM prices
ORDER BY time DESC
LIMIT 1`
);
if (!rates || rates.length === 0) {
return oldestRow[0] ? oldestRow[0].id : null;
}
public async $getLatestPriceTime(): Promise<number> {
const [oldestRow] = await DB.query(`
SELECT UNIX_TIMESTAMP(time) AS time
FROM prices
ORDER BY time DESC
LIMIT 1`
);
return oldestRow[0] ? oldestRow[0].time : 0;
}
public async $getPricesTimes(): Promise<number[]> {
const [times] = await DB.query(`
SELECT UNIX_TIMESTAMP(time) AS time
FROM prices
WHERE USD != -1
ORDER BY time
`);
if (!Array.isArray(times)) {
return [];
}
return times.map(time => time.time);
}
public async $getPricesTimesAndId(): Promise<{time: number, id: number, USD: number}[]> {
const [times] = await DB.query(`
SELECT
UNIX_TIMESTAMP(time) AS time,
id,
USD
FROM prices
ORDER BY time
`);
return times as {time: number, id: number, USD: number}[];
}
public async $getLatestConversionRates(): Promise<ApiPrice> {
const [rates] = await DB.query(`
SELECT ${ApiPriceFields}
FROM prices
ORDER BY time DESC
LIMIT 1`
);
if (!Array.isArray(rates) || rates.length === 0) {
return priceUpdater.getEmptyPricesObj();
}
return rates[0];
return rates[0] as ApiPrice;
}
public async $getNearestHistoricalPrice(timestamp: number | undefined): Promise<Conversion | null> {
try {
const [rates]: any[] = await DB.query(`
SELECT *, UNIX_TIMESTAMP(time) AS time
const [rates] = await DB.query(`
SELECT ${ApiPriceFields}
FROM prices
WHERE UNIX_TIMESTAMP(time) < ?
ORDER BY time DESC
LIMIT 1`,
[timestamp]
);
if (!rates) {
if (!Array.isArray(rates)) {
throw Error(`Cannot get single historical price from the database`);
}
// Compute fiat exchange rates
const latestPrice = await this.$getLatestConversionRates();
let latestPrice = rates[0] as ApiPrice;
if (latestPrice.USD === -1) {
latestPrice = priceUpdater.getEmptyPricesObj();
}
const computeFx = (usd: number, other: number): number =>
Math.round(Math.max(other, 0) / Math.max(usd, 1) * 100) / 100;
const exchangeRates: ExchangeRates = {
USDEUR: Math.round(latestPrice.EUR / latestPrice.USD * 100) / 100,
USDGBP: Math.round(latestPrice.GBP / latestPrice.USD * 100) / 100,
USDCAD: Math.round(latestPrice.CAD / latestPrice.USD * 100) / 100,
USDCHF: Math.round(latestPrice.CHF / latestPrice.USD * 100) / 100,
USDAUD: Math.round(latestPrice.AUD / latestPrice.USD * 100) / 100,
USDJPY: Math.round(latestPrice.JPY / latestPrice.USD * 100) / 100,
USDEUR: computeFx(latestPrice.USD, latestPrice.EUR),
USDGBP: computeFx(latestPrice.USD, latestPrice.GBP),
USDCAD: computeFx(latestPrice.USD, latestPrice.CAD),
USDCHF: computeFx(latestPrice.USD, latestPrice.CHF),
USDAUD: computeFx(latestPrice.USD, latestPrice.AUD),
USDJPY: computeFx(latestPrice.USD, latestPrice.JPY),
};
return {
prices: rates,
prices: rates as ApiPrice[],
exchangeRates: exchangeRates
};
} catch (e) {
@@ -141,28 +188,35 @@ class PricesRepository {
public async $getHistoricalPrices(): Promise<Conversion | null> {
try {
const [rates]: any[] = await DB.query(`
SELECT *, UNIX_TIMESTAMP(time) AS time
const [rates] = await DB.query(`
SELECT ${ApiPriceFields}
FROM prices
ORDER BY time DESC
`);
if (!rates) {
if (!Array.isArray(rates)) {
throw Error(`Cannot get average historical price from the database`);
}
// Compute fiat exchange rates
const latestPrice: ApiPrice = rates[0];
let latestPrice = rates[0] as ApiPrice;
if (latestPrice.USD === -1) {
latestPrice = priceUpdater.getEmptyPricesObj();
}
const computeFx = (usd: number, other: number): number =>
Math.round(Math.max(other, 0) / Math.max(usd, 1) * 100) / 100;
const exchangeRates: ExchangeRates = {
USDEUR: Math.round(latestPrice.EUR / latestPrice.USD * 100) / 100,
USDGBP: Math.round(latestPrice.GBP / latestPrice.USD * 100) / 100,
USDCAD: Math.round(latestPrice.CAD / latestPrice.USD * 100) / 100,
USDCHF: Math.round(latestPrice.CHF / latestPrice.USD * 100) / 100,
USDAUD: Math.round(latestPrice.AUD / latestPrice.USD * 100) / 100,
USDJPY: Math.round(latestPrice.JPY / latestPrice.USD * 100) / 100,
USDEUR: computeFx(latestPrice.USD, latestPrice.EUR),
USDGBP: computeFx(latestPrice.USD, latestPrice.GBP),
USDCAD: computeFx(latestPrice.USD, latestPrice.CAD),
USDCHF: computeFx(latestPrice.USD, latestPrice.CHF),
USDAUD: computeFx(latestPrice.USD, latestPrice.AUD),
USDJPY: computeFx(latestPrice.USD, latestPrice.JPY),
};
return {
prices: rates,
prices: rates as ApiPrice[],
exchangeRates: exchangeRates
};
} catch (e) {

2
backend/src/tasks/lightning/sync-tasks/stats-importer.ts
View File

@@ -411,7 +411,7 @@ class LightningStatsImporter {
}
if (totalProcessed > 0) {
logger.notice(`Lightning network stats historical import completed`, logger.tags.ln);
logger.info(`Lightning network stats historical import completed`, logger.tags.ln);
}
} catch (e) {
logger.err(`Lightning network stats historical failed. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.ln);

22
backend/src/tasks/pools-updater.ts
View File

@@ -12,7 +12,7 @@ import * as https from 'https';
*/
class PoolsUpdater {
lastRun: number = 0;
currentSha: string | undefined = undefined;
currentSha: string | null = null;
poolsUrl: string = config.MEMPOOL.POOLS_JSON_URL;
treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL;
@@ -33,7 +33,7 @@ class PoolsUpdater {
try {
const githubSha = await this.fetchPoolsSha(); // Fetch pools-v2.json sha from github
if (githubSha === undefined) {
if (githubSha === null) {
return;
}
@@ -42,12 +42,12 @@ class PoolsUpdater {
}
logger.debug(`pools-v2.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
if (this.currentSha !== undefined && this.currentSha === githubSha) {
if (this.currentSha !== null && this.currentSha === githubSha) {
return;
}
// See backend README for more details about the mining pools update process
if (this.currentSha !== undefined && // If we don't have any mining pool, download it at least once
if (this.currentSha !== null && // If we don't have any mining pool, download it at least once
config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING !== true && // Automatic pools update is disabled
!process.env.npm_config_update_pools // We're not manually updating mining pool
) {
@@ -57,7 +57,7 @@ class PoolsUpdater {
}
const network = config.SOCKS5PROXY.ENABLED ? 'tor' : 'clearnet';
if (this.currentSha === undefined) {
if (this.currentSha === null) {
logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining);
} else {
logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
@@ -82,7 +82,7 @@ class PoolsUpdater {
logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
await DB.query('ROLLBACK;');
}
logger.notice('PoolsUpdater completed');
logger.info('PoolsUpdater completed');
} catch (e) {
this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week
@@ -108,20 +108,20 @@ class PoolsUpdater {
/**
* Fetch our latest pools-v2.json sha from the db
*/
private async getShaFromDb(): Promise<string | undefined> {
private async getShaFromDb(): Promise<string | null> {
try {
const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
return (rows.length > 0 ? rows[0].string : undefined);
return (rows.length > 0 ? rows[0].string : null);
} catch (e) {
logger.err('Cannot fetch pools-v2.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
return undefined;
return null;
}
}
/**
* Fetch our latest pools-v2.json sha from github
*/
private async fetchPoolsSha(): Promise<string | undefined> {
private async fetchPoolsSha(): Promise<string | null> {
const response = await this.query(this.treeUrl);
if (response !== undefined) {
@@ -133,7 +133,7 @@ class PoolsUpdater {
}
logger.err(`Cannot find "pools-v2.json" in git tree (${this.treeUrl})`, logger.tags.mining);
return undefined;
return null;
}
/**

3
backend/src/tasks/price-feeds/bitfinex-api.ts
View File

@@ -8,9 +8,6 @@ class BitfinexApi implements PriceFeed {
public url: string = 'https://api.bitfinex.com/v1/pubticker/BTC';
public urlHist: string = 'https://api-pub.bitfinex.com/v2/candles/trade:{GRANULARITY}:tBTC{CURRENCY}/hist';
constructor() {
}
public async $fetchPrice(currency): Promise<number> {
const response = await query(this.url + currency);
if (response && response['last_price']) {

2
backend/src/tasks/price-feeds/kraken-api.ts
View File

@@ -98,7 +98,7 @@ class KrakenApi implements PriceFeed {
}
if (Object.keys(priceHistory).length > 0) {
logger.notice(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`, logger.tags.mining);
logger.info(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`, logger.tags.mining);
}
}
}

36
backend/src/tasks/price-updater.ts
View File

@@ -2,8 +2,7 @@ import * as fs from 'fs';
import path from 'path';
import config from '../config';
import logger from '../logger';
import { IConversionRates } from '../mempool.interfaces';
import PricesRepository, { MAX_PRICES } from '../repositories/PricesRepository';
import PricesRepository, { ApiPrice, MAX_PRICES } from '../repositories/PricesRepository';
import BitfinexApi from './price-feeds/bitfinex-api';
import BitflyerApi from './price-feeds/bitflyer-api';
import CoinbaseApi from './price-feeds/coinbase-api';
@@ -21,18 +20,18 @@ export interface PriceFeed {
}
export interface PriceHistory {
[timestamp: number]: IConversionRates;
[timestamp: number]: ApiPrice;
}
class PriceUpdater {
public historyInserted = false;
lastRun = 0;
lastHistoricalRun = 0;
running = false;
feeds: PriceFeed[] = [];
currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
latestPrices: IConversionRates;
private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined;
private lastRun = 0;
private lastHistoricalRun = 0;
private running = false;
private feeds: PriceFeed[] = [];
private currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
private latestPrices: ApiPrice;
private ratesChangedCallback: ((rates: ApiPrice) => void) | undefined;
constructor() {
this.latestPrices = this.getEmptyPricesObj();
@@ -44,8 +43,13 @@ class PriceUpdater {
this.feeds.push(new GeminiApi());
}
public getEmptyPricesObj(): IConversionRates {
public getLatestPrices(): ApiPrice {
return this.latestPrices;
}
public getEmptyPricesObj(): ApiPrice {
return {
time: 0,
USD: -1,
EUR: -1,
GBP: -1,
@@ -56,7 +60,7 @@ class PriceUpdater {
};
}
public setRatesChangedCallback(fn: (rates: IConversionRates) => void) {
public setRatesChangedCallback(fn: (rates: ApiPrice) => void): void {
this.ratesChangedCallback = fn;
}
@@ -156,6 +160,10 @@ class PriceUpdater {
}
this.lastRun = new Date().getTime() / 1000;
if (this.latestPrices.USD === -1) {
this.latestPrices = await PricesRepository.$getLatestConversionRates();
}
}
/**
@@ -224,7 +232,7 @@ class PriceUpdater {
// Group them by timestamp and currency, for example
// grouped[123456789]['USD'] = [1, 2, 3, 4];
const grouped: any = {};
const grouped = {};
for (const historicalEntry of historicalPrices) {
for (const time in historicalEntry) {
if (existingPriceTimes.includes(parseInt(time, 10))) {
@@ -249,7 +257,7 @@ class PriceUpdater {
// Average prices and insert everything into the db
let totalInserted = 0;
for (const time in grouped) {
const prices: IConversionRates = this.getEmptyPricesObj();
const prices: ApiPrice = this.getEmptyPricesObj();
for (const currency in grouped[time]) {
if (grouped[time][currency].length === 0) {
continue;

29
backend/src/utils/format.ts Normal file
View File

@@ -0,0 +1,29 @@
const byteUnits = ['B', 'kB', 'MB', 'GB', 'TB'];
export function getBytesUnit(bytes: number): string {
if (isNaN(bytes) || !isFinite(bytes)) {
return 'B';
}
let unitIndex = 0;
while (unitIndex < byteUnits.length && bytes > 1024) {
unitIndex++;
bytes /= 1024;
}
return byteUnits[unitIndex];
}
export function formatBytes(bytes: number, toUnit: string, skipUnit = false): string {
if (isNaN(bytes) || !isFinite(bytes)) {
return `${bytes}`;
}
let unitIndex = 0;
while (unitIndex < byteUnits.length && (toUnit && byteUnits[unitIndex] !== toUnit || (!toUnit && bytes > 1024))) {
unitIndex++;
bytes /= 1024;
}
return `${bytes.toFixed(2)}${skipUnit ? '' : ' ' + byteUnits[unitIndex]}`;
}