machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into nymkappa/bugfix/fast-forward-re-org

Browse Source
This commit is contained in:
wiz
2022-04-24 05:54:42 +00:00
committed by GitHub
parent 84b72f8b32 e15abe6851
commit 8d279edfe7
42 changed files with 852 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/api/common.ts
View File

@@ -77,7 +77,7 @@ export class Common {
};
}
static sleep(ms: number): Promise<void> {
static sleep$(ms: number): Promise<void> {
return new Promise((resolve) => {
setTimeout(() => {
resolve();

9
backend/src/api/database-migration.ts
View File

@@ -1,8 +1,7 @@
import config from '../config';
import DB from '../database';
import logger from '../logger';
const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 17;
@@ -25,7 +24,7 @@ class DatabaseMigration {
await this.$createMigrationStateTable();
} catch (e) {
logger.err('MIGRATIONS: Unable to create `state` table, aborting in 10 seconds. ' + e);
await sleep(10000);
await Common.sleep$(10000);
process.exit(-1);
}
logger.debug('MIGRATIONS: `state` table initialized.');
@@ -36,7 +35,7 @@ class DatabaseMigration {
databaseSchemaVersion = await this.$getSchemaVersionFromDatabase();
} catch (e) {
logger.err('MIGRATIONS: Unable to get current database migration version, aborting in 10 seconds. ' + e);
await sleep(10000);
await Common.sleep$(10000);
process.exit(-1);
}
@@ -52,7 +51,7 @@ class DatabaseMigration {
await this.$createMissingTablesAndIndexes(databaseSchemaVersion);
} catch (e) {
logger.err('MIGRATIONS: Unable to create required tables, aborting in 10 seconds. ' + e);
await sleep(10000);
await Common.sleep$(10000);
process.exit(-1);
}

1
backend/src/api/liquid/icons.ts
View File

@@ -1,5 +1,4 @@
import * as fs from 'fs';
import config from '../../config';
import logger from '../../logger';
class Icons {

12
backend/src/api/mining.ts
View File

@@ -15,7 +15,7 @@ class Mining {
}
/**
* Get historical block reward and total fee
* Get historical block total fee
*/
public async $getHistoricalBlockFees(interval: string | null = null): Promise<any> {
return await BlocksRepository.$getHistoricalBlockFees(
@@ -34,6 +34,16 @@ class Mining {
);
}
/**
* Get historical block fee rates percentiles
*/
public async $getHistoricalBlockFeeRates(interval: string | null = null): Promise<any> {
return await BlocksRepository.$getHistoricalBlockFeeRates(
this.getTimeRange(interval),
Common.getSqlInterval(interval)
);
}
/**
* Generate high level overview of the pool ranks and general stats
*/

7
backend/src/index.ts
View File

@@ -85,16 +85,16 @@ class Server {
this.setUpWebsocketHandling();
await syncAssets.syncAssets();
await syncAssets.syncAssets$();
diskCache.loadMempoolCache();
if (config.DATABASE.ENABLED) {
await DB.checkDbConnection();
try {
if (process.env.npm_config_reindex != undefined) { // Re-index requests
if (process.env.npm_config_reindex !== undefined) { // Re-index requests
const tables = process.env.npm_config_reindex.split(',');
logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds (using '--reindex')`);
await Common.sleep(5000);
await Common.sleep$(5000);
await databaseMigration.$truncateIndexedData(tables);
}
await databaseMigration.$initializeOrMigrateDatabase();
@@ -322,6 +322,7 @@ class Server {
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/reward-stats/:blockCount', routes.$getRewardStats)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fees/:interval', routes.$getHistoricalBlockFees)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', routes.$getHistoricalBlockRewards)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', routes.$getHistoricalBlockFeeRates)
;
}

39
backend/src/repositories/BlocksRepository.ts
View File

@@ -416,7 +416,9 @@ class BlocksRepository {
*/
public async $getHistoricalBlockFees(div: number, interval: string | null): Promise<any> {
try {
let query = `SELECT CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
let query = `SELECT
CAST(AVG(height) as INT) as avg_height,
CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
CAST(AVG(fees) as INT) as avg_fees
FROM blocks`;
@@ -439,7 +441,9 @@ class BlocksRepository {
*/
public async $getHistoricalBlockRewards(div: number, interval: string | null): Promise<any> {
try {
let query = `SELECT CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
let query = `SELECT
CAST(AVG(height) as INT) as avg_height,
CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
CAST(AVG(reward) as INT) as avg_rewards
FROM blocks`;
@@ -456,6 +460,37 @@ class BlocksRepository {
throw e;
}
}
/**
* Get the historical averaged block fee rate percentiles
*/
public async $getHistoricalBlockFeeRates(div: number, interval: string | null): Promise<any> {
try {
let query = `SELECT
CAST(AVG(height) as INT) as avg_height,
CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
CAST(AVG(JSON_EXTRACT(fee_span, '$[0]')) as INT) as avg_fee_0,
CAST(AVG(JSON_EXTRACT(fee_span, '$[1]')) as INT) as avg_fee_10,
CAST(AVG(JSON_EXTRACT(fee_span, '$[2]')) as INT) as avg_fee_25,
CAST(AVG(JSON_EXTRACT(fee_span, '$[3]')) as INT) as avg_fee_50,
CAST(AVG(JSON_EXTRACT(fee_span, '$[4]')) as INT) as avg_fee_75,
CAST(AVG(JSON_EXTRACT(fee_span, '$[5]')) as INT) as avg_fee_90,
CAST(AVG(JSON_EXTRACT(fee_span, '$[6]')) as INT) as avg_fee_100
FROM blocks`;
if (interval !== null) {
query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
}
query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
const [rows]: any = await DB.query(query);
return rows;
} catch (e) {
logger.err('Cannot generate block fee rates history. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
}
export default new BlocksRepository();

2
backend/src/repositories/HashratesRepository.ts
View File

@@ -102,7 +102,7 @@ class HashratesRepository {
/**
* Returns a pool hashrate history
*/
public async $getPoolWeeklyHashrate(slug: string): Promise<any[]> {
public async $getPoolWeeklyHashrate(slug: string): Promise<any[]> {
const pool = await PoolsRepository.$getPool(slug);
if (!pool) {
throw new Error(`This mining pool does not exist`);

48
backend/src/routes.ts
View File

@@ -575,8 +575,10 @@ class Routes {
public async $getPools(interval: string, req: Request, res: Response) {
try {
const stats = await miningStats.$getPoolsStats(interval);
const blockCount = await BlocksRepository.$blockCount(null, null);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.header('X-total-count', blockCount.toString());
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(stats);
} catch (e) {
@@ -587,14 +589,12 @@ class Routes {
public async $getPoolsHistoricalHashrate(req: Request, res: Response) {
try {
const hashrates = await HashratesRepository.$getPoolsWeeklyHashrate(req.params.interval ?? null);
const oldestIndexedBlockTimestamp = await BlocksRepository.$oldestBlockTimestamp();
const blockCount = await BlocksRepository.$blockCount(null, null);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.header('X-total-count', blockCount.toString());
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
res.json({
oldestIndexedBlockTimestamp: oldestIndexedBlockTimestamp,
hashrates: hashrates,
});
res.json(hashrates);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
@@ -603,14 +603,12 @@ class Routes {
public async $getPoolHistoricalHashrate(req: Request, res: Response) {
try {
const hashrates = await HashratesRepository.$getPoolWeeklyHashrate(req.params.slug);
const oldestIndexedBlockTimestamp = await BlocksRepository.$oldestBlockTimestamp();
const blockCount = await BlocksRepository.$blockCount(null, null);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.header('X-total-count', blockCount.toString());
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
res.json({
oldestIndexedBlockTimestamp: oldestIndexedBlockTimestamp,
hashrates: hashrates,
});
res.json(hashrates);
} catch (e) {
if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
res.status(404).send(e.message);
@@ -624,12 +622,12 @@ class Routes {
try {
const hashrates = await HashratesRepository.$getNetworkDailyHashrate(req.params.interval ?? null);
const difficulty = await BlocksRepository.$getBlocksDifficulty(req.params.interval ?? null);
const oldestIndexedBlockTimestamp = await BlocksRepository.$oldestBlockTimestamp();
const blockCount = await BlocksRepository.$blockCount(null, null);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.header('X-total-count', blockCount.toString());
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
res.json({
oldestIndexedBlockTimestamp: oldestIndexedBlockTimestamp,
hashrates: hashrates,
difficulty: difficulty,
});
@@ -641,14 +639,12 @@ class Routes {
public async $getHistoricalBlockFees(req: Request, res: Response) {
try {
const blockFees = await mining.$getHistoricalBlockFees(req.params.interval ?? null);
const oldestIndexedBlockTimestamp = await BlocksRepository.$oldestBlockTimestamp();
const blockCount = await BlocksRepository.$blockCount(null, null);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.header('X-total-count', blockCount.toString());
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
res.json({
oldestIndexedBlockTimestamp: oldestIndexedBlockTimestamp,
blockFees: blockFees,
});
res.json(blockFees);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
@@ -657,13 +653,27 @@ class Routes {
public async $getHistoricalBlockRewards(req: Request, res: Response) {
try {
const blockRewards = await mining.$getHistoricalBlockRewards(req.params.interval ?? null);
const blockCount = await BlocksRepository.$blockCount(null, null);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.header('X-total-count', blockCount.toString());
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
res.json(blockRewards);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
public async $getHistoricalBlockFeeRates(req: Request, res: Response) {
try {
const blockFeeRates = await mining.$getHistoricalBlockFeeRates(req.params.interval ?? null);
const oldestIndexedBlockTimestamp = await BlocksRepository.$oldestBlockTimestamp();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
res.json({
oldestIndexedBlockTimestamp: oldestIndexedBlockTimestamp,
blockRewards: blockRewards,
blockFeeRates: blockFeeRates,
});
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
@@ -970,7 +980,7 @@ class Routes {
public async $getRewardStats(req: Request, res: Response) {
try {
const response = await mining.$getRewardStats(parseInt(req.params.blockCount))
const response = await mining.$getRewardStats(parseInt(req.params.blockCount, 10));
res.json(response);
} catch (e) {
res.status(500).end();

96
backend/src/sync-assets.ts
View File

@@ -9,55 +9,69 @@ const PATH = './';
class SyncAssets {
constructor() { }
public async syncAssets() {
public async syncAssets$() {
for (const url of config.MEMPOOL.EXTERNAL_ASSETS) {
await this.downloadFile(url);
try {
await this.downloadFile$(url);
} catch (e) {
throw new Error(`Failed to download external asset. ` + (e instanceof Error ? e.message : e));
}
}
}
private async downloadFile(url: string) {
const fileName = url.split('/').slice(-1)[0];
private async downloadFile$(url: string) {
return new Promise((resolve, reject) => {
const fileName = url.split('/').slice(-1)[0];
try {
if (config.SOCKS5PROXY.ENABLED) {
let socksOptions: any = {
agentOptions: {
keepAlive: true,
},
host: config.SOCKS5PROXY.HOST,
port: config.SOCKS5PROXY.PORT
};
try {
if (config.SOCKS5PROXY.ENABLED) {
const socksOptions: any = {
agentOptions: {
keepAlive: true,
},
host: config.SOCKS5PROXY.HOST,
port: config.SOCKS5PROXY.PORT
};
if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
socksOptions.username = config.SOCKS5PROXY.USERNAME;
socksOptions.password = config.SOCKS5PROXY.PASSWORD;
if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
socksOptions.username = config.SOCKS5PROXY.USERNAME;
socksOptions.password = config.SOCKS5PROXY.PASSWORD;
}
const agent = new SocksProxyAgent(socksOptions);
logger.info(`Downloading external asset ${fileName} over the Tor network...`);
return axios.get(url, {
httpAgent: agent,
httpsAgent: agent,
responseType: 'stream',
timeout: 30000
}).then(function (response) {
const writer = fs.createWriteStream(PATH + fileName);
writer.on('finish', () => {
logger.info(`External asset ${fileName} saved to ${PATH + fileName}`);
resolve(0);
});
response.data.pipe(writer);
});
} else {
logger.info(`Downloading external asset ${fileName} over clearnet...`);
return axios.get(url, {
responseType: 'stream',
timeout: 30000
}).then(function (response) {
const writer = fs.createWriteStream(PATH + fileName);
writer.on('finish', () => {
logger.info(`External asset ${fileName} saved to ${PATH + fileName}`);
resolve(0);
});
response.data.pipe(writer);
});
}
const agent = new SocksProxyAgent(socksOptions);
logger.info(`Downloading external asset ${fileName} over the Tor network...`);
await axios.get(url, {
httpAgent: agent,
httpsAgent: agent,
responseType: 'stream',
timeout: 30000
}).then(function (response) {
response.data.pipe(fs.createWriteStream(PATH + fileName));
logger.info(`External asset ${fileName} saved to ${PATH + fileName}`);
});
} else {
logger.info(`Downloading external asset ${fileName} over clearnet...`);
await axios.get(url, {
responseType: 'stream',
timeout: 30000
}).then(function (response) {
response.data.pipe(fs.createWriteStream(PATH + fileName));
logger.info(`External asset ${fileName} saved to ${PATH + fileName}`);
});
} catch (e: any) {
reject(e);
}
} catch (e: any) {
throw new Error(`Failed to download external asset. ` + e);
}
});
}
}