machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into simon/database-migration-feature

Browse Source 
# Conflicts:
#	backend/src/index.ts
This commit is contained in:
softsimon
2021-12-24 00:26:33 +04:00
parent 2281116504 5b22e2a000
commit 91e30fbc3c
54 changed files with 2668 additions and 1963 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
backend/.gitignore vendored
View File

@@ -1,7 +1,10 @@
# See http://help.github.com/ignore-files/ for more about ignoring files.
# production config
mempool-config.json
# production config and external assets
*.json
!mempool-config.sample.json
icons.json
# compiled output
/dist

3
backend/mempool-config.sample.json
View File

@@ -13,7 +13,8 @@
"INITIAL_BLOCKS_AMOUNT": 8,
"MEMPOOL_BLOCKS_AMOUNT": 8,
"PRICE_FEED_UPDATE_INTERVAL": 3600,
"USE_SECOND_NODE_FOR_MINFEE": false
"USE_SECOND_NODE_FOR_MINFEE": false,
"EXTERNAL_ASSETS": []
},
"CORE_RPC": {
"HOST": "127.0.0.1",

2
backend/src/api/bitcoin/bitcoin-api.interface.ts
View File

@@ -108,7 +108,7 @@ export namespace IBitcoinApi {
scriptPubKey: string; // (string) The hex-encoded scriptPubKey generated by the address
isscript: boolean; // (boolean) If the key is a script
iswitness: boolean; // (boolean) If the address is a witness
witness_version?: boolean; // (numeric, optional) The version number of the witness program
witness_version?: number; // (numeric, optional) The version number of the witness program
witness_program: string; // (string, optional) The hex value of the witness program
confidential_key?: string; // (string) Elements only
unconfidential?: string; // (string) Elements only

39
backend/src/api/liquid/icons.ts Normal file
View File

@@ -0,0 +1,39 @@
import * as fs from 'fs';
import config from '../../config';
import logger from '../../logger';
class Icons {
private static FILE_NAME = './icons.json';
private iconIds: string[] = [];
private icons: { [assetId: string]: string; } = {};
constructor() {}
public loadIcons() {
if (!fs.existsSync(Icons.FILE_NAME)) {
logger.warn(`${Icons.FILE_NAME} does not exist. No Liquid icons loaded.`);
return;
}
const cacheData = fs.readFileSync(Icons.FILE_NAME, 'utf8');
this.icons = JSON.parse(cacheData);
for (const i in this.icons) {
this.iconIds.push(i);
}
logger.debug(`Liquid icons has been loaded.`);
}
public getIconByAssetId(assetId: string): Buffer | undefined {
const icon = this.icons[assetId];
if (icon) {
return Buffer.from(icon, 'base64');
}
}
public getAllIconIds() {
return this.iconIds;
}
}
export default new Icons();

79
backend/src/api/statistics.ts
View File

@@ -267,8 +267,57 @@ class Statistics {
}
}
private getQueryForDays(div: number) {
return `SELECT id, added, unconfirmed_transactions,
private getQueryForDaysAvg(div: number, interval: string) {
return `SELECT id, UNIX_TIMESTAMP(added) as added,
CAST(avg(unconfirmed_transactions) as FLOAT) as unconfirmed_transactions,
CAST(avg(tx_per_second) as FLOAT) as tx_per_second,
CAST(avg(vbytes_per_second) as FLOAT) as vbytes_per_second,
CAST(avg(vsize_1) as FLOAT) as vsize_1,
CAST(avg(vsize_2) as FLOAT) as vsize_2,
CAST(avg(vsize_3) as FLOAT) as vsize_3,
CAST(avg(vsize_4) as FLOAT) as vsize_4,
CAST(avg(vsize_5) as FLOAT) as vsize_5,
CAST(avg(vsize_6) as FLOAT) as vsize_6,
CAST(avg(vsize_8) as FLOAT) as vsize_8,
CAST(avg(vsize_10) as FLOAT) as vsize_10,
CAST(avg(vsize_12) as FLOAT) as vsize_12,
CAST(avg(vsize_15) as FLOAT) as vsize_15,
CAST(avg(vsize_20) as FLOAT) as vsize_20,
CAST(avg(vsize_30) as FLOAT) as vsize_30,
CAST(avg(vsize_40) as FLOAT) as vsize_40,
CAST(avg(vsize_50) as FLOAT) as vsize_50,
CAST(avg(vsize_60) as FLOAT) as vsize_60,
CAST(avg(vsize_70) as FLOAT) as vsize_70,
CAST(avg(vsize_80) as FLOAT) as vsize_80,
CAST(avg(vsize_90) as FLOAT) as vsize_90,
CAST(avg(vsize_100) as FLOAT) as vsize_100,
CAST(avg(vsize_125) as FLOAT) as vsize_125,
CAST(avg(vsize_150) as FLOAT) as vsize_150,
CAST(avg(vsize_175) as FLOAT) as vsize_175,
CAST(avg(vsize_200) as FLOAT) as vsize_200,
CAST(avg(vsize_250) as FLOAT) as vsize_250,
CAST(avg(vsize_300) as FLOAT) as vsize_300,
CAST(avg(vsize_350) as FLOAT) as vsize_350,
CAST(avg(vsize_400) as FLOAT) as vsize_400,
CAST(avg(vsize_500) as FLOAT) as vsize_500,
CAST(avg(vsize_600) as FLOAT) as vsize_600,
CAST(avg(vsize_700) as FLOAT) as vsize_700,
CAST(avg(vsize_800) as FLOAT) as vsize_800,
CAST(avg(vsize_900) as FLOAT) as vsize_900,
CAST(avg(vsize_1000) as FLOAT) as vsize_1000,
CAST(avg(vsize_1200) as FLOAT) as vsize_1200,
CAST(avg(vsize_1400) as FLOAT) as vsize_1400,
CAST(avg(vsize_1600) as FLOAT) as vsize_1600,
CAST(avg(vsize_1800) as FLOAT) as vsize_1800,
CAST(avg(vsize_2000) as FLOAT) as vsize_2000 \
FROM statistics \
WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() \
GROUP BY UNIX_TIMESTAMP(added) DIV ${div} \
ORDER BY id DESC;`;
}
private getQueryForDays(div: number, interval: string) {
return `SELECT id, UNIX_TIMESTAMP(added) as added, unconfirmed_transactions,
tx_per_second,
vbytes_per_second,
vsize_1,
@@ -308,13 +357,17 @@ class Statistics {
vsize_1400,
vsize_1600,
vsize_1800,
vsize_2000 FROM statistics GROUP BY UNIX_TIMESTAMP(added) DIV ${div} ORDER BY id DESC LIMIT 480`;
vsize_2000 \
FROM statistics \
WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() \
GROUP BY UNIX_TIMESTAMP(added) DIV ${div} \
ORDER BY id DESC;`;
}
public async $get(id: number): Promise<OptimizedStatistic | undefined> {
try {
const connection = await DB.pool.getConnection();
const query = `SELECT * FROM statistics WHERE id = ?`;
const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics WHERE id = ?`;
const [rows] = await connection.query<any>(query, [id]);
connection.release();
if (rows[0]) {
@@ -328,7 +381,7 @@ class Statistics {
public async $list2H(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const query = `SELECT * FROM statistics ORDER BY id DESC LIMIT 120`;
const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY id DESC LIMIT 120`;
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
return this.mapStatisticToOptimizedStatistic(rows);
@@ -341,7 +394,7 @@ class Statistics {
public async $list24H(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const query = this.getQueryForDays(180);
const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY id DESC LIMIT 1440`;
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
return this.mapStatisticToOptimizedStatistic(rows);
@@ -354,7 +407,7 @@ class Statistics {
public async $list1W(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const query = this.getQueryForDays(1260);
const query = this.getQueryForDaysAvg(600, '1 WEEK'); // 10m interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
return this.mapStatisticToOptimizedStatistic(rows);
@@ -367,7 +420,7 @@ class Statistics {
public async $list1M(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const query = this.getQueryForDays(5040);
const query = this.getQueryForDaysAvg(3600, '1 MONTH'); // 1h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
return this.mapStatisticToOptimizedStatistic(rows);
@@ -380,7 +433,7 @@ class Statistics {
public async $list3M(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const query = this.getQueryForDays(15120);
const query = this.getQueryForDaysAvg(14400, '3 MONTH'); // 4h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
return this.mapStatisticToOptimizedStatistic(rows);
@@ -393,7 +446,7 @@ class Statistics {
public async $list6M(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const query = this.getQueryForDays(30240);
const query = this.getQueryForDaysAvg(21600, '6 MONTH'); // 6h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
return this.mapStatisticToOptimizedStatistic(rows);
@@ -406,7 +459,7 @@ class Statistics {
public async $list1Y(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const query = this.getQueryForDays(60480);
const query = this.getQueryForDays(43200, '1 YEAR'); // 12h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
return this.mapStatisticToOptimizedStatistic(rows);
@@ -419,7 +472,7 @@ class Statistics {
public async $list2Y(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const query = this.getQueryForDays(120960);
const query = this.getQueryForDays(86400, "2 YEAR"); // 1d interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
return this.mapStatisticToOptimizedStatistic(rows);
@@ -432,7 +485,7 @@ class Statistics {
public async $list3Y(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const query = this.getQueryForDays(181440);
const query = this.getQueryForDays(86400, "3 YEAR"); // 1d interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
return this.mapStatisticToOptimizedStatistic(rows);

2
backend/src/config.ts
View File

@@ -16,6 +16,7 @@ interface IConfig {
MEMPOOL_BLOCKS_AMOUNT: number;
PRICE_FEED_UPDATE_INTERVAL: number;
USE_SECOND_NODE_FOR_MINFEE: boolean;
EXTERNAL_ASSETS: string[];
};
ESPLORA: {
REST_API_URL: string;
@@ -78,6 +79,7 @@ const defaults: IConfig = {
'MEMPOOL_BLOCKS_AMOUNT': 8,
'PRICE_FEED_UPDATE_INTERVAL': 3600,
'USE_SECOND_NODE_FOR_MINFEE': false,
'EXTERNAL_ASSETS': [],
},
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',

14
backend/src/index.ts
View File

@@ -22,6 +22,8 @@ import loadingIndicators from './api/loading-indicators';
import mempool from './api/mempool';
import elementsParser from './api/liquid/elements-parser';
import databaseMigration from './api/database-migration';
import syncAssets from './sync-assets';
import icons from './api/liquid/icons';
class Server {
private wss: WebSocket.Server | undefined;
@@ -78,6 +80,7 @@ class Server {
this.setUpWebsocketHandling();
await syncAssets.syncAssets();
diskCache.loadMempoolCache();
if (config.DATABASE.ENABLED) {
@@ -93,6 +96,10 @@ class Server {
statistics.startStatistics();
}
if (config.MEMPOOL.NETWORK === 'liquid') {
icons.loadIcons();
}
fiatConversion.startService();
this.setUpHttpApiRoutes();
@@ -276,6 +283,13 @@ class Server {
;
}
if (config.MEMPOOL.NETWORK === 'liquid') {
this.app
.get(config.MEMPOOL.API_URL_PREFIX + 'assets/icons', routes.getAllLiquidIcon)
.get(config.MEMPOOL.API_URL_PREFIX + 'asset/:assetId/icon', routes.getLiquidIcon)
;
}
if (config.MEMPOOL.NETWORK === 'liquid' && config.DATABASE.ENABLED) {
this.app
.get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/month', routes.$getElementsPegsByMonth)

21
backend/src/routes.ts
View File

@@ -19,6 +19,7 @@ import loadingIndicators from './api/loading-indicators';
import { Common } from './api/common';
import bitcoinClient from './api/bitcoin/bitcoin-client';
import elementsParser from './api/liquid/elements-parser';
import icons from './api/liquid/icons';
class Routes {
constructor() {}
@@ -807,6 +808,26 @@ class Routes {
: (e.message || 'Error'));
}
}
public getLiquidIcon(req: Request, res: Response) {
const result = icons.getIconByAssetId(req.params.assetId);
if (result) {
res.setHeader('content-type', 'image/png');
res.setHeader('content-length', result.length);
res.send(result);
} else {
res.status(404).send('Asset icon not found');
}
}
public getAllLiquidIcon(req: Request, res: Response) {
const result = icons.getAllIconIds();
if (result) {
res.json(result);
} else {
res.status(404).send('Asset icons not found');
}
}
}
export default new Routes();

32
backend/src/sync-assets.ts Normal file
View File

@@ -0,0 +1,32 @@
import axios from 'axios';
import * as fs from 'fs';
const fsPromises = fs.promises;
import config from './config';
import logger from './logger';
const PATH = './';
class SyncAssets {
constructor() { }
public async syncAssets() {
for (const url of config.MEMPOOL.EXTERNAL_ASSETS) {
await this.downloadFile(url);
}
}
private async downloadFile(url: string) {
const fileName = url.split('/').slice(-1)[0];
logger.info(`Downloading external asset: ${fileName}...`);
try {
const response = await axios.get(url, {
responseType: 'stream', timeout: 30000
});
await fsPromises.writeFile(PATH + fileName, response.data);
} catch (e: any) {
throw new Error(`Failed to download external asset. ` + e);
}
}
}
export default new SyncAssets();