machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactoring price update config. Fixing last price time.

Browse Source
This commit is contained in:
softsimon 2023-07-29 19:27:19 +09:00
parent bb7f73637b
commit 8c442fe207
9 changed files with 25 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
backend/mempool-config.sample.json
View File

@ -32,7 +32,8 @@
"CPFP_INDEXING": false,
"DISK_CACHE_BLOCK_INTERVAL": 6,
"MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
"ALLOW_UNREACHABLE": true
"ALLOW_UNREACHABLE": true,
"PRICE_UPDATE_FREQUENCY": 3600
},
"CORE_RPC": {
"HOST": "127.0.0.1",
@ -115,11 +116,6 @@
"USERNAME": "",
"PASSWORD": ""
},
"PRICE_DATA_SERVER": {
"UPDATE_FREQUENCY": 300,
"TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices",
"CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices"
},
"EXTERNAL_DATA_SERVER": {
"MEMPOOL_API": "https://mempool.space/api/v1",
"MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1",

12
backend/src/__fixtures__/mempool-config.template.json
View File

@ -23,8 +23,8 @@
"USER_AGENT": "__MEMPOOL_USER_AGENT__",
"STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
"INDEXING_BLOCKS_AMOUNT": 14,
"POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
"POOLS_JSON_URL": "__POOLS_JSON_URL__",
"POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__",
"POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
"AUDIT": true,
"ADVANCED_GBT_AUDIT": true,
"ADVANCED_GBT_MEMPOOL": true,
@ -33,7 +33,8 @@
"MAX_BLOCKS_BULK_QUERY": 999,
"DISK_CACHE_BLOCK_INTERVAL": 999,
"MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
"ALLOW_UNREACHABLE": true
"ALLOW_UNREACHABLE": true,
"PRICE_UPDATE_FREQUENCY": "__MEMPOOL_PRICE_UPDATE_FREQUENCY__"
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",
@ -92,11 +93,6 @@
"USERNAME": "__SOCKS5PROXY_USERNAME__",
"PASSWORD": "__SOCKS5PROXY_PASSWORD__"
},
"PRICE_DATA_SERVER": {
"UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY__",
"TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
"CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
},
"EXTERNAL_DATA_SERVER": {
"MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
"MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",

9
backend/src/__tests__/config.test.ts
View File

@ -47,6 +47,7 @@ describe('Mempool Backend Config', () => {
DISK_CACHE_BLOCK_INTERVAL: 6,
MAX_PUSH_TX_SIZE_WEIGHT: 400000,
ALLOW_UNREACHABLE: true,
PRICE_UPDATE_FREQUENCY: 3600,
});
expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
@ -101,12 +102,6 @@ describe('Mempool Backend Config', () => {
PASSWORD: ''
});
expect(config.PRICE_DATA_SERVER).toStrictEqual({
UPDATE_FREQUENCY: 3600,
TOR_URL: 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
CLEARNET_URL: 'https://price.bisq.wiz.biz/getAllMarketPrices'
});
expect(config.EXTERNAL_DATA_SERVER).toStrictEqual({
MEMPOOL_API: 'https://mempool.space/api/v1',
MEMPOOL_ONION: 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
@ -164,8 +159,6 @@ describe('Mempool Backend Config', () => {
expect(config.SOCKS5PROXY).toStrictEqual(fixture.SOCKS5PROXY);
expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER);
expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER);
expect(config.REDIS).toStrictEqual(fixture.REDIS);

14
backend/src/config.ts
View File

@ -38,6 +38,7 @@ interface IConfig {
DISK_CACHE_BLOCK_INTERVAL: number;
MAX_PUSH_TX_SIZE_WEIGHT: number;
ALLOW_UNREACHABLE: boolean;
PRICE_UPDATE_FREQUENCY: number;
};
ESPLORA: {
REST_API_URL: string;
@ -115,11 +116,6 @@ interface IConfig {
USERNAME: string;
PASSWORD: string;
};
PRICE_DATA_SERVER: {
UPDATE_FREQUENCY: number;
TOR_URL: string;
CLEARNET_URL: string;
};
EXTERNAL_DATA_SERVER: {
MEMPOOL_API: string;
MEMPOOL_ONION: string;
@ -182,6 +178,7 @@ const defaults: IConfig = {
'DISK_CACHE_BLOCK_INTERVAL': 6,
'MAX_PUSH_TX_SIZE_WEIGHT': 400000,
'ALLOW_UNREACHABLE': true,
'PRICE_UPDATE_FREQUENCY': 3600,
},
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',
@ -259,11 +256,6 @@ const defaults: IConfig = {
'USERNAME': '',
'PASSWORD': ''
},
'PRICE_DATA_SERVER': {
'UPDATE_FREQUENCY': 3600,
'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices'
},
'EXTERNAL_DATA_SERVER': {
'MEMPOOL_API': 'https://mempool.space/api/v1',
'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
@ -304,7 +296,6 @@ class Config implements IConfig {
LND: IConfig['LND'];
CLIGHTNING: IConfig['CLIGHTNING'];
SOCKS5PROXY: IConfig['SOCKS5PROXY'];
PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
MAXMIND: IConfig['MAXMIND'];
REPLICATION: IConfig['REPLICATION'];
@ -325,7 +316,6 @@ class Config implements IConfig {
this.LND = configs.LND;
this.CLIGHTNING = configs.CLIGHTNING;
this.SOCKS5PROXY = configs.SOCKS5PROXY;
this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
this.MAXMIND = configs.MAXMIND;
this.REPLICATION = configs.REPLICATION;

8
backend/src/indexer.ts
View File

@ -105,6 +105,12 @@ class Indexer {
return;
}
try {
await priceUpdater.$run();
} catch (e) {
logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e));
}
// Do not attempt to index anything unless Bitcoin Core is fully synced
const blockchainInfo = await bitcoinClient.getBlockchainInfo();
if (blockchainInfo.blocks !== blockchainInfo.headers) {
@ -119,8 +125,6 @@ class Indexer {
await this.checkAvailableCoreIndexes();
try {
await priceUpdater.$run();
const chainValid = await blocks.$generateBlockDatabase();
if (chainValid === false) {
// Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration

2
backend/src/repositories/PricesRepository.ts
View File

@ -65,7 +65,7 @@ class PricesRepository {
try {
await DB.query(`
INSERT INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY)
INSERT IGNORE INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY)
VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ? )`,
[time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY]
);

10
backend/src/tasks/price-updater.ts
View File

@ -108,8 +108,8 @@ class PriceUpdater {
this.lastRun = await PricesRepository.$getLatestPriceTime();
}
if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < config.PRICE_DATA_SERVER.UPDATE_FREQUENCY) {
// Refresh every UPDATE_FREQUENCY seconds at most
if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < config.MEMPOOL.PRICE_UPDATE_FREQUENCY) {
// Refresh every PRICE_UPDATE_FREQUENCY seconds at most
return;
}
@ -146,14 +146,11 @@ class PriceUpdater {
}
}
logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
if (config.DATABASE.ENABLED === true) {
// Save everything in db
try {
const p = 60 * 60 * 1000; // milliseconds in an hour
const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042
this.latestPrices.time = nowRounded.getTime() / 1000;
await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices);
} catch (e) {
this.lastRun = previousRun + 5 * 60;
@ -161,6 +158,9 @@ class PriceUpdater {
}
}
this.latestPrices.time = Math.round(new Date().getTime() / 1000);
logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
if (this.ratesChangedCallback) {
this.ratesChangedCallback(this.latestPrices);
}

8
docker/backend/mempool-config.json
View File

@ -33,7 +33,8 @@
"MAX_PUSH_TX_SIZE_WEIGHT": __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__,
"ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__,
"POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__",
"POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__"
"POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
"PRICE_UPDATE_FREQUENCY": __MEMPOOL_PRICE_UPDATE_FREQUENCY__
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",
@ -111,11 +112,6 @@
"USERNAME": "__SOCKS5PROXY_USERNAME__",
"PASSWORD": "__SOCKS5PROXY_PASSWORD__"
},
"PRICE_DATA_SERVER": {
"UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY__",
"TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
"CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
},
"EXTERNAL_DATA_SERVER": {
"MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
"MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",

6
production/mempool-config.mainnet.json
View File

@ -18,7 +18,8 @@
"USE_SECOND_NODE_FOR_MINFEE": true,
"DISK_CACHE_BLOCK_INTERVAL": 1,
"MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
"ALLOW_UNREACHABLE": true
"ALLOW_UNREACHABLE": true,
"PRICE_UPDATE_FREQUENCY": 300
},
"SYSLOG" : {
"MIN_PRIORITY": "debug"
@ -49,9 +50,6 @@
"ENABLED": true,
"TX_PER_SECOND_SAMPLE_PERIOD": 150
},
"PRICE_DATA_SERVER": {
"UPDATE_FREQUENCY": 300
},
"REPLICATION": {
"ENABLED": true,
"AUDIT": true,