machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Markets API: Use file mtime to only parse files that has updated.

Browse Source
This commit is contained in:
softsimon 2020-09-18 23:43:45 +07:00
parent 55852bcf62
commit 5d2c0d2e0a
No known key found for this signature in database
GPG Key ID: 488D7DCFB5A430D7
3 changed files with 71 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/api/bisq/interfaces.ts
View File

@ -142,7 +142,7 @@ interface OffersMarket {
sells: Offer[] | null;
}
export interface OffsersData {
export interface OffersData {
direction: string;
currencyCode: string;
minAmount: number;

37
backend/src/api/bisq/markets-api.ts
View File

@ -1,4 +1,4 @@
import { Currencies, OffsersData, TradesData, Depth, Currency, Interval, HighLowOpenClose,
import { Currencies, OffersData, TradesData, Depth, Currency, Interval, HighLowOpenClose,
Markets, Offers, Offer, BisqTrade, MarketVolume, Tickers, Ticker, SummarizedIntervals, SummarizedInterval } from './interfaces';
import * as datetime from 'locutus/php/datetime';
@ -6,7 +6,7 @@ import * as datetime from 'locutus/php/datetime';
class BisqMarketsApi {
private cryptoCurrencyData: Currency[] = [];
private fiatCurrencyData: Currency[] = [];
private offersData: OffsersData[] = [];
private offersData: OffersData[] = [];
private tradesData: TradesData[] = [];
private fiatCurrenciesIndexed: { [code: string]: true } = {};
private allCurrenciesIndexed: { [code: string]: Currency } = {};
@ -14,15 +14,30 @@ class BisqMarketsApi {
constructor() { }
public setData(cryptoCurrency: Currency[], fiatCurrency: Currency[], offers: OffsersData[], trades: TradesData[]) {
setOffersData(offers: OffersData[]) {
this.offersData = offers;
}
setTradesData(trades: TradesData[]) {
this.tradesData = trades;
this.tradeDataByMarket = {};
this.tradesData.forEach((trade) => {
trade._market = trade.currencyPair.toLowerCase().replace('/', '_');
if (!this.tradeDataByMarket[trade._market]) {
this.tradeDataByMarket[trade._market] = [];
}
this.tradeDataByMarket[trade._market].push(trade);
});
}
setCurrencyData(cryptoCurrency: Currency[], fiatCurrency: Currency[]) {
this.cryptoCurrencyData = cryptoCurrency,
this.fiatCurrencyData = fiatCurrency;
this.offersData = offers;
this.tradesData = trades;
// Handle data for smarter memory caching
this.fiatCurrenciesIndexed = {};
this.allCurrenciesIndexed = {};
this.fiatCurrencyData.forEach((currency) => {
currency._type = 'fiat';
this.fiatCurrenciesIndexed[currency.code] = true;
@ -32,14 +47,6 @@ class BisqMarketsApi {
currency._type = 'crypto';
this.allCurrenciesIndexed[currency.code] = currency;
});
this.tradeDataByMarket = {};
this.tradesData.forEach((trade) => {
trade._market = trade.currencyPair.toLowerCase().replace('/', '_');
if (!this.tradeDataByMarket[trade._market]) {
this.tradeDataByMarket[trade._market] = [];
}
this.tradeDataByMarket[trade._market].push(trade);
});
}
getCurrencies(
@ -600,7 +607,7 @@ class BisqMarketsApi {
}
}
private offerDataToOffer(offer: OffsersData, market: string): Offer {
private offerDataToOffer(offer: OffersData, market: string): Offer {
const currencyPairs = market.split('_');
const currencyRight = this.allCurrenciesIndexed[currencyPairs[1].toUpperCase()];
const currencyLeft = this.allCurrenciesIndexed[currencyPairs[0].toUpperCase()];

61
backend/src/api/bisq/markets.ts
View File

@ -1,9 +1,10 @@
const config = require('../../../mempool-config.json');
import * as fs from 'fs';
import { OffsersData, TradesData, Currency } from './interfaces';
import { OffersData as OffersData, TradesData, Currency } from './interfaces';
import bisqMarket from './markets-api';
class Bisq {
private static FOLDER_WATCH_CHANGE_DETECTION_DEBOUNCE = 4000;
private static MARKET_JSON_PATH = config.BISQ_MARKETS_DATA_PATH + '/btc_mainnet/db';
private static MARKET_JSON_FILE_PATHS = {
cryptoCurrency: '/crypto_currency_list.json',
@ -12,6 +13,11 @@ class Bisq {
trades: '/trade_statistics.json',
};
private cryptoCurrencyLastMtime = new Date('2016-01-01');
private fiatCurrencyLastMtime = new Date('2016-01-01');
private offersLastMtime = new Date('2016-01-01');
private tradesLastMtime = new Date('2016-01-01');
private subdirectoryWatcher: fs.FSWatcher | undefined;
constructor() {}
@ -46,32 +52,61 @@ class Bisq {
fsWait = setTimeout(() => {
console.log(`Change detected in the Bisq market data folder.`);
this.loadBisqDumpFile();
}, 2000);
}, Bisq.FOLDER_WATCH_CHANGE_DETECTION_DEBOUNCE);
});
}
private async loadBisqDumpFile(): Promise<void> {
const start = new Date().getTime();
console.log('Processing Bisq market data...');
try {
const cryptoCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency);
const fiatCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.fiatCurrency);
const offersData = await this.loadData<OffsersData[]>(Bisq.MARKET_JSON_FILE_PATHS.offers);
const tradesData = await this.loadData<TradesData[]>(Bisq.MARKET_JSON_FILE_PATHS.trades);
bisqMarket.setData(cryptoCurrencyData, fiatCurrencyData, offersData, tradesData);
let marketsDataUpdated = false;
const cryptoMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency);
const fiatMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.fiatCurrency);
if (cryptoMtime > this.cryptoCurrencyLastMtime || fiatMtime > this.fiatCurrencyLastMtime) {
const cryptoCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency);
const fiatCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.fiatCurrency);
console.log('Updating Bisq Market Currency Data');
bisqMarket.setCurrencyData(cryptoCurrencyData, fiatCurrencyData);
if (cryptoMtime > this.cryptoCurrencyLastMtime) {
this.cryptoCurrencyLastMtime = cryptoMtime;
}
if (fiatMtime > this.fiatCurrencyLastMtime) {
this.fiatCurrencyLastMtime = fiatMtime;
}
marketsDataUpdated = true;
}
const offersMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.offers);
if (offersMtime > this.offersLastMtime) {
const offersData = await this.loadData<OffersData[]>(Bisq.MARKET_JSON_FILE_PATHS.offers);
console.log('Updating Bisq Market Offers Data');
bisqMarket.setOffersData(offersData);
this.offersLastMtime = offersMtime;
marketsDataUpdated = true;
}
const tradesMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.trades);
if (tradesMtime > this.tradesLastMtime) {
const tradesData = await this.loadData<TradesData[]>(Bisq.MARKET_JSON_FILE_PATHS.trades);
console.log('Updating Bisq Market Trades Data');
bisqMarket.setTradesData(tradesData);
this.tradesLastMtime = tradesMtime;
marketsDataUpdated = true;
}
const time = new Date().getTime() - start;
console.log('Bisq market data processed in ' + time + ' ms');
if (marketsDataUpdated) {
console.log('Bisq market data updated in ' + time + ' ms');
}
} catch (e) {
console.log('loadBisqMarketDataDumpFile() error.', e.message);
}
}
private getFileMtime(path: string): Date {
const stats = fs.statSync(Bisq.MARKET_JSON_PATH + path);
return stats.mtime;
}
private loadData<T>(path: string): Promise<T> {
return new Promise((resolve, reject) => {
if (!fs.existsSync(Bisq.MARKET_JSON_PATH + path)) {
return reject(path + ` doesn't exist`);
}
fs.readFile(Bisq.MARKET_JSON_PATH + path, 'utf8', (err, data) => {
if (err) {
reject(err);