machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Markets API: Use file mtime to only parse files that has updated.

Browse Source
This commit is contained in:
softsimon 2020-09-18 23:43:45 +07:00
parent 55852bcf62
commit 5d2c0d2e0a
No known key found for this signature in database
GPG Key ID: 488D7DCFB5A430D7
3 changed files with 71 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/api/bisq/interfaces.ts
View File

@ -142,7 +142,7 @@ interface OffersMarket {
sells: Offer[] | null; sells: Offer[] | null;
} }
export interface OffsersData { export interface OffersData {
direction: string; direction: string;
currencyCode: string; currencyCode: string;
minAmount: number; minAmount: number;

37
backend/src/api/bisq/markets-api.ts
View File

@ -1,4 +1,4 @@
import { Currencies, OffsersData, TradesData, Depth, Currency, Interval, HighLowOpenClose, import { Currencies, OffersData, TradesData, Depth, Currency, Interval, HighLowOpenClose,
Markets, Offers, Offer, BisqTrade, MarketVolume, Tickers, Ticker, SummarizedIntervals, SummarizedInterval } from './interfaces'; Markets, Offers, Offer, BisqTrade, MarketVolume, Tickers, Ticker, SummarizedIntervals, SummarizedInterval } from './interfaces';
import * as datetime from 'locutus/php/datetime'; import * as datetime from 'locutus/php/datetime';
@ -6,7 +6,7 @@ import * as datetime from 'locutus/php/datetime';
class BisqMarketsApi { class BisqMarketsApi {
private cryptoCurrencyData: Currency[] = []; private cryptoCurrencyData: Currency[] = [];
private fiatCurrencyData: Currency[] = []; private fiatCurrencyData: Currency[] = [];
private offersData: OffsersData[] = []; private offersData: OffersData[] = [];
private tradesData: TradesData[] = []; private tradesData: TradesData[] = [];
private fiatCurrenciesIndexed: { [code: string]: true } = {}; private fiatCurrenciesIndexed: { [code: string]: true } = {};
private allCurrenciesIndexed: { [code: string]: Currency } = {}; private allCurrenciesIndexed: { [code: string]: Currency } = {};
@ -14,15 +14,30 @@ class BisqMarketsApi {
constructor() { } constructor() { }
public setData(cryptoCurrency: Currency[], fiatCurrency: Currency[], offers: OffsersData[], trades: TradesData[]) { setOffersData(offers: OffersData[]) {
this.offersData = offers;
}
setTradesData(trades: TradesData[]) {
this.tradesData = trades;
this.tradeDataByMarket = {};
this.tradesData.forEach((trade) => {
trade._market = trade.currencyPair.toLowerCase().replace('/', '_');
if (!this.tradeDataByMarket[trade._market]) {
this.tradeDataByMarket[trade._market] = [];
}
this.tradeDataByMarket[trade._market].push(trade);
});
}
setCurrencyData(cryptoCurrency: Currency[], fiatCurrency: Currency[]) {
this.cryptoCurrencyData = cryptoCurrency, this.cryptoCurrencyData = cryptoCurrency,
this.fiatCurrencyData = fiatCurrency; this.fiatCurrencyData = fiatCurrency;
this.offersData = offers;
this.tradesData = trades;
// Handle data for smarter memory caching
this.fiatCurrenciesIndexed = {}; this.fiatCurrenciesIndexed = {};
this.allCurrenciesIndexed = {}; this.allCurrenciesIndexed = {};
this.fiatCurrencyData.forEach((currency) => { this.fiatCurrencyData.forEach((currency) => {
currency._type = 'fiat'; currency._type = 'fiat';
this.fiatCurrenciesIndexed[currency.code] = true; this.fiatCurrenciesIndexed[currency.code] = true;
@ -32,14 +47,6 @@ class BisqMarketsApi {
currency._type = 'crypto'; currency._type = 'crypto';
this.allCurrenciesIndexed[currency.code] = currency; this.allCurrenciesIndexed[currency.code] = currency;
}); });
this.tradeDataByMarket = {};
this.tradesData.forEach((trade) => {
trade._market = trade.currencyPair.toLowerCase().replace('/', '_');
if (!this.tradeDataByMarket[trade._market]) {
this.tradeDataByMarket[trade._market] = [];
}
this.tradeDataByMarket[trade._market].push(trade);
});
} }
getCurrencies( getCurrencies(
@ -600,7 +607,7 @@ class BisqMarketsApi {
} }
} }
private offerDataToOffer(offer: OffsersData, market: string): Offer { private offerDataToOffer(offer: OffersData, market: string): Offer {
const currencyPairs = market.split('_'); const currencyPairs = market.split('_');
const currencyRight = this.allCurrenciesIndexed[currencyPairs[1].toUpperCase()]; const currencyRight = this.allCurrenciesIndexed[currencyPairs[1].toUpperCase()];
const currencyLeft = this.allCurrenciesIndexed[currencyPairs[0].toUpperCase()]; const currencyLeft = this.allCurrenciesIndexed[currencyPairs[0].toUpperCase()];

61
backend/src/api/bisq/markets.ts
View File

@ -1,9 +1,10 @@
const config = require('../../../mempool-config.json'); const config = require('../../../mempool-config.json');
import * as fs from 'fs'; import * as fs from 'fs';
import { OffsersData, TradesData, Currency } from './interfaces'; import { OffersData as OffersData, TradesData, Currency } from './interfaces';
import bisqMarket from './markets-api'; import bisqMarket from './markets-api';
class Bisq { class Bisq {
private static FOLDER_WATCH_CHANGE_DETECTION_DEBOUNCE = 4000;
private static MARKET_JSON_PATH = config.BISQ_MARKETS_DATA_PATH + '/btc_mainnet/db'; private static MARKET_JSON_PATH = config.BISQ_MARKETS_DATA_PATH + '/btc_mainnet/db';
private static MARKET_JSON_FILE_PATHS = { private static MARKET_JSON_FILE_PATHS = {
cryptoCurrency: '/crypto_currency_list.json', cryptoCurrency: '/crypto_currency_list.json',
@ -12,6 +13,11 @@ class Bisq {
trades: '/trade_statistics.json', trades: '/trade_statistics.json',
}; };
private cryptoCurrencyLastMtime = new Date('2016-01-01');
private fiatCurrencyLastMtime = new Date('2016-01-01');
private offersLastMtime = new Date('2016-01-01');
private tradesLastMtime = new Date('2016-01-01');
private subdirectoryWatcher: fs.FSWatcher | undefined; private subdirectoryWatcher: fs.FSWatcher | undefined;
constructor() {} constructor() {}
@ -46,32 +52,61 @@ class Bisq {
fsWait = setTimeout(() => { fsWait = setTimeout(() => {
console.log(`Change detected in the Bisq market data folder.`); console.log(`Change detected in the Bisq market data folder.`);
this.loadBisqDumpFile(); this.loadBisqDumpFile();
}, 2000); }, Bisq.FOLDER_WATCH_CHANGE_DETECTION_DEBOUNCE);
}); });
} }
private async loadBisqDumpFile(): Promise<void> { private async loadBisqDumpFile(): Promise<void> {
const start = new Date().getTime(); const start = new Date().getTime();
console.log('Processing Bisq market data...');
try { try {
const cryptoCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency); let marketsDataUpdated = false;
const fiatCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.fiatCurrency); const cryptoMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency);
const offersData = await this.loadData<OffsersData[]>(Bisq.MARKET_JSON_FILE_PATHS.offers); const fiatMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.fiatCurrency);
const tradesData = await this.loadData<TradesData[]>(Bisq.MARKET_JSON_FILE_PATHS.trades); if (cryptoMtime > this.cryptoCurrencyLastMtime || fiatMtime > this.fiatCurrencyLastMtime) {
const cryptoCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency);
bisqMarket.setData(cryptoCurrencyData, fiatCurrencyData, offersData, tradesData); const fiatCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.fiatCurrency);
console.log('Updating Bisq Market Currency Data');
bisqMarket.setCurrencyData(cryptoCurrencyData, fiatCurrencyData);
if (cryptoMtime > this.cryptoCurrencyLastMtime) {
this.cryptoCurrencyLastMtime = cryptoMtime;
}
if (fiatMtime > this.fiatCurrencyLastMtime) {
this.fiatCurrencyLastMtime = fiatMtime;
}
marketsDataUpdated = true;
}
const offersMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.offers);
if (offersMtime > this.offersLastMtime) {
const offersData = await this.loadData<OffersData[]>(Bisq.MARKET_JSON_FILE_PATHS.offers);
console.log('Updating Bisq Market Offers Data');
bisqMarket.setOffersData(offersData);
this.offersLastMtime = offersMtime;
marketsDataUpdated = true;
}
const tradesMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.trades);
if (tradesMtime > this.tradesLastMtime) {
const tradesData = await this.loadData<TradesData[]>(Bisq.MARKET_JSON_FILE_PATHS.trades);
console.log('Updating Bisq Market Trades Data');
bisqMarket.setTradesData(tradesData);
this.tradesLastMtime = tradesMtime;
marketsDataUpdated = true;
}
const time = new Date().getTime() - start; const time = new Date().getTime() - start;
console.log('Bisq market data processed in ' + time + ' ms'); if (marketsDataUpdated) {
console.log('Bisq market data updated in ' + time + ' ms');
}
} catch (e) { } catch (e) {
console.log('loadBisqMarketDataDumpFile() error.', e.message); console.log('loadBisqMarketDataDumpFile() error.', e.message);
} }
} }
private getFileMtime(path: string): Date {
const stats = fs.statSync(Bisq.MARKET_JSON_PATH + path);
return stats.mtime;
}
private loadData<T>(path: string): Promise<T> { private loadData<T>(path: string): Promise<T> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
if (!fs.existsSync(Bisq.MARKET_JSON_PATH + path)) {
return reject(path + ` doesn't exist`);
}
fs.readFile(Bisq.MARKET_JSON_PATH + path, 'utf8', (err, data) => { fs.readFile(Bisq.MARKET_JSON_PATH + path, 'utf8', (err, data) => {
if (err) { if (err) {
reject(err); reject(err);