machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HLOC markets api.

Browse Source
This commit is contained in:
softsimon 2020-09-13 17:51:53 +07:00
parent 23b2b108a6
commit a3f8555087
8 changed files with 282 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/mempool-config.sample.json
View File

@ -17,7 +17,7 @@
"BISQ_ENABLED": false,
"BISQ_MARKET_ENABLED": false,
"BSQ_BLOCKS_DATA_PATH": "/bisq/data",
"BSQ_MARKETS_DATA_PATH": "/bisq-markets/data",
"BSQ_MARKETS_DATA_PATH": "/bisq-folder/Bisq",
"SSL": false,
"SSL_CERT_FILE_PATH": "/etc/letsencrypt/live/mysite/fullchain.pem",
"SSL_KEY_FILE_PATH": "/etc/letsencrypt/live/mysite/privkey.pem"

18
backend/package-lock.json generated
View File

@ -86,6 +86,11 @@
"@types/range-parser": "*"
}
},
"@types/locutus": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/@types/locutus/-/locutus-0.0.6.tgz",
"integrity": "sha512-P+BQds4wrJhqKiIOBWAYpbsE9UOztnnqW9zHk4Bci7kCXjEQAA7FJrD9HX5JU2Z36fhE2WDctuuIpLvqDsciWQ=="
},
"@types/mime": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.3.tgz",
@ -446,6 +451,11 @@
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
"integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k="
},
"es6-promise": {
"version": "4.2.8",
"resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz",
"integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w=="
},
"escape-html": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
@ -733,6 +743,14 @@
"verror": "1.10.0"
}
},
"locutus": {
"version": "2.0.12",
"resolved": "https://registry.npmjs.org/locutus/-/locutus-2.0.12.tgz",
"integrity": "sha512-wnzhY9xOdDb2djr17kQhTh9oZgEfp78zI27KRRiiV1GnPXWA2xfVODbpH3QgpIuUMLupM02+6X/rJXvktTpnoA==",
"requires": {
"es6-promise": "^4.2.5"
}
},
"long": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz",

2
backend/package.json
View File

@ -29,6 +29,7 @@
"express": "^4.17.1",
"mysql2": "^1.6.1",
"request": "^2.88.2",
"locutus": "^2.0.12",
"ws": "^7.3.1"
},
"devDependencies": {
@ -36,6 +37,7 @@
"@types/express": "^4.17.2",
"@types/request": "^2.48.2",
"@types/ws": "^6.0.4",
"@types/locutus": "^0.0.6",
"tslint": "~6.1.0",
"typescript": "~3.9.7"
}

12
backend/src/api/bisq/interfaces.ts
View File

@ -222,10 +222,14 @@ export interface TradesData {
primaryMarketTradeVolume: number;
_market: string;
_tradePrice: string;
_tradeAmount: string;
_tradeVolume: string;
_offerAmount: string;
_tradePriceStr: string;
_tradeAmountStr: string;
_tradeVolumeStr: string;
_offerAmountStr: string;
_tradePrice: number;
_tradeAmount: number;
_tradeVolume: number;
_offerAmount: number;
}
export interface MarketVolume {

330
backend/src/api/bisq/markets-api.ts
View File

@ -1,6 +1,8 @@
import { Currencies, OffsersData, TradesData, Depth, Currency, Interval, HighLowOpenClose,
Markets, Offers, Offer, BisqTrade, MarketVolume, Tickers } from './interfaces';
import * as datetime from 'locutus/php/datetime';
class BisqMarketsApi {
private cryptoCurrencyData: Currency[] = [];
private fiatCurrencyData: Currency[] = [];
@ -149,92 +151,18 @@ class BisqMarketsApi {
direction?: 'buy' | 'sell',
limit: number = 100,
sort: 'asc' | 'desc' = 'desc',
): BisqTrade[] {
): BisqTrade[] {
limit = Math.min(limit, 2000);
let trade_id_from_ts: number | null = null;
let trade_id_to_ts: number | null = null;
const _market = market === 'all' ? null : market;
if (!timestamp_from) {
timestamp_from = new Date('2016-01-01').getTime();
} else {
timestamp_from = timestamp_from * 1000;
timestamp_from = new Date('2016-01-01').getTime() / 1000;
}
if (!timestamp_to) {
timestamp_to = new Date().getTime();
} else {
timestamp_to = timestamp_to * 1000;
timestamp_to = new Date().getTime() / 1000;
}
const allCurrencies = this.getCurrencies();
// note: the offer_id_from/to depends on iterating over trades in
// descending chronological order.
const tradesDataSorted = this.tradesData.slice();
if (sort === 'asc') {
tradesDataSorted.reverse();
}
let matches: TradesData[] = [];
for (const trade of tradesDataSorted) {
if (trade_id_from === trade.offerId) {
trade_id_from_ts = trade.tradeDate;
}
if (trade_id_to === trade.offerId) {
trade_id_to_ts = trade.tradeDate;
}
if (trade_id_to && trade_id_to_ts === null) {
continue;
}
if (trade_id_from && trade_id_from_ts != null && trade_id_from_ts !== trade.tradeDate ) {
continue;
}
if (_market && _market !== trade._market) {
continue;
}
if (timestamp_from && timestamp_from > trade.tradeDate) {
continue;
}
if (timestamp_to && timestamp_to < trade.tradeDate) {
continue;
}
if (direction && direction !== trade.direction.toLowerCase() ) {
continue;
}
// Filter out bogus trades with BTC/BTC or XXX/XXX market.
// See github issue: https://github.com/bitsquare/bitsquare/issues/883
const currencyPairs = trade.currencyPair.split('/');
if (currencyPairs[0] === currencyPairs[1]) {
continue;
}
const currencyLeft = allCurrencies[currencyPairs[0]];
const currencyRight = allCurrencies[currencyPairs[1]];
if (!currencyLeft || !currencyRight) {
continue;
}
const tradePrice = trade.primaryMarketTradePrice * Math.pow(10, 8 - currencyRight.precision);
const tradeAmount = trade.primaryMarketTradeAmount * Math.pow(10, 8 - currencyLeft.precision);
const tradeVolume = trade.primaryMarketTradeVolume * Math.pow(10, 8 - currencyRight.precision);
trade._tradePrice = this.intToBtc(tradePrice);
trade._tradeAmount = this.intToBtc(tradeAmount);
trade._tradeVolume = this.intToBtc(tradeVolume);
trade._offerAmount = this.intToBtc(trade.offerAmount);
matches.push(trade);
if (matches.length >= limit) {
break;
}
}
if ((trade_id_from && !trade_id_from_ts) || (trade_id_to && !trade_id_to_ts) ) {
matches = [];
}
const matches = this.getTradesByCriteria(_market, timestamp_to, timestamp_from, trade_id_to, trade_id_from, direction, sort, limit);
if (sort === 'asc') {
matches.sort((a, b) => a.tradeDate - b.tradeDate);
@ -245,9 +173,9 @@ class BisqMarketsApi {
return matches.map((trade) => {
const bsqTrade: BisqTrade = {
direction: trade.primaryMarketDirection,
price: trade._tradePrice,
amount: trade._tradeAmount,
volume: trade._tradeVolume,
price: trade._tradePriceStr,
amount: trade._tradeAmountStr,
volume: trade._tradeVolumeStr,
payment_method: trade.paymentMethod,
trade_id: trade.offerId,
trade_date: trade.tradeDate,
@ -279,20 +207,246 @@ class BisqMarketsApi {
interval: Interval = 'auto',
timestamp_from?: number,
timestamp_to?: number,
milliseconds?: boolean,
): HighLowOpenClose[] {
if (milliseconds) {
timestamp_from = timestamp_from ? timestamp_from / 1000 : timestamp_from;
timestamp_to = timestamp_to ? timestamp_to / 1000 : timestamp_to;
}
if (!timestamp_from) {
timestamp_from = new Date('2016-01-01').getTime();
} else {
timestamp_from = timestamp_from * 1000;
timestamp_from = new Date('2016-01-01').getTime() / 1000;
}
if (!timestamp_to) {
timestamp_to = new Date().getTime();
} else {
timestamp_to = timestamp_to * 1000;
timestamp_to = new Date().getTime() / 1000;
}
return [];
const range = timestamp_to - timestamp_from;
const trades = this.getTradesByCriteria(market, timestamp_to, timestamp_from,
undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER);
if (interval === 'auto') {
// two days range loads minute data
if (range <= 3600) {
// up to one hour range loads minutely data
interval = 'minute';
} else if (range <= 1 * 24 * 3600) {
// up to one day range loads half-hourly data
interval = 'half_hour';
} else if (range <= 3 * 24 * 3600) {
// up to 3 day range loads hourly data
interval = 'hour';
} else if (range <= 7 * 24 * 3600) {
// up to 7 day range loads half-daily data
interval = 'half_day';
} else if (range <= 60 * 24 * 3600) {
// up to 2 month range loads daily data
interval = 'day';
} else if (range <= 12 * 31 * 24 * 3600) {
// up to one year range loads weekly data
interval = 'week';
} else if (range <= 12 * 31 * 24 * 3600) {
// up to 5 year range loads monthly data
interval = 'month';
} else {
// greater range loads yearly data
interval = 'year';
}
}
const hlocs = this.getTradesSummarized(trades, timestamp_from, interval);
return hlocs;
}
private getTradesSummarized(trades: TradesData[], timestamp_from, interval: string): HighLowOpenClose[] {
const intervals: any = {};
const intervals_prices: any = {};
const one_period = false;
for (const trade of trades) {
const traded_at = trade.tradeDate / 1000;
const interval_start = one_period ? timestamp_from : this.intervalStart(traded_at, interval);
if (!intervals[interval_start]) {
intervals[interval_start] = {
'open': 0,
'close': 0,
'high': 0,
'low': 0,
'avg': 0,
'volume_right': 0,
'volume_left': 0,
};
intervals_prices[interval_start] = [];
}
const period = intervals[interval_start];
const price = trade._tradePrice;
if (!intervals_prices[interval_start]['leftvol']) {
intervals_prices[interval_start]['leftvol'] = [];
}
if (!intervals_prices[interval_start]['rightvol']) {
intervals_prices[interval_start]['rightvol'] = [];
}
intervals_prices[interval_start]['leftvol'].push(trade._tradeAmount);
intervals_prices[interval_start]['rightvol'].push(trade._tradeVolume);
if (price) {
const plow = period['low'];
period['period_start'] = interval_start;
period['open'] = period['open'] || price;
period['close'] = price;
period['high'] = price > period['high'] ? price : period['high'];
period['low'] = (plow && price > plow) ? period['low'] : price;
period['avg'] = intervals_prices[interval_start]['rightvol'].reduce((p: number, c: number) => c + p, 0)
/ intervals_prices[interval_start]['leftvol'].reduce((c: number, p: number) => c + p, 0) * 100000000;
period['volume_left'] += trade._tradeAmount;
period['volume_right'] += trade._tradeVolume;
}
}
const hloc: HighLowOpenClose[] = [];
for (const p in intervals) {
if (intervals.hasOwnProperty(p)) {
const period = intervals[p];
hloc.push({
period_start: period['period_start'],
open: this.intToBtc(period['open']),
close: this.intToBtc(period['close']),
high: this.intToBtc(period['high']),
low: this.intToBtc(period['low']),
avg: this.intToBtc(period['avg']),
volume_right: this.intToBtc(period['volume_right']),
volume_left: this.intToBtc(period['volume_left']),
});
}
}
return hloc;
}
private getTradesByCriteria(
market: string | null,
timestamp_to: number,
timestamp_from: number,
trade_id_to: string | undefined,
trade_id_from: string | undefined,
direction: 'buy' | 'sell' | undefined,
sort: string, limit: number,
integerAmounts: boolean = true,
): TradesData[] {
let trade_id_from_ts: number | null = null;
let trade_id_to_ts: number | null = null;
const allCurrencies = this.getCurrencies();
const timestampFromMilli = timestamp_from * 1000;
const timestampToMilli = timestamp_to * 1000;
// note: the offer_id_from/to depends on iterating over trades in
// descending chronological order.
const tradesDataSorted = this.tradesData.slice();
if (sort === 'asc') {
tradesDataSorted.reverse();
}
let matches: TradesData[] = [];
for (const trade of tradesDataSorted) {
if (trade_id_from === trade.offerId) {
trade_id_from_ts = trade.tradeDate;
}
if (trade_id_to === trade.offerId) {
trade_id_to_ts = trade.tradeDate;
}
if (trade_id_to && trade_id_to_ts === null) {
continue;
}
if (trade_id_from && trade_id_from_ts != null && trade_id_from_ts !== trade.tradeDate) {
continue;
}
if (market && market !== trade._market) {
continue;
}
if (timestampFromMilli && timestampFromMilli > trade.tradeDate) {
continue;
}
if (timestampToMilli && timestampToMilli < trade.tradeDate) {
continue;
}
if (direction && direction !== trade.direction.toLowerCase()) {
continue;
}
// Filter out bogus trades with BTC/BTC or XXX/XXX market.
// See github issue: https://github.com/bitsquare/bitsquare/issues/883
const currencyPairs = trade.currencyPair.split('/');
if (currencyPairs[0] === currencyPairs[1]) {
continue;
}
const currencyLeft = allCurrencies[currencyPairs[0]];
const currencyRight = allCurrencies[currencyPairs[1]];
if (!currencyLeft || !currencyRight) {
continue;
}
const tradePrice = trade.primaryMarketTradePrice * Math.pow(10, 8 - currencyRight.precision);
const tradeAmount = trade.primaryMarketTradeAmount * Math.pow(10, 8 - currencyLeft.precision);
const tradeVolume = trade.primaryMarketTradeVolume * Math.pow(10, 8 - currencyRight.precision);
if (integerAmounts) {
trade._tradePrice = tradePrice;
trade._tradeAmount = tradeAmount;
trade._tradeVolume = tradeVolume;
trade._offerAmount = trade.offerAmount;
} else {
trade._tradePriceStr = this.intToBtc(tradePrice);
trade._tradeAmountStr = this.intToBtc(tradeAmount);
trade._tradeVolumeStr = this.intToBtc(tradeVolume);
trade._offerAmountStr = this.intToBtc(trade.offerAmount);
}
matches.push(trade);
if (matches.length >= limit) {
break;
}
}
if ((trade_id_from && !trade_id_from_ts) || (trade_id_to && !trade_id_to_ts)) {
matches = [];
}
return matches;
}
private intervalStart(ts: number, interval: string) {
switch (interval) {
case 'minute':
return (ts - (ts % 60));
case '10_minute':
return (ts - (ts % 600));
case 'half_hour':
return (ts - (ts % 1800));
case 'hour':
return (ts - (ts % 3600));
case 'half_day':
return (ts - (ts % (3600 * 12)));
case 'day':
return datetime.strtotime('midnight today', ts);
case 'week':
return datetime.strtotime('midnight sunday last week', ts);
case 'month':
return datetime.strtotime('midnight first day of this month', ts);
case 'year':
return datetime.strtotime('midnight first day of january', ts);
default:
throw new Error('Unsupported interval: ' + interval);
}
}
private offerDataToOffer(offer: OffsersData): Offer {
return {
offer_id: offer.id,

6
backend/src/api/bisq/markets.ts
View File

@ -19,7 +19,7 @@ class Bisq {
startBisqService(): void {
this.checkForBisqDataFolder();
this.loadBisqDumpFile();
this.startSubDirectoryWatcher();
this.startBisqDirectoryWatcher();
}
private checkForBisqDataFolder() {
@ -29,13 +29,13 @@ class Bisq {
}
}
private startSubDirectoryWatcher() {
private startBisqDirectoryWatcher() {
if (this.subdirectoryWatcher) {
this.subdirectoryWatcher.close();
}
if (!fs.existsSync(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency)) {
console.log(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency + ` doesn't exist. Trying to restart sub directory watcher again in 3 minutes.`);
setTimeout(() => this.startSubDirectoryWatcher(), 180000);
setTimeout(() => this.startBisqDirectoryWatcher(), 180000);
return;
}
let fsWait: NodeJS.Timeout | null = null;

2
backend/src/interfaces.ts
View File

@ -235,5 +235,5 @@ export interface RequiredSpec { [name: string]: RequiredParams; }
interface RequiredParams {
required: boolean;
types: ('@string' | '@number' | string)[];
types: ('@string' | '@number' | '@boolean' | string)[];
}

8
backend/src/routes.ts
View File

@ -347,6 +347,10 @@ class Routes {
required: false,
types: ['@number']
},
'milliseconds': {
required: false,
types: ['@boolean']
},
};
const p = this.parseRequestParameters(req, constraints);
@ -355,7 +359,7 @@ class Routes {
return;
}
const result = bisqMarket.getHloc(p.market, p.interval, p.timestamp_from, p.timestamp_to);
const result = bisqMarket.getHloc(p.market, p.interval, p.timestamp_from, p.timestamp_to, p.milliseconds);
if (result) {
res.json(result);
} else {
@ -399,6 +403,8 @@ class Routes {
final[i] = number;
} else if (params[i].types.indexOf('@string') > -1) {
final[i] = str;
} else if (params[i].types.indexOf('@boolean') > -1) {
final[i] = str === 'true' || str === 'yes';
} else if (params[i].types.indexOf(str) > -1) {
final[i] = str;
} else {