machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #4786 from mempool/simon/remove-bisq

Browse Source 
Removing Bisq
This commit is contained in:
wiz
2024-04-02 15:06:23 +09:00
committed by GitHub
parent ce057bad20 8bec0cacdb
commit 83dbae3a70
128 changed files with 57 additions and 8764 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
backend/mempool-config.sample.json
View File

@@ -97,10 +97,6 @@
"GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
"GEOIP2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
},
"BISQ": {
"ENABLED": false,
"DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db"
},
"LIGHTNING": {
"ENABLED": false,
"BACKEND": "lnd",
@@ -131,9 +127,7 @@
"MEMPOOL_API": "https://mempool.space/api/v1",
"MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1",
"LIQUID_API": "https://liquid.network/api/v1",
"LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1",
"BISQ_URL": "https://bisq.markets/api",
"BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api"
"LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1"
},
"REDIS": {
"ENABLED": false,

8
backend/src/__fixtures__/mempool-config.template.json
View File

@@ -93,10 +93,6 @@
"ENABLED": false,
"TX_PER_SECOND_SAMPLE_PERIOD": 20
},
"BISQ": {
"ENABLED": true,
"DATA_PATH": "__BISQ_DATA_PATH__"
},
"SOCKS5PROXY": {
"ENABLED": true,
"USE_ONION": true,
@@ -109,9 +105,7 @@
"MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
"MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",
"LIQUID_API": "__EXTERNAL_DATA_SERVER_LIQUID_API__",
"LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__",
"BISQ_URL": "__EXTERNAL_DATA_SERVER_BISQ_URL__",
"BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__"
"LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__"
},
"LIGHTNING": {
"ENABLED": true,

8
backend/src/__tests__/config.test.ts
View File

@@ -107,8 +107,6 @@ describe('Mempool Backend Config', () => {
expect(config.STATISTICS).toStrictEqual({ ENABLED: true, TX_PER_SECOND_SAMPLE_PERIOD: 150 });
expect(config.BISQ).toStrictEqual({ ENABLED: false, DATA_PATH: '/bisq/statsnode-data/btc_mainnet/db' });
expect(config.SOCKS5PROXY).toStrictEqual({
ENABLED: false,
USE_ONION: true,
@@ -122,9 +120,7 @@ describe('Mempool Backend Config', () => {
MEMPOOL_API: 'https://mempool.space/api/v1',
MEMPOOL_ONION: 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
LIQUID_API: 'https://liquid.network/api/v1',
LIQUID_ONION: 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1',
BISQ_URL: 'https://bisq.markets/api',
BISQ_ONION: 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
LIQUID_ONION: 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1'
});
expect(config.MAXMIND).toStrictEqual({
@@ -182,8 +178,6 @@ describe('Mempool Backend Config', () => {
expect(config.STATISTICS).toStrictEqual(fixture.STATISTICS);
expect(config.BISQ).toStrictEqual(fixture.BISQ);
expect(config.SOCKS5PROXY).toStrictEqual(fixture.SOCKS5PROXY);
expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER);

381
backend/src/api/bisq/bisq.routes.ts
View File

@@ -1,381 +0,0 @@
import { Application, Request, Response } from 'express';
import config from '../../config';
import { RequiredSpec } from '../../mempool.interfaces';
import bisq from './bisq';
import { MarketsApiError } from './interfaces';
import marketsApi from './markets-api';
class BisqRoutes {
public initRoutes(app: Application) {
app
.get(config.MEMPOOL.API_URL_PREFIX + 'bisq/stats', this.getBisqStats)
.get(config.MEMPOOL.API_URL_PREFIX + 'bisq/tx/:txId', this.getBisqTransaction)
.get(config.MEMPOOL.API_URL_PREFIX + 'bisq/block/:hash', this.getBisqBlock)
.get(config.MEMPOOL.API_URL_PREFIX + 'bisq/blocks/tip/height', this.getBisqTip)
.get(config.MEMPOOL.API_URL_PREFIX + 'bisq/blocks/:index/:length', this.getBisqBlocks)
.get(config.MEMPOOL.API_URL_PREFIX + 'bisq/address/:address', this.getBisqAddress)
.get(config.MEMPOOL.API_URL_PREFIX + 'bisq/txs/:index/:length', this.getBisqTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/currencies', this.getBisqMarketCurrencies.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/depth', this.getBisqMarketDepth.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/hloc', this.getBisqMarketHloc.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/markets', this.getBisqMarketMarkets.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/offers', this.getBisqMarketOffers.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/ticker', this.getBisqMarketTicker.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/trades', this.getBisqMarketTrades.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/volumes', this.getBisqMarketVolumes.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/volumes/7d', this.getBisqMarketVolumes7d.bind(this))
;
}
private getBisqStats(req: Request, res: Response) {
const result = bisq.getStats();
res.json(result);
}
private getBisqTip(req: Request, res: Response) {
const result = bisq.getLatestBlockHeight();
res.type('text/plain');
res.send(result.toString());
}
private getBisqTransaction(req: Request, res: Response) {
const result = bisq.getTransaction(req.params.txId);
if (result) {
res.json(result);
} else {
res.status(404).send('Bisq transaction not found');
}
}
private getBisqTransactions(req: Request, res: Response) {
const types: string[] = [];
req.query.types = req.query.types || [];
if (!Array.isArray(req.query.types)) {
res.status(500).send('Types is not an array');
return;
}
for (const _type in req.query.types) {
if (typeof req.query.types[_type] === 'string') {
types.push(req.query.types[_type].toString());
}
}
const index = parseInt(req.params.index, 10) || 0;
const length = parseInt(req.params.length, 10) > 100 ? 100 : parseInt(req.params.length, 10) || 25;
const [transactions, count] = bisq.getTransactions(index, length, types);
res.header('X-Total-Count', count.toString());
res.json(transactions);
}
private getBisqBlock(req: Request, res: Response) {
const result = bisq.getBlock(req.params.hash);
if (result) {
res.json(result);
} else {
res.status(404).send('Bisq block not found');
}
}
private getBisqBlocks(req: Request, res: Response) {
const index = parseInt(req.params.index, 10) || 0;
const length = parseInt(req.params.length, 10) > 100 ? 100 : parseInt(req.params.length, 10) || 25;
const [transactions, count] = bisq.getBlocks(index, length);
res.header('X-Total-Count', count.toString());
res.json(transactions);
}
private getBisqAddress(req: Request, res: Response) {
const result = bisq.getAddress(req.params.address.substr(1));
if (result) {
res.json(result);
} else {
res.status(404).send('Bisq address not found');
}
}
private getBisqMarketCurrencies(req: Request, res: Response) {
const constraints: RequiredSpec = {
'type': {
required: false,
types: ['crypto', 'fiat', 'all']
},
};
const p = this.parseRequestParameters(req.query, constraints);
if (p.error) {
res.status(400).json(this.getBisqMarketErrorResponse(p.error));
return;
}
const result = marketsApi.getCurrencies(p.type);
if (result) {
res.json(result);
} else {
res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketCurrencies error'));
}
}
private getBisqMarketDepth(req: Request, res: Response) {
const constraints: RequiredSpec = {
'market': {
required: true,
types: ['@string']
},
};
const p = this.parseRequestParameters(req.query, constraints);
if (p.error) {
res.status(400).json(this.getBisqMarketErrorResponse(p.error));
return;
}
const result = marketsApi.getDepth(p.market);
if (result) {
res.json(result);
} else {
res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketDepth error'));
}
}
private getBisqMarketMarkets(req: Request, res: Response) {
const result = marketsApi.getMarkets();
if (result) {
res.json(result);
} else {
res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketMarkets error'));
}
}
private getBisqMarketTrades(req: Request, res: Response) {
const constraints: RequiredSpec = {
'market': {
required: true,
types: ['@string']
},
'timestamp_from': {
required: false,
types: ['@number']
},
'timestamp_to': {
required: false,
types: ['@number']
},
'trade_id_to': {
required: false,
types: ['@string']
},
'trade_id_from': {
required: false,
types: ['@string']
},
'direction': {
required: false,
types: ['buy', 'sell']
},
'limit': {
required: false,
types: ['@number']
},
'sort': {
required: false,
types: ['asc', 'desc']
}
};
const p = this.parseRequestParameters(req.query, constraints);
if (p.error) {
res.status(400).json(this.getBisqMarketErrorResponse(p.error));
return;
}
const result = marketsApi.getTrades(p.market, p.timestamp_from,
p.timestamp_to, p.trade_id_from, p.trade_id_to, p.direction, p.limit, p.sort);
if (result) {
res.json(result);
} else {
res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketTrades error'));
}
}
private getBisqMarketOffers(req: Request, res: Response) {
const constraints: RequiredSpec = {
'market': {
required: true,
types: ['@string']
},
'direction': {
required: false,
types: ['buy', 'sell']
},
};
const p = this.parseRequestParameters(req.query, constraints);
if (p.error) {
res.status(400).json(this.getBisqMarketErrorResponse(p.error));
return;
}
const result = marketsApi.getOffers(p.market, p.direction);
if (result) {
res.json(result);
} else {
res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketOffers error'));
}
}
private getBisqMarketVolumes(req: Request, res: Response) {
const constraints: RequiredSpec = {
'market': {
required: false,
types: ['@string']
},
'interval': {
required: false,
types: ['minute', 'half_hour', 'hour', 'half_day', 'day', 'week', 'month', 'year', 'auto']
},
'timestamp_from': {
required: false,
types: ['@number']
},
'timestamp_to': {
required: false,
types: ['@number']
},
'milliseconds': {
required: false,
types: ['@boolean']
},
'timestamp': {
required: false,
types: ['no', 'yes']
},
};
const p = this.parseRequestParameters(req.query, constraints);
if (p.error) {
res.status(400).json(this.getBisqMarketErrorResponse(p.error));
return;
}
const result = marketsApi.getVolumes(p.market, p.timestamp_from, p.timestamp_to, p.interval, p.milliseconds, p.timestamp);
if (result) {
res.json(result);
} else {
res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketVolumes error'));
}
}
private getBisqMarketHloc(req: Request, res: Response) {
const constraints: RequiredSpec = {
'market': {
required: true,
types: ['@string']
},
'interval': {
required: false,
types: ['minute', 'half_hour', 'hour', 'half_day', 'day', 'week', 'month', 'year', 'auto']
},
'timestamp_from': {
required: false,
types: ['@number']
},
'timestamp_to': {
required: false,
types: ['@number']
},
'milliseconds': {
required: false,
types: ['@boolean']
},
'timestamp': {
required: false,
types: ['no', 'yes']
},
};
const p = this.parseRequestParameters(req.query, constraints);
if (p.error) {
res.status(400).json(this.getBisqMarketErrorResponse(p.error));
return;
}
const result = marketsApi.getHloc(p.market, p.interval, p.timestamp_from, p.timestamp_to, p.milliseconds, p.timestamp);
if (result) {
res.json(result);
} else {
res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketHloc error'));
}
}
private getBisqMarketTicker(req: Request, res: Response) {
const constraints: RequiredSpec = {
'market': {
required: false,
types: ['@string']
},
};
const p = this.parseRequestParameters(req.query, constraints);
if (p.error) {
res.status(400).json(this.getBisqMarketErrorResponse(p.error));
return;
}
const result = marketsApi.getTicker(p.market);
if (result) {
res.json(result);
} else {
res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketTicker error'));
}
}
private getBisqMarketVolumes7d(req: Request, res: Response) {
const result = marketsApi.getVolumesByTime(604800);
if (result) {
res.json(result);
} else {
res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketVolumes7d error'));
}
}
private parseRequestParameters(requestParams: object, params: RequiredSpec): { [name: string]: any; } {
const final = {};
for (const i in params) {
if (params.hasOwnProperty(i)) {
if (params[i].required && requestParams[i] === undefined) {
return { error: i + ' parameter missing'};
}
if (typeof requestParams[i] === 'string') {
const str = (requestParams[i] || '').toString().toLowerCase();
if (params[i].types.indexOf('@number') > -1) {
const number = parseInt((str).toString(), 10);
final[i] = number;
} else if (params[i].types.indexOf('@string') > -1) {
final[i] = str;
} else if (params[i].types.indexOf('@boolean') > -1) {
final[i] = str === 'true' || str === 'yes';
} else if (params[i].types.indexOf(str) > -1) {
final[i] = str;
} else {
return { error: i + ' parameter invalid'};
}
} else if (typeof requestParams[i] === 'number') {
final[i] = requestParams[i];
}
}
}
return final;
}
private getBisqMarketErrorResponse(message: string): MarketsApiError {
return {
'success': 0,
'error': message
};
}
}
export default new BisqRoutes;

359
backend/src/api/bisq/bisq.ts
View File

@@ -1,359 +0,0 @@
import config from '../../config';
import * as fs from 'fs';
import axios, { AxiosResponse } from 'axios';
import * as http from 'http';
import * as https from 'https';
import { SocksProxyAgent } from 'socks-proxy-agent';
import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces';
import { Common } from '../common';
import { BlockExtended } from '../../mempool.interfaces';
import backendInfo from '../backend-info';
import logger from '../../logger';
class Bisq {
private static BLOCKS_JSON_FILE_PATH = config.BISQ.DATA_PATH + '/json/all/blocks.json';
private latestBlockHeight = 0;
private blocks: BisqBlock[] = [];
private allBlocks: BisqBlock[] = [];
private transactions: BisqTransaction[] = [];
private transactionIndex: { [txId: string]: BisqTransaction } = {};
private blockIndex: { [hash: string]: BisqBlock } = {};
private addressIndex: { [address: string]: BisqTransaction[] } = {};
private stats: BisqStats = {
minted: 0,
burnt: 0,
addresses: 0,
unspent_txos: 0,
spent_txos: 0,
};
private price: number = 0;
private priceUpdateCallbackFunction: ((price: number) => void) | undefined;
private topDirectoryWatcher: fs.FSWatcher | undefined;
private subdirectoryWatcher: fs.FSWatcher | undefined;
constructor() {}
startBisqService(): void {
try {
this.checkForBisqDataFolder();
} catch (e) {
logger.info('Retrying to start bisq service in 3 minutes');
setTimeout(this.startBisqService.bind(this), 180000);
return;
}
this.loadBisqDumpFile();
setInterval(this.updatePrice.bind(this), 1000 * 60 * 60);
this.updatePrice();
this.startTopDirectoryWatcher();
this.startSubDirectoryWatcher();
}
handleNewBitcoinBlock(block: BlockExtended): void {
if (block.height - 10 > this.latestBlockHeight && this.latestBlockHeight !== 0) {
logger.warn(`Bitcoin block height (#${block.height}) has diverged from the latest Bisq block height (#${this.latestBlockHeight}). Restarting watchers...`);
this.startTopDirectoryWatcher();
this.startSubDirectoryWatcher();
}
}
getTransaction(txId: string): BisqTransaction | undefined {
return this.transactionIndex[txId];
}
getTransactions(start: number, length: number, types: string[]): [BisqTransaction[], number] {
let transactions = this.transactions;
if (types.length) {
transactions = transactions.filter((tx) => types.indexOf(tx.txType) > -1);
}
return [transactions.slice(start, length + start), transactions.length];
}
getBlock(hash: string): BisqBlock | undefined {
return this.blockIndex[hash];
}
getAddress(hash: string): BisqTransaction[] {
return this.addressIndex[hash];
}
getBlocks(start: number, length: number): [BisqBlock[], number] {
return [this.blocks.slice(start, length + start), this.blocks.length];
}
getStats(): BisqStats {
return this.stats;
}
setPriceCallbackFunction(fn: (price: number) => void) {
this.priceUpdateCallbackFunction = fn;
}
getLatestBlockHeight(): number {
return this.latestBlockHeight;
}
private checkForBisqDataFolder() {
if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
logger.warn(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Make sure Bisq is running and the config is correct before starting the server.`);
throw new Error(`Cannot load BISQ ${Bisq.BLOCKS_JSON_FILE_PATH} file`);
}
}
private startTopDirectoryWatcher() {
if (this.topDirectoryWatcher) {
this.topDirectoryWatcher.close();
}
let fsWait: NodeJS.Timeout | null = null;
this.topDirectoryWatcher = fs.watch(config.BISQ.DATA_PATH + '/json', () => {
if (fsWait) {
clearTimeout(fsWait);
}
if (this.subdirectoryWatcher) {
this.subdirectoryWatcher.close();
}
fsWait = setTimeout(() => {
logger.debug(`Bisq restart detected. Resetting both watchers in 3 minutes.`);
setTimeout(() => {
this.startTopDirectoryWatcher();
this.startSubDirectoryWatcher();
this.loadBisqDumpFile();
}, 180000);
}, 15000);
});
}
private startSubDirectoryWatcher() {
if (this.subdirectoryWatcher) {
this.subdirectoryWatcher.close();
}
if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
logger.warn(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Trying to restart sub directory watcher again in 3 minutes.`);
setTimeout(() => this.startSubDirectoryWatcher(), 180000);
return;
}
let fsWait: NodeJS.Timeout | null = null;
this.subdirectoryWatcher = fs.watch(config.BISQ.DATA_PATH + '/json/all', () => {
if (fsWait) {
clearTimeout(fsWait);
}
fsWait = setTimeout(() => {
logger.debug(`Change detected in the Bisq data folder.`);
this.loadBisqDumpFile();
}, 2000);
});
}
private async updatePrice() {
type axiosOptions = {
headers: {
'User-Agent': string
};
timeout: number;
httpAgent?: http.Agent;
httpsAgent?: https.Agent;
}
const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
const BISQ_URL = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.EXTERNAL_DATA_SERVER.BISQ_ONION : config.EXTERNAL_DATA_SERVER.BISQ_URL;
const isHTTP = (new URL(BISQ_URL).protocol.split(':')[0] === 'http') ? true : false;
const axiosOptions: axiosOptions = {
headers: {
'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
},
timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
};
let retry = 0;
while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
try {
if (config.SOCKS5PROXY.ENABLED) {
const socksOptions: any = {
agentOptions: {
keepAlive: true,
},
hostname: config.SOCKS5PROXY.HOST,
port: config.SOCKS5PROXY.PORT
};
if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
socksOptions.username = config.SOCKS5PROXY.USERNAME;
socksOptions.password = config.SOCKS5PROXY.PASSWORD;
} else {
// Retry with different tor circuits https://stackoverflow.com/a/64960234
socksOptions.username = `circuit${retry}`;
}
// Handle proxy agent for onion addresses
if (isHTTP) {
axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
} else {
axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
}
}
const data: AxiosResponse = await axios.get(`${BISQ_URL}/trades/?market=bsq_btc`, axiosOptions);
if (data.statusText === 'error' || !data.data) {
throw new Error(`Could not fetch data from Bisq market, Error: ${data.status}`);
}
const prices: number[] = [];
data.data.forEach((trade) => {
prices.push(parseFloat(trade.price) * 100000000);
});
prices.sort((a, b) => a - b);
this.price = Common.median(prices);
if (this.priceUpdateCallbackFunction) {
this.priceUpdateCallbackFunction(this.price);
}
logger.debug('Successfully updated Bisq market price');
break;
} catch (e) {
logger.err('Error updating Bisq market price: ' + (e instanceof Error ? e.message : e));
await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
retry++;
}
}
}
private async loadBisqDumpFile(): Promise<void> {
this.allBlocks = [];
try {
await this.loadData();
this.buildIndex();
this.calculateStats();
} catch (e) {
logger.info('Cannot load bisq dump file because: ' + (e instanceof Error ? e.message : e));
}
}
private buildIndex() {
const start = new Date().getTime();
this.transactions = [];
this.transactionIndex = {};
this.addressIndex = {};
this.allBlocks.forEach((block) => {
/* Build block index */
if (!this.blockIndex[block.hash]) {
this.blockIndex[block.hash] = block;
}
/* Build transactions index */
block.txs.forEach((tx) => {
this.transactions.push(tx);
this.transactionIndex[tx.id] = tx;
});
});
/* Build address index */
this.transactions.forEach((tx) => {
tx.inputs.forEach((input) => {
if (!this.addressIndex[input.address]) {
this.addressIndex[input.address] = [];
}
if (this.addressIndex[input.address].indexOf(tx) === -1) {
this.addressIndex[input.address].push(tx);
}
});
tx.outputs.forEach((output) => {
if (!this.addressIndex[output.address]) {
this.addressIndex[output.address] = [];
}
if (this.addressIndex[output.address].indexOf(tx) === -1) {
this.addressIndex[output.address].push(tx);
}
});
});
const time = new Date().getTime() - start;
logger.debug('Bisq data index rebuilt in ' + time + ' ms');
}
private calculateStats() {
let minted = 0;
let burned = 0;
let unspent = 0;
let spent = 0;
this.transactions.forEach((tx) => {
tx.outputs.forEach((output) => {
if (output.opReturn) {
return;
}
if (output.txOutputType === 'GENESIS_OUTPUT' || output.txOutputType === 'ISSUANCE_CANDIDATE_OUTPUT' && output.isVerified) {
minted += output.bsqAmount;
}
if (output.isUnspent) {
unspent++;
} else {
spent++;
}
});
burned += tx['burntFee'];
});
this.stats = {
addresses: Object.keys(this.addressIndex).length,
minted: minted / 100,
burnt: burned / 100,
spent_txos: spent,
unspent_txos: unspent,
};
}
private async loadData(): Promise<any> {
if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
throw new Error(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist`);
}
const readline = require('readline');
const events = require('events');
const rl = readline.createInterface({
input: fs.createReadStream(Bisq.BLOCKS_JSON_FILE_PATH),
crlfDelay: Infinity
});
let blockBuffer = '';
let readingBlock = false;
let lineCount = 1;
const start = new Date().getTime();
logger.debug('Processing Bisq data dump...');
rl.on('line', (line) => {
if (lineCount === 2) {
line = line.replace(' "chainHeight": ', '');
this.latestBlockHeight = parseInt(line, 10);
}
if (line === ' {') {
readingBlock = true;
} else if (line === ' },') {
blockBuffer += '}';
try {
const block: BisqBlock = JSON.parse(blockBuffer);
this.allBlocks.push(block);
readingBlock = false;
blockBuffer = '';
} catch (e) {
logger.debug(blockBuffer);
throw Error(`Unable to parse Bisq data dump at line ${lineCount}` + (e instanceof Error ? e.message : e));
}
}
if (readingBlock === true) {
blockBuffer += line;
}
++lineCount;
});
await events.once(rl, 'close');
this.allBlocks.reverse();
this.blocks = this.allBlocks.filter((block) => block.txs.length > 0);
const time = new Date().getTime() - start;
logger.debug('Bisq dump processed in ' + time + ' ms');
}
}
export default new Bisq();

258
backend/src/api/bisq/interfaces.ts
View File

@@ -1,258 +0,0 @@
export interface BisqBlocks {
chainHeight: number;
blocks: BisqBlock[];
}
export interface BisqBlock {
height: number;
time: number;
hash: string;
previousBlockHash: string;
txs: BisqTransaction[];
}
export interface BisqTransaction {
txVersion: string;
id: string;
blockHeight: number;
blockHash: string;
time: number;
inputs: BisqInput[];
outputs: BisqOutput[];
txType: string;
txTypeDisplayString: string;
burntFee: number;
invalidatedBsq: number;
unlockBlockHeight: number;
}
export interface BisqStats {
minted: number;
burnt: number;
addresses: number;
unspent_txos: number;
spent_txos: number;
}
interface BisqInput {
spendingTxOutputIndex: number;
spendingTxId: string;
bsqAmount: number;
isVerified: boolean;
address: string;
time: number;
}
interface BisqOutput {
txVersion: string;
txId: string;
index: number;
bsqAmount: number;
btcAmount: number;
height: number;
isVerified: boolean;
burntFee: number;
invalidatedBsq: number;
address: string;
scriptPubKey: BisqScriptPubKey;
time: any;
txType: string;
txTypeDisplayString: string;
txOutputType: string;
txOutputTypeDisplayString: string;
lockTime: number;
isUnspent: boolean;
spentInfo: SpentInfo;
opReturn?: string;
}
interface BisqScriptPubKey {
addresses: string[];
asm: string;
hex: string;
reqSigs?: number;
type: string;
}
interface SpentInfo {
height: number;
inputIndex: number;
txId: string;
}
export interface BisqTrade {
direction: string;
price: string;
amount: string;
volume: string;
payment_method: string;
trade_id: string;
trade_date: number;
market?: string;
}
export interface Currencies { [txid: string]: Currency; }
export interface Currency {
code: string;
name: string;
precision: number;
_type: string;
}
export interface Depth { [market: string]: Market; }
interface Market {
'buys': string[];
'sells': string[];
}
export interface HighLowOpenClose {
period_start: number | string;
open: string;
high: string;
low: string;
close: string;
volume_left: string;
volume_right: string;
avg: string;
}
export interface Markets { [txid: string]: Pair; }
interface Pair {
pair: string;
lname: string;
rname: string;
lsymbol: string;
rsymbol: string;
lprecision: number;
rprecision: number;
ltype: string;
rtype: string;
name: string;
}
export interface Offers { [market: string]: OffersMarket; }
interface OffersMarket {
buys: Offer[] | null;
sells: Offer[] | null;
}
export interface OffersData {
direction: string;
currencyCode: string;
minAmount: number;
amount: number;
price: number;
date: number;
useMarketBasedPrice: boolean;
marketPriceMargin: number;
paymentMethod: string;
id: string;
currencyPair: string;
primaryMarketDirection: string;
priceDisplayString: string;
primaryMarketAmountDisplayString: string;
primaryMarketMinAmountDisplayString: string;
primaryMarketVolumeDisplayString: string;
primaryMarketMinVolumeDisplayString: string;
primaryMarketPrice: number;
primaryMarketAmount: number;
primaryMarketMinAmount: number;
primaryMarketVolume: number;
primaryMarketMinVolume: number;
}
export interface Offer {
offer_id: string;
offer_date: number;
direction: string;
min_amount: string;
amount: string;
price: string;
volume: string;
payment_method: string;
offer_fee_txid: any;
}
export interface Tickers { [market: string]: Ticker | null; }
export interface Ticker {
last: string;
high: string;
low: string;
volume_left: string;
volume_right: string;
buy: string | null;
sell: string | null;
}
export interface Trade {
direction: string;
price: string;
amount: string;
volume: string;
payment_method: string;
trade_id: string;
trade_date: number;
}
export interface TradesData {
currency: string;
direction: string;
tradePrice: number;
tradeAmount: number;
tradeDate: number;
paymentMethod: string;
offerDate: number;
useMarketBasedPrice: boolean;
marketPriceMargin: number;
offerAmount: number;
offerMinAmount: number;
offerId: string;
depositTxId?: string;
currencyPair: string;
primaryMarketDirection: string;
primaryMarketTradePrice: number;
primaryMarketTradeAmount: number;
primaryMarketTradeVolume: number;
_market: string;
_tradePriceStr: string;
_tradeAmountStr: string;
_tradeVolumeStr: string;
_offerAmountStr: string;
_tradePrice: number;
_tradeAmount: number;
_tradeVolume: number;
_offerAmount: number;
}
export interface MarketVolume {
period_start: number;
num_trades: number;
volume: string;
}
export interface MarketsApiError {
success: number;
error: string;
}
export type Interval = 'minute' | 'half_hour' | 'hour' | 'half_day' | 'day' | 'week' | 'month' | 'year' | 'auto';
export interface SummarizedIntervals { [market: string]: SummarizedInterval; }
export interface SummarizedInterval {
'period_start': number;
'open': number;
'close': number;
'high': number;
'low': number;
'avg': number;
'volume_right': number;
'volume_left': number;
}

679
backend/src/api/bisq/markets-api.ts
View File

@@ -1,679 +0,0 @@
import { Currencies, OffersData, TradesData, Depth, Currency, Interval, HighLowOpenClose,
Markets, Offers, Offer, BisqTrade, MarketVolume, Tickers, Ticker, SummarizedIntervals, SummarizedInterval } from './interfaces';
const strtotime = require('./strtotime');
class BisqMarketsApi {
private cryptoCurrencyData: Currency[] = [];
private fiatCurrencyData: Currency[] = [];
private activeCryptoCurrencyData: Currency[] = [];
private activeFiatCurrencyData: Currency[] = [];
private offersData: OffersData[] = [];
private tradesData: TradesData[] = [];
private fiatCurrenciesIndexed: { [code: string]: true } = {};
private allCurrenciesIndexed: { [code: string]: Currency } = {};
private tradeDataByMarket: { [market: string]: TradesData[] } = {};
private tickersCache: Ticker | Tickers | null = null;
constructor() { }
setOffersData(offers: OffersData[]) {
this.offersData = offers;
}
setTradesData(trades: TradesData[]) {
this.tradesData = trades;
this.tradeDataByMarket = {};
this.tradesData.forEach((trade) => {
trade._market = trade.currencyPair.toLowerCase().replace('/', '_');
if (!this.tradeDataByMarket[trade._market]) {
this.tradeDataByMarket[trade._market] = [];
}
this.tradeDataByMarket[trade._market].push(trade);
});
}
setCurrencyData(cryptoCurrency: Currency[], fiatCurrency: Currency[], activeCryptoCurrency: Currency[], activeFiatCurrency: Currency[]) {
this.cryptoCurrencyData = cryptoCurrency,
this.fiatCurrencyData = fiatCurrency,
this.activeCryptoCurrencyData = activeCryptoCurrency,
this.activeFiatCurrencyData = activeFiatCurrency;
this.fiatCurrenciesIndexed = {};
this.allCurrenciesIndexed = {};
this.fiatCurrencyData.forEach((currency) => {
currency._type = 'fiat';
this.fiatCurrenciesIndexed[currency.code] = true;
this.allCurrenciesIndexed[currency.code] = currency;
});
this.cryptoCurrencyData.forEach((currency) => {
currency._type = 'crypto';
this.allCurrenciesIndexed[currency.code] = currency;
});
}
updateCache() {
this.tickersCache = null;
this.tickersCache = this.getTicker();
}
getCurrencies(
type: 'crypto' | 'fiat' | 'active' | 'all' = 'all',
): Currencies {
let currencies: Currency[];
switch (type) {
case 'fiat':
currencies = this.fiatCurrencyData;
break;
case 'crypto':
currencies = this.cryptoCurrencyData;
break;
case 'active':
currencies = this.activeCryptoCurrencyData.concat(this.activeFiatCurrencyData);
break;
case 'all':
default:
currencies = this.cryptoCurrencyData.concat(this.fiatCurrencyData);
}
const result = {};
currencies.forEach((currency) => {
result[currency.code] = currency;
});
return result;
}
getDepth(
market: string,
): Depth {
const currencyPair = market.replace('_', '/').toUpperCase();
const buys = this.offersData
.filter((offer) => offer.currencyPair === currencyPair && offer.primaryMarketDirection === 'BUY')
.map((offer) => offer.price)
.sort((a, b) => b - a)
.map((price) => this.intToBtc(price));
const sells = this.offersData
.filter((offer) => offer.currencyPair === currencyPair && offer.primaryMarketDirection === 'SELL')
.map((offer) => offer.price)
.sort((a, b) => a - b)
.map((price) => this.intToBtc(price));
const result = {};
result[market] = {
'buys': buys,
'sells': sells,
};
return result;
}
getOffers(
market: string,
direction?: 'buy' | 'sell',
): Offers {
const currencyPair = market.replace('_', '/').toUpperCase();
let buys: Offer[] | null = null;
let sells: Offer[] | null = null;
if (!direction || direction === 'buy') {
buys = this.offersData
.filter((offer) => offer.currencyPair === currencyPair && offer.primaryMarketDirection === 'BUY')
.sort((a, b) => b.price - a.price)
.map((offer) => this.offerDataToOffer(offer, market));
}
if (!direction || direction === 'sell') {
sells = this.offersData
.filter((offer) => offer.currencyPair === currencyPair && offer.primaryMarketDirection === 'SELL')
.sort((a, b) => a.price - b.price)
.map((offer) => this.offerDataToOffer(offer, market));
}
const result: Offers = {};
result[market] = {
'buys': buys,
'sells': sells,
};
return result;
}
getMarkets(): Markets {
const allCurrencies = this.getCurrencies();
const activeCurrencies = this.getCurrencies('active');
const markets = {};
for (const currency of Object.keys(activeCurrencies)) {
if (allCurrencies[currency].code === 'BTC') {
continue;
}
const isFiat = allCurrencies[currency]._type === 'fiat';
const pmarketname = allCurrencies['BTC']['name'];
const lsymbol = isFiat ? 'BTC' : currency;
const rsymbol = isFiat ? currency : 'BTC';
const lname = isFiat ? pmarketname : allCurrencies[currency].name;
const rname = isFiat ? allCurrencies[currency].name : pmarketname;
const ltype = isFiat ? 'crypto' : allCurrencies[currency]._type;
const rtype = isFiat ? 'fiat' : 'crypto';
const lprecision = 8;
const rprecision = isFiat ? 2 : 8;
const pair = lsymbol.toLowerCase() + '_' + rsymbol.toLowerCase();
markets[pair] = {
'pair': pair,
'lname': lname,
'rname': rname,
'lsymbol': lsymbol,
'rsymbol': rsymbol,
'lprecision': lprecision,
'rprecision': rprecision,
'ltype': ltype,
'rtype': rtype,
'name': lname + '/' + rname,
};
}
return markets;
}
getTrades(
market: string,
timestamp_from?: number,
timestamp_to?: number,
trade_id_from?: string,
trade_id_to?: string,
direction?: 'buy' | 'sell',
limit: number = 100,
sort: 'asc' | 'desc' = 'desc',
): BisqTrade[] {
limit = Math.min(limit, 2000);
const _market = market === 'all' ? undefined : market;
if (!timestamp_from) {
timestamp_from = new Date('2016-01-01').getTime() / 1000;
}
if (!timestamp_to) {
timestamp_to = new Date().getTime() / 1000;
}
const matches = this.getTradesByCriteria(_market, timestamp_to, timestamp_from,
trade_id_to, trade_id_from, direction, sort, limit, false);
if (sort === 'asc') {
matches.sort((a, b) => a.tradeDate - b.tradeDate);
} else {
matches.sort((a, b) => b.tradeDate - a.tradeDate);
}
return matches.map((trade) => {
const bsqTrade: BisqTrade = {
direction: trade.primaryMarketDirection,
price: trade._tradePriceStr,
amount: trade._tradeAmountStr,
volume: trade._tradeVolumeStr,
payment_method: trade.paymentMethod,
trade_id: trade.offerId,
trade_date: trade.tradeDate,
};
if (market === 'all') {
bsqTrade.market = trade._market;
}
return bsqTrade;
});
}
getVolumes(
market?: string,
timestamp_from?: number,
timestamp_to?: number,
interval: Interval = 'auto',
milliseconds?: boolean,
timestamp: 'no' | 'yes' = 'yes',
): MarketVolume[] {
if (milliseconds) {
timestamp_from = timestamp_from ? timestamp_from / 1000 : timestamp_from;
timestamp_to = timestamp_to ? timestamp_to / 1000 : timestamp_to;
}
if (!timestamp_from) {
timestamp_from = new Date('2016-01-01').getTime() / 1000;
}
if (!timestamp_to) {
timestamp_to = new Date().getTime() / 1000;
}
const trades = this.getTradesByCriteria(market, timestamp_to, timestamp_from,
undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER);
if (interval === 'auto') {
const range = timestamp_to - timestamp_from;
interval = this.getIntervalFromRange(range);
}
const intervals: any = {};
const marketVolumes: MarketVolume[] = [];
for (const trade of trades) {
const traded_at = trade['tradeDate'] / 1000;
const interval_start = this.intervalStart(traded_at, interval);
if (!intervals[interval_start]) {
intervals[interval_start] = {
'volume': 0,
'num_trades': 0,
};
}
const period = intervals[interval_start];
period['period_start'] = interval_start;
period['volume'] += this.fiatCurrenciesIndexed[trade.currency] ? trade._tradeAmount : trade._tradeVolume;
period['num_trades']++;
}
for (const p in intervals) {
if (intervals.hasOwnProperty(p)) {
const period = intervals[p];
marketVolumes.push({
period_start: timestamp === 'no' ? new Date(period['period_start'] * 1000).toISOString() : period['period_start'],
num_trades: period['num_trades'],
volume: this.intToBtc(period['volume']),
});
}
}
return marketVolumes;
}
getTicker(
market?: string,
): Tickers | Ticker | null {
if (market) {
return this.getTickerFromMarket(market);
}
if (this.tickersCache) {
return this.tickersCache;
}
const allMarkets = this.getMarkets();
const tickers = {};
for (const m in allMarkets) {
if (allMarkets.hasOwnProperty(m)) {
tickers[allMarkets[m].pair] = this.getTickerFromMarket(allMarkets[m].pair);
}
}
return tickers;
}
getTickerFromMarket(market: string): Ticker | null {
let ticker: Ticker;
const timestamp_from = strtotime('-24 hour');
const timestamp_to = new Date().getTime() / 1000;
const trades = this.getTradesByCriteria(market, timestamp_to, timestamp_from,
undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER);
const periods: SummarizedInterval[] = Object.values(this.getTradesSummarized(trades, timestamp_from));
const allCurrencies = this.getCurrencies();
const currencyRight = allCurrencies[market.split('_')[1].toUpperCase()];
if (periods[0]) {
ticker = {
'last': this.intToBtc(periods[0].close),
'high': this.intToBtc(periods[0].high),
'low': this.intToBtc(periods[0].low),
'volume_left': this.intToBtc(periods[0].volume_left),
'volume_right': this.intToBtc(periods[0].volume_right),
'buy': null,
'sell': null,
};
} else {
const lastTrade = this.tradeDataByMarket[market];
if (!lastTrade) {
return null;
}
const tradePrice = lastTrade[0].primaryMarketTradePrice * Math.pow(10, 8 - currencyRight.precision);
const lastTradePrice = this.intToBtc(tradePrice);
ticker = {
'last': lastTradePrice,
'high': lastTradePrice,
'low': lastTradePrice,
'volume_left': '0',
'volume_right': '0',
'buy': null,
'sell': null,
};
}
const timestampFromMilli = timestamp_from * 1000;
const timestampToMilli = timestamp_to * 1000;
const currencyPair = market.replace('_', '/').toUpperCase();
const offersData = this.offersData.slice().sort((a, b) => a.price - b.price);
const buy = offersData.find((offer) => offer.currencyPair === currencyPair
&& offer.primaryMarketDirection === 'BUY'
&& offer.date >= timestampFromMilli
&& offer.date <= timestampToMilli
);
const sell = offersData.find((offer) => offer.currencyPair === currencyPair
&& offer.primaryMarketDirection === 'SELL'
&& offer.date >= timestampFromMilli
&& offer.date <= timestampToMilli
);
if (buy) {
ticker.buy = this.intToBtc(buy.primaryMarketPrice * Math.pow(10, 8 - currencyRight.precision));
}
if (sell) {
ticker.sell = this.intToBtc(sell.primaryMarketPrice * Math.pow(10, 8 - currencyRight.precision));
}
return ticker;
}
getHloc(
market: string,
interval: Interval = 'auto',
timestamp_from?: number,
timestamp_to?: number,
milliseconds?: boolean,
timestamp: 'no' | 'yes' = 'yes',
): HighLowOpenClose[] {
if (milliseconds) {
timestamp_from = timestamp_from ? timestamp_from / 1000 : timestamp_from;
timestamp_to = timestamp_to ? timestamp_to / 1000 : timestamp_to;
}
if (!timestamp_from) {
timestamp_from = new Date('2016-01-01').getTime() / 1000;
}
if (!timestamp_to) {
timestamp_to = new Date().getTime() / 1000;
}
const trades = this.getTradesByCriteria(market, timestamp_to, timestamp_from,
undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER);
if (interval === 'auto') {
const range = timestamp_to - timestamp_from;
interval = this.getIntervalFromRange(range);
}
const intervals = this.getTradesSummarized(trades, timestamp_from, interval);
const hloc: HighLowOpenClose[] = [];
for (const p in intervals) {
if (intervals.hasOwnProperty(p)) {
const period = intervals[p];
hloc.push({
period_start: timestamp === 'no' ? new Date(period['period_start'] * 1000).toISOString() : period['period_start'],
open: this.intToBtc(period['open']),
close: this.intToBtc(period['close']),
high: this.intToBtc(period['high']),
low: this.intToBtc(period['low']),
avg: this.intToBtc(period['avg']),
volume_right: this.intToBtc(period['volume_right']),
volume_left: this.intToBtc(period['volume_left']),
});
}
}
return hloc;
}
private getIntervalFromRange(range: number): Interval {
// two days range loads minute data
if (range <= 3600) {
// up to one hour range loads minutely data
return 'minute';
} else if (range <= 1 * 24 * 3600) {
// up to one day range loads half-hourly data
return 'half_hour';
} else if (range <= 3 * 24 * 3600) {
// up to 3 day range loads hourly data
return 'hour';
} else if (range <= 7 * 24 * 3600) {
// up to 7 day range loads half-daily data
return 'half_day';
} else if (range <= 60 * 24 * 3600) {
// up to 2 month range loads daily data
return 'day';
} else if (range <= 12 * 31 * 24 * 3600) {
// up to one year range loads weekly data
return 'week';
} else if (range <= 12 * 31 * 24 * 3600) {
// up to 5 year range loads monthly data
return 'month';
} else {
// greater range loads yearly data
return 'year';
}
}
getVolumesByTime(time: number): MarketVolume[] {
const timestamp_from = new Date().getTime() / 1000 - time;
const timestamp_to = new Date().getTime() / 1000;
const trades = this.getTradesByCriteria(undefined, timestamp_to, timestamp_from,
undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER);
const markets: any = {};
for (const trade of trades) {
if (!markets[trade._market]) {
markets[trade._market] = {
'volume': 0,
'num_trades': 0,
};
}
markets[trade._market]['volume'] += this.fiatCurrenciesIndexed[trade.currency] ? trade._tradeAmount : trade._tradeVolume;
markets[trade._market]['num_trades']++;
}
return markets;
}
private getTradesSummarized(trades: TradesData[], timestamp_from: number, interval?: string): SummarizedIntervals {
const intervals: any = {};
const intervals_prices: any = {};
for (const trade of trades) {
const traded_at = trade.tradeDate / 1000;
const interval_start = !interval ? timestamp_from : this.intervalStart(traded_at, interval);
if (!intervals[interval_start]) {
intervals[interval_start] = {
'open': 0,
'close': 0,
'high': 0,
'low': 0,
'avg': 0,
'volume_right': 0,
'volume_left': 0,
};
intervals_prices[interval_start] = [];
}
const period = intervals[interval_start];
const price = trade._tradePrice;
if (!intervals_prices[interval_start]['leftvol']) {
intervals_prices[interval_start]['leftvol'] = [];
}
if (!intervals_prices[interval_start]['rightvol']) {
intervals_prices[interval_start]['rightvol'] = [];
}
intervals_prices[interval_start]['leftvol'].push(trade._tradeAmount);
intervals_prices[interval_start]['rightvol'].push(trade._tradeVolume);
if (price) {
const plow = period['low'];
period['period_start'] = interval_start;
period['open'] = period['open'] || price;
period['close'] = price;
period['high'] = price > period['high'] ? price : period['high'];
period['low'] = (plow && price > plow) ? period['low'] : price;
period['avg'] = intervals_prices[interval_start]['rightvol'].reduce((p: number, c: number) => c + p, 0)
/ intervals_prices[interval_start]['leftvol'].reduce((c: number, p: number) => c + p, 0) * 100000000;
period['volume_left'] += trade._tradeAmount;
period['volume_right'] += trade._tradeVolume;
}
}
return intervals;
}
private getTradesByCriteria(
market: string | undefined,
timestamp_to: number,
timestamp_from: number,
trade_id_to: string | undefined,
trade_id_from: string | undefined,
direction: 'buy' | 'sell' | undefined,
sort: string,
limit: number,
integerAmounts: boolean = true,
): TradesData[] {
let trade_id_from_ts: number | null = null;
let trade_id_to_ts: number | null = null;
const allCurrencies = this.getCurrencies();
const timestampFromMilli = timestamp_from * 1000;
const timestampToMilli = timestamp_to * 1000;
// note: the offer_id_from/to depends on iterating over trades in
// descending chronological order.
const tradesDataSorted = this.tradesData.slice();
if (sort === 'asc') {
tradesDataSorted.reverse();
}
let matches: TradesData[] = [];
for (const trade of tradesDataSorted) {
if (trade_id_from === trade.offerId) {
trade_id_from_ts = trade.tradeDate;
}
if (trade_id_to === trade.offerId) {
trade_id_to_ts = trade.tradeDate;
}
if (trade_id_to && trade_id_to_ts === null) {
continue;
}
if (trade_id_from && trade_id_from_ts != null && trade_id_from_ts !== trade.tradeDate) {
continue;
}
if (market && market !== trade._market) {
continue;
}
if (timestampFromMilli && timestampFromMilli > trade.tradeDate) {
continue;
}
if (timestampToMilli && timestampToMilli < trade.tradeDate) {
continue;
}
if (direction && direction !== trade.direction.toLowerCase()) {
continue;
}
// Filter out bogus trades with BTC/BTC or XXX/XXX market.
// See github issue: https://github.com/bitsquare/bitsquare/issues/883
const currencyPairs = trade.currencyPair.split('/');
if (currencyPairs[0] === currencyPairs[1]) {
continue;
}
const currencyLeft = allCurrencies[currencyPairs[0]];
const currencyRight = allCurrencies[currencyPairs[1]];
if (!currencyLeft || !currencyRight) {
continue;
}
const tradePrice = trade.primaryMarketTradePrice * Math.pow(10, 8 - currencyRight.precision);
const tradeAmount = trade.primaryMarketTradeAmount * Math.pow(10, 8 - currencyLeft.precision);
const tradeVolume = trade.primaryMarketTradeVolume * Math.pow(10, 8 - currencyRight.precision);
if (integerAmounts) {
trade._tradePrice = tradePrice;
trade._tradeAmount = tradeAmount;
trade._tradeVolume = tradeVolume;
trade._offerAmount = trade.offerAmount;
} else {
trade._tradePriceStr = this.intToBtc(tradePrice);
trade._tradeAmountStr = this.intToBtc(tradeAmount);
trade._tradeVolumeStr = this.intToBtc(tradeVolume);
trade._offerAmountStr = this.intToBtc(trade.offerAmount);
}
matches.push(trade);
if (matches.length >= limit) {
break;
}
}
if ((trade_id_from && !trade_id_from_ts) || (trade_id_to && !trade_id_to_ts)) {
matches = [];
}
return matches;
}
private intervalStart(ts: number, interval: string): number {
switch (interval) {
case 'minute':
return (ts - (ts % 60));
case '10_minute':
return (ts - (ts % 600));
case 'half_hour':
return (ts - (ts % 1800));
case 'hour':
return (ts - (ts % 3600));
case 'half_day':
return (ts - (ts % (3600 * 12)));
case 'day':
return strtotime('midnight today', ts);
case 'week':
return strtotime('midnight sunday last week', ts);
case 'month':
return strtotime('midnight first day of this month', ts);
case 'year':
return strtotime('midnight first day of january', ts);
default:
throw new Error('Unsupported interval');
}
}
private offerDataToOffer(offer: OffersData, market: string): Offer {
const currencyPairs = market.split('_');
const currencyRight = this.allCurrenciesIndexed[currencyPairs[1].toUpperCase()];
const currencyLeft = this.allCurrenciesIndexed[currencyPairs[0].toUpperCase()];
const price = offer['primaryMarketPrice'] * Math.pow( 10, 8 - currencyRight['precision']);
const amount = offer['primaryMarketAmount'] * Math.pow( 10, 8 - currencyLeft['precision']);
const volume = offer['primaryMarketVolume'] * Math.pow( 10, 8 - currencyRight['precision']);
return {
offer_id: offer.id,
offer_date: offer.date,
direction: offer.primaryMarketDirection,
min_amount: this.intToBtc(offer.minAmount),
amount: this.intToBtc(amount),
price: this.intToBtc(price),
volume: this.intToBtc(volume),
payment_method: offer.paymentMethod,
offer_fee_txid: null,
};
}
private intToBtc(val: number): string {
return (val / 100000000).toFixed(8);
}
}
export default new BisqMarketsApi();

137
backend/src/api/bisq/markets.ts
View File

@@ -1,137 +0,0 @@
import config from '../../config';
import * as fs from 'fs';
import { OffersData as OffersData, TradesData, Currency } from './interfaces';
import bisqMarket from './markets-api';
import logger from '../../logger';
class Bisq {
private static FOLDER_WATCH_CHANGE_DETECTION_DEBOUNCE = 4000;
private static MARKET_JSON_PATH = config.BISQ.DATA_PATH;
private static MARKET_JSON_FILE_PATHS = {
activeCryptoCurrency: '/active_crypto_currency_list.json',
activeFiatCurrency: '/active_fiat_currency_list.json',
cryptoCurrency: '/crypto_currency_list.json',
fiatCurrency: '/fiat_currency_list.json',
offers: '/offers_statistics.json',
trades: '/trade_statistics.json',
};
private cryptoCurrencyLastMtime = new Date('2016-01-01');
private fiatCurrencyLastMtime = new Date('2016-01-01');
private offersLastMtime = new Date('2016-01-01');
private tradesLastMtime = new Date('2016-01-01');
private subdirectoryWatcher: fs.FSWatcher | undefined;
constructor() {}
startBisqService(): void {
try {
this.checkForBisqDataFolder();
} catch (e) {
logger.info('Retrying to start bisq service (markets) in 3 minutes');
setTimeout(this.startBisqService.bind(this), 180000);
return;
}
this.loadBisqDumpFile();
this.startBisqDirectoryWatcher();
}
private checkForBisqDataFolder() {
if (!fs.existsSync(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency)) {
logger.err(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency + ` doesn't exist. Make sure Bisq is running and the config is correct before starting the server.`);
throw new Error(`Cannot load BISQ ${Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency} file`);
}
}
private startBisqDirectoryWatcher() {
if (this.subdirectoryWatcher) {
this.subdirectoryWatcher.close();
}
if (!fs.existsSync(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency)) {
logger.warn(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency + ` doesn't exist. Trying to restart sub directory watcher again in 3 minutes.`);
setTimeout(() => this.startBisqDirectoryWatcher(), 180000);
return;
}
let fsWait: NodeJS.Timeout | null = null;
this.subdirectoryWatcher = fs.watch(Bisq.MARKET_JSON_PATH, () => {
if (fsWait) {
clearTimeout(fsWait);
}
fsWait = setTimeout(() => {
logger.debug(`Change detected in the Bisq market data folder.`);
this.loadBisqDumpFile();
}, Bisq.FOLDER_WATCH_CHANGE_DETECTION_DEBOUNCE);
});
}
private async loadBisqDumpFile(): Promise<void> {
const start = new Date().getTime();
try {
let marketsDataUpdated = false;
const cryptoMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency);
const fiatMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.fiatCurrency);
if (cryptoMtime > this.cryptoCurrencyLastMtime || fiatMtime > this.fiatCurrencyLastMtime) {
const cryptoCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency);
const fiatCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.fiatCurrency);
const activeCryptoCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.activeCryptoCurrency);
const activeFiatCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.activeFiatCurrency);
logger.debug('Updating Bisq Market Currency Data');
bisqMarket.setCurrencyData(cryptoCurrencyData, fiatCurrencyData, activeCryptoCurrencyData, activeFiatCurrencyData);
if (cryptoMtime > this.cryptoCurrencyLastMtime) {
this.cryptoCurrencyLastMtime = cryptoMtime;
}
if (fiatMtime > this.fiatCurrencyLastMtime) {
this.fiatCurrencyLastMtime = fiatMtime;
}
marketsDataUpdated = true;
}
const offersMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.offers);
if (offersMtime > this.offersLastMtime) {
const offersData = await this.loadData<OffersData[]>(Bisq.MARKET_JSON_FILE_PATHS.offers);
logger.debug('Updating Bisq Market Offers Data');
bisqMarket.setOffersData(offersData);
this.offersLastMtime = offersMtime;
marketsDataUpdated = true;
}
const tradesMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.trades);
if (tradesMtime > this.tradesLastMtime) {
const tradesData = await this.loadData<TradesData[]>(Bisq.MARKET_JSON_FILE_PATHS.trades);
logger.debug('Updating Bisq Market Trades Data');
bisqMarket.setTradesData(tradesData);
this.tradesLastMtime = tradesMtime;
marketsDataUpdated = true;
}
if (marketsDataUpdated) {
bisqMarket.updateCache();
const time = new Date().getTime() - start;
logger.debug('Bisq market data updated in ' + time + ' ms');
}
} catch (e) {
logger.err('loadBisqMarketDataDumpFile() error.' + (e instanceof Error ? e.message : e));
}
}
private getFileMtime(path: string): Date {
const stats = fs.statSync(Bisq.MARKET_JSON_PATH + path);
return stats.mtime;
}
private loadData<T>(path: string): Promise<T> {
return new Promise((resolve, reject) => {
fs.readFile(Bisq.MARKET_JSON_PATH + path, 'utf8', (err, data) => {
if (err) {
reject(err);
}
try {
const parsedData = JSON.parse(data);
resolve(parsedData);
} catch (e) {
reject('JSON parse error (' + path + ')');
}
});
});
}
}
export default new Bisq();

1375
backend/src/api/bisq/strtotime.ts
View File

File diff suppressed because it is too large Load Diff

4
backend/src/api/bitcoin/bitcoin.routes.ts
View File

@@ -418,7 +418,7 @@ class BitcoinRoutes {
const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(await blocks.$getBlocks(height, 15));
} else { // Liquid, Bisq
} else { // Liquid
return await this.getLegacyBlocks(req, res);
}
} catch (e) {
@@ -428,7 +428,7 @@ class BitcoinRoutes {
private async getBlocksByBulk(req: Request, res: Response) {
try {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid - Not implemented
return res.status(404).send(`This API is only available for Bitcoin networks`);
}
if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) {

8
backend/src/api/websocket-handler.ts
View File

@@ -360,14 +360,6 @@ class WebsocketHandler {
client['track-donation'] = parsedMessage['track-donation'];
}
if (parsedMessage['track-bisq-market']) {
if (/^[a-z]{3}_[a-z]{3}$/.test(parsedMessage['track-bisq-market'])) {
client['track-bisq-market'] = parsedMessage['track-bisq-market'];
} else {
client['track-bisq-market'] = null;
}
}
if (Object.keys(response).length) {
client.send(this.serializeResponse(response));
}

16
backend/src/config.ts
View File

@@ -116,10 +116,6 @@ interface IConfig {
ENABLED: boolean;
TX_PER_SECOND_SAMPLE_PERIOD: number;
};
BISQ: {
ENABLED: boolean;
DATA_PATH: string;
};
SOCKS5PROXY: {
ENABLED: boolean;
USE_ONION: boolean;
@@ -133,8 +129,6 @@ interface IConfig {
MEMPOOL_ONION: string;
LIQUID_API: string;
LIQUID_ONION: string;
BISQ_URL: string;
BISQ_ONION: string;
};
MAXMIND: {
ENABLED: boolean;
@@ -258,10 +252,6 @@ const defaults: IConfig = {
'ENABLED': true,
'TX_PER_SECOND_SAMPLE_PERIOD': 150
},
'BISQ': {
'ENABLED': false,
'DATA_PATH': '/bisq/statsnode-data/btc_mainnet/db'
},
'LIGHTNING': {
'ENABLED': false,
'BACKEND': 'lnd',
@@ -293,9 +283,7 @@ const defaults: IConfig = {
'MEMPOOL_API': 'https://mempool.space/api/v1',
'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
'LIQUID_API': 'https://liquid.network/api/v1',
'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1',
'BISQ_URL': 'https://bisq.markets/api',
'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1'
},
'MAXMIND': {
'ENABLED': false,
@@ -333,7 +321,6 @@ class Config implements IConfig {
DATABASE: IConfig['DATABASE'];
SYSLOG: IConfig['SYSLOG'];
STATISTICS: IConfig['STATISTICS'];
BISQ: IConfig['BISQ'];
LIGHTNING: IConfig['LIGHTNING'];
LND: IConfig['LND'];
CLIGHTNING: IConfig['CLIGHTNING'];
@@ -355,7 +342,6 @@ class Config implements IConfig {
this.DATABASE = configs.DATABASE;
this.SYSLOG = configs.SYSLOG;
this.STATISTICS = configs.STATISTICS;
this.BISQ = configs.BISQ;
this.LIGHTNING = configs.LIGHTNING;
this.LND = configs.LND;
this.CLIGHTNING = configs.CLIGHTNING;

13
backend/src/index.ts
View File

@@ -11,8 +11,6 @@ import memPool from './api/mempool';
import diskCache from './api/disk-cache';
import statistics from './api/statistics/statistics';
import websocketHandler from './api/websocket-handler';
import bisq from './api/bisq/bisq';
import bisqMarkets from './api/bisq/markets';
import logger from './logger';
import backendInfo from './api/backend-info';
import loadingIndicators from './api/loading-indicators';
@@ -32,7 +30,6 @@ import networkSyncService from './tasks/lightning/network-sync.service';
import statisticsRoutes from './api/statistics/statistics.routes';
import pricesRoutes from './api/prices/prices.routes';
import miningRoutes from './api/mining/mining-routes';
import bisqRoutes from './api/bisq/bisq.routes';
import liquidRoutes from './api/liquid/liquid.routes';
import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
@@ -182,13 +179,6 @@ class Server {
setInterval(() => { this.healthCheck(); }, 2500);
if (config.BISQ.ENABLED) {
bisq.startBisqService();
bisq.setPriceCallbackFunction((price) => websocketHandler.setExtraInitData('bsq-price', price));
blocks.setNewBlockCallback(bisq.handleNewBitcoinBlock.bind(bisq));
bisqMarkets.startBisqService();
}
if (config.LIGHTNING.ENABLED) {
this.$runLightningBackend();
}
@@ -307,9 +297,6 @@ class Server {
if (Common.indexingEnabled() && config.MEMPOOL.ENABLED) {
miningRoutes.initRoutes(this.app);
}
if (config.BISQ.ENABLED) {
bisqRoutes.initRoutes(this.app);
}
if (Common.isLiquid()) {
liquidRoutes.initRoutes(this.app);
}

3
backend/src/logger.ts
View File

@@ -86,9 +86,6 @@ class Logger {
if (config.LIGHTNING.ENABLED) {
return config.MEMPOOL.NETWORK === 'mainnet' ? 'lightning' : `${config.MEMPOOL.NETWORK}-lightning`;
}
if (config.BISQ.ENABLED) {
return 'bisq';
}
if (config.MEMPOOL.NETWORK && config.MEMPOOL.NETWORK !== 'mainnet') {
return config.MEMPOOL.NETWORK;
}

1
backend/src/mempool.interfaces.ts
View File

@@ -435,7 +435,6 @@ export interface WebsocketResponse {
'track-tx': string;
'track-address': string;
'watch-mempool': boolean;
'track-bisq-market': string;
}
export interface VbytesPerSecond {