477 lines
18 KiB
TypeScript
477 lines
18 KiB
TypeScript
import * as fs from 'fs';
|
|
import path from 'path';
|
|
import config from '../config';
|
|
import logger from '../logger';
|
|
import PricesRepository, { ApiPrice, MAX_PRICES } from '../repositories/PricesRepository';
|
|
import BitfinexApi from './price-feeds/bitfinex-api';
|
|
import BitflyerApi from './price-feeds/bitflyer-api';
|
|
import CoinbaseApi from './price-feeds/coinbase-api';
|
|
import GeminiApi from './price-feeds/gemini-api';
|
|
import KrakenApi from './price-feeds/kraken-api';
|
|
import FreeCurrencyApi from './price-feeds/free-currency-api';
|
|
|
|
export interface PriceFeed {
|
|
name: string;
|
|
url: string;
|
|
urlHist: string;
|
|
currencies: string[];
|
|
|
|
$fetchPrice(currency): Promise<number>;
|
|
$fetchRecentPrice(currencies: string[], type: string): Promise<PriceHistory>;
|
|
}
|
|
|
|
export interface PriceHistory {
|
|
[timestamp: number]: ApiPrice;
|
|
}
|
|
|
|
export interface ConversionFeed {
|
|
$getQuota(): Promise<any>;
|
|
$fetchLatestConversionRates(): Promise<ConversionRates>;
|
|
$fetchConversionRates(date: string): Promise<ConversionRates>;
|
|
}
|
|
|
|
export interface ConversionRates {
|
|
[currency: string]: number
|
|
}
|
|
|
|
function getMedian(arr: number[]): number {
|
|
const sortedArr = arr.slice().sort((a, b) => a - b);
|
|
const mid = Math.floor(sortedArr.length / 2);
|
|
return sortedArr.length % 2 !== 0
|
|
? sortedArr[mid]
|
|
: (sortedArr[mid - 1] + sortedArr[mid]) / 2;
|
|
}
|
|
|
|
class PriceUpdater {
|
|
public historyInserted = false;
|
|
private additionalCurrenciesHistoryInserted = false;
|
|
private additionalCurrenciesHistoryRunning = false;
|
|
private lastFailedHistoricalRun = 0;
|
|
private timeBetweenUpdatesMs = 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR;
|
|
private cyclePosition = -1;
|
|
private firstRun = true;
|
|
private lastTime = -1;
|
|
private lastHistoricalRun = 0;
|
|
private running = false;
|
|
private feeds: PriceFeed[] = [];
|
|
private currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
|
|
private latestPrices: ApiPrice;
|
|
private currencyConversionFeed: ConversionFeed | undefined;
|
|
private newCurrencies: string[] = ['BGN', 'BRL', 'CNY', 'CZK', 'DKK', 'HKD', 'HRK', 'HUF', 'IDR', 'ILS', 'INR', 'ISK', 'KRW', 'MXN', 'MYR', 'NOK', 'NZD', 'PHP', 'PLN', 'RON', 'RUB', 'SEK', 'SGD', 'THB', 'TRY', 'ZAR'];
|
|
private lastTimeConversionsRatesFetched: number = 0;
|
|
private latestConversionsRatesFromFeed: ConversionRates = {};
|
|
private ratesChangedCallback: ((rates: ApiPrice) => void) | undefined;
|
|
|
|
constructor() {
|
|
this.latestPrices = this.getEmptyPricesObj();
|
|
|
|
this.feeds.push(new BitflyerApi()); // Does not have historical endpoint
|
|
this.feeds.push(new KrakenApi());
|
|
this.feeds.push(new CoinbaseApi());
|
|
this.feeds.push(new BitfinexApi());
|
|
this.feeds.push(new GeminiApi());
|
|
|
|
this.currencyConversionFeed = new FreeCurrencyApi(config.FIAT_PRICE.API_KEY);
|
|
this.setCyclePosition();
|
|
}
|
|
|
|
public getLatestPrices(): ApiPrice {
|
|
return this.latestPrices;
|
|
}
|
|
|
|
public getEmptyPricesObj(): ApiPrice {
|
|
return {
|
|
time: 0,
|
|
USD: -1,
|
|
EUR: -1,
|
|
GBP: -1,
|
|
CAD: -1,
|
|
CHF: -1,
|
|
AUD: -1,
|
|
JPY: -1,
|
|
BGN: -1,
|
|
BRL: -1,
|
|
CNY: -1,
|
|
CZK: -1,
|
|
DKK: -1,
|
|
HKD: -1,
|
|
HRK: -1,
|
|
HUF: -1,
|
|
IDR: -1,
|
|
ILS: -1,
|
|
INR: -1,
|
|
ISK: -1,
|
|
KRW: -1,
|
|
MXN: -1,
|
|
MYR: -1,
|
|
NOK: -1,
|
|
NZD: -1,
|
|
PHP: -1,
|
|
PLN: -1,
|
|
RON: -1,
|
|
RUB: -1,
|
|
SEK: -1,
|
|
SGD: -1,
|
|
THB: -1,
|
|
TRY: -1,
|
|
ZAR: -1,
|
|
};
|
|
}
|
|
|
|
public setRatesChangedCallback(fn: (rates: ApiPrice) => void): void {
|
|
this.ratesChangedCallback = fn;
|
|
}
|
|
|
|
/**
|
|
* We execute this function before the websocket initialization since
|
|
* the websocket init is not done asyncronously
|
|
*/
|
|
public async $initializeLatestPriceWithDb(): Promise<void> {
|
|
this.latestPrices = await PricesRepository.$getLatestConversionRates();
|
|
}
|
|
|
|
public async $run(): Promise<void> {
|
|
if (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'testnet') {
|
|
// Coins have no value on testnet/signet, so we want to always show 0
|
|
return;
|
|
}
|
|
|
|
if (this.running === true) {
|
|
return;
|
|
}
|
|
this.running = true;
|
|
|
|
if ((Math.round(new Date().getTime() / 1000) - this.lastHistoricalRun) > 3600 * 24) {
|
|
// Once a day, look for missing prices (could happen due to network connectivity issues)
|
|
this.historyInserted = false;
|
|
this.additionalCurrenciesHistoryInserted = false;
|
|
}
|
|
|
|
if (this.lastFailedHistoricalRun > 0 && (Math.round(new Date().getTime() / 1000) - this.lastFailedHistoricalRun) > 60) {
|
|
// If the last attempt to insert missing prices failed, we try again after 60 seconds
|
|
this.additionalCurrenciesHistoryInserted = false;
|
|
}
|
|
|
|
if (config.FIAT_PRICE.API_KEY && this.currencyConversionFeed && (Math.round(new Date().getTime() / 1000) - this.lastTimeConversionsRatesFetched) > 3600 * 24) {
|
|
// Once a day, fetch conversion rates from api: we don't need more granularity for fiat currencies and have a limited number of requests
|
|
try {
|
|
this.latestConversionsRatesFromFeed = await this.currencyConversionFeed.$fetchLatestConversionRates();
|
|
this.lastTimeConversionsRatesFetched = Math.round(new Date().getTime() / 1000);
|
|
logger.debug(`Fetched currencies conversion rates from external API: ${JSON.stringify(this.latestConversionsRatesFromFeed)}`);
|
|
} catch (e) {
|
|
logger.err(`Cannot fetch conversion rates from the API. Reason: ${(e instanceof Error ? e.message : e)}`);
|
|
}
|
|
}
|
|
|
|
try {
|
|
await this.$updatePrice();
|
|
if (this.historyInserted === false && config.DATABASE.ENABLED === true) {
|
|
await this.$insertHistoricalPrices();
|
|
}
|
|
if (this.additionalCurrenciesHistoryInserted === false && config.DATABASE.ENABLED === true && config.FIAT_PRICE.API_KEY && !this.additionalCurrenciesHistoryRunning) {
|
|
await this.$insertMissingAdditionalPrices();
|
|
}
|
|
|
|
} catch (e: any) {
|
|
logger.err(`Cannot save BTC prices in db. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
|
|
}
|
|
|
|
this.running = false;
|
|
}
|
|
|
|
private getMillisecondsSinceBeginningOfHour(): number {
|
|
const now = new Date();
|
|
const beginningOfHour = new Date(now);
|
|
beginningOfHour.setMinutes(0, 0, 0);
|
|
return now.getTime() - beginningOfHour.getTime();
|
|
}
|
|
|
|
private setCyclePosition(): void {
|
|
const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour();
|
|
for (let i = 0; i < config.MEMPOOL.PRICE_UPDATES_PER_HOUR; i++) {
|
|
if (this.timeBetweenUpdatesMs * i > millisecondsSinceBeginningOfHour) {
|
|
this.cyclePosition = i;
|
|
return;
|
|
}
|
|
}
|
|
this.cyclePosition = config.MEMPOOL.PRICE_UPDATES_PER_HOUR;
|
|
}
|
|
|
|
/**
|
|
* Fetch last BTC price from exchanges, average them, and save it in the database once every hour
|
|
*/
|
|
private async $updatePrice(): Promise<void> {
|
|
let forceUpdate = false;
|
|
if (this.firstRun === true && config.DATABASE.ENABLED === true) {
|
|
const lastUpdate = await PricesRepository.$getLatestPriceTime();
|
|
if (new Date().getTime() / 1000 - lastUpdate > this.timeBetweenUpdatesMs / 1000) {
|
|
forceUpdate = true;
|
|
}
|
|
this.firstRun = false;
|
|
}
|
|
|
|
const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour();
|
|
|
|
// Reset the cycle on new hour
|
|
if (this.lastTime > millisecondsSinceBeginningOfHour) {
|
|
this.cyclePosition = 0;
|
|
}
|
|
this.lastTime = millisecondsSinceBeginningOfHour;
|
|
if (millisecondsSinceBeginningOfHour < this.timeBetweenUpdatesMs * this.cyclePosition && !forceUpdate && this.cyclePosition !== 0) {
|
|
return;
|
|
}
|
|
|
|
for (const currency of this.currencies) {
|
|
let prices: number[] = [];
|
|
|
|
for (const feed of this.feeds) {
|
|
// Fetch prices from API which supports `currency`
|
|
if (feed.currencies.includes(currency)) {
|
|
try {
|
|
const price = await feed.$fetchPrice(currency);
|
|
if (price > -1 && price < MAX_PRICES[currency]) {
|
|
prices.push(price);
|
|
}
|
|
logger.debug(`${feed.name} BTC/${currency} price: ${price}`, logger.tags.mining);
|
|
} catch (e) {
|
|
logger.debug(`Could not fetch BTC/${currency} price at ${feed.name}. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
|
|
}
|
|
}
|
|
}
|
|
if (prices.length === 1) {
|
|
logger.debug(`Only ${prices.length} feed available for BTC/${currency} price`, logger.tags.mining);
|
|
}
|
|
|
|
// Compute average price, non weighted
|
|
prices = prices.filter(price => price > 0);
|
|
if (prices.length === 0) {
|
|
this.latestPrices[currency] = -1;
|
|
} else {
|
|
this.latestPrices[currency] = Math.round(getMedian(prices));
|
|
}
|
|
}
|
|
|
|
if (config.FIAT_PRICE.API_KEY && this.latestPrices.USD > 0 && Object.keys(this.latestConversionsRatesFromFeed).length > 0) {
|
|
for (const conversionCurrency of this.newCurrencies) {
|
|
if (this.latestConversionsRatesFromFeed[conversionCurrency] > 0 && this.latestPrices.USD * this.latestConversionsRatesFromFeed[conversionCurrency] < MAX_PRICES[conversionCurrency]) {
|
|
this.latestPrices[conversionCurrency] = Math.round(this.latestPrices.USD * this.latestConversionsRatesFromFeed[conversionCurrency]);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (config.DATABASE.ENABLED === true && this.cyclePosition === 0) {
|
|
// Save everything in db
|
|
try {
|
|
const p = 60 * 60 * 1000; // milliseconds in an hour
|
|
const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042
|
|
await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices);
|
|
} catch (e) {
|
|
logger.err(`Cannot save latest prices into db. Trying again in 5 minutes. Reason: ${(e instanceof Error ? e.message : e)}`);
|
|
}
|
|
}
|
|
|
|
this.latestPrices.time = Math.round(new Date().getTime() / 1000);
|
|
logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
|
|
|
|
if (this.ratesChangedCallback) {
|
|
this.ratesChangedCallback(this.latestPrices);
|
|
}
|
|
|
|
if (!forceUpdate) {
|
|
this.cyclePosition++;
|
|
}
|
|
|
|
if (this.latestPrices.USD === -1) {
|
|
this.latestPrices = await PricesRepository.$getLatestConversionRates();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Called once by the database migration to initialize historical prices data (weekly)
|
|
* We use MtGox weekly price from July 19, 2010 to September 30, 2013
|
|
* We use Kraken weekly price from October 3, 2013 up to last month
|
|
* We use Kraken hourly price for the past month
|
|
*/
|
|
private async $insertHistoricalPrices(): Promise<void> {
|
|
const existingPriceTimes = await PricesRepository.$getPricesTimes();
|
|
|
|
// Insert MtGox weekly prices
|
|
const pricesJson: any[] = JSON.parse(fs.readFileSync(path.join(__dirname, 'mtgox-weekly.json')).toString());
|
|
const prices = this.getEmptyPricesObj();
|
|
let insertedCount: number = 0;
|
|
for (const price of pricesJson) {
|
|
if (existingPriceTimes.includes(price['ct'])) {
|
|
continue;
|
|
}
|
|
|
|
// From 1380758400 we will use Kraken price as it follows closely MtGox, but was not affected as much
|
|
// by the MtGox exchange collapse a few months later
|
|
if (price['ct'] > 1380758400) {
|
|
break;
|
|
}
|
|
prices.USD = price['c'];
|
|
await PricesRepository.$savePrices(price['ct'], prices);
|
|
++insertedCount;
|
|
}
|
|
if (insertedCount > 0) {
|
|
logger.notice(`Inserted ${insertedCount} MtGox USD weekly price history into db`, logger.tags.mining);
|
|
} else {
|
|
logger.debug(`Inserted ${insertedCount} MtGox USD weekly price history into db`, logger.tags.mining);
|
|
}
|
|
|
|
// Insert Kraken weekly prices
|
|
await new KrakenApi().$insertHistoricalPrice();
|
|
|
|
// Insert missing recent hourly prices
|
|
await this.$insertMissingRecentPrices('day');
|
|
await this.$insertMissingRecentPrices('hour');
|
|
|
|
this.historyInserted = true;
|
|
this.lastHistoricalRun = Math.round(new Date().getTime() / 1000);
|
|
}
|
|
|
|
/**
|
|
* Find missing hourly prices and insert them in the database
|
|
* It has a limited backward range and it depends on which API are available
|
|
*/
|
|
private async $insertMissingRecentPrices(type: 'hour' | 'day'): Promise<void> {
|
|
const existingPriceTimes = await PricesRepository.$getPricesTimes();
|
|
|
|
logger.debug(`Fetching ${type === 'day' ? 'dai' : 'hour'}ly price history from exchanges and saving missing ones into the database`, logger.tags.mining);
|
|
|
|
const historicalPrices: PriceHistory[] = [];
|
|
|
|
// Fetch all historical hourly prices
|
|
for (const feed of this.feeds) {
|
|
try {
|
|
historicalPrices.push(await feed.$fetchRecentPrice(this.currencies, type));
|
|
} catch (e) {
|
|
logger.err(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
|
|
}
|
|
}
|
|
|
|
// Group them by timestamp and currency, for example
|
|
// grouped[123456789]['USD'] = [1, 2, 3, 4];
|
|
const grouped = {};
|
|
for (const historicalEntry of historicalPrices) {
|
|
for (const time in historicalEntry) {
|
|
if (existingPriceTimes.includes(parseInt(time, 10))) {
|
|
continue;
|
|
}
|
|
|
|
if (grouped[time] === undefined) {
|
|
grouped[time] = {
|
|
USD: [], EUR: [], GBP: [], CAD: [], CHF: [], AUD: [], JPY: []
|
|
};
|
|
}
|
|
|
|
for (const currency of this.currencies) {
|
|
const price = historicalEntry[time][currency];
|
|
if (price > -1 && price < MAX_PRICES[currency]) {
|
|
grouped[time][currency].push(typeof price === 'string' ? parseInt(price, 10) : price);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Average prices and insert everything into the db
|
|
let totalInserted = 0;
|
|
for (const time in grouped) {
|
|
const prices: ApiPrice = this.getEmptyPricesObj();
|
|
for (const currency in grouped[time]) {
|
|
if (grouped[time][currency].length === 0) {
|
|
continue;
|
|
}
|
|
prices[currency] = Math.round(getMedian(grouped[time][currency]));
|
|
}
|
|
await PricesRepository.$savePrices(parseInt(time, 10), prices);
|
|
++totalInserted;
|
|
}
|
|
|
|
if (totalInserted > 0) {
|
|
logger.notice(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`, logger.tags.mining);
|
|
} else {
|
|
logger.debug(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`, logger.tags.mining);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Find missing prices for additional currencies and insert them in the database
|
|
* We calculate the additional prices from the USD price and the conversion rates
|
|
*/
|
|
private async $insertMissingAdditionalPrices(): Promise<void> {
|
|
this.lastFailedHistoricalRun = 0;
|
|
const priceTimesToFill = await PricesRepository.$getPricesTimesWithMissingFields();
|
|
if (priceTimesToFill.length === 0) {
|
|
return;
|
|
}
|
|
try {
|
|
const remainingQuota = await this.currencyConversionFeed?.$getQuota();
|
|
if (remainingQuota['month']['remaining'] < 500) { // We need some calls left for the daily updates
|
|
logger.debug(`Not enough currency API credit to insert missing prices in ${priceTimesToFill.length} rows (${remainingQuota['month']['remaining']} calls left).`, logger.tags.mining);
|
|
this.additionalCurrenciesHistoryInserted = true; // Do not try again until next day
|
|
return;
|
|
}
|
|
} catch (e) {
|
|
logger.err(`Cannot fetch currency API credit, insertion of missing prices aborted. Reason: ${(e instanceof Error ? e.message : e)}`);
|
|
return;
|
|
}
|
|
|
|
this.additionalCurrenciesHistoryRunning = true;
|
|
logger.debug(`Fetching missing conversion rates from external API to fill ${priceTimesToFill.length} rows`, logger.tags.mining);
|
|
|
|
let conversionRates: { [timestamp: number]: ConversionRates } = {};
|
|
let totalInserted = 0;
|
|
|
|
for (let i = 0; i < priceTimesToFill.length; i++) {
|
|
const priceTime = priceTimesToFill[i];
|
|
const missingLegacyCurrencies = this.getMissingLegacyCurrencies(priceTime); // In the case a legacy currency (EUR, GBP, CAD, CHF, AUD, JPY)
|
|
const year = new Date(priceTime.time * 1000).getFullYear(); // is missing, we use the same process as for the new currencies
|
|
const month = new Date(priceTime.time * 1000).getMonth();
|
|
const yearMonthTimestamp = new Date(year, month, 1).getTime() / 1000;
|
|
if (conversionRates[yearMonthTimestamp] === undefined) {
|
|
conversionRates[yearMonthTimestamp] = await this.currencyConversionFeed?.$fetchConversionRates(`${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-01`) || { USD: -1 };
|
|
if (conversionRates[yearMonthTimestamp]['USD'] < 0) {
|
|
logger.err(`Cannot fetch conversion rates from the API for ${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-01. Aborting insertion of missing prices.`, logger.tags.mining);
|
|
this.lastFailedHistoricalRun = Math.round(new Date().getTime() / 1000);
|
|
break;
|
|
}
|
|
}
|
|
|
|
const prices: ApiPrice = this.getEmptyPricesObj();
|
|
|
|
let willInsert = false;
|
|
for (const conversionCurrency of this.newCurrencies.concat(missingLegacyCurrencies)) {
|
|
if (conversionRates[yearMonthTimestamp][conversionCurrency] > 0 && priceTime.USD * conversionRates[yearMonthTimestamp][conversionCurrency] < MAX_PRICES[conversionCurrency]) {
|
|
prices[conversionCurrency] = year >= 2013 ? Math.round(priceTime.USD * conversionRates[yearMonthTimestamp][conversionCurrency]) : Math.round(priceTime.USD * conversionRates[yearMonthTimestamp][conversionCurrency] * 100) / 100;
|
|
willInsert = true;
|
|
} else {
|
|
prices[conversionCurrency] = 0;
|
|
}
|
|
}
|
|
|
|
if (willInsert) {
|
|
await PricesRepository.$saveAdditionalCurrencyPrices(priceTime.time, prices, missingLegacyCurrencies);
|
|
++totalInserted;
|
|
}
|
|
}
|
|
|
|
logger.debug(`Inserted ${totalInserted} missing additional currency prices into the db`, logger.tags.mining);
|
|
this.additionalCurrenciesHistoryInserted = true;
|
|
this.additionalCurrenciesHistoryRunning = false;
|
|
}
|
|
|
|
// Helper function to get legacy missing currencies in a row (EUR, GBP, CAD, CHF, AUD, JPY)
|
|
private getMissingLegacyCurrencies(priceTime: any): string[] {
|
|
const missingCurrencies: string[] = [];
|
|
['eur', 'gbp', 'cad', 'chf', 'aud', 'jpy'].forEach(currency => {
|
|
if (priceTime[`${currency}_missing`]) {
|
|
missingCurrencies.push(currency.toUpperCase());
|
|
}
|
|
});
|
|
return missingCurrencies;
|
|
}
|
|
}
|
|
|
|
export default new PriceUpdater();
|