machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into nymkappa/bugfix/db-disabled

Browse Source
This commit is contained in:
wiz
2022-06-08 01:56:52 +09:00
committed by GitHub
parent 21ae1fce2a e7c5307ca4
commit 0b50c17ed0
13 changed files with 259 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
backend/src/tasks/pools-updater.ts
View File

@@ -1,7 +1,8 @@
import axios from 'axios';
import axios, { AxiosResponse } from 'axios';
import poolsParser from '../api/pools-parser';
import config from '../config';
import DB from '../database';
import backendInfo from '../api/backend-info';
import logger from '../logger';
import { SocksProxyAgent } from 'socks-proxy-agent';
import * as https from 'https';
@@ -119,33 +120,45 @@ class PoolsUpdater {
*/
private async query(path): Promise<object | undefined> {
type axiosOptions = {
headers: {
'User-Agent': string
};
timeout: number;
httpsAgent?: https.Agent;
}
const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
const axiosOptions: axiosOptions = {};
const axiosOptions: axiosOptions = {
headers: {
'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
},
timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
};
let retry = 0;
if (config.SOCKS5PROXY.ENABLED) {
const socksOptions: any = {
agentOptions: {
keepAlive: true,
},
hostname: config.SOCKS5PROXY.HOST,
port: config.SOCKS5PROXY.PORT
};
if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
socksOptions.username = config.SOCKS5PROXY.USERNAME;
socksOptions.password = config.SOCKS5PROXY.PASSWORD;
}
axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
}
while(retry < 5) {
while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
try {
const data = await axios.get(path, axiosOptions);
if (data.statusText !== 'OK' || !data.data) {
if (config.SOCKS5PROXY.ENABLED) {
const socksOptions: any = {
agentOptions: {
keepAlive: true,
},
hostname: config.SOCKS5PROXY.HOST,
port: config.SOCKS5PROXY.PORT
};
if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
socksOptions.username = config.SOCKS5PROXY.USERNAME;
socksOptions.password = config.SOCKS5PROXY.PASSWORD;
} else {
// Retry with different tor circuits https://stackoverflow.com/a/64960234
socksOptions.username = `circuit${retry}`;
}
axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
}
const data: AxiosResponse = await axios.get(path, axiosOptions);
if (data.statusText === 'error' || !data.data) {
throw new Error(`Could not fetch data from Github, Error: ${data.status}`);
}
return data.data;
@@ -153,7 +166,7 @@ class PoolsUpdater {
logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e));
retry++;
}
await setDelay();
await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
}
return undefined;
}