Merge pull request #1649 from ayanamitech/fix-poolupdater

pools-updater: Support secure Tor connection to sync data with Github

backend/src/tasks/pools-updater.ts

@@ -1,8 +1,10 @@
-const https = require('https');
+import axios from 'axios';
 import poolsParser from '../api/pools-parser';
 import config from '../config';
 import DB from '../database';
 import logger from '../logger';
+import { SocksProxyAgent } from 'socks-proxy-agent';
+import * as https from 'https';
 
 /**
  * Maintain the most recent version of pools.json
@@ -28,6 +30,13 @@ class PoolsUpdater {
 
     this.lastRun = now;
 
+    logger.info('Updating latest mining pools from Github');
+    if (config.SOCKS5PROXY.ENABLED) {
+      logger.info('List of public pools will be queried over the Tor network');
+    } else {
+      logger.info('List of public pools will be queried over clearnet');
+    }
+
     try {
       const dbSha = await this.getShaFromDb();
       const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github
@@ -41,7 +50,10 @@ class PoolsUpdater {
       }
 
       logger.warn('Pools.json is outdated, fetch latest from github');
-      const poolsJson = await this.fetchPools();
+      const poolsJson = await this.query('https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json');
+      if (poolsJson === undefined) {
+        return;
+      }
       await poolsParser.migratePoolsJson(poolsJson);
       await this.updateDBSha(githubSha);
       logger.notice('PoolsUpdater completed');
@@ -52,14 +64,6 @@ class PoolsUpdater {
     }
   }
 
-  /**
-   * Fetch pools.json from github repo
-   */
-  private async fetchPools(): Promise<object> {
-    const response = await this.query('/repos/mempool/mining-pools/contents/pools.json');
-    return JSON.parse(Buffer.from(response['content'], 'base64').toString('utf8'));
-  }
-
   /**
    * Fetch our latest pools.json sha from the db
    */
@@ -90,13 +94,15 @@ class PoolsUpdater {
    * Fetch our latest pools.json sha from github
    */
   private async fetchPoolsSha(): Promise<string | undefined> {
-    const response = await this.query('/repos/mempool/mining-pools/git/trees/master');
+    const response = await this.query('https://api.github.com/repos/mempool/mining-pools/git/trees/master');
 
+    if (response !== undefined) {
       for (const file of response['tree']) {
         if (file['path'] === 'pools.json') {
           return file['sha'];
         }
       }
+    }
 
     logger.err('Cannot to find latest pools.json sha from github api response');
     return undefined;
@@ -105,35 +111,45 @@ class PoolsUpdater {
   /**
    * Http request wrapper
    */
-  private query(path): Promise<string> {
+  private async query(path): Promise<object | undefined> {
-    return new Promise((resolve, reject) => {
+    type axiosOptions = {
-      const options = {
+      httpsAgent?: https.Agent;
-        host: 'api.github.com',
+    }
-        path: path,
+    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
-        method: 'GET',
+    const axiosOptions: axiosOptions = {};
-        headers: { 'user-agent': 'node.js' }
+    let retry = 0;
+
+    if (config.SOCKS5PROXY.ENABLED) {
+      const socksOptions: any = {
+        agentOptions: {
+          keepAlive: true,
+        },
+        hostname: config.SOCKS5PROXY.HOST,
+        port: config.SOCKS5PROXY.PORT
       };
 
-      logger.debug('Querying: api.github.com' + path);
+      if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+        socksOptions.username = config.SOCKS5PROXY.USERNAME;
+        socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+      }
 
-      const request = https.get(options, (response) => {
+      axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
-        const chunks_of_data: any[] = [];
+    }
-        response.on('data', (fragments) => {
-          chunks_of_data.push(fragments);
-        });
-        response.on('end', () => {
-          resolve(JSON.parse(Buffer.concat(chunks_of_data).toString()));
-        });
-        response.on('error', (error) => {
-          reject(error);
-        });
-      });
 
-      request.on('error', (error) => {
+    while(retry < 5) {
-        logger.err('Github API query failed. Reason: '  + error);
+      try {
-        reject(error);
+        const data = await axios.get(path, axiosOptions);
-      });
+        if (data.statusText !== 'OK' || !data.data) {
-    });
+          throw new Error(`Could not fetch data from Github, Error: ${data.status}`);
+        }
+        return data.data;
+      } catch (e) {
+        logger.err('Could not connect to Github. Reason: '  + (e instanceof Error ? e.message : e));
+        retry++;
+      }
+      await setDelay();
+    }
+    return undefined;
   }
 }
 

contributors/ayanamidev.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of May 15, 2022.
+
+Signed: ayanamidev