Merge pull request #3668 from mempool/simon/ignore-existing-mining-pools

Skip existing mining pool logos when syncing assets

frontend/package.json

@@ -35,7 +35,7 @@
     "start:local-staging": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-staging",
     "start:mixed": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c mixed",
     "build": "npm run generate-config && npm run ng -- build --configuration production --localize && npm run sync-assets && npm run build-mempool.js",
-    "sync-assets": "rsync -av ./src/resources ./dist/mempool/browser && node sync-assets.js 'dist/mempool/browser/resources/'",
+    "sync-assets": "rsync -av ./src/resources ./dist/mempool/browser && node sync-assets.js 'dist/mempool/browser/resources'",
     "sync-assets-dev": "node sync-assets.js 'src/resources/'",
     "generate-config": "node generate-config.js",
     "build-mempool.js": "npm run build-mempool-js && npm run build-mempool-liquid-js && npm run build-mempool-bisq-js",

frontend/sync-assets.js

@@ -1,5 +1,6 @@
 var https = require('https');
 var fs = require('fs');
+var crypto = require('crypto');
 
 const CONFIG_FILE_NAME = 'mempool-frontend-config.json';
 let configContent = {};
@@ -37,39 +38,180 @@ function download(filename, url) {
   });
 }
 
-function downloadMiningPoolLogos() {
+function getLocalHash(filePath) {
-  const options = {
+  const size = fs.statSync(filePath);
-    host: 'api.github.com',
+  const buffer = fs.readFileSync(filePath);
-    path: '/repos/mempool/mining-pool-logos/contents/',
+  const bufferWithHeader = Buffer.concat([Buffer.from('blob '), Buffer.from(`${size.size}`), Buffer.from('\0'), buffer]);
-    method: 'GET',
+  return crypto.createHash('sha1').update(bufferWithHeader).digest('hex');
-    headers: {'user-agent': 'node.js'}
-  };
-
-  https.get(options, (response) => {
-    let chunks_of_data = [];
-
-    response.on('data', (fragments) => {
-      chunks_of_data.push(fragments);
-    });
-
-    response.on('end', () => {
-      let response_body = Buffer.concat(chunks_of_data);
-      try {
-        const poolLogos = JSON.parse(response_body.toString());
-        for (const poolLogo of poolLogos) {
-            download(`${PATH}/mining-pools/${poolLogo.name}`, poolLogo.download_url);
-        }
-      } catch (e) {
-        console.error(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
-      }
-    });
-
-    response.on('error', (error) => {
-      throw new Error(error);
-    });
-  })
 }
 
+function downloadMiningPoolLogos$() {
+  return new Promise((resolve, reject) => {
+    console.log('Checking if mining pool logos needs downloading or updating...');
+    const options = {
+      host: 'api.github.com',
+      path: '/repos/mempool/mining-pool-logos/contents/',
+      method: 'GET',
+      headers: {'user-agent': 'node.js'}
+    };
+
+    https.get(options, (response) => {
+      const chunks_of_data = [];
+
+      response.on('data', (fragments) => {
+        chunks_of_data.push(fragments);
+      });
+
+      response.on('end', () => {
+        const response_body = Buffer.concat(chunks_of_data);
+        try {
+          const poolLogos = JSON.parse(response_body.toString());
+          if (poolLogos.message) {
+            reject(poolLogos.message);
+          }
+          let downloadedCount = 0;
+          for (const poolLogo of poolLogos) {
+            const filePath = `${PATH}/mining-pools/${poolLogo.name}`;
+            if (fs.existsSync(filePath)) {
+              const localHash = getLocalHash(filePath);
+              if (localHash !== poolLogo.sha) {
+                console.log(`${poolLogo.name} is different on the remote, downloading...`);
+                download(filePath, poolLogo.download_url);
+                downloadedCount++;
+              }
+            } else {
+              console.log(`${poolLogo.name} is missing, downloading...`);
+              download(filePath, poolLogo.download_url);
+              downloadedCount++;
+            }
+          }
+          console.log(`Downloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`);
+          resolve();
+        } catch (e) {
+          reject(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
+        }
+      });
+
+      response.on('error', (error) => {
+        reject(error);
+      });
+    });
+  });
+}
+
+function downloadPromoVideoSubtiles$() {
+  return new Promise((resolve, reject) => {
+    console.log('Checking if promo video subtitles needs downloading or updating...');
+    const options = {
+      host: 'api.github.com',
+      path: '/repos/mempool/mempool-promo/contents/subtitles',
+      method: 'GET',
+      headers: {'user-agent': 'node.js'}
+    };
+
+    https.get(options, (response) => {
+      const chunks_of_data = [];
+
+      response.on('data', (fragments) => {
+        chunks_of_data.push(fragments);
+      });
+
+      response.on('end', () => {
+        const response_body = Buffer.concat(chunks_of_data);
+        try {
+          const videoLanguages = JSON.parse(response_body.toString());
+          if (videoLanguages.message) {
+            reject(videoLanguages.message);
+          }
+          let downloadedCount = 0;
+          for (const language of videoLanguages) {
+            const filePath = `${PATH}/promo-video/${language.name}`;
+            if (fs.existsSync(filePath)) {
+              const localHash = getLocalHash(filePath);
+              if (localHash !== language.sha) {
+                console.log(`${language.name} is different on the remote, updating`);
+                download(filePath, language.download_url);
+                downloadedCount++;
+              }
+            } else {
+              console.log(`${language.name} is missing, downloading`);
+              download(filePath, language.download_url);
+              downloadedCount++;
+            }
+          }
+          console.log(`Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`);
+          resolve();
+        } catch (e) {
+          reject(`Unable to download video subtitles. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
+        }
+      });
+
+      response.on('error', (error) => {
+        reject(error);
+      });
+    });
+  });
+}
+
+function downloadPromoVideo$() {
+  return new Promise((resolve, reject) => {
+    console.log('Checking if promo video needs downloading or updating...');
+    const options = {
+      host: 'api.github.com',
+      path: '/repos/mempool/mempool-promo/contents',
+      method: 'GET',
+      headers: {'user-agent': 'node.js'}
+    };
+
+    https.get(options, (response) => {
+      const chunks_of_data = [];
+
+      response.on('data', (fragments) => {
+        chunks_of_data.push(fragments);
+      });
+
+      response.on('end', () => {
+        const response_body = Buffer.concat(chunks_of_data);
+        try {
+          const contents = JSON.parse(response_body.toString());
+          if (contents.message) {
+            reject(contents.message);
+          }
+          for (const item of contents) {
+            if (item.name !== 'promo.mp4') {
+              continue;
+            }
+            const filePath = `${PATH}/promo-video/mempool-promo.mp4`;
+            if (fs.existsSync(filePath)) {
+              const localHash = getLocalHash(filePath);
+              if (localHash !== item.sha) {
+                console.log(`mempool-promo.mp4 is different on the remote, updating`);
+                download(filePath, item.download_url);
+                console.log('mempool-promo.mp4 downloaded.');
+              } else {
+                console.log(`mempool-promo.mp4 is already up to date. Skipping.`);
+              }
+            } else {
+              console.log(`mempool-promo.mp4 is missing, downloading`);
+              download(filePath, item.download_url);
+            }
+          }
+          resolve();
+        } catch (e) {
+          reject(`Unable to download video. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
+        }
+      });
+
+      response.on('error', (error) => {
+        reject(error);
+      });
+    });
+  });
+
+}
+
+
+
 let assetsJsonUrl = 'https://raw.githubusercontent.com/mempool/asset_registry_db/master/index.json';
 let assetsMinimalJsonUrl = 'https://raw.githubusercontent.com/mempool/asset_registry_db/master/index.minimal.json';
 
@@ -81,11 +223,6 @@ if (configContent.BASE_MODULE && configContent.BASE_MODULE === 'liquid') {
 const testnetAssetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.json';
 const testnetAssetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json';
 
-const promoPrefix = PATH + 'promo-video/';
-const promoVideoFile = promoPrefix + 'mempool-promo.mp4';
-const promoVideoUrl = 'https://raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4';
-const promoVideoLanguages = ['en','sv','ja','zh','cs','fi','fr','de','it','lt','nb','fa','pl','ro','pt'];
-
 console.log('Downloading assets');
 download(PATH + 'assets.json', assetsJsonUrl);
 console.log('Downloading assets minimal');
@@ -94,13 +231,10 @@ console.log('Downloading testnet assets');
 download(PATH + 'assets-testnet.json', testnetAssetsJsonUrl);
 console.log('Downloading testnet assets minimal');
 download(PATH + 'assets-testnet.minimal.json', testnetAssetsMinimalJsonUrl);
-if (!fs.existsSync(promoVideoFile)) {
+
-  console.log('Downloading promo video');
+downloadMiningPoolLogos$()
-  download(promoVideoFile, promoVideoUrl);
+  .then(() => downloadPromoVideoSubtiles$())
-}
+  .then(() => downloadPromoVideo$())
-console.log('Downloading promo video subtitles');
+  .catch((error) => {
-for( const l of promoVideoLanguages ) {
+    throw new Error(error);
-  download(promoPrefix + l + ".vtt", "https://raw.githubusercontent.com/mempool/mempool-promo/master/subtitles/" + l + ".vtt");
+  });
-}
-console.log('Downloading mining pool logos');
-downloadMiningPoolLogos();