From 7f3e4eb534375511f6f8fc301ef8f6dfe69ee2a9 Mon Sep 17 00:00:00 2001 From: softsimon Date: Wed, 19 Apr 2023 13:10:49 +0400 Subject: [PATCH 1/7] Skip existing mining pool logos when syncing assets --- frontend/sync-assets.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frontend/sync-assets.js b/frontend/sync-assets.js index 953811fae..4a397e53d 100644 --- a/frontend/sync-assets.js +++ b/frontend/sync-assets.js @@ -56,9 +56,15 @@ function downloadMiningPoolLogos() { let response_body = Buffer.concat(chunks_of_data); try { const poolLogos = JSON.parse(response_body.toString()); + let downloadedCount = 0; for (const poolLogo of poolLogos) { - download(`${PATH}/mining-pools/${poolLogo.name}`, poolLogo.download_url); + const filePath = `${PATH}/mining-pools/${poolLogo.name}`; + if (!fs.existsSync(filePath)) { + download(filePath, poolLogo.download_url); + downloadedCount++; + } } + console.log(`Downloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`); } catch (e) { console.error(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); } From 49e057e726943abc06527ff4f54ac6bb8ac483f0 Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 20 Apr 2023 16:39:50 +0400 Subject: [PATCH 2/7] Fixing trailing slash issue --- frontend/package.json | 2 +- frontend/sync-assets.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index 883fe4017..c06fe74a7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -35,7 +35,7 @@ "start:local-staging": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-staging", "start:mixed": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c mixed", "build": "npm run generate-config && npm run ng -- build --configuration production --localize && npm run sync-assets && npm run build-mempool.js", - "sync-assets": "rsync -av ./src/resources ./dist/mempool/browser && node sync-assets.js 'dist/mempool/browser/resources/'", + "sync-assets": "rsync -av ./src/resources ./dist/mempool/browser && node sync-assets.js 'dist/mempool/browser/resources'", "sync-assets-dev": "node sync-assets.js 'src/resources/'", "generate-config": "node generate-config.js", "build-mempool.js": "npm run build-mempool-js && npm run build-mempool-liquid-js && npm run build-mempool-bisq-js", diff --git a/frontend/sync-assets.js b/frontend/sync-assets.js index 4a397e53d..eefa1f6c8 100644 --- a/frontend/sync-assets.js +++ b/frontend/sync-assets.js @@ -87,7 +87,7 @@ if (configContent.BASE_MODULE && configContent.BASE_MODULE === 'liquid') { const testnetAssetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.json'; const testnetAssetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json'; -const promoPrefix = PATH + 'promo-video/'; +const promoPrefix = PATH + '/promo-video/'; const promoVideoFile = promoPrefix + 'mempool-promo.mp4'; const promoVideoUrl = 'https://raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4'; const promoVideoLanguages = ['en','sv','ja','zh','cs','fi','fr','de','it','lt','nb','fa','pl','ro','pt']; From d3bd4342555e599f589d91c510e4a6cb21deddf4 Mon Sep 17 00:00:00 2001 From: softsimon Date: Fri, 12 May 2023 16:01:45 -0500 Subject: [PATCH 3/7] Use sha hashes to compare before downloading --- frontend/sync-assets.js | 86 +++++++++++++++++++++++++++++++++++------ 1 file changed, 75 insertions(+), 11 deletions(-) diff --git a/frontend/sync-assets.js b/frontend/sync-assets.js index eefa1f6c8..b41a927f2 100644 --- a/frontend/sync-assets.js +++ b/frontend/sync-assets.js @@ -1,5 +1,6 @@ var https = require('https'); var fs = require('fs'); +var crypto = require('crypto'); const CONFIG_FILE_NAME = 'mempool-frontend-config.json'; let configContent = {}; @@ -37,6 +38,13 @@ function download(filename, url) { }); } +function getLocalHash(filePath) { + const size = fs.statSync(filePath); + const buffer = fs.readFileSync(filePath); + const bufferWithHeader = Buffer.concat([Buffer.from('blob '), Buffer.from(`${size.size}`), Buffer.from('\0'), buffer]); + return crypto.createHash('sha1').update(bufferWithHeader).digest('hex'); +} + function downloadMiningPoolLogos() { const options = { host: 'api.github.com', @@ -46,21 +54,29 @@ function downloadMiningPoolLogos() { }; https.get(options, (response) => { - let chunks_of_data = []; + const chunks_of_data = []; response.on('data', (fragments) => { chunks_of_data.push(fragments); }); response.on('end', () => { - let response_body = Buffer.concat(chunks_of_data); + const response_body = Buffer.concat(chunks_of_data); try { const poolLogos = JSON.parse(response_body.toString()); let downloadedCount = 0; for (const poolLogo of poolLogos) { const filePath = `${PATH}/mining-pools/${poolLogo.name}`; - if (!fs.existsSync(filePath)) { - download(filePath, poolLogo.download_url); + if (fs.existsSync(filePath)) { + const localHash = getLocalHash(filePath); + if (localHash !== poolLogo.sha) { + console.log(`${poolLogo.name} is different on the remote, updating`); + download(filePath, poolLogo.download_url); + downloadedCount++; + } + } else { + console.log(`${poolLogo.name} is missing, downloading`); + download(`${PATH}mining-pools/${poolLogo.name}`, poolLogo.download_url); downloadedCount++; } } @@ -73,7 +89,58 @@ function downloadMiningPoolLogos() { response.on('error', (error) => { throw new Error(error); }); - }) + }); +} + +function downloadPromoVideoSubtiles() { + // for( const l of promoVideoLanguages ) { + // download(promoPrefix + l + '.vtt', 'https://raw.githubusercontent.com/mempool/mempool-promo/master/subtitles/' + l + '.vtt'); + // } + + const options = { + host: 'api.github.com', + path: '/repos/mempool/mempool-promo/contents/subtitles', + method: 'GET', + headers: {'user-agent': 'node.js'} + }; + + https.get(options, (response) => { + const chunks_of_data = []; + + response.on('data', (fragments) => { + chunks_of_data.push(fragments); + }); + + response.on('end', () => { + const response_body = Buffer.concat(chunks_of_data); + try { + const videoLanguages = JSON.parse(response_body.toString()); + let downloadedCount = 0; + for (const language of videoLanguages) { + const filePath = `${PATH}/promo-video/${language.name}`; + if (fs.existsSync(filePath)) { + const localHash = getLocalHash(filePath); + if (localHash !== language.sha) { + console.log(`${language.name} is different on the remote, updating`); + download(filePath, language.download_url); + downloadedCount++; + } + } else { + console.log(`${language.name} is missing, downloading`); + download(`${PATH}promo-video/${language.name}`, language.download_url); + downloadedCount++; + } + } + console.log(`Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`); + } catch (e) { + console.error(`Unable to download video subtitles. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); + } + }); + + response.on('error', (error) => { + throw new Error(error); + }); + }); } let assetsJsonUrl = 'https://raw.githubusercontent.com/mempool/asset_registry_db/master/index.json'; @@ -87,10 +154,9 @@ if (configContent.BASE_MODULE && configContent.BASE_MODULE === 'liquid') { const testnetAssetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.json'; const testnetAssetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json'; -const promoPrefix = PATH + '/promo-video/'; -const promoVideoFile = promoPrefix + 'mempool-promo.mp4'; + +const promoVideoFile = PATH + '/promo-video/mempool-promo.mp4'; const promoVideoUrl = 'https://raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4'; -const promoVideoLanguages = ['en','sv','ja','zh','cs','fi','fr','de','it','lt','nb','fa','pl','ro','pt']; console.log('Downloading assets'); download(PATH + 'assets.json', assetsJsonUrl); @@ -105,8 +171,6 @@ if (!fs.existsSync(promoVideoFile)) { download(promoVideoFile, promoVideoUrl); } console.log('Downloading promo video subtitles'); -for( const l of promoVideoLanguages ) { - download(promoPrefix + l + ".vtt", "https://raw.githubusercontent.com/mempool/mempool-promo/master/subtitles/" + l + ".vtt"); -} +downloadPromoVideoSubtiles(); console.log('Downloading mining pool logos'); downloadMiningPoolLogos(); From a4478879016744632c2325fa4b093a6dc7b756f0 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 13 May 2023 13:23:27 -0500 Subject: [PATCH 4/7] Adding missing slash --- frontend/sync-assets.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/frontend/sync-assets.js b/frontend/sync-assets.js index b41a927f2..605a663f8 100644 --- a/frontend/sync-assets.js +++ b/frontend/sync-assets.js @@ -93,10 +93,6 @@ function downloadMiningPoolLogos() { } function downloadPromoVideoSubtiles() { - // for( const l of promoVideoLanguages ) { - // download(promoPrefix + l + '.vtt', 'https://raw.githubusercontent.com/mempool/mempool-promo/master/subtitles/' + l + '.vtt'); - // } - const options = { host: 'api.github.com', path: '/repos/mempool/mempool-promo/contents/subtitles', @@ -127,7 +123,7 @@ function downloadPromoVideoSubtiles() { } } else { console.log(`${language.name} is missing, downloading`); - download(`${PATH}promo-video/${language.name}`, language.download_url); + download(`${PATH}/promo-video/${language.name}`, language.download_url); downloadedCount++; } } From da3c3e8f5c00159d4d745818339a2c91469d0d99 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 13 May 2023 13:24:44 -0500 Subject: [PATCH 5/7] Re-use variable fix --- frontend/sync-assets.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/sync-assets.js b/frontend/sync-assets.js index 605a663f8..8556cac75 100644 --- a/frontend/sync-assets.js +++ b/frontend/sync-assets.js @@ -123,7 +123,7 @@ function downloadPromoVideoSubtiles() { } } else { console.log(`${language.name} is missing, downloading`); - download(`${PATH}/promo-video/${language.name}`, language.download_url); + download(filePath, language.download_url); downloadedCount++; } } From f12403747de48e667806e8f6698b4343412f7bec Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 14 May 2023 12:45:42 -0500 Subject: [PATCH 6/7] Adding video to sha1 check and chaining requests in promises. --- frontend/sync-assets.js | 230 ++++++++++++++++++++++++++-------------- 1 file changed, 149 insertions(+), 81 deletions(-) diff --git a/frontend/sync-assets.js b/frontend/sync-assets.js index 8556cac75..4d7b06124 100644 --- a/frontend/sync-assets.js +++ b/frontend/sync-assets.js @@ -45,100 +45,173 @@ function getLocalHash(filePath) { return crypto.createHash('sha1').update(bufferWithHeader).digest('hex'); } -function downloadMiningPoolLogos() { - const options = { - host: 'api.github.com', - path: '/repos/mempool/mining-pool-logos/contents/', - method: 'GET', - headers: {'user-agent': 'node.js'} - }; +function downloadMiningPoolLogos$() { + return new Promise((resolve, reject) => { + console.log('Checking if mining pool logos needs downloading or updating...'); + const options = { + host: 'api.github.com', + path: '/repos/mempool/mining-pool-logos/contents/', + method: 'GET', + headers: {'user-agent': 'node.js'} + }; - https.get(options, (response) => { - const chunks_of_data = []; + https.get(options, (response) => { + const chunks_of_data = []; - response.on('data', (fragments) => { - chunks_of_data.push(fragments); - }); + response.on('data', (fragments) => { + chunks_of_data.push(fragments); + }); - response.on('end', () => { - const response_body = Buffer.concat(chunks_of_data); - try { - const poolLogos = JSON.parse(response_body.toString()); - let downloadedCount = 0; - for (const poolLogo of poolLogos) { - const filePath = `${PATH}/mining-pools/${poolLogo.name}`; - if (fs.existsSync(filePath)) { - const localHash = getLocalHash(filePath); - if (localHash !== poolLogo.sha) { - console.log(`${poolLogo.name} is different on the remote, updating`); - download(filePath, poolLogo.download_url); + response.on('end', () => { + const response_body = Buffer.concat(chunks_of_data); + try { + const poolLogos = JSON.parse(response_body.toString()); + if (poolLogos.message) { + reject(poolLogos.message); + } + let downloadedCount = 0; + for (const poolLogo of poolLogos) { + const filePath = `${PATH}/mining-pools/${poolLogo.name}`; + if (fs.existsSync(filePath)) { + const localHash = getLocalHash(filePath); + if (localHash !== poolLogo.sha) { + console.log(`${poolLogo.name} is different on the remote, downloading...`); + download(filePath, poolLogo.download_url); + downloadedCount++; + } + } else { + console.log(`${poolLogo.name} is missing, downloading...`); + download(`${PATH}mining-pools/${poolLogo.name}`, poolLogo.download_url); downloadedCount++; } - } else { - console.log(`${poolLogo.name} is missing, downloading`); - download(`${PATH}mining-pools/${poolLogo.name}`, poolLogo.download_url); - downloadedCount++; } + console.log(`Downloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`); + resolve(); + } catch (e) { + reject(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); } - console.log(`Downloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`); - } catch (e) { - console.error(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); - } - }); + }); - response.on('error', (error) => { - throw new Error(error); + response.on('error', (error) => { + reject(error); + }); }); }); } -function downloadPromoVideoSubtiles() { - const options = { - host: 'api.github.com', - path: '/repos/mempool/mempool-promo/contents/subtitles', - method: 'GET', - headers: {'user-agent': 'node.js'} - }; +function downloadPromoVideoSubtiles$() { + return new Promise((resolve, reject) => { + console.log('Checking if promo video subtitles needs downloading or updating...'); + const options = { + host: 'api.github.com', + path: '/repos/mempool/mempool-promo/contents/subtitles', + method: 'GET', + headers: {'user-agent': 'node.js'} + }; - https.get(options, (response) => { - const chunks_of_data = []; + https.get(options, (response) => { + const chunks_of_data = []; - response.on('data', (fragments) => { - chunks_of_data.push(fragments); - }); + response.on('data', (fragments) => { + chunks_of_data.push(fragments); + }); - response.on('end', () => { - const response_body = Buffer.concat(chunks_of_data); - try { - const videoLanguages = JSON.parse(response_body.toString()); - let downloadedCount = 0; - for (const language of videoLanguages) { - const filePath = `${PATH}/promo-video/${language.name}`; - if (fs.existsSync(filePath)) { - const localHash = getLocalHash(filePath); - if (localHash !== language.sha) { - console.log(`${language.name} is different on the remote, updating`); + response.on('end', () => { + const response_body = Buffer.concat(chunks_of_data); + try { + const videoLanguages = JSON.parse(response_body.toString()); + if (videoLanguages.message) { + reject(videoLanguages.message); + } + let downloadedCount = 0; + for (const language of videoLanguages) { + const filePath = `${PATH}/promo-video/${language.name}`; + if (fs.existsSync(filePath)) { + const localHash = getLocalHash(filePath); + if (localHash !== language.sha) { + console.log(`${language.name} is different on the remote, updating`); + download(filePath, language.download_url); + downloadedCount++; + } + } else { + console.log(`${language.name} is missing, downloading`); download(filePath, language.download_url); downloadedCount++; } - } else { - console.log(`${language.name} is missing, downloading`); - download(filePath, language.download_url); - downloadedCount++; } + console.log(`Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`); + resolve(); + } catch (e) { + reject(`Unable to download video subtitles. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); } - console.log(`Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`); - } catch (e) { - console.error(`Unable to download video subtitles. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); - } - }); + }); - response.on('error', (error) => { - throw new Error(error); + response.on('error', (error) => { + reject(error); + }); }); }); } +function downloadPromoVideo$() { + return new Promise((resolve, reject) => { + console.log('Checking if promo video needs downloading or updating...'); + const options = { + host: 'api.github.com', + path: '/repos/mempool/mempool-promo/contents', + method: 'GET', + headers: {'user-agent': 'node.js'} + }; + + https.get(options, (response) => { + const chunks_of_data = []; + + response.on('data', (fragments) => { + chunks_of_data.push(fragments); + }); + + response.on('end', () => { + const response_body = Buffer.concat(chunks_of_data); + try { + const contents = JSON.parse(response_body.toString()); + if (contents.message) { + reject(contents.message); + } + for (const item of contents) { + if (item.name !== 'promo.mp4') { + continue; + } + const filePath = `${PATH}/promo-video/mempool-promo.mp4`; + if (fs.existsSync(filePath)) { + const localHash = getLocalHash(filePath); + if (localHash !== item.sha) { + console.log(`mempool-promo.mp4 is different on the remote, updating`); + download(filePath, item.download_url); + console.log('mempool-promo.mp4 downloaded.'); + } else { + console.log(`mempool-promo.mp4 is already up to date. Skipping.`); + } + } else { + console.log(`mempool-promo.mp4 is missing, downloading`); + download(filePath, item.download_url); + } + } + resolve(); + } catch (e) { + reject(`Unable to download video. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); + } + }); + + response.on('error', (error) => { + reject(error); + }); + }); + }); + +} + + + let assetsJsonUrl = 'https://raw.githubusercontent.com/mempool/asset_registry_db/master/index.json'; let assetsMinimalJsonUrl = 'https://raw.githubusercontent.com/mempool/asset_registry_db/master/index.minimal.json'; @@ -150,10 +223,6 @@ if (configContent.BASE_MODULE && configContent.BASE_MODULE === 'liquid') { const testnetAssetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.json'; const testnetAssetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json'; - -const promoVideoFile = PATH + '/promo-video/mempool-promo.mp4'; -const promoVideoUrl = 'https://raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4'; - console.log('Downloading assets'); download(PATH + 'assets.json', assetsJsonUrl); console.log('Downloading assets minimal'); @@ -162,11 +231,10 @@ console.log('Downloading testnet assets'); download(PATH + 'assets-testnet.json', testnetAssetsJsonUrl); console.log('Downloading testnet assets minimal'); download(PATH + 'assets-testnet.minimal.json', testnetAssetsMinimalJsonUrl); -if (!fs.existsSync(promoVideoFile)) { - console.log('Downloading promo video'); - download(promoVideoFile, promoVideoUrl); -} -console.log('Downloading promo video subtitles'); -downloadPromoVideoSubtiles(); -console.log('Downloading mining pool logos'); -downloadMiningPoolLogos(); + +downloadMiningPoolLogos$() + .then(() => downloadPromoVideoSubtiles$()) + .then(() => downloadPromoVideo$()) + .catch((error) => { + throw new Error(error); + }); From aa9fd845efd13d21e1f906cbf11fbc4bda4b4a19 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 14 May 2023 14:21:41 -0500 Subject: [PATCH 7/7] Path fix --- frontend/sync-assets.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/sync-assets.js b/frontend/sync-assets.js index 4d7b06124..26d84e319 100644 --- a/frontend/sync-assets.js +++ b/frontend/sync-assets.js @@ -81,7 +81,7 @@ function downloadMiningPoolLogos$() { } } else { console.log(`${poolLogo.name} is missing, downloading...`); - download(`${PATH}mining-pools/${poolLogo.name}`, poolLogo.download_url); + download(filePath, poolLogo.download_url); downloadedCount++; } }