diff --git a/frontend/package.json b/frontend/package.json index 883fe4017..c06fe74a7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -35,7 +35,7 @@ "start:local-staging": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-staging", "start:mixed": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c mixed", "build": "npm run generate-config && npm run ng -- build --configuration production --localize && npm run sync-assets && npm run build-mempool.js", - "sync-assets": "rsync -av ./src/resources ./dist/mempool/browser && node sync-assets.js 'dist/mempool/browser/resources/'", + "sync-assets": "rsync -av ./src/resources ./dist/mempool/browser && node sync-assets.js 'dist/mempool/browser/resources'", "sync-assets-dev": "node sync-assets.js 'src/resources/'", "generate-config": "node generate-config.js", "build-mempool.js": "npm run build-mempool-js && npm run build-mempool-liquid-js && npm run build-mempool-bisq-js", diff --git a/frontend/sync-assets.js b/frontend/sync-assets.js index 953811fae..26d84e319 100644 --- a/frontend/sync-assets.js +++ b/frontend/sync-assets.js @@ -1,5 +1,6 @@ var https = require('https'); var fs = require('fs'); +var crypto = require('crypto'); const CONFIG_FILE_NAME = 'mempool-frontend-config.json'; let configContent = {}; @@ -37,39 +38,180 @@ function download(filename, url) { }); } -function downloadMiningPoolLogos() { - const options = { - host: 'api.github.com', - path: '/repos/mempool/mining-pool-logos/contents/', - method: 'GET', - headers: {'user-agent': 'node.js'} - }; - - https.get(options, (response) => { - let chunks_of_data = []; - - response.on('data', (fragments) => { - chunks_of_data.push(fragments); - }); - - response.on('end', () => { - let response_body = Buffer.concat(chunks_of_data); - try { - const poolLogos = JSON.parse(response_body.toString()); - for (const poolLogo of poolLogos) { - download(`${PATH}/mining-pools/${poolLogo.name}`, poolLogo.download_url); - } - } catch (e) { - console.error(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); - } - }); - - response.on('error', (error) => { - throw new Error(error); - }); - }) +function getLocalHash(filePath) { + const size = fs.statSync(filePath); + const buffer = fs.readFileSync(filePath); + const bufferWithHeader = Buffer.concat([Buffer.from('blob '), Buffer.from(`${size.size}`), Buffer.from('\0'), buffer]); + return crypto.createHash('sha1').update(bufferWithHeader).digest('hex'); } +function downloadMiningPoolLogos$() { + return new Promise((resolve, reject) => { + console.log('Checking if mining pool logos needs downloading or updating...'); + const options = { + host: 'api.github.com', + path: '/repos/mempool/mining-pool-logos/contents/', + method: 'GET', + headers: {'user-agent': 'node.js'} + }; + + https.get(options, (response) => { + const chunks_of_data = []; + + response.on('data', (fragments) => { + chunks_of_data.push(fragments); + }); + + response.on('end', () => { + const response_body = Buffer.concat(chunks_of_data); + try { + const poolLogos = JSON.parse(response_body.toString()); + if (poolLogos.message) { + reject(poolLogos.message); + } + let downloadedCount = 0; + for (const poolLogo of poolLogos) { + const filePath = `${PATH}/mining-pools/${poolLogo.name}`; + if (fs.existsSync(filePath)) { + const localHash = getLocalHash(filePath); + if (localHash !== poolLogo.sha) { + console.log(`${poolLogo.name} is different on the remote, downloading...`); + download(filePath, poolLogo.download_url); + downloadedCount++; + } + } else { + console.log(`${poolLogo.name} is missing, downloading...`); + download(filePath, poolLogo.download_url); + downloadedCount++; + } + } + console.log(`Downloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`); + resolve(); + } catch (e) { + reject(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); + } + }); + + response.on('error', (error) => { + reject(error); + }); + }); + }); +} + +function downloadPromoVideoSubtiles$() { + return new Promise((resolve, reject) => { + console.log('Checking if promo video subtitles needs downloading or updating...'); + const options = { + host: 'api.github.com', + path: '/repos/mempool/mempool-promo/contents/subtitles', + method: 'GET', + headers: {'user-agent': 'node.js'} + }; + + https.get(options, (response) => { + const chunks_of_data = []; + + response.on('data', (fragments) => { + chunks_of_data.push(fragments); + }); + + response.on('end', () => { + const response_body = Buffer.concat(chunks_of_data); + try { + const videoLanguages = JSON.parse(response_body.toString()); + if (videoLanguages.message) { + reject(videoLanguages.message); + } + let downloadedCount = 0; + for (const language of videoLanguages) { + const filePath = `${PATH}/promo-video/${language.name}`; + if (fs.existsSync(filePath)) { + const localHash = getLocalHash(filePath); + if (localHash !== language.sha) { + console.log(`${language.name} is different on the remote, updating`); + download(filePath, language.download_url); + downloadedCount++; + } + } else { + console.log(`${language.name} is missing, downloading`); + download(filePath, language.download_url); + downloadedCount++; + } + } + console.log(`Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`); + resolve(); + } catch (e) { + reject(`Unable to download video subtitles. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); + } + }); + + response.on('error', (error) => { + reject(error); + }); + }); + }); +} + +function downloadPromoVideo$() { + return new Promise((resolve, reject) => { + console.log('Checking if promo video needs downloading or updating...'); + const options = { + host: 'api.github.com', + path: '/repos/mempool/mempool-promo/contents', + method: 'GET', + headers: {'user-agent': 'node.js'} + }; + + https.get(options, (response) => { + const chunks_of_data = []; + + response.on('data', (fragments) => { + chunks_of_data.push(fragments); + }); + + response.on('end', () => { + const response_body = Buffer.concat(chunks_of_data); + try { + const contents = JSON.parse(response_body.toString()); + if (contents.message) { + reject(contents.message); + } + for (const item of contents) { + if (item.name !== 'promo.mp4') { + continue; + } + const filePath = `${PATH}/promo-video/mempool-promo.mp4`; + if (fs.existsSync(filePath)) { + const localHash = getLocalHash(filePath); + if (localHash !== item.sha) { + console.log(`mempool-promo.mp4 is different on the remote, updating`); + download(filePath, item.download_url); + console.log('mempool-promo.mp4 downloaded.'); + } else { + console.log(`mempool-promo.mp4 is already up to date. Skipping.`); + } + } else { + console.log(`mempool-promo.mp4 is missing, downloading`); + download(filePath, item.download_url); + } + } + resolve(); + } catch (e) { + reject(`Unable to download video. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); + } + }); + + response.on('error', (error) => { + reject(error); + }); + }); + }); + +} + + + let assetsJsonUrl = 'https://raw.githubusercontent.com/mempool/asset_registry_db/master/index.json'; let assetsMinimalJsonUrl = 'https://raw.githubusercontent.com/mempool/asset_registry_db/master/index.minimal.json'; @@ -81,11 +223,6 @@ if (configContent.BASE_MODULE && configContent.BASE_MODULE === 'liquid') { const testnetAssetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.json'; const testnetAssetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json'; -const promoPrefix = PATH + 'promo-video/'; -const promoVideoFile = promoPrefix + 'mempool-promo.mp4'; -const promoVideoUrl = 'https://raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4'; -const promoVideoLanguages = ['en','sv','ja','zh','cs','fi','fr','de','it','lt','nb','fa','pl','ro','pt']; - console.log('Downloading assets'); download(PATH + 'assets.json', assetsJsonUrl); console.log('Downloading assets minimal'); @@ -94,13 +231,10 @@ console.log('Downloading testnet assets'); download(PATH + 'assets-testnet.json', testnetAssetsJsonUrl); console.log('Downloading testnet assets minimal'); download(PATH + 'assets-testnet.minimal.json', testnetAssetsMinimalJsonUrl); -if (!fs.existsSync(promoVideoFile)) { - console.log('Downloading promo video'); - download(promoVideoFile, promoVideoUrl); -} -console.log('Downloading promo video subtitles'); -for( const l of promoVideoLanguages ) { - download(promoPrefix + l + ".vtt", "https://raw.githubusercontent.com/mempool/mempool-promo/master/subtitles/" + l + ".vtt"); -} -console.log('Downloading mining pool logos'); -downloadMiningPoolLogos(); + +downloadMiningPoolLogos$() + .then(() => downloadPromoVideoSubtiles$()) + .then(() => downloadPromoVideo$()) + .catch((error) => { + throw new Error(error); + });