diff --git a/frontend/package.json b/frontend/package.json index 31e584cb3..214cf9ff6 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -35,7 +35,7 @@ "start:local-staging": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-staging", "start:mixed": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c mixed", "build": "npm run generate-config && npm run ng -- build --configuration production --localize && npm run sync-assets && npm run build-mempool.js", - "sync-assets": "rsync -av ./src/resources ./dist/mempool/browser && node sync-assets.js 'dist/mempool/browser/resources'", + "sync-assets": "rsync -av ./src/resources ./dist/mempool/browser && node sync-assets.js 'dist/mempool/browser/resources/'", "sync-assets-dev": "node sync-assets.js 'src/resources/'", "generate-config": "node generate-config.js", "build-mempool.js": "npm run build-mempool-js && npm run build-mempool-liquid-js && npm run build-mempool-bisq-js", diff --git a/frontend/sync-assets.js b/frontend/sync-assets.js index 49618dea3..a92c3b21d 100644 --- a/frontend/sync-assets.js +++ b/frontend/sync-assets.js @@ -1,6 +1,19 @@ var https = require('https'); var fs = require('fs'); var crypto = require('crypto'); +var path = require('node:path'); +const LOG_TAG = '[sync-assets]'; +let verbose = false; + +if (parseInt(process.env.SKIP_SYNC) === 1) { + console.log(`${LOG_TAG} SKIP_SYNC is set, not checking any assets`); + process.exit(0); +} + +if (parseInt(process.env.VERBOSE) === 1) { + console.log(`${LOG_TAG} VERBOSE is set, logs will be more verbose`); + verbose = true; +} const CONFIG_FILE_NAME = 'mempool-frontend-config.json'; let configContent = {}; @@ -8,6 +21,13 @@ let configContent = {}; var PATH; if (process.argv[2]) { PATH = process.argv[2]; + PATH += PATH.endsWith("/") ? "" : "/" + PATH = path.normalize(PATH); + console.log(`[sync-assets] using PATH ${PATH}`); + if (!fs.existsSync(PATH)){ + console.log(`${LOG_TAG} ${PATH} does not exist, creating`); + fs.mkdirSync(PATH, { recursive: true }); + } } if (!PATH) { @@ -17,12 +37,12 @@ if (!PATH) { try { const rawConfig = fs.readFileSync(CONFIG_FILE_NAME); configContent = JSON.parse(rawConfig); - console.log(`${CONFIG_FILE_NAME} file found, using provided config`); + console.log(`${LOG_TAG} ${CONFIG_FILE_NAME} file found, using provided config`); } catch (e) { if (e.code !== 'ENOENT') { throw new Error(e); } else { - console.log(`${CONFIG_FILE_NAME} file not found, using default config`); + console.log(`${LOG_TAG} ${CONFIG_FILE_NAME} file not found, using default config`); } } @@ -37,6 +57,11 @@ function download(filename, url) { }) .on('error', function(e) { throw new Error(e); + }) + .on('finish', () => { + if (verbose) { + console.log(`${LOG_TAG} Finished downloading ${url} to ${filename}`); + } }); } @@ -44,12 +69,18 @@ function getLocalHash(filePath) { const size = fs.statSync(filePath); const buffer = fs.readFileSync(filePath); const bufferWithHeader = Buffer.concat([Buffer.from('blob '), Buffer.from(`${size.size}`), Buffer.from('\0'), buffer]); - return crypto.createHash('sha1').update(bufferWithHeader).digest('hex'); + const hash = crypto.createHash('sha1').update(bufferWithHeader).digest('hex'); + + if (verbose) { + console.log(`${LOG_TAG} \tgetLocalHash ${filePath} ${hash}`); + } + + return hash; } function downloadMiningPoolLogos$() { return new Promise((resolve, reject) => { - console.log('Checking if mining pool logos needs downloading or updating...'); + console.log(`${LOG_TAG} Checking if mining pool logos needs downloading or updating...`); const options = { host: 'api.github.com', path: '/repos/mempool/mining-pool-logos/contents/', @@ -58,7 +89,7 @@ function downloadMiningPoolLogos$() { }; if (githubSecret) { - console.log('Downloading the mining pool logos with authentication'); + console.log(`${LOG_TAG} Downloading the mining pool logos with authentication`); options.headers['authorization'] = `Bearer ${githubSecret}`; options.headers['X-GitHub-Api-Version'] = '2022-11-28'; } @@ -79,21 +110,29 @@ function downloadMiningPoolLogos$() { } let downloadedCount = 0; for (const poolLogo of poolLogos) { - const filePath = `${PATH}/mining-pools/${poolLogo.name}`; + const filePath = PATH + `mining-pools/${poolLogo.name}`; if (fs.existsSync(filePath)) { const localHash = getLocalHash(filePath); + if (verbose) { + console.log(`${LOG_TAG} Remote ${poolLogo.name} logo hash ${poolLogo.sha}`); + console.log(`${LOG_TAG} \tchecking if ${filePath} exists: ${fs.existsSync(filePath)}`); + } if (localHash !== poolLogo.sha) { - console.log(`${poolLogo.name} is different on the remote, downloading...`); + console.log(`${LOG_TAG} \t\t${poolLogo.name} is different on the remote, downloading...`); download(filePath, poolLogo.download_url); downloadedCount++; } } else { - console.log(`${poolLogo.name} is missing, downloading...`); + console.log(`${LOG_TAG} ${poolLogo.name} is missing, downloading...`); + const miningPoolsDir = PATH + `mining-pools/`; + if (!fs.existsSync(miningPoolsDir)){ + fs.mkdirSync(miningPoolsDir, { recursive: true }); + } download(filePath, poolLogo.download_url); downloadedCount++; } } - console.log(`Downloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`); + console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`); resolve(); } catch (e) { reject(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); @@ -109,7 +148,7 @@ function downloadMiningPoolLogos$() { function downloadPromoVideoSubtiles$() { return new Promise((resolve, reject) => { - console.log('Checking if promo video subtitles needs downloading or updating...'); + console.log(`${LOG_TAG} Checking if promo video subtitles needs downloading or updating...`); const options = { host: 'api.github.com', path: '/repos/mempool/mempool-promo/contents/subtitles', @@ -118,7 +157,7 @@ function downloadPromoVideoSubtiles$() { }; if (githubSecret) { - console.log('Downloading the promo video subtitles with authentication'); + console.log(`${LOG_TAG} Downloading the promo video subtitles with authentication`); options.headers['authorization'] = `Bearer ${githubSecret}`; options.headers['X-GitHub-Api-Version'] = '2022-11-28'; } @@ -140,21 +179,30 @@ function downloadPromoVideoSubtiles$() { } let downloadedCount = 0; for (const language of videoLanguages) { - const filePath = `${PATH}/promo-video/${language.name}`; + const filePath = PATH + `promo-video/${language.name}`; if (fs.existsSync(filePath)) { + if (verbose) { + console.log(`${LOG_TAG} ${language.name} remote promo video hash ${language.sha}`); + } const localHash = getLocalHash(filePath); + if (localHash !== language.sha) { - console.log(`${language.name} is different on the remote, updating`); + console.log(`${LOG_TAG} ${language.name} is different on the remote, updating`); download(filePath, language.download_url); downloadedCount++; } } else { - console.log(`${language.name} is missing, downloading`); + console.log(`${LOG_TAG} ${language.name} is missing, downloading`); + const promoVideosDir = PATH + `promo-video/`; + if (!fs.existsSync(promoVideosDir)){ + fs.mkdirSync(promoVideosDir, { recursive: true }); + } + download(filePath, language.download_url); downloadedCount++; } } - console.log(`Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`); + console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`); resolve(); } catch (e) { reject(`Unable to download video subtitles. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); @@ -170,7 +218,7 @@ function downloadPromoVideoSubtiles$() { function downloadPromoVideo$() { return new Promise((resolve, reject) => { - console.log('Checking if promo video needs downloading or updating...'); + console.log(`${LOG_TAG} Checking if promo video needs downloading or updating...`); const options = { host: 'api.github.com', path: '/repos/mempool/mempool-promo/contents', @@ -179,7 +227,7 @@ function downloadPromoVideo$() { }; if (githubSecret) { - console.log('Downloading the promo videos with authentication'); + console.log(`${LOG_TAG} Downloading the promo video with authentication`); options.headers['authorization'] = `Bearer ${githubSecret}`; options.headers['X-GitHub-Api-Version'] = '2022-11-28'; } @@ -202,18 +250,19 @@ function downloadPromoVideo$() { if (item.name !== 'promo.mp4') { continue; } - const filePath = `${PATH}/promo-video/mempool-promo.mp4`; + const filePath = PATH + `promo-video/mempool-promo.mp4`; if (fs.existsSync(filePath)) { const localHash = getLocalHash(filePath); + if (localHash !== item.sha) { - console.log(`mempool-promo.mp4 is different on the remote, updating`); + console.log(`${LOG_TAG} \tmempool-promo.mp4 is different on the remote, updating`); download(filePath, item.download_url); - console.log('mempool-promo.mp4 downloaded.'); + console.log(`${LOG_TAG} \tmempool-promo.mp4 downloaded.`); } else { - console.log(`mempool-promo.mp4 is already up to date. Skipping.`); + console.log(`${LOG_TAG} \tmempool-promo.mp4 is already up to date. Skipping.`); } } else { - console.log(`mempool-promo.mp4 is missing, downloading`); + console.log(`${LOG_TAG} \tmempool-promo.mp4 is missing, downloading`); download(filePath, item.download_url); } } @@ -232,30 +281,47 @@ function downloadPromoVideo$() { } - -let assetsJsonUrl = 'https://raw.githubusercontent.com/mempool/asset_registry_db/master/index.json'; -let assetsMinimalJsonUrl = 'https://raw.githubusercontent.com/mempool/asset_registry_db/master/index.minimal.json'; - if (configContent.BASE_MODULE && configContent.BASE_MODULE === 'liquid') { - assetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_db/master/index.json'; - assetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_db/master/index.minimal.json'; + const assetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_db/master/index.json'; + const assetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_db/master/index.minimal.json'; + const testnetAssetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.json'; + const testnetAssetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json'; + + console.log(`${LOG_TAG} Downloading assets`); + download(PATH + 'assets.json', assetsJsonUrl); + + console.log(`${LOG_TAG} Downloading assets minimal`); + download(PATH + 'assets.minimal.json', assetsMinimalJsonUrl); + + console.log(`${LOG_TAG} Downloading testnet assets`); + download(PATH + 'assets-testnet.json', testnetAssetsJsonUrl); + + console.log(`${LOG_TAG} Downloading testnet assets minimal`); + download(PATH + 'assets-testnet.minimal.json', testnetAssetsMinimalJsonUrl); +} else { + if (verbose) { + console.log(`${LOG_TAG} BASE_MODULE is not set to Liquid (${configContent.BASE_MODULE}), skipping downloading assets`); + } } -const testnetAssetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.json'; -const testnetAssetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json'; - -console.log('Downloading assets'); -download(PATH + 'assets.json', assetsJsonUrl); -console.log('Downloading assets minimal'); -download(PATH + 'assets.minimal.json', assetsMinimalJsonUrl); -console.log('Downloading testnet assets'); -download(PATH + 'assets-testnet.json', testnetAssetsJsonUrl); -console.log('Downloading testnet assets minimal'); -download(PATH + 'assets-testnet.minimal.json', testnetAssetsMinimalJsonUrl); - -downloadMiningPoolLogos$() - .then(() => downloadPromoVideoSubtiles$()) - .then(() => downloadPromoVideo$()) +(() => { + if (verbose) { + console.log(`${LOG_TAG} Downloading mining pool logos`); + } + downloadMiningPoolLogos$() + .then(() => { + if (verbose) { + console.log(`${LOG_TAG} Downloading promo video subtitles`); + } + downloadPromoVideoSubtiles$(); + }) + .then(() => { + if (verbose) { + console.log(`${LOG_TAG} Downloading promo video`); + } + downloadPromoVideo$(); + }) .catch((error) => { throw new Error(error); }); +})(); \ No newline at end of file