machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add SKIP_SYNC and verbosity from previous PR

Browse Source
This commit is contained in:
Felipe Knorr Kuhn 2023-09-23 15:39:00 -07:00
parent 4ed629a8ea
commit c35f8a3f08
No known key found for this signature in database
GPG Key ID: 79619B52BB097C1A
1 changed files with 74 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
frontend/sync-assets.js
View File

@ -2,6 +2,18 @@ var https = require('https');
var fs = require('fs'); var fs = require('fs');
var crypto = require('crypto'); var crypto = require('crypto');
var path = require('node:path'); var path = require('node:path');
const LOG_TAG = '[sync-assets]';
let verbose = false;
if (parseInt(process.env.SKIP_SYNC) === 1) {
console.log(`${LOG_TAG} SKIP_SYNC is set, not checking any assets`);
process.exit(0);
}
if (parseInt(process.env.VERBOSE) === 1) {
console.log(`${LOG_TAG} VERBOSE is set, logs will be more verbose`);
verbose = true;
}
const CONFIG_FILE_NAME = 'mempool-frontend-config.json'; const CONFIG_FILE_NAME = 'mempool-frontend-config.json';
let configContent = {}; let configContent = {};
@ -13,7 +25,7 @@ if (process.argv[2]) {
PATH = path.normalize(PATH); PATH = path.normalize(PATH);
console.log(`[sync-assets] using PATH ${PATH}`); console.log(`[sync-assets] using PATH ${PATH}`);
if (!fs.existsSync(PATH)){ if (!fs.existsSync(PATH)){
console.log(`${PATH} does not exist, creating`); console.log(`${LOG_TAG} ${PATH} does not exist, creating`);
fs.mkdirSync(PATH, { recursive: true }); fs.mkdirSync(PATH, { recursive: true });
} }
} }
@ -25,12 +37,12 @@ if (!PATH) {
try { try {
const rawConfig = fs.readFileSync(CONFIG_FILE_NAME); const rawConfig = fs.readFileSync(CONFIG_FILE_NAME);
configContent = JSON.parse(rawConfig); configContent = JSON.parse(rawConfig);
console.log(`${CONFIG_FILE_NAME} file found, using provided config`); console.log(`${LOG_TAG} ${CONFIG_FILE_NAME} file found, using provided config`);
} catch (e) { } catch (e) {
if (e.code !== 'ENOENT') { if (e.code !== 'ENOENT') {
throw new Error(e); throw new Error(e);
} else { } else {
console.log(`${CONFIG_FILE_NAME} file not found, using default config`); console.log(`${LOG_TAG} ${CONFIG_FILE_NAME} file not found, using default config`);
} }
} }
@ -45,6 +57,11 @@ function download(filename, url) {
}) })
.on('error', function(e) { .on('error', function(e) {
throw new Error(e); throw new Error(e);
})
.on('finish', () => {
if (verbose) {
console.log(`${LOG_TAG} Finished downloading ${url} to ${filename}`);
}
}); });
} }
@ -52,12 +69,18 @@ function getLocalHash(filePath) {
const size = fs.statSync(filePath); const size = fs.statSync(filePath);
const buffer = fs.readFileSync(filePath); const buffer = fs.readFileSync(filePath);
const bufferWithHeader = Buffer.concat([Buffer.from('blob '), Buffer.from(`${size.size}`), Buffer.from('\0'), buffer]); const bufferWithHeader = Buffer.concat([Buffer.from('blob '), Buffer.from(`${size.size}`), Buffer.from('\0'), buffer]);
return crypto.createHash('sha1').update(bufferWithHeader).digest('hex'); const hash = crypto.createHash('sha1').update(bufferWithHeader).digest('hex');
if (verbose) {
console.log(`${LOG_TAG} \tgetLocalHash ${filePath} ${hash}`);
}
return hash;
} }
function downloadMiningPoolLogos$() { function downloadMiningPoolLogos$() {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
console.log('Checking if mining pool logos needs downloading or updating...'); console.log(`${LOG_TAG} Checking if mining pool logos needs downloading or updating...`);
const options = { const options = {
host: 'api.github.com', host: 'api.github.com',
path: '/repos/mempool/mining-pool-logos/contents/', path: '/repos/mempool/mining-pool-logos/contents/',
@ -66,7 +89,7 @@ function downloadMiningPoolLogos$() {
}; };
if (githubSecret) { if (githubSecret) {
console.log('Downloading the mining pool logos with authentication'); console.log(`${LOG_TAG} Downloading the mining pool logos with authentication`);
options.headers['authorization'] = `Bearer ${githubSecret}`; options.headers['authorization'] = `Bearer ${githubSecret}`;
options.headers['X-GitHub-Api-Version'] = '2022-11-28'; options.headers['X-GitHub-Api-Version'] = '2022-11-28';
} }
@ -90,13 +113,17 @@ function downloadMiningPoolLogos$() {
const filePath = PATH + `mining-pools/${poolLogo.name}`; const filePath = PATH + `mining-pools/${poolLogo.name}`;
if (fs.existsSync(filePath)) { if (fs.existsSync(filePath)) {
const localHash = getLocalHash(filePath); const localHash = getLocalHash(filePath);
if (verbose) {
console.log(`${LOG_TAG} Remote ${poolLogo.name} logo hash ${poolLogo.sha}`);
console.log(`${LOG_TAG} \tchecking if ${filePath} exists: ${fs.existsSync(filePath)}`);
}
if (localHash !== poolLogo.sha) { if (localHash !== poolLogo.sha) {
console.log(`${poolLogo.name} is different on the remote, downloading...`); console.log(`${LOG_TAG} \t\t${poolLogo.name} is different on the remote, downloading...`);
download(filePath, poolLogo.download_url); download(filePath, poolLogo.download_url);
downloadedCount++; downloadedCount++;
} }
} else { } else {
console.log(`${poolLogo.name} is missing, downloading...`); console.log(`${LOG_TAG} ${poolLogo.name} is missing, downloading...`);
const miningPoolsDir = PATH + `mining-pools/`; const miningPoolsDir = PATH + `mining-pools/`;
if (!fs.existsSync(miningPoolsDir)){ if (!fs.existsSync(miningPoolsDir)){
fs.mkdirSync(miningPoolsDir, { recursive: true }); fs.mkdirSync(miningPoolsDir, { recursive: true });
@ -105,7 +132,7 @@ function downloadMiningPoolLogos$() {
downloadedCount++; downloadedCount++;
} }
} }
console.log(`Downloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`); console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`);
resolve(); resolve();
} catch (e) { } catch (e) {
reject(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); reject(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
@ -121,7 +148,7 @@ function downloadMiningPoolLogos$() {
function downloadPromoVideoSubtiles$() { function downloadPromoVideoSubtiles$() {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
console.log('Checking if promo video subtitles needs downloading or updating...'); console.log(`${LOG_TAG} Checking if promo video subtitles needs downloading or updating...`);
const options = { const options = {
host: 'api.github.com', host: 'api.github.com',
path: '/repos/mempool/mempool-promo/contents/subtitles', path: '/repos/mempool/mempool-promo/contents/subtitles',
@ -130,7 +157,7 @@ function downloadPromoVideoSubtiles$() {
}; };
if (githubSecret) { if (githubSecret) {
console.log('Downloading the promo video subtitles with authentication'); console.log(`${LOG_TAG} Downloading the promo video subtitles with authentication`);
options.headers['authorization'] = `Bearer ${githubSecret}`; options.headers['authorization'] = `Bearer ${githubSecret}`;
options.headers['X-GitHub-Api-Version'] = '2022-11-28'; options.headers['X-GitHub-Api-Version'] = '2022-11-28';
} }
@ -154,14 +181,18 @@ function downloadPromoVideoSubtiles$() {
for (const language of videoLanguages) { for (const language of videoLanguages) {
const filePath = PATH + `promo-video/${language.name}`; const filePath = PATH + `promo-video/${language.name}`;
if (fs.existsSync(filePath)) { if (fs.existsSync(filePath)) {
if (verbose) {
console.log(`${LOG_TAG} ${language.name} remote promo video hash ${language.sha}`);
}
const localHash = getLocalHash(filePath); const localHash = getLocalHash(filePath);
if (localHash !== language.sha) { if (localHash !== language.sha) {
console.log(`${language.name} is different on the remote, updating`); console.log(`${LOG_TAG} ${language.name} is different on the remote, updating`);
download(filePath, language.download_url); download(filePath, language.download_url);
downloadedCount++; downloadedCount++;
} }
} else { } else {
console.log(`${language.name} is missing, downloading`); console.log(`${LOG_TAG} ${language.name} is missing, downloading`);
const promoVideosDir = PATH + `promo-video/`; const promoVideosDir = PATH + `promo-video/`;
if (!fs.existsSync(promoVideosDir)){ if (!fs.existsSync(promoVideosDir)){
fs.mkdirSync(promoVideosDir, { recursive: true }); fs.mkdirSync(promoVideosDir, { recursive: true });
@ -171,7 +202,7 @@ function downloadPromoVideoSubtiles$() {
downloadedCount++; downloadedCount++;
} }
} }
console.log(`Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`); console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`);
resolve(); resolve();
} catch (e) { } catch (e) {
reject(`Unable to download video subtitles. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); reject(`Unable to download video subtitles. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
@ -187,7 +218,7 @@ function downloadPromoVideoSubtiles$() {
function downloadPromoVideo$() { function downloadPromoVideo$() {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
console.log('Checking if promo video needs downloading or updating...'); console.log(`${LOG_TAG} Checking if promo video needs downloading or updating...`);
const options = { const options = {
host: 'api.github.com', host: 'api.github.com',
path: '/repos/mempool/mempool-promo/contents', path: '/repos/mempool/mempool-promo/contents',
@ -196,7 +227,7 @@ function downloadPromoVideo$() {
}; };
if (githubSecret) { if (githubSecret) {
console.log('Downloading the promo videos with authentication'); console.log(`${LOG_TAG} Downloading the promo video with authentication`);
options.headers['authorization'] = `Bearer ${githubSecret}`; options.headers['authorization'] = `Bearer ${githubSecret}`;
options.headers['X-GitHub-Api-Version'] = '2022-11-28'; options.headers['X-GitHub-Api-Version'] = '2022-11-28';
} }
@ -222,15 +253,16 @@ function downloadPromoVideo$() {
const filePath = PATH + `promo-video/mempool-promo.mp4`; const filePath = PATH + `promo-video/mempool-promo.mp4`;
if (fs.existsSync(filePath)) { if (fs.existsSync(filePath)) {
const localHash = getLocalHash(filePath); const localHash = getLocalHash(filePath);
if (localHash !== item.sha) { if (localHash !== item.sha) {
console.log(`mempool-promo.mp4 is different on the remote, updating`); console.log(`${LOG_TAG} \tmempool-promo.mp4 is different on the remote, updating`);
download(filePath, item.download_url); download(filePath, item.download_url);
console.log('mempool-promo.mp4 downloaded.'); console.log(`${LOG_TAG} \tmempool-promo.mp4 downloaded.`);
} else { } else {
console.log(`mempool-promo.mp4 is already up to date. Skipping.`); console.log(`${LOG_TAG} \tmempool-promo.mp4 is already up to date. Skipping.`);
} }
} else { } else {
console.log(`mempool-promo.mp4 is missing, downloading`); console.log(`${LOG_TAG} \tmempool-promo.mp4 is missing, downloading`);
download(filePath, item.download_url); download(filePath, item.download_url);
} }
} }
@ -261,21 +293,36 @@ if (configContent.BASE_MODULE && configContent.BASE_MODULE === 'liquid') {
const testnetAssetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.json'; const testnetAssetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.json';
const testnetAssetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json'; const testnetAssetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json';
console.log('Downloading assets'); console.log(`${LOG_TAG} Downloading assets`);
download(PATH + 'assets.json', assetsJsonUrl); download(PATH + 'assets.json', assetsJsonUrl);
console.log('Downloading assets minimal'); console.log(`${LOG_TAG} Downloading assets minimal`);
download(PATH + 'assets.minimal.json', assetsMinimalJsonUrl); download(PATH + 'assets.minimal.json', assetsMinimalJsonUrl);
console.log('Downloading testnet assets'); console.log(`${LOG_TAG} Downloading testnet assets`);
download(PATH + 'assets-testnet.json', testnetAssetsJsonUrl); download(PATH + 'assets-testnet.json', testnetAssetsJsonUrl);
console.log('Downloading testnet assets minimal'); console.log(`${LOG_TAG} Downloading testnet assets minimal`);
download(PATH + 'assets-testnet.minimal.json', testnetAssetsMinimalJsonUrl); download(PATH + 'assets-testnet.minimal.json', testnetAssetsMinimalJsonUrl);
downloadMiningPoolLogos$() (() => {
.then(() => downloadPromoVideoSubtiles$()) if (verbose) {
.then(() => downloadPromoVideo$()) console.log(`${LOG_TAG} Downloading mining pool logos`);
}
downloadMiningPoolLogos$()
.then(() => {
if (verbose) {
console.log(`${LOG_TAG} Downloading promo video subtitles`);
}
downloadPromoVideoSubtiles$();
})
.then(() => {
if (verbose) {
console.log(`${LOG_TAG} Downloading promo video`);
}
downloadPromoVideo$();
})
.catch((error) => { .catch((error) => {
throw new Error(error); throw new Error(error);
}); });
})();