machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add DRY_RUN and MEMPOOL_CDN options

Browse Source
This commit is contained in:
Felipe Knorr Kuhn 2024-02-04 09:17:42 -08:00
parent a15729c38f
commit 933bc47b44
No known key found for this signature in database
GPG Key ID: 79619B52BB097C1A
1 changed files with 116 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

147
frontend/sync-assets.js
View File

@ -4,6 +4,8 @@ var crypto = require('crypto');
var path = require('node:path'); var path = require('node:path');
const LOG_TAG = '[sync-assets]'; const LOG_TAG = '[sync-assets]';
let verbose = false; let verbose = false;
let MEMPOOL_CDN = false;
let DRY_RUN = false;
if (parseInt(process.env.SKIP_SYNC) === 1) { if (parseInt(process.env.SKIP_SYNC) === 1) {
console.log(`${LOG_TAG} SKIP_SYNC is set, not checking any assets`); console.log(`${LOG_TAG} SKIP_SYNC is set, not checking any assets`);
@ -15,6 +17,18 @@ if (parseInt(process.env.VERBOSE) === 1) {
verbose = true; verbose = true;
} }
if (parseInt(process.env.MEMPOOL_CDN) === 1) {
console.log(`${LOG_TAG} MEMPOOL_CDN is set, assets will be downloaded from mempool.space`);
MEMPOOL_CDN = true;
}
if (parseInt(process.env.DRY_RUN) === 1) {
console.log(`${LOG_TAG} DRY_RUN is set, not downloading any assets`);
DRY_RUN = true;
}
const githubSecret = process.env.GITHUB_TOKEN;
const CONFIG_FILE_NAME = 'mempool-frontend-config.json'; const CONFIG_FILE_NAME = 'mempool-frontend-config.json';
let configContent = {}; let configContent = {};
@ -46,8 +60,6 @@ try {
} }
} }
const githubSecret = process.env.GITHUB_TOKEN;
function download(filename, url) { function download(filename, url) {
https.get(url, (response) => { https.get(url, (response) => {
if (response.statusCode < 200 || response.statusCode > 299) { if (response.statusCode < 200 || response.statusCode > 299) {
@ -60,7 +72,7 @@ function download(filename, url) {
}) })
.on('finish', () => { .on('finish', () => {
if (verbose) { if (verbose) {
console.log(`${LOG_TAG} Finished downloading ${url} to ${filename}`); console.log(`${LOG_TAG} \tFinished downloading ${url} to ${filename}`);
} }
}); });
} }
@ -72,7 +84,7 @@ function getLocalHash(filePath) {
const hash = crypto.createHash('sha1').update(bufferWithHeader).digest('hex'); const hash = crypto.createHash('sha1').update(bufferWithHeader).digest('hex');
if (verbose) { if (verbose) {
console.log(`${LOG_TAG} \tgetLocalHash ${filePath} ${hash}`); console.log(`${LOG_TAG} \t\tgetLocalHash ${filePath} ${hash}`);
} }
return hash; return hash;
@ -80,7 +92,7 @@ function getLocalHash(filePath) {
function downloadMiningPoolLogos$() { function downloadMiningPoolLogos$() {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
console.log(`${LOG_TAG} Checking if mining pool logos needs downloading or updating...`); console.log(`${LOG_TAG} \tChecking if mining pool logos needs downloading or updating...`);
const options = { const options = {
host: 'api.github.com', host: 'api.github.com',
path: '/repos/mempool/mining-pool-logos/contents/', path: '/repos/mempool/mining-pool-logos/contents/',
@ -110,29 +122,54 @@ function downloadMiningPoolLogos$() {
} }
let downloadedCount = 0; let downloadedCount = 0;
for (const poolLogo of poolLogos) { for (const poolLogo of poolLogos) {
if (verbose) {
console.log(`${LOG_TAG} Processing ${poolLogo.name}`);
}
const filePath = `${PATH}/mining-pools/${poolLogo.name}`; const filePath = `${PATH}/mining-pools/${poolLogo.name}`;
if (fs.existsSync(filePath)) { if (fs.existsSync(filePath)) {
const localHash = getLocalHash(filePath); const localHash = getLocalHash(filePath);
if (verbose) { if (verbose) {
console.log(`${LOG_TAG} Remote ${poolLogo.name} logo hash ${poolLogo.sha}`); console.log(`${LOG_TAG} \t\tremote ${poolLogo.name} logo hash ${poolLogo.sha}`);
console.log(`${LOG_TAG} \tchecking if ${filePath} exists: ${fs.existsSync(filePath)}`); console.log(`${LOG_TAG} \t\t\tchecking if ${filePath} exists: ${fs.existsSync(filePath)}`);
} }
if (localHash !== poolLogo.sha) { if (localHash !== poolLogo.sha) {
console.log(`${LOG_TAG} \t\t${poolLogo.name} is different on the remote, downloading...`); console.log(`${LOG_TAG} \t\t\t\t${poolLogo.name} is different on the remote, downloading...`);
download(filePath, poolLogo.download_url); let download_url = poolLogo.download_url;
downloadedCount++; if (MEMPOOL_CDN) {
download_url = download_url.replace("raw.githubusercontent.com/mempool/mining-pool-logos/master", "mempool.space/resources/mining-pools");
}
if (DRY_RUN) {
console.log(`${LOG_TAG} \t\tDRY_RUN is set, not downloading ${poolLogo.name} but we should`);
} else {
if (verbose) {
console.log(`${LOG_TAG} \t\tDownloading ${download_url} to ${filePath}`);
}
download(filePath, download_url);
downloadedCount++;
}
} else {
console.log(`${LOG_TAG} \t\t${poolLogo.name} is already up to date. Skipping.`);
} }
} else { } else {
console.log(`${LOG_TAG} ${poolLogo.name} is missing, downloading...`); console.log(`${LOG_TAG} \t\t${poolLogo.name} is missing, downloading...`);
const miningPoolsDir = `${PATH}/mining-pools/`; const miningPoolsDir = `${PATH}/mining-pools/`;
if (!fs.existsSync(miningPoolsDir)){ if (!fs.existsSync(miningPoolsDir)){
fs.mkdirSync(miningPoolsDir, { recursive: true }); fs.mkdirSync(miningPoolsDir, { recursive: true });
} }
download(filePath, poolLogo.download_url); let download_url = poolLogo.download_url;
downloadedCount++; if (MEMPOOL_CDN) {
download_url = download_url.replace("raw.githubusercontent.com/mempool/mining-pool-logos/master", "mempool.space/resources/mining-pools");
}
if (DRY_RUN) {
console.log(`${LOG_TAG} DRY_RUN is set, not downloading ${poolLogo.name} but it should`);
} else {
console.log(`${LOG_TAG} \tDownloading ${download_url} to ${filePath}`);
download(filePath, download_url);
downloadedCount++;
}
} }
} }
console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`); console.log(`${LOG_TAG} \t\tDownloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`);
resolve(); resolve();
} catch (e) { } catch (e) {
reject(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); reject(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
@ -148,7 +185,7 @@ function downloadMiningPoolLogos$() {
function downloadPromoVideoSubtiles$() { function downloadPromoVideoSubtiles$() {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
console.log(`${LOG_TAG} Checking if promo video subtitles needs downloading or updating...`); console.log(`${LOG_TAG} \tChecking if promo video subtitles needs downloading or updating...`);
const options = { const options = {
host: 'api.github.com', host: 'api.github.com',
path: '/repos/mempool/mempool-promo/contents/subtitles', path: '/repos/mempool/mempool-promo/contents/subtitles',
@ -157,7 +194,7 @@ function downloadPromoVideoSubtiles$() {
}; };
if (githubSecret) { if (githubSecret) {
console.log(`${LOG_TAG} Downloading the promo video subtitles with authentication`); console.log(`${LOG_TAG} \tDownloading the promo video subtitles with authentication`);
options.headers['authorization'] = `Bearer ${githubSecret}`; options.headers['authorization'] = `Bearer ${githubSecret}`;
options.headers['X-GitHub-Api-Version'] = '2022-11-28'; options.headers['X-GitHub-Api-Version'] = '2022-11-28';
} }
@ -179,27 +216,53 @@ function downloadPromoVideoSubtiles$() {
} }
let downloadedCount = 0; let downloadedCount = 0;
for (const language of videoLanguages) { for (const language of videoLanguages) {
if (verbose) {
console.log(`${LOG_TAG} Processing ${language.name}`);
}
const filePath = `${PATH}/promo-video/${language.name}`; const filePath = `${PATH}/promo-video/${language.name}`;
if (fs.existsSync(filePath)) { if (fs.existsSync(filePath)) {
if (verbose) { if (verbose) {
console.log(`${LOG_TAG} ${language.name} remote promo video hash ${language.sha}`); console.log(`${LOG_TAG} \t${language.name} remote promo video hash ${language.sha}`);
} }
const localHash = getLocalHash(filePath); const localHash = getLocalHash(filePath);
if (localHash !== language.sha) { if (localHash !== language.sha) {
console.log(`${LOG_TAG} ${language.name} is different on the remote, updating`); console.log(`${LOG_TAG} \t\t${language.name} is different on the remote, updating`);
download(filePath, language.download_url); let download_url = language.download_url;
downloadedCount++; if (MEMPOOL_CDN) {
download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/subtitles", "mempool.space/resources/promo-video");
}
if (DRY_RUN) {
console.log(`${LOG_TAG} \t\tDRY_RUN is set, not downloading ${language.name} but we should`);
} else {
if (verbose) {
console.log(`${LOG_TAG} \t\tdownloading ${download_url} to ${filePath}`);
}
download(filePath, download_url);
downloadedCount++;
}
} else {
console.log(`${LOG_TAG} \t\t${language.name} is already up to date. Skipping.`);
} }
} else { } else {
console.log(`${LOG_TAG} ${language.name} is missing, downloading`); console.log(`${LOG_TAG} \t\t${language.name} is missing, downloading`);
const promoVideosDir = `${PATH}/promo-video/`; const promoVideosDir = `${PATH}/promo-video/`;
if (!fs.existsSync(promoVideosDir)){ if (!fs.existsSync(promoVideosDir)){
fs.mkdirSync(promoVideosDir, { recursive: true }); fs.mkdirSync(promoVideosDir, { recursive: true });
} }
download(filePath, language.download_url); let download_url = language.download_url;
downloadedCount++; if (MEMPOOL_CDN) {
download_url = downloadownload_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/subtitles", "mempool.space/resources/promo-video");
}
if (DRY_RUN) {
console.log(`${LOG_TAG} \tDRY_RUN is set, not downloading ${language.name} but we should`);
} else {
if (verbose) {
console.log(`${LOG_TAG} downloading ${download_url} to ${filePath}`);
}
download(filePath, download_url);
downloadedCount++;
}
} }
} }
console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`); console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`);
@ -218,7 +281,7 @@ function downloadPromoVideoSubtiles$() {
function downloadPromoVideo$() { function downloadPromoVideo$() {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
console.log(`${LOG_TAG} Checking if promo video needs downloading or updating...`); console.log(`${LOG_TAG} \tChecking if promo video needs downloading or updating...`);
const options = { const options = {
host: 'api.github.com', host: 'api.github.com',
path: '/repos/mempool/mempool-promo/contents', path: '/repos/mempool/mempool-promo/contents',
@ -227,7 +290,7 @@ function downloadPromoVideo$() {
}; };
if (githubSecret) { if (githubSecret) {
console.log(`${LOG_TAG} Downloading the promo video with authentication`); console.log(`${LOG_TAG} \tDownloading the promo video with authentication`);
options.headers['authorization'] = `Bearer ${githubSecret}`; options.headers['authorization'] = `Bearer ${githubSecret}`;
options.headers['X-GitHub-Api-Version'] = '2022-11-28'; options.headers['X-GitHub-Api-Version'] = '2022-11-28';
} }
@ -256,14 +319,36 @@ function downloadPromoVideo$() {
if (localHash !== item.sha) { if (localHash !== item.sha) {
console.log(`${LOG_TAG} \tmempool-promo.mp4 is different on the remote, updating`); console.log(`${LOG_TAG} \tmempool-promo.mp4 is different on the remote, updating`);
download(filePath, item.download_url); let download_url = item.download_url;
console.log(`${LOG_TAG} \tmempool-promo.mp4 downloaded.`); if (MEMPOOL_CDN) {
download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4", "mempool.space/resources/promo-video/mempool-promo.mp4");
}
if (DRY_RUN) {
console.log(`${LOG_TAG} DRY_RUN is set, not downloading mempool-promo.mp4 but we should`);
} else {
if (verbose) {
console.log(`${LOG_TAG} downloading ${download_url} to ${filePath}`);
}
download(filePath, download_url);
console.log(`${LOG_TAG} \tmempool-promo.mp4 downloaded.`);
}
} else { } else {
console.log(`${LOG_TAG} \tmempool-promo.mp4 is already up to date. Skipping.`); console.log(`${LOG_TAG} \t\tmempool-promo.mp4 is already up to date. Skipping.`);
} }
} else { } else {
console.log(`${LOG_TAG} \tmempool-promo.mp4 is missing, downloading`); console.log(`${LOG_TAG} \tmempool-promo.mp4 is missing, downloading`);
download(filePath, item.download_url); let download_url = item.download_url;
if (MEMPOOL_CDN) {
download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4", "mempool.space/resources/promo-video/mempool-promo.mp4");
}
if (DRY_RUN) {
console.log(`${LOG_TAG} DRY_RUN is set, not downloading mempool-promo.mp4 but we should`);
} else {
if (verbose) {
console.log(`${LOG_TAG} downloading ${download_url} to ${filePath}`);
}
download(filePath, download_url);
}
} }
} }
resolve(); resolve();
@ -300,7 +385,7 @@ if (configContent.BASE_MODULE && configContent.BASE_MODULE === 'liquid') {
download(`${PATH}/assets-testnet.minimal.json`, testnetAssetsMinimalJsonUrl); download(`${PATH}/assets-testnet.minimal.json`, testnetAssetsMinimalJsonUrl);
} else { } else {
if (verbose) { if (verbose) {
console.log(`${LOG_TAG} BASE_MODULE is not set to Liquid (${configContent.BASE_MODULE}), skipping downloading assets`); console.log(`${LOG_TAG} BASE_MODULE is not set to Liquid (currently ${configContent.BASE_MODULE}), skipping downloading assets`);
} }
} }