Adding video to sha1 check and chaining requests in promises.

This commit is contained in:
softsimon 2023-05-14 12:45:42 -05:00
parent da3c3e8f5c
commit f12403747d
No known key found for this signature in database
GPG Key ID: 488D7DCFB5A430D7

View File

@ -45,100 +45,173 @@ function getLocalHash(filePath) {
return crypto.createHash('sha1').update(bufferWithHeader).digest('hex'); return crypto.createHash('sha1').update(bufferWithHeader).digest('hex');
} }
function downloadMiningPoolLogos() { function downloadMiningPoolLogos$() {
const options = { return new Promise((resolve, reject) => {
host: 'api.github.com', console.log('Checking if mining pool logos needs downloading or updating...');
path: '/repos/mempool/mining-pool-logos/contents/', const options = {
method: 'GET', host: 'api.github.com',
headers: {'user-agent': 'node.js'} path: '/repos/mempool/mining-pool-logos/contents/',
}; method: 'GET',
headers: {'user-agent': 'node.js'}
};
https.get(options, (response) => { https.get(options, (response) => {
const chunks_of_data = []; const chunks_of_data = [];
response.on('data', (fragments) => { response.on('data', (fragments) => {
chunks_of_data.push(fragments); chunks_of_data.push(fragments);
}); });
response.on('end', () => { response.on('end', () => {
const response_body = Buffer.concat(chunks_of_data); const response_body = Buffer.concat(chunks_of_data);
try { try {
const poolLogos = JSON.parse(response_body.toString()); const poolLogos = JSON.parse(response_body.toString());
let downloadedCount = 0; if (poolLogos.message) {
for (const poolLogo of poolLogos) { reject(poolLogos.message);
const filePath = `${PATH}/mining-pools/${poolLogo.name}`; }
if (fs.existsSync(filePath)) { let downloadedCount = 0;
const localHash = getLocalHash(filePath); for (const poolLogo of poolLogos) {
if (localHash !== poolLogo.sha) { const filePath = `${PATH}/mining-pools/${poolLogo.name}`;
console.log(`${poolLogo.name} is different on the remote, updating`); if (fs.existsSync(filePath)) {
download(filePath, poolLogo.download_url); const localHash = getLocalHash(filePath);
if (localHash !== poolLogo.sha) {
console.log(`${poolLogo.name} is different on the remote, downloading...`);
download(filePath, poolLogo.download_url);
downloadedCount++;
}
} else {
console.log(`${poolLogo.name} is missing, downloading...`);
download(`${PATH}mining-pools/${poolLogo.name}`, poolLogo.download_url);
downloadedCount++; downloadedCount++;
} }
} else {
console.log(`${poolLogo.name} is missing, downloading`);
download(`${PATH}mining-pools/${poolLogo.name}`, poolLogo.download_url);
downloadedCount++;
} }
console.log(`Downloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`);
resolve();
} catch (e) {
reject(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
} }
console.log(`Downloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`); });
} catch (e) {
console.error(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
}
});
response.on('error', (error) => { response.on('error', (error) => {
throw new Error(error); reject(error);
});
}); });
}); });
} }
function downloadPromoVideoSubtiles() { function downloadPromoVideoSubtiles$() {
const options = { return new Promise((resolve, reject) => {
host: 'api.github.com', console.log('Checking if promo video subtitles needs downloading or updating...');
path: '/repos/mempool/mempool-promo/contents/subtitles', const options = {
method: 'GET', host: 'api.github.com',
headers: {'user-agent': 'node.js'} path: '/repos/mempool/mempool-promo/contents/subtitles',
}; method: 'GET',
headers: {'user-agent': 'node.js'}
};
https.get(options, (response) => { https.get(options, (response) => {
const chunks_of_data = []; const chunks_of_data = [];
response.on('data', (fragments) => { response.on('data', (fragments) => {
chunks_of_data.push(fragments); chunks_of_data.push(fragments);
}); });
response.on('end', () => { response.on('end', () => {
const response_body = Buffer.concat(chunks_of_data); const response_body = Buffer.concat(chunks_of_data);
try { try {
const videoLanguages = JSON.parse(response_body.toString()); const videoLanguages = JSON.parse(response_body.toString());
let downloadedCount = 0; if (videoLanguages.message) {
for (const language of videoLanguages) { reject(videoLanguages.message);
const filePath = `${PATH}/promo-video/${language.name}`; }
if (fs.existsSync(filePath)) { let downloadedCount = 0;
const localHash = getLocalHash(filePath); for (const language of videoLanguages) {
if (localHash !== language.sha) { const filePath = `${PATH}/promo-video/${language.name}`;
console.log(`${language.name} is different on the remote, updating`); if (fs.existsSync(filePath)) {
const localHash = getLocalHash(filePath);
if (localHash !== language.sha) {
console.log(`${language.name} is different on the remote, updating`);
download(filePath, language.download_url);
downloadedCount++;
}
} else {
console.log(`${language.name} is missing, downloading`);
download(filePath, language.download_url); download(filePath, language.download_url);
downloadedCount++; downloadedCount++;
} }
} else {
console.log(`${language.name} is missing, downloading`);
download(filePath, language.download_url);
downloadedCount++;
} }
console.log(`Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`);
resolve();
} catch (e) {
reject(`Unable to download video subtitles. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
} }
console.log(`Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`); });
} catch (e) {
console.error(`Unable to download video subtitles. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
}
});
response.on('error', (error) => { response.on('error', (error) => {
throw new Error(error); reject(error);
});
}); });
}); });
} }
function downloadPromoVideo$() {
return new Promise((resolve, reject) => {
console.log('Checking if promo video needs downloading or updating...');
const options = {
host: 'api.github.com',
path: '/repos/mempool/mempool-promo/contents',
method: 'GET',
headers: {'user-agent': 'node.js'}
};
https.get(options, (response) => {
const chunks_of_data = [];
response.on('data', (fragments) => {
chunks_of_data.push(fragments);
});
response.on('end', () => {
const response_body = Buffer.concat(chunks_of_data);
try {
const contents = JSON.parse(response_body.toString());
if (contents.message) {
reject(contents.message);
}
for (const item of contents) {
if (item.name !== 'promo.mp4') {
continue;
}
const filePath = `${PATH}/promo-video/mempool-promo.mp4`;
if (fs.existsSync(filePath)) {
const localHash = getLocalHash(filePath);
if (localHash !== item.sha) {
console.log(`mempool-promo.mp4 is different on the remote, updating`);
download(filePath, item.download_url);
console.log('mempool-promo.mp4 downloaded.');
} else {
console.log(`mempool-promo.mp4 is already up to date. Skipping.`);
}
} else {
console.log(`mempool-promo.mp4 is missing, downloading`);
download(filePath, item.download_url);
}
}
resolve();
} catch (e) {
reject(`Unable to download video. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
}
});
response.on('error', (error) => {
reject(error);
});
});
});
}
let assetsJsonUrl = 'https://raw.githubusercontent.com/mempool/asset_registry_db/master/index.json'; let assetsJsonUrl = 'https://raw.githubusercontent.com/mempool/asset_registry_db/master/index.json';
let assetsMinimalJsonUrl = 'https://raw.githubusercontent.com/mempool/asset_registry_db/master/index.minimal.json'; let assetsMinimalJsonUrl = 'https://raw.githubusercontent.com/mempool/asset_registry_db/master/index.minimal.json';
@ -150,10 +223,6 @@ if (configContent.BASE_MODULE && configContent.BASE_MODULE === 'liquid') {
const testnetAssetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.json'; const testnetAssetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.json';
const testnetAssetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json'; const testnetAssetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json';
const promoVideoFile = PATH + '/promo-video/mempool-promo.mp4';
const promoVideoUrl = 'https://raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4';
console.log('Downloading assets'); console.log('Downloading assets');
download(PATH + 'assets.json', assetsJsonUrl); download(PATH + 'assets.json', assetsJsonUrl);
console.log('Downloading assets minimal'); console.log('Downloading assets minimal');
@ -162,11 +231,10 @@ console.log('Downloading testnet assets');
download(PATH + 'assets-testnet.json', testnetAssetsJsonUrl); download(PATH + 'assets-testnet.json', testnetAssetsJsonUrl);
console.log('Downloading testnet assets minimal'); console.log('Downloading testnet assets minimal');
download(PATH + 'assets-testnet.minimal.json', testnetAssetsMinimalJsonUrl); download(PATH + 'assets-testnet.minimal.json', testnetAssetsMinimalJsonUrl);
if (!fs.existsSync(promoVideoFile)) {
console.log('Downloading promo video'); downloadMiningPoolLogos$()
download(promoVideoFile, promoVideoUrl); .then(() => downloadPromoVideoSubtiles$())
} .then(() => downloadPromoVideo$())
console.log('Downloading promo video subtitles'); .catch((error) => {
downloadPromoVideoSubtiles(); throw new Error(error);
console.log('Downloading mining pool logos'); });
downloadMiningPoolLogos();