machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Wait for external assets file writers to complete

Browse Source
This commit is contained in:
nymkappa
2022-04-22 04:03:08 -04:00
parent e8d6872620
commit 0565471ecf
5 changed files with 63 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
backend/src/sync-assets.ts
View File

@@ -9,55 +9,69 @@ const PATH = './';
class SyncAssets {
constructor() { }
public async syncAssets() {
public async syncAssets$() {
for (const url of config.MEMPOOL.EXTERNAL_ASSETS) {
await this.downloadFile(url);
try {
await this.downloadFile$(url);
} catch (e) {
throw new Error(`Failed to download external asset. ` + (e instanceof Error ? e.message : e));
}
}
}
private async downloadFile(url: string) {
const fileName = url.split('/').slice(-1)[0];
private async downloadFile$(url: string) {
return new Promise((resolve, reject) => {
const fileName = url.split('/').slice(-1)[0];
try {
if (config.SOCKS5PROXY.ENABLED) {
let socksOptions: any = {
agentOptions: {
keepAlive: true,
},
host: config.SOCKS5PROXY.HOST,
port: config.SOCKS5PROXY.PORT
};
try {
if (config.SOCKS5PROXY.ENABLED) {
const socksOptions: any = {
agentOptions: {
keepAlive: true,
},
host: config.SOCKS5PROXY.HOST,
port: config.SOCKS5PROXY.PORT
};
if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
socksOptions.username = config.SOCKS5PROXY.USERNAME;
socksOptions.password = config.SOCKS5PROXY.PASSWORD;
if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
socksOptions.username = config.SOCKS5PROXY.USERNAME;
socksOptions.password = config.SOCKS5PROXY.PASSWORD;
}
const agent = new SocksProxyAgent(socksOptions);
logger.info(`Downloading external asset ${fileName} over the Tor network...`);
return axios.get(url, {
httpAgent: agent,
httpsAgent: agent,
responseType: 'stream',
timeout: 30000
}).then(function (response) {
const writer = fs.createWriteStream(PATH + fileName);
writer.on('finish', () => {
logger.info(`External asset ${fileName} saved to ${PATH + fileName}`);
resolve(0);
});
response.data.pipe(writer);
});
} else {
logger.info(`Downloading external asset ${fileName} over clearnet...`);
return axios.get(url, {
responseType: 'stream',
timeout: 30000
}).then(function (response) {
const writer = fs.createWriteStream(PATH + fileName);
writer.on('finish', () => {
logger.info(`External asset ${fileName} saved to ${PATH + fileName}`);
resolve(0);
});
response.data.pipe(writer);
});
}
const agent = new SocksProxyAgent(socksOptions);
logger.info(`Downloading external asset ${fileName} over the Tor network...`);
await axios.get(url, {
httpAgent: agent,
httpsAgent: agent,
responseType: 'stream',
timeout: 30000
}).then(function (response) {
response.data.pipe(fs.createWriteStream(PATH + fileName));
logger.info(`External asset ${fileName} saved to ${PATH + fileName}`);
});
} else {
logger.info(`Downloading external asset ${fileName} over clearnet...`);
await axios.get(url, {
responseType: 'stream',
timeout: 30000
}).then(function (response) {
response.data.pipe(fs.createWriteStream(PATH + fileName));
logger.info(`External asset ${fileName} saved to ${PATH + fileName}`);
});
} catch (e: any) {
reject(e);
}
} catch (e: any) {
throw new Error(`Failed to download external asset. ` + e);
}
});
}
}