machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1569 from mempool/nymkappa/bugfix/bisq-dump-loading

Browse Source 
Read bisq dump blocks line by line and parse manually
This commit is contained in:
wiz 2022-04-25 23:06:03 +00:00 committed by GitHub
commit 14e23831eb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 53 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
backend/src/api/bisq/bisq.ts
View File

@ -157,8 +157,7 @@ class Bisq {
private async loadBisqDumpFile(): Promise<void> {
try {
const data = await this.loadData();
await this.loadBisqBlocksDump(data);
await this.loadData();
this.buildIndex();
this.calculateStats();
} catch (e) {
@ -241,36 +240,61 @@ class Bisq {
};
}
private async loadBisqBlocksDump(cacheData: string): Promise<void> {
const start = new Date().getTime();
if (cacheData && cacheData.length !== 0) {
logger.debug('Processing Bisq data dump...');
const data: BisqBlocks = await this.jsonParsePool.exec(cacheData);
if (data.blocks && data.blocks.length !== this.allBlocks.length) {
this.allBlocks = data.blocks;
this.allBlocks.reverse();
this.blocks = this.allBlocks.filter((block) => block.txs.length > 0);
this.latestBlockHeight = data.chainHeight;
const time = new Date().getTime() - start;
logger.debug('Bisq dump processed in ' + time + ' ms (worker thread)');
} else {
throw new Error(`Bisq dump didn't contain any blocks`);
private async loadData(): Promise<any> {
if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
throw new Error(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist`);
}
const readline = require('readline');
const events = require('events');
const rl = readline.createInterface({
input: fs.createReadStream(Bisq.BLOCKS_JSON_FILE_PATH),
crlfDelay: Infinity
});
let blockBuffer = '';
let readingBlock = false;
let lineCount = 1;
const start = new Date().getTime();
logger.debug('Processing Bisq data dump...');
rl.on('line', (line) => {
if (lineCount === 2) {
line = line.replace(' "chainHeight": ', '');
this.latestBlockHeight = parseInt(line, 10);
}
if (line === ' {') {
readingBlock = true;
} else if (line === ' },') {
blockBuffer += '}';
try {
const block: BisqBlock = JSON.parse(blockBuffer);
this.allBlocks.push(block);
readingBlock = false;
blockBuffer = '';
} catch (e) {
logger.debug(blockBuffer);
throw Error(`Unable to parse Bisq data dump at line ${lineCount}` + (e instanceof Error ? e.message : e));
}
}
private loadData(): Promise<string> {
return new Promise((resolve, reject) => {
if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
return reject(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist`);
if (readingBlock === true) {
blockBuffer += line;
}
fs.readFile(Bisq.BLOCKS_JSON_FILE_PATH, 'utf8', (err, data) => {
if (err) {
reject(err);
}
resolve(data);
});
++lineCount;
});
await events.once(rl, 'close');
this.allBlocks.reverse();
this.blocks = this.allBlocks.filter((block) => block.txs.length > 0);
const time = new Date().getTime() - start;
logger.debug('Bisq dump processed in ' + time + ' ms');
}
}