machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve statistics replication service

Browse Source
This commit is contained in:
natsoni 2024-05-21 11:33:37 +02:00
parent b0630de3cc
commit 60a07fb093
No known key found for this signature in database
GPG Key ID: C65917583181743B
1 changed files with 61 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
backend/src/replication/StatisticsReplication.ts
View File

@ -6,13 +6,13 @@ import { Common } from '../api/common';
import statistics from '../api/statistics/statistics-api'; import statistics from '../api/statistics/statistics-api';
interface MissingStatistics { interface MissingStatistics {
'24h': number[]; '24h': Set<number>;
'1w': number[]; '1w': Set<number>;
'1m': number[]; '1m': Set<number>;
'3m': number[]; '3m': Set<number>;
'6m': number[]; '6m': Set<number>;
'2y': number[]; '2y': Set<number>;
'all': number[]; 'all': Set<number>;
} }
const steps = { const steps = {
@ -30,7 +30,6 @@ const steps = {
*/ */
class StatisticsReplication { class StatisticsReplication {
inProgress: boolean = false; inProgress: boolean = false;
skip: Set<number> = new Set();
public async $sync(): Promise<void> { public async $sync(): Promise<void> {
if (!config.REPLICATION.ENABLED || !config.REPLICATION.STATISTICS || !config.STATISTICS.ENABLED) { if (!config.REPLICATION.ENABLED || !config.REPLICATION.STATISTICS || !config.STATISTICS.ENABLED) {
@ -44,8 +43,8 @@ class StatisticsReplication {
this.inProgress = true; this.inProgress = true;
const missingStatistics = await this.$getMissingStatistics(); const missingStatistics = await this.$getMissingStatistics();
const missingIntervals = Object.keys(missingStatistics).filter(key => missingStatistics[key].length > 0); const missingIntervals = Object.keys(missingStatistics).filter(key => missingStatistics[key].size > 0);
const totalMissing = missingIntervals.reduce((total, key) => total + missingStatistics[key].length, 0); const totalMissing = missingIntervals.reduce((total, key) => total + missingStatistics[key].size, 0);
if (totalMissing === 0) { if (totalMissing === 0) {
this.inProgress = false; this.inProgress = false;
@ -54,7 +53,7 @@ class StatisticsReplication {
} }
for (const interval of missingIntervals) { for (const interval of missingIntervals) {
logger.debug(`Missing ${missingStatistics[interval].length} statistics rows in '${interval}' timespan`, 'Replication'); logger.debug(`Missing ${missingStatistics[interval].size} statistics rows in '${interval}' timespan`, 'Replication');
} }
logger.debug(`Fetching ${missingIntervals.join(', ')} statistics endpoints from trusted servers to fill ${totalMissing} rows missing in statistics`, 'Replication'); logger.debug(`Fetching ${missingIntervals.join(', ')} statistics endpoints from trusted servers to fill ${totalMissing} rows missing in statistics`, 'Replication');
@ -75,31 +74,35 @@ class StatisticsReplication {
this.inProgress = false; this.inProgress = false;
} }
private async $syncStatistics(interval: string, missingTimes: number[]): Promise<any> { private async $syncStatistics(interval: string, missingTimes: Set<number>): Promise<any> {
let success = false; let success = false;
let synced = 0; let synced = 0;
let missed = 0; let missed = new Set(missingTimes);
const syncResult = await $sync(`/api/v1/statistics/${interval}`); const syncResult = await $sync(`/api/v1/statistics/${interval}`);
if (syncResult) { if (syncResult && syncResult.data?.length) {
if (syncResult.data?.length) { success = true;
success = true; logger.info(`Fetched /api/v1/statistics/${interval} from ${syncResult.server}`);
logger.info(`Fetched /api/v1/statistics/${interval} from ${syncResult.server}`);
const fetchedTimes = syncResult.data.map((stat: any) => stat.added); for (const stat of syncResult.data) {
for (const time of missingTimes) { const time = this.roundToNearestStep(stat.added, steps[interval]);
const closest = syncResult.data[this.getClosestElement(time, fetchedTimes).index]; if (missingTimes.has(time)) {
if (Math.abs(time - closest.added) < steps[interval]) { try {
await statistics.$create(statistics.mapOptimizedStatisticToStatistic([closest])[0], true); await statistics.$create(statistics.mapOptimizedStatisticToStatistic([stat])[0], true);
synced++; if (missed.delete(time)) {
} else { synced++;
missed++; }
this.skip.add(time); } catch (e: any) {
logger.err(`Failed to insert statistics row at ${stat.added} (${interval}) from ${syncResult.server}. Reason: ` + (e instanceof Error ? e.message : e));
} }
} }
} }
} else {
logger.warn(`An error occured when trying to fetch /api/v1/statistics/${interval}`);
} }
return { success, synced, missed }; return { success, synced, missed: missed.size };
} }
private async $getMissingStatistics(): Promise<MissingStatistics> { private async $getMissingStatistics(): Promise<MissingStatistics> {
@ -109,7 +112,15 @@ class StatisticsReplication {
const startTime = this.getStartTimeFromConfig(); const startTime = this.getStartTimeFromConfig();
const missingStatistics: MissingStatistics = { '24h': [], '1w': [], '1m': [], '3m': [], '6m': [], '2y': [], 'all': [] }; const missingStatistics: MissingStatistics = {
'24h': new Set<number>(),
'1w': new Set<number>(),
'1m': new Set<number>(),
'3m': new Set<number>(),
'6m': new Set<number>(),
'2y': new Set<number>(),
'all': new Set<number>()
};
const intervals = [ // [start, end, label ] const intervals = [ // [start, end, label ]
[now - day, now - 60, '24h'] , // from 24 hours ago to now = 1 minute granularity [now - day, now - 60, '24h'] , // from 24 hours ago to now = 1 minute granularity
@ -135,7 +146,7 @@ class StatisticsReplication {
} }
} }
private async $getMissingStatisticsInterval(interval: any, startTime: number): Promise<number[]> { private async $getMissingStatisticsInterval(interval: any, startTime: number): Promise<Set<number>> {
try { try {
const start = interval[0]; const start = interval[0];
const end = interval[1]; const end = interval[1];
@ -145,23 +156,26 @@ class StatisticsReplication {
SELECT UNIX_TIMESTAMP(added) as added SELECT UNIX_TIMESTAMP(added) as added
FROM statistics FROM statistics
WHERE added >= FROM_UNIXTIME(?) AND added <= FROM_UNIXTIME(?) WHERE added >= FROM_UNIXTIME(?) AND added <= FROM_UNIXTIME(?)
GROUP BY UNIX_TIMESTAMP(added) DIV ${step} ORDER BY statistics.added DESC
`, [start, end]); `, [start, end]);
const timesAlreadyHere = rows.map(row => row.added);
const missingTimes: number[] = []; const startingTime = Math.max(startTime, start) - Math.max(startTime, start) % step;
if (timesAlreadyHere.length === 0) { const timeSteps: number[] = [];
for (let time = Math.max(startTime, start); time < end; time += step) { for (let time = startingTime; time < end; time += step) {
missingTimes.push(time); timeSteps.push(time);
}
return missingTimes;
} }
for (let time = Math.max(startTime, start); time < end; time += step) { if (timeSteps.length === 0) {
const closest = this.getClosestElement(time, timesAlreadyHere); return new Set<number>();
if (Math.abs(time - closest.value) > step && !this.skip.has(time)) { }
missingTimes.push(time);
} const roundedTimesAlreadyHere = new Set(rows.map(row => this.roundToNearestStep(row.added, step)));
const missingTimes = new Set(timeSteps.filter(time => !roundedTimesAlreadyHere.has(time)));
// Don't bother fetching if very few rows are missing
if (missingTimes.size < timeSteps.length * 0.005) {
return new Set();
} }
return missingTimes; return missingTimes;
@ -171,16 +185,13 @@ class StatisticsReplication {
} }
} }
private getClosestElement(element: number, list: number[]): {index: number, value: number} { private roundToNearestStep(time: number, step: number): number {
let closest = list[0]; const remainder = time % step;
let index = 0; if (remainder < step / 2) {
for (let i = 0; i < list.length; i++) { return time - remainder;
if (Math.abs(list[i] - element) < Math.abs(closest - element)) { } else {
closest = list[i]; return time + (step - remainder);
index = i;
}
} }
return { index, value: closest };
} }
private getStartTimeFromConfig(): number { private getStartTimeFromConfig(): number {