machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add FIRST_SEEN_INDEXING_DAYS config, auto-deletion

Browse Source
This commit is contained in:
Mononaut 2022-12-20 11:12:23 -06:00
parent c454ef0655
commit 28a10f2aaa
No known key found for this signature in database
GPG Key ID: A3F058E41374C04E
11 changed files with 46 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backend/mempool-config.sample.json
View File

@ -27,7 +27,8 @@
"POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
"ADVANCED_GBT_AUDIT": false,
"ADVANCED_GBT_MEMPOOL": false,
"TRANSACTION_INDEXING": false
"TRANSACTION_INDEXING": false,
"FIRST_SEEN_INDEXING_DAYS": 0
},
"CORE_RPC": {
"HOST": "127.0.0.1",

3
backend/src/__fixtures__/mempool-config.template.json
View File

@ -28,7 +28,8 @@
"POOLS_JSON_URL": "__POOLS_JSON_URL__",
"ADVANCED_GBT_AUDIT": "__ADVANCED_GBT_AUDIT__",
"ADVANCED_GBT_MEMPOOL": "__ADVANCED_GBT_MEMPOOL__",
"TRANSACTION_INDEXING": "__TRANSACTION_INDEXING__"
"TRANSACTION_INDEXING": "__TRANSACTION_INDEXING__",
"FIRST_SEEN_INDEXING_DAYS": "__FIRST_SEEN_INDEXING_DAYS__"
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",

1
backend/src/__tests__/config.test.ts
View File

@ -41,6 +41,7 @@ describe('Mempool Backend Config', () => {
ADVANCED_GBT_AUDIT: false,
ADVANCED_GBT_MEMPOOL: false,
TRANSACTION_INDEXING: false,
FIRST_SEEN_INDEXING_DAYS: 0,
});
expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });

7
backend/src/api/common.ts
View File

@ -194,6 +194,13 @@ export class Common {
);
}
static firstSeenIndexingEnabled(): boolean {
return (
Common.indexingEnabled() &&
config.MEMPOOL.FIRST_SEEN_INDEXING_DAYS !== 0
);
}
static setDateMidnight(date: Date): void {
date.setUTCHours(0);
date.setUTCMinutes(0);

4
backend/src/api/database-migration.ts
View File

@ -444,8 +444,8 @@ class DatabaseMigration {
}
if (databaseSchemaVersion < 50 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `transactions` ADD first_seen datetime DEFAULT NULL');
await this.updateToSchemaVersion(49);
await this.$executeQuery('ALTER TABLE `transactions` ADD first_seen datetime DEFAULT NULL, ADD INDEX (first_seen)');
await this.updateToSchemaVersion(50);
}
}

2
backend/src/api/websocket-handler.ts
View File

@ -463,7 +463,7 @@ class WebsocketHandler {
}
}
if (config.MEMPOOL.TRANSACTION_INDEXING) {
if (Common.firstSeenIndexingEnabled()) {
await mempool.$saveTxFirstSeenTimes(transactions, _memPool);
}

2
backend/src/config.ts
View File

@ -32,6 +32,7 @@ interface IConfig {
ADVANCED_GBT_AUDIT: boolean;
ADVANCED_GBT_MEMPOOL: boolean;
TRANSACTION_INDEXING: boolean;
FIRST_SEEN_INDEXING_DAYS: number;
};
ESPLORA: {
REST_API_URL: string;
@ -153,6 +154,7 @@ const defaults: IConfig = {
'ADVANCED_GBT_AUDIT': false,
'ADVANCED_GBT_MEMPOOL': false,
'TRANSACTION_INDEXING': false,
'FIRST_SEEN_INDEXING_DAYS': 0,
},
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',

2
backend/src/indexer.ts
View File

@ -7,6 +7,7 @@ import HashratesRepository from './repositories/HashratesRepository';
import bitcoinClient from './api/bitcoin/bitcoin-client';
import priceUpdater from './tasks/price-updater';
import PricesRepository from './repositories/PricesRepository';
import TransactionRepository from './repositories/TransactionRepository';
class Indexer {
runIndexer = true;
@ -78,6 +79,7 @@ class Indexer {
await mining.$generatePoolHashrateHistory();
await blocks.$generateBlocksSummariesDatabase();
await blocks.$generateCPFPDatabase();
await TransactionRepository.$clearOldFirstSeen();
} catch (e) {
this.indexerRunning = false;
logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));

25
backend/src/repositories/TransactionRepository.ts
View File

@ -1,6 +1,7 @@
import config from '../config';
import DB from '../database';
import logger from '../logger';
import { Ancestor, CpfpInfo, TransactionExtended, TransactionExtras } from '../mempool.interfaces';
import { Ancestor, CpfpInfo, TransactionExtras } from '../mempool.interfaces';
interface TxInfo {
txid: string;
@ -95,6 +96,28 @@ class TransactionRepository {
}
}
public async $clearOldFirstSeen() {
if (config.MEMPOOL.FIRST_SEEN_INDEXING_DAYS > 0) {
const cutoff = Math.floor(Date.now() / 1000) - (config.MEMPOOL.FIRST_SEEN_INDEXING_DAYS * 86400);
await this.$clearFirstSeenBefore(cutoff);
}
}
private async $clearFirstSeenBefore(cutoff: number) {
try {
const result = await DB.query(
`
DELETE FROM transactions
WHERE cluster is null AND first_seen < FROM_UNIXTIME(?)
;`,
[cutoff]
);
} catch (e: any) {
logger.err(`Cannot clear old tx first seen times from db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
private convertCpfp(info: TxInfo): TransactionExtras {
const descendants: Ancestor[] = [];
const ancestors: Ancestor[] = [];

3
frontend/src/app/components/time-span/time-span.component.ts
View File

@ -53,9 +53,6 @@ export class TimeSpanComponent implements OnInit, OnChanges, OnDestroy {
calculate() {
const seconds = Math.floor(this.time);
if (seconds < 60) {
return $localize`:@@date-base.just-now:Just now`;
}
let counter: number;
for (const i in this.intervals) {
if (this.intervals.hasOwnProperty(i)) {

6
frontend/src/app/components/transaction/transaction.component.html
View File

@ -75,10 +75,10 @@
<ng-template [ngIf]="transactionTime > 0">
<tr>
<td i18n="transaction.confirmed|Transaction Confirmed state">Confirmed</td>
<td><app-time-span [time]="tx.status.block_time - transactionTime" [fastRender]="true"></app-time-span></td>
<td><app-time-span [time]="tx.status.block_time - transactionTime"></app-time-span></td>
</tr>
</ng-template>
<tr *ngIf="network !== 'liquid' && network !== 'liquidtestnet' && cpfpInfo && (cpfpInfo?.bestDescendant || cpfpInfo?.descendants?.length || cpfpInfo?.ancestors?.length)">
<tr *ngIf="network !== 'liquid' && network !== 'liquidtestnet' && (cpfpInfo && (cpfpInfo?.bestDescendant || cpfpInfo?.descendants?.length || cpfpInfo?.ancestors?.length) || !(transactionTime > 0))">
<td class="td-width" i18n="transaction.features|Transaction features">Features</td>
<td>
<app-tx-features [tx]="tx"></app-tx-features>
@ -509,7 +509,7 @@
<button type="button" class="btn btn-outline-info btn-sm btn-small-height float-right" (click)="showCpfpDetails = !showCpfpDetails">CPFP <fa-icon [icon]="['fas', 'info-circle']" [fixedWidth]="true"></fa-icon></button>
</td>
</tr>
<tr *ngIf="tx?.status?.confirmed && (!cpfpInfo || (!cpfpInfo?.bestDescendant && !cpfpInfo?.descendants?.length && !cpfpInfo?.ancestors?.length)) && network !== 'liquid' && network !== 'liquidtestnet'">
<tr *ngIf="tx?.status?.confirmed && (!cpfpInfo || (!cpfpInfo?.bestDescendant && !cpfpInfo?.descendants?.length && !cpfpInfo?.ancestors?.length)) && transactionTime > 0 && network !== 'liquid' && network !== 'liquidtestnet'">
<td class="td-width" i18n="transaction.features|Transaction Features">Features</td>
<td>
<app-tx-features [tx]="tx"></app-tx-features>