machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into nymkappa/configurable-timeout

Browse Source
This commit is contained in:
wiz 2023-03-20 17:38:40 +09:00 committed by GitHub
commit 6a3f35f0f8
15 changed files with 64 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -1,13 +1,13 @@
# The Mempool Open Source Project™ [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs)
https://user-images.githubusercontent.com/232186/222445818-234aa6c9-c233-4c52-b3f0-e32b8232893b.mp4
https://user-images.githubusercontent.com/93150691/226236121-375ea64f-b4a1-4cc0-8fad-a6fb33226840.mp4
<br>
Mempool is the fully-featured mempool visualizer, explorer, and API service running at [mempool.space](https://mempool.space/).
It is an open-source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market that is evolving Bitcoin into a multi-layer ecosystem.
![mempool](https://mempool.space/resources/screenshots/v2.4.0-dashboard.png)
# Installation Methods
Mempool can be self-hosted on a wide variety of your own hardware, ranging from a simple one-click installation on a Raspberry Pi full-node distro all the way to a robust production instance on a powerful FreeBSD server.

3
backend/mempool-config.sample.json
View File

@ -27,7 +27,8 @@
"AUDIT": false,
"ADVANCED_GBT_AUDIT": false,
"ADVANCED_GBT_MEMPOOL": false,
"CPFP_INDEXING": false
"CPFP_INDEXING": false,
"DISK_CACHE_BLOCK_INTERVAL": 6
},
"CORE_RPC": {
"HOST": "127.0.0.1",

3
backend/src/__fixtures__/mempool-config.template.json
View File

@ -28,7 +28,8 @@
"ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
"ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
"CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__",
"MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__"
"MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__",
"DISK_CACHE_BLOCK_INTERVAL": "__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__"
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",

1
backend/src/__tests__/config.test.ts
View File

@ -42,6 +42,7 @@ describe('Mempool Backend Config', () => {
ADVANCED_GBT_MEMPOOL: false,
CPFP_INDEXING: false,
MAX_BLOCKS_BULK_QUERY: 0,
DISK_CACHE_BLOCK_INTERVAL: 6,
});
expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });

2
backend/src/api/blocks.ts
View File

@ -651,7 +651,7 @@ class Blocks {
if (this.newBlockCallbacks.length) {
this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
}
if (!memPool.hasPriority() && (block.height % 6 === 0)) {
if (!memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
diskCache.$saveCacheToDisk();
}

80
backend/src/api/disk-cache.ts
View File

@ -19,20 +19,16 @@ class DiskCache {
private isWritingCache = false;
constructor() {
if (!cluster.isMaster) {
if (!cluster.isPrimary) {
return;
}
process.on('SIGINT', (e) => {
this.saveCacheToDiskSync();
process.exit(2);
});
process.on('SIGTERM', (e) => {
this.saveCacheToDiskSync();
process.exit(2);
this.$saveCacheToDisk(true);
process.exit(0);
});
}
async $saveCacheToDisk(): Promise<void> {
async $saveCacheToDisk(sync: boolean = false): Promise<void> {
if (!cluster.isPrimary) {
return;
}
@ -41,51 +37,7 @@ class DiskCache {
return;
}
try {
logger.debug('Writing mempool and blocks data to disk cache (async)...');
this.isWritingCache = true;
const mempool = memPool.getMempool();
const mempoolArray: TransactionExtended[] = [];
for (const tx in mempool) {
mempoolArray.push(mempool[tx]);
}
Common.shuffleArray(mempoolArray);
const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
network: config.MEMPOOL.NETWORK,
cacheSchemaVersion: this.cacheSchemaVersion,
blocks: blocks.getBlocks(),
blockSummaries: blocks.getBlockSummaries(),
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), { flag: 'w' });
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), { flag: 'w' });
}
logger.debug('Mempool and blocks data saved to disk cache');
this.isWritingCache = false;
} catch (e) {
logger.warn('Error writing to cache file: ' + (e instanceof Error ? e.message : e));
this.isWritingCache = false;
}
}
saveCacheToDiskSync(): void {
if (!cluster.isPrimary) {
return;
}
if (this.isWritingCache) {
logger.debug('Saving cache already in progress. Skipping.');
return;
}
try {
logger.debug('Writing mempool and blocks data to disk cache (sync)...');
logger.debug(`Writing mempool and blocks data to disk cache (${ sync ? 'sync' : 'async' })...`);
this.isWritingCache = true;
const mempool = memPool.getMempool();
@ -98,6 +50,7 @@ class DiskCache {
const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
if (sync) {
fs.writeFileSync(DiskCache.TMP_FILE_NAME, JSON.stringify({
network: config.MEMPOOL.NETWORK,
cacheSchemaVersion: this.cacheSchemaVersion,
@ -117,6 +70,27 @@ class DiskCache {
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
fs.renameSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
}
} else {
await fsPromises.writeFile(DiskCache.TMP_FILE_NAME, JSON.stringify({
network: config.MEMPOOL.NETWORK,
cacheSchemaVersion: this.cacheSchemaVersion,
blocks: blocks.getBlocks(),
blockSummaries: blocks.getBlockSummaries(),
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), { flag: 'w' });
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
await fsPromises.writeFile(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), { flag: 'w' });
}
await fsPromises.rename(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
await fsPromises.rename(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
}
}
logger.debug('Mempool and blocks data saved to disk cache');
this.isWritingCache = false;

2
backend/src/config.ts
View File

@ -33,6 +33,7 @@ interface IConfig {
ADVANCED_GBT_MEMPOOL: boolean;
CPFP_INDEXING: boolean;
MAX_BLOCKS_BULK_QUERY: number;
DISK_CACHE_BLOCK_INTERVAL: number;
};
ESPLORA: {
REST_API_URL: string;
@ -158,6 +159,7 @@ const defaults: IConfig = {
'ADVANCED_GBT_MEMPOOL': false,
'CPFP_INDEXING': false,
'MAX_BLOCKS_BULK_QUERY': 0,
'DISK_CACHE_BLOCK_INTERVAL': 6,
},
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',

2
docker/README.md
View File

@ -113,6 +113,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over
"ADVANCED_GBT_MEMPOOL": false,
"CPFP_INDEXING": false,
"MAX_BLOCKS_BULK_QUERY": 0,
"DISK_CACHE_BLOCK_INTERVAL": 6
},
```
@ -144,6 +145,7 @@ Corresponding `docker-compose.yml` overrides:
MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
MEMPOOL_CPFP_INDEXING: ""
MAX_BLOCKS_BULK_QUERY: ""
DISK_CACHE_BLOCK_INTERVAL: ""
...
```

3
docker/backend/mempool-config.json
View File

@ -26,7 +26,8 @@
"ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
"ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
"CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
"MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__
"MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__,
"DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",

2
docker/backend/start.sh
View File

@ -30,6 +30,7 @@ __MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
__MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
__MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
__MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6}
# CORE_RPC
__CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@ -152,6 +153,7 @@ sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g
sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json
sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json

3
production/mempool-config.liquid.json
View File

@ -6,7 +6,8 @@
"MINED_BLOCKS_CACHE": 144,
"SPAWN_CLUSTER_PROCS": 0,
"API_URL_PREFIX": "/api/v1/",
"POLL_RATE_MS": 1000
"POLL_RATE_MS": 1000,
"DISK_CACHE_BLOCK_INTERVAL": 1
},
"SYSLOG" : {
"MIN_PRIORITY": "debug"

3
production/mempool-config.liquidtestnet.json
View File

@ -6,7 +6,8 @@
"MINED_BLOCKS_CACHE": 144,
"SPAWN_CLUSTER_PROCS": 0,
"API_URL_PREFIX": "/api/v1/",
"POLL_RATE_MS": 1000
"POLL_RATE_MS": 1000,
"DISK_CACHE_BLOCK_INTERVAL": 1
},
"SYSLOG" : {
"MIN_PRIORITY": "debug"

3
production/mempool-config.mainnet.json
View File

@ -14,7 +14,8 @@
"CPFP_INDEXING": true,
"ADVANCED_GBT_AUDIT": true,
"ADVANCED_GBT_MEMPOOL": true,
"USE_SECOND_NODE_FOR_MINFEE": true
"USE_SECOND_NODE_FOR_MINFEE": true,
"DISK_CACHE_BLOCK_INTERVAL": 1
},
"SYSLOG" : {
"MIN_PRIORITY": "debug"

3
production/mempool-config.signet.json
View File

@ -10,7 +10,8 @@
"AUDIT": true,
"ADVANCED_GBT_AUDIT": true,
"ADVANCED_GBT_MEMPOOL": true,
"POLL_RATE_MS": 1000
"POLL_RATE_MS": 1000,
"DISK_CACHE_BLOCK_INTERVAL": 1
},
"SYSLOG" : {
"MIN_PRIORITY": "debug"

3
production/mempool-config.testnet.json
View File

@ -10,7 +10,8 @@
"AUDIT": true,
"ADVANCED_GBT_AUDIT": true,
"ADVANCED_GBT_MEMPOOL": true,
"POLL_RATE_MS": 1000
"POLL_RATE_MS": 1000,
"DISK_CACHE_BLOCK_INTERVAL": 1
},
"SYSLOG" : {
"MIN_PRIORITY": "debug"