Compare commits

..

1 Commits

Author SHA1 Message Date
wiz
ff8495f7cf Add Maxmind GeoIP Lite download to Docker build 2023-03-19 17:33:51 +09:00
20 changed files with 101 additions and 126 deletions

View File

@@ -27,8 +27,7 @@
"AUDIT": false,
"ADVANCED_GBT_AUDIT": false,
"ADVANCED_GBT_MEMPOOL": false,
"CPFP_INDEXING": false,
"DISK_CACHE_BLOCK_INTERVAL": 6
"CPFP_INDEXING": false
},
"CORE_RPC": {
"HOST": "127.0.0.1",

View File

@@ -28,8 +28,7 @@
"ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
"ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
"CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__",
"MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__",
"DISK_CACHE_BLOCK_INTERVAL": "__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__"
"MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__"
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",

View File

@@ -42,7 +42,6 @@ describe('Mempool Backend Config', () => {
ADVANCED_GBT_MEMPOOL: false,
CPFP_INDEXING: false,
MAX_BLOCKS_BULK_QUERY: 0,
DISK_CACHE_BLOCK_INTERVAL: 6,
});
expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
@@ -109,10 +108,10 @@ describe('Mempool Backend Config', () => {
});
expect(config.MAXMIND).toStrictEqual({
ENABLED: false,
GEOLITE2_CITY: '/usr/local/share/GeoIP/GeoLite2-City.mmdb',
GEOLITE2_ASN: '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
GEOIP2_ISP: '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
ENABLED: true,
GEOLITE2_CITY: './backend/GeoIP/GeoLite2-City.mmdb',
GEOLITE2_ASN: './backend/GeoIP/GeoLite2-ASN.mmdb',
GEOIP2_ISP: ''
});
});
});

View File

@@ -651,7 +651,7 @@ class Blocks {
if (this.newBlockCallbacks.length) {
this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
}
if (!memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
if (!memPool.hasPriority() && (block.height % 6 === 0)) {
diskCache.$saveCacheToDisk();
}

View File

@@ -19,16 +19,20 @@ class DiskCache {
private isWritingCache = false;
constructor() {
if (!cluster.isPrimary) {
if (!cluster.isMaster) {
return;
}
process.on('SIGINT', (e) => {
this.$saveCacheToDisk(true);
process.exit(0);
this.saveCacheToDiskSync();
process.exit(2);
});
process.on('SIGTERM', (e) => {
this.saveCacheToDiskSync();
process.exit(2);
});
}
async $saveCacheToDisk(sync: boolean = false): Promise<void> {
async $saveCacheToDisk(): Promise<void> {
if (!cluster.isPrimary) {
return;
}
@@ -37,7 +41,7 @@ class DiskCache {
return;
}
try {
logger.debug(`Writing mempool and blocks data to disk cache (${ sync ? 'sync' : 'async' })...`);
logger.debug('Writing mempool and blocks data to disk cache (async)...');
this.isWritingCache = true;
const mempool = memPool.getMempool();
@@ -50,46 +54,68 @@ class DiskCache {
const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
if (sync) {
fs.writeFileSync(DiskCache.TMP_FILE_NAME, JSON.stringify({
network: config.MEMPOOL.NETWORK,
cacheSchemaVersion: this.cacheSchemaVersion,
blocks: blocks.getBlocks(),
blockSummaries: blocks.getBlockSummaries(),
await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
network: config.MEMPOOL.NETWORK,
cacheSchemaVersion: this.cacheSchemaVersion,
blocks: blocks.getBlocks(),
blockSummaries: blocks.getBlockSummaries(),
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), { flag: 'w' });
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), { flag: 'w' });
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
fs.writeFileSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), { flag: 'w' });
}
}
logger.debug('Mempool and blocks data saved to disk cache');
this.isWritingCache = false;
} catch (e) {
logger.warn('Error writing to cache file: ' + (e instanceof Error ? e.message : e));
this.isWritingCache = false;
}
}
fs.renameSync(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
fs.renameSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
}
} else {
await fsPromises.writeFile(DiskCache.TMP_FILE_NAME, JSON.stringify({
network: config.MEMPOOL.NETWORK,
cacheSchemaVersion: this.cacheSchemaVersion,
blocks: blocks.getBlocks(),
blockSummaries: blocks.getBlockSummaries(),
saveCacheToDiskSync(): void {
if (!cluster.isPrimary) {
return;
}
if (this.isWritingCache) {
logger.debug('Saving cache already in progress. Skipping.');
return;
}
try {
logger.debug('Writing mempool and blocks data to disk cache (sync)...');
this.isWritingCache = true;
const mempool = memPool.getMempool();
const mempoolArray: TransactionExtended[] = [];
for (const tx in mempool) {
mempoolArray.push(mempool[tx]);
}
Common.shuffleArray(mempoolArray);
const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
fs.writeFileSync(DiskCache.TMP_FILE_NAME, JSON.stringify({
network: config.MEMPOOL.NETWORK,
cacheSchemaVersion: this.cacheSchemaVersion,
blocks: blocks.getBlocks(),
blockSummaries: blocks.getBlockSummaries(),
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), { flag: 'w' });
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
fs.writeFileSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), { flag: 'w' });
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
await fsPromises.writeFile(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), { flag: 'w' });
}
}
await fsPromises.rename(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
await fsPromises.rename(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
}
fs.renameSync(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
fs.renameSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
}
logger.debug('Mempool and blocks data saved to disk cache');

View File

@@ -33,7 +33,6 @@ interface IConfig {
ADVANCED_GBT_MEMPOOL: boolean;
CPFP_INDEXING: boolean;
MAX_BLOCKS_BULK_QUERY: number;
DISK_CACHE_BLOCK_INTERVAL: number;
};
ESPLORA: {
REST_API_URL: string;
@@ -156,7 +155,6 @@ const defaults: IConfig = {
'ADVANCED_GBT_MEMPOOL': false,
'CPFP_INDEXING': false,
'MAX_BLOCKS_BULK_QUERY': 0,
'DISK_CACHE_BLOCK_INTERVAL': 6,
},
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',

View File

@@ -112,7 +112,6 @@ Below we list all settings from `mempool-config.json` and the corresponding over
"ADVANCED_GBT_MEMPOOL": false,
"CPFP_INDEXING": false,
"MAX_BLOCKS_BULK_QUERY": 0,
"DISK_CACHE_BLOCK_INTERVAL": 6
},
```
@@ -144,7 +143,6 @@ Corresponding `docker-compose.yml` overrides:
MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
MEMPOOL_CPFP_INDEXING: ""
MAX_BLOCKS_BULK_QUERY: ""
DISK_CACHE_BLOCK_INTERVAL: ""
...
```
@@ -452,8 +450,8 @@ Corresponding `docker-compose.yml` overrides:
api:
environment:
MAXMIND_ENABLED: true,
MAXMIND_GEOLITE2_CITY: "/backend/GeoIP/GeoLite2-City.mmdb",
MAXMIND_GEOLITE2_ASN": "/backend/GeoIP/GeoLite2-ASN.mmdb",
MAXMIND_GEOIP2_ISP": "/backend/GeoIP/GeoIP2-ISP.mmdb"
MAXMIND_GEOLITE2_CITY: "./backend/GeoIP/GeoLite2-City.mmdb",
MAXMIND_GEOLITE2_ASN": "./backend/GeoIP/GeoLite2-ASN.mmdb",
MAXMIND_GEOIP2_ISP": "./backend/GeoIP/GeoIP2-ISP.mmdb"
...
```

View File

@@ -17,7 +17,6 @@ WORKDIR /backend
RUN chown 1000:1000 ./
COPY --from=builder --chown=1000:1000 /build/package ./package/
COPY --from=builder --chown=1000:1000 /build/GeoIP ./GeoIP/
COPY --from=builder --chown=1000:1000 /build/mempool-config.json /build/start.sh /build/wait-for-it.sh ./
USER 1000

View File

@@ -26,8 +26,7 @@
"ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
"ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
"CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
"MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__,
"DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__
"MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",
@@ -110,7 +109,7 @@
"BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__"
},
"MAXMIND": {
"ENABLED": __MAXMIND_ENABLED__,
"ENABLED": "__MAXMIND_ENABLED__",
"GEOLITE2_CITY": "__MAXMIND_GEOLITE2_CITY__",
"GEOLITE2_ASN": "__MAXMIND_GEOLITE2_ASN__",
"GEOIP2_ISP": "__MAXMIND_GEOIP2_ISP__"

View File

@@ -30,7 +30,6 @@ __MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
__MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
__MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
__MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6}
# CORE_RPC
__CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@@ -114,8 +113,8 @@ __CLIGHTNING_SOCKET__=${CLIGHTNING_SOCKET:=""}
# MAXMIND
__MAXMIND_ENABLED__=${MAXMIND_ENABLED:=true}
__MAXMIND_GEOLITE2_CITY__=${MAXMIND_GEOLITE2_CITY:="/backend/GeoIP/GeoLite2-City.mmdb"}
__MAXMIND_GEOLITE2_ASN__=${MAXMIND_GEOLITE2_ASN:="/backend/GeoIP/GeoLite2-ASN.mmdb"}
__MAXMIND_GEOLITE2_CITY__=${MAXMIND_GEOLITE2_CITY:="./backend/GeoIP/GeoLite2-City.mmdb"}
__MAXMIND_GEOLITE2_ASN__=${MAXMIND_GEOLITE2_ASN:="./backend/GeoIP/GeoLite2-ASN.mmdb"}
__MAXMIND_GEOIP2_ISP__=${MAXMIND_GEOIP2_ISP:=""}
@@ -150,7 +149,6 @@ sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g
sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json
sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json

View File

@@ -4,7 +4,6 @@
cp ./docker/backend/* ./backend/
#geoip-data
mkdir -p ./backend/GeoIP/
wget -O ./backend/GeoIP/GeoLite2-City.mmdb https://raw.githubusercontent.com/mempool/geoip-data/master/GeoLite2-City.mmdb
wget -O ./backend/GeoIP/GeoLite2-ASN.mmdb https://raw.githubusercontent.com/mempool/geoip-data/master/GeoLite2-ASN.mmdb

View File

@@ -26,7 +26,7 @@
</thead>
<tbody *ngIf="blocks$ | async as blocks; else skeleton" [style]="isLoading ? 'opacity: 0.75' : ''">
<tr *ngFor="let block of blocks; let i= index; trackBy: trackByBlock">
<td class="height text-left" [class]="widget ? 'widget' : ''">
<td class="text-left" [class]="widget ? 'widget' : ''">
<a [routerLink]="['/block' | relativeUrl, block.id]" [state]="{ data: { block: block } }">{{ block.height }}</a>
</td>
<td *ngIf="indexingAvailable" class="pool text-left" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
@@ -89,6 +89,7 @@
<span class="skeleton-loader" style="max-width: 75px"></span>
</td>
<td *ngIf="indexingAvailable" class="pool text-left" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
<img width="1" height="25" style="opacity: 0">
<span class="skeleton-loader" style="max-width: 125px"></span>
</td>
<td class="timestamp" *ngIf="!widget" [class]="indexingAvailable ? '' : 'legacy'">
@@ -97,7 +98,7 @@
<td class="mined" *ngIf="!widget" [class]="indexingAvailable ? '' : 'legacy'">
<span class="skeleton-loader" style="max-width: 125px"></span>
</td>
<td *ngIf="auditAvailable" class="health text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
<td *ngIf="auditAvailable" class="health text-left" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
<span class="skeleton-loader" style="max-width: 75px"></span>
</td>
<td *ngIf="indexingAvailable" class="reward text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">

View File

@@ -51,12 +51,7 @@ tr, td, th {
.pool.widget {
width: 40%;
padding-left: 24px;
@media (min-width: 768px) AND (max-width: 926px) {
padding-left: 0px;
width: 60%;
}
@media (max-width: 430px) {
padding-left: 0px;
@media (max-width: 376px) {
width: 60%;
}
}
@@ -64,10 +59,6 @@ tr, td, th {
display: inline-block;
vertical-align: text-top;
padding-left: 10px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
max-width: 160px;
}
.height {
@@ -78,12 +69,6 @@ tr, td, th {
@media (max-width: 576px) {
width: 10%;
}
@media (min-width: 768px) AND (max-width: 926px) {
width: 30%;
}
@media (max-width: 430px) {
width: 30%;
}
}
.height.legacy {
width: 15%;
@@ -107,7 +92,7 @@ tr, td, th {
.mined {
width: 13%;
@media (max-width: 730px) {
@media (max-width: 576px) {
display: none;
}
}
@@ -153,7 +138,7 @@ tr, td, th {
.fees {
width: 8%;
@media (max-width: 820px) {
@media (max-width: 650px) {
display: none;
}
}
@@ -178,16 +163,6 @@ tr, td, th {
width: 30%;
padding-right: 0;
}
@media (min-width: 768px) AND (max-width: 926px) {
overflow: hidden;
text-overflow: ellipsis;
max-width: 90px;
}
@media (max-width: 430px) {
overflow: hidden;
text-overflow: ellipsis;
max-width: 90px;
}
}
.size {
@@ -214,10 +189,10 @@ tr, td, th {
.health {
width: 10%;
@media (max-width: 1105px) {
@media (max-width: 1000px) {
width: 13%;
}
@media (max-width: 560px) {
@media (max-width: 950px) {
display: none;
}
@@ -227,7 +202,7 @@ tr, td, th {
}
.health.widget {
width: 25%;
@media (max-width: 1105px) {
@media (max-width: 1000px) {
display: none;
}
@media (max-width: 767px) {
@@ -267,4 +242,4 @@ tr, td, th {
vertical-align: middle;
max-width: 50vw;
text-align: left;
}
}

View File

@@ -2,18 +2,17 @@
<table class="table latest-adjustments">
<thead>
<tr>
<th class="" i18n="block.height">Height</th>
<th class="date text-left" i18n="mining.adjusted">Adjusted</th>
<th class="text-right" i18n="mining.difficulty">Difficulty</th>
<th class="text-right" i18n="mining.change">Change</th>
<th class="d-none d-md-block" i18n="block.height">Height</th>
<th i18n="mining.adjusted" class="text-left">Adjusted</th>
<th i18n="mining.difficulty" class="text-right">Difficulty</th>
<th i18n="mining.change" class="text-right">Change</th>
</tr>
</thead>
<tbody *ngIf="(hashrateObservable$ | async) as data">
<tr *ngFor="let diffChange of data">
<td class="">
<a [routerLink]="['/block' | relativeUrl, diffChange.height]">{{ diffChange.height }}</a>
</td>
<td class="date text-left">
<td class="d-none d-md-block"><a [routerLink]="['/block' | relativeUrl, diffChange.height]">{{ diffChange.height
}}</a></td>
<td class="text-left">
<app-time kind="since" [time]="diffChange.timestamp" [fastRender]="true"></app-time>
</td>
<td class="text-right">{{ diffChange.difficultyShorten }}</td>
@@ -24,8 +23,8 @@
</tbody>
<tbody *ngIf="isLoading">
<tr *ngFor="let item of [1,2,3,4,5,6]">
<td class=""><span class="skeleton-loader"></span></td>
<td class="date text-left"><span class="skeleton-loader w-75"></span></td>
<td class="d-none d-md-block w-75"><span class="skeleton-loader"></span></td>
<td class="text-left"><span class="skeleton-loader w-75"></span></td>
<td class="text-right"><span class="skeleton-loader w-75"></span></td>
<td class="text-right"><span class="skeleton-loader w-75"></span></td>
</tr>

View File

@@ -17,12 +17,3 @@
}
}
}
.date {
@media (min-width: 767px) AND (max-width: 991px) {
display: none;
}
@media (max-width: 500px) {
display: none;
}
}

View File

@@ -6,8 +6,7 @@
"MINED_BLOCKS_CACHE": 144,
"SPAWN_CLUSTER_PROCS": 0,
"API_URL_PREFIX": "/api/v1/",
"POLL_RATE_MS": 1000,
"DISK_CACHE_BLOCK_INTERVAL": 1
"POLL_RATE_MS": 1000
},
"SYSLOG" : {
"MIN_PRIORITY": "debug"

View File

@@ -6,8 +6,7 @@
"MINED_BLOCKS_CACHE": 144,
"SPAWN_CLUSTER_PROCS": 0,
"API_URL_PREFIX": "/api/v1/",
"POLL_RATE_MS": 1000,
"DISK_CACHE_BLOCK_INTERVAL": 1
"POLL_RATE_MS": 1000
},
"SYSLOG" : {
"MIN_PRIORITY": "debug"

View File

@@ -14,8 +14,7 @@
"CPFP_INDEXING": true,
"ADVANCED_GBT_AUDIT": true,
"ADVANCED_GBT_MEMPOOL": true,
"USE_SECOND_NODE_FOR_MINFEE": true,
"DISK_CACHE_BLOCK_INTERVAL": 1
"USE_SECOND_NODE_FOR_MINFEE": true
},
"SYSLOG" : {
"MIN_PRIORITY": "debug"

View File

@@ -10,8 +10,7 @@
"AUDIT": true,
"ADVANCED_GBT_AUDIT": true,
"ADVANCED_GBT_MEMPOOL": true,
"POLL_RATE_MS": 1000,
"DISK_CACHE_BLOCK_INTERVAL": 1
"POLL_RATE_MS": 1000
},
"SYSLOG" : {
"MIN_PRIORITY": "debug"

View File

@@ -10,8 +10,7 @@
"AUDIT": true,
"ADVANCED_GBT_AUDIT": true,
"ADVANCED_GBT_MEMPOOL": true,
"POLL_RATE_MS": 1000,
"DISK_CACHE_BLOCK_INTERVAL": 1
"POLL_RATE_MS": 1000
},
"SYSLOG" : {
"MIN_PRIORITY": "debug"