Merge branch 'master' into ops/esplora-unix-sockets

2023-03-21 14:29:06 +09:00
parent 7970f4ae88 cd2bda4b49
commit ccab8b16bf
113 changed files with 14872 additions and 8274 deletions
--- a/docker/README.md
+++ b/docker/README.md
@@ -34,6 +34,7 @@ If you want to use different credentials, specify them in the `docker-compose.ym
      CORE_RPC_PORT: "8332"
      CORE_RPC_USERNAME: "customuser"
      CORE_RPC_PASSWORD: "custompassword"
+      CORE_RPC_TIMEOUT: "60000"
 ```

 The IP address in the example above refers to Docker's default gateway IP address so that the container can hit the `bitcoind` instance running on the host machine. If your setup is different, update it accordingly.
@@ -112,6 +113,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over
    "ADVANCED_GBT_MEMPOOL": false,
    "CPFP_INDEXING": false,
    "MAX_BLOCKS_BULK_QUERY": 0,
+    "DISK_CACHE_BLOCK_INTERVAL": 6
  },
 ```

@@ -143,6 +145,7 @@ Corresponding `docker-compose.yml` overrides:
      MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
      MEMPOOL_CPFP_INDEXING: ""
      MAX_BLOCKS_BULK_QUERY: ""
+      DISK_CACHE_BLOCK_INTERVAL: ""
      ...
 ```

@@ -158,7 +161,8 @@ Corresponding `docker-compose.yml` overrides:
    "HOST": "127.0.0.1",
    "PORT": 8332,
    "USERNAME": "mempool",
-    "PASSWORD": "mempool"
+    "PASSWORD": "mempool",
+    "TIMEOUT": 60000
  },
 ```

@@ -170,6 +174,7 @@ Corresponding `docker-compose.yml` overrides:
      CORE_RPC_PORT: ""
      CORE_RPC_USERNAME: ""
      CORE_RPC_PASSWORD: ""
+      CORE_RPC_TIMEOUT: 60000
      ...
 ```

@@ -221,7 +226,8 @@ Corresponding `docker-compose.yml` overrides:
    "HOST": "127.0.0.1",
    "PORT": 8332,
    "USERNAME": "mempool",
-    "PASSWORD": "mempool"
+    "PASSWORD": "mempool",
+    "TIMEOUT": 60000
  },
 ```

@@ -233,6 +239,7 @@ Corresponding `docker-compose.yml` overrides:
      SECOND_CORE_RPC_PORT: ""
      SECOND_CORE_RPC_USERNAME: ""
      SECOND_CORE_RPC_PASSWORD: ""
+      SECOND_CORE_RPC_TIMEOUT: ""
      ...
 ```

@@ -405,6 +412,7 @@ Corresponding `docker-compose.yml` overrides:
    "TLS_CERT_PATH": ""
    "MACAROON_PATH": ""
    "REST_API_URL": "https://localhost:8080"
+    "TIMEOUT": 10000
  }
 ```

@@ -415,6 +423,7 @@ Corresponding `docker-compose.yml` overrides:
      LND_TLS_CERT_PATH: ""
      LND_MACAROON_PATH: ""
      LND_REST_API_URL: "https://localhost:8080"
+      LND_TIMEOUT: 10000
      ...
 ```

@@ -434,3 +443,26 @@ Corresponding `docker-compose.yml` overrides:
      CLIGHTNING_SOCKET: ""
      ...
 ```
+
+<br/>
+
+`mempool-config.json`:
+```json
+  "MAXMIND": {
+    "ENABLED": true,
+    "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
+    "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
+    "GEOIP2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
+  }
+```
+
+Corresponding `docker-compose.yml` overrides:
+```yaml
+  api:
+    environment:
+      MAXMIND_ENABLED: true,
+      MAXMIND_GEOLITE2_CITY: "/backend/GeoIP/GeoLite2-City.mmdb",
+      MAXMIND_GEOLITE2_ASN": "/backend/GeoIP/GeoLite2-ASN.mmdb",
+      MAXMIND_GEOIP2_ISP": "/backend/GeoIP/GeoIP2-ISP.mmdb"
+      ...
+```
--- a/docker/backend/Dockerfile
+++ b/docker/backend/Dockerfile
@@ -17,6 +17,7 @@ WORKDIR /backend

 RUN chown 1000:1000 ./
 COPY --from=builder --chown=1000:1000 /build/package ./package/
+COPY --from=builder --chown=1000:1000 /build/GeoIP ./GeoIP/
 COPY --from=builder --chown=1000:1000 /build/mempool-config.json /build/start.sh /build/wait-for-it.sh ./

 USER 1000
--- a/docker/backend/mempool-config.json
+++ b/docker/backend/mempool-config.json
@@ -26,13 +26,15 @@
    "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
    "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
    "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
-    "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__
+    "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__,
+    "DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__
  },
  "CORE_RPC": {
    "HOST": "__CORE_RPC_HOST__",
    "PORT": __CORE_RPC_PORT__,
    "USERNAME": "__CORE_RPC_USERNAME__",
-    "PASSWORD": "__CORE_RPC_PASSWORD__"
+    "PASSWORD": "__CORE_RPC_PASSWORD__",
+    "TIMEOUT": __CORE_RPC_TIMEOUT__
  },
  "ELECTRUM": {
    "HOST": "__ELECTRUM_HOST__",
@@ -47,7 +49,8 @@
    "HOST": "__SECOND_CORE_RPC_HOST__",
    "PORT": __SECOND_CORE_RPC_PORT__,
    "USERNAME": "__SECOND_CORE_RPC_USERNAME__",
-    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__"
+    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__",
+    "TIMEOUT": __SECOND_CORE_RPC_TIMEOUT__
  },
  "DATABASE": {
    "ENABLED": __DATABASE_ENABLED__,
@@ -84,7 +87,8 @@
  "LND": {
    "TLS_CERT_PATH": "__LND_TLS_CERT_PATH__",
    "MACAROON_PATH": "__LND_MACAROON_PATH__",
-    "REST_API_URL": "__LND_REST_API_URL__"
+    "REST_API_URL": "__LND_REST_API_URL__",
+    "TIMEOUT": "__LND_TIMEOUT__"
  },
  "CLIGHTNING": {
    "SOCKET": "__CLIGHTNING_SOCKET__"
@@ -108,5 +112,11 @@
    "LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__",
    "BISQ_URL": "__EXTERNAL_DATA_SERVER_BISQ_URL__",
    "BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__"
+  },
+  "MAXMIND": {
+    "ENABLED": __MAXMIND_ENABLED__,
+    "GEOLITE2_CITY": "__MAXMIND_GEOLITE2_CITY__",
+    "GEOLITE2_ASN": "__MAXMIND_GEOLITE2_ASN__",
+    "GEOIP2_ISP": "__MAXMIND_GEOIP2_ISP__"
  }
-}
+}
--- a/docker/backend/start.sh
+++ b/docker/backend/start.sh
@@ -22,7 +22,6 @@ __MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=1}
 __MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=0}
 __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool}
 __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
-__MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false}
 __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false}
 __MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json}
 __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
@@ -31,12 +30,14 @@ __MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
 __MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
 __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
 __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
+__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6}

 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
 __CORE_RPC_PORT__=${CORE_RPC_PORT:=8332}
 __CORE_RPC_USERNAME__=${CORE_RPC_USERNAME:=mempool}
 __CORE_RPC_PASSWORD__=${CORE_RPC_PASSWORD:=mempool}
+__CORE_RPC_TIMEOUT__=${CORE_RPC_TIMEOUT:=60000}

 # ELECTRUM
 __ELECTRUM_HOST__=${ELECTRUM_HOST:=127.0.0.1}
@@ -52,6 +53,7 @@ __SECOND_CORE_RPC_HOST__=${SECOND_CORE_RPC_HOST:=127.0.0.1}
 __SECOND_CORE_RPC_PORT__=${SECOND_CORE_RPC_PORT:=8332}
 __SECOND_CORE_RPC_USERNAME__=${SECOND_CORE_RPC_USERNAME:=mempool}
 __SECOND_CORE_RPC_PASSWORD__=${SECOND_CORE_RPC_PASSWORD:=mempool}
+__SECOND_CORE_RPC_TIMEOUT__=${SECOND_CORE_RPC_TIMEOUT:=60000}

 # DATABASE
 __DATABASE_ENABLED__=${DATABASE_ENABLED:=true}
@@ -109,10 +111,18 @@ __LIGHTNING_LOGGER_UPDATE_INTERVAL__=${LIGHTNING_LOGGER_UPDATE_INTERVAL:=30}
 __LND_TLS_CERT_PATH__=${LND_TLS_CERT_PATH:=""}
 __LND_MACAROON_PATH__=${LND_MACAROON_PATH:=""}
 __LND_REST_API_URL__=${LND_REST_API_URL:="https://localhost:8080"}
+__LND_TIMEOUT__=${LND_TIMEOUT:=10000}

 # CLN
 __CLIGHTNING_SOCKET__=${CLIGHTNING_SOCKET:=""}

+# MAXMIND
+__MAXMIND_ENABLED__=${MAXMIND_ENABLED:=true}
+__MAXMIND_GEOLITE2_CITY__=${MAXMIND_GEOLITE2_CITY:="/backend/GeoIP/GeoLite2-City.mmdb"}
+__MAXMIND_GEOLITE2_ASN__=${MAXMIND_GEOLITE2_ASN:="/backend/GeoIP/GeoLite2-ASN.mmdb"}
+__MAXMIND_GEOIP2_ISP__=${MAXMIND_GEOIP2_ISP:=""}
+
+
 mkdir -p "${__MEMPOOL_CACHE_DIR__}"

 sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json
@@ -136,7 +146,6 @@ sed -i "s!__MEMPOOL_EXTERNAL_MAX_RETRY__!${__MEMPOOL_EXTERNAL_MAX_RETRY__}!g" me
 sed -i "s!__MEMPOOL_EXTERNAL_RETRY_INTERVAL__!${__MEMPOOL_EXTERNAL_RETRY_INTERVAL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_USER_AGENT__!${__MEMPOOL_USER_AGENT__}!g" mempool-config.json
 sed -i "s/__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__/${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}/g" mempool-config.json
-sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json
 sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json
@@ -145,11 +154,13 @@ sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g
 sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json

 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_USERNAME__/${__CORE_RPC_USERNAME__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PASSWORD__/${__CORE_RPC_PASSWORD__}/g" mempool-config.json
+sed -i "s/__CORE_RPC_TIMEOUT__/${__CORE_RPC_TIMEOUT__}/g" mempool-config.json

 sed -i "s/__ELECTRUM_HOST__/${__ELECTRUM_HOST__}/g" mempool-config.json
 sed -i "s/__ELECTRUM_PORT__/${__ELECTRUM_PORT__}/g" mempool-config.json
@@ -162,6 +173,7 @@ sed -i "s/__SECOND_CORE_RPC_HOST__/${__SECOND_CORE_RPC_HOST__}/g" mempool-config
 sed -i "s/__SECOND_CORE_RPC_PORT__/${__SECOND_CORE_RPC_PORT__}/g" mempool-config.json
 sed -i "s/__SECOND_CORE_RPC_USERNAME__/${__SECOND_CORE_RPC_USERNAME__}/g" mempool-config.json
 sed -i "s/__SECOND_CORE_RPC_PASSWORD__/${__SECOND_CORE_RPC_PASSWORD__}/g" mempool-config.json
+sed -i "s/__SECOND_CORE_RPC_TIMEOUT__/${__SECOND_CORE_RPC_TIMEOUT__}/g" mempool-config.json

 sed -i "s/__DATABASE_ENABLED__/${__DATABASE_ENABLED__}/g" mempool-config.json
 sed -i "s/__DATABASE_HOST__/${__DATABASE_HOST__}/g" mempool-config.json
@@ -213,8 +225,16 @@ sed -i "s!__LIGHTNING_LOGGER_UPDATE_INTERVAL__!${__LIGHTNING_LOGGER_UPDATE_INTER
 sed -i "s!__LND_TLS_CERT_PATH__!${__LND_TLS_CERT_PATH__}!g" mempool-config.json
 sed -i "s!__LND_MACAROON_PATH__!${__LND_MACAROON_PATH__}!g" mempool-config.json
 sed -i "s!__LND_REST_API_URL__!${__LND_REST_API_URL__}!g" mempool-config.json
+sed -i "s!__LND_TIMEOUT__!${__LND_TIMEOUT__}!g" mempool-config.json

 # CLN
 sed -i "s!__CLIGHTNING_SOCKET__!${__CLIGHTNING_SOCKET__}!g" mempool-config.json

+# MAXMIND
+sed -i "s!__MAXMIND_ENABLED__!${__MAXMIND_ENABLED__}!g" mempool-config.json
+sed -i "s!__MAXMIND_GEOLITE2_CITY__!${__MAXMIND_GEOLITE2_CITY__}!g" mempool-config.json
+sed -i "s!__MAXMIND_GEOLITE2_ASN__!${__MAXMIND_GEOLITE2_ASN__}!g" mempool-config.json
+sed -i "s!__MAXMIND_GEOIP2_ISP__!${__MAXMIND_GEOIP2_ISP__}!g" mempool-config.json
+
+
 node /backend/package/index.js
--- a/docker/init.sh
+++ b/docker/init.sh
@@ -3,6 +3,11 @@
 #backend
 cp ./docker/backend/* ./backend/

+#geoip-data
+mkdir -p ./backend/GeoIP/
+wget -O ./backend/GeoIP/GeoLite2-City.mmdb https://raw.githubusercontent.com/mempool/geoip-data/master/GeoLite2-City.mmdb
+wget -O ./backend/GeoIP/GeoLite2-ASN.mmdb https://raw.githubusercontent.com/mempool/geoip-data/master/GeoLite2-ASN.mmdb
+
 #frontend
 localhostIP="127.0.0.1"
 cp ./docker/frontend/* ./frontend