Merge pull request #322 from bguillaumat/update-dockerization

Update dockerization
2021-02-03 13:03:44 +09:00 · 2021-02-03 13:03:44 +09:00 · 448cb8e264
commit 448cb8e264
parent 03179e34fb 6782229a3d
17 changed files with 309 additions and 78 deletions
--- a/docker/backend/Dockerfile
+++ b/docker/backend/Dockerfile
@ -2,12 +2,17 @@ FROM node:12-buster-slim AS builder

 WORKDIR /build
 COPY . .
+
+RUN sed -i "s!../.git/refs/heads/master!master!g" ./src/api/backend-info.ts
+
 RUN apt-get update
 RUN apt-get install -y build-essential python3 pkg-config
 RUN npm ci --production
 RUN npm i typescript
 RUN npm run build

+RUN mv ./docker/* .
+
 FROM node:12-buster-slim

 WORKDIR /backend
--- a/backend/docker/master
+++ b/backend/docker/master
@ -0,0 +1 @@
+9d02ab1eb5ffb60d38128df903e47e11b95f13d5
--- a/backend/docker/start.sh
+++ b/backend/docker/start.sh
@ -2,7 +2,7 @@

 #MEMPOOL
 __MEMPOOL_BACKEND_MAINNET_HTTP_PORT__=${BACKEND_MAINNET_HTTP_PORT:=8999}
-__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__=${CACHE_DIR:=./cache}
+__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__=${CACHE_DIR:=./}
 # BITCOIN
 __BITCOIN_MAINNET_RPC_HOST__=${RPC_HOST:=127.0.0.1}
 __BITCOIN_MAINNET_RPC_PORT__=${RPC_PORT:=8332}
@ -15,7 +15,7 @@ __ELECTRS_MAINNET_HTTP_PORT__=${ELECTRS_PORT:=50002}
 __MYSQL_HOST__=${MYSQL_HOST:=127.0.0.1}
 __MYSQL_PORT__=${MYSQL_PORT:=3306}

-mkdir -p "${__MEMPOOL_BACKEND_CACHE_DIR__}"
+mkdir -p "${__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__}"

 sed -i "s/__BITCOIN_MAINNET_RPC_HOST__/${__BITCOIN_MAINNET_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__BITCOIN_MAINNET_RPC_PORT__/${__BITCOIN_MAINNET_RPC_PORT__}/g" mempool-config.json
--- a/backend/docker/wait-for-it.sh
+++ b/backend/docker/wait-for-it.sh
--- a/backend/package.json
+++ b/backend/package.json
@ -36,7 +36,6 @@
    "locutus": "^2.0.12",
    "mysql2": "2.2.5",
    "node-worker-threads-pool": "^1.4.2",
-    "typescript": "~3.9.7",
    "ws": "^7.3.1"
  },
  "devDependencies": {
--- a/docker/README.md
+++ b/docker/README.md
@ -1,5 +0,0 @@
-## Init
-./docker/init.sh
-
-## Build
-./docker/build.sh YOUR_DOCKER_HUB_NAME VERSION
--- a/docker/backend/mempool-config.json
+++ b/docker/backend/mempool-config.json
@ -1,38 +0,0 @@
-{
-  "MEMPOOL": {
-    "NETWORK": "mainnet",
-    "BACKEND": "electrum",
-    "HTTP_PORT": __MEMPOOL_BACKEND_MAINNET_HTTP_PORT__,
-    "SPAWN_CLUSTER_PROCS": 0,
-    "API_URL_PREFIX": "/api/v1/",
-    "POLL_RATE_MS": 2000,
-    "CACHE_DIR": "__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__"
-  },
-  "CORE_RPC": {
-    "HOST": "__BITCOIN_MAINNET_RPC_HOST__",
-    "PORT": __BITCOIN_MAINNET_RPC_PORT__,
-    "USERNAME": "__BITCOIN_MAINNET_RPC_USER__",
-    "PASSWORD": "__BITCOIN_MAINNET_RPC_PASS__"
-  },
-  "ELECTRUM": {
-    "HOST": "__ELECTRS_MAINNET_HTTP_HOST__",
-    "PORT": __ELECTRS_MAINNET_HTTP_PORT__,
-    "TLS_ENABLED": false,
-    "TX_LOOKUPS": true
-  },
-  "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:3000"
-  },
-  "DATABASE": {
-    "ENABLED": true,
-    "HOST": "__MYSQL_HOST__",
-    "PORT": __MYSQL_PORT__,
-    "DATABASE": "mempool",
-    "USERNAME": "mempool",
-    "PASSWORD": "mempool"
-  },
-  "STATISTICS": {
-    "ENABLED": true,
-    "TX_PER_SECOND_SAMPLE_PERIOD": 150
-  }
-}
--- a/docker/build.sh
+++ b/docker/build.sh
@ -1,10 +0,0 @@
-#!/bin/sh
-
-cd ./backend
-docker buildx build --platform linux/amd64,linux/arm64 -t $1/mempool-backend:$2 --push .
-
-cd ../frontend
-docker buildx build --platform linux/amd64,linux/arm64 -t $1/mempool-frontend:$2 --push .
-
-cd ../mysql
-docker buildx build --platform linux/amd64,linux/arm64 -t $1/mempool-db:$2 --push .
--- a/docker/init.sh
+++ b/docker/init.sh
@ -1,22 +0,0 @@
-#!/bin/sh
-
-#backend
-gitMaster="\.\.\/\.git\/refs\/heads\/master"
-cp .git/refs/heads/master ./backend
-cp ./docker/backend/* ./backend/
-sed -i "s/${gitMaster}/master/g" ./backend/src/api/backend-info.ts
-
-#frontend
-localhostIP="127.0.0.1"
-cp ./docker/frontend/* ./frontend
-cp ./nginx.conf ./frontend/
-cp ./nginx-mempool.conf ./frontend/
-sed -i "s/${localhostIP}:/0.0.0.0:80/g" ./frontend/nginx.conf
-sed -i "s/${localhostIP}/0.0.0.0/g" ./frontend/nginx.conf
-sed -i "s/user nobody;//g" ./frontend/nginx.conf
-sed -i "s!/etc/nginx/nginx-mempool.conf!/etc/nginx/conf.d/nginx-mempool.conf!g" ./frontend/nginx.conf
-sed -i "s/${localhostIP}:/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:/g" ./frontend/nginx-mempool.conf
-
-#db
-cp -fr docker/mysql .
-cp ./mariadb-structure.sql ./mysql/setup.sql
--- a/docker/frontend/Dockerfile
+++ b/docker/frontend/Dockerfile
@ -7,6 +7,8 @@ RUN apt-get install -y build-essential rsync
 RUN npm i
 RUN npm run build

+RUN mv ./docker/* .
+
 FROM nginx:1.17.8-alpine

 WORKDIR /patch
--- a/frontend/docker/entrypoint.sh
+++ b/frontend/docker/entrypoint.sh
@ -1,6 +1,8 @@
 #!/bin/sh
 __MEMPOOL_BACKEND_MAINNET_HTTP_HOST__=${BACKEND_MAINNET_HTTP_HOST:=127.0.0.1}
+__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__=${BACKEND_MAINNET_HTTP_PORT:=8999}

 sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__/${__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__}/g" /etc/nginx/conf.d/nginx-mempool.conf
+sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/${__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__}/g" /etc/nginx/conf.d/nginx-mempool.conf

 exec "$@"
--- a/frontend/docker/nginx-mempool.conf
+++ b/frontend/docker/nginx-mempool.conf
@ -0,0 +1,62 @@
+	access_log /var/log/nginx/access_mempool.log;
+	error_log /var/log/nginx/error_mempool.log;
+
+	root /var/www/mempool/browser;
+
+	index index.html;
+
+	# fallback for all URLs i.e. /address/foo /tx/foo /block/000
+	location / {
+		try_files /$lang/$uri /$lang/$uri/ $uri $uri/ /en-US/$uri @index-redirect;
+	}
+	location @index-redirect {
+		add_header vary accept-language;
+		rewrite (.*) /$lang/index.html;
+	}
+
+	# location block using regex are matched in order
+
+	# used to rewrite resources from /<lang>/ to /en-US/
+	location ~ ^/(ar|bg|bs|ca|cs|da|de|et|el|es|eo|eu|fa|fr|gl|ko|hr|id|it|he|ka|lv|lt|hu|mk|ms|nl|ja|ka|no|nb|nn|pl|pt|pt-BR|ro|ru|sk|sl|sr|sh|fi|sv|th|tr|uk|vi|zh)/resources/ {
+		rewrite ^/[a-zA-Z-]*/resources/(.*) /en-US/resources/$1;
+	}
+	# used for cookie override
+	location ~ ^/(ar|bg|bs|ca|cs|da|de|et|el|es|eo|eu|fa|fr|gl|ko|hr|id|it|he|ka|lv|lt|hu|mk|ms|nl|ja|ka|no|nb|nn|pl|pt|pt-BR|ro|ru|sk|sl|sr|sh|fi|sv|th|tr|uk|vi|zh)/ {
+		try_files $uri $uri/ /$1/index.html =404;
+	}
+
+	# static API docs
+	location = /api {
+		try_files $uri $uri/ /en-US/index.html =404;
+	}
+	location = /api/ {
+		try_files $uri $uri/ /en-US/index.html =404;
+	}
+
+	# mainnet API
+	location /api/v1/donations {
+		proxy_pass https://mempool.space;
+	}
+	location /api/v1/donations/images {
+		proxy_pass https://mempool.space;
+	}
+	location /api/v1/ws {
+		proxy_pass http://__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/;
+		proxy_http_version 1.1;
+		proxy_set_header Upgrade $http_upgrade;
+		proxy_set_header Connection "Upgrade";
+	}
+	location /api/v1 {
+		proxy_pass http://__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/api/v1;
+	}
+	location /api/ {
+		proxy_pass http://__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/api/v1/;
+	}
+
+	# mainnet API
+	location /ws {
+		proxy_pass http://__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/;
+		proxy_http_version 1.1;
+		proxy_set_header Upgrade $http_upgrade;
+		proxy_set_header Connection "Upgrade";
+	}
--- a/frontend/docker/nginx.conf
+++ b/frontend/docker/nginx.conf
@ -0,0 +1,128 @@
+
+pid /var/run/nginx.pid;
+
+worker_processes auto;
+worker_rlimit_nofile 100000;
+
+events {
+	worker_connections 9000;
+	multi_accept on;
+}
+
+http {
+	sendfile on;
+	tcp_nopush on;
+	tcp_nodelay on;
+
+	server_tokens off;
+	server_name_in_redirect off;
+
+	include /etc/nginx/mime.types;
+	default_type application/octet-stream;
+
+	access_log /var/log/nginx/access.log;
+	error_log /var/log/nginx/error.log;
+
+	# reset timed out connections freeing ram
+	reset_timedout_connection on;
+	# maximum time between packets the client can pause when sending nginx any data
+	client_body_timeout 10s;
+	# maximum time the client has to send the entire header to nginx
+	client_header_timeout 10s;
+	# timeout which a single keep-alive client connection will stay open
+	keepalive_timeout 69s;
+	# maximum time between packets nginx is allowed to pause when sending the client data
+	send_timeout 10s;
+
+	# number of requests per connection, does not affect SPDY
+	keepalive_requests 100;
+
+	# enable gzip compression
+	gzip on;
+	gzip_vary on;
+	gzip_comp_level 6;
+	gzip_min_length 1000;
+	gzip_proxied expired no-cache no-store private auth;
+	# text/html is always compressed by gzip module
+	gzip_types application/javascript application/json application/ld+json application/manifest+json application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard;
+
+	# limit request body size
+	client_max_body_size 10m;
+
+	# proxy cache
+	proxy_cache off;
+	proxy_cache_path /var/cache/nginx keys_zone=cache:20m levels=1:2 inactive=600s max_size=500m;
+	types_hash_max_size 2048;
+
+	# exempt localhost from rate limit
+	geo $limited_ip {
+		default		1;
+		0.0.0.0		0;
+	}
+	map $limited_ip $limited_ip_key {
+		1 $binary_remote_addr;
+		0 '';
+	}
+
+	# rate limit requests
+	limit_req_zone $limited_ip_key zone=api:5m rate=200r/m;
+	limit_req_zone $limited_ip_key zone=electrs:5m rate=2000r/m;
+	limit_req_status 429;
+
+	# rate limit connections
+	limit_conn_zone $limited_ip_key zone=websocket:10m;
+	limit_conn_status 429;
+
+	map $http_accept_language $header_lang {
+		default en-US;
+		~*^en-US en-US;
+		~*^en en-US;
+	        ~*^ar ar;
+	        ~*^cs cs;
+	        ~*^de de;
+	        ~*^es es;
+	        ~*^fa fa;
+	        ~*^fr fr;
+	        ~*^ja ja;
+	        ~*^ka ka;
+	        ~*^hu hu;
+	        ~*^nl nl;
+	        ~*^nn nn;
+	        ~*^pt pt;
+	        ~*^sl sl;
+	        ~*^sv sv;
+	        ~*^tr tr;
+	        ~*^uk uk;
+	        ~*^vi vi;
+	        ~*^zh zh;
+	}
+
+	map $cookie_lang $lang {
+		default $header_lang;
+		~*^en-US en-US;
+		~*^en en-US;
+	        ~*^ar ar;
+	        ~*^cs cs;
+	        ~*^de de;
+	        ~*^es es;
+	        ~*^fa fa;
+	        ~*^fr fr;
+	        ~*^ja ja;
+	        ~*^ka ka;
+	        ~*^hu hu;
+	        ~*^nl nl;
+	        ~*^nn nn;
+	        ~*^pt pt;
+	        ~*^sl sl;
+	        ~*^sv sv;
+	        ~*^tr tr;
+	        ~*^uk uk;
+	        ~*^vi vi;
+	        ~*^zh zh;
+	}
+
+	server {
+		listen 0.0.0.0:8080;
+		include /etc/nginx/conf.d/nginx-mempool.conf;
+	}
+}
--- a/frontend/docker/wait-for
+++ b/frontend/docker/wait-for
--- a/docker/mysql/Dockerfile
+++ b/docker/mysql/Dockerfile
--- a/docker/mysql/mysql-mempool.cnf
+++ b/docker/mysql/mysql-mempool.cnf
--- a/mysql/setup.sql
+++ b/mysql/setup.sql
@ -0,0 +1,107 @@
+SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO";
+SET time_zone = "+00:00";
+
+CREATE TABLE `blocks` (
+  `height` int(11) NOT NULL,
+  `hash` varchar(65) NOT NULL,
+  `size` int(11) NOT NULL,
+  `weight` int(11) NOT NULL,
+  `minFee` int(11) NOT NULL,
+  `maxFee` int(11) NOT NULL,
+  `time` int(11) NOT NULL,
+  `fees` double NOT NULL,
+  `nTx` int(11) NOT NULL,
+  `medianFee` double NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+CREATE TABLE `statistics` (
+  `id` int(11) NOT NULL,
+  `added` datetime NOT NULL,
+  `unconfirmed_transactions` int(11) UNSIGNED NOT NULL,
+  `tx_per_second` float UNSIGNED NOT NULL,
+  `vbytes_per_second` int(10) UNSIGNED NOT NULL,
+  `mempool_byte_weight` int(10) UNSIGNED NOT NULL,
+  `fee_data` longtext NOT NULL,
+  `total_fee` double UNSIGNED NOT NULL,
+  `vsize_1` int(11) NOT NULL,
+  `vsize_2` int(11) NOT NULL,
+  `vsize_3` int(11) NOT NULL,
+  `vsize_4` int(11) NOT NULL,
+  `vsize_5` int(11) NOT NULL,
+  `vsize_6` int(11) NOT NULL,
+  `vsize_8` int(11) NOT NULL,
+  `vsize_10` int(11) NOT NULL,
+  `vsize_12` int(11) NOT NULL,
+  `vsize_15` int(11) NOT NULL,
+  `vsize_20` int(11) NOT NULL,
+  `vsize_30` int(11) NOT NULL,
+  `vsize_40` int(11) NOT NULL,
+  `vsize_50` int(11) NOT NULL,
+  `vsize_60` int(11) NOT NULL,
+  `vsize_70` int(11) NOT NULL,
+  `vsize_80` int(11) NOT NULL,
+  `vsize_90` int(11) NOT NULL,
+  `vsize_100` int(11) NOT NULL,
+  `vsize_125` int(11) NOT NULL,
+  `vsize_150` int(11) NOT NULL,
+  `vsize_175` int(11) NOT NULL,
+  `vsize_200` int(11) NOT NULL,
+  `vsize_250` int(11) NOT NULL,
+  `vsize_300` int(11) NOT NULL,
+  `vsize_350` int(11) NOT NULL,
+  `vsize_400` int(11) NOT NULL,
+  `vsize_500` int(11) NOT NULL,
+  `vsize_600` int(11) NOT NULL,
+  `vsize_700` int(11) NOT NULL,
+  `vsize_800` int(11) NOT NULL,
+  `vsize_900` int(11) NOT NULL,
+  `vsize_1000` int(11) NOT NULL,
+  `vsize_1200` int(11) NOT NULL,
+  `vsize_1400` int(11) NOT NULL,
+  `vsize_1600` int(11) NOT NULL,
+  `vsize_1800` int(11) NOT NULL,
+  `vsize_2000` int(11) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+CREATE TABLE `transactions` (
+  `blockheight` int(11) NOT NULL,
+  `txid` varchar(65) NOT NULL,
+  `fee` double NOT NULL,
+  `feePerVsize` double NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+
+ALTER TABLE `blocks`
+  ADD PRIMARY KEY (`height`);
+
+ALTER TABLE `statistics`
+  ADD PRIMARY KEY (`id`);
+
+ALTER TABLE `transactions`
+  ADD PRIMARY KEY (`txid`),
+  ADD KEY `blockheight` (`blockheight`);
+
+
+ALTER TABLE `statistics`
+  MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
+
+
+CREATE TABLE `donations` (
+  `id` int(11) NOT NULL,
+  `added` datetime NOT NULL,
+  `amount` float NOT NULL,
+  `handle` varchar(250) NOT NULL,
+  `order_id` varchar(25) NOT NULL,
+  `imageUrl` varchar(250) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+ALTER TABLE `donations`
+  ADD PRIMARY KEY (`id`);
+
+ALTER TABLE `donations`
+  MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
+
+ALTER TABLE `donations` ADD UNIQUE(`order_id`);
+
+ALTER TABLE `donations` ADD `image` MEDIUMBLOB NULL AFTER `imageUrl`;
+ALTER TABLE `donations` ADD `twitter_id` VARCHAR(250) NULL AFTER `handle`;