Re-implement Docker workflow using GitHub Actions (#326)

* Recreate a docker dir with all docker files inside and add init.sh
* Add docker/README.md with example docker-compose.xml
* Add mysql vars for mempool-config.json
* Port can be set from env var in docker-compose.xml
* Update docker-compose.xml example to use latest tag
* Remove armv7 architecture from build workflow
* Remove master git-hash file
* Remove useless ':' in sed

.github/workflows/on-tag.yml

@@ -34,6 +34,9 @@ jobs:
       - name: Checkout project
         uses: actions/checkout@v2
 
+      - name: Init repo for Dockerization
+        run: docker/init.sh "$TAG"
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
         id: qemu

backend/docker/master

@@ -1 +0,0 @@
-9d02ab1eb5ffb60d38128df903e47e11b95f13d5

docker/README.md

@@ -0,0 +1,99 @@
+# Docker
+
+## Initialization
+
+In an empty dir create 2 sub-dirs
+
+```bash
+mkdir -p data mysql/data mysql/db-scripts
+```
+
+In the mysql/db-scripts sub-dir add the mariadb-structure.sql file from the mempool repo
+
+Your dir should now look like that:
+
+```bash
+$ls -R
+.:
+data mysql
+
+./data:
+
+./mysql:
+data  db-scripts
+
+./mysql/data:
+
+./mysql/db-scripts:
+mariadb-structure.sql
+```
+
+In the main dir add the following docker-compose.yml
+
+```bash
+version: "3.7"
+
+services:
+  web:
+    image: mempool/frontend:latest
+    user: "1000:1000"
+    restart: on-failure
+    stop_grace_period: 1m
+    command: "./wait-for db:3306 --timeout=720 -- nginx -g 'daemon off;'"
+    ports:
+      - 80:8080
+    environment:
+      FRONTEND_HTTP_PORT: "8080"
+      BACKEND_MAINNET_HTTP_HOST: "api"
+  api:
+    image: mempool/backend:v2.1.0
+    user: "1000:1000"
+    restart: on-failure
+    stop_grace_period: 1m
+    command: "./wait-for-it.sh db:3306 --timeout=720 --strict -- ./start.sh"
+    volumes:
+      - ./data:/backend/cache
+    environment:
+      RPC_HOST: "127.0.0.1"
+      RPC_PORT: "8332"
+      RPC_USER: "mempool"
+      RPC_PASS: "mempool"
+      ELECTRS_HOST: "127.0.0.1"
+      ELECTRS_PORT: "50002"
+      MYSQL_HOST: "db"
+      MYSQL_PORT: "3306"
+      MYSQL_DATABASE: "mempool"
+      MYSQL_USER: "mempool"
+      MYSQL_PASS: "mempool"
+      BACKEND_MAINNET_HTTP_PORT: "8999"
+      CACHE_DIR: "/backend/cache/"
+  db:
+    image: mariadb:10.5.8
+    user: "1000:1000"
+    restart: on-failure
+    stop_grace_period: 1m
+    volumes:
+      - ./mysql/data:/var/lib/mysql
+      - ./mysql/db-scripts:/docker-entrypoint-initdb.d
+    environment:
+      MYSQL_DATABASE: "mempool"
+      MYSQL_USER: "mempool"
+      MYSQL_PASSWORD: "mempool"
+      MYSQL_ROOT_PASSWORD: "admin"
+
+```
+
+You can update all the environment variables inside the API container, especially the RPC and ELECTRS ones
+
+## Run it
+
+To run our docker-compose use the following cmd:
+
+```bash
+docker-compose up
+```
+
+If everything went okay you should see the beautiful mempool :grin:
+
+If you get stuck on "loading blocks", this means the websocket can't connect.
+Check your nginx proxy setup, firewalls, etc. and open an issue if you need help.

docker/backend/Dockerfile

@@ -3,17 +3,12 @@ FROM node:12-buster-slim AS builder
 WORKDIR /build
 COPY . .
 
-RUN sed -i "s!../.git/refs/heads/master!master!g" ./src/api/backend-info.ts
-
 RUN apt-get update
 RUN apt-get install -y build-essential python3 pkg-config
 RUN npm ci --production
 RUN npm i typescript
 RUN npm run build
 
-RUN mv ./docker/* .
-RUN mv ./mempool-config-docker.json ./mempool-config.json
-
 FROM node:12-buster-slim
 
 WORKDIR /backend

docker/backend/mempool-config.json

@@ -27,9 +27,9 @@
     "ENABLED": true,
     "HOST": "__MYSQL_HOST__",
     "PORT": __MYSQL_PORT__,
-    "DATABASE": "mempool",
+    "DATABASE": "__MYSQL_DATABASE__",
-    "USERNAME": "mempool",
+    "USERNAME": "__MYSQL_USERNAME__",
-    "PASSWORD": "mempool"
+    "PASSWORD": "__MYSQL_PASSWORD__"
   },
   "STATISTICS": {
     "ENABLED": true,

docker/backend/start.sh

@@ -14,6 +14,9 @@ __ELECTRS_MAINNET_HTTP_PORT__=${ELECTRS_PORT:=50002}
 # MYSQL
 __MYSQL_HOST__=${MYSQL_HOST:=127.0.0.1}
 __MYSQL_PORT__=${MYSQL_PORT:=3306}
+__MYSQL_DATABASE__=${MYSQL_DATABASE:=mempool}
+__MYSQL_USERNAME__=${MYSQL_USER:=mempool}
+__MYSQL_PASSWORD__=${MYSQL_PASS:=mempool}
 
 mkdir -p "${__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__}"
 
@@ -25,6 +28,9 @@ sed -i "s/__ELECTRS_MAINNET_HTTP_HOST__/${__ELECTRS_MAINNET_HTTP_HOST__}/g" memp
 sed -i "s/__ELECTRS_MAINNET_HTTP_PORT__/${__ELECTRS_MAINNET_HTTP_PORT__}/g" mempool-config.json
 sed -i "s/__MYSQL_HOST__/${__MYSQL_HOST__}/g" mempool-config.json
 sed -i "s/__MYSQL_PORT__/${__MYSQL_PORT__}/g" mempool-config.json
+sed -i "s/__MYSQL_DATABASE__/${__MYSQL_DATABASE__}/g" mempool-config.json
+sed -i "s/__MYSQL_USERNAME__/${__MYSQL_USERNAME__}/g" mempool-config.json
+sed -i "s/__MYSQL_PASSWORD__/${__MYSQL_PASSWORD__}/g" mempool-config.json
 sed -i "s!__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__!${__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__}!g" mempool-config.json
 sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/${__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__}/g" mempool-config.json
 

docker/frontend/Dockerfile

@@ -7,8 +7,6 @@ RUN apt-get install -y build-essential rsync
 RUN npm i
 RUN npm run build
 
-RUN mv ./docker/* .
-
 FROM nginx:1.17.8-alpine
 
 WORKDIR /patch
@@ -25,13 +23,12 @@ RUN chmod +x /patch/wait-for
 RUN chown -R 1000:1000 /patch && chmod -R 755 /patch && \
         chown -R 1000:1000 /var/cache/nginx && \
         chown -R 1000:1000 /var/log/nginx && \
+        chown -R 1000:1000 /etc/nginx/nginx.conf && \
         chown -R 1000:1000 /etc/nginx/conf.d
 RUN touch /var/run/nginx.pid && \
         chown -R 1000:1000 /var/run/nginx.pid
 
 USER 1000
 
-EXPOSE 8080
-
 ENTRYPOINT ["/patch/entrypoint.sh"]
 CMD ["nginx", "-g", "daemon off;"]

docker/frontend/entrypoint.sh

@@ -1,8 +1,13 @@
 #!/bin/sh
 __MEMPOOL_BACKEND_MAINNET_HTTP_HOST__=${BACKEND_MAINNET_HTTP_HOST:=127.0.0.1}
 __MEMPOOL_BACKEND_MAINNET_HTTP_PORT__=${BACKEND_MAINNET_HTTP_PORT:=8999}
+__MEMPOOL_FRONTEND_HTTP_PORT__=${FRONTEND_HTTP_PORT:=8080}
 
 sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__/${__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__}/g" /etc/nginx/conf.d/nginx-mempool.conf
 sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/${__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__}/g" /etc/nginx/conf.d/nginx-mempool.conf
 
+cp /etc/nginx/nginx.conf /patch/nginx.conf
+sed -i "s/__MEMPOOL_FRONTEND_HTTP_PORT__/${__MEMPOOL_FRONTEND_HTTP_PORT__}/g" /patch/nginx.conf
+cat /patch/nginx.conf > /etc/nginx/nginx.conf
+
 exec "$@"

docker/init.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+#backend
+gitMaster="\.\.\/\.git\/refs\/heads\/master"
+git ls-remote https://github.com/mempool/mempool.git $1 | awk '{ print $1}' > ./backend/master
+cp ./docker/backend/* ./backend/
+sed -i "s/${gitMaster}/master/g" ./backend/src/api/backend-info.ts
+
+#frontend
+localhostIP="127.0.0.1"
+cp ./docker/frontend/* ./frontend
+cp ./nginx.conf ./frontend/
+cp ./nginx-mempool.conf ./frontend/
+sed -i "s/${localhostIP}:80/0.0.0.0:__MEMPOOL_FRONTEND_HTTP_PORT__/g" ./frontend/nginx.conf
+sed -i "s/${localhostIP}/0.0.0.0/g" ./frontend/nginx.conf
+sed -i "s/user nobody;//g" ./frontend/nginx.conf
+sed -i "s!/etc/nginx/nginx-mempool.conf!/etc/nginx/conf.d/nginx-mempool.conf!g" ./frontend/nginx.conf
+sed -i "s/${localhostIP}:8999/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/g" ./frontend/nginx-mempool.conf

frontend/docker/nginx-mempool.conf

@@ -1,62 +0,0 @@
-	access_log /var/log/nginx/access_mempool.log;
-	error_log /var/log/nginx/error_mempool.log;
-
-	root /var/www/mempool/browser;
-
-	index index.html;
-
-	# fallback for all URLs i.e. /address/foo /tx/foo /block/000
-	location / {
-		try_files /$lang/$uri /$lang/$uri/ $uri $uri/ /en-US/$uri @index-redirect;
-	}
-	location @index-redirect {
-		add_header vary accept-language;
-		rewrite (.*) /$lang/index.html;
-	}
-
-	# location block using regex are matched in order
-
-	# used to rewrite resources from /<lang>/ to /en-US/
-	location ~ ^/(ar|bg|bs|ca|cs|da|de|et|el|es|eo|eu|fa|fr|gl|ko|hr|id|it|he|ka|lv|lt|hu|mk|ms|nl|ja|ka|no|nb|nn|pl|pt|pt-BR|ro|ru|sk|sl|sr|sh|fi|sv|th|tr|uk|vi|zh)/resources/ {
-		rewrite ^/[a-zA-Z-]*/resources/(.*) /en-US/resources/$1;
-	}
-	# used for cookie override
-	location ~ ^/(ar|bg|bs|ca|cs|da|de|et|el|es|eo|eu|fa|fr|gl|ko|hr|id|it|he|ka|lv|lt|hu|mk|ms|nl|ja|ka|no|nb|nn|pl|pt|pt-BR|ro|ru|sk|sl|sr|sh|fi|sv|th|tr|uk|vi|zh)/ {
-		try_files $uri $uri/ /$1/index.html =404;
-	}
-
-	# static API docs
-	location = /api {
-		try_files $uri $uri/ /en-US/index.html =404;
-	}
-	location = /api/ {
-		try_files $uri $uri/ /en-US/index.html =404;
-	}
-
-	# mainnet API
-	location /api/v1/donations {
-		proxy_pass https://mempool.space;
-	}
-	location /api/v1/donations/images {
-		proxy_pass https://mempool.space;
-	}
-	location /api/v1/ws {
-		proxy_pass http://__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/;
-		proxy_http_version 1.1;
-		proxy_set_header Upgrade $http_upgrade;
-		proxy_set_header Connection "Upgrade";
-	}
-	location /api/v1 {
-		proxy_pass http://__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/api/v1;
-	}
-	location /api/ {
-		proxy_pass http://__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/api/v1/;
-	}
-
-	# mainnet API
-	location /ws {
-		proxy_pass http://__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/;
-		proxy_http_version 1.1;
-		proxy_set_header Upgrade $http_upgrade;
-		proxy_set_header Connection "Upgrade";
-	}

frontend/docker/nginx.conf

@@ -1,128 +0,0 @@
-
-pid /var/run/nginx.pid;
-
-worker_processes auto;
-worker_rlimit_nofile 100000;
-
-events {
-	worker_connections 9000;
-	multi_accept on;
-}
-
-http {
-	sendfile on;
-	tcp_nopush on;
-	tcp_nodelay on;
-
-	server_tokens off;
-	server_name_in_redirect off;
-
-	include /etc/nginx/mime.types;
-	default_type application/octet-stream;
-
-	access_log /var/log/nginx/access.log;
-	error_log /var/log/nginx/error.log;
-
-	# reset timed out connections freeing ram
-	reset_timedout_connection on;
-	# maximum time between packets the client can pause when sending nginx any data
-	client_body_timeout 10s;
-	# maximum time the client has to send the entire header to nginx
-	client_header_timeout 10s;
-	# timeout which a single keep-alive client connection will stay open
-	keepalive_timeout 69s;
-	# maximum time between packets nginx is allowed to pause when sending the client data
-	send_timeout 10s;
-
-	# number of requests per connection, does not affect SPDY
-	keepalive_requests 100;
-
-	# enable gzip compression
-	gzip on;
-	gzip_vary on;
-	gzip_comp_level 6;
-	gzip_min_length 1000;
-	gzip_proxied expired no-cache no-store private auth;
-	# text/html is always compressed by gzip module
-	gzip_types application/javascript application/json application/ld+json application/manifest+json application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard;
-
-	# limit request body size
-	client_max_body_size 10m;
-
-	# proxy cache
-	proxy_cache off;
-	proxy_cache_path /var/cache/nginx keys_zone=cache:20m levels=1:2 inactive=600s max_size=500m;
-	types_hash_max_size 2048;
-
-	# exempt localhost from rate limit
-	geo $limited_ip {
-		default		1;
-		0.0.0.0		0;
-	}
-	map $limited_ip $limited_ip_key {
-		1 $binary_remote_addr;
-		0 '';
-	}
-
-	# rate limit requests
-	limit_req_zone $limited_ip_key zone=api:5m rate=200r/m;
-	limit_req_zone $limited_ip_key zone=electrs:5m rate=2000r/m;
-	limit_req_status 429;
-
-	# rate limit connections
-	limit_conn_zone $limited_ip_key zone=websocket:10m;
-	limit_conn_status 429;
-
-	map $http_accept_language $header_lang {
-		default en-US;
-		~*^en-US en-US;
-		~*^en en-US;
-	        ~*^ar ar;
-	        ~*^cs cs;
-	        ~*^de de;
-	        ~*^es es;
-	        ~*^fa fa;
-	        ~*^fr fr;
-	        ~*^ja ja;
-	        ~*^ka ka;
-	        ~*^hu hu;
-	        ~*^nl nl;
-	        ~*^nn nn;
-	        ~*^pt pt;
-	        ~*^sl sl;
-	        ~*^sv sv;
-	        ~*^tr tr;
-	        ~*^uk uk;
-	        ~*^vi vi;
-	        ~*^zh zh;
-	}
-
-	map $cookie_lang $lang {
-		default $header_lang;
-		~*^en-US en-US;
-		~*^en en-US;
-	        ~*^ar ar;
-	        ~*^cs cs;
-	        ~*^de de;
-	        ~*^es es;
-	        ~*^fa fa;
-	        ~*^fr fr;
-	        ~*^ja ja;
-	        ~*^ka ka;
-	        ~*^hu hu;
-	        ~*^nl nl;
-	        ~*^nn nn;
-	        ~*^pt pt;
-	        ~*^sl sl;
-	        ~*^sv sv;
-	        ~*^tr tr;
-	        ~*^uk uk;
-	        ~*^vi vi;
-	        ~*^zh zh;
-	}
-
-	server {
-		listen 0.0.0.0:8080;
-		include /etc/nginx/conf.d/nginx-mempool.conf;
-	}
-}

mysql-mempool.cnf

@@ -1,2 +0,0 @@
-[mysqld]
-sql_mode="NO_AUTO_VALUE_ON_ZERO"