I find out that sometimes the docker swarm network is not synced.
And sometimes the network port is not binded synced too.
Nginx proxy API to the node1 node2 node3 servers.
I have node1 leader node and node2 node3 work node on docker swarm mode.
I have to redeploy the API service again then the API can response correct.
Sorry for my pool english
#my nginx conf looks like this
upstream spweb{
ip_hash;
server node1.x.com:8284;
server node2.x.com:8284;
server node3.x.com:8284;
}
[root@node1 sp]# cat sp-web-stack.yml
# this is one of my API stack conf
version: "3.7"
services:
sp-web-prod:
image: x/sp-web-prod:3.1
restart: always
deploy:
replicas: 2
update_config:
parallelism: 1
delay: 1m
restart_policy:
condition: on-failure
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8284/web/actuator/health"]
interval: 1m
timeout: 5s
retries: 3
start_period: 30s
environment:
- "TZ=Asia/Shanghai"
- "SPRING_PROFILES_ACTIVE=prod"
volumes:
- /app:/app
ports:
- "8284:8284"
- "8285:8285"
networks:
- sp-web
networks:
sp-web:
driver: overlay
attachable: true
The OS is CentOS 7.9
Kernel version 5.7.0-1.el7.elrepo.x86_64
docker/docker-compose version is
[root@node1 sp]# docker -v
Docker version 20.10.1, build 831ebea
[root@node1 sp]# docker-compose -v
docker-compose version 1.27.4, build 40524192
#node1
[root@node1 sp]# docker node ls
ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION
ryi98e44x4gndqwv0ej40gju4 * node1 Ready Active Leader 20.10.1
ahjqosvzffqdcp77rcuow37be node2 Ready Active 20.10.1
srlbttf3la5hm6l064ih1iqj9 node3 Ready Active 20.10.1
[root@node1 sp]# docker network ls
NETWORK ID NAME DRIVER SCOPE
0901227ab441 bridge bridge local
vjk8auyypswk consul_consul overlay swarm
d439ee06c2ed docker_gwbridge bridge local
460936d2aba7 es-prod_default bridge local
s96o38pqhptx es-prod_es overlay swarm
36cc5a078a0b host host local
mmnb3aqmyl44 ingress overlay swarm
268d64514238 nginx_default bridge local
6af3297290c6 none null local
e9xzpvzaqe4a portainer_agent_network overlay swarm
t3va2ukvkzai prod overlay swarm
xjwnn2h3eee7 rabbitmq-prod_rabbitmq-cluster overlay swarm
bwrwxl7ut6ul redis_default overlay swarm
73e94ad9141c redis_redis_prod_default bridge local
l46j1mbl9ssv salary-calculator-prod_salary-calculator overlay swarm
ibrbge28n5z7 short-url-prod_short-url overlay swarm
p4ja5gmqd57z sp-management-prod_sp-management overlay swarm
axxgbemcnntw sp-management-test_sp-management overlay swarm
ujbxsjeppwlr sp-pay-prod_sp-pay overlay swarm
rhqqxnmyf0zm sp-pdf-prod_sp-pdf overlay swarm
dxainbwgqun8 sp-web-prod_sp-web overlay swarm
[root@node1 sp]#
[root@node1 sp]# netstat -ntpl
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp6 0 0 :::9000 :::* LISTEN 1095/dockerd
tcp6 0 0 :::2377 :::* LISTEN 1095/dockerd
tcp6 0 0 :::7946 :::* LISTEN 1095/dockerd
tcp6 0 0 :::8784 :::* LISTEN 1095/dockerd
tcp6 0 0 :::80 :::* LISTEN 1429/docker-proxy
tcp6 0 0 :::8785 :::* LISTEN 1095/dockerd
tcp6 0 0 :::9201 :::* LISTEN 1457/docker-proxy
tcp6 0 0 :::8082 :::* LISTEN 1095/dockerd
tcp6 0 0 :::8083 :::* LISTEN 1095/dockerd
tcp6 0 0 :::8084 :::* LISTEN 1095/dockerd
tcp6 0 0 :::8884 :::* LISTEN 1095/dockerd
tcp6 0 0 :::8085 :::* LISTEN 1095/dockerd
tcp6 0 0 :::8885 :::* LISTEN 1095/dockerd
tcp6 0 0 :::9301 :::* LISTEN 1445/docker-proxy
tcp6 0 0 :::35671 :::* LISTEN 1095/dockerd
tcp6 0 0 :::35672 :::* LISTEN 1095/dockerd
tcp6 0 0 :::8088 :::* LISTEN 1095/dockerd
tcp6 0 0 :::8185 :::* LISTEN 1095/dockerd
tcp6 0 0 :::35673 :::* LISTEN 1095/dockerd
tcp6 0 0 :::8089 :::* LISTEN 1095/dockerd
tcp6 0 0 :::8186 :::* LISTEN 1095/dockerd
tcp6 0 0 :::56379 :::* LISTEN 1095/dockerd
tcp6 0 0 :::443 :::* LISTEN 1417/docker-proxy
tcp6 0 0 :::8284 :::* LISTEN 1095/dockerd
tcp6 0 0 :::56380 :::* LISTEN 1095/dockerd
tcp6 0 0 :::8285 :::* LISTEN 1095/dockerd
tcp6 0 0 :::8000 :::* LISTEN 1095/dockerd
#node2
[root@node2 ~]# docker network ls
NETWORK ID NAME DRIVER SCOPE
98e13850bf2b bridge bridge local
fd3b36a780a2 docker_gwbridge bridge local
151bdb911836 host host local
mmnb3aqmyl44 ingress overlay swarm
12cfc7eafcfa nginx_default bridge local
72460f72a929 none null local
e9xzpvzaqe4a portainer_agent_network overlay swarm
xjwnn2h3eee7 rabbitmq-prod_rabbitmq-cluster overlay swarm
l46j1mbl9ssv salary-calculator-prod_salary-calculator overlay swarm
axxgbemcnntw sp-management-test_sp-management overlay swarm
ujbxsjeppwlr sp-pay-prod_sp-pay overlay swarm
rhqqxnmyf0zm sp-pdf-prod_sp-pdf overlay swarm
[root@node2 ~]#
[root@node2 ~]#
[root@node2 ~]# netstat -ntpl
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp6 0 0 :::7946 :::* LISTEN 5999/dockerd
tcp6 0 0 :::8784 :::* LISTEN 5999/dockerd
tcp6 0 0 :::80 :::* LISTEN 6174/docker-proxy
tcp6 0 0 :::8785 :::* LISTEN 5999/dockerd
tcp6 0 0 :::8082 :::* LISTEN 5999/dockerd
tcp6 0 0 :::8083 :::* LISTEN 5999/dockerd
tcp6 0 0 :::8084 :::* LISTEN 5999/dockerd
tcp6 0 0 :::8884 :::* LISTEN 5999/dockerd
tcp6 0 0 :::8085 :::* LISTEN 5999/dockerd
tcp6 0 0 :::8885 :::* LISTEN 5999/dockerd
tcp6 0 0 :::35671 :::* LISTEN 5999/dockerd
tcp6 0 0 :::35672 :::* LISTEN 5999/dockerd
tcp6 0 0 :::8088 :::* LISTEN 5999/dockerd
tcp6 0 0 :::8185 :::* LISTEN 5999/dockerd
tcp6 0 0 :::35673 :::* LISTEN 5999/dockerd
tcp6 0 0 :::8089 :::* LISTEN 5999/dockerd
tcp6 0 0 :::8186 :::* LISTEN 5999/dockerd
tcp6 0 0 :::56379 :::* LISTEN 5999/dockerd
tcp6 0 0 :::443 :::* LISTEN 6162/docker-proxy
tcp6 0 0 :::8284 :::* LISTEN 5999/dockerd
tcp6 0 0 :::56380 :::* LISTEN 5999/dockerd
tcp6 0 0 :::8285 :::* LISTEN 5999/dockerd
tcp6 0 0 :::8000 :::* LISTEN 5999/dockerd
tcp6 0 0 :::9000 :::* LISTEN 5999/dockerd
#node3
[root@node3 ~]# docker network ls
NETWORK ID NAME DRIVER SCOPE
48636387560b bridge bridge local
ebe02370d888 docker_gwbridge bridge local
d2b8fa7bb178 host host local
mmnb3aqmyl44 ingress overlay swarm
772412d4f717 none null local
e9xzpvzaqe4a portainer_agent_network overlay swarm
xjwnn2h3eee7 rabbitmq-prod_rabbitmq-cluster overlay swarm
bwrwxl7ut6ul redis_default overlay swarm
dxainbwgqun8 sp-web-prod_sp-web overlay swarm
[root@node3 ~]#
[root@node3 ~]#
[root@node3 ~]# netstat -ntpl
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp6 0 0 :::9000 :::* LISTEN 6254/dockerd
tcp6 0 0 :::7946 :::* LISTEN 6254/dockerd
tcp6 0 0 :::8784 :::* LISTEN 6254/dockerd
tcp6 0 0 :::8785 :::* LISTEN 6254/dockerd
tcp6 0 0 :::8082 :::* LISTEN 6254/dockerd
tcp6 0 0 :::8083 :::* LISTEN 6254/dockerd
tcp6 0 0 :::8084 :::* LISTEN 6254/dockerd
tcp6 0 0 :::8884 :::* LISTEN 6254/dockerd
tcp6 0 0 :::8085 :::* LISTEN 6254/dockerd
tcp6 0 0 :::8885 :::* LISTEN 6254/dockerd
tcp6 0 0 :::35671 :::* LISTEN 6254/dockerd
tcp6 0 0 :::35672 :::* LISTEN 6254/dockerd
tcp6 0 0 :::8088 :::* LISTEN 6254/dockerd
tcp6 0 0 :::8185 :::* LISTEN 6254/dockerd
tcp6 0 0 :::35673 :::* LISTEN 6254/dockerd
tcp6 0 0 :::8089 :::* LISTEN 6254/dockerd
tcp6 0 0 :::8186 :::* LISTEN 6254/dockerd
tcp6 0 0 :::56379 :::* LISTEN 6254/dockerd
tcp6 0 0 :::8284 :::* LISTEN 6254/dockerd
tcp6 0 0 :::56380 :::* LISTEN 6254/dockerd
tcp6 0 0 :::8285 :::* LISTEN 6254/dockerd
tcp6 0 0 :::8000 :::* LISTEN 6254/dockerd