Docker Community Forums

Share and learn in the Docker community.

Docker swarm network not synced on docker work node sometimes

I find out that sometimes the docker swarm network is not synced.
And sometimes the network port is not binded synced too.
Nginx proxy API to the node1 node2 node3 servers.
I have node1 leader node and node2 node3 work node on docker swarm mode.
I have to redeploy the API service again then the API can response correct.
Sorry for my pool english

#my nginx conf looks like this
    upstream spweb{
        ip_hash;
        server node1.x.com:8284;
        server node2.x.com:8284;
        server node3.x.com:8284;
    }
[root@node1 sp]# cat sp-web-stack.yml 
# this is one of my API stack conf
version: "3.7"
services:
  sp-web-prod:
    image: x/sp-web-prod:3.1
    restart: always
    deploy:
      replicas: 2
      update_config:
        parallelism: 1
        delay: 1m
      restart_policy:
        condition: on-failure
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8284/web/actuator/health"]
      interval: 1m
      timeout: 5s
      retries: 3
      start_period: 30s
    environment:
      - "TZ=Asia/Shanghai"
      - "SPRING_PROFILES_ACTIVE=prod"
    volumes:
      - /app:/app
    ports:
      - "8284:8284"
      - "8285:8285"
    networks:
      - sp-web
networks:
  sp-web:
    driver: overlay
    attachable: true

The OS is CentOS 7.9
Kernel version 5.7.0-1.el7.elrepo.x86_64
docker/docker-compose version is

[root@node1 sp]# docker -v
Docker version 20.10.1, build 831ebea
[root@node1 sp]# docker-compose -v
docker-compose version 1.27.4, build 40524192
#node1 
[root@node1 sp]# docker node ls
ID                            HOSTNAME   STATUS    AVAILABILITY   MANAGER STATUS   ENGINE VERSION
ryi98e44x4gndqwv0ej40gju4 *   node1      Ready     Active         Leader           20.10.1
ahjqosvzffqdcp77rcuow37be     node2      Ready     Active                          20.10.1
srlbttf3la5hm6l064ih1iqj9     node3      Ready     Active                          20.10.1

[root@node1 sp]# docker network ls
NETWORK ID     NAME                                       DRIVER    SCOPE
0901227ab441   bridge                                     bridge    local
vjk8auyypswk   consul_consul                              overlay   swarm
d439ee06c2ed   docker_gwbridge                            bridge    local
460936d2aba7   es-prod_default                            bridge    local
s96o38pqhptx   es-prod_es                                 overlay   swarm
36cc5a078a0b   host                                       host      local
mmnb3aqmyl44   ingress                                    overlay   swarm
268d64514238   nginx_default                              bridge    local
6af3297290c6   none                                       null      local
e9xzpvzaqe4a   portainer_agent_network                    overlay   swarm
t3va2ukvkzai   prod                                       overlay   swarm
xjwnn2h3eee7   rabbitmq-prod_rabbitmq-cluster             overlay   swarm
bwrwxl7ut6ul   redis_default                              overlay   swarm
73e94ad9141c   redis_redis_prod_default                   bridge    local
l46j1mbl9ssv   salary-calculator-prod_salary-calculator   overlay   swarm
ibrbge28n5z7   short-url-prod_short-url                   overlay   swarm
p4ja5gmqd57z   sp-management-prod_sp-management           overlay   swarm
axxgbemcnntw   sp-management-test_sp-management           overlay   swarm
ujbxsjeppwlr   sp-pay-prod_sp-pay                         overlay   swarm
rhqqxnmyf0zm   sp-pdf-prod_sp-pdf                         overlay   swarm
dxainbwgqun8   sp-web-prod_sp-web                         overlay   swarm
[root@node1 sp]# 
[root@node1 sp]# netstat -ntpl
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name              
tcp6       0      0 :::9000                 :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::2377                 :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::7946                 :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::8784                 :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::80                   :::*                    LISTEN      1429/docker-proxy   
tcp6       0      0 :::8785                 :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::9201                 :::*                    LISTEN      1457/docker-proxy   
tcp6       0      0 :::8082                 :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::8083                 :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::8084                 :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::8884                 :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::8085                 :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::8885                 :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::9301                 :::*                    LISTEN      1445/docker-proxy   
tcp6       0      0 :::35671                :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::35672                :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::8088                 :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::8185                 :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::35673                :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::8089                 :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::8186                 :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::56379                :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::443                  :::*                    LISTEN      1417/docker-proxy   
tcp6       0      0 :::8284                 :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::56380                :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::8285                 :::*                    LISTEN      1095/dockerd        
tcp6       0      0 :::8000                 :::*                    LISTEN      1095/dockerd
#node2
[root@node2 ~]# docker network ls
NETWORK ID     NAME                                       DRIVER    SCOPE
98e13850bf2b   bridge                                     bridge    local
fd3b36a780a2   docker_gwbridge                            bridge    local
151bdb911836   host                                       host      local
mmnb3aqmyl44   ingress                                    overlay   swarm
12cfc7eafcfa   nginx_default                              bridge    local
72460f72a929   none                                       null      local
e9xzpvzaqe4a   portainer_agent_network                    overlay   swarm
xjwnn2h3eee7   rabbitmq-prod_rabbitmq-cluster             overlay   swarm
l46j1mbl9ssv   salary-calculator-prod_salary-calculator   overlay   swarm
axxgbemcnntw   sp-management-test_sp-management           overlay   swarm
ujbxsjeppwlr   sp-pay-prod_sp-pay                         overlay   swarm
rhqqxnmyf0zm   sp-pdf-prod_sp-pdf                         overlay   swarm
[root@node2 ~]# 
[root@node2 ~]# 
[root@node2 ~]# netstat -ntpl
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name    
tcp6       0      0 :::7946                 :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::8784                 :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::80                   :::*                    LISTEN      6174/docker-proxy   
tcp6       0      0 :::8785                 :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::8082                 :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::8083                 :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::8084                 :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::8884                 :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::8085                 :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::8885                 :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::35671                :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::35672                :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::8088                 :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::8185                 :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::35673                :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::8089                 :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::8186                 :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::56379                :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::443                  :::*                    LISTEN      6162/docker-proxy   
tcp6       0      0 :::8284                 :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::56380                :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::8285                 :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::8000                 :::*                    LISTEN      5999/dockerd        
tcp6       0      0 :::9000                 :::*                    LISTEN      5999/dockerd
#node3
[root@node3 ~]# docker network ls
NETWORK ID     NAME                             DRIVER    SCOPE
48636387560b   bridge                           bridge    local
ebe02370d888   docker_gwbridge                  bridge    local
d2b8fa7bb178   host                             host      local
mmnb3aqmyl44   ingress                          overlay   swarm
772412d4f717   none                             null      local
e9xzpvzaqe4a   portainer_agent_network          overlay   swarm
xjwnn2h3eee7   rabbitmq-prod_rabbitmq-cluster   overlay   swarm
bwrwxl7ut6ul   redis_default                    overlay   swarm
dxainbwgqun8   sp-web-prod_sp-web               overlay   swarm
[root@node3 ~]# 
[root@node3 ~]# 
[root@node3 ~]# netstat -ntpl
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name             
tcp6       0      0 :::9000                 :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::7946                 :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::8784                 :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::8785                 :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::8082                 :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::8083                 :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::8084                 :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::8884                 :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::8085                 :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::8885                 :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::35671                :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::35672                :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::8088                 :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::8185                 :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::35673                :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::8089                 :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::8186                 :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::56379                :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::8284                 :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::56380                :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::8285                 :::*                    LISTEN      6254/dockerd        
tcp6       0      0 :::8000                 :::*                    LISTEN      6254/dockerd