Docker Community Forums

Share and learn in the Docker community.

Traefik not work (404) with containers inside a worker node [docker swarm]

Hi All,

my problem is the following:

I created a small swarm cluster consisting of 2 simple nodes: a manager and a worker.
On the manager node I installed a stack with a Traefik service and a stack with the services for the Swarmpit dashboard.
On the worker node I installed a stack with wordpress service.
I installed an “overlay” network, named “web”, for the correct traefik configuration.

The result is that both the Traefik dashboard and the Swarmpit dashboard work properly. Anything that I put on the worker node is not “recognized” by traefik so it doesn’t work. And if I try to reach the web page of the service, error 404 is returned

If I move the wordpress service from the worker node to the manager node it starts working correctly.
In addition I am able to ping from inside the traefik container to the wordpress container via the “web” network and vice versa

What could be the problem?

Below some configurations and logs:

Traefik docker-conpose.yml configuration:

version: "3.7"
networks:
  web:
    external: true
services:
  traefik:
    image: traefik:alpine
    command:
      - --api
      - --docker
      - --logLevel=DEBUG
    networks:
      - web
    labels:
      - traefik.frontend.rule=Host:traefik.mydomain.cloud
      - traefik.docker.network=web
      - traefik.backend=traefik.mydomain.cloud
      - traefik.enable=true
      - traefik.port=8080
    ports:
      - 80:80
      - 443:443
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - data:/etc/traefik
    deploy:
      mode: global
      placement:
        constraints:
          - node.role == manager
volumes:
  data:
    driver_opts:
      type: nfs
      o: addr=x.x.x.x,nolock,soft,rw
      device: :/var/nfs/traefik


Wordpress test docker-conpose.yml configuration:

version: "3.7"
networks:
  web:
    external: true
  internal:
    external: false
services:
  app:
    image: wordpress:5.2.1-php7.1-apache
    depends_on:
      - mysql
    networks:
      - web
      - internal
    labels:
      - traefik.enable=true
      - traefik.frontend.rule=Host:www.mywordpress_test.com
      - traefik.backend=www.mywordpress_test.com
      - traefik.docker.network=web
      - traefik.port=80
    environment:
      WORDPRESS_DB_HOST: mysql:3306
      WORDPRESS_DB_USER: xxxxx
      WORDPRESS_DB_PASSWORD: xxxxx
      WORDPRESS_DB_NAME: xxxxx
    volumes:
      - www:/var/www/html
    deploy:
      placement:
        constraints:
          - node.role == worker
  mysql:
    image: mysql:5.7
    networks:
      - web
      - internal
    environment:
      MYSQL_ROOT_PASSWORD: xxxxx
      MYSQL_DATABASE: xxxxx
      MYSQL_USER: xxxxx
      MYSQL_PASSWORD: xxxxx
    volumes:
      - db_data:/var/lib/mysql/
      - db_init:/docker-entrypoint-initdb.d/
    deploy:
      placement:
        constraints:
          - node.role == worker
volumes:
  db_init:
    driver_opts:
      type: nfs
      o: addr=x.x.x.x,nolock,soft,rw
      device: :/var/nfs/wordpress_test/db/initdb.d
  db_data:
    driver_opts:
      type: nfs
      o: addr=x.x.x.x,nolock,soft,rw
      device: :/var/nfs/wordpress_test/db/data
  www:
    driver_opts:
      type: nfs
      o: addr=x.x.x.x,nolock,soft,rw
      device: :/var/nfs/wordpress_test/www


On the manager node:

    sudo docker node ls
    
    ID          HOSTNAME    STATUS  AVAILABILITY    MANAGER STATUS  ENGINE VERSION
    xxxxxx *    sw-man-01   Ready   Active          Leader          18.09.6
    xxxxxx      sw-wk-01    Ready   Active                          18.09.6

On the manager node:

    sudo docker network ls

    NETWORK ID          NAME                        DRIVER              SCOPE
    xxxxxx              bridge                      bridge              local
    xxxxxx              docker_gwbridge             bridge              local
    xxxxxx              swarmpit_net                overlay             swarm
    xxxxxx              wordpress_test_internal     overlay             swarm
    xxxxxx              host                        host                local
    xxxxxx              ingress                     overlay             swarm
    xxxxxx              none                        null                local
    xxxxxx              web                         overlay             swarm

On the manager node:

    sudo docker network inspect web

    [
    {
        "Name": "web",
        "Id": "vdvnirom26xxxxxx",
        "Created": "2019-06-17T17:59:52.865557645+02:00",
        "Scope": "swarm",
        "Driver": "overlay",
        "EnableIPv6": false,
        "IPAM": {
            "Driver": "default",
            "Options": null,
            "Config": [
                {
                    "Subnet": "10.0.0.0/24",
                    "Gateway": "10.0.0.1"
                }
            ]
        },
        "Internal": false,
        "Attachable": true,
        "Ingress": false,
        "ConfigFrom": {
            "Network": ""
        },
        "ConfigOnly": false,
        "Containers": {
            "xxxxxx": {
                "Name": "swarmpit_app.1.pefyqnzqyhk4lrnkexgs7k4wu",
                "EndpointID": "xxxxxx",
                "MacAddress": "02:42:0a:00:00:6f",
                "IPv4Address": "10.0.0.111/24",
                "IPv6Address": ""
            },
            "xxxxxx": {
                "Name": "traefik_traefik.pttpmxhv0fsq02zcew5i2qylw.qfrllyuio7yau74ablv0amqut",
                "EndpointID": "xxxxxx",
                "MacAddress": "02:42:0a:00:00:1e",
                "IPv4Address": "10.0.0.30/24",
                "IPv6Address": ""
            },
            "lb-web": {
                "Name": "web-endpoint",
                "EndpointID": "xxxxxx",
                "MacAddress": "02:42:0a:00:00:04",
                "IPv4Address": "10.0.0.4/24",
                "IPv6Address": ""
            }
        },
        "Options": {
            "com.docker.network.driver.overlay.vxlanid_list": "4097"
        },
        "Labels": {},
        "Peers": [
            {
                "Name": "xxxxxx",
                "IP": "192.168.2.120"
            },
            {
                "Name": "xxxxxx",
                "IP": "192.168.2.100"
            }
        ]
    }
]

On the manager node:

    sudo docker container inspect [traefik container id]

    [
    {
        ...,
        "NetworkSettings": {
            "Bridge": "",
            "SandboxID": "xxxxxxxx",
            "HairpinMode": false,
            "LinkLocalIPv6Address": "",
            "LinkLocalIPv6PrefixLen": 0,
            "Ports": {
                "80/tcp": null
            },
            "SandboxKey": "/var/run/docker/netns/xxxxxxxx",
            "SecondaryIPAddresses": null,
            "SecondaryIPv6Addresses": null,
            "EndpointID": "",
            "Gateway": "",
            "GlobalIPv6Address": "",
            "GlobalIPv6PrefixLen": 0,
            "IPAddress": "",
            "IPPrefixLen": 0,
            "IPv6Gateway": "",
            "MacAddress": "",
            "Networks": {
                "ingress": {
                    "IPAMConfig": {
                        "IPv4Address": "10.255.0.117"
                    },
                    "Links": null,
                    "Aliases": [
                        "xxxxxxxx"
                    ],
                    "NetworkID": "xxxxxxxx",
                    "EndpointID": "xxxxxxxx",
                    "Gateway": "",
                    "IPAddress": "10.255.0.117",
                    "IPPrefixLen": 16,
                    "IPv6Gateway": "",
                    "GlobalIPv6Address": "",
                    "GlobalIPv6PrefixLen": 0,
                    "MacAddress": "02:42:0a:ff:00:75",
                    "DriverOpts": null
                },
                "web": {
                    "IPAMConfig": {
                        "IPv4Address": "10.0.0.30"
                    },
                    "Links": null,
                    "Aliases": [
                        "xxxxxxxx"
                    ],
                    "NetworkID": "vdvnirom26xxxxxxxx",
                    "EndpointID": "56a914884xxxxxxxx",
                    "Gateway": "",
                    "IPAddress": "10.0.0.30",
                    "IPPrefixLen": 24,
                    "IPv6Gateway": "",
                    "GlobalIPv6Address": "",
                    "GlobalIPv6PrefixLen": 0,
                    "MacAddress": "02:42:0a:00:00:1e",
                    "DriverOpts": null
                }
            }
        }
    }
]

On the worker node:

    sudo docker network inspect web

    [
    {
        "Name": "web",
        "Id": "vdvnirom26xxxxxx",
        "Created": "2019-06-19T18:21:15.839889513+02:00",
        "Scope": "swarm",
        "Driver": "overlay",
        "EnableIPv6": false,
        "IPAM": {
            "Driver": "default",
            "Options": null,
            "Config": [
                {
                    "Subnet": "10.0.0.0/24",
                    "Gateway": "10.0.0.1"
                }
            ]
        },
        "Internal": false,
        "Attachable": true,
        "Ingress": false,
        "ConfigFrom": {
            "Network": ""
        },
        "ConfigOnly": false,
        "Containers": {
            "xxxxxx": {
                "Name": "wordpress_test_app.1.2mbi7i0u9rn14od5fzw1ycbuk",
                "EndpointID": "xxxxxx",
                "MacAddress": "02:42:0a:00:00:19",
                "IPv4Address": "10.0.0.25/24",
                "IPv6Address": ""
            },
            "xxxxxx": {
                "Name": "wordpress_test_mysql.1.7qktlbc9ksn164gc292zva12w",
                "EndpointID": "xxxxxx",
                "MacAddress": "02:42:0a:00:00:17",
                "IPv4Address": "10.0.0.23/24",
                "IPv6Address": ""
            },
            "lb-web": {
                "Name": "web-endpoint",
                "EndpointID": "xxxxxx",
                "MacAddress": "02:42:0a:00:00:1a",
                "IPv4Address": "10.0.0.26/24",
                "IPv6Address": ""
            }
        },
        "Options": {
            "com.docker.network.driver.overlay.vxlanid_list": "4097"
        },
        "Labels": {},
        "Peers": [
            {
                "Name": "xxxxxx",
                "IP": "192.168.2.120"
            },
            {
                "Name": "xxxxxx",
                "IP": "192.168.2.100"
            }
        ]
    }
]

On the worker node:

    sudo docker container inspect [wordpress_test container id]

    [
    {
        ...,
        "NetworkSettings": {
            "Bridge": "",
            "SandboxID": "xxxxxxx",
            "HairpinMode": false,
            "LinkLocalIPv6Address": "",
            "LinkLocalIPv6PrefixLen": 0,
            "Ports": {
                "80/tcp": null,
                "9000/tcp": null
            },
            "SandboxKey": "/var/run/docker/netns/xxxxxxx",
            "SecondaryIPAddresses": null,
            "SecondaryIPv6Addresses": null,
            "EndpointID": "",
            "Gateway": "",
            "GlobalIPv6Address": "",
            "GlobalIPv6PrefixLen": 0,
            "IPAddress": "",
            "IPPrefixLen": 0,
            "IPv6Gateway": "",
            "MacAddress": "",
            "Networks": {
                "wordpress_test_internal": {
                    "IPAMConfig": {
                        "IPv4Address": "10.0.29.35"
                    },
                    "Links": null,
                    "Aliases": [
                        "xxxxxxx"
                    ],
                    "NetworkID": "xxxxxxx",
                    "EndpointID": "xxxxxxx",
                    "Gateway": "",
                    "IPAddress": "10.0.29.35",
                    "IPPrefixLen": 24,
                    "IPv6Gateway": "",
                    "GlobalIPv6Address": "",
                    "GlobalIPv6PrefixLen": 0,
                    "MacAddress": "02:42:0a:00:1d:23",
                    "DriverOpts": null
                },
                "web": {
                    "IPAMConfig": {
                        "IPv4Address": "10.0.0.25"
                    },
                    "Links": null,
                    "Aliases": [
                        "xxxxxxx"
                    ],
                    "NetworkID": "vdvnirom26xxxxxxx",
                    "EndpointID": "cabe06bc3b1c48acxxxxxxx",
                    "Gateway": "",
                    "IPAddress": "10.0.0.25",
                    "IPPrefixLen": 24,
                    "IPv6Gateway": "",
                    "GlobalIPv6Address": "",
                    "GlobalIPv6PrefixLen": 0,
                    "MacAddress": "02:42:0a:00:00:19",
                    "DriverOpts": null
                }
            }
        }
    }
]

You need to add the labels undearneath deploy and not on the container level.