Container UDP port unreachable

Hi everyone,

We’re having issues of missing syslog message, after further debugging I’ve seen that sometimes a container with exposed port 514 gives port unreachable even when trying to send a log from the host running the container:

IP 100.96.6.1.53833 > 100.96.6.29.syslog: [|syslog]
IP 100.96.6.1.35763 > 100.96.6.29.syslog: [|syslog]
IP 100.96.6.29 > 100.96.6.1: ICMP 100.96.6.29 udp port syslog unreachable, length 60
IP 100.96.6.1.38080 > 100.96.6.29.syslog: [|syslog]
IP 100.96.6.29 > 100.96.6.1: ICMP 100.96.6.29 udp port syslog unreachable, length 60
IP 100.96.6.1.60517 > 100.96.6.29.syslog: [|syslog]
IP 100.96.6.29 > 100.96.6.1: ICMP 100.96.6.29 udp port syslog unreachable, length 60
IP 100.96.6.1.40651 > 100.96.6.29.syslog: [|syslog]
IP 100.96.6.29 > 100.96.6.1: ICMP 100.96.6.29 udp port syslog unreachable, length 60

as you can see it randomly returns port 514 unreachable without a reason. There is nothing in host’s syslog.

Any idea?

Update:

Docker version is 1.12.6, build 78d1802 (latest available for kubernetes)
I’ve attached docker inspect for container

[
    {
        "Id": "e2aca1ca2a37e5597398e22090555c2c29a5bafd329f5606713baa110f644d3c",
        "Created": "2017-08-22T21:14:24.044482116Z",
        "Path": "/tmp/run.sh",
        "Args": [],
        "State": {
            "Status": "running",
            "Running": true,
            "Paused": false,
            "Restarting": false,
            "OOMKilled": false,
            "Dead": false,
            "Pid": 16156,
            "ExitCode": 0,
            "Error": "",
            "StartedAt": "2017-08-22T21:14:24.175314428Z",
            "FinishedAt": "0001-01-01T00:00:00Z"
        },
        "Image": "sha256:aec3cdaf0112a16b2ba7bf4bcdb5edec15a6af82f7c04ab5c5039547f18f1f5e",
        "ResolvConfPath": "/var/lib/docker/containers/93fa707b378b82094b2052585aaacd70f2b5763dece430225d7858ecacd86d14/resolv.conf",
        "HostnamePath": "/var/lib/docker/containers/93fa707b378b82094b2052585aaacd70f2b5763dece430225d7858ecacd86d14/hostname",
        "HostsPath": "/var/lib/kubelet/pods/de4f8a7f-877e-11e7-a6d8-0acffb3485de/etc-hosts",
        "LogPath": "/var/lib/docker/containers/e2aca1ca2a37e5597398e22090555c2c29a5bafd329f5606713baa110f644d3c/e2aca1ca2a37e5597398e22090555c2c29a5bafd329f5606713baa110f644d3c-json.log",
        "Name": "/k8s_loggly_loggly-2860579233-5p126_goalshouter-test_de4f8a7f-877e-11e7-a6d8-0acffb3485de_0",
        "RestartCount": 0,
        "Driver": "overlay",
        "MountLabel": "",
        "ProcessLabel": "",
        "AppArmorProfile": "",
        "ExecIDs": [
            "d9b0b4f0cc26475a0de7de5a3111dd3bf99ee7df79e7284ffeecedb18522eb66",
            "7ebf1eaf7e598a14d88e35fdcea3994f9f1622272e1dd955f73b0d16320efc3b"
        ],
        "HostConfig": {
            "Binds": [
                "/var/lib/kubelet/pods/de4f8a7f-877e-11e7-a6d8-0acffb3485de/volumes/kubernetes.io~secret/default-token-l9ndt:/var/run/secrets/kubernetes.io/serviceaccount:ro",
                "/var/lib/kubelet/pods/de4f8a7f-877e-11e7-a6d8-0acffb3485de/etc-hosts:/etc/hosts",
                "/var/lib/kubelet/pods/de4f8a7f-877e-11e7-a6d8-0acffb3485de/containers/loggly/9e6c89e2:/dev/termination-log"
            ],
            "ContainerIDFile": "",
            "LogConfig": {
                "Type": "json-file",
                "Config": {
                    "max-file": "5",
                    "max-size": "10m"
                }
            },
            "NetworkMode": "container:93fa707b378b82094b2052585aaacd70f2b5763dece430225d7858ecacd86d14",
            "PortBindings": null,
            "RestartPolicy": {
                "Name": "",
                "MaximumRetryCount": 0
            },
            "AutoRemove": false,
            "VolumeDriver": "",
            "VolumesFrom": null,
            "CapAdd": null,
            "CapDrop": null,
            "Dns": null,
            "DnsOptions": null,
            "DnsSearch": null,
            "ExtraHosts": null,
            "GroupAdd": null,
            "IpcMode": "container:93fa707b378b82094b2052585aaacd70f2b5763dece430225d7858ecacd86d14",
            "Cgroup": "",
            "Links": null,
            "OomScoreAdj": 994,
            "PidMode": "",
            "Privileged": false,
            "PublishAllPorts": false,
            "ReadonlyRootfs": false,
            "SecurityOpt": [
                "seccomp=unconfined"
            ],
            "UTSMode": "",
            "UsernsMode": "",
            "ShmSize": 67108864,
            "Runtime": "runc",
            "ConsoleSize": [
                0,
                0
            ],
            "Isolation": "",
            "CpuShares": 2,
            "Memory": 52428800,
            "CgroupParent": "/kubepods/burstable/podde4f8a7f-877e-11e7-a6d8-0acffb3485de",
            "BlkioWeight": 0,
            "BlkioWeightDevice": null,
            "BlkioDeviceReadBps": null,
            "BlkioDeviceWriteBps": null,
            "BlkioDeviceReadIOps": null,
            "BlkioDeviceWriteIOps": null,
            "CpuPeriod": 0,
            "CpuQuota": 0,
            "CpusetCpus": "",
            "CpusetMems": "",
            "Devices": [],
            "DiskQuota": 0,
            "KernelMemory": 0,
            "MemoryReservation": 0,
            "MemorySwap": -1,
            "MemorySwappiness": -1,
            "OomKillDisable": false,
            "PidsLimit": 0,
            "Ulimits": null,
            "CpuCount": 0,
            "CpuPercent": 0,
            "IOMaximumIOps": 0,
            "IOMaximumBandwidth": 0
        },
        "GraphDriver": {
            "Name": "overlay",
            "Data": {
                "LowerDir": "/var/lib/docker/overlay/eca00431ae79c222147d0e020665e36dea1f08f40f6e120a8a66c2645313f3d7/root",
                "MergedDir": "/var/lib/docker/overlay/84a5d5e0a525423813eb871ba3bae2e87cfd78c27e5e67cf23bbb60c7ea52d61/merged",
                "UpperDir": "/var/lib/docker/overlay/84a5d5e0a525423813eb871ba3bae2e87cfd78c27e5e67cf23bbb60c7ea52d61/upper",
                "WorkDir": "/var/lib/docker/overlay/84a5d5e0a525423813eb871ba3bae2e87cfd78c27e5e67cf23bbb60c7ea52d61/work"
            }
        },
        "Mounts": [
            {
                "Source": "/var/lib/kubelet/pods/de4f8a7f-877e-11e7-a6d8-0acffb3485de/volumes/kubernetes.io~secret/default-token-l9ndt",
                "Destination": "/var/run/secrets/kubernetes.io/serviceaccount",
                "Mode": "ro",
                "RW": false,
                "Propagation": "rprivate"
            },
            {
                "Source": "/var/lib/kubelet/pods/de4f8a7f-877e-11e7-a6d8-0acffb3485de/etc-hosts",
                "Destination": "/etc/hosts",
                "Mode": "",
                "RW": true,
                "Propagation": "rprivate"
            },
            {
                "Source": "/var/lib/kubelet/pods/de4f8a7f-877e-11e7-a6d8-0acffb3485de/containers/loggly/9e6c89e2",
                "Destination": "/dev/termination-log",
                "Mode": "",
                "RW": true,
                "Propagation": "rprivate"
            }
        ],
        "Config": {
            "Hostname": "loggly-2860579233-5p126",
            "Domainname": "",
            "User": "0",
            "AttachStdin": false,
            "AttachStdout": false,
            "AttachStderr": false,
            "ExposedPorts": {
                "514/tcp": {},
                "514/udp": {}
            },
            "Tty": false,
            "OpenStdin": false,
            "StdinOnce": false,
            "Env": [
                ....redacted....
            ],
            "Cmd": [
                "/tmp/run.sh"
            ],
            "Image": "sendgridlabs/loggly-docker@sha256:fcdc482a079dc98a0b78e8e75340f89ec9710dec95ad29452f46433d2a2119c7",
            "Volumes": null,
            "WorkingDir": "",
            "Entrypoint": null,
            "OnBuild": null,
            "Labels": {
                "annotation.io.kubernetes.container.hash": "25ff262e",
                "annotation.io.kubernetes.container.ports": "[{\"containerPort\":514,\"protocol\":\"UDP\"}]",
                "annotation.io.kubernetes.container.restartCount": "0",
                "annotation.io.kubernetes.container.terminationMessagePath": "/dev/termination-log",
                "annotation.io.kubernetes.container.terminationMessagePolicy": "File",
                "annotation.io.kubernetes.pod.terminationGracePeriod": "30",
                "io.kubernetes.container.logpath": "/var/log/pods/de4f8a7f-877e-11e7-a6d8-0acffb3485de/loggly_0.log",
                "io.kubernetes.container.name": "loggly",
                "io.kubernetes.docker.type": "container",
                "io.kubernetes.pod.name": "loggly-2860579233-5p126",
                "io.kubernetes.pod.uid": "de4f8a7f-877e-11e7-a6d8-0acffb3485de",
                "io.kubernetes.sandbox.id": "93fa707b378b82094b2052585aaacd70f2b5763dece430225d7858ecacd86d14"
            }
        },
        "NetworkSettings": {
            "Bridge": "",
            "SandboxID": "",
            "HairpinMode": false,
            "LinkLocalIPv6Address": "",
            "LinkLocalIPv6PrefixLen": 0,
            "Ports": null,
            "SandboxKey": "",
            "SecondaryIPAddresses": null,
            "SecondaryIPv6Addresses": null,
            "EndpointID": "",
            "Gateway": "",
            "GlobalIPv6Address": "",
            "GlobalIPv6PrefixLen": 0,
            "IPAddress": "",
            "IPPrefixLen": 0,
            "IPv6Gateway": "",
            "MacAddress": "",
            "Networks": null
        }
    }
]

check in the container config using netstat whether the given port is listening to incoming traffic.