Hi everyone,
We’re having issues of missing syslog message, after further debugging I’ve seen that sometimes a container with exposed port 514 gives port unreachable even when trying to send a log from the host running the container:
IP 100.96.6.1.53833 > 100.96.6.29.syslog: [|syslog]
IP 100.96.6.1.35763 > 100.96.6.29.syslog: [|syslog]
IP 100.96.6.29 > 100.96.6.1: ICMP 100.96.6.29 udp port syslog unreachable, length 60
IP 100.96.6.1.38080 > 100.96.6.29.syslog: [|syslog]
IP 100.96.6.29 > 100.96.6.1: ICMP 100.96.6.29 udp port syslog unreachable, length 60
IP 100.96.6.1.60517 > 100.96.6.29.syslog: [|syslog]
IP 100.96.6.29 > 100.96.6.1: ICMP 100.96.6.29 udp port syslog unreachable, length 60
IP 100.96.6.1.40651 > 100.96.6.29.syslog: [|syslog]
IP 100.96.6.29 > 100.96.6.1: ICMP 100.96.6.29 udp port syslog unreachable, length 60
as you can see it randomly returns port 514 unreachable without a reason. There is nothing in host’s syslog.
Any idea?
Update:
Docker version is 1.12.6, build 78d1802 (latest available for kubernetes)
I’ve attached docker inspect for container
[
{
"Id": "e2aca1ca2a37e5597398e22090555c2c29a5bafd329f5606713baa110f644d3c",
"Created": "2017-08-22T21:14:24.044482116Z",
"Path": "/tmp/run.sh",
"Args": [],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 16156,
"ExitCode": 0,
"Error": "",
"StartedAt": "2017-08-22T21:14:24.175314428Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:aec3cdaf0112a16b2ba7bf4bcdb5edec15a6af82f7c04ab5c5039547f18f1f5e",
"ResolvConfPath": "/var/lib/docker/containers/93fa707b378b82094b2052585aaacd70f2b5763dece430225d7858ecacd86d14/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/93fa707b378b82094b2052585aaacd70f2b5763dece430225d7858ecacd86d14/hostname",
"HostsPath": "/var/lib/kubelet/pods/de4f8a7f-877e-11e7-a6d8-0acffb3485de/etc-hosts",
"LogPath": "/var/lib/docker/containers/e2aca1ca2a37e5597398e22090555c2c29a5bafd329f5606713baa110f644d3c/e2aca1ca2a37e5597398e22090555c2c29a5bafd329f5606713baa110f644d3c-json.log",
"Name": "/k8s_loggly_loggly-2860579233-5p126_goalshouter-test_de4f8a7f-877e-11e7-a6d8-0acffb3485de_0",
"RestartCount": 0,
"Driver": "overlay",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": [
"d9b0b4f0cc26475a0de7de5a3111dd3bf99ee7df79e7284ffeecedb18522eb66",
"7ebf1eaf7e598a14d88e35fdcea3994f9f1622272e1dd955f73b0d16320efc3b"
],
"HostConfig": {
"Binds": [
"/var/lib/kubelet/pods/de4f8a7f-877e-11e7-a6d8-0acffb3485de/volumes/kubernetes.io~secret/default-token-l9ndt:/var/run/secrets/kubernetes.io/serviceaccount:ro",
"/var/lib/kubelet/pods/de4f8a7f-877e-11e7-a6d8-0acffb3485de/etc-hosts:/etc/hosts",
"/var/lib/kubelet/pods/de4f8a7f-877e-11e7-a6d8-0acffb3485de/containers/loggly/9e6c89e2:/dev/termination-log"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {
"max-file": "5",
"max-size": "10m"
}
},
"NetworkMode": "container:93fa707b378b82094b2052585aaacd70f2b5763dece430225d7858ecacd86d14",
"PortBindings": null,
"RestartPolicy": {
"Name": "",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"Dns": null,
"DnsOptions": null,
"DnsSearch": null,
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "container:93fa707b378b82094b2052585aaacd70f2b5763dece430225d7858ecacd86d14",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 994,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": [
"seccomp=unconfined"
],
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 2,
"Memory": 52428800,
"CgroupParent": "/kubepods/burstable/podde4f8a7f-877e-11e7-a6d8-0acffb3485de",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DiskQuota": 0,
"KernelMemory": 0,
"MemoryReservation": 0,
"MemorySwap": -1,
"MemorySwappiness": -1,
"OomKillDisable": false,
"PidsLimit": 0,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0
},
"GraphDriver": {
"Name": "overlay",
"Data": {
"LowerDir": "/var/lib/docker/overlay/eca00431ae79c222147d0e020665e36dea1f08f40f6e120a8a66c2645313f3d7/root",
"MergedDir": "/var/lib/docker/overlay/84a5d5e0a525423813eb871ba3bae2e87cfd78c27e5e67cf23bbb60c7ea52d61/merged",
"UpperDir": "/var/lib/docker/overlay/84a5d5e0a525423813eb871ba3bae2e87cfd78c27e5e67cf23bbb60c7ea52d61/upper",
"WorkDir": "/var/lib/docker/overlay/84a5d5e0a525423813eb871ba3bae2e87cfd78c27e5e67cf23bbb60c7ea52d61/work"
}
},
"Mounts": [
{
"Source": "/var/lib/kubelet/pods/de4f8a7f-877e-11e7-a6d8-0acffb3485de/volumes/kubernetes.io~secret/default-token-l9ndt",
"Destination": "/var/run/secrets/kubernetes.io/serviceaccount",
"Mode": "ro",
"RW": false,
"Propagation": "rprivate"
},
{
"Source": "/var/lib/kubelet/pods/de4f8a7f-877e-11e7-a6d8-0acffb3485de/etc-hosts",
"Destination": "/etc/hosts",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
},
{
"Source": "/var/lib/kubelet/pods/de4f8a7f-877e-11e7-a6d8-0acffb3485de/containers/loggly/9e6c89e2",
"Destination": "/dev/termination-log",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
}
],
"Config": {
"Hostname": "loggly-2860579233-5p126",
"Domainname": "",
"User": "0",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"514/tcp": {},
"514/udp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
....redacted....
],
"Cmd": [
"/tmp/run.sh"
],
"Image": "sendgridlabs/loggly-docker@sha256:fcdc482a079dc98a0b78e8e75340f89ec9710dec95ad29452f46433d2a2119c7",
"Volumes": null,
"WorkingDir": "",
"Entrypoint": null,
"OnBuild": null,
"Labels": {
"annotation.io.kubernetes.container.hash": "25ff262e",
"annotation.io.kubernetes.container.ports": "[{\"containerPort\":514,\"protocol\":\"UDP\"}]",
"annotation.io.kubernetes.container.restartCount": "0",
"annotation.io.kubernetes.container.terminationMessagePath": "/dev/termination-log",
"annotation.io.kubernetes.container.terminationMessagePolicy": "File",
"annotation.io.kubernetes.pod.terminationGracePeriod": "30",
"io.kubernetes.container.logpath": "/var/log/pods/de4f8a7f-877e-11e7-a6d8-0acffb3485de/loggly_0.log",
"io.kubernetes.container.name": "loggly",
"io.kubernetes.docker.type": "container",
"io.kubernetes.pod.name": "loggly-2860579233-5p126",
"io.kubernetes.pod.uid": "de4f8a7f-877e-11e7-a6d8-0acffb3485de",
"io.kubernetes.sandbox.id": "93fa707b378b82094b2052585aaacd70f2b5763dece430225d7858ecacd86d14"
}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": null,
"SandboxKey": "",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": null
}
}
]