Hi guys!
I have a network related issues. I have a container running gunicorn app, with port 8000 published.
I’m use echo 'exit' | telnet 172.31.3.10 8000 to connect to this port, from my laptop.
Docker host has ip 172.31.3.10, my laptop is 172.31.3.246, both connected by wire to that LAN.
What I see is a pretty big time for telnet to establish connection, sometimes, but sometimes not.
I’ve looked a little deeper and wireshark showed me SYN packets re-transmitted in such cases.
I have dumps from laptop and host, for both cases: with and without SYN re-transmission, here is one of it:
- docker host, SYN re-transmission
OS version
$ uname -a
Linux remy-base-1 4.15.0-58-generic #64~16.04.1-Ubuntu SMP Wed Aug 7 14:10:35 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux
$ cat /etc/*elease
DISTRIB_ID=Ubuntu
DISTRIB_RELEASE=16.04
DISTRIB_CODENAME=xenial
DISTRIB_DESCRIPTION="Ubuntu 16.04.6 LTS"
NAME="Ubuntu"
VERSION="16.04.6 LTS (Xenial Xerus)"
ID=ubuntu
ID_LIKE=debian
PRETTY_NAME="Ubuntu 16.04.6 LTS"
VERSION_ID="16.04"
HOME_URL="http://www.ubuntu.com/"
SUPPORT_URL="http://help.ubuntu.com/"
BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/"
VERSION_CODENAME=xenial
UBUNTU_CODENAME=xenial
$ sudo ufw status
Status: inactive
Docker version
$ docker version
Client:
Version: 18.09.5
API version: 1.39
Go version: go1.10.8
Git commit: e8ff056
Built: Thu Apr 11 04:44:24 2019
OS/Arch: linux/amd64
Experimental: false
Server: Docker Engine - Community
Engine:
Version: 18.09.5
API version: 1.39 (minimum version 1.12)
Go version: go1.10.8
Git commit: e8ff056
Built: Thu Apr 11 04:10:53 2019
OS/Arch: linux/amd64
Experimental: false
$ docker system info
Containers: 5
Running: 5
Paused: 0
Stopped: 0
Images: 145
Server Version: 18.09.5
Storage Driver: overlay2
Backing Filesystem: extfs
Supports d_type: true
Native Overlay Diff: true
Logging Driver: json-file
Cgroup Driver: cgroupfs
Plugins:
Volume: local
Network: bridge host macvlan null overlay
Log: awslogs fluentd gcplogs gelf journald json-file local logentries splunk syslog
Swarm: inactive
Runtimes: nvidia runc
Default Runtime: runc
Init Binary: docker-init
containerd version: bb71b10fd8f58240ca47fbb579b9d1028eea7c84
runc version: 2b18fe1d885ee5083ef9f0838fee39b62d653e30
init version: fec3683
Security Options:
apparmor
seccomp
Profile: default
Kernel Version: 4.15.0-58-generic
Operating System: Ubuntu 16.04.6 LTS
OSType: linux
Architecture: x86_64
CPUs: 12
Total Memory: 15.58GiB
Name: remy-base-1
ID: YC2W:NOUT:FD7R:4E54:CS2P:Y6JE:EI35:BZDA:BRL6:PXPJ:E43O:YOQS
Docker Root Dir: /var/lib/docker
Debug Mode (client): false
Debug Mode (server): false
Registry: https://index.docker.io/v1/
Labels:
Experimental: false
Insecure Registries:
127.0.0.0/8
Live Restore Enabled: false
Product License: Community Engine
WARNING: No swap limit support
My container
$ docker inspect controllers_service
[
{
"Id": "d1bd7eb9537b6c534dd384b33ff33d072687fdb6b11821dfc5fc7f6f4c588355",
"Created": "2019-08-14T15:21:25.77887618Z",
"Path": "gunicorn",
"Args": [
"-c",
"config.py",
"controllers:app"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 3726,
"ExitCode": 0,
"Error": "",
"StartedAt": "2019-09-04T07:46:50.304673906Z",
"FinishedAt": "2019-09-04T10:46:48.1642967+03:00"
},
"Image": "sha256:954a51ab10b535b797ce27a04dd1963170db4f6a44f26f0d1869865b0669ee9d",
"ResolvConfPath": "/var/lib/docker/containers/d1bd7eb9537b6c534dd384b33ff33d072687fdb6b11821dfc5fc7f6f4c588355/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/d1bd7eb9537b6c534dd384b33ff33d072687fdb6b11821dfc5fc7f6f4c588355/hostname",
"HostsPath": "/var/lib/docker/containers/d1bd7eb9537b6c534dd384b33ff33d072687fdb6b11821dfc5fc7f6f4c588355/hosts",
"LogPath": "/var/lib/docker/containers/d1bd7eb9537b6c534dd384b33ff33d072687fdb6b11821dfc5fc7f6f4c588355/d1bd7eb9537b6c534dd384b33ff33d072687fdb6b11821dfc5fc7f6f4c588355-json.log",
"Name": "/controllers_service",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "unconfined",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"/dev:/dev"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "bridge",
"PortBindings": {
"8000/tcp": [
{
"HostIp": "",
"HostPort": "8000"
}
]
},
"RestartPolicy": {
"Name": "unless-stopped",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": [
"AUDIT_WRITE",
"CHOWN",
"DAC_OVERRIDE",
"FOWNER",
"FSETID",
"KILL",
"MKNOD",
"NET_BIND_SERVICE",
"NET_RAW",
"SETFCAP",
"SETGID",
"SETPCAP",
"SETUID",
"SYS_CHROOT"
],
"CapDrop": [
"AUDIT_CONTROL",
"BLOCK_SUSPEND",
"DAC_READ_SEARCH",
"IPC_LOCK",
"IPC_OWNER",
"LEASE",
"LINUX_IMMUTABLE",
"MAC_ADMIN",
"MAC_OVERRIDE",
"NET_ADMIN",
"NET_BROADCAST",
"SYSLOG",
"SYS_ADMIN",
"SYS_BOOT",
"SYS_MODULE",
"SYS_NICE",
"SYS_PACCT",
"SYS_PTRACE",
"SYS_RAWIO",
"SYS_RESOURCE",
"SYS_TIME",
"SYS_TTY_CONFIG",
"WAKE_ALARM"
],
"Dns": [],
"DnsOptions": [],
"DnsSearch": [],
"ExtraHosts": [],
"GroupAdd": null,
"IpcMode": "shareable",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": true,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": [
"label=disable"
],
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 0,
"Memory": 1073741824,
"NanoCpus": 3000000000,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DeviceCgroupRules": null,
"DiskQuota": 0,
"KernelMemory": 0,
"MemoryReservation": 536870912,
"MemorySwap": -1,
"MemorySwappiness": null,
"OomKillDisable": false,
"PidsLimit": 0,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": null,
"ReadonlyPaths": null
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/2d61fa4bdcd60acebbaff47dbe60ed2013226b15bbcd779b5cabcb20b1c6ee89-init/diff:/var/lib/docker/overlay2/8414fa036dc668a031f32eabd00453ca2ee6582042dab3c71fd412438086db26/diff:/var/lib/docker/overlay2/c6f56b6a1d8b6eb2951278989c275c5a112c4c9cb85c95a4cf4a51cab723e3f9/diff:/var/lib/docker/overlay2/5f6307eac7e35d0e23b023999df51dd7d827b02b6ef258621c694da0d1eacd8e/diff:/var/lib/docker/overlay2/2dca0e566480a1d498941d28d8d2dcf1f02d7df069e5276bd19a4348d9dfdd27/diff:/var/lib/docker/overlay2/c09e3fcb3199d7a6dea9614f4be958afe4e238e93d79a693bc45140f89a23e7b/diff:/var/lib/docker/overlay2/cdd192d528150e6f2d9879ce3247f08f839680b375ed1319dab2c7d7ac1268e8/diff:/var/lib/docker/overlay2/934e930253feeb6bbc4af2980b568ef2898eee7217ee05c43a4de3df34799171/diff",
"MergedDir": "/var/lib/docker/overlay2/2d61fa4bdcd60acebbaff47dbe60ed2013226b15bbcd779b5cabcb20b1c6ee89/merged",
"UpperDir": "/var/lib/docker/overlay2/2d61fa4bdcd60acebbaff47dbe60ed2013226b15bbcd779b5cabcb20b1c6ee89/diff",
"WorkDir": "/var/lib/docker/overlay2/2d61fa4bdcd60acebbaff47dbe60ed2013226b15bbcd779b5cabcb20b1c6ee89/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "bind",
"Source": "/dev",
"Destination": "/dev",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
}
],
"Config": {
"Hostname": "1dc4badbefe3",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"8000/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"PYTHONPATH=/"
],
"Cmd": [],
"ArgsEscaped": true,
"Image": "controllers_service:0.2.0",
"Volumes": {
"/dev": {}
},
"WorkingDir": "/",
"Entrypoint": [
"gunicorn",
"-c",
"config.py",
"dispensers:app"
],
"MacAddress": "02:42:ac:11:00:02",
"OnBuild": null,
"Labels": {}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "51e4c73e9e8586b36007acf4e7dc68f369191f4e5f8c838b47cbb0774ef9726a",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {
"8000/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "8000"
}
]
},
"SandboxKey": "/var/run/docker/netns/51e4c73e9e85",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "d23ea731f678708dc67b23754a19d8a6a791995e94c254f69e142c5001f0af07",
"Gateway": "172.17.0.1",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "172.17.0.3",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"MacAddress": "02:42:ac:11:00:02",
"Networks": {
"bridge": {
"IPAMConfig": {},
"Links": null,
"Aliases": null,
"NetworkID": "7d7e76a89e2fae4ba1cec2118119c1c803cbf5a557f5a780c8c7a87084716861",
"EndpointID": "d23ea731f678708dc67b23754a19d8a6a791995e94c254f69e142c5001f0af07",
"Gateway": "172.17.0.1",
"IPAddress": "172.17.0.3",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "02:42:ac:11:00:02",
"DriverOpts": null
}
}
}
}
]
I’ve tried to search similar issues, but found only two:
- https://tech.xing.com/a-reason-for-unexplained-connection-timeouts-on-kubernetes-docker-abd041cf7e02
- https://tech.vijayp.ca/linux-kernel-bug-delivers-corrupt-tcp-ip-data-to-mesos-kubernetes-docker-containers-4986f88f7a19
but not sure, is it relevant to my case.
Please, help me with investigating and fixing it
Thanks a lot!