I have an application that uses short-lived containers (for automated grading of student work - unit testing of very simple programs in Python and C++). Typically a job should last < 10 s. This is running on a linux server with 12C/24T CPU and 32GB memory. When the number of created containers is still relatively small (~20-30), it seems like containers are being created, but are slow to start - 30s or more - at least that seems to be what docker ps -a is reporting to me (output below). Only a handful of containers appear to be started at any time - much fewer than the number of CPUs. I’m wondering why - the server is definitely not running out of CPU, tons of memory free, iostat is not reporting excessive disk usage. The slow start means students are waiting too long to get their results.
Thank you for any insights, suggestions, etc.
OS: Ubuntu 20.04 Linux 5.4.0-81-generic #91-Ubuntu SMP Thu Jul 15 19:09:17 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux
Docker version: 20.10.17
Image is 206MB, based on python:3.8-slim
Output of docker ps -a:
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
927a3f7e9c50 d0a8b718a8a4 "/run.sh" 6 seconds ago Created elastic_ptolemy
440ff35c0b19 d0a8b718a8a4 "/run.sh" 8 seconds ago Created gracious_swartz
e40a7944c224 d0a8b718a8a4 "/run.sh" 9 seconds ago Created compassionate_curie
948dff1e1551 d0a8b718a8a4 "/run.sh" 10 seconds ago Created funny_antonelli
0eaa0a0b7f44 d0a8b718a8a4 "/run.sh" 12 seconds ago Created gallant_margulis
a03730880023 d0a8b718a8a4 "/run.sh" 14 seconds ago Created distracted_wilson
ec39006de4d4 d0a8b718a8a4 "/run.sh" 14 seconds ago Created nice_meitner
0c2a5ca2ca8c d0a8b718a8a4 "/run.sh" 14 seconds ago Created goofy_dubinsky
d028cbb36aa9 d0a8b718a8a4 "/run.sh" 15 seconds ago Created crazy_ellis
421f3c7c7cf2 d0a8b718a8a4 "/run.sh" 15 seconds ago Created zealous_kilby
ced484139e6d 630e9680e389 "/run.sh" 16 seconds ago Created practical_wescoff
1b4f6a4c7119 d0a8b718a8a4 "/run.sh" 18 seconds ago Created distracted_almeida
25506f710f55 d0a8b718a8a4 "/run.sh" 18 seconds ago Created suspicious_jones
506ada18b569 d0a8b718a8a4 "/run.sh" 22 seconds ago Created nice_cray
2749d57a1333 d0a8b718a8a4 "/run.sh" 23 seconds ago Created charming_kare
286107e099b4 d0a8b718a8a4 "/run.sh" 25 seconds ago Created vigorous_blackwell
c943746e33e1 d0a8b718a8a4 "/run.sh" 27 seconds ago Created wizardly_almeida
42851177e0d8 d0a8b718a8a4 "/run.sh" 30 seconds ago Created stoic_hamilton
17ff4cc1915f d0a8b718a8a4 "/run.sh" 40 seconds ago Created epic_bardeen
a5ff4b1a9925 d0a8b718a8a4 "/run.sh" 43 seconds ago Up 11 seconds admiring_ramanujan
8743a5585f66 d0a8b718a8a4 "/run.sh" 47 seconds ago Up 13 seconds funny_dijkstra
0f8e8abd4a9f d0a8b718a8a4 "/run.sh" 50 seconds ago Up Less than a second sharp_ishizaka
43aec90c52d8 d0a8b718a8a4 "/run.sh" 50 seconds ago Up 4 seconds loving_mendeleev
b6d7c575bdc1 d0a8b718a8a4 "/run.sh" 50 seconds ago Up 2 seconds elegant_herschel
36e2d099aca3 d0a8b718a8a4 "/run.sh" 55 seconds ago Up 4 seconds mystifying_cohen
30167fa7a960 d0a8b718a8a4 "/run.sh" 55 seconds ago Up 9 seconds awesome_chaplygin
6dae9414f4bf d0a8b718a8a4 "/run.sh" 55 seconds ago Up 9 seconds boring_mendel
60cb3992d828 d0a8b718a8a4 "/run.sh" 56 seconds ago Up 9 seconds crazy_kare
Docker container inspect of one container:
[
{
"Id": "927a3f7e9c502b2bdd7cc3d3a0696a707ca08e306fd3c390f020f79786830dc5",
"Created": "2021-08-26T16:45:39.13730698Z",
"Path": "/run.sh",
"Args": [],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 1055078,
"ExitCode": 0,
"Error": "",
"StartedAt": "2021-08-26T16:46:19.614199339Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:d0a8b718a8a42104952cbfc22e2a3d41f9023edc052aab7dca2d19e5a10b742b",
"ResolvConfPath": "",
"HostnamePath": "/var/lib/docker/containers/927a3f7e9c502b2bdd7cc3d3a0696a707ca08e306fd3c390f020f79786830dc5/hostname",
"HostsPath": "",
"LogPath": "/var/lib/docker/containers/927a3f7e9c502b2bdd7cc3d3a0696a707ca08e306fd3c390f020f79786830dc5/927a3f7e9c502b2bdd7cc3d3a0696a707ca08e306fd3c390f020f79786830dc5-json.log",
"Name": "/elastic_ptolemy",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "docker-default",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"/home/autograder/autograder/working/autograder20210826-946195-12lxins-10_45_39:/tests/"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "default",
"PortBindings": null,
"RestartPolicy": {
"Name": "",
"MaximumRetryCount": 0
},
"AutoRemove": true,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"CgroupnsMode": "host",
"Dns": null,
"DnsOptions": null,
"DnsSearch": null,
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "private",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": null,
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 0,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": null,
"DeviceCgroupRules": null,
"DeviceRequests": null,
"KernelMemory": 0,
"KernelMemoryTCP": 0,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": null,
"OomKillDisable": false,
"PidsLimit": null,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": [
"/proc/asound",
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware"
],
"ReadonlyPaths": [
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger"
]
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/4dc81d4b18cfe5a4dc706a2c451ecf4733fe0d4845ec927e193d16b5d0782e52-init/diff:/var/lib/docker/overlay2/d10954fa0ef91936e6b769c6dcbe760ad0a5f2512812de3bd77d6cf66a6874e1/diff:/var/lib/docker/overlay2/7161fe677e6d769cddfd6ac0e240aa99ba6d1aabcc476b6d04750b360685ac0c/diff:/var/lib/docker/overlay2/f7a952ac97e464ab70cec81b08c1185f4790b44faad810d98e49fdee7c4c7db1/diff:/var/lib/docker/overlay2/df8dba83b475e85536ccb9500948ddf1fd2041780239a02cb55d16a8f860d3e0/diff:/var/lib/docker/overlay2/e827e77ea4843b3371ca86748b932041ead0a6d04afec3a992c30b0b9e0ecbac/diff:/var/lib/docker/overlay2/456c8386e123787d9779e467a52e2b1b591175fd507e0379c4e0f29321d43315/diff:/var/lib/docker/overlay2/92a87522a73b6ddec507e58a87170c89687fb066ee3148dc125b6e9f4e9090cf/diff:/var/lib/docker/overlay2/3569f62c1d947960af20590b1c6963001ef4ef5abf04644a4a51012f024333bc/diff",
"MergedDir": "/var/lib/docker/overlay2/4dc81d4b18cfe5a4dc706a2c451ecf4733fe0d4845ec927e193d16b5d0782e52/merged",
"UpperDir": "/var/lib/docker/overlay2/4dc81d4b18cfe5a4dc706a2c451ecf4733fe0d4845ec927e193d16b5d0782e52/diff",
"WorkDir": "/var/lib/docker/overlay2/4dc81d4b18cfe5a4dc706a2c451ecf4733fe0d4845ec927e193d16b5d0782e52/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "bind",
"Source": "/home/autograder/autograder/working/autograder20210826-946195-12lxins-10_45_39",
"Destination": "/tests",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
}
],
"Config": {
"Hostname": "927a3f7e9c50",
"Domainname": "",
"User": "1001",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"test_num=5",
"timeout=1.0",
"PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"LANG=C.UTF-8",
"GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568",
"PYTHON_VERSION=3.8.11",
"PYTHON_PIP_VERSION=21.2.4",
"PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/c20b0cfd643cd4a19246ccf204e2997af70f6b21/public/get-pip.py",
"PYTHON_GET_PIP_SHA256=fa6f3fb93cce234cd4e8dd2beb54a51ab9c247653b52855a48dd44e6b21ff28b"
],
"Cmd": [
"/run.sh"
],
"Image": "d0a8b718a8a4",
"Volumes": null,
"WorkingDir": "/tests",
"Entrypoint": null,
"NetworkDisabled": true,
"OnBuild": null,
"Labels": {}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {},
"SandboxKey": "",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {}
}
}
]