Docker Community Forums

Share and learn in the Docker community.

Docker Process Exited with Code 137, but not due to OEM


(Jcald1) #1

I’m executing the following subprocess.

    print(f'runner executing the simulation for {simulation_file_url} ')
    process = Popen(['gridlabd', simulation_file_url], cwd=temp_dir,
                    stdout=PIPE, stderr=PIPE)

The main app periodically dies with code 137. I’m running this on Ubuntu 17.10 in Docker. I inspected the Docker container and I didn’t see anything like an out of memory issue. The subprocess is a C++ program installed locally and is consuming up to 800M of memory. I have about 10G available in my VirtualBox VM where this Docker container is running. So it doesn’t appear to be a memory issue on my host, unless there’s some memory setting in Docker I need to change.

web_1  | runner executing the simulation for /code/components/g_sim_runner/test/simulation_models/i/8 
g-server_web_1 exited with code 137

docker inspect 8eeb7235b12d

[
    {
        "Id": "8eeb7235b12d84b85b9a7b8f90e56e204ceceaf31638451df83fd0f537477dde",
        "Created": "2018-10-12T20:17:57.038522287Z",
        "Path": "./start.sh",
        "Args": [],
        "State": {
            "Status": "exited",
            "Running": false,
            "Paused": false,
            "Restarting": false,
            "OOMKilled": false,
            "Dead": false,
            "Pid": 0,
            "ExitCode": 137,
            "Error": "",
            "StartedAt": "2018-10-12T21:16:08.144554771Z",
            "FinishedAt": "2018-10-12T21:16:58.272396915Z"
        },
        "Image": "sha256:8271cae1b4b9ecf54d98913d56903d40befd9ae29f1940c8dc6721a518d8c855",
        "ResolvConfPath": "/var/lib/docker/containers/8eeb7235b12d84b85b9a7b8f90e56e204ceceaf31638451df83fd0f537477dde/resolv.conf",
        "HostnamePath": "/var/lib/docker/containers/8eeb7235b12d84b85b9a7b8f90e56e204ceceaf31638451df83fd0f537477dde/hostname",
        "HostsPath": "/var/lib/docker/containers/8eeb7235b12d84b85b9a7b8f90e56e204ceceaf31638451df83fd0f537477dde/hosts",
        "LogPath": "/var/lib/docker/containers/8eeb7235b12d84b85b9a7b8f90e56e204ceceaf31638451df83fd0f537477dde/8eeb7235b12d84b85b9a7b8f90e56e204ceceaf31638451df83fd0f537477dde-json.log",
        "Name": "/g-server_web_1",
        "RestartCount": 0,
        "Driver": "overlay2",
        "Platform": "linux",
        "MountLabel": "",
        "ProcessLabel": "",
        "AppArmorProfile": "docker-default",
        "ExecIDs": null,
        "HostConfig": {
            "Binds": [
                "/home/jorge/projects/g/g-server:/code:rw"
            ],
            "ContainerIDFile": "",
            "LogConfig": {
                "Type": "json-file",
                "Config": {}
            },
            "NetworkMode": "g-server_default",
            "PortBindings": {
                "5000/tcp": [
                    {
                        "HostIp": "",
                        "HostPort": "5000"
                    }
                ],
                "5432/tcp": [
                    {
                        "HostIp": "",
                        "HostPort": "5432"
                    }
                ]
            },
            "RestartPolicy": {
                "Name": "",
                "MaximumRetryCount": 0
            },
            "AutoRemove": false,
            "VolumeDriver": "",
            "VolumesFrom": [],
            "CapAdd": null,
            "CapDrop": null,
            "Dns": null,
            "DnsOptions": null,
            "DnsSearch": null,
            "ExtraHosts": null,
            "GroupAdd": null,
            "IpcMode": "shareable",
            "Cgroup": "",
            "Links": null,
            "OomScoreAdj": 0,
            "PidMode": "",
            "Privileged": false,
            "PublishAllPorts": false,
            "ReadonlyRootfs": false,
            "SecurityOpt": null,
            "UTSMode": "",
            "UsernsMode": "",
            "ShmSize": 67108864,
            "Runtime": "runc",
            "ConsoleSize": [
                0,
                0
            ],
            "Isolation": "",
            "CpuShares": 0,
            "Memory": 0,
            "NanoCpus": 0,
            "CgroupParent": "",
            "BlkioWeight": 0,
            "BlkioWeightDevice": null,
            "BlkioDeviceReadBps": null,
            "BlkioDeviceWriteBps": null,
            "BlkioDeviceReadIOps": null,
            "BlkioDeviceWriteIOps": null,
            "CpuPeriod": 0,
            "CpuQuota": 0,
            "CpuRealtimePeriod": 0,
            "CpuRealtimeRuntime": 0,
            "CpusetCpus": "",
            "CpusetMems": "",
            "Devices": null,
            "DeviceCgroupRules": null,
            "DiskQuota": 0,
            "KernelMemory": 0,
            "MemoryReservation": 0,
            "MemorySwap": 0,
            "MemorySwappiness": null,
            "OomKillDisable": false,
            "PidsLimit": 0,
            "Ulimits": null,
            "CpuCount": 0,
            "CpuPercent": 0,
            "IOMaximumIOps": 0,
            "IOMaximumBandwidth": 0,
            "MaskedPaths": [
                "/proc/acpi",
                "/proc/kcore",
                "/proc/keys",
                "/proc/latency_stats",
                "/proc/timer_list",
                "/proc/timer_stats",
                "/proc/sched_debug",
                "/proc/scsi",
                "/sys/firmware"
            ],
            "ReadonlyPaths": [
                "/proc/asound",
                "/proc/bus",
                "/proc/fs",
                "/proc/irq",
                "/proc/sys",
                "/proc/sysrq-trigger"
            ]
        },
        "GraphDriver": {
            "Data": {
                "LowerDir": "/var/lib/docker/overlay2/358589d9656a048a81b98b2adeba4b234cadf0059e99c556b9ad7b19af57c682-init/diff:/var/lib/docker/overlay2/ab885975de023327d90fdbb400df3d86315b337cd0278a05653f0041c50aa44e/diff:/var/lib/docker/overlay2/2c2d41377887c338b4f96e6d297f3d46f64c5f2fc7875e7da3487dbddcac4c36/diff:/var/lib/docker/overlay2/268d8b51611d020aaeb0b6d9031d20a4b1e253689b05ec268c27e0b0954670a2/diff:/var/lib/docker/overlay2/fcf703d53e20141f797a5291390f6b73cd92fd93d326a31ec7d4a4f0df757539/diff:/var/lib/docker/overlay2/e91de505943b0fccb7eebc275280d031faa252aa72002e3e42303cbd6693487a/diff:/var/lib/docker/overlay2/5804a5985a4bccd7c4e353a4a6c41644f4a64c74ee1ace858f4061834f1337c7/diff:/var/lib/docker/overlay2/ef14b1f95c409965b029fbab38e645ffe06cfa5e0d2c139745e93afa79ba4592/diff:/var/lib/docker/overlay2/a781784fdb0ab751bd23b5f65afebb01ad41fba2d7237bbbe5bae51883458adb/diff:/var/lib/docker/overlay2/197b25c1fd9081ed2617b293e75ecf98a53964efd13673a7365abd852e2f8348/diff:/var/lib/docker/overlay2/aedfffc33d88b63f5b4d17d7293628ffc78bf65b76c926947e72d5a405b03672/diff:/var/lib/docker/overlay2/867e84ffb7eac77a9797ca7eae91708b43b84de50a2d314f05480f96994f9874/diff:/var/lib/docker/overlay2/8e0902348ae523ce672e3831729144f6a9161d470a782a4f391fd44dbd5a15dd/diff:/var/lib/docker/overlay2/22d4637ecfaccdb482fe3274f5bd67617fa9e6651787ba63ca359e19625633f2/diff:/var/lib/docker/overlay2/1b7873370612180526eae10d431ea3c89a2654e15e3d770207b28ec2122db034/diff:/var/lib/docker/overlay2/94d7b0908aece6dbab4b4d69b072b48547a926050531061ec64652b8130089be/diff:/var/lib/docker/overlay2/d944746aa2ac3863214c5ed04a194f4abe6818352c90553c8ee3bf37f4a7bd79/diff:/var/lib/docker/overlay2/f35cab877637a359c793d52c43cd57bc5fd31f10268e68409c785513c0244d50/diff:/var/lib/docker/overlay2/86f005796210c70c4f933d2969116d3540ddc5202eb7d399895906aabd4f936b/diff:/var/lib/docker/overlay2/abf882ac5c8992bf3c5bc56991fc79c60cfbfa031bdab4e8d027a8c760b19c0e/diff:/var/lib/docker/overlay2/c7cba72ce1dd8e2ec45d7b20f78373e308836fad12e128a63b8eef188f1c1ffb/diff:/var/lib/docker/overlay2/cda8d88b1c063e83faa6722cfe923852327b00157bbe489e140f7eeeb759ed8b/diff:/var/lib/docker/overlay2/3561cefaf04a50b8fbc0660b851650622782b1d95e0d172a9d7bcad143d4b18e/diff:/var/lib/docker/overlay2/374fb975e13a3a514bd694c126265c19d4fff4ba2a66b303da9fcedc52122d42/diff",
                "MergedDir": "/var/lib/docker/overlay2/358589d9656a048a81b98b2adeba4b234cadf0059e99c556b9ad7b19af57c682/merged",
                "UpperDir": "/var/lib/docker/overlay2/358589d9656a048a81b98b2adeba4b234cadf0059e99c556b9ad7b19af57c682/diff",
                "WorkDir": "/var/lib/docker/overlay2/358589d9656a048a81b98b2adeba4b234cadf0059e99c556b9ad7b19af57c682/work"
            },
            "Name": "overlay2"
        },
        "Mounts": [
            {
                "Type": "bind",
                "Source": "/home/jorge/projects/g/g-server",
                "Destination": "/code",
                "Mode": "rw",
                "RW": true,
                "Propagation": "rprivate"
            }
        ],
        "Config": {
            "Hostname": "8eeb7235b12d",
            "Domainname": "",
            "User": "",
            "AttachStdin": false,
            "AttachStdout": false,
            "AttachStderr": false,
            "ExposedPorts": {
                "5000/tcp": {},
                "5432/tcp": {}
            },
            "Tty": false,
            "OpenStdin": false,
            "StdinOnce": false,
            "Env": [
                "PORT=5000",
                "PYTHONUNBUFFERED=1",
                "PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
                "LANG=C.UTF-8",
                "GPG_KEY=0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D",
                "PYTHON_VERSION=3.7.0",
                "PYTHON_PIP_VERSION=18.1"
            ],
            "Cmd": [
                "./start.sh"
            ],
            "ArgsEscaped": true,
            "Image": "g-server_web",
            "Volumes": {
                "/code": {}
            },
            "WorkingDir": "/code",
            "Entrypoint": null,
            "OnBuild": null,
            "Labels": {
                "com.docker.compose.config-hash": "828f89f5804dd723836da5e1b9172b72bb282ab266272acfefbad89d6a52b610",
                "com.docker.compose.container-number": "1",
                "com.docker.compose.oneoff": "False",
                "com.docker.compose.project": "g-server",
                "com.docker.compose.service": "web",
                "com.docker.compose.version": "1.22.0"
            }
        },
        "NetworkSettings": {
            "Bridge": "",
            "SandboxID": "e54f428ae9e4991662dd96de555e28e45c3b3f276f06512fbaf9952c5555c0bf",
            "HairpinMode": false,
            "LinkLocalIPv6Address": "",
            "LinkLocalIPv6PrefixLen": 0,
            "Ports": {},
            "SandboxKey": "/var/run/docker/netns/e54f428ae9e4",
            "SecondaryIPAddresses": null,
            "SecondaryIPv6Addresses": null,
            "EndpointID": "",
            "Gateway": "",
            "GlobalIPv6Address": "",
            "GlobalIPv6PrefixLen": 0,
            "IPAddress": "",
            "IPPrefixLen": 0,
            "IPv6Gateway": "",
            "MacAddress": "",
            "Networks": {
                "g-server_default": {
                    "IPAMConfig": null,
                    "Links": null,
                    "Aliases": [
                        "8eeb7235b12d",
                        "web"
                    ],
                    "NetworkID": "e2b78a8e1bf9e40de0ae75fde8640df9d26f46262e38e1b973342ae51036d899",
                    "EndpointID": "",
                    "Gateway": "",
                    "IPAddress": "",
                    "IPPrefixLen": 0,
                    "IPv6Gateway": "",
                    "GlobalIPv6Address": "",
                    "GlobalIPv6PrefixLen": 0,
                    "MacAddress": "",
                    "DriverOpts": null
                }
            }
        }
    }
]