Docker space issue

Hi,

We are facing some issue for our docker engine(version 18.09.1) running on Ubuntu, sometimes it pop-up with the No space left on device error although we have lot of spaces present on the mount point wherever docker is installed and lot of resources(CPU, memory etc.) available on the host.

I’m getting the below error while I’m running any docker command. Please note, after restarting the docker engine the issue is going away.

docker ps
runtime/cgo: pthread_create failed: No space left on device
SIGABRT: abort
PC=0x7f0d28c81e97 m=14 sigcode=18446744073709551610

goroutine 0 [idle]:
runtime: unknown pc 0x7f0d28c81e97
stack: frame={sp:0x7f0d01ffa890, fp:0x0} stack=[0x7f0d017fb290,0x7f0d01ffae90)
00007f0d01ffa790: 0000000000000000 0000000000000000
00007f0d01ffa7a0: 0000000000000000 0000000000000000
00007f0d01ffa7b0: 0000000000000000 000055e3b6e55cae <runtime.netpoll+302>
00007f0d01ffa7c0: 000055e300000004 00007f0d01ffa7f8
00007f0d01ffa7d0: 0000000000000080 0000000000000000
00007f0d01ffa7e0: 0000000000000000 0000000000000000
00007f0d01ffa7f0: 0000000000000000 0000000000000000
00007f0d01ffa800: 0000000000000000 0000000000000000
00007f0d01ffa810: 0000000000000000 0000000000000000
00007f0d01ffa820: 0000000000000000 0000000000000000
00007f0d01ffa830: 0000000000000000 0000000000000000
00007f0d01ffa840: 0000000000000000 0000000000000000
00007f0d01ffa850: 0000000000000000 0000000000000000
00007f0d01ffa860: 0000000000000000 0000000000000000
00007f0d01ffa870: 0000000000000000 0000000000000000
00007f0d01ffa880: 0000000000000000 0000000000000000
00007f0d01ffa890: <0000000000000000 0000000000000000
00007f0d01ffa8a0: 0000000000000000 0000000000000000
00007f0d01ffa8b0: 0000000000000000 0000000000000000
00007f0d01ffa8c0: 0000000000000000 0000000000000000
00007f0d01ffa8d0: 0000000000000000 0000000000000000
00007f0d01ffa8e0: 0000000000000000 0000000000000000
00007f0d01ffa8f0: 0000000000000000 0000000000000000
00007f0d01ffa900: 0000000000000000 0000000000000000
00007f0d01ffa910: fffffffe7fffffff ffffffffffffffff
00007f0d01ffa920: ffffffffffffffff ffffffffffffffff
00007f0d01ffa930: ffffffffffffffff ffffffffffffffff
00007f0d01ffa940: ffffffffffffffff ffffffffffffffff
00007f0d01ffa950: ffffffffffffffff ffffffffffffffff
00007f0d01ffa960: ffffffffffffffff ffffffffffffffff
00007f0d01ffa970: ffffffffffffffff ffffffffffffffff
00007f0d01ffa980: ffffffffffffffff ffffffffffffffff
runtime: unknown pc 0x7f0d28c81e97
stack: frame={sp:0x7f0d01ffa890, fp:0x0} stack=[0x7f0d017fb290,0x7f0d01ffae90)
00007f0d01ffa790: 0000000000000000 0000000000000000
00007f0d01ffa7a0: 0000000000000000 0000000000000000
00007f0d01ffa7b0: 0000000000000000 000055e3b6e55cae <runtime.netpoll+302>
00007f0d01ffa7c0: 000055e300000004 00007f0d01ffa7f8
00007f0d01ffa7d0: 0000000000000080 0000000000000000
00007f0d01ffa7e0: 0000000000000000 0000000000000000
00007f0d01ffa7f0: 0000000000000000 0000000000000000
00007f0d01ffa800: 0000000000000000 0000000000000000
00007f0d01ffa810: 0000000000000000 0000000000000000
00007f0d01ffa820: 0000000000000000 0000000000000000
00007f0d01ffa830: 0000000000000000 0000000000000000
00007f0d01ffa840: 0000000000000000 0000000000000000
00007f0d01ffa850: 0000000000000000 0000000000000000
00007f0d01ffa860: 0000000000000000 0000000000000000
00007f0d01ffa870: 0000000000000000 0000000000000000
00007f0d01ffa880: 0000000000000000 0000000000000000
00007f0d01ffa890: <0000000000000000 0000000000000000
00007f0d01ffa8a0: 0000000000000000 0000000000000000
00007f0d01ffa8b0: 0000000000000000 0000000000000000
00007f0d01ffa8c0: 0000000000000000 0000000000000000
00007f0d01ffa8d0: 0000000000000000 0000000000000000
00007f0d01ffa8e0: 0000000000000000 0000000000000000
00007f0d01ffa8f0: 0000000000000000 0000000000000000
00007f0d01ffa900: 0000000000000000 0000000000000000
00007f0d01ffa910: fffffffe7fffffff ffffffffffffffff
00007f0d01ffa920: ffffffffffffffff ffffffffffffffff
00007f0d01ffa930: ffffffffffffffff ffffffffffffffff
00007f0d01ffa940: ffffffffffffffff ffffffffffffffff
00007f0d01ffa950: ffffffffffffffff ffffffffffffffff
00007f0d01ffa960: ffffffffffffffff ffffffffffffffff
00007f0d01ffa970: ffffffffffffffff ffffffffffffffff
00007f0d01ffa980: ffffffffffffffff ffffffffffffffff

goroutine 1 [running]:
runtime.systemstack_switch()
/usr/local/go/src/runtime/asm_amd64.s:363 fp=0xc4207796c8 sp=0xc4207796c0 pc=0x55e3b6e82a20
runtime.gcStart(0x0, 0x1, 0x0, 0x7f0d00000000)
/usr/local/go/src/runtime/mgc.go:1371 +0x343 fp=0xc4207796f8 sp=0xc4207796c8 pc=0x55e3b6e463e3
runtime.mallocgc(0x900, 0x55e3b8821460, 0x7f0d2983e101, 0xc4207797d0)
/usr/local/go/src/runtime/malloc.go:803 +0x425 fp=0xc420779798 sp=0xc4207796f8 pc=0x55e3b6e3e435
runtime.makeslice(0x55e3b8821460, 0x81c, 0x81c, 0x7f0d297902b0, 0x301, 0x1000000000003e5)
/usr/local/go/src/runtime/slice.go:61 +0x79 fp=0xc4207797c8 sp=0xc420779798 pc=0x55e3b6e6e689
bytes.makeSlice(0x81c, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:230 +0x6f fp=0xc420779808 sp=0xc4207797c8 pc=0x55e3b6f0eb4f
bytes.(*Buffer).grow(0xc4209dab38, 0x52, 0xc4209c0e78)
/usr/local/go/src/bytes/buffer.go:144 +0x153 fp=0xc420779858 sp=0xc420779808 pc=0x55e3b6f0e4f3
bytes.(*Buffer).Write(0xc4209dab38, 0xc4209803b8, 0x52, 0xab76, 0x20, 0x27, 0xc4209c0e60)
/usr/local/go/src/bytes/buffer.go:174 +0xdb fp=0xc420779888 sp=0xc420779858 pc=0x55e3b6f0e7cb
text/tabwriter.(*Writer).append(0xc4209dab00, 0xc4209803b8, 0x52, 0xab76)
/usr/local/go/src/text/tabwriter/tabwriter.go:390 +0x51 fp=0xc4207798d0 sp=0xc420779888 pc=0x55e3b73c2701
text/tabwriter.(*Writer).Write(0xc4209dab00, 0xc420980000, 0xa38c, 0xaf2e, 0x3b8, 0x0, 0x0)
/usr/local/go/src/text/tabwriter/tabwriter.go:505 +0x218 fp=0xc420779970 sp=0xc4207798d0 pc=0x55e3b73c2eb8
bytes.(*Buffer).WriteTo(0xc420270000, 0x55e3b8b27000, 0xc4209dab00, 0x20, 0x1, 0x0)
/usr/local/go/src/bytes/buffer.go:240 +0xb8 fp=0xc4207799c0 sp=0xc420779970 pc=0x55e3b6f0ec58
github.com/docker/cli/cli/command/formatter.(*Context).postFormat(0xc4200fe1e0, 0xc420784080, 0x55e3b8b24940, 0xc420048300)
/go/src/github.com/docker/cli/cli/command/formatter/formatter.go:79 +0x235 fp=0xc420779a60 sp=0xc4207799c0 pc=0x55e3b7411755
/github.com/docker/cli/cli/command/formatter.(*Context).Write(0xc4200fe1e0, 0x55e3b8b24940, 0xc420048300, 0xc4208de000, 0xc4202f6338, 0x0)
/go/src/github.com/docker/cli/cli/command/formatter/formatter.go:117 +0x18e fp=0xc420779ac0 sp=0xc420779a60 pc=0x55e3b7411b3e
github.com/docker/cli/cli/command/formatter.ContainerWrite(0x55e3b8b248a0, 0xc4202191d0, 0x55e3b8191a78, 0x57, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, …)
/go/src/github.com/docker/cli/cli/command/formatter/container.go:74 +0x59e fp=0xc420779b20 sp=0xc420779ac0 pc=0x55e3b740979e
/github.com/docker/cli/cli/command/container.runPs(0x55e3b8b65e40, 0xc420553950, 0xc4205fb170, 0x0, 0x0)
/go/src/github.com/docker/cli/cli/command/container/list.go:139 +0x273 fp=0xc420779c38 sp=0xc420779b20 pc=0x55e3b76f5483
/github.com/docker/cli/cli/command/container.NewPsCommand.func1(0xc42060f400, 0x55e3b9646be0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/docker/cli/cli/command/container/list.go:36 +0x3e fp=0xc420779c70 sp=0xc420779c38 pc=0x55e3b770ce3e
/github.com/docker/cli/vendor/github.com/spf13/cobra.(*Command).execute(0xc42060f400, 0xc42000c070, 0x0, 0x0, 0xc42060f400, 0xc42000c070)
/go/src/github.com/docker/cli/vendor/github.com/spf13/cobra/command.go:762 +0x46a fp=0xc420779d60 sp=0xc420779c70 pc=0x55e3b6fc8dba
/github.com/docker/cli/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0xc42012c780, 0xc4200c7fa0, 0x55e3b88200a0, 0xc4200c7fb0)
/go/src/github.com/docker/cli/vendor/github.com/spf13/cobra/command.go:852 +0x30c fp=0xc420779ea0 sp=0xc420779d60 pc=0x55e3b6fc97dc
/github.com/docker/cli/vendor/github.com/spf13/cobra.(*Command).Execute(0xc42012c780, 0xc42012c780, 0x55e3b8b26dc0)
/go/src/github.com/docker/cli/vendor/github.com/spf13/cobra/command.go:800 +0x2d fp=0xc420779ed0 sp=0xc420779ea0 pc=0x55e3b6fc949d
main.main()
/go/src/github.com/docker/cli/cmd/docker/docker.go:180 +0xde fp=0xc420779f88 sp=0xc420779ed0 pc=0x55e3b80fb54e
runtime.main()
/usr/local/go/src/runtime/proc.go:198 +0x21a fp=0xc420779fe0 sp=0xc420779f88 pc=0x55e3b6e59cda
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1 fp=0xc420779fe8 sp=0xc420779fe0 pc=0x55e3b6e855e1

goroutine 5 [syscall]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:139 +0xa8
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:22 +0x24
created by os/signal.init.0
/usr/local/go/src/os/signal/signal_unix.go:28 +0x43

goroutine 56 [chan receive]:
/github.com/docker/cli/vendor/github.com/golang/glog.(*loggingT).flushDaemon(0x55e3b96282a0)
/go/src/github.com/docker/cli/vendor/github.com/golang/glog/glog.go:882 +0x8d
created by /github.com/docker/cli/vendor/github.com/golang/glog.init.0
/go/src/github.com/docker/cli/vendor/github.com/golang/glog/glog.go:410 +0x205

goroutine 59 [IO wait]:
internal/poll.runtime_pollWait(0x7f0d297e0f00, 0x72, 0xc42063b9a8)
/usr/local/go/src/runtime/netpoll.go:173 +0x59
internal/poll.(*pollDesc).wait(0xc420630018, 0x72, 0xffffffffffffff00, 0x55e3b8b29100, 0x55e3b95baa68)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9d
internal/poll.(*pollDesc).waitRead(0xc420630018, 0xc420601000, 0x1000, 0x1000)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3f
internal/poll.(*FD).Read(0xc420630000, 0xc420601000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17f
net.(*netFD).Read(0xc420630000, 0xc420601000, 0x1000, 0x1000, 0x55e3b6e81460, 0xc420000180, 0x4)
/usr/local/go/src/net/fd_unix.go:202 +0x51
net.(*conn).Read(0xc42000e8b0, 0xc420601000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:176 +0x6c
net/http.(*persistConn).Read(0xc4205fc6c0, 0xc420601000, 0x1000, 0x1000, 0xc42063bb98, 0x55e3b6e31635, 0xc42063c000)
/usr/local/go/src/net/http/transport.go:1453 +0x138
bufio.(*Reader).fill(0xc4202baae0)
/usr/local/go/src/bufio/bufio.go:100 +0x120
bufio.(*Reader).Peek(0xc4202baae0, 0x1, 0x0, 0x0, 0x0, 0xc42063c0c0, 0x0)
/usr/local/go/src/bufio/bufio.go:132 +0x3c
net/http.(*persistConn).readLoop(0xc4205fc6c0)
/usr/local/go/src/net/http/transport.go:1601 +0x187
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1237 +0x95c

goroutine 60 [select]:
net/http.(*persistConn).writeLoop(0xc4205fc6c0)
/usr/local/go/src/net/http/transport.go:1822 +0x14d
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1238 +0x981

rax 0x0
rbx 0x7f0d2902f840
rcx 0x7f0d28c81e97
rdx 0x0
rdi 0x2
rsi 0x7f0d01ffa890
rbp 0x55e3b82de458
rsp 0x7f0d01ffa890
r8 0x0
r9 0x7f0d01ffa890
r10 0x8
r11 0x246
r12 0x7f0ce4000c50
r13 0xf1
r14 0x11
r15 0x0
rip 0x7f0d28c81e97
rflags 0x246
cs 0x33
fs 0x0
gs 0x0

Any help to resolve this issue is really appreciated.

Thanks
Himanshu

Hello @hmaity
Can you First Check Your HardDrive
Use Of Below Commands

After Check docker volume ls
Then Remove unwanted volume From docker.

As I mentioned, there is lot of free space on the server. Not sure if the problem is with underlying hardware associated with VM or not.