Crash when starting many services via Docker Compose

lukad · April 22, 2016, 3:19pm

I have a docker-compose file which defines ~15 rails apps and some backing services like a Postgres db, elasticsearch and elasticsearch. When I try to start all services via docker-compose up -d I can see that docker starts some apps by looking at docker-compose logs -f.
At some point docker becomes unresponsive and eventually it crashes. This works fine with docker-machine.

~/w/docker-compose (develop) $ pinata diagnose -u
OS X: version 10.10.5 (build: 14F1605)
Docker.app: version v1.11.0-beta8.2
Running diagnostic tests:
[OK]      docker-cli
[OK]      Moby booted
[OK]      driver.amd64-linux
[OK]      vmnetd
[OK]      osxfs
[OK]      db
[OK]      slirp
[OK]      menubar
[OK]      environment
[OK]      Docker
[OK]      VT-x
Docker logs are being collected into /tmp/20160422-171255.tar.gz
Most specific failure is: No error was detected
Your unique id is: 9E42786C-8AE4-4659-A2A1-C0E3301E9151
Please quote this in all correspondence.

Is there any additional info I can provide besides the pinata diagnose -u?

jredl · April 23, 2016, 1:10am

I had a very similar issue, I ended up increasing the memory of the hypervisor:

pinata set hypervisor native memory=4

dsheets · April 25, 2016, 9:43am

Hi Luka,

Could you share the compose file or the non-sensitive parts of it? We’d really like to get a reproduction case for this issue so that we can begin reducing it. Any amount of information toward a reproduction would be helpful.

Thanks,

David

lukad · April 25, 2016, 9:52am

Hi David,

Here is my docker-compose.yml. It’s quite large but we don’t really do anything special. But I don’t think it will really help you to reproduce my crash. I guess booting all services just uses a lot of memory…

version: '2'

services:
  nginx:
    image: nginx
    volumes:
      - ./nginx:/etc/nginx
      - app6-data:/app6
    ports:
      - "80:80"
    depends_on:
      - app3
      - app8
      - app9
      - app10
      - app13
      - app14
      - app7
      - app6
      - app12
      - app1
      - app15
      - app16

  app1:
    build:
      context: ../app1
    command: bundle exec puma -C /shared/puma.rb
    volumes:
      - ./shared:/shared
      - ../app1:/app
    env_file: ./env/app1.env

  app2:
    build:
      context: ../app2
    command: bundle exec puma -C /shared/puma.rb
    volumes:
      - ./shared:/shared
      - ../app2:/app
    env_file: ./env/app2.env
    depends_on:
      - mail

  app2_worker:
    image: dockercompose_app2
    command: bundle exec rake jobs:work
    volumes_from:
      - app2
    env_file:
      - ./env/app2.env
      - ./env/app2_worker.env
    depends_on:
      - mail

  app3:
    build:
      context: ../app3_app
    command: bundle exec puma -C /shared/puma.rb
    volumes:
      - ./shared:/shared
      - ../app3_app:/app
    env_file: ./env/app3.env
    depends_on:
      - db
      - redis
      - elasticsearch
      - rabbitmq
      - app4
      - app13
      - app14

  app3_consumer:
    image: dockercompose_app3
    command: bin/amqp_worker run
    volumes_from:
      - app3
    env_file: ./env/app3.env
    depends_on:
      - elasticsearch
      - db
      - rabbitmq
      - app4

  app4:
    build:
      context: ../wlw
    command: bundle exec puma -C /shared/puma.rb
    volumes:
      - ./shared:/shared
      - ../wlw:/app
    env_file: ./env/app4.env
    depends_on:
      - db
      - redis
      - elasticsearch
      - rabbitmq
      - mail

  app5:
    build:
      context: ../app5
    command: bundle exec puma -C /shared/puma.rb
    volumes:
      - ./shared:/shared
      - ../app5:/app
    env_file: ./env/app5.env
    depends_on:
      - db
      - app7
      - rabbitmq
      - mail

  app6:
    build:
      context: ../app6_service
    command: bundle exec puma -C /shared/puma.rb
    volumes:
      - ./shared:/shared
      - ../app6_service:/app
      - app6-data:/app6
    env_file: ./env/app6.env
    depends_on:
      - db
      - rabbitmq

  app6-product-image-worker:
    image: dockercompose_app6
    command: bundle exec rake sneakers:run
    volumes_from:
      - app6
    env_file:
      - ./env/app6.env
      - ./env/app6-product-image-worker.env
    depends_on:
      - db
      - rabbitmq

  app6-medium-worker:
    image: dockercompose_app6
    command: bundle exec rake sneakers:run
    volumes_from:
      - app6
    env_file:
      - ./env/app6.env
      - ./env/app6-medium-worker.env
    depends_on:
      - db
      - rabbitmq

  app7:
    build:
      context: ../app7
    command: bundle exec puma -C /shared/puma.rb
    volumes:
      - ./shared:/shared
      - ../app7:/app
    env_file: ./env/app7.env
    depends_on:
      - db
      - rabbitmq
      - redis
      - mail

  app8:
    build:
      context: ../app8
    command: bundle exec puma -C /shared/puma.rb
    volumes:
      - ./shared:/shared
      - ../app8:/app
    volumes_from:
      - geoipupdate
    env_file: ./env/app8.env
    extra_hosts:
      local.wlw.at: 127.0.0.1
      local.wlw.ch: 127.0.0.1
      local.wlw.de: 127.0.0.1
    depends_on:
      - db
      - redis

  app9:
    build:
      context: ../app9
    command: bundle exec puma -C /shared/puma.rb
    volumes:
      - ./shared:/shared
      - ../app9:/app
    env_file: ./env/app9.env
    depends_on:
      - db
      - rabbitmq
      - redis

  app10:
    build:
      context: ../app10
    command: bundle exec puma -C /shared/puma.rb
    volumes:
      - ./shared:/shared
      - ../app10:/app
    env_file: ./env/app10.env
    depends_on:
      - db
      - rabbitmq
      - redis

  app11:
    build:
      context: ../app11
    command: bundle exec puma -C /shared/puma.rb
    volumes:
      - ../app11:/app
      - ./shared:/shared
      - app6-data:/app6
    env_file: ./env/app11.env
    depends_on:
      - db
      - redis

  app12:
    build:
      context: ../app12
    command: bundle exec puma -C /shared/puma.rb
    volumes:
      - ../app12:/app
      - ./shared:/shared
    env_file: ./env/app12.env
    depends_on:
      - db
      - redis
      - app5

  app13:
    build:
      context: ../app13
    command: bundle exec puma -C /shared/puma.rb
    volumes:
      - ./shared:/shared
      - ../app13:/app
    env_file: ./env/app13.env
    depends_on:
      - db
      - rabbitmq
      - redis
      - app14

  app13-worker:
    image: dockercompose_app13
    command: bundle exec rake sneakers:run
    volumes_from:
      - app13
    env_file:
      - ./env/app13.env
      - ./env/app13-worker.env
    links:
      - nginx:local.wlw.de
    depends_on:
      - db
      - rabbitmq
      - redis
      - app14
      - download_service
      - app6

  app14:
    build:
      context: ../app14
    command: bundle exec puma -C /shared/puma.rb
    volumes:
      - ./shared:/shared
      - ../app14:/app
    env_file: ./env/app14.env
    depends_on:
      - db
      - rabbitmq

  app15:
    build:
      context: ../app15
    command: bundle exec puma -C /shared/puma.rb
    volumes:
      - ./shared:/shared
      - ../app15:/app
    env_file: ./env/app15.env
    depends_on:
      - db

  app16:
    build:
      context: ../app16
    command: bundle exec puma -C /shared/puma.rb
    volumes:
      - ./shared:/shared
      - ../app16:/app
    env_file: ./env/app16.env
    depends_on:
      - db

  rabbitmq:
    image: rabbitmq:management
    ports:
      - "15672:15672"

  download_service:
    build:
      context: ../download_service
    command: bundle exec puma -C /shared/puma.rb
    volumes:
      - ./shared:/shared
      - ../download_service:/app
    env_file: ./env/download_service.env
    depends_on:
      - db
      - rabbitmq
      - redis
      - app6

  db:
    image: postgres
    ports:
      - "5432:5432"
    volumes:
      - postgres-data:/var/lib/postgresql/data

  mail:
    image: schickling/mailcatcher
    ports:
      - "1080:1080"

  redis:
    image: redis

  elasticsearch:
    build:
      context: ./elasticsearch
    ports:
      - "9200:9200"
    environment:
      ES_HEAP_SIZE: 2g
    volumes:
      - elasticsearch-data:/usr/share/elasticsearch/data
      - ./elasticsearch:/usr/share/elasticsearch/config
      - ./elasticsearch_app4:/app/config/elasticsearch

  geoipupdate:
    image: tkrs/maxmind-geoipupdate
    volumes:
      - /usr/share/GeoIP

volumes:
  postgres-data:
    driver: local

  elasticsearch-data:
    driver: local

  app6-data:
    driver: local

When I work with docker-machine I usually start everything at first and then stop some services that I don’t need at that moment.

Topic		Replies	Views
Compose crashes Docker Desktop	2	1326	August 29, 2016
Docker-compose on mac is "failing" without indicating "why" Compose	0	872	May 11, 2017
Docker-compose slow, crashes app, unusable in beta13 Docker Desktop	3	2146	May 31, 2016
Mac Beta Crashed during docker-compose up Docker Desktop	3	1800	May 27, 2016
Intermittent "i/o timeout" composing multiple services Compose	1	905	April 6, 2023

Crash when starting many services via Docker Compose

Related topics