Hi,
We have an environment where docker runs ins swarm mode (1.13.1).
Several services are deployed which includes the following two:
0aaujuxxiusx service-skeleton replicated 1/1 dockerreg.smec.xx:5000/service-skeleton:0.22.0
cofi5jfvwsmg ad-engine-x replicated 1/1 dockerreg.smec.xx:5000/ad-engine-x:0.10.0
All our services are connected with an overlay network.
When I now go into the started container of service-skeleton and execute the following commands:
> nslookup ad-engine-x
Server: 127.0.0.11
Address: 127.0.0.11#53
Non-authoritative answer:
Name: ad-engine-x
Address: 10.0.1.33
> nslookup tasks.ad-engine-x
Server: 127.0.0.11
Address: 127.0.0.11#53
Non-authoritative answer:
Name: tasks.ad-engine-x
Address: 10.0.1.32
> curl 10.0.1.32:8081/health --> {"status":"UP","diskSpace":{"status":"UP","free":1751977357312,"threshold":...
> curl 10.0.1.32:8081/health --> {"status":"UP","diskSpace":{"status":"UP","free":1751977357312,"threshold":...
> curl 10.0.1.32:8081/health --> {"status":"UP","diskSpace":{"status":"UP","free":1751977357312,"threshold":...
> curl 10.0.1.32:8081/health --> {"status":"UP","diskSpace":{"status":"UP","free":1751977357312,"threshold":...
> curl 10.0.1.33:8081/health --> {"status":"UP","diskSpace":{"status":"UP","free":1751977357312,"threshold":...
> curl 10.0.1.33:8081/health --> curl: (7) Failed to connect to 10.0.1.33 port 8081: Connection refused
> curl 10.0.1.33:8081/health --> {"status":"UP","diskSpace":{"status":"UP","free":1751977357312,"threshold":...
> curl 10.0.1.33:8081/health --> curl: (7) Failed to connect to 10.0.1.33 port 8081: Connection refused
> curl 10.0.1.33:8081/health --> {"status":"UP","diskSpace":{"status":"UP","free":1751977357312,"threshold":...
> curl 10.0.1.33:8081/health --> curl: (7) Failed to connect to 10.0.1.33 port 8081: Connection refused
> curl ad-engine-x:8081/health --> {"status":"UP","diskSpace":{"status":"UP","free":1751977357312,"threshold":..
> curl ad-engine-x:8081/health --> curl: (7) Failed to connect to 10.0.1.33 port 8081: Connection refused
> curl ad-engine-x:8081/health --> {"status":"UP","diskSpace":{"status":"UP","free":1751977357312,"threshold":..
> curl ad-engine-x:8081/health --> curl: (7) Failed to connect to 10.0.1.33 port 8081: Connection refused
> curl ad-engine-x:8081/health --> {"status":"UP","diskSpace":{"status":"UP","free":1751977357312,"threshold":..
> curl ad-engine-x:8081/health --> curl: (7) Failed to connect to 10.0.1.33 port 8081: Connection refused
Inspecting the service:
> docker service inspect ad-engine-x
[
{
"ID": "cofi5jfvwsmgp4etnpdtsnnjh",
"Version": {
"Index": 72002
},
"CreatedAt": "2017-01-12T08:19:03.764158902Z",
"UpdatedAt": "2017-02-21T09:47:49.420612622Z",
"Spec": {
"Name": "ad-engine-x",
"TaskTemplate": {
"ContainerSpec": {
"Image": "dockerreg.smec.xx:5000/ad-engine-x:0.10.0@sha256:b15c130a8b5c84ce84e9f9515e40a26c342fa237156616539dcb4dc1ee46cb20",
"Mounts": [
{
"Type": "bind",
"Source": "/etc/hosts",
"Target": "/etc/hosts_from_parent",
"ReadOnly": true
},
{
"Type": "bind",
"Source": "/etc/localtime",
"Target": "/etc/localtime",
"ReadOnly": true
},
{
"Type": "bind",
"Source": "/etc/timezone",
"Target": "/etc/timezone",
"ReadOnly": true
},
{
"Type": "bind",
"Source": "/srv/plugins/ad-engine-x",
"Target": "/srv/plugins"
},
{
"Type": "bind",
"Source": "/var/log/smec/ad-engine-x",
"Target": "/var/log/smec"
}
]
},
"Resources": {
"Limits": {},
"Reservations": {
"NanoCPUs": 1000000000,
"MemoryBytes": 1073741824
}
},
"RestartPolicy": {
"Condition": "any",
"MaxAttempts": 0
},
"Placement": {},
"LogDriver": {
"Name": "json-file",
"Options": {
"max-size": "20m"
}
},
"ForceUpdate": 0
},
"Mode": {
"Replicated": {
"Replicas": 1
}
},
"UpdateConfig": {
"Parallelism": 1,
"FailureAction": "pause",
"MaxFailureRatio": 0
},
"Networks": [
{
"Target": "a3akeoamu7fvpqx4g14mjcrnr"
}
],
"EndpointSpec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 80,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 8080,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 8081,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 8082,
"PublishMode": "ingress"
}
]
}
},
"PreviousSpec": {
"Name": "ad-engine-x",
"TaskTemplate": {
"ContainerSpec": {
"Image": "dockerreg.smec.xx:5000/ad-engine-x:0.10.0@sha256:b15c130a8b5c84ce84e9f9515e40a26c342fa237156616539dcb4dc1ee46cb20",
"Mounts": [
{
"Type": "bind",
"Source": "/etc/hosts",
"Target": "/etc/hosts_from_parent",
"ReadOnly": true
},
{
"Type": "bind",
"Source": "/etc/localtime",
"Target": "/etc/localtime",
"ReadOnly": true
},
{
"Type": "bind",
"Source": "/etc/timezone",
"Target": "/etc/timezone",
"ReadOnly": true
},
{
"Type": "bind",
"Source": "/srv/plugins/ad-engine-x",
"Target": "/srv/plugins"
},
{
"Type": "bind",
"Source": "/var/log/smec/ad-engine-x",
"Target": "/var/log/smec"
}
]
},
"Resources": {
"Limits": {},
"Reservations": {
"NanoCPUs": 1000000000,
"MemoryBytes": 1073741824
}
},
"RestartPolicy": {
"Condition": "any",
"MaxAttempts": 0
},
"Placement": {},
"LogDriver": {
"Name": "json-file",
"Options": {
"max-size": "20m"
}
},
"ForceUpdate": 0
},
"Mode": {
"Replicated": {
"Replicas": 0
}
},
"UpdateConfig": {
"Parallelism": 1,
"FailureAction": "pause",
"MaxFailureRatio": 0
},
"Networks": [
{
"Target": "a3akeoamu7fvpqx4g14mjcrnr"
}
],
"EndpointSpec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 80,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 8080,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 8081,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 8082,
"PublishMode": "ingress"
}
]
}
},
"Endpoint": {
"Spec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 80,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 8080,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 8081,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 8082,
"PublishMode": "ingress"
}
]
},
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 80,
"PublishedPort": 30020,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 8080,
"PublishedPort": 30021,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 8081,
"PublishedPort": 30022,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 8082,
"PublishedPort": 30023,
"PublishMode": "ingress"
}
],
"VirtualIPs": [
{
"NetworkID": "emyw7txpxh31kxvhxeb8jl0a9",
"Addr": "10.255.0.21/16"
},
{
"NetworkID": "a3akeoamu7fvpqx4g14mjcrnr",
"Addr": "10.0.1.33/24"
}
]
},
"UpdateStatus": {
"StartedAt": "0001-01-01T00:00:00Z",
"CompletedAt": "0001-01-01T00:00:00Z"
}
}
]
On the other hand
> curl service-skeleton:8081/health -> good result
> curl service-skeleton:8081/health -> good result
> curl service-skeleton:8081/health -> good result
> curl service-skeleton:8081/health -> good result
> curl service-skeleton:8081/health -> good result
So what can it be, that the ad-engine-x service flaps every second call? It seems really odd.