Nomad: consul/connect: sidecar healthchecks are failing when host_networks are defined
Nomad version
v1.0.1
Issue
Sidecar healthchecks for a job with Connect-enabled service are failing on Nomad clients with host_networks defined and pass on clients without host_networks. Sidecar's docker inspect logs are given below for both cases
Job file
job "redis" {
datacenters = ["fsn1"]
type = "system"
constraint {
attribute = "${node.class}"
operator = "regexp"
value = "(cloud-)?cache"
}
constraint {
operator = "distinct_property"
attribute = "${node.datacenter}"
value = "1"
}
group "redis" {
network {
mode = "bridge"
}
count = 1
ephemeral_disk {
sticky = true
migrate = true
size = 4000
}
restart {
mode = "delay"
}
service {
name = "redis"
port = "6379"
connect {
sidecar_service {}
}
check {
type = "script"
name = "redis"
task = "redis_server"
command = "/bin/sh"
args = ["-c", "[ \"$(redis-cli ping)\" = 'PONG' ] && exit 0; exit 1"]
interval = "60s"
timeout = "5s"
}
}
task "redis_server" {
driver = "docker"
config {
image = "redis:latest"
sysctl {
net.core.somaxconn = 1024
}
}
resources {
cpu = 1000
memory = 3072
}
}
}
}
Docker inspect output for a sidecar when host_networks are defined
[
{
"Id": "eccb3872c50933f12d300f451f582c489a3acf4d68430bc22ed176fa9875d825",
"Created": "2020-12-18T17:37:21.484075946Z",
"Path": "/docker-entrypoint.sh",
"Args": [
"-c",
"/secrets/envoy_bootstrap.json",
"-l",
"info",
"--concurrency",
"1",
"--disable-hot-restart"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 20238,
"ExitCode": 0,
"Error": "",
"StartedAt": "2020-12-18T17:37:21.765901968Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:a438abf4c3fdcaa9c7d74dd178429e4bc8b5edba9590c429a12cf08dc8d44399",
"ResolvConfPath": "/var/lib/docker/containers/d2e6aabee7da0fa824f8b33f0d3927ec01b86612aaaeb9e96038f24ba280c964/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/d2e6aabee7da0fa824f8b33f0d3927ec01b86612aaaeb9e96038f24ba280c964/hostname",
"HostsPath": "/var/lib/docker/containers/d2e6aabee7da0fa824f8b33f0d3927ec01b86612aaaeb9e96038f24ba280c964/hosts",
"LogPath": "/var/lib/docker/containers/eccb3872c50933f12d300f451f582c489a3acf4d68430bc22ed176fa9875d825/eccb3872c50933f12d300f451f582c489a3acf4d68430bc22ed176fa9875d825-json.log",
"Name": "/connect-proxy-redis-0c1dad91-9076-31d4-0bd3-ce5d3a55e93e",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"/mnt/data/nomad/alloc/0c1dad91-9076-31d4-0bd3-ce5d3a55e93e/alloc:/alloc",
"/mnt/data/nomad/alloc/0c1dad91-9076-31d4-0bd3-ce5d3a55e93e/connect-proxy-redis/local:/local",
"/mnt/data/nomad/alloc/0c1dad91-9076-31d4-0bd3-ce5d3a55e93e/connect-proxy-redis/secrets:/secrets"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {
"max-file": "2",
"max-size": "2m"
}
},
"NetworkMode": "container:d2e6aabee7da0fa824f8b33f0d3927ec01b86612aaaeb9e96038f24ba280c964",
"PortBindings": null,
"RestartPolicy": {
"Name": "",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"Dns": null,
"DnsOptions": null,
"DnsSearch": null,
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "shareable",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": null,
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 250,
"Memory": 134217728,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": null,
"DeviceCgroupRules": null,
"DiskQuota": 0,
"KernelMemory": 0,
"MemoryReservation": 0,
"MemorySwap": 134217728,
"MemorySwappiness": 0,
"OomKillDisable": false,
"PidsLimit": 0,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": [
"/proc/asound",
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware"
],
"ReadonlyPaths": [
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger"
]
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/05ac2766679cb28bea6b8707fd354e02a0d915388fb3c1c3dfcd27ecbe2c9794-init/diff:/var/lib/docker/overlay2/599a4324268414bbee1313c1a8228fc04a23dbbaa7e217781b13068aba76cac2/diff:/var/lib/docker/overlay2/f18ed9ac9064b91bc969ef7d5464e5d4137cb45259140bcf9aad6fd33c943f5e/diff:/var/lib/docker/overlay2/5309f6e870cbb0ddbbef3c050ed846938caf914ad7a6fd70213038f283443e1b/diff:/var/lib/docker/overlay2/c906779f27cb173cc0e909f176d2ae771707fd5de6c616d5bb48b9aa43c8a724/diff:/var/lib/docker/overlay2/97f0674e04138268879db58b8199f9edd2ea8fea0cf666e3f1ea00705cf4afa2/diff:/var/lib/docker/overlay2/89781cc021e7703728ee2521fd570f68e589bb4f611f727bb122e28280877dec/diff:/var/lib/docker/overlay2/b06ad4f2a5d1c3664670680d80a9ccc565c10092de595b5a237eb8ab3eb40926/diff:/var/lib/docker/overlay2/9f9632f2a682c8d6e16695a28a00fa5d8c529f618fcd0fc81e245ff509365b7d/diff:/var/lib/docker/overlay2/f30016271bcc92a3d764944f9444a662e3ef10543a6586c27fa626c0d58a96ac/diff:/var/lib/docker/overlay2/e2046c6de4c07c84110e401bf2fa73b58f0aeebc806d15ecae1b9ffb609d943a/diff",
"MergedDir": "/var/lib/docker/overlay2/05ac2766679cb28bea6b8707fd354e02a0d915388fb3c1c3dfcd27ecbe2c9794/merged",
"UpperDir": "/var/lib/docker/overlay2/05ac2766679cb28bea6b8707fd354e02a0d915388fb3c1c3dfcd27ecbe2c9794/diff",
"WorkDir": "/var/lib/docker/overlay2/05ac2766679cb28bea6b8707fd354e02a0d915388fb3c1c3dfcd27ecbe2c9794/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "bind",
"Source": "/mnt/data/nomad/alloc/0c1dad91-9076-31d4-0bd3-ce5d3a55e93e/alloc",
"Destination": "/alloc",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
},
{
"Type": "bind",
"Source": "/mnt/data/nomad/alloc/0c1dad91-9076-31d4-0bd3-ce5d3a55e93e/connect-proxy-redis/local",
"Destination": "/local",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
},
{
"Type": "bind",
"Source": "/mnt/data/nomad/alloc/0c1dad91-9076-31d4-0bd3-ce5d3a55e93e/connect-proxy-redis/secrets",
"Destination": "/secrets",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
}
],
"Config": {
"Hostname": "d2e6aabee7da",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"10000/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"NOMAD_ADDR_connect_proxy_redis=2a01:4f8:1c17:7a7b::1:26705",
"NOMAD_ALLOC_DIR=/alloc",
"NOMAD_ALLOC_ID=0c1dad91-9076-31d4-0bd3-ce5d3a55e93e",
"NOMAD_ALLOC_INDEX=0",
"NOMAD_ALLOC_NAME=redis.redis[0]",
"NOMAD_ALLOC_PORT_connect-proxy-redis=26705",
"NOMAD_CPU_LIMIT=250",
"NOMAD_DC=fsn1",
"NOMAD_ENVOY_ADMIN_ADDR_redis=localhost:19001",
"NOMAD_GROUP_NAME=redis",
"NOMAD_HOST_ADDR_connect-proxy-redis=2a01:4f8:1c17:7a7b::1:26705",
"NOMAD_HOST_IP_connect-proxy-redis=2a01:4f8:1c17:7a7b::1",
"NOMAD_HOST_PORT_connect_proxy_redis=26705",
"NOMAD_IP_connect_proxy_redis=2a01:4f8:1c17:7a7b::1",
"NOMAD_JOB_ID=redis",
"NOMAD_JOB_NAME=redis",
"NOMAD_MEMORY_LIMIT=128",
"NOMAD_NAMESPACE=default",
"NOMAD_PORT_connect_proxy_redis=26705",
"NOMAD_REGION=global",
"NOMAD_SECRETS_DIR=/secrets",
"NOMAD_TASK_DIR=/local",
"NOMAD_TASK_NAME=connect-proxy-redis",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Cmd": [
"-c",
"/secrets/envoy_bootstrap.json",
"-l",
"info",
"--concurrency",
"1",
"--disable-hot-restart"
],
"Image": "envoyproxy/envoy:v1.16.0",
"Volumes": null,
"WorkingDir": "",
"Entrypoint": [
"/docker-entrypoint.sh"
],
"OnBuild": null,
"Labels": {
"com.hashicorp.nomad.alloc_id": "0c1dad91-9076-31d4-0bd3-ce5d3a55e93e"
}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {},
"SandboxKey": "",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {}
}
}
]
Consul service addresses
{
"_nomad-task-9908af05-a7bb-ca40-80e2-eed84c3cdfce-group-redis-redis-6379": {
"ID": "_nomad-task-9908af05-a7bb-ca40-80e2-eed84c3cdfce-group-redis-redis-6379",
"Service": "redis",
"Tags": [],
"Meta": {
"external-source": "nomad"
},
"Port": 6379,
"Address": "",
"Weights": {
"Passing": 1,
"Warning": 1
},
"EnableTagOverride": false,
"Datacenter": "fsn1"
},
"_nomad-task-9908af05-a7bb-ca40-80e2-eed84c3cdfce-group-redis-redis-6379-sidecar-proxy": {
"Kind": "connect-proxy",
"ID": "_nomad-task-9908af05-a7bb-ca40-80e2-eed84c3cdfce-group-redis-redis-6379-sidecar-proxy",
"Service": "redis-sidecar-proxy",
"Tags": [],
"Meta": {
"external-source": "nomad"
},
"Port": 26705,
"Address": "10.1.25.13",
"TaggedAddresses": {
"lan_ipv4": {
"Address": "10.1.25.13",
"Port": 26705
},
"wan_ipv4": {
"Address": "10.1.25.13",
"Port": 26705
}
},
"Weights": {
"Passing": 1,
"Warning": 1
},
"EnableTagOverride": false,
"Proxy": {
"DestinationServiceName": "redis",
"DestinationServiceID": "_nomad-task-9908af05-a7bb-ca40-80e2-eed84c3cdfce-group-redis-redis-6379",
"LocalServiceAddress": "127.0.0.1",
"LocalServicePort": 6379,
"Config": {
"bind_address": "0.0.0.0",
"bind_port": 26705
},
"MeshGateway": {},
"Expose": {}
},
"Datacenter": "fsn1"
}
}
Docker inspect output for a sidecar when host_networks are not defined
[
{
"Id": "85889682da02adfe8404ef22bb8f93276b730bb39b0aa8d946afefa231adc2d9",
"Created": "2020-12-18T17:31:31.983985542Z",
"Path": "/docker-entrypoint.sh",
"Args": [
"-c",
"/secrets/envoy_bootstrap.json",
"-l",
"info",
"--concurrency",
"1",
"--disable-hot-restart"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 17077,
"ExitCode": 0,
"Error": "",
"StartedAt": "2020-12-18T17:31:32.293876824Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:a438abf4c3fdcaa9c7d74dd178429e4bc8b5edba9590c429a12cf08dc8d44399",
"ResolvConfPath": "/var/lib/docker/containers/0fe0b6c22fcc7bb3662561b1028dc71b6702366015b4509f93e5222c1d670fea/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/0fe0b6c22fcc7bb3662561b1028dc71b6702366015b4509f93e5222c1d670fea/hostname",
"HostsPath": "/var/lib/docker/containers/0fe0b6c22fcc7bb3662561b1028dc71b6702366015b4509f93e5222c1d670fea/hosts",
"LogPath": "/var/lib/docker/containers/85889682da02adfe8404ef22bb8f93276b730bb39b0aa8d946afefa231adc2d9/85889682da02adfe8404ef22bb8f93276b730bb39b0aa8d946afefa231adc2d9-json.log",
"Name": "/connect-proxy-redis-566e7f47-4ec8-f276-eb16-7c2284053b3d",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"/mnt/data/nomad/alloc/566e7f47-4ec8-f276-eb16-7c2284053b3d/alloc:/alloc",
"/mnt/data/nomad/alloc/566e7f47-4ec8-f276-eb16-7c2284053b3d/connect-proxy-redis/local:/local",
"/mnt/data/nomad/alloc/566e7f47-4ec8-f276-eb16-7c2284053b3d/connect-proxy-redis/secrets:/secrets"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {
"max-file": "2",
"max-size": "2m"
}
},
"NetworkMode": "container:0fe0b6c22fcc7bb3662561b1028dc71b6702366015b4509f93e5222c1d670fea",
"PortBindings": null,
"RestartPolicy": {
"Name": "",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"Dns": null,
"DnsOptions": null,
"DnsSearch": null,
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "shareable",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": null,
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 250,
"Memory": 134217728,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": null,
"DeviceCgroupRules": null,
"DiskQuota": 0,
"KernelMemory": 0,
"MemoryReservation": 0,
"MemorySwap": 134217728,
"MemorySwappiness": 0,
"OomKillDisable": false,
"PidsLimit": 0,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": [
"/proc/asound",
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware"
],
"ReadonlyPaths": [
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger"
]
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/5c7cf9a8c42bdd02f707357f7501d67a3e31305248e2c67cef3a0408a0ca2f51-init/diff:/var/lib/docker/overlay2/599a4324268414bbee1313c1a8228fc04a23dbbaa7e217781b13068aba76cac2/diff:/var/lib/docker/overlay2/f18ed9ac9064b91bc969ef7d5464e5d4137cb45259140bcf9aad6fd33c943f5e/diff:/var/lib/docker/overlay2/5309f6e870cbb0ddbbef3c050ed846938caf914ad7a6fd70213038f283443e1b/diff:/var/lib/docker/overlay2/c906779f27cb173cc0e909f176d2ae771707fd5de6c616d5bb48b9aa43c8a724/diff:/var/lib/docker/overlay2/97f0674e04138268879db58b8199f9edd2ea8fea0cf666e3f1ea00705cf4afa2/diff:/var/lib/docker/overlay2/89781cc021e7703728ee2521fd570f68e589bb4f611f727bb122e28280877dec/diff:/var/lib/docker/overlay2/b06ad4f2a5d1c3664670680d80a9ccc565c10092de595b5a237eb8ab3eb40926/diff:/var/lib/docker/overlay2/9f9632f2a682c8d6e16695a28a00fa5d8c529f618fcd0fc81e245ff509365b7d/diff:/var/lib/docker/overlay2/f30016271bcc92a3d764944f9444a662e3ef10543a6586c27fa626c0d58a96ac/diff:/var/lib/docker/overlay2/e2046c6de4c07c84110e401bf2fa73b58f0aeebc806d15ecae1b9ffb609d943a/diff",
"MergedDir": "/var/lib/docker/overlay2/5c7cf9a8c42bdd02f707357f7501d67a3e31305248e2c67cef3a0408a0ca2f51/merged",
"UpperDir": "/var/lib/docker/overlay2/5c7cf9a8c42bdd02f707357f7501d67a3e31305248e2c67cef3a0408a0ca2f51/diff",
"WorkDir": "/var/lib/docker/overlay2/5c7cf9a8c42bdd02f707357f7501d67a3e31305248e2c67cef3a0408a0ca2f51/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "bind",
"Source": "/mnt/data/nomad/alloc/566e7f47-4ec8-f276-eb16-7c2284053b3d/alloc",
"Destination": "/alloc",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
},
{
"Type": "bind",
"Source": "/mnt/data/nomad/alloc/566e7f47-4ec8-f276-eb16-7c2284053b3d/connect-proxy-redis/local",
"Destination": "/local",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
},
{
"Type": "bind",
"Source": "/mnt/data/nomad/alloc/566e7f47-4ec8-f276-eb16-7c2284053b3d/connect-proxy-redis/secrets",
"Destination": "/secrets",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
}
],
"Config": {
"Hostname": "0fe0b6c22fcc",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"10000/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"NOMAD_ADDR_connect_proxy_redis=10.1.25.13:24731",
"NOMAD_ALLOC_DIR=/alloc",
"NOMAD_ALLOC_ID=566e7f47-4ec8-f276-eb16-7c2284053b3d",
"NOMAD_ALLOC_INDEX=0",
"NOMAD_ALLOC_NAME=redis.redis[0]",
"NOMAD_ALLOC_PORT_connect-proxy-redis=24731",
"NOMAD_CPU_LIMIT=250",
"NOMAD_DC=fsn1",
"NOMAD_ENVOY_ADMIN_ADDR_redis=localhost:19001",
"NOMAD_GROUP_NAME=redis",
"NOMAD_HOST_ADDR_connect-proxy-redis=10.1.25.13:24731",
"NOMAD_HOST_IP_connect-proxy-redis=10.1.25.13",
"NOMAD_HOST_PORT_connect_proxy_redis=24731",
"NOMAD_IP_connect_proxy_redis=10.1.25.13",
"NOMAD_JOB_ID=redis",
"NOMAD_JOB_NAME=redis",
"NOMAD_MEMORY_LIMIT=128",
"NOMAD_NAMESPACE=default",
"NOMAD_PORT_connect_proxy_redis=24731",
"NOMAD_REGION=global",
"NOMAD_SECRETS_DIR=/secrets",
"NOMAD_TASK_DIR=/local",
"NOMAD_TASK_NAME=connect-proxy-redis",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Cmd": [
"-c",
"/secrets/envoy_bootstrap.json",
"-l",
"info",
"--concurrency",
"1",
"--disable-hot-restart"
],
"Image": "envoyproxy/envoy:v1.16.0",
"Volumes": null,
"WorkingDir": "",
"Entrypoint": [
"/docker-entrypoint.sh"
],
"OnBuild": null,
"Labels": {
"com.hashicorp.nomad.alloc_id": "566e7f47-4ec8-f276-eb16-7c2284053b3d"
}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {},
"SandboxKey": "",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {}
}
}
]
Consul health status for a sidecar service when host_networks are defined
{
"service:_nomad-task-9908af05-a7bb-ca40-80e2-eed84c3cdfce-group-redis-redis-6379-sidecar-proxy:1": {
"Node": "qa-1-fsn1-cache-0",
"CheckID": "service:_nomad-task-9908af05-a7bb-ca40-80e2-eed84c3cdfce-group-redis-redis-6379-sidecar-proxy:1",
"Name": "Connect Sidecar Listening",
"Status": "critical",
"Notes": "",
"Output": "dial tcp 127.0.0.1:26705: connect: connection refused",
"ServiceID": "_nomad-task-9908af05-a7bb-ca40-80e2-eed84c3cdfce-group-redis-redis-6379-sidecar-proxy",
"ServiceName": "redis-sidecar-proxy",
"ServiceTags": [],
"Type": "tcp",
"Definition": {},
"CreateIndex": 0,
"ModifyIndex": 0
}
}
AndrewChubatiuk
👍4
All 6 comments
Looks like the same issue is described here
https://discuss.hashicorp.com/t/connect-sidecar-listening-healthcheck-fail/14574
AndrewChubatiuk
on 18 Dec 2020
@AndrewChubatiuk - Have you found a fix? Trying to setup consul-connect by following the tutorial from here, running into this issue:
Nomad version:
Nomad v0.12.9 (45c139e53f2407a44b1290385b5818b46ea3a62c)
Consul Version
Consul v1.9.1
Revision ca5c38943
Protocol 2 spoken by default, understands 2 to 3 (agent will automatically use protocol >2 when speaking to compatible agents)
jsanant
on 24 Dec 2020
So I found the issue, turns out it was because of the envoy version that the sidecar was using v1.11.0 and consul no longer supports it so I upgraded it to v1.14.6 and the connect sidecar listening health check is passing.
Hope this helps!
jsanant
on 28 Dec 2020
@jsanant
I'm using nomad 1.0.1, which uses envoy proxy of version 1.16.0 by default
Have problems with consul connect only when host_network is defined for nomad agent.
AndrewChubatiuk
on 28 Dec 2020
👍2
Hi, I am getting the same issue with Nomad 1.0.1 using the dashcount example with host_networks defined and network_interface:
#nomad config
data_dir = "/opt/nomad/data"
bind_addr = "10.1.1.1"
region = "europe"
server {
enabled = true
bootstrap_expect = 1
}
client {
servers = ["10.1.1.1:4647"]
enabled = true
network_interface = "ens10"
host_network "public" {
interface = "{{ GetPublicInterfaces | limit 1 | attr \"name\" }}"
cidr = "<INSTANCE PUBLIC IP>/32"
reserved_ports = "22,80,443,8080"
}
}
addresses {
http = "0.0.0.0"
}
advertise {
http = "10.1.1.1"
}
When checking the sidecar service in consul I get the following: (curl http://127.0.0.1:8500/v1/health/service/count-api-sidecar-proxy | jq)
[
{
"Node": {
"ID": "4d863b15-935b-3191-a383-1933d4d334db",
"Node": "vps-de01-dev-001",
"Address": "10.1.1.1",
"Datacenter": "dc1",
"TaggedAddresses": {
"lan": "10.1.1.1",
"lan_ipv4": "10.1.1.1",
"wan": "10.1.1.1",
"wan_ipv4": "10.1.1.1"
},
"Meta": {
"consul-network-segment": ""
},
"CreateIndex": 12,
"ModifyIndex": 13
},
"Service": {
"Kind": "connect-proxy",
"ID": "_nomad-task-839535c0-9640-0b7b-c82d-c274c43c0fb5-group-api-count-api--sidecar-proxy",
"Service": "count-api-sidecar-proxy",
"Tags": [],
"Address": "<INSTANCE PUBLIC IP>",
"TaggedAddresses": {
"lan_ipv4": {
"Address": "<INSTANCE PUBLIC IP>",
"Port": 30628
},
"wan_ipv4": {
"Address": "<INSTANCE PUBLIC IP>",
"Port": 30628
}
},
"Meta": {
"external-source": "nomad"
},
"Port": 30628,
"Weights": {
"Passing": 1,
"Warning": 1
},
"EnableTagOverride": false,
"Proxy": {
"DestinationServiceName": "count-api",
"DestinationServiceID": "_nomad-task-839535c0-9640-0b7b-c82d-c274c43c0fb5-group-api-count-api-",
"LocalServiceAddress": "127.0.0.1",
"Config": {
"bind_address": "0.0.0.0",
"bind_port": 30628
},
"MeshGateway": {},
"Expose": {}
},
"Connect": {},
"CreateIndex": 912,
"ModifyIndex": 912
},
"Checks": [
{
"Node": "vps-de01-dev-001",
"CheckID": "serfHealth",
"Name": "Serf Health Status",
"Status": "passing",
"Notes": "",
"Output": "Agent alive and reachable",
"ServiceID": "",
"ServiceName": "",
"ServiceTags": [],
"Type": "",
"Definition": {},
"CreateIndex": 12,
"ModifyIndex": 12
},
{
"Node": "vps-de01-dev-001",
"CheckID": "service:_nomad-task-839535c0-9640-0b7b-c82d-c274c43c0fb5-group-api-count-api--sidecar-proxy:1",
"Name": "Connect Sidecar Listening",
"Status": "critical",
"Notes": "",
"Output": "dial tcp 127.0.0.1:30628: connect: connection refused",
"ServiceID": "_nomad-task-839535c0-9640-0b7b-c82d-c274c43c0fb5-group-api-count-api--sidecar-proxy",
"ServiceName": "count-api-sidecar-proxy",
"ServiceTags": [],
"Type": "tcp",
"Definition": {},
"CreateIndex": 912,
"ModifyIndex": 941
},
{
"Node": "vps-de01-dev-001",
"CheckID": "service:_nomad-task-839535c0-9640-0b7b-c82d-c274c43c0fb5-group-api-count-api--sidecar-proxy:2",
"Name": "Connect Sidecar Aliasing _nomad-task-839535c0-9640-0b7b-c82d-c274c43c0fb5-group-api-count-api-",
"Status": "passing",
"Notes": "",
"Output": "No checks found.",
"ServiceID": "_nomad-task-839535c0-9640-0b7b-c82d-c274c43c0fb5-group-api-count-api--sidecar-proxy",
"ServiceName": "count-api-sidecar-proxy",
"ServiceTags": [],
"Type": "alias",
"Definition": {},
"CreateIndex": 912,
"ModifyIndex": 912
}
]
}
]
and the job configuration:
job "countdash" {
datacenters = ["dc1"]
group "api" {
network {
mode = "bridge"
}
service {
name = "count-api"
connect {
sidecar_service {}
}
}
task "web" {
driver = "docker"
config {
image = "hashicorpnomad/counter-api:v2"
}
}
}
group "dashboard" {
network {
mode ="bridge"
port "http" {
host_network = "public"
static = 9002
to = 9002
}
}
service {
name = "count-dashboard"
port = "9002"
connect {
sidecar_service {
proxy {
upstreams {
destination_name = "count-api"
local_bind_port = 8080
}
}
}
}
}
task "dashboard" {
driver = "docker"
env {
COUNTING_SERVICE_URL = "http://${NOMAD_UPSTREAM_ADDR_count_api}"
}
config {
image = "hashicorpnomad/counter-dashboard:v2"
}
}
}
}
cpl
on 6 Jan 2021
Making the following change to my nomad config fixes the healthcheck issue for the sidecars but creates some undefined/unwished behaviour
network_interface = "lo"
host_network "public" {
interface = "{{ GetPublicInterfaces | limit 1 | attr \"name\" }}"
cidr = "<PUBLIC IP>/32"
reserved_ports = "22,80,443,8080"
}
host_network "private" {
interface = "ens10"
cidr = "10.1.1.1/32"
reserved_ports = "22,80,443,8080"
}
Was this page helpful?
0 / 5 - 0 ratings
Related issues
samber
路
3Comments
hynek
路
3Comments
dvusboy
路
3Comments
funkytaco
路
3Comments
jippi
路
3Comments
Most helpful comment
@jsanant
I'm using nomad 1.0.1, which uses envoy proxy of version 1.16.0 by default
Have problems with consul connect only when host_network is defined for nomad agent.